Skip to content
Snippets Groups Projects
Commit 68979a76 authored by Maaijke Mevius's avatar Maaijke Mevius
Browse files

add functionality

parent b1b1db7a
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 223 additions and 5 deletions
query_data.py
+ 223
− 5
View file @ 68979a76
...@@ -28,6 +28,21 @@ def get_png(my_list): ...@@ -28,6 +28,21 @@ def get_png(my_list):
S4file = [f for f in my_list if 'png' in f][0] S4file = [f for f in my_list if 'png' in f][0]
return S4file return S4file
def get_velocity(result):
ew = result['EW_velocity']
ns = result['NS_velocity']
return ew,ns
def get_azel(result):
az = result['azimuth']
el = result['elevation']
return az,el
def get_S4_data(result):
return result['S4_data']
def get_S4_180_data(result):
return result['S4_data_180']
def get_files(date,timest,timeend): def get_files(date,timest,timeend):
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/' url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
time1 = Time(date+"T"+timest,format='isot', scale='utc') time1 = Time(date+"T"+timest,format='isot', scale='utc')
...@@ -64,14 +79,217 @@ def get_files(date,timest,timeend): ...@@ -64,14 +79,217 @@ def get_files(date,timest,timeend):
#print("No S4 here", result) #print("No S4 here", result)
pass pass
url = response_json['next'] url = response_json['next']
try:
print("trying..")
response = requests.get(url)
except:
print("failed.. try again?")
response = requests.get(url) response = requests.get(url)
return S4files, fitsfiles,all_results return S4files, fitsfiles,all_results
def get_obsid(obsid):
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
myfilter = json.dumps({"OBSERVATION_ID":obsid})
response = requests.get(url, params={'filter': myfilter})
all_results = []
while response.status_code == 200:
print(response)
response_json = response.json()
results = response_json['results']
if len(results) == 0:
break
print(f"Number of results: {len(results)}")
all_results += response_json['results']
url = response_json['next']
try:
print("trying..")
response = requests.get(url)
except:
print("failed.. try again?")
response = requests.get(url)
return all_results
def get_timerange(timest, timeend):
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
start_time_iso = timest.isot
end_time_iso = timeend.isot
myfilter = json.dumps({"$and":[{'sample_start_datetime': {'$lte':str(end_time_iso[:-4])}},
{'sample_end_datetime':{'$gte':str(start_time_iso[:-4])}}]})
response = requests.get(url, params={'filter': myfilter})
all_results = []
while response.status_code == 200:
print(response)
response_json = response.json()
results = response_json['results']
if len(results) == 0:
break
print(f"Number of results: {len(results)}")
all_results += response_json['results']
url = response_json['next']
try:
print("trying..")
response = requests.get(url)
except:
print("failed.. try again?")
response = requests.get(url)
return all_results
def plot_S4(timest=Time("2022-01-14T22:00:00"), timeend=Time("2022-01-15T22:00:00")):
def test(): results = get_timerange(timest=timest, timeend=timeend)
stations = set([i['BEAM_STATIONS_LIST'][0] for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys()])
stations = sorted(list(stations))
S4data = []
times = []
maxlength =0
selectstat = []
for stat in stations:
S4datatmp = ([(i['S4_data'], i['S4_data_180'], i['sample_start_datetime'], i['azimuth'], i['elevation']) for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys() and stat in i['BEAM_STATIONS_LIST'] ])
timestmp = [Time(j)+np.arange(len(i))*u.min for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
aztmp = [az[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
eltmp = [el[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
S4data180tmp = [i2 for i,i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
S4datatmp = [i for i,i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
S4datatmp = np.concatenate(S4datatmp)
timestmp = Time(np.concatenate(timestmp))
S4data180tmp = np.concatenate(S4data180tmp)
aztmp = np.concatenate(aztmp)
eltmp = np.concatenate(eltmp)
if S4datatmp.shape[0] > maxlength:
maxlength = S4datatmp.shape[0]
S4data = [S4datatmp]
S4180data = [S4data180tmp]
azdata = [aztmp]
eldata = [eltmp]
times = [timestmp]
selectstat = [stat]
elif S4datatmp.shape[0] == maxlength:
S4data.append(S4datatmp)
S4180data.append(S4data180tmp)
azdata.append(aztmp)
eldata.append(eltmp)
times.append(timestmp)
selectstat.append(stat)
S4,fits,results = get_files(date = "2020-04-15",timest="00:00:00",timeend = "23:59") S4data = np.array(S4data)
S4data = [i['S4_data'] for i in results] S4180data = np.array(S4180data)
plt.plot(np.concatenate(S4data)) azdata = np.array(azdata)
eldata = np.array(eldata)
times = times[0]
plt.plot(times.datetime, np.array(S4data).T ,'o')
plt.legend(selectstat)
plt.show() plt.show()
return S4data, S4180data, azdata, eldata, times, selectstat
def get_all_velocities():
mydict = {}
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
myfilter = json.dumps({'EW_velocity': {'$exists': True}})
response = requests.get(url, params={'filter': myfilter})
vel = []
skip = 100
while response.status_code == 200:
response_json = response.json()
results = response_json['results']
if len(results) == 0:
break
print(f"Number of results: {len(results)}")
for result in response_json['results'][:]:
vel.append(get_velocity(result))
mydict[result['sample_start_datetime']] = {}
mydict[result['sample_start_datetime']]['velocities'] = vel[-1]
# get azel
#myfilter2 = json.dumps({"$and":[ {'sample_start_datetime': result['sample_start_datetime'] },
# {'azimuth': {'$exists': True}}] })
myfilter2 = json.dumps({'sample_start_datetime': result['sample_start_datetime'] })
try:
print("trying..",url, myfilter2)
response = requests.get(url, params = {'filter': myfilter2})
except:
print("failed.. try again?")
response = requests.get(url, params = {'filter': myfilter2})
for result2 in response.json()['results']:
if not "LOFAR_CORE" in result2['filename'] and 'azimuth' in result2.keys():
break;
if not 'azimuth' in result2.keys():
print (result2['filename'])
az,el = [-9999],[-9999]
else:
az,el = get_azel(result2)
if not 'S4_data' in result2.keys():
s4 = [-9999]
else:
s4 = get_S4_data(result2)
if not 'S4_data_180' in result2.keys():
s4_180 = [-9999]
else:
s4_180 = get_S4_180_data(result2)
mydict[result['sample_start_datetime']]['azimuth'] = az
mydict[result['sample_start_datetime']]['elevation'] = el
mydict[result['sample_start_datetime']]['target'] = result2['TARGET']
mydict[result['sample_start_datetime']]['s4'] = s4
mydict[result['sample_start_datetime']]['s4_180'] = s4_180
url_next = response_json['next']
url_next = f"{url}?{url_next.split('?')[-1]}"
response = requests.get(url_next)
target = [mydict[i]['target'] for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
vel = [np.array(mydict[i]['velocities']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
s4 = [np.array(mydict[i]['s4']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
s4_180 = [np.array(mydict[i]['s4_180']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
az = [np.array(mydict[i]['azimuth']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
el = [np.array(mydict[i]['elevation']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
times = [i for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
dtimes = [np.arange(np.array(mydict[i]['velocities']).shape[1])*u.min for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
times = [Time(i) + dt for i,dt in zip (times,dtimes)]
times = Time(times)
#vel = np.concatenate(vel,axis=1)
#s4 = np.concatenate(s4)
#el = np.concatenate(el)
#az = np.concatenate(az)
#avel = np.ma.array(vel, mask = np.abs(vel)>2000)
target = [[i]*j.shape[1] for i,j,k in zip(target,vel,s4_180) if k.shape[0]>=j.shape[1]]
s4_2 = [i[:j.shape[1]] for i,j,k in zip(s4,vel,s4_180) if k.shape[0]>=j.shape[1]]
az_2 = [i[:j.shape[1]] for i,j,k in zip(az,vel,s4_180) if k.shape[0]>=j.shape[1]]
el_2 = [i[:j.shape[1]] for i,j,k in zip(el,vel,s4_180) if k.shape[0]>=j.shape[1]]
s4_180_2 = [i[:j.shape[1]] for i,j in zip(s4_180,vel) if i.shape[0]>=j.shape[1]]
vel_2 = [j for i,j in zip(s4_180,vel) if i.shape[0]>=j.shape[1]]
idx=0; skipidx=[];
for i, (j,v) in enumerate(zip(s4_180,vel)):
if j.shape[0]<v.shape[1]:
skipidx += range(idx,idx+v.shape[1])
print(f"skipping {i} {idx} {v.shape[1]}")
idx+=v.shape[1]
times_skip = np.copy(times)
times_skip = np.delete(times_skip, skipidx)
return vel_2,s4_2,el_2,az_2, times_skip, s4_180_2, target, mydict
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment