diff --git a/query_data.py b/query_data.py index 9625be83c2c10d4f17143c8ce2d12b231623801d..2435a98692e8f443a2fd571908a65b547c939f54 100755 --- a/query_data.py +++ b/query_data.py @@ -2,157 +2,226 @@ import numpy as np import matplotlib.pyplot as plt -from urllib.request import urlopen,urlretrieve +from urllib.request import urlopen, urlretrieve import requests import json from astropy.time import Time import astropy.units as u -months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] +months = [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", +] + def reformat_date(datestr): - + month = datestr.split("-") - month[1] = months.index(month[1])+1 + month[1] = months.index(month[1]) + 1 month = [str(i) for i in month[::-1]] return "-".join(month) - + def get_fits(my_list): - fits_file = [f for f in my_list if 'fits' in f][0] + fits_file = [f for f in my_list if "fits" in f][0] return fits_file + + def get_S4(my_list): - S4file = [f for f in my_list if 'S4.png' in f][0] + S4file = [f for f in my_list if "S4.png" in f][0] return S4file + + def get_png(my_list): - S4file = [f for f in my_list if 'png' in f][0] + S4file = [f for f in my_list if "png" in f][0] return S4file + def get_velocity(result): - ew = result['EW_velocity'] - ns = result['NS_velocity'] - return ew,ns - + ew = result["EW_velocity"] + ns = result["NS_velocity"] + return ew, ns + + def get_azel(result): - az = result['azimuth'] - el = result['elevation'] - return az,el + az = result["azimuth"] + el = result["elevation"] + return az, el + def get_S4_data(result): - return result['S4_data'] + return result["S4_data"] + + def get_S4_180_data(result): - return result['S4_data_180'] - -def get_files(date,timest,timeend): - url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/' - time1 = Time(date+"T"+timest,format='isot', scale='utc') + return result["S4_data_180"] + + +def get_files(date, timest, timeend): + url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/" + time1 = Time(date + "T" + timest, format="isot", scale="utc") start_time_iso = (time1).isot - time2 = Time(date+"T"+timeend,format='isot', scale='utc') + time2 = Time(date + "T" + timeend, format="isot", scale="utc") end_time_iso = (time2).isot - myfilter = json.dumps({"$and":[{'sample_start_datetime': {'$lte':str(end_time_iso[:-4])}}, - {'sample_end_datetime':{'$gte':str(start_time_iso[:-4])}}]}) - #myfilter = json.dumps({"OBSERVATION_ID":obsid}) - response = requests.get(url, params={'filter': myfilter}) + myfilter = json.dumps( + { + "$and": [ + {"sample_start_datetime": {"$lte": str(end_time_iso[:-4])}}, + {"sample_end_datetime": {"$gte": str(start_time_iso[:-4])}}, + ] + } + ) + # myfilter = json.dumps({"OBSERVATION_ID":obsid}) + response = requests.get(url, params={"filter": myfilter}) all_results = [] fitsfiles = [] - S4files=[] + S4files = [] while response.status_code == 200: print(response) response_json = response.json() - results = response_json['results'] + results = response_json["results"] if len(results) == 0: break print(f"Number of results: {len(results)}") - all_results += response_json['results'] - for result in response_json['results'][:]: + all_results += response_json["results"] + for result in response_json["results"][:]: try: - fitsfiles.append(get_fits(result['additional_files_url'])) + fitsfiles.append(get_fits(result["additional_files_url"])) except IndexError: - #print("No fits here", result['filename']) + # print("No fits here", result['filename']) pass try: - S4files.append(get_png(result['additional_files_url'])) + S4files.append(get_png(result["additional_files_url"])) except IndexError: - #print("No S4 here", result) + # print("No S4 here", result) pass - url = response_json['next'] + url = response_json["next"] try: print("trying..") response = requests.get(url) except: print("failed.. try again?") response = requests.get(url) - return S4files, fitsfiles,all_results + return S4files, fitsfiles, all_results + def get_obsid(obsid): - url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/' - myfilter = json.dumps({"OBSERVATION_ID":obsid}) - response = requests.get(url, params={'filter': myfilter}) + url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/" + myfilter = json.dumps({"OBSERVATION_ID": obsid}) + response = requests.get(url, params={"filter": myfilter}) all_results = [] while response.status_code == 200: print(response) response_json = response.json() - results = response_json['results'] + results = response_json["results"] if len(results) == 0: break print(f"Number of results: {len(results)}") - all_results += response_json['results'] - url = response_json['next'] + all_results += response_json["results"] + url = response_json["next"] try: print("trying..") response = requests.get(url) except: print("failed.. try again?") response = requests.get(url) - + return all_results -def get_timerange(timest, timeend): - url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/' + +def get_timerange(timest, timeend, extra_filter=[]): + url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/" start_time_iso = timest.isot end_time_iso = timeend.isot - - myfilter = json.dumps({"$and":[{'sample_start_datetime': {'$lte':str(end_time_iso[:-4])}}, - {'sample_end_datetime':{'$gte':str(start_time_iso[:-4])}}]}) - response = requests.get(url, params={'filter': myfilter}) + + myfilterlist = [ + {"sample_start_datetime": {"$lte": str(end_time_iso[:-4])}}, + {"sample_end_datetime": {"$gte": str(start_time_iso[:-4])}}, + ] + myfilterlist += extra_filter + + myfilter = json.dumps({"$and": myfilterlist}) + response = requests.get(url, params={"filter": myfilter}) all_results = [] while response.status_code == 200: print(response) response_json = response.json() - results = response_json['results'] + results = response_json["results"] if len(results) == 0: break print(f"Number of results: {len(results)}") - all_results += response_json['results'] - url = response_json['next'] + all_results += response_json["results"] + url = response_json["next"] try: print("trying..") response = requests.get(url) except: print("failed.. try again?") response = requests.get(url) - + return all_results -def plot_S4(timest=Time("2022-01-14T22:00:00"), timeend=Time("2022-01-15T22:00:00")): - - results = get_timerange(timest=timest, timeend=timeend) - stations = set([i['BEAM_STATIONS_LIST'][0] for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys()]) + +def plot_S4( + timest=Time("2022-01-14T22:00:00"), + timeend=Time("2022-01-15T22:00:00"), + extra_filter=[], +): + + results = get_timerange(timest=timest, timeend=timeend, extra_filter=extra_filter) + stations = set( + [ + i["BEAM_STATIONS_LIST"][0] + for i in results + if "S4_data" in i.keys() and "BEAM_STATIONS_LIST" in i.keys() + ] + ) stations = sorted(list(stations)) S4data = [] times = [] - maxlength =0 + maxlength = 0 selectstat = [] for stat in stations: - S4datatmp = ([(i['S4_data'], i['S4_data_180'], i['sample_start_datetime'], i['azimuth'], i['elevation']) for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys() and stat in i['BEAM_STATIONS_LIST'] ]) - timestmp = [Time(j)+np.arange(len(i))*u.min for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])] - aztmp = [az[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])] - eltmp = [el[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])] - S4data180tmp = [i2 for i,i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])] - S4datatmp = [i for i,i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])] + S4datatmp = [ + ( + i["S4_data"], + i["S4_data_180"], + i["sample_start_datetime"], + i["azimuth"], + i["elevation"], + ) + for i in results + if "S4_data" in i.keys() + and "BEAM_STATIONS_LIST" in i.keys() + and stat in i["BEAM_STATIONS_LIST"] + ] + timestmp = [ + Time(j) + np.arange(len(i)) * u.min + for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2]) + ] + aztmp = [ + az[: len(i)] for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2]) + ] + eltmp = [ + el[: len(i)] for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2]) + ] + S4data180tmp = [ + i2 for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2]) + ] + S4datatmp = [i for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])] S4datatmp = np.concatenate(S4datatmp) timestmp = Time(np.concatenate(timestmp)) S4data180tmp = np.concatenate(S4data180tmp) @@ -173,123 +242,172 @@ def plot_S4(timest=Time("2022-01-14T22:00:00"), timeend=Time("2022-01-15T22:00:0 eldata.append(eltmp) times.append(timestmp) selectstat.append(stat) - + S4data = np.array(S4data) S4180data = np.array(S4180data) azdata = np.array(azdata) eldata = np.array(eldata) times = times[0] - plt.plot(times.datetime, np.array(S4data).T ,'o') + plt.plot(times.datetime, np.array(S4data).T, "o") plt.legend(selectstat) plt.show() return S4data, S4180data, azdata, eldata, times, selectstat + def get_all_velocities(): mydict = {} - url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/' - myfilter = json.dumps({'EW_velocity': {'$exists': True}}) - response = requests.get(url, params={'filter': myfilter}) + url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/" + myfilter = json.dumps({"EW_velocity": {"$exists": True}}) + response = requests.get(url, params={"filter": myfilter}) vel = [] skip = 100 while response.status_code == 200: response_json = response.json() - results = response_json['results'] + results = response_json["results"] if len(results) == 0: break print(f"Number of results: {len(results)}") - for result in response_json['results'][:]: + for result in response_json["results"][:]: vel.append(get_velocity(result)) - mydict[result['sample_start_datetime']] = {} - mydict[result['sample_start_datetime']]['velocities'] = vel[-1] + mydict[result["sample_start_datetime"]] = {} + mydict[result["sample_start_datetime"]]["velocities"] = vel[-1] # get azel - #myfilter2 = json.dumps({"$and":[ {'sample_start_datetime': result['sample_start_datetime'] }, + # myfilter2 = json.dumps({"$and":[ {'sample_start_datetime': result['sample_start_datetime'] }, # {'azimuth': {'$exists': True}}] }) - myfilter2 = json.dumps({'sample_start_datetime': result['sample_start_datetime'] }) + myfilter2 = json.dumps( + {"sample_start_datetime": result["sample_start_datetime"]} + ) try: - print("trying..",url, myfilter2) - response = requests.get(url, params = {'filter': myfilter2}) + print("trying..", url, myfilter2) + response = requests.get(url, params={"filter": myfilter2}) except: print("failed.. try again?") - response = requests.get(url, params = {'filter': myfilter2}) - for result2 in response.json()['results']: - if not "LOFAR_CORE" in result2['filename'] and 'azimuth' in result2.keys(): - break; - if not 'azimuth' in result2.keys(): - print (result2['filename']) - az,el = [-9999],[-9999] + response = requests.get(url, params={"filter": myfilter2}) + for result2 in response.json()["results"]: + if ( + not "LOFAR_CORE" in result2["filename"] + and "azimuth" in result2.keys() + ): + break + if not "azimuth" in result2.keys(): + print(result2["filename"]) + az, el = [-9999], [-9999] else: - az,el = get_azel(result2) - if not 'S4_data' in result2.keys(): + az, el = get_azel(result2) + if not "S4_data" in result2.keys(): s4 = [-9999] else: s4 = get_S4_data(result2) - if not 'S4_data_180' in result2.keys(): + if not "S4_data_180" in result2.keys(): s4_180 = [-9999] else: s4_180 = get_S4_180_data(result2) - mydict[result['sample_start_datetime']]['azimuth'] = az - mydict[result['sample_start_datetime']]['elevation'] = el - mydict[result['sample_start_datetime']]['target'] = result2['TARGET'] - mydict[result['sample_start_datetime']]['s4'] = s4 - mydict[result['sample_start_datetime']]['s4_180'] = s4_180 - url_next = response_json['next'] + mydict[result["sample_start_datetime"]]["azimuth"] = az + mydict[result["sample_start_datetime"]]["elevation"] = el + mydict[result["sample_start_datetime"]]["target"] = result2["TARGET"] + mydict[result["sample_start_datetime"]]["s4"] = s4 + mydict[result["sample_start_datetime"]]["s4_180"] = s4_180 + url_next = response_json["next"] url_next = f"{url}?{url_next.split('?')[-1]}" response = requests.get(url_next) - target = [mydict[i]['target'] for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - vel = [np.array(mydict[i]['velocities']) for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - s4 = [np.array(mydict[i]['s4']) for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - s4_180 = [np.array(mydict[i]['s4_180']) for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - - az = [np.array(mydict[i]['azimuth']) for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - el = [np.array(mydict[i]['elevation']) for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - times = [i for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - dtimes = [np.arange(np.array(mydict[i]['velocities']).shape[1])*u.min for i in sorted(list(mydict.keys())) - if not np.any(np.array(mydict[i]['velocities']) == None) - and mydict[i]['velocities'][0] != -9999 - and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ] - times = [Time(i) + dt for i,dt in zip (times,dtimes)] + target = [ + mydict[i]["target"] + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + vel = [ + np.array(mydict[i]["velocities"]) + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + s4 = [ + np.array(mydict[i]["s4"]) + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + s4_180 = [ + np.array(mydict[i]["s4_180"]) + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + + az = [ + np.array(mydict[i]["azimuth"]) + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + el = [ + np.array(mydict[i]["elevation"]) + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + times = [ + i + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + dtimes = [ + np.arange(np.array(mydict[i]["velocities"]).shape[1]) * u.min + for i in sorted(list(mydict.keys())) + if not np.any(np.array(mydict[i]["velocities"]) == None) + and mydict[i]["velocities"][0] != -9999 + and type(mydict[i]["azimuth"]) == list + and mydict[i]["azimuth"][0] != -9999 + ] + times = [Time(i) + dt for i, dt in zip(times, dtimes)] times = Time(times) - #vel = np.concatenate(vel,axis=1) - #s4 = np.concatenate(s4) - #el = np.concatenate(el) - #az = np.concatenate(az) - #avel = np.ma.array(vel, mask = np.abs(vel)>2000) - target = [[i]*j.shape[1] for i,j,k in zip(target,vel,s4_180) if k.shape[0]>=j.shape[1]] - s4_2 = [i[:j.shape[1]] for i,j,k in zip(s4,vel,s4_180) if k.shape[0]>=j.shape[1]] - az_2 = [i[:j.shape[1]] for i,j,k in zip(az,vel,s4_180) if k.shape[0]>=j.shape[1]] - el_2 = [i[:j.shape[1]] for i,j,k in zip(el,vel,s4_180) if k.shape[0]>=j.shape[1]] - s4_180_2 = [i[:j.shape[1]] for i,j in zip(s4_180,vel) if i.shape[0]>=j.shape[1]] - vel_2 = [j for i,j in zip(s4_180,vel) if i.shape[0]>=j.shape[1]] - idx=0; skipidx=[]; - for i, (j,v) in enumerate(zip(s4_180,vel)): - if j.shape[0]<v.shape[1]: - skipidx += range(idx,idx+v.shape[1]) + # vel = np.concatenate(vel,axis=1) + # s4 = np.concatenate(s4) + # el = np.concatenate(el) + # az = np.concatenate(az) + # avel = np.ma.array(vel, mask = np.abs(vel)>2000) + target = [ + [i] * j.shape[1] + for i, j, k in zip(target, vel, s4_180) + if k.shape[0] >= j.shape[1] + ] + s4_2 = [ + i[: j.shape[1]] for i, j, k in zip(s4, vel, s4_180) if k.shape[0] >= j.shape[1] + ] + az_2 = [ + i[: j.shape[1]] for i, j, k in zip(az, vel, s4_180) if k.shape[0] >= j.shape[1] + ] + el_2 = [ + i[: j.shape[1]] for i, j, k in zip(el, vel, s4_180) if k.shape[0] >= j.shape[1] + ] + s4_180_2 = [ + i[: j.shape[1]] for i, j in zip(s4_180, vel) if i.shape[0] >= j.shape[1] + ] + vel_2 = [j for i, j in zip(s4_180, vel) if i.shape[0] >= j.shape[1]] + idx = 0 + skipidx = [] + for i, (j, v) in enumerate(zip(s4_180, vel)): + if j.shape[0] < v.shape[1]: + skipidx += range(idx, idx + v.shape[1]) print(f"skipping {i} {idx} {v.shape[1]}") - idx+=v.shape[1] + idx += v.shape[1] times_skip = np.copy(times) times_skip = np.delete(times_skip, skipidx) - - return vel_2,s4_2,el_2,az_2, times_skip, s4_180_2, target, mydict - + return vel_2, s4_2, el_2, az_2, times_skip, s4_180_2, target, mydict