Skip to content
Snippets Groups Projects
Commit 275c7384 authored by Maaijke Mevius's avatar Maaijke Mevius
Browse files

layout

parent 68979a76
Branches
Tags
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 264 additions and 146 deletions
query_data.py
+ 264
− 146
View file @ 275c7384
......@@ -8,7 +8,21 @@ import json
from astropy.time import Time
import astropy.units as u
months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
months = [
"Jan",
"Feb",
"Mar",
"Apr",
"May",
"Jun",
"Jul",
"Aug",
"Sep",
"Oct",
"Nov",
"Dec",
]
def reformat_date(datestr):
......@@ -19,40 +33,56 @@ def reformat_date(datestr):
def get_fits(my_list):
fits_file = [f for f in my_list if 'fits' in f][0]
fits_file = [f for f in my_list if "fits" in f][0]
return fits_file
def get_S4(my_list):
S4file = [f for f in my_list if 'S4.png' in f][0]
S4file = [f for f in my_list if "S4.png" in f][0]
return S4file
def get_png(my_list):
S4file = [f for f in my_list if 'png' in f][0]
S4file = [f for f in my_list if "png" in f][0]
return S4file
def get_velocity(result):
ew = result['EW_velocity']
ns = result['NS_velocity']
ew = result["EW_velocity"]
ns = result["NS_velocity"]
return ew, ns
def get_azel(result):
az = result['azimuth']
el = result['elevation']
az = result["azimuth"]
el = result["elevation"]
return az, el
def get_S4_data(result):
return result['S4_data']
return result["S4_data"]
def get_S4_180_data(result):
return result['S4_data_180']
return result["S4_data_180"]
def get_files(date, timest, timeend):
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
time1 = Time(date+"T"+timest,format='isot', scale='utc')
url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
time1 = Time(date + "T" + timest, format="isot", scale="utc")
start_time_iso = (time1).isot
time2 = Time(date+"T"+timeend,format='isot', scale='utc')
time2 = Time(date + "T" + timeend, format="isot", scale="utc")
end_time_iso = (time2).isot
myfilter = json.dumps({"$and":[{'sample_start_datetime': {'$lte':str(end_time_iso[:-4])}},
{'sample_end_datetime':{'$gte':str(start_time_iso[:-4])}}]})
myfilter = json.dumps(
{
"$and": [
{"sample_start_datetime": {"$lte": str(end_time_iso[:-4])}},
{"sample_end_datetime": {"$gte": str(start_time_iso[:-4])}},
]
}
)
# myfilter = json.dumps({"OBSERVATION_ID":obsid})
response = requests.get(url, params={'filter': myfilter})
response = requests.get(url, params={"filter": myfilter})
all_results = []
fitsfiles = []
......@@ -60,25 +90,25 @@ def get_files(date,timest,timeend):
while response.status_code == 200:
print(response)
response_json = response.json()
results = response_json['results']
results = response_json["results"]
if len(results) == 0:
break
print(f"Number of results: {len(results)}")
all_results += response_json['results']
for result in response_json['results'][:]:
all_results += response_json["results"]
for result in response_json["results"][:]:
try:
fitsfiles.append(get_fits(result['additional_files_url']))
fitsfiles.append(get_fits(result["additional_files_url"]))
except IndexError:
# print("No fits here", result['filename'])
pass
try:
S4files.append(get_png(result['additional_files_url']))
S4files.append(get_png(result["additional_files_url"]))
except IndexError:
# print("No S4 here", result)
pass
url = response_json['next']
url = response_json["next"]
try:
print("trying..")
response = requests.get(url)
......@@ -87,20 +117,21 @@ def get_files(date,timest,timeend):
response = requests.get(url)
return S4files, fitsfiles, all_results
def get_obsid(obsid):
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
myfilter = json.dumps({"OBSERVATION_ID": obsid})
response = requests.get(url, params={'filter': myfilter})
response = requests.get(url, params={"filter": myfilter})
all_results = []
while response.status_code == 200:
print(response)
response_json = response.json()
results = response_json['results']
results = response_json["results"]
if len(results) == 0:
break
print(f"Number of results: {len(results)}")
all_results += response_json['results']
url = response_json['next']
all_results += response_json["results"]
url = response_json["next"]
try:
print("trying..")
response = requests.get(url)
......@@ -110,24 +141,30 @@ def get_obsid(obsid):
return all_results
def get_timerange(timest, timeend):
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
def get_timerange(timest, timeend, extra_filter=[]):
url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
start_time_iso = timest.isot
end_time_iso = timeend.isot
myfilter = json.dumps({"$and":[{'sample_start_datetime': {'$lte':str(end_time_iso[:-4])}},
{'sample_end_datetime':{'$gte':str(start_time_iso[:-4])}}]})
response = requests.get(url, params={'filter': myfilter})
myfilterlist = [
{"sample_start_datetime": {"$lte": str(end_time_iso[:-4])}},
{"sample_end_datetime": {"$gte": str(start_time_iso[:-4])}},
]
myfilterlist += extra_filter
myfilter = json.dumps({"$and": myfilterlist})
response = requests.get(url, params={"filter": myfilter})
all_results = []
while response.status_code == 200:
print(response)
response_json = response.json()
results = response_json['results']
results = response_json["results"]
if len(results) == 0:
break
print(f"Number of results: {len(results)}")
all_results += response_json['results']
url = response_json['next']
all_results += response_json["results"]
url = response_json["next"]
try:
print("trying..")
response = requests.get(url)
......@@ -137,21 +174,53 @@ def get_timerange(timest, timeend):
return all_results
def plot_S4(timest=Time("2022-01-14T22:00:00"), timeend=Time("2022-01-15T22:00:00")):
results = get_timerange(timest=timest, timeend=timeend)
stations = set([i['BEAM_STATIONS_LIST'][0] for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys()])
def plot_S4(
timest=Time("2022-01-14T22:00:00"),
timeend=Time("2022-01-15T22:00:00"),
extra_filter=[],
):
results = get_timerange(timest=timest, timeend=timeend, extra_filter=extra_filter)
stations = set(
[
i["BEAM_STATIONS_LIST"][0]
for i in results
if "S4_data" in i.keys() and "BEAM_STATIONS_LIST" in i.keys()
]
)
stations = sorted(list(stations))
S4data = []
times = []
maxlength = 0
selectstat = []
for stat in stations:
S4datatmp = ([(i['S4_data'], i['S4_data_180'], i['sample_start_datetime'], i['azimuth'], i['elevation']) for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys() and stat in i['BEAM_STATIONS_LIST'] ])
timestmp = [Time(j)+np.arange(len(i))*u.min for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
aztmp = [az[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
eltmp = [el[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
S4data180tmp = [i2 for i,i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
S4datatmp = [
(
i["S4_data"],
i["S4_data_180"],
i["sample_start_datetime"],
i["azimuth"],
i["elevation"],
)
for i in results
if "S4_data" in i.keys()
and "BEAM_STATIONS_LIST" in i.keys()
and stat in i["BEAM_STATIONS_LIST"]
]
timestmp = [
Time(j) + np.arange(len(i)) * u.min
for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
]
aztmp = [
az[: len(i)] for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
]
eltmp = [
el[: len(i)] for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
]
S4data180tmp = [
i2 for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
]
S4datatmp = [i for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
S4datatmp = np.concatenate(S4datatmp)
timestmp = Time(np.concatenate(timestmp))
......@@ -179,95 +248,133 @@ def plot_S4(timest=Time("2022-01-14T22:00:00"), timeend=Time("2022-01-15T22:00:0
azdata = np.array(azdata)
eldata = np.array(eldata)
times = times[0]
plt.plot(times.datetime, np.array(S4data).T ,'o')
plt.plot(times.datetime, np.array(S4data).T, "o")
plt.legend(selectstat)
plt.show()
return S4data, S4180data, azdata, eldata, times, selectstat
def get_all_velocities():
mydict = {}
url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
myfilter = json.dumps({'EW_velocity': {'$exists': True}})
response = requests.get(url, params={'filter': myfilter})
url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
myfilter = json.dumps({"EW_velocity": {"$exists": True}})
response = requests.get(url, params={"filter": myfilter})
vel = []
skip = 100
while response.status_code == 200:
response_json = response.json()
results = response_json['results']
results = response_json["results"]
if len(results) == 0:
break
print(f"Number of results: {len(results)}")
for result in response_json['results'][:]:
for result in response_json["results"][:]:
vel.append(get_velocity(result))
mydict[result['sample_start_datetime']] = {}
mydict[result['sample_start_datetime']]['velocities'] = vel[-1]
mydict[result["sample_start_datetime"]] = {}
mydict[result["sample_start_datetime"]]["velocities"] = vel[-1]
# get azel
# myfilter2 = json.dumps({"$and":[ {'sample_start_datetime': result['sample_start_datetime'] },
# {'azimuth': {'$exists': True}}] })
myfilter2 = json.dumps({'sample_start_datetime': result['sample_start_datetime'] })
myfilter2 = json.dumps(
{"sample_start_datetime": result["sample_start_datetime"]}
)
try:
print("trying..", url, myfilter2)
response = requests.get(url, params = {'filter': myfilter2})
response = requests.get(url, params={"filter": myfilter2})
except:
print("failed.. try again?")
response = requests.get(url, params = {'filter': myfilter2})
for result2 in response.json()['results']:
if not "LOFAR_CORE" in result2['filename'] and 'azimuth' in result2.keys():
break;
if not 'azimuth' in result2.keys():
print (result2['filename'])
response = requests.get(url, params={"filter": myfilter2})
for result2 in response.json()["results"]:
if (
not "LOFAR_CORE" in result2["filename"]
and "azimuth" in result2.keys()
):
break
if not "azimuth" in result2.keys():
print(result2["filename"])
az, el = [-9999], [-9999]
else:
az, el = get_azel(result2)
if not 'S4_data' in result2.keys():
if not "S4_data" in result2.keys():
s4 = [-9999]
else:
s4 = get_S4_data(result2)
if not 'S4_data_180' in result2.keys():
if not "S4_data_180" in result2.keys():
s4_180 = [-9999]
else:
s4_180 = get_S4_180_data(result2)
mydict[result['sample_start_datetime']]['azimuth'] = az
mydict[result['sample_start_datetime']]['elevation'] = el
mydict[result['sample_start_datetime']]['target'] = result2['TARGET']
mydict[result['sample_start_datetime']]['s4'] = s4
mydict[result['sample_start_datetime']]['s4_180'] = s4_180
url_next = response_json['next']
mydict[result["sample_start_datetime"]]["azimuth"] = az
mydict[result["sample_start_datetime"]]["elevation"] = el
mydict[result["sample_start_datetime"]]["target"] = result2["TARGET"]
mydict[result["sample_start_datetime"]]["s4"] = s4
mydict[result["sample_start_datetime"]]["s4_180"] = s4_180
url_next = response_json["next"]
url_next = f"{url}?{url_next.split('?')[-1]}"
response = requests.get(url_next)
target = [mydict[i]['target'] for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
vel = [np.array(mydict[i]['velocities']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
s4 = [np.array(mydict[i]['s4']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
s4_180 = [np.array(mydict[i]['s4_180']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
az = [np.array(mydict[i]['azimuth']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
el = [np.array(mydict[i]['elevation']) for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
times = [i for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
dtimes = [np.arange(np.array(mydict[i]['velocities']).shape[1])*u.min for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]['velocities']) == None)
and mydict[i]['velocities'][0] != -9999
and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
target = [
mydict[i]["target"]
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
vel = [
np.array(mydict[i]["velocities"])
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
s4 = [
np.array(mydict[i]["s4"])
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
s4_180 = [
np.array(mydict[i]["s4_180"])
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
az = [
np.array(mydict[i]["azimuth"])
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
el = [
np.array(mydict[i]["elevation"])
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
times = [
i
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
dtimes = [
np.arange(np.array(mydict[i]["velocities"]).shape[1]) * u.min
for i in sorted(list(mydict.keys()))
if not np.any(np.array(mydict[i]["velocities"]) == None)
and mydict[i]["velocities"][0] != -9999
and type(mydict[i]["azimuth"]) == list
and mydict[i]["azimuth"][0] != -9999
]
times = [Time(i) + dt for i, dt in zip(times, dtimes)]
times = Time(times)
# vel = np.concatenate(vel,axis=1)
......@@ -275,13 +382,26 @@ def get_all_velocities():
# el = np.concatenate(el)
# az = np.concatenate(az)
# avel = np.ma.array(vel, mask = np.abs(vel)>2000)
target = [[i]*j.shape[1] for i,j,k in zip(target,vel,s4_180) if k.shape[0]>=j.shape[1]]
s4_2 = [i[:j.shape[1]] for i,j,k in zip(s4,vel,s4_180) if k.shape[0]>=j.shape[1]]
az_2 = [i[:j.shape[1]] for i,j,k in zip(az,vel,s4_180) if k.shape[0]>=j.shape[1]]
el_2 = [i[:j.shape[1]] for i,j,k in zip(el,vel,s4_180) if k.shape[0]>=j.shape[1]]
s4_180_2 = [i[:j.shape[1]] for i,j in zip(s4_180,vel) if i.shape[0]>=j.shape[1]]
target = [
[i] * j.shape[1]
for i, j, k in zip(target, vel, s4_180)
if k.shape[0] >= j.shape[1]
]
s4_2 = [
i[: j.shape[1]] for i, j, k in zip(s4, vel, s4_180) if k.shape[0] >= j.shape[1]
]
az_2 = [
i[: j.shape[1]] for i, j, k in zip(az, vel, s4_180) if k.shape[0] >= j.shape[1]
]
el_2 = [
i[: j.shape[1]] for i, j, k in zip(el, vel, s4_180) if k.shape[0] >= j.shape[1]
]
s4_180_2 = [
i[: j.shape[1]] for i, j in zip(s4_180, vel) if i.shape[0] >= j.shape[1]
]
vel_2 = [j for i, j in zip(s4_180, vel) if i.shape[0] >= j.shape[1]]
idx=0; skipidx=[];
idx = 0
skipidx = []
for i, (j, v) in enumerate(zip(s4_180, vel)):
if j.shape[0] < v.shape[1]:
skipidx += range(idx, idx + v.shape[1])
......@@ -291,5 +411,3 @@ def get_all_velocities():
times_skip = np.delete(times_skip, skipidx)
return vel_2, s4_2, el_2, az_2, times_skip, s4_180_2, target, mydict
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment