diff --git a/query_data.py b/query_data.py
index 9625be83c2c10d4f17143c8ce2d12b231623801d..2435a98692e8f443a2fd571908a65b547c939f54 100755
--- a/query_data.py
+++ b/query_data.py
@@ -2,157 +2,226 @@
 
 import numpy as np
 import matplotlib.pyplot as plt
-from urllib.request import urlopen,urlretrieve
+from urllib.request import urlopen, urlretrieve
 import requests
 import json
 from astropy.time import Time
 import astropy.units as u
 
-months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
+months = [
+    "Jan",
+    "Feb",
+    "Mar",
+    "Apr",
+    "May",
+    "Jun",
+    "Jul",
+    "Aug",
+    "Sep",
+    "Oct",
+    "Nov",
+    "Dec",
+]
+
 
 def reformat_date(datestr):
-    
+
     month = datestr.split("-")
-    month[1] = months.index(month[1])+1
+    month[1] = months.index(month[1]) + 1
     month = [str(i) for i in month[::-1]]
     return "-".join(month)
 
-    
+
 def get_fits(my_list):
-    fits_file = [f for f in my_list if 'fits' in f][0]
+    fits_file = [f for f in my_list if "fits" in f][0]
     return fits_file
+
+
 def get_S4(my_list):
-    S4file = [f for f in my_list if 'S4.png' in f][0]
+    S4file = [f for f in my_list if "S4.png" in f][0]
     return S4file
+
+
 def get_png(my_list):
-    S4file = [f for f in my_list if 'png' in f][0]
+    S4file = [f for f in my_list if "png" in f][0]
     return S4file
 
+
 def get_velocity(result):
-    ew = result['EW_velocity']
-    ns = result['NS_velocity']
-    return ew,ns
-    
+    ew = result["EW_velocity"]
+    ns = result["NS_velocity"]
+    return ew, ns
+
+
 def get_azel(result):
-    az = result['azimuth']
-    el = result['elevation']
-    return az,el
+    az = result["azimuth"]
+    el = result["elevation"]
+    return az, el
+
 
 def get_S4_data(result):
-    return result['S4_data']
+    return result["S4_data"]
+
+
 def get_S4_180_data(result):
-    return result['S4_data_180']
-    
-def get_files(date,timest,timeend):
-    url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
-    time1 = Time(date+"T"+timest,format='isot', scale='utc')
+    return result["S4_data_180"]
+
+
+def get_files(date, timest, timeend):
+    url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
+    time1 = Time(date + "T" + timest, format="isot", scale="utc")
     start_time_iso = (time1).isot
-    time2 = Time(date+"T"+timeend,format='isot', scale='utc')
+    time2 = Time(date + "T" + timeend, format="isot", scale="utc")
     end_time_iso = (time2).isot
-    myfilter = json.dumps({"$and":[{'sample_start_datetime': {'$lte':str(end_time_iso[:-4])}},
-                                   {'sample_end_datetime':{'$gte':str(start_time_iso[:-4])}}]})
-    #myfilter = json.dumps({"OBSERVATION_ID":obsid})
-    response = requests.get(url, params={'filter': myfilter})
+    myfilter = json.dumps(
+        {
+            "$and": [
+                {"sample_start_datetime": {"$lte": str(end_time_iso[:-4])}},
+                {"sample_end_datetime": {"$gte": str(start_time_iso[:-4])}},
+            ]
+        }
+    )
+    # myfilter = json.dumps({"OBSERVATION_ID":obsid})
+    response = requests.get(url, params={"filter": myfilter})
 
     all_results = []
     fitsfiles = []
-    S4files=[]
+    S4files = []
     while response.status_code == 200:
         print(response)
         response_json = response.json()
-        results = response_json['results']
+        results = response_json["results"]
 
         if len(results) == 0:
             break
         print(f"Number of results: {len(results)}")
-        all_results += response_json['results']
-        for result in response_json['results'][:]:
+        all_results += response_json["results"]
+        for result in response_json["results"][:]:
 
             try:
-                fitsfiles.append(get_fits(result['additional_files_url']))
+                fitsfiles.append(get_fits(result["additional_files_url"]))
             except IndexError:
-                    #print("No fits here", result['filename'])
+                # print("No fits here", result['filename'])
                 pass
             try:
-                S4files.append(get_png(result['additional_files_url']))
+                S4files.append(get_png(result["additional_files_url"]))
             except IndexError:
-                #print("No S4 here", result)
+                # print("No S4 here", result)
                 pass
-        url = response_json['next']
+        url = response_json["next"]
         try:
             print("trying..")
             response = requests.get(url)
         except:
             print("failed.. try again?")
             response = requests.get(url)
-    return S4files, fitsfiles,all_results
+    return S4files, fitsfiles, all_results
+
 
 def get_obsid(obsid):
-    url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
-    myfilter = json.dumps({"OBSERVATION_ID":obsid})
-    response = requests.get(url, params={'filter': myfilter})
+    url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
+    myfilter = json.dumps({"OBSERVATION_ID": obsid})
+    response = requests.get(url, params={"filter": myfilter})
     all_results = []
     while response.status_code == 200:
         print(response)
         response_json = response.json()
-        results = response_json['results']
+        results = response_json["results"]
         if len(results) == 0:
             break
         print(f"Number of results: {len(results)}")
-        all_results += response_json['results']
-        url = response_json['next']
+        all_results += response_json["results"]
+        url = response_json["next"]
         try:
             print("trying..")
             response = requests.get(url)
         except:
             print("failed.. try again?")
             response = requests.get(url)
-  
+
     return all_results
 
-def get_timerange(timest, timeend):
-    url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
+
+def get_timerange(timest, timeend, extra_filter=[]):
+    url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
     start_time_iso = timest.isot
     end_time_iso = timeend.isot
-    
-    myfilter = json.dumps({"$and":[{'sample_start_datetime': {'$lte':str(end_time_iso[:-4])}},
-                                   {'sample_end_datetime':{'$gte':str(start_time_iso[:-4])}}]})
-    response = requests.get(url, params={'filter': myfilter})
+
+    myfilterlist = [
+        {"sample_start_datetime": {"$lte": str(end_time_iso[:-4])}},
+        {"sample_end_datetime": {"$gte": str(start_time_iso[:-4])}},
+    ]
+    myfilterlist += extra_filter
+
+    myfilter = json.dumps({"$and": myfilterlist})
+    response = requests.get(url, params={"filter": myfilter})
     all_results = []
     while response.status_code == 200:
         print(response)
         response_json = response.json()
-        results = response_json['results']
+        results = response_json["results"]
         if len(results) == 0:
             break
         print(f"Number of results: {len(results)}")
-        all_results += response_json['results']
-        url = response_json['next']
+        all_results += response_json["results"]
+        url = response_json["next"]
         try:
             print("trying..")
             response = requests.get(url)
         except:
             print("failed.. try again?")
             response = requests.get(url)
-  
+
     return all_results
 
-def plot_S4(timest=Time("2022-01-14T22:00:00"), timeend=Time("2022-01-15T22:00:00")):
-    
-    results = get_timerange(timest=timest, timeend=timeend)
-    stations = set([i['BEAM_STATIONS_LIST'][0] for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys()])
+
+def plot_S4(
+    timest=Time("2022-01-14T22:00:00"),
+    timeend=Time("2022-01-15T22:00:00"),
+    extra_filter=[],
+):
+
+    results = get_timerange(timest=timest, timeend=timeend, extra_filter=extra_filter)
+    stations = set(
+        [
+            i["BEAM_STATIONS_LIST"][0]
+            for i in results
+            if "S4_data" in i.keys() and "BEAM_STATIONS_LIST" in i.keys()
+        ]
+    )
     stations = sorted(list(stations))
     S4data = []
     times = []
-    maxlength =0
+    maxlength = 0
     selectstat = []
     for stat in stations:
-        S4datatmp = ([(i['S4_data'], i['S4_data_180'], i['sample_start_datetime'], i['azimuth'], i['elevation']) for i in results if 'S4_data' in i.keys() and 'BEAM_STATIONS_LIST' in i.keys() and stat in i['BEAM_STATIONS_LIST'] ])
-        timestmp = [Time(j)+np.arange(len(i))*u.min for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
-        aztmp = [az[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
-        eltmp = [el[:len(i)] for i, i2,j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
-        S4data180tmp = [i2 for i,i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
-        S4datatmp = [i for i,i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
+        S4datatmp = [
+            (
+                i["S4_data"],
+                i["S4_data_180"],
+                i["sample_start_datetime"],
+                i["azimuth"],
+                i["elevation"],
+            )
+            for i in results
+            if "S4_data" in i.keys()
+            and "BEAM_STATIONS_LIST" in i.keys()
+            and stat in i["BEAM_STATIONS_LIST"]
+        ]
+        timestmp = [
+            Time(j) + np.arange(len(i)) * u.min
+            for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
+        ]
+        aztmp = [
+            az[: len(i)] for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
+        ]
+        eltmp = [
+            el[: len(i)] for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
+        ]
+        S4data180tmp = [
+            i2 for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])
+        ]
+        S4datatmp = [i for i, i2, j, az, el in sorted(S4datatmp, key=lambda x: x[2])]
         S4datatmp = np.concatenate(S4datatmp)
         timestmp = Time(np.concatenate(timestmp))
         S4data180tmp = np.concatenate(S4data180tmp)
@@ -173,123 +242,172 @@ def plot_S4(timest=Time("2022-01-14T22:00:00"), timeend=Time("2022-01-15T22:00:0
             eldata.append(eltmp)
             times.append(timestmp)
             selectstat.append(stat)
-            
+
     S4data = np.array(S4data)
     S4180data = np.array(S4180data)
     azdata = np.array(azdata)
     eldata = np.array(eldata)
     times = times[0]
-    plt.plot(times.datetime, np.array(S4data).T ,'o')
+    plt.plot(times.datetime, np.array(S4data).T, "o")
     plt.legend(selectstat)
     plt.show()
     return S4data, S4180data, azdata, eldata, times, selectstat
 
+
 def get_all_velocities():
     mydict = {}
-    url = 'https://spaceweather.astron.nl/api2/scintillation_preview/tasks/'
-    myfilter = json.dumps({'EW_velocity': {'$exists': True}})
-    response = requests.get(url, params={'filter': myfilter})
+    url = "https://spaceweather.astron.nl/api2/scintillation_preview/tasks/"
+    myfilter = json.dumps({"EW_velocity": {"$exists": True}})
+    response = requests.get(url, params={"filter": myfilter})
     vel = []
     skip = 100
     while response.status_code == 200:
         response_json = response.json()
-        results = response_json['results']
+        results = response_json["results"]
         if len(results) == 0:
             break
         print(f"Number of results: {len(results)}")
-        for result in response_json['results'][:]:
+        for result in response_json["results"][:]:
             vel.append(get_velocity(result))
-            mydict[result['sample_start_datetime']] = {}
-            mydict[result['sample_start_datetime']]['velocities'] = vel[-1]
+            mydict[result["sample_start_datetime"]] = {}
+            mydict[result["sample_start_datetime"]]["velocities"] = vel[-1]
             # get azel
-            #myfilter2 = json.dumps({"$and":[ {'sample_start_datetime': result['sample_start_datetime'] },
+            # myfilter2 = json.dumps({"$and":[ {'sample_start_datetime': result['sample_start_datetime'] },
             #                                 {'azimuth': {'$exists': True}}] })
-            myfilter2 = json.dumps({'sample_start_datetime': result['sample_start_datetime'] })
+            myfilter2 = json.dumps(
+                {"sample_start_datetime": result["sample_start_datetime"]}
+            )
             try:
-                print("trying..",url, myfilter2)
-                response = requests.get(url, params = {'filter': myfilter2})
+                print("trying..", url, myfilter2)
+                response = requests.get(url, params={"filter": myfilter2})
             except:
                 print("failed.. try again?")
-                response = requests.get(url, params = {'filter': myfilter2})
-            for result2 in response.json()['results']:
-                if not "LOFAR_CORE" in result2['filename'] and 'azimuth' in result2.keys():
-                    break;
-            if not 'azimuth' in result2.keys():
-                print (result2['filename'])
-                az,el = [-9999],[-9999]
+                response = requests.get(url, params={"filter": myfilter2})
+            for result2 in response.json()["results"]:
+                if (
+                    not "LOFAR_CORE" in result2["filename"]
+                    and "azimuth" in result2.keys()
+                ):
+                    break
+            if not "azimuth" in result2.keys():
+                print(result2["filename"])
+                az, el = [-9999], [-9999]
             else:
-                az,el = get_azel(result2)
-            if not 'S4_data' in result2.keys():
+                az, el = get_azel(result2)
+            if not "S4_data" in result2.keys():
                 s4 = [-9999]
             else:
                 s4 = get_S4_data(result2)
-            if not 'S4_data_180' in result2.keys():
+            if not "S4_data_180" in result2.keys():
                 s4_180 = [-9999]
             else:
                 s4_180 = get_S4_180_data(result2)
-            mydict[result['sample_start_datetime']]['azimuth'] = az
-            mydict[result['sample_start_datetime']]['elevation'] = el
-            mydict[result['sample_start_datetime']]['target'] = result2['TARGET']
-            mydict[result['sample_start_datetime']]['s4'] = s4
-            mydict[result['sample_start_datetime']]['s4_180'] = s4_180
-        url_next = response_json['next']
+            mydict[result["sample_start_datetime"]]["azimuth"] = az
+            mydict[result["sample_start_datetime"]]["elevation"] = el
+            mydict[result["sample_start_datetime"]]["target"] = result2["TARGET"]
+            mydict[result["sample_start_datetime"]]["s4"] = s4
+            mydict[result["sample_start_datetime"]]["s4_180"] = s4_180
+        url_next = response_json["next"]
         url_next = f"{url}?{url_next.split('?')[-1]}"
         response = requests.get(url_next)
-    target = [mydict[i]['target'] for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-           and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-    vel = [np.array(mydict[i]['velocities']) for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-           and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-    s4 = [np.array(mydict[i]['s4']) for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-           and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-    s4_180 = [np.array(mydict[i]['s4_180']) for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-           and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-
-    az = [np.array(mydict[i]['azimuth']) for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-           and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-    el = [np.array(mydict[i]['elevation']) for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-           and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-    times = [i for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-           and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-    dtimes = [np.arange(np.array(mydict[i]['velocities']).shape[1])*u.min for i in sorted(list(mydict.keys()))
-           if not np.any(np.array(mydict[i]['velocities']) == None)
-           and mydict[i]['velocities'][0] != -9999
-          and type(mydict[i]['azimuth'])==list and mydict[i]['azimuth'][0] != -9999 ]
-    times = [Time(i) + dt for i,dt in zip (times,dtimes)]
+    target = [
+        mydict[i]["target"]
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+    vel = [
+        np.array(mydict[i]["velocities"])
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+    s4 = [
+        np.array(mydict[i]["s4"])
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+    s4_180 = [
+        np.array(mydict[i]["s4_180"])
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+
+    az = [
+        np.array(mydict[i]["azimuth"])
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+    el = [
+        np.array(mydict[i]["elevation"])
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+    times = [
+        i
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+    dtimes = [
+        np.arange(np.array(mydict[i]["velocities"]).shape[1]) * u.min
+        for i in sorted(list(mydict.keys()))
+        if not np.any(np.array(mydict[i]["velocities"]) == None)
+        and mydict[i]["velocities"][0] != -9999
+        and type(mydict[i]["azimuth"]) == list
+        and mydict[i]["azimuth"][0] != -9999
+    ]
+    times = [Time(i) + dt for i, dt in zip(times, dtimes)]
     times = Time(times)
-    #vel = np.concatenate(vel,axis=1)
-    #s4 = np.concatenate(s4)
-    #el = np.concatenate(el)
-    #az = np.concatenate(az)
-    #avel = np.ma.array(vel, mask = np.abs(vel)>2000)
-    target = [[i]*j.shape[1] for i,j,k in zip(target,vel,s4_180) if k.shape[0]>=j.shape[1]]
-    s4_2 = [i[:j.shape[1]] for i,j,k in zip(s4,vel,s4_180) if k.shape[0]>=j.shape[1]]
-    az_2 = [i[:j.shape[1]] for i,j,k in zip(az,vel,s4_180) if k.shape[0]>=j.shape[1]]
-    el_2 = [i[:j.shape[1]] for i,j,k in zip(el,vel,s4_180) if k.shape[0]>=j.shape[1]]
-    s4_180_2 = [i[:j.shape[1]] for i,j in zip(s4_180,vel) if i.shape[0]>=j.shape[1]]
-    vel_2 = [j for i,j in zip(s4_180,vel) if i.shape[0]>=j.shape[1]]
-    idx=0; skipidx=[]; 
-    for i, (j,v) in enumerate(zip(s4_180,vel)):
-        if j.shape[0]<v.shape[1]:
-            skipidx += range(idx,idx+v.shape[1])
+    # vel = np.concatenate(vel,axis=1)
+    # s4 = np.concatenate(s4)
+    # el = np.concatenate(el)
+    # az = np.concatenate(az)
+    # avel = np.ma.array(vel, mask = np.abs(vel)>2000)
+    target = [
+        [i] * j.shape[1]
+        for i, j, k in zip(target, vel, s4_180)
+        if k.shape[0] >= j.shape[1]
+    ]
+    s4_2 = [
+        i[: j.shape[1]] for i, j, k in zip(s4, vel, s4_180) if k.shape[0] >= j.shape[1]
+    ]
+    az_2 = [
+        i[: j.shape[1]] for i, j, k in zip(az, vel, s4_180) if k.shape[0] >= j.shape[1]
+    ]
+    el_2 = [
+        i[: j.shape[1]] for i, j, k in zip(el, vel, s4_180) if k.shape[0] >= j.shape[1]
+    ]
+    s4_180_2 = [
+        i[: j.shape[1]] for i, j in zip(s4_180, vel) if i.shape[0] >= j.shape[1]
+    ]
+    vel_2 = [j for i, j in zip(s4_180, vel) if i.shape[0] >= j.shape[1]]
+    idx = 0
+    skipidx = []
+    for i, (j, v) in enumerate(zip(s4_180, vel)):
+        if j.shape[0] < v.shape[1]:
+            skipidx += range(idx, idx + v.shape[1])
             print(f"skipping {i} {idx} {v.shape[1]}")
-        idx+=v.shape[1]
+        idx += v.shape[1]
     times_skip = np.copy(times)
     times_skip = np.delete(times_skip, skipidx)
-    
-    return vel_2,s4_2,el_2,az_2, times_skip, s4_180_2, target, mydict
-
 
+    return vel_2, s4_2, el_2, az_2, times_skip, s4_180_2, target, mydict