diff --git a/database/models.py b/database/models.py index 79e2428381828c2eb80a3abdeb48addf82cd6381..98ad27fdfd3637a15451b256ccb07f03dd6a35e3 100644 --- a/database/models.py +++ b/database/models.py @@ -10,6 +10,7 @@ class SkyView(Base): ra = Column(Float, index=True) dec = Column(Float, index=True) observation = Column(String) + beam = Column(Integer) collection = Column(String) level = Column(Integer) dataproduct_type = Column(String) diff --git a/database/schemas.py b/database/schemas.py index 16ff5d3c583bff656b0e9c9ee925dc7348dfd6c1..19893eeecc6b4f4f0be60b43f2035f7e588fce61 100644 --- a/database/schemas.py +++ b/database/schemas.py @@ -11,6 +11,7 @@ class SkyView(BaseModel): ra: float dec: float observation: str + beam: int collection: str level: int dataproduct_type: str diff --git a/dev_scripts/__init__.py b/dev_scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/convert_alta_to_adex_cache.py b/dev_scripts/convert_alta_to_adex_cache.py similarity index 59% rename from utils/convert_alta_to_adex_cache.py rename to dev_scripts/convert_alta_to_adex_cache.py index 4fcd49bc42b9ff5b2db97ffe989d93729b1b6556..b8f9467507e5465d09faf880f112bc0fdd6b5d40 100644 --- a/utils/convert_alta_to_adex_cache.py +++ b/dev_scripts/convert_alta_to_adex_cache.py @@ -4,7 +4,8 @@ import psycopg2 from psycopg2 import Error import argparse -from utils.sql_scripts import select_from_alta, insert_into_skyview + +from schemas import sql_scripts def parse_database_url(url): # parse database url like: postgres:postgres@localhost/altadb_1sept2022:5432 @@ -47,18 +48,55 @@ def do_convert(source, target): ) source_cursor = source_connection.cursor() - source_cursor.execute(select_from_alta) + source_cursor.execute(sql_scripts.select_from_alta) target_cursor = target_connection.cursor() + # first drop the existing table and recreate it + target_cursor.execute(sql_scripts.drop_table_skyviews) + target_cursor.execute(sql_scripts.create_table_skyviews) + target_connection.commit() + + print('fetching records from ALTA...') rows = source_cursor.fetchall() + count = len(rows) + print(str(count) + ' records fetched') + + print('inserting records into adex_cache...') + insert_count = 0 for row in rows: - print(row) access_url,ra,dec,dt,dst,observation = row - record_to_insert = (observation, observation, ra, dec, "alta",0, dt,dst, access_url) - target_cursor.execute(insert_into_skyview,record_to_insert) + + # TODO: move this algorithm to a sane place, finish it and have scientists review it. + # determine which dataproducts to skip + insert_this_record = True + if dt in ['inspectionPlot']: + continue + + if dst in ['calibrationTable']: + continue + + # determine collection + collection = 'apertif-imaging' + if dt == 'timeSeries': + collection = 'apertif-timeseries' + + level = 0 + if dst == 'calibratedVisibility': + level=1 + if 'cube' in dt: + level=2 + + # todo: extract beam from name, but first JOIN with api_dataentity (see sql_scripts) + title = "Not available yet" + beam = 0 + + record_to_insert = (title, observation, beam, ra, dec, collection, level, dt,dst, access_url) + target_cursor.execute(sql_scripts.insert_into_skyviews,record_to_insert) target_connection.commit() + insert_count = insert_count + 1 + print(str(insert_count) + ' inserted') except Error as e: print(e) diff --git a/requirements.txt b/requirements.txt index 0722f0fe9eb465af0ada6c205d143f95c1297dc1..b1201c4fb2878db03ef26628d801447f7c30a6e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ colorama==0.4.5 databases==0.6.1 fastapi==0.79.0 greenlet==1.1.2 -gunicorn==20.1.0 h11==0.13.0 idna==3.3 pydantic==1.9.2 diff --git a/schemas/__init__.py b/schemas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/sql_scripts.py b/schemas/sql_scripts.py similarity index 76% rename from utils/sql_scripts.py rename to schemas/sql_scripts.py index ef4ee16f4e6406492f056e8244b2ce9fbea57a54..e7d98895035c45cc95d487781cfbadf6f0821617 100644 --- a/utils/sql_scripts.py +++ b/schemas/sql_scripts.py @@ -6,12 +6,17 @@ CREATE DATABASE adex_cache CONNECTION LIMIT = -1; """ -create_table = """ +drop_table_skyviews = """ + DROP TABLE IF EXISTS public.skyviews; +""" + +create_table_skyviews = """ CREATE TABLE public.skyviews ( "id" SERIAL, "title" character varying(50), "observation" character varying(50), + "beam" integer, "ra" double precision, "dec" double precision, "collection" character varying(50), @@ -24,11 +29,12 @@ CREATE TABLE public.skyviews """ -insert_into_skyview = """ +insert_into_skyviews = """ INSERT INTO public.skyviews ( title, observation, + beam, ra, dec, collection, @@ -36,9 +42,10 @@ INSERT INTO public.skyviews dataproduct_type, dataproduct_subtype, access_url) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) """ +# todo: join with api_dataentity table to retrieve the name (which also holds the beam) select_from_alta = """ SELECT "storageRef" as access_url,