Skip to content
Snippets Groups Projects
Commit c0d832c5 authored by Nico Vermaas's avatar Nico Vermaas
Browse files

Merge branch 'add-some-alta-logic' into 'main'

adding better ALTA logic

See merge request !1
parents e8b4ae83 c8204beb
Branches
No related tags found
2 merge requests!2Main,!1adding better ALTA logic
Pipeline #35386 passed
......@@ -10,6 +10,7 @@ class SkyView(Base):
ra = Column(Float, index=True)
dec = Column(Float, index=True)
observation = Column(String)
beam = Column(Integer)
collection = Column(String)
level = Column(Integer)
dataproduct_type = Column(String)
......
......@@ -11,6 +11,7 @@ class SkyView(BaseModel):
ra: float
dec: float
observation: str
beam: int
collection: str
level: int
dataproduct_type: str
......
......@@ -4,7 +4,8 @@
import psycopg2
from psycopg2 import Error
import argparse
from utils.sql_scripts import select_from_alta, insert_into_skyview
from schemas import sql_scripts
def parse_database_url(url):
# parse database url like: postgres:postgres@localhost/altadb_1sept2022:5432
......@@ -47,18 +48,55 @@ def do_convert(source, target):
)
source_cursor = source_connection.cursor()
source_cursor.execute(select_from_alta)
source_cursor.execute(sql_scripts.select_from_alta)
target_cursor = target_connection.cursor()
# first drop the existing table and recreate it
target_cursor.execute(sql_scripts.drop_table_skyviews)
target_cursor.execute(sql_scripts.create_table_skyviews)
target_connection.commit()
print('fetching records from ALTA...')
rows = source_cursor.fetchall()
count = len(rows)
print(str(count) + ' records fetched')
print('inserting records into adex_cache...')
insert_count = 0
for row in rows:
print(row)
access_url,ra,dec,dt,dst,observation = row
record_to_insert = (observation, observation, ra, dec, "alta",0, dt,dst, access_url)
target_cursor.execute(insert_into_skyview,record_to_insert)
# TODO: move this algorithm to a sane place, finish it and have scientists review it.
# determine which dataproducts to skip
insert_this_record = True
if dt in ['inspectionPlot']:
continue
if dst in ['calibrationTable']:
continue
# determine collection
collection = 'apertif-imaging'
if dt == 'timeSeries':
collection = 'apertif-timeseries'
level = 0
if dst == 'calibratedVisibility':
level=1
if 'cube' in dt:
level=2
# todo: extract beam from name, but first JOIN with api_dataentity (see sql_scripts)
title = "Not available yet"
beam = 0
record_to_insert = (title, observation, beam, ra, dec, collection, level, dt,dst, access_url)
target_cursor.execute(sql_scripts.insert_into_skyviews,record_to_insert)
target_connection.commit()
insert_count = insert_count + 1
print(str(insert_count) + ' inserted')
except Error as e:
print(e)
......
......@@ -5,7 +5,6 @@ colorama==0.4.5
databases==0.6.1
fastapi==0.79.0
greenlet==1.1.2
gunicorn==20.1.0
h11==0.13.0
idna==3.3
pydantic==1.9.2
......
......@@ -6,12 +6,17 @@ CREATE DATABASE adex_cache
CONNECTION LIMIT = -1;
"""
create_table = """
drop_table_skyviews = """
DROP TABLE IF EXISTS public.skyviews;
"""
create_table_skyviews = """
CREATE TABLE public.skyviews
(
"id" SERIAL,
"title" character varying(50),
"observation" character varying(50),
"beam" integer,
"ra" double precision,
"dec" double precision,
"collection" character varying(50),
......@@ -24,11 +29,12 @@ CREATE TABLE public.skyviews
"""
insert_into_skyview = """
insert_into_skyviews = """
INSERT INTO public.skyviews
(
title,
observation,
beam,
ra,
dec,
collection,
......@@ -36,9 +42,10 @@ INSERT INTO public.skyviews
dataproduct_type,
dataproduct_subtype,
access_url)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
"""
# todo: join with api_dataentity table to retrieve the name (which also holds the beam)
select_from_alta = """
SELECT
"storageRef" as access_url,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment