From f6331a88a62e650548abdb54da627a7d274c2ab3 Mon Sep 17 00:00:00 2001
From: Vermaas <vermaas@astron.nl>
Date: Tue, 4 Apr 2023 10:44:15 +0200
Subject: [PATCH] finish Focus connector adding fake secondary connector
because of missing ancillary data in VO
---
..._secondary_fake_12ab3d_surf_localhost.args | 9 ++
.../primary_calvis_surf_localhost.args | 2 +-
.../primary_skymap_surf_localhost.args | 9 ++
scraper/vo/connectors/Focus.py | 98 +++++++++++++++++++
4 files changed, 117 insertions(+), 1 deletion(-)
create mode 100644 examples/vo/focusproject/ancillary_secondary_fake_12ab3d_surf_localhost.args
create mode 100644 examples/vo/focusproject/primary_skymap_surf_localhost.args
diff --git a/examples/vo/focusproject/ancillary_secondary_fake_12ab3d_surf_localhost.args b/examples/vo/focusproject/ancillary_secondary_fake_12ab3d_surf_localhost.args
new file mode 100644
index 0000000..f841226
--- /dev/null
+++ b/examples/vo/focusproject/ancillary_secondary_fake_12ab3d_surf_localhost.args
@@ -0,0 +1,9 @@
+--datasource=vo
+--connector=FOCUS.LincSecondaryFake12ab3d
+--data_host=http://145.38.188.72/tap/focusproject.secondary
+--batch_size=1000
+--adex_backend_host=http://localhost:8000/adex_backend/
+--adex_resource=ancillary_dp/create
+--adex_token=9519e433ba37487f1a18121dfb1957d992fbb790
+--clear_collection
+--collection=linc_ancillary
\ No newline at end of file
diff --git a/examples/vo/focusproject/primary_calvis_surf_localhost.args b/examples/vo/focusproject/primary_calvis_surf_localhost.args
index 69c2376..8e26425 100644
--- a/examples/vo/focusproject/primary_calvis_surf_localhost.args
+++ b/examples/vo/focusproject/primary_calvis_surf_localhost.args
@@ -6,4 +6,4 @@
--adex_resource=primary_dp/create
--adex_token=9519e433ba37487f1a18121dfb1957d992fbb790
--clear_collection
---collection=focus
\ No newline at end of file
+--collection=linc_visibilities
\ No newline at end of file
diff --git a/examples/vo/focusproject/primary_skymap_surf_localhost.args b/examples/vo/focusproject/primary_skymap_surf_localhost.args
new file mode 100644
index 0000000..f5df0d2
--- /dev/null
+++ b/examples/vo/focusproject/primary_skymap_surf_localhost.args
@@ -0,0 +1,9 @@
+--datasource=vo
+--connector=FOCUS.LincSkymap
+--data_host=http://145.38.188.72/tap/focusproject.skymap
+--batch_size=1000
+--adex_backend_host=http://localhost:8000/adex_backend/
+--adex_resource=primary_dp/create
+--adex_token=9519e433ba37487f1a18121dfb1957d992fbb790
+--clear_collection
+--collection=linc_skymap
\ No newline at end of file
diff --git a/scraper/vo/connectors/Focus.py b/scraper/vo/connectors/Focus.py
index daf7170..3d79857 100644
--- a/scraper/vo/connectors/Focus.py
+++ b/scraper/vo/connectors/Focus.py
@@ -1,5 +1,60 @@
import datetime
+class LincSkymap():
+
+ def translate(self, row, args):
+ """
+ parse the specific row that comes from the VO adql query,
+ and translate it into the standard json payload for posting to the ADEX backend REST API
+
+ :param row: the results from the ADQL query to a VO service
+ :param args: the commandline arguments, but only args.collection is currently used
+ :return: user, password, host, database, db_port
+ """
+ freq_min = float(row['f_min'])
+ freq_max = float(row['f_max'])
+ freq_avg = (freq_max + freq_min) / 2
+
+ duration = float(row['t_end']) - float(row['t_start'])
+ #sampling_time = duration / time_samples
+
+ payload = dict(
+ pid=row['imageTitle'],
+ name=row['imageTitle'],
+ dp_type="skymap",
+ format=row['mime'], #this yields 'archive/tar', which is not a defined format by stakeholders
+ #format="other",
+ locality="online",
+ access_url=row['accref'],
+ ra=float(row['centerAlpha']),
+ dec=float(row['centerDelta']),
+ equinox=str(row['wcs_equinox']),
+
+ exposure_time=duration,
+ central_frequency=freq_avg,
+ frequency_resolution=None,
+ time_resolution=None,
+ bandwidth=None,
+
+ release_date=None,
+ data_provider="ASTRON",
+
+ PSF_size=None,
+ #sky_footprint=str(row['s_region']),
+ sky_footprint=None,
+ dataset_id=str(row['pipeline_run_id']),
+ collection=args.collection,
+ parent=None
+
+ )
+
+ # are these parameters that we can create the Acitivies with?
+ pipeline_version = row['pipeline_version'],
+ pipeline_access_url = row['pipeline_access_url'],
+ #related_products = row['related_products'],
+ return payload
+
+
class LincCalibratedVisibilities():
def translate(self, row, args):
@@ -109,4 +164,47 @@ class LincSecondary():
collection = args.collection
)
+ return payload
+
+class LincSecondaryFake12ab3d():
+
+ def translate(self, row, args):
+ """"
+ parse the specific row that comes from the VO adql query,
+ and translate it into the standard json payload for posting to the ADEX backend REST API
+
+ :param row: the results from the ADQL query to a VO service
+ :param args: the commandline arguments, but only args.collection is currently used
+ :return: user, password, host, database, db_port
+ """
+
+ # check for these types from VO: inputs, inspection-plot, execution-logs, calibration-file
+ # translate VO speak to ADEX speak
+ dp_type = row['type']
+ if dp_type == 'inspection-plot':
+ dp_type = 'diagnostic-plot'
+
+ if dp_type == 'calibration-file':
+ dp_type = 'calibration-solution'
+
+ if dp_type == 'inputs':
+ dp_type = 'pipeline-config'
+
+ if dp_type == 'execution-log':
+ dp_type = 'execution-log'
+
+ payload = dict(
+ pid=row['access_url'],
+ name=row['access_url'],
+ dp_type=dp_type,
+ format="other",
+ locality="online",
+ access_url=row['access_url'],
+
+ dataset_id="12ab3d",
+ parent=None,
+
+ collection = args.collection
+ )
+
return payload
\ No newline at end of file
--
GitLab