From d74e18ff3e38b218d74eeae812b82f0e846c2272 Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Mon, 5 Sep 2022 10:59:21 +0200
Subject: [PATCH] Add fields to additional_meta
---
.../ldv_migrate/migrate_ldvadmin_to_ldvspec.py | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py
index 1ae4251..901f274 100644
--- a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py
+++ b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py
@@ -15,13 +15,12 @@ Some examples:
python ./ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py --limit 1000 --host prod
"""
-import json
-import os
-import time
-import logging
import argparse
-import sys
+import logging
import math
+import os
+import sys
+import time
import ldv_migrate.connection.retrieve_db_connection as connector
from ldv_migrate.ldv_specification_interface import LDVSpecInterface
@@ -67,7 +66,10 @@ def execute_query(connection, sql_query, data=None, batch_size=1000):
def row_to_dict(dps):
dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA",
"dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5],
- "filesize": dps[6], "additional_meta": {"dysco_compression": dps[7]}, "location": dps[5]}
+ "filesize": dps[6],
+ "additional_meta": {"dysco_compression": dps[7], "antenna_set": dps[8], "instrument_filter": dps[9]},
+ "location": dps[5]}
+
return dps_dict
@@ -162,7 +164,7 @@ def main():
query_count_all_raw_dataproducts = "select count(*) from astrowise.raw_dataproducts"
query_count_all_pipeline_dataproducts = "select count(*) from astrowise.pl_dataproducts"
query_all_required_fields_raw_dataproducts = \
- "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco from astrowise.raw_dataproducts order by id {}" \
+ "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco, antenna_set, instrument_filter, from astrowise.raw_dataproducts order by id {}" \
.format(limit_str)
# Create connection using ssh tunnel with the ldvadmin database
@@ -179,7 +181,7 @@ def main():
no_limit_to_insert, args.max_nbr_dps_to_insert_per_request)
# Are there still dataproducts left to query?
- nbr_dps_left = args.limits - n_inserted_items
+ nbr_dps_left = args.limit - n_inserted_items
if nbr_dps_left > 0 and args.limit > 0:
logging.debug("Limit on number of leftover dataproducts to query is set to {}".format(nbr_dps_left))
--
GitLab