diff --git a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py index 1ae4251324fc500557105b35d4f6ede2df971075..901f2740006835115209713b798190dbae05d643 100644 --- a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py +++ b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py @@ -15,13 +15,12 @@ Some examples: python ./ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py --limit 1000 --host prod """ -import json -import os -import time -import logging import argparse -import sys +import logging import math +import os +import sys +import time import ldv_migrate.connection.retrieve_db_connection as connector from ldv_migrate.ldv_specification_interface import LDVSpecInterface @@ -67,7 +66,10 @@ def execute_query(connection, sql_query, data=None, batch_size=1000): def row_to_dict(dps): dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA", "dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5], - "filesize": dps[6], "additional_meta": {"dysco_compression": dps[7]}, "location": dps[5]} + "filesize": dps[6], + "additional_meta": {"dysco_compression": dps[7], "antenna_set": dps[8], "instrument_filter": dps[9]}, + "location": dps[5]} + return dps_dict @@ -162,7 +164,7 @@ def main(): query_count_all_raw_dataproducts = "select count(*) from astrowise.raw_dataproducts" query_count_all_pipeline_dataproducts = "select count(*) from astrowise.pl_dataproducts" query_all_required_fields_raw_dataproducts = \ - "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco from astrowise.raw_dataproducts order by id {}" \ + "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco, antenna_set, instrument_filter, from astrowise.raw_dataproducts order by id {}" \ .format(limit_str) # Create connection using ssh tunnel with the ldvadmin database @@ -179,7 +181,7 @@ def main(): no_limit_to_insert, args.max_nbr_dps_to_insert_per_request) # Are there still dataproducts left to query? - nbr_dps_left = args.limits - n_inserted_items + nbr_dps_left = args.limit - n_inserted_items if nbr_dps_left > 0 and args.limit > 0: logging.debug("Limit on number of leftover dataproducts to query is set to {}".format(nbr_dps_left))