Merge branch 'add_additional_fields' into 'master'

Add fields to additional_meta

See merge request !12

ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py

@@ -15,13 +15,12 @@ Some examples:
       python ./ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py --limit 1000 --host prod
 
 """
-import json
-import os
-import time
-import logging
 import argparse
-import sys
+import logging
 import math
+import os
+import sys
+import time
 
 import ldv_migrate.connection.retrieve_db_connection as connector
 from ldv_migrate.ldv_specification_interface import LDVSpecInterface
@@ -67,7 +66,10 @@ def execute_query(connection, sql_query, data=None, batch_size=1000):
 def row_to_dict(dps):
     dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA",
                 "dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5],
-                "filesize": dps[6], "additional_meta": {"dysco_compression": dps[7]}, "location": dps[5]}
+                "filesize": dps[6],
+                "additional_meta": {"dysco_compression": dps[7], "antenna_set": dps[8], "instrument_filter": dps[9]},
+                "location": dps[5]}
+
     return dps_dict
 
 
@@ -162,7 +164,7 @@ def main():
     query_count_all_raw_dataproducts = "select count(*) from astrowise.raw_dataproducts"
     query_count_all_pipeline_dataproducts = "select count(*) from astrowise.pl_dataproducts"
     query_all_required_fields_raw_dataproducts = \
-        "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco from astrowise.raw_dataproducts order by id {}" \
+        "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco, antenna_set, instrument_filter, from astrowise.raw_dataproducts order by id {}" \
             .format(limit_str)
 
     # Create connection using ssh tunnel with the ldvadmin database
@@ -179,7 +181,7 @@ def main():
                                         no_limit_to_insert, args.max_nbr_dps_to_insert_per_request)
 
     # Are there still dataproducts left to query?
-    nbr_dps_left = args.limits - n_inserted_items
+    nbr_dps_left = args.limit - n_inserted_items
 
     if nbr_dps_left > 0 and args.limit > 0:
         logging.debug("Limit on number of leftover dataproducts to query is set to {}".format(nbr_dps_left))