SDC-685: Update after review comment

ldvspec/lofardata/scripts/ldv_specification_interface.py

@@ -98,8 +98,6 @@ class LDVSpecInterface():
 
         if isinstance(payload, str):
             response = self.session().request(type, url, data=payload, headers=self.header, params=query_parameters)
-        elif isinstance(payload, dict):
-            response = self.session().request(type, url, json=payload, headers=self.header, params=query_parameters)
         else:
             response = self.session().request(type, url, json=payload, headers=self.header, params=query_parameters)
 

ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py

@@ -84,10 +84,10 @@ def main():
     if args.version:
         # Get file's Last modification time stamp only in terms of seconds since epoch and convert in timestamp
         modification_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getmtime(__file__)))
-        print("--- {} (last updated {}) ---".format(os.path.basename(__file__), modification_time))
+        logging.info("--- {} (last updated {}) ---".format(os.path.basename(__file__), modification_time))
         return
 
-    print("\nMoment please....\n")
+    logging.info("\nMoment please....\n")
 
     if args.verbose:
         change_logging_level(logging.DEBUG)
@@ -104,23 +104,28 @@ def main():
         no_limit_to_insert = False
         logging.debug("Limit on number of dataproducts to insert REST is set to {}".format(args.max_nbr_dps_to_insert_per_request))
 
+    query_count_all_raw_dataproducts = "select count(*) from astrowise.raw_dataproducts"
+    query_count_all_pipeline_dataproducts = "select count(*) from astrowise.pl_dataproducts"
+    query_all_required_fields_raw_dataproducts = \
+        "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco from astrowise.raw_dataproducts {}"\
+            .format(limit_str)
+
     # Create connection using ssh tunnel with the ldvadmin database
     conn, tunnel = connector.connect_postgresql(args.section)
-    count_raw_dps = execute_query(conn, "select count(*) from astrowise.raw_dataproducts")[0][0]
-    count_pl_dps = execute_query(conn, "select count(*) from astrowise.pl_dataproducts")[0][0]
+    count_raw_dps = execute_query(conn, query_count_all_raw_dataproducts)[0][0]
+    count_pl_dps = execute_query(conn, query_count_all_pipeline_dataproducts)[0][0]
     logging.info(f"There are {count_raw_dps} raw dataproducts and {count_pl_dps} pipeline dataproduct in the ldvadmin.astrowise table!!")
+    result_query_all_dps = execute_query(conn, query_all_required_fields_raw_dataproducts)
 
-    result_query_all_dps = execute_query(conn,
-                                             "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco "
-                                             "from astrowise.raw_dataproducts {}".format(limit_str))
     # Are there still dataproducts left to query?
     nbr_dps_left = len(result_query_all_dps) - args.limit
     if nbr_dps_left > 0 and args.limit > 0:
         logging.debug("Limit on number of leftover dataproducts to query is set to {}".format(nbr_dps_left))
         limit_str = "limit {}".format(nbr_dps_left)
-        result_query_all_dps.extend(execute_query(conn,
-                                                  "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco "
-                                                  "from astrowise.pl_dataproducts {}".format(limit_str)))
+        query_all_required_fields_pipeline_dataproducts = \
+            "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco from astrowise.pl_dataproducts {}" \
+                .format(limit_str)
+        result_query_all_dps.extend(execute_query(conn, query_all_required_fields_pipeline_dataproducts))
 
     logging.info("{} dataproduct retrieved from ldvadmin".format(len(result_query_all_dps)))
     # Create connection with ldv-spec-db using REST API, use temp. token created in my test-env
@@ -134,7 +139,6 @@ def main():
         dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA",
                     "dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5],
                     "filesize": dps[6], "additional_meta": metadata_str, "location": dps[5]}
-        # We can do a bulk insert (there should be a max in 'payload length somewhere -> TODO
         lst_all_dps.append(dps_dict)
 
     if no_limit_to_insert:
@@ -153,7 +157,7 @@ def main():
                          format(cnt, nbr_required_inserts, args.max_nbr_dps_to_insert_per_request, start, end))
             logging.debug("Added with ids={}".format(res_lst_ids))
 
-    print("\nThat's All Folks!\n")
+    logging.info("\nThat's All Folks!\n")
 
 
 if __name__ == "__main__":