diff --git a/ldvspec/lofardata/scripts/ldv_specification_interface.py b/ldvspec/lofardata/scripts/ldv_specification_interface.py index f9568b435b09860506fd8e8714f473a8da40e706..d0469ad552c8646e191c6bdebc55387e75e72451 100644 --- a/ldvspec/lofardata/scripts/ldv_specification_interface.py +++ b/ldvspec/lofardata/scripts/ldv_specification_interface.py @@ -98,8 +98,6 @@ class LDVSpecInterface(): if isinstance(payload, str): response = self.session().request(type, url, data=payload, headers=self.header, params=query_parameters) - elif isinstance(payload, dict): - response = self.session().request(type, url, json=payload, headers=self.header, params=query_parameters) else: response = self.session().request(type, url, json=payload, headers=self.header, params=query_parameters) diff --git a/ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py b/ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py index 0cd9dd8598d0b3721f8d06c6eba5c831977dbfea..e85e7961adcad6d321d9c0579492986eb80ea675 100644 --- a/ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py +++ b/ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py @@ -84,10 +84,10 @@ def main(): if args.version: # Get file's Last modification time stamp only in terms of seconds since epoch and convert in timestamp modification_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getmtime(__file__))) - print("--- {} (last updated {}) ---".format(os.path.basename(__file__), modification_time)) + logging.info("--- {} (last updated {}) ---".format(os.path.basename(__file__), modification_time)) return - print("\nMoment please....\n") + logging.info("\nMoment please....\n") if args.verbose: change_logging_level(logging.DEBUG) @@ -104,23 +104,28 @@ def main(): no_limit_to_insert = False logging.debug("Limit on number of dataproducts to insert REST is set to {}".format(args.max_nbr_dps_to_insert_per_request)) + query_count_all_raw_dataproducts = "select count(*) from astrowise.raw_dataproducts" + query_count_all_pipeline_dataproducts = "select count(*) from astrowise.pl_dataproducts" + query_all_required_fields_raw_dataproducts = \ + "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco from astrowise.raw_dataproducts {}"\ + .format(limit_str) + # Create connection using ssh tunnel with the ldvadmin database conn, tunnel = connector.connect_postgresql(args.section) - count_raw_dps = execute_query(conn, "select count(*) from astrowise.raw_dataproducts")[0][0] - count_pl_dps = execute_query(conn, "select count(*) from astrowise.pl_dataproducts")[0][0] + count_raw_dps = execute_query(conn, query_count_all_raw_dataproducts)[0][0] + count_pl_dps = execute_query(conn, query_count_all_pipeline_dataproducts)[0][0] logging.info(f"There are {count_raw_dps} raw dataproducts and {count_pl_dps} pipeline dataproduct in the ldvadmin.astrowise table!!") + result_query_all_dps = execute_query(conn, query_all_required_fields_raw_dataproducts) - result_query_all_dps = execute_query(conn, - "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco " - "from astrowise.raw_dataproducts {}".format(limit_str)) # Are there still dataproducts left to query? nbr_dps_left = len(result_query_all_dps) - args.limit if nbr_dps_left > 0 and args.limit > 0: logging.debug("Limit on number of leftover dataproducts to query is set to {}".format(nbr_dps_left)) limit_str = "limit {}".format(nbr_dps_left) - result_query_all_dps.extend(execute_query(conn, - "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco " - "from astrowise.pl_dataproducts {}".format(limit_str))) + query_all_required_fields_pipeline_dataproducts = \ + "select obsid, obsid_source, dp_type, project, activity, uri, size, dysco from astrowise.pl_dataproducts {}" \ + .format(limit_str) + result_query_all_dps.extend(execute_query(conn, query_all_required_fields_pipeline_dataproducts)) logging.info("{} dataproduct retrieved from ldvadmin".format(len(result_query_all_dps))) # Create connection with ldv-spec-db using REST API, use temp. token created in my test-env @@ -134,7 +139,6 @@ def main(): dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA", "dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5], "filesize": dps[6], "additional_meta": metadata_str, "location": dps[5]} - # We can do a bulk insert (there should be a max in 'payload length somewhere -> TODO lst_all_dps.append(dps_dict) if no_limit_to_insert: @@ -153,7 +157,7 @@ def main(): format(cnt, nbr_required_inserts, args.max_nbr_dps_to_insert_per_request, start, end)) logging.debug("Added with ids={}".format(res_lst_ids)) - print("\nThat's All Folks!\n") + logging.info("\nThat's All Folks!\n") if __name__ == "__main__":