diff --git a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py index 9cfb88dcd0219c5f90ddb53a19d8faa131355a0c..3966ad13b43b77372675745554f9cbdf08e05291 100644 --- a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py +++ b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py @@ -15,6 +15,7 @@ Some examples: python ./ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py --limit 1000 --host prod """ +import json import os import time import logging @@ -139,10 +140,9 @@ def main(): lst_all_dps = [] for dps in result_query_all_dps: logging.debug(dps) - metadata_str = "{'dysco_compression': %s}" % dps[7] dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA", "dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5], - "filesize": dps[6], "additional_meta": metadata_str, "location": dps[5]} + "filesize": dps[6], "additional_meta": {"dysco_compression": dps[7]}, "location": dps[5]} lst_all_dps.append(dps_dict) if no_limit_to_insert: @@ -150,12 +150,10 @@ def main(): logging.info("Added {} DataProduct objects".format(len(res_lst_ids))) logging.debug("Added with ids={}".format(res_lst_ids)) else: - cnt = 0 nbr_required_inserts = math.ceil(len(lst_all_dps)/args.max_nbr_dps_to_insert_per_request) for cnt in range(nbr_required_inserts): start = cnt * args.max_nbr_dps_to_insert_per_request end = start + args.max_nbr_dps_to_insert_per_request - cnt += 1 res_lst_ids = ldvspec_interface.insert_multiple_dataproduct(payload=lst_all_dps[start:end]) logging.info("Insert count {} of {}: Added {} DataProduct objects [{} till {}]". format(cnt, nbr_required_inserts, args.max_nbr_dps_to_insert_per_request, start, end))