Skip to content
Snippets Groups Projects
Commit 3987ddac authored by Fanna Lautenbach's avatar Fanna Lautenbach
Browse files

Fix script errors

parent 315471d4
No related branches found
No related tags found
1 merge request!10Fix script errors
...@@ -15,6 +15,7 @@ Some examples: ...@@ -15,6 +15,7 @@ Some examples:
python ./ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py --limit 1000 --host prod python ./ldvspec/lofardata/scripts/migrate_ldvadmin_to_ldvspec.py --limit 1000 --host prod
""" """
import json
import os import os
import time import time
import logging import logging
...@@ -139,10 +140,9 @@ def main(): ...@@ -139,10 +140,9 @@ def main():
lst_all_dps = [] lst_all_dps = []
for dps in result_query_all_dps: for dps in result_query_all_dps:
logging.debug(dps) logging.debug(dps)
metadata_str = "{'dysco_compression': %s}" % dps[7]
dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA", dps_dict = {"obs_id": dps[0], "oid_source": dps[1], "dataproduct_source": "LOFAR LTA",
"dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5], "dataproduct_type": dps[2], "project": dps[3], "activity": dps[4], "surl": dps[5],
"filesize": dps[6], "additional_meta": metadata_str, "location": dps[5]} "filesize": dps[6], "additional_meta": {"dysco_compression": dps[7]}, "location": dps[5]}
lst_all_dps.append(dps_dict) lst_all_dps.append(dps_dict)
if no_limit_to_insert: if no_limit_to_insert:
...@@ -150,12 +150,10 @@ def main(): ...@@ -150,12 +150,10 @@ def main():
logging.info("Added {} DataProduct objects".format(len(res_lst_ids))) logging.info("Added {} DataProduct objects".format(len(res_lst_ids)))
logging.debug("Added with ids={}".format(res_lst_ids)) logging.debug("Added with ids={}".format(res_lst_ids))
else: else:
cnt = 0
nbr_required_inserts = math.ceil(len(lst_all_dps)/args.max_nbr_dps_to_insert_per_request) nbr_required_inserts = math.ceil(len(lst_all_dps)/args.max_nbr_dps_to_insert_per_request)
for cnt in range(nbr_required_inserts): for cnt in range(nbr_required_inserts):
start = cnt * args.max_nbr_dps_to_insert_per_request start = cnt * args.max_nbr_dps_to_insert_per_request
end = start + args.max_nbr_dps_to_insert_per_request end = start + args.max_nbr_dps_to_insert_per_request
cnt += 1
res_lst_ids = ldvspec_interface.insert_multiple_dataproduct(payload=lst_all_dps[start:end]) res_lst_ids = ldvspec_interface.insert_multiple_dataproduct(payload=lst_all_dps[start:end])
logging.info("Insert count {} of {}: Added {} DataProduct objects [{} till {}]". logging.info("Insert count {} of {}: Added {} DataProduct objects [{} till {}]".
format(cnt, nbr_required_inserts, args.max_nbr_dps_to_insert_per_request, start, end)) format(cnt, nbr_required_inserts, args.max_nbr_dps_to_insert_per_request, start, end))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment