From 7de831ba879d7aa7a706bf53482ce9d8aafc30d8 Mon Sep 17 00:00:00 2001 From: Nico Vermaas <vermaas@astron.nl> Date: Wed, 23 Dec 2020 13:20:51 +0100 Subject: [PATCH] updating add_dataproducts functionality --- .idea/atdb-ldv.iml | 6 +++-- .idea/misc.xml | 2 +- atdb/taskdatabase/models.py | 1 - atdb/taskdatabase/serializers.py | 4 +-- atdb/taskdatabase/services/algorithms.py | 26 ++++++++++++------- .../templates/taskdatabase/index.html | 2 +- atdb/taskdatabase/views.py | 10 +++---- 7 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.idea/atdb-ldv.iml b/.idea/atdb-ldv.iml index c5bb47a1..50cd9802 100644 --- a/.idea/atdb-ldv.iml +++ b/.idea/atdb-ldv.iml @@ -1,8 +1,10 @@ <?xml version="1.0" encoding="UTF-8"?> <module type="PYTHON_MODULE" version="4"> <component name="NewModuleRootManager"> - <content url="file://$MODULE_DIR$" /> - <orderEntry type="jdk" jdkName="Python 3.7 (atdb-ldv)" jdkType="Python SDK" /> + <content url="file://$MODULE_DIR$"> + <excludeFolder url="file://$MODULE_DIR$/venv" /> + </content> + <orderEntry type="jdk" jdkName="Python 3.7 (atdb-ldv) (2)" jdkType="Python SDK" /> <orderEntry type="sourceFolder" forTests="false" /> </component> <component name="PyDocumentationSettings"> diff --git a/.idea/misc.xml b/.idea/misc.xml index 1a05b2d0..85a13228 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <project version="4"> - <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (atdb-ldv)" project-jdk-type="Python SDK" /> + <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (atdb-ldv) (2)" project-jdk-type="Python SDK" /> <component name="PyCharmProfessionalAdvertiser"> <option name="shown" value="true" /> </component> diff --git a/atdb/taskdatabase/models.py b/atdb/taskdatabase/models.py index c6eedf59..8fa8af5b 100644 --- a/atdb/taskdatabase/models.py +++ b/atdb/taskdatabase/models.py @@ -115,7 +115,6 @@ class DataProduct(TaskObject): description = models.CharField(max_length=255, default="unknown") dataproduct_type = models.CharField('type', default=TYPE_VISIBILITY, max_length=50) size = models.BigIntegerField(default=0) - #quality = models.CharField(max_length=30, default="unknown") # relationships parent = models.ForeignKey(Observation, related_name='generated_dataproducts', on_delete=models.CASCADE, null=False) diff --git a/atdb/taskdatabase/serializers.py b/atdb/taskdatabase/serializers.py index 96987703..39040fcd 100644 --- a/atdb/taskdatabase/serializers.py +++ b/atdb/taskdatabase/serializers.py @@ -29,9 +29,9 @@ class DataProductSerializer(serializers.ModelSerializer): class Meta: model = DataProduct fields = ('id','task_type','name','filename','description', - 'taskID','creationTime','size','quality', + 'taskID','creationTime','size', 'my_status','new_status','status_history','parent', - 'data_location','node') + 'data_location','node','quality','metadata') class ObservationSerializer(serializers.ModelSerializer): diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py index ba27357b..5e61721a 100644 --- a/atdb/taskdatabase/services/algorithms.py +++ b/atdb/taskdatabase/services/algorithms.py @@ -115,22 +115,26 @@ def get_next_observation(my_status, observing_mode, datawriter): return taskID,minutes_left -# /atdb/post_dataproducts?taskid=190405001 -@timeit -def add_dataproducts(taskID, dataproducts): +# /atdb/post_dataproducts +def add_dataproducts(dataproducts): """ :param taskID: taskid of the observation :param dataproducts: json list of dataproducts to be added to the provided taskid :return: """ - number_of_dataproducts = len(dataproducts) - logger.info("add_dataproducts("+taskID+','+str(number_of_dataproducts)+")") - # get the common fields from the observation based on the given taskid - parent = Observation.objects.get(taskID=taskID) - parent_data_location = parent.data_location + taskID = None + dps = dataproducts.get('dps') + number_of_dataproducts = len(dps) + + logger.info("add_dataproducts("+str(number_of_dataproducts)+")") + + for dp in dps: + # get the common fields from the observation based on the given taskid + taskID = dp.get('taskID') + parent = Observation.objects.get(taskID=taskID) + parent_data_location = parent.data_location - for dp in dataproducts: node=dp.get('node',None) new_status = dp.get('new_status','defined') data_location = dp.get('data_dir',parent_data_location) @@ -141,6 +145,8 @@ def add_dataproducts(taskID, dataproducts): filename=dp['filename'], name=dp['filename'], description=dp['filename'], + quality=dp['quality'], + metadata=dp['metadata'], task_type='dataproduct', new_status=new_status, parent=parent, @@ -150,4 +156,4 @@ def add_dataproducts(taskID, dataproducts): logger.info('addding dataproduct: '+str(myDataProduct)) myDataProduct.save() - + return taskID diff --git a/atdb/taskdatabase/templates/taskdatabase/index.html b/atdb/taskdatabase/templates/taskdatabase/index.html index cbe91a7a..44440253 100644 --- a/atdb/taskdatabase/templates/taskdatabase/index.html +++ b/atdb/taskdatabase/templates/taskdatabase/index.html @@ -46,7 +46,7 @@ </div> {% include 'taskdatabase/pagination.html' %} </div> - <p class="footer"> Version 1.0.0 (22 dec 2020 - 14:30) + <p class="footer"> Version 1.0.0 (23 dec 2020 - 08:00) <script type="text/javascript"> (function(seconds) { var refresh, diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py index 8c44ff23..5d41f6f5 100644 --- a/atdb/taskdatabase/views.py +++ b/atdb/taskdatabase/views.py @@ -452,19 +452,15 @@ class PostDataproductsView(generics.CreateAPIView): pagination_class = DataProductsPagination def post(self, request, *args, **kwargs): - # read the arguments from the request - try: - taskID = self.request.query_params['taskID'] - except: - taskID = None try: body_unicode = request.body.decode('utf-8') dataproducts = json.loads(body_unicode) - except: + except Exception as e: + print(e) dataproducts = None - taskID = algorithms.add_dataproducts(taskID, dataproducts) + taskID = algorithms.add_dataproducts(dataproducts) # return a response return Response({ -- GitLab