From f76c9c8091822b10ceb1c81f07ff1c8491f4b1f3 Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Thu, 4 Jun 2020 13:59:34 +0200
Subject: [PATCH] TMSS-207: call generic create_subtasks_from_task_blueprint
from REST action, lookup generator method(s), and create the actual subtasks.
---
SAS/TMSS/src/tmss/exceptions.py | 3 +
SAS/TMSS/src/tmss/tmssapp/subtasks.py | 55 +++++++++++--------
.../tmss/tmssapp/viewsets/specification.py | 5 +-
3 files changed, 39 insertions(+), 24 deletions(-)
diff --git a/SAS/TMSS/src/tmss/exceptions.py b/SAS/TMSS/src/tmss/exceptions.py
index c77b7226f04..ccdf97362bd 100644
--- a/SAS/TMSS/src/tmss/exceptions.py
+++ b/SAS/TMSS/src/tmss/exceptions.py
@@ -8,6 +8,9 @@ class SchemaValidationException(TMSSException):
class ConversionException(TMSSException):
pass
+class SubtaskCreationException(ConversionException):
+ pass
+
class SchedulingException(TMSSException):
pass
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index 418d4bf1e03..abb418e029d 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -3,7 +3,7 @@ logger = logging.getLogger(__name__)
from lofar.common.json_utils import add_defaults_to_json_object_for_schema, get_default_json_object_for_schema
-from lofar.sas.tmss.tmss.exceptions import SubtaskSchedulingException, ConversionException
+from lofar.sas.tmss.tmss.exceptions import SubtaskCreationException, SubtaskSchedulingException
from datetime import datetime, timedelta
from lofar.common.datetimeutils import parseDatetime
@@ -12,6 +12,31 @@ from lofar.sas.tmss.tmss.tmssapp.models import *
# ==== various create* methods to convert/create a TaskBlueprint into one or more Subtasks ====
+def check_prerequities_for_subtask_creation(task_blueprint: TaskBlueprint) -> bool:
+ if task_blueprint.do_cancel:
+ raise SubtaskCreationException("Cannot create subtasks from task_blueprint id=%d, because the task_blueprint is explicit set to cancel." % task_blueprint.id)
+
+ return True
+
+def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint) -> [Subtask]:
+ '''Generic create-method for subtasks. Calls the appropiate create method based on the task_blueprint specifications_template name.'''
+ check_prerequities_for_subtask_creation(task_blueprint)
+
+ # fixed mapping from template name to generator functions which create the list of subtask(s) for this task_blueprint
+ generators_mapping = {'correlator schema': [create_observation_control_subtask_from_task_blueprint,
+ create_qafile_subtask_from_task_blueprint,
+ create_qaplots_subtask_from_task_blueprint],
+ 'preprocessing schema': [create_preprocessing_subtask_from_task_blueprint]}
+
+ template_name = task_blueprint.specifications_template.name
+ if template_name in generators_mapping:
+ generators = generators_mapping[template_name]
+ subtasks = [generator(task_blueprint) for generator in generators]
+ return subtasks
+ else:
+ raise SubtaskCreationException('Cannot create subtasks for task id=%s because no generator exists for its schema name=%s' % (task_blueprint.pk, template_name))
+
+
def create_observation_control_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) -> Subtask:
"""
Create an observation control subtask .
@@ -19,9 +44,7 @@ def create_observation_control_subtask_from_task_blueprint(task_blueprint: TaskB
https://support.astron.nl/confluence/display/TMSS/Specification+Flow
"""
# step 0: check pre-requisites
- if task_blueprint.do_cancel:
- raise ValueError("Cancel create subtasks from blueprint task id=%d, because its explicit set to cancel" %
- task_blueprint.id)
+ check_prerequities_for_subtask_creation(task_blueprint)
# step 1: create subtask in defining state
subtask_template = SubtaskTemplate.objects.get(name='observationcontrol schema')
@@ -43,7 +66,7 @@ def create_observation_control_subtask_from_task_blueprint(task_blueprint: TaskB
}
}
specifications_doc = add_defaults_to_json_object_for_schema(extra_specifications_doc, subtask_template.schema)
- cancel = datetime.datetime.utcnow().isoformat() # I dont understand why this should be a dateformat and not a boolean ?
+ cancel = datetime.utcnow().isoformat() # I dont understand why this should be a dateformat and not a boolean ?
cluster_name = task_blueprint.specifications_doc.get("storage_cluster", "CEP4")
subtask_data = { "start_time": None,
"stop_time": None,
@@ -72,10 +95,10 @@ def create_observation_control_subtask_from_task_blueprint(task_blueprint: TaskB
def create_qafile_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) -> Subtask:
observation_subtasks = [st for st in task_blueprint.subtasks.all() if st.specifications_template.type.value == SubtaskType.Choices.OBSERVATION.value]
if not observation_subtasks:
- raise ValueError("Cannot create %s subtask for task_blueprint id=%d because it has no observation subtask(s)" % (
+ raise SubtaskCreationException("Cannot create %s subtask for task_blueprint id=%d because it has no observation subtask(s)" % (
SubtaskType.Choices.QA_FILES.value, task_blueprint.pk))
- observation_subtask = observation_subtasks.first() # TODO: decide what to do when there are multiple observation subtasks?
+ observation_subtask = observation_subtasks[0] # TODO: decide what to do when there are multiple observation subtasks?
return create_qafile_subtask_from_observation_subtask(observation_subtask)
@@ -139,7 +162,7 @@ def create_qaplots_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) ->
raise ValueError("Cannot create %s subtask for task_blueprint id=%d because it has no qafile subtask(s)" % (
SubtaskType.Choices.QA_FILES.value, task_blueprint.pk))
- qafile_subtask = qafile_subtasks.first() # TODO: decide what to do when there are multiple qafile subtasks?
+ qafile_subtask = qafile_subtasks[0] # TODO: decide what to do when there are multiple qafile subtasks?
return create_qaplots_subtask_from_qafile_subtask(qafile_subtask)
@@ -368,34 +391,20 @@ def connect_observation_subtask_to_preprocessing_subtask(observation_subtask: Su
pipeline_subtask_output.dataproducts.set(output_dps)
-def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint):
- generator_mapping = {'preprocessing schema': create_preprocessing_subtask_from_task_blueprint }
- template_name = task_blueprint.specifications_template.name
- if template_name in generator_mapping:
- generator = generator_mapping[template_name]
- return generator(task_blueprint)
- else:
- raise ValueError('Cannot create subtasks for task id=%s since no generator exists for its schema name=%s' % (task_blueprint.pk, template_name))
-
-
def create_preprocessing_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) -> Subtask:
# todo: check whether already created to avoid duplication?
subtask_template = SubtaskTemplate.objects.get(name='pipelinecontrol schema')
default_subtask_specs = get_default_json_object_for_schema(subtask_template.schema)
- subtasks = []
subtask_specs = _generate_subtask_specs_from_preprocessing_task_specs(task_blueprint.specifications_doc,
default_subtask_specs)
subtask = create_subtask(subtask_template, subtask_specs)
subtask.task_blueprint = task_blueprint
subtask.cluster = Cluster.objects.get(name="CEP4") # todo: probably should not be hardcoded? Can be optional in parset?
subtask.save()
- subtasks.append(subtask)
- SubtaskTemplate.objects.get(name='pipelinecontrol schema')
-
- return {'subtasks_created': [s.pk for s in subtasks]}
# todo: determine observation subtask, then call connect_observation_subtask_to_preprocessing_subtask to create inputs (not sure where exactly this should happen)
+ return subtask
def _generate_subtask_specs_from_preprocessing_task_specs(preprocessing_task_specs, default_subtask_specs):
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index 9edd4e04e40..98781f242cb 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -259,7 +259,10 @@ class TaskBlueprintViewSet(LOFARViewSet):
def create_subtasks(self, request, pk=None):
task_blueprint = get_object_or_404(models.TaskBlueprint, pk=pk)
subtasks = create_subtasks_from_task_blueprint(task_blueprint)
- return JsonResponse(subtasks)
+
+ # return a response with the new serialized Subtasks
+ serialized_subtasks = serializers.SubtaskSerializer(subtasks, many=True, context={'request':request}).data
+ return Response(serialized_subtasks, status=status.HTTP_201_CREATED)
class TaskBlueprintNestedViewSet(LOFARNestedViewSet):
--
GitLab