diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py index d60555f635901124bf2ae1003f13f8049f0d2bc8..e78bb4d9167f402eff1c7ddef1690aa1da2728b1 100644 --- a/SAS/TMSS/src/tmss/tmssapp/populate.py +++ b/SAS/TMSS/src/tmss/tmssapp/populate.py @@ -196,6 +196,70 @@ def _populate_correlator_calibrator_schema(): "CEP4", "DragNet" ] + }, + "QA": { + "type": "object", + "title": "Quality Assurance", + "default": {}, + "description": "Specify Quality Assurance steps for this observation", + "properties": { + "file_conversion": { + "type": "object", + "title": "File Conversion", + "default": {}, + "description": "Create a QA file for the observation", + "properties": { + "enabled": { + "type": "boolean", + "title": "enabled", + "default": true, + "description": "Do/Don't create a QA file for the observation" + }, + "nr_of_subbands": { + "type": "integer", + "title": "#subbands", + "default": -1, + "description": "Keep this number of subbands from the observation in the QA file, or all if -1" + }, + "nr_of_timestamps": { + "type": "integer", + "title": "#timestamps", + "default": 256, + "minimum": 1, + "description": "Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)" + } + }, + "additionalProperties": false + }, + "plots": { + "type": "object", + "title": "Plots", + "default": {}, + "description": "Create dynamic spectrum plots", + "properties": { + "enabled": { + "type": "boolean", + "title": "enabled", + "default": true, + "description": "Do/Don't create plots from the QA file from the observation" + }, + "autocorrelation": { + "type": "boolean", + "title": "autocorrelation", + "default": true, + "description": "Create autocorrelation plots for all stations" + }, + "crosscorrelation": { + "type": "boolean", + "title": "crosscorrelation", + "default": true, + "description": "Create crosscorrelation plots for all baselines" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } } }'''), "tags": []} @@ -807,10 +871,22 @@ def _populate_qa_files_subtask_template(): "$id": "http://example.com/example.json", "type": "object", "$schema": "http://json-schema.org/draft-06/schema#", - "definitions": { - }, + "definitions": {}, "additionalProperties": false, "properties": { + "nr_of_subbands": { + "type": "integer", + "title": "#subbands", + "default": -1, + "description": "Keep this number of subbands from the observation in the QA file, or all if -1" + }, + "nr_of_timestamps": { + "type": "integer", + "title": "#timestamps", + "default": 256, + "minimum": 1, + "description": "Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)" + } } }'''), "realtime": False, @@ -829,10 +905,21 @@ def _populate_qa_plots_subtask_template(): "$id": "http://example.com/example.json", "type": "object", "$schema": "http://json-schema.org/draft-06/schema#", - "definitions": { - }, + "definitions": {}, "additionalProperties": false, "properties": { + "autocorrelation": { + "type": "boolean", + "title": "autocorrelation", + "default": true, + "description": "Create autocorrelation plots for all stations" + }, + "crosscorrelation": { + "type": "boolean", + "title": "crosscorrelation", + "default": true, + "description": "Create crosscorrelation plots for all baselines" + } } }'''), "realtime": False, diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py index f6fd537561fcf422238af57845126a6f4081da92..1e6542a5ff0e120accf43b509d61aef0a9befc04 100644 --- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py +++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py @@ -3,6 +3,7 @@ logger = logging.getLogger(__name__) from datetime import datetime, timedelta from lofar.common.datetimeutils import parseDatetime +from lofar.common.json_utils import add_defaults_to_json_object_for_schema from lofar.sas.tmss.tmss.tmssapp.models.specification import * from lofar.sas.tmss.tmss.tmssapp.models.scheduling import * @@ -22,14 +23,26 @@ def create_observation_to_qafile_subtask(observation_subtask: Subtask): raise ValueError("Cannot create %s subtask for subtask id=%d because it is not DEFINED yet" % ( SubtaskType.Choices.QA_FILES.value, observation_subtask.pk)) - # step 1: create subtask in defining state + obs_task_spec = observation_subtask.task_blueprint.specifications_doc + obs_task_qafile_spec = obs_task_spec.get("QA", {}).get("file_conversion", {}) + + if not obs_task_qafile_spec.get("enabled", False): + logger.debug("Skipping creation of qafile_subtask because QA.file_conversion is not enabled") + return None + + # step 1: create subtask in defining state, with filled-in subtask_template qafile_subtask_template = SubtaskTemplate.objects.get(name="QA file conversion") + qafile_subtask_spec = add_defaults_to_json_object_for_schema({}, qafile_subtask_template.schema) + qafile_subtask_spec['nr_of_subbands'] = obs_task_qafile_spec.get("nr_of_subbands") + qafile_subtask_spec['nr_of_timestamps'] = obs_task_qafile_spec.get("nr_of_timestamps") + validate_json_against_schema(qafile_subtask_spec, qafile_subtask_template.schema) + qafile_subtask_data = { "start_time": None, "stop_time": None, "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value), "task_blueprint": observation_subtask.task_blueprint, "specifications_template": qafile_subtask_template, - "specifications_doc": "", + "specifications_doc": qafile_subtask_spec, "priority": 1, "schedule_method": ScheduleMethod.objects.get(value=ScheduleMethod.Choices.DYNAMIC.value), "cluster": observation_subtask.cluster} @@ -109,14 +122,26 @@ def create_qafile_to_qaplots_subtask(qafile_subtask: Subtask): raise ValueError("Cannot create %s subtask for subtask id=%d because it is not DEFINED yet" % ( SubtaskType.Choices.QA_PLOTS.value, qafile_subtask.pk)) - # step 1: create subtask in defining state + obs_task_spec = qafile_subtask.task_blueprint.specifications_doc + obs_task_qaplots_spec = obs_task_spec.get("QA", {}).get("plots", {}) + + if not obs_task_qaplots_spec.get("enabled", False): + logger.debug("Skipping creation of qaplots_subtask because QA.plots is not enabled") + return None + + # step 1: create subtask in defining state, with filled-in subtask_template qaplots_subtask_template = SubtaskTemplate.objects.get(name="QA plots") + qaplots_subtask_spec_doc = add_defaults_to_json_object_for_schema({}, qaplots_subtask_template.schema) + qaplots_subtask_spec_doc['autocorrelation'] = obs_task_qaplots_spec.get("autocorrelation") + qaplots_subtask_spec_doc['crosscorrelation'] = obs_task_qaplots_spec.get("crosscorrelation") + validate_json_against_schema(qaplots_subtask_spec_doc, qaplots_subtask_template.schema) + qaplots_subtask_data = { "start_time": None, "stop_time": None, "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value), "task_blueprint": qafile_subtask.task_blueprint, "specifications_template": qaplots_subtask_template, - "specifications_doc": "", + "specifications_doc": qaplots_subtask_spec_doc, "priority": 1, "schedule_method": ScheduleMethod.objects.get(value=ScheduleMethod.Choices.DYNAMIC.value), "cluster": qafile_subtask.cluster} diff --git a/SAS/TMSS/src/tmss/tmssapp/tasks.py b/SAS/TMSS/src/tmss/tmssapp/tasks.py index 8e9b27665f394c5d14957c44194756892333fc29..0296932af9f1d9e1a45396585a07cdd56ddb7c7c 100644 --- a/SAS/TMSS/src/tmss/tmssapp/tasks.py +++ b/SAS/TMSS/src/tmss/tmssapp/tasks.py @@ -14,7 +14,7 @@ import logging logger = logging.getLogger(__name__) -def run_specify_observation(task_draft: models.TaskDraft): +def create_task_blueprint_from_task_draft_and_instantiate_subtasks_from_template(task_draft: models.TaskDraft): """ Create a task_blueprint from the task_draft For every subtask specified in task blueprint: @@ -23,31 +23,28 @@ def run_specify_observation(task_draft: models.TaskDraft): - link subtask inputs to predecessor outputs - set subtask to DEFINED """ - logger.debug("run_specify_observation...") - task_blueprint = create_taskblueprint_from_taskdraft(task_draft) - results_str = "# BLUEPRINT TASK ID=%d CREATED FROM TASK DRAFT ID=%d\n" % (task_blueprint.id, task_draft.pk) + logger.debug("create_task_blueprint_from_task_draft_and_instantiate_subtasks_from_template(task_draft.id=%s)...", task_draft.pk) + task_blueprint = create_task_blueprint_from_task_draft(task_draft) obs_subtask = create_subtask_observation_control(task_blueprint) pipe_subtask = create_subtask_pipeline_control(task_blueprint) - results_str += "# SUBTASKS %d and %d CREATED FROM THE TASK BLUEPRINT ID=%d\n" %\ - (obs_subtask.id, pipe_subtask.id, task_blueprint.id) - connect_observation_subtask_to_preprocessing_subtask(obs_subtask, pipe_subtask) - qa_file_subtask = create_observation_to_qafile_subtask(obs_subtask) - qa_plots_subtask = create_qafile_to_qaplots_subtask(qa_file_subtask) - results_str += "# SUBTASK %d CREATED FROM SUBTASK ID=%d\n" % (qa_file_subtask.id, obs_subtask.id) - results_str += "# SUBTASK %d CREATED FROM SUBTASK ID=%d\n" % (qa_plots_subtask.id, qa_file_subtask.id) - logger.info(results_str) - return results_str + if task_blueprint.specifications_doc.get("QA",{}).get("file_conversion",{}).get("enabled", False): + qa_file_subtask = create_observation_to_qafile_subtask(obs_subtask) + + if qa_file_subtask is not None and task_blueprint.specifications_doc.get("QA", {}).get("plots", {}).get("enabled", False): + qa_plots_subtask = create_qafile_to_qaplots_subtask(qa_file_subtask) + return task_blueprint -def create_taskblueprint_from_taskdraft(task_draft: models.TaskDraft): + +def create_task_blueprint_from_task_draft(task_draft: models.TaskDraft): """ Create a task_blueprint from the task_draft :raises Exception if instantiate fails. """ - logger.debug("create_taskblueprint_from_taskdraft") + logger.debug("create_task_blueprint_from_task_draft(task_draft.id=%s)", task_draft.pk) # Get scheduling unit blueprint from scheduling unit draft, but that is a multi object relation # so which one is related to this task_draft? @@ -73,8 +70,8 @@ def create_taskblueprint_from_taskdraft(task_draft: models.TaskDraft): specifications_doc=task_draft.specifications_doc, specifications_template=task_draft.specifications_template ) - # Add blueprint id as relation in task_draftmodels - print("Return the blueprint id " + str(task_blueprint.id)) + + logger.info("create_task_blueprint_from_task_draft(task_draft.id=%s) created task_blueprint: %s", task_draft.pk, task_blueprint.pk) return task_blueprint @@ -126,7 +123,7 @@ def create_subtask_observation_control(task_blueprint: models.TaskBlueprint): subtask = Subtask.objects.create(**subtask_data) # step 2: create and link subtask input/output - # Observation does not have input so only create output + # an observation has no input, it just produces output data subtask_output = SubtaskOutput.objects.create(subtask=subtask) # step 3: set state to DEFINED diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py index cd8de190e9cdbb9c42eb5d52f3838f9f9817e70e..9c28360f2884cce25dbe292b903cf5ba9bb50a4a 100644 --- a/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py +++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/scheduling.py @@ -94,6 +94,14 @@ class SubtaskTemplateViewSet(LOFARViewSet): return queryset + @swagger_auto_schema(responses={200: 'The schema as a JSON object', + 403: 'forbidden'}, + operation_description="Get the schema as a JSON object.") + @action(methods=['get'], detail=True) + def schema(self, request, pk=None): + subtask_template = get_object_or_404(models.SubtaskTemplate, pk=pk) + return JsonResponse(subtask_template.schema) + @swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in', 403: 'forbidden'}, operation_description="Get a JSON object with all the defaults from the schema filled in.") diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py index 96674287c91312e7749191882ac50e50ba6806a3..59625e2617b34fc46b757de9fb2d4b66ba6cab0a 100644 --- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py +++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py @@ -3,9 +3,12 @@ This file contains the viewsets (based on the elsewhere defined data models and """ from django.shortcuts import get_object_or_404 -from django.http import HttpResponse, JsonResponse +from django.http import JsonResponse +from django.utils.cache import add_never_cache_headers from django.contrib.auth.models import User from rest_framework.viewsets import ReadOnlyModelViewSet +from rest_framework import status +from rest_framework.response import Response from rest_framework.decorators import permission_classes from rest_framework.permissions import IsAuthenticatedOrReadOnly, DjangoModelPermissions @@ -20,7 +23,7 @@ from lofar.sas.tmss.tmss.tmssapp import serializers from datetime import datetime from lofar.common.json_utils import get_default_json_object_for_schema from lofar.common.datetimeutils import formatDatetime -from lofar.sas.tmss.tmss.tmssapp.tasks import run_specify_observation +from lofar.sas.tmss.tmss.tmssapp.tasks import create_task_blueprint_from_task_draft_and_instantiate_subtasks_from_template # This is required for keeping a user reference as ForeignKey in other models @@ -59,6 +62,14 @@ class TaskTemplateViewSet(LOFARViewSet): queryset = models.TaskTemplate.objects.all() serializer_class = serializers.TaskTemplateSerializer + @swagger_auto_schema(responses={200: 'The schema as a JSON object', + 403: 'forbidden'}, + operation_description="Get the schema as a JSON object.") + @action(methods=['get'], detail=True) + def schema(self, request, pk=None): + template = get_object_or_404(models.TaskTemplate, pk=pk) + return JsonResponse(template.schema) + @swagger_auto_schema(responses={200: 'JSON object with all the defaults from the schema filled in', 403: 'forbidden'}, operation_description="Get a JSON object with all the defaults from the schema filled in.") @@ -250,10 +261,21 @@ class TaskDraftViewSetJSONeditorOnline(LOFARViewSet): else: return models.TaskDraft.objects.all() - @action(methods=['get'], detail=True) - def specify_observation(self, request, pk=None): - task = get_object_or_404(models.TaskDraft, pk=pk) - results_str = run_specify_observation(task) - # Just for demo purpose show some result - return HttpResponse(results_str, content_type='text/plain') + @swagger_auto_schema(responses={201: 'Created task blueprint, see Location in Response header', + 403: 'forbidden'}, + operation_description="Carve this draft task specification in stone, and make an (uneditable) blueprint out of it.") + @action(methods=['get'], detail=True, url_name="create_task_blueprint") + def create_task_blueprint(self, request, pk=None): + task_draft = get_object_or_404(models.TaskDraft, pk=pk) + task_blueprint = create_task_blueprint_from_task_draft_and_instantiate_subtasks_from_template(task_draft) + + # url path magic to construct the new task_blueprint_path url + task_draft_path = request._request.path + base_path = task_draft_path[:task_draft_path.find('/task_draft')] + task_blueprint_path = '%s/task_blueprint/%s/' % (base_path, task_blueprint.id,) + + # return a response with the new serialized TaskBlueprint, and a Location to the new instance in the header + return Response(serializers.TaskBlueprintSerializerJSONeditorOnline(task_blueprint, context={'request':request}).data, + status=status.HTTP_201_CREATED, + headers={'Location': task_blueprint_path})