diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index 0d5940a18a8bd7bfd9bc3b1c79831c7f5c9abe19..4c6c82d7970435613cafe260fe89aa5978cc3444 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -43,9 +43,9 @@ def populate_lofar_json_schemas(apps, schema_editor):
_populate_scheduling_unit_schema()
# populate task schema's
- _populate_stations_schema()
+ _populate_stations_schema() # obsolete
_populate_preprocessing_schema()
- _populate_correlator_calibrator_schema()
+ _populate_correlator_calibrator_schema() # obsolete
_populate_observation_with_stations_schema()
_populate_calibrator_addon_schema()
@@ -134,7 +134,7 @@ def _populate_scheduling_unit_schema():
with open(os.path.join(working_dir, "schemas/scheduling-unit.json")) as json_file:
json_data = json.loads(json_file.read())
scheduling_unit_template_data = {"name": "scheduling unit schema",
- "description": 'Schema for UC1 HBA scheduling unit',
+ "description": 'Schema for scheduling unit',
"version": '0.1',
"tags": ["UC1"],
"schema": json_data}
@@ -155,7 +155,7 @@ def _populate_observation_with_stations_schema():
def _populate_calibrator_addon_schema():
with open(os.path.join(working_dir, "schemas/task-calibrator-addon.json")) as json_file:
json_data = json.loads(json_file.read())
- task_template_data = {"name": "calibrator addon schema",
+ task_template_data = {"name": "calibrator schema",
"description": 'addon schema for calibrator observations',
"version": '0.1',
"tags": [],
@@ -169,7 +169,7 @@ def _populate_correlator_calibrator_schema():
task_template_data = {"name": "correlator schema",
"description": 'correlator schema for calibrator observations',
"version": '0.1',
- "tags": [],
+ "tags": ["obsolete?"],
"schema": json_data}
TaskTemplate.objects.create(**task_template_data)
@@ -893,3 +893,12 @@ def _populate_connectors():
output_of=TaskTemplate.objects.get(name='correlator schema'),
input_of=TaskTemplate.objects.get(name='preprocessing schema'))
+ TaskConnector.objects.create(role=Role.objects.get(value=Role.Choices.INPUT.value),
+ datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
+ output_of=TaskTemplate.objects.get(name='observation schema'),
+ input_of=TaskTemplate.objects.get(name='preprocessing schema'))
+
+ TaskConnector.objects.create(role=Role.objects.get(value=Role.Choices.CORRELATOR.value),
+ datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
+ output_of=TaskTemplate.objects.get(name='calibrator schema'),
+ input_of=TaskTemplate.objects.get(name='preprocessing schema'))
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/example_CTC_observation.json b/SAS/TMSS/src/tmss/tmssapp/schemas/example_CTC_observation.json
new file mode 100644
index 0000000000000000000000000000000000000000..db4da481d0b1513b4b935286809ce35c6c59640b
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/example_CTC_observation.json
@@ -0,0 +1,267 @@
+{
+ "tasks": [
+ {
+ "name": "Calibrator Observation 1",
+ "description": "Calibrator Observation for UC1 HBA scheduling block",
+ "tags": [],
+ "specifications_doc": {
+ "duration": 600,
+ "autoselect": false,
+ "pointing": {
+ "direction_type": "J2000",
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0
+ }
+ },
+ "specifications_template": "calibrator schema"
+ },
+ {
+ "name": "Pipeline 1",
+ "description": "Preprocessing Pipeline for Calibrator Observation 1",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ {
+ "name": "Target Observation",
+ "description": "Target Observation for UC1 HBA scheduling block",
+ "tags": [],
+ "specifications_doc": {
+ "QA": {
+ "plots": {
+ "enabled": true,
+ "autocorrelation": true,
+ "crosscorrelation": true
+ },
+ "file_conversion": {
+ "enabled": true,
+ "nr_of_subbands": -1,
+ "nr_of_timestamps": 256
+ }
+ },
+ "duration": 28800,
+ "correlator": {
+ "storage_cluster": "CEP4",
+ "integration_time": 1,
+ "channels_per_subband": 64
+ },
+ "antenna_set": "HBA_DUAL_INNER",
+ "filter": "HBA_110_190",
+ "stations": [
+ {
+ "group": "ALL",
+ "min_stations": 1
+ }
+ ],
+ "tile_beam": {
+ "direction_type": "J2000",
+ "angle1": 42,
+ "angle2": 42,
+ "angle3": 42
+ },
+ "SAPs": [
+ {
+ "name": "target0",
+ "digital_pointing": {
+ "direction_type": "J2000",
+ "angle1": 24,
+ "angle2": 24,
+ "angle3": 24
+ },
+ "subbands": [
+ 349,
+ 372
+ ]
+ }
+ ]
+ },
+ "specifications_template": "observation schema"
+ },
+ {
+ "name": "Pipeline SAP0",
+ "description": "Preprocessing Pipeline for Target Observation SAP0",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ {
+ "name": "Pipeline SAP1",
+ "description": "Preprocessing Pipeline for Target Observation SAP1",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ {
+ "name": "Calibrator Observation 2",
+ "description": "Calibrator Observation for UC1 HBA scheduling block",
+ "tags": [],
+ "specifications_doc": {
+ "duration": 600,
+ "autoselect": false,
+ "pointing": {
+ "direction_type": "J2000",
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0
+ }
+ },
+ "specifications_template": "calibrator schema"
+ },
+ {
+ "name": "Pipeline 2",
+ "description": "Preprocessing Pipeline for Calibrator Observation 2",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ }
+ ],
+ "task_relations": [
+ {
+ "producer": "Calibrator Observation 1",
+ "consumer": "Pipeline 1",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "producer": "Calibrator Observation 2",
+ "consumer": "Pipeline 2",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "producer": "Target Observation",
+ "consumer": "Pipeline SAP0",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {"sap":[0]},
+ "selection_template": "SAP"
+ },
+ {
+ "producer": "Target Observation",
+ "consumer": "Pipeline SAP1",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {"sap":[1]},
+ "selection_template": "SAP"
+ }
+ ],
+ "task_scheduling_relations": [
+ {
+ "first": "Calibrator Observation 1",
+ "second": "Target Observation",
+ "placement": "before",
+ "time_offset": 60
+ },
+ {
+ "first": "Calibration Observation 2",
+ "second": "Target Observation",
+ "placement": "after",
+ "time_offset": 60
+ }
+ ]
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json
index 0d86b1861788243b88d6ec47fbff626da42f999a..11f98660cda0aff9518ac184687f11b805f697ce 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task-calibrator-addon.json
@@ -15,6 +15,8 @@
"default": "J2000",
"enum": [
"J2000",
+ "AZELGEO",
+ "LMN",
"SUN",
"MOON",
"MERCURY",
@@ -38,8 +40,18 @@
"title": "Angle 2",
"description": "Second angle (f.e. DEC)",
"default": 0
+ },
+ "angle3": {
+ "type": "number",
+ "title": "Angle 3",
+ "description": "Third angle (f.e. N in LMN)",
+ "default": 0
}
- }
+ },
+ "required": [
+ "angle1",
+ "angle2"
+ ]
}
},
"format": "grid",
@@ -64,5 +76,8 @@
"description": "Manually selected calibrator",
"$ref": "#/definitions/pointing"
}
- }, "required": ["autoselect"]
+ },
+ "required": [
+ "autoselect"
+ ]
}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
index 9bfccc111092ca5946078edd7331cb29faf8c567..b5dfccb433abb2bab71eb7e83449a2e461f140f1 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task-observation-with-stations.json
@@ -15,6 +15,8 @@
"default": "J2000",
"enum": [
"J2000",
+ "AZELGEO",
+ "LMN",
"SUN",
"MOON",
"MERCURY",
@@ -38,6 +40,12 @@
"title": "Angle 2",
"description": "Second angle (f.e. DEC)",
"default": 0
+ },
+ "angle3": {
+ "type": "number",
+ "title": "Angle 3",
+ "description": "Third angle (f.e. N in LMN)",
+ "default": 0
}
},
"required": [
@@ -197,18 +205,19 @@
"HBA_210_250"
]
},
- "analog_pointing": {
- "title": "Analog pointing",
+ "tile_beam": {
+ "title": "Tile beam",
"description": "HBA only",
"$ref": "#/definitions/pointing"
},
- "beams": {
+ "SAPs": {
"type": "array",
- "title": "Beams",
+ "title": "SAPs",
+ "description": "Station beams",
"format": "tabs",
"additionalItems": false,
"items": {
- "title": "Beam",
+ "title": "SAP",
"headerTemplate": "{{ i0 }} - {{ self.name }}",
"type": "object",
"additionalProperties": false,
@@ -294,13 +303,77 @@
"integration_time",
"storage_cluster"
]
+ },
+ "QA": {
+ "type": "object",
+ "title": "Quality Assurance",
+ "default": {},
+ "description": "Specify Quality Assurance steps for this observation",
+ "properties": {
+ "file_conversion": {
+ "type": "object",
+ "title": "File Conversion",
+ "default": {},
+ "description": "Create a QA file for the observation",
+ "properties": {
+ "enabled": {
+ "type": "boolean",
+ "title": "enabled",
+ "default": true,
+ "description": "Do/Don't create a QA file for the observation"
+ },
+ "nr_of_subbands": {
+ "type": "integer",
+ "title": "#subbands",
+ "default": -1,
+ "description": "Keep this number of subbands from the observation in the QA file, or all if -1"
+ },
+ "nr_of_timestamps": {
+ "type": "integer",
+ "title": "#timestamps",
+ "default": 256,
+ "minimum": 1,
+ "description": "Extract this number of timestamps from the observation in the QA file (equidistantanly sampled, no averaging/interpolation)"
+ }
+ },
+ "additionalProperties": false
+ },
+ "plots": {
+ "type": "object",
+ "title": "Plots",
+ "default": {},
+ "description": "Create dynamic spectrum plots",
+ "properties": {
+ "enabled": {
+ "type": "boolean",
+ "title": "enabled",
+ "default": true,
+ "description": "Do/Don't create plots from the QA file from the observation"
+ },
+ "autocorrelation": {
+ "type": "boolean",
+ "title": "autocorrelation",
+ "default": true,
+ "description": "Create autocorrelation plots for all stations"
+ },
+ "crosscorrelation": {
+ "type": "boolean",
+ "title": "crosscorrelation",
+ "default": true,
+ "description": "Create crosscorrelation plots for all baselines"
+ }
+ },
+ "additionalProperties": false
+ }
+ },
+ "additionalProperties": false
}
},
"required": [
"stations",
"antenna_set",
"filter",
- "beams",
+ "SAPs",
"duration",
"correlator"
]
diff --git a/SAS/TMSS/src/tmss/tmssapp/tasks.py b/SAS/TMSS/src/tmss/tmssapp/tasks.py
index b97afa0eb13b3714d55586c27c03b265ab7524d6..84cd542871c03b8b88f0cd258ba6ce606d26191a 100644
--- a/SAS/TMSS/src/tmss/tmssapp/tasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/tasks.py
@@ -1,5 +1,5 @@
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.sas.tmss.tmss.tmssapp.models.specification import TaskBlueprint, SchedulingUnitBlueprint
+from lofar.sas.tmss.tmss.tmssapp.models.specification import TaskBlueprint, SchedulingUnitBlueprint, TaskDraft
from lofar.sas.tmss.tmss.tmssapp.subtasks import create_and_schedule_subtasks_from_task_blueprint
from functools import cmp_to_key
@@ -27,6 +27,51 @@ def create_scheduling_unit_blueprint_from_scheduling_unit_draft(scheduling_unit_
return scheduling_unit_blueprint
+def create_tasks_draft_from_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft) -> [TaskDraft]:
+ """
+ Generic create-method for tasks draft. Calls the appropriate create method based on the scheduling_unit_blueprint
+ specifications_template name.
+ """
+ list_tasks = scheduling_unit_draft.requirements_doc["tasks"]
+ list_task_relations = scheduling_unit_draft.requirements_doc["task_relations"]
+ list_task_scheduling_relations = scheduling_unit_draft.requirements_doc["task_scheduling_relations"]
+
+ print("create_tasks_draft_from_scheduling_unit_draft with scheduling_unit_draft.id=%s, nbr_tasks=%d" %
+ (scheduling_unit_draft.pk, len(list_tasks)))
+ for task in list_tasks:
+ task_template_name = task["specifications_template"]
+ print("task name is '%s', task_template_name '%s'" % (task["name"], task_template_name))
+ task_template = models.TaskTemplate.objects.get(name=task_template_name)
+ task_draft = models.TaskDraft.objects.create(
+ name=task["name"],
+ description=task["description"],
+ tags=task["tags"],
+ specifications_doc=task["specifications_doc"],
+ copy_reason=models.CopyReason.objects.get(value='template'),
+ copies=None,
+ scheduling_unit_draft=scheduling_unit_draft,
+ specifications_template=task_template)
+ print("task draft with id %s created succesfully" % task_draft.id)
+
+ # Now create task relation
+ print("create_tasks_draft_from_scheduling_unit_draft, nbr of task relations=%d" % len(list_task_relations))
+ for task_relation in list_task_relations:
+ task_rel_obj = models.TaskRelationDraft.objects.create(
+ tags=task_relation["tags"],
+ selection_doc=task_relation["selection_doc"],
+ dataformat=models.Dataformat.objects.get(value=task_relation["dataformat"]),
+ producer=models.TaskDraft.objects.get(name=task_relation["producer"]),
+ consumer=models.TaskDraft.objects.get(name=task_relation["consumer"]),
+ input=models.TaskConnector.objects.get(role=task_relation["input"]["role"]),
+ output=models.TaskConnector.objects.get(role=task_relation["output"]["role"]),
+ selection_template=models.TaskRelationSelectionTemplate.objects.get(name=task_relation["selection_template"]))
+ print("task relation draft object with id %s created successfully" % task_rel_obj.id)
+
+ # TODO task_scheduling_relation ??
+ # Apparently a task scheduling relation draft and a task scheduling relation blueprint table must be created
+ # Also have to get the QA properties and maybe more from Target Observation before and after
+
+
def create_task_blueprint_from_task_draft(task_draft: models.TaskDraft) -> models.TaskBlueprint:
"""
Create a task_blueprint from the task_draft
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index 4516d434acbf9e031a0c6daad5d01a019668cea2..d5febba4edd187a6bb2349707a4de2652103739e 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -202,6 +202,20 @@ class SchedulingUnitDraftViewSet(LOFARViewSet):
status=status.HTTP_201_CREATED,
headers={'Location': scheduling_unit_blueprint_path})
+ @swagger_auto_schema(responses={201: 'The Created Task Draft, see Location in Response header',
+ 403: 'forbidden'},
+ operation_description="Create Tasks Draft from SchedulingUnitDraft.")
+ @action(methods=['get'], detail=True, url_name="create_tasks_draft", name="Create Tasks Draft from Requirement doc")
+ def create_tasks_draft(self, request, pk=None):
+ scheduling_unit_draft = get_object_or_404(models.SchedulingUnitDraft, pk=pk)
+ create_tasks_draft_from_scheduling_unit_draft(scheduling_unit_draft)
+
+ # return a response with the new serialized scheduling_unit_blueprint (with references to the created task_blueprint(s) and (scheduled) subtasks)
+ return Response(serializers.SchedulingUnitDraftSerializer(scheduling_unit_draft,
+ context={'request':request}).data,
+ status=status.HTTP_201_CREATED)
+
+
class SchedulingUnitDraftNestedViewSet(LOFARNestedViewSet):
queryset = models.SchedulingUnitDraft.objects.all()
serializer_class = serializers.SchedulingUnitDraftSerializer