diff --git a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
index a6932d705e358db231d0fe22f47496a970591b29..00d119f380cb0b035aba9e7e432dc94cec708df2 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
@@ -15,6 +15,7 @@ from django.contrib.auth.models import User
from .specification import AbstractChoice, BasicCommon, Template, NamedCommon, annotate_validate_add_defaults_to_doc_using_template
from enum import Enum
from django.db.models.expressions import RawSQL
+from django.core.exceptions import ValidationError
from lofar.sas.tmss.tmss.exceptions import SubtaskSchedulingException
from lofar.messaging.messagebus import ToBus, DEFAULT_BROKER, DEFAULT_BUSNAME
@@ -195,6 +196,17 @@ class Subtask(BasicCommon):
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ # check for uniqueness of SAP names:
+ # todo: this is a very specific check, that depends on the template. On the task level, we have a javascript
+ # field for complex validation on the template (I suspect that's for the frontend), so maybe we want to have
+ # a similar mechanism to describe complex validation on the backend?
+ if self.specifications_doc and 'stations' in self.specifications_doc and 'digital_pointings' in self.specifications_doc['stations']:
+ sap_names = [pointing['name'] for pointing in self.specifications_doc['stations']['digital_pointings']]
+ duplicate_names = [name for name in set(sap_names) if sap_names.count(name) > 1]
+ if duplicate_names:
+ raise ValidationError("Pointings defined in the same Subtask must have unique names. Duplicate names are %s" % duplicate_names)
+
+ # check if we have a start time or there were predecessors
if self.state.value == SubtaskState.Choices.SCHEDULED.value and self.__original_state_id == SubtaskState.Choices.SCHEDULING.value:
if self.start_time is None:
if self.predecessors.all().count() == 0:
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json b/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
index 183f8933332dd016194b939315b4bc25a9c3d183..e70a9b49e4e356ff90b1d0d302cda4a00b603d4f 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
@@ -11,7 +11,8 @@
"angle1": 0,
"angle2": 0,
"angle3": 0
- }
+ },
+ "name": "calibrator1"
},
"specifications_template": "calibrator observation"
},
@@ -71,7 +72,7 @@
},
"SAPs": [
{
- "name": "target0",
+ "name": "target1",
"digital_pointing": {
"direction_type": "J2000",
"angle1": 0.24,
@@ -84,7 +85,7 @@
]
},
{
- "name": "target1",
+ "name": "target2",
"digital_pointing": {
"direction_type": "J2000",
"angle1": 0.27,
@@ -100,8 +101,8 @@
},
"specifications_template": "target observation"
},
- "Pipeline SAP0": {
- "description": "Preprocessing Pipeline for Target Observation SAP0",
+ "Pipeline target1": {
+ "description": "Preprocessing Pipeline for Target Observation target1",
"tags": [],
"specifications_doc": {
"flag": {
@@ -123,8 +124,8 @@
},
"specifications_template": "preprocessing pipeline"
},
- "Pipeline SAP1": {
- "description": "Preprocessing Pipeline for Target Observation SAP1",
+ "Pipeline target2": {
+ "description": "Preprocessing Pipeline for Target Observation target2",
"tags": [],
"specifications_doc": {
"flag": {
@@ -157,7 +158,8 @@
"angle1": 0,
"angle2": 0,
"angle3": 0
- }
+ },
+ "name": "calibrator2"
},
"specifications_template": "calibrator observation"
},
@@ -220,7 +222,7 @@
},
{
"producer": "Target Observation",
- "consumer": "Pipeline SAP0",
+ "consumer": "Pipeline target1",
"tags": [],
"input": {
"role": "input",
@@ -233,14 +235,14 @@
"dataformat": "MeasurementSet",
"selection_doc": {
"sap": [
- 0
+ "target1"
]
},
"selection_template": "SAP"
},
{
"producer": "Target Observation",
- "consumer": "Pipeline SAP1",
+ "consumer": "Pipeline target2",
"tags": [],
"input": {
"role": "input",
@@ -253,7 +255,7 @@
"dataformat": "MeasurementSet",
"selection_doc": {
"sap": [
- 1
+ "target2"
]
},
"selection_template": "SAP"
@@ -276,14 +278,14 @@
"parameters": [
{
"refs": [
- "#/tasks/Target Observation/specifications_doc/SAPs/0/digital_pointing"
+ "#/tasks/Target Observation/specifications_doc/SAPs/target1/digital_pointing"
],
- "name": "Target Pointing 0"
+ "name": "Target Pointing 1"
},{
"refs": [
- "#/tasks/Target Observation/specifications_doc/SAPs/1/digital_pointing"
+ "#/tasks/Target Observation/specifications_doc/SAPs/target2/digital_pointing"
],
- "name": "Target Pointing 1"
+ "name": "Target Pointing 2"
},{
"refs": [
"#/tasks/Target Observation/specifications_doc/tile_beam"
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-SAP-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-SAP-1.json
index 16265b5fcc2f0080cef17c7c927ea7ea369bfe85..d78d9ec50bdc3e59f71688ec0acf2e1bbe04b178 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-SAP-1.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/dataproduct_specifications_template-SAP-1.json
@@ -12,10 +12,8 @@
"additionalItems": false,
"default": [],
"items": {
- "type": "integer",
- "title": "sap",
- "minimum": 0,
- "maximum": 1
+ "type": "string",
+ "title": "sap"
}
}
}
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
index c016a1412a74225fa66047319f0d2bba2f75a89e..7beb99e31f76dbc154fb28e8e2739c424e51a3a8 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-observation-1.json
@@ -46,10 +46,16 @@
"properties": {
"name": {
"type": "string",
- "title": "Name/target",
+ "title": "Name",
"description": "Custom identifier for this beam. Same name is same beam.",
"default": ""
},
+ "target": {
+ "type": "string",
+ "title": "Target",
+ "description": "Name of the target",
+ "default": ""
+ },
"pointing": {
"title": "Digital pointing",
"$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing",
@@ -67,7 +73,12 @@
"maximum": 511
}
}
- }
+ },
+ "required": [
+ "target",
+ "name",
+ "pointing",
+ "subbands"]
}
}
},
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
index cc6b1e86bdb5d0b1145042a323672c1228d9767f..8307de613566df0b7a19d2417a24b740d3f41e7a 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-pipeline-1.json
@@ -5,7 +5,6 @@
"description":"This schema defines the parameters to setup and control a (preprocessing) pipeline subtask.",
"version":1,
"type": "object",
- "$schema": "http://json-schema.org/draft-06/schema#",
"properties": {
"preflagger0": {
"title": "Preflagger0",
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json
index bc62ea8d7b4493cf3ff11bea012a9f962229fbd9..57b57d9c63e1829bec3b9aeb194922513da974cd 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_relation_selection_template-SAP-1.json
@@ -12,10 +12,8 @@
"additionalItems": false,
"default": [],
"items": {
- "type": "integer",
- "title": "sap",
- "minimum": 0,
- "maximum": 1
+ "type": "string",
+ "title": "sap"
}
}
}
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json
index cd606bf6794ef166c491a80cff583e0838d8d788..ec6696b0efcfc82bf577b6640b112006a5dc7b47 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-calibrator_observation-1.json
@@ -25,9 +25,15 @@
"$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/pointing/1/#/definitions/pointing",
"description": "Manually selected calibrator",
"default": {}
+ },
+ "name": {
+ "title": "Name",
+ "description": "Name of the calibrator SAP",
+ "type": "string",
+ "default": "calibrator"
}
},
"required": [
- "autoselect", "duration", "pointing"
+ "autoselect", "duration", "pointing", "name"
]
}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json
index f6e92bc010d32d7e20cc879dfe7576fd18dfdb71..0962ebf708493f0aaf2e1d7f50fc34f056d975ba 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/task_template-target_observation-1.json
@@ -41,10 +41,16 @@
"properties": {
"name": {
"type": "string",
- "title": "Name/target",
+ "title": "Name",
"description": "Identifier for this beam",
"default": ""
},
+ "target": {
+ "type": "string",
+ "title": "Target",
+ "description": "Description of where this beam points at",
+ "default": ""
+ },
"digital_pointing": {
"$id": "#target_pointing",
"title": "Digital pointing",
@@ -65,6 +71,8 @@
}
},
"required": [
+ "target",
+ "name",
"digital_pointing",
"subbands"
]
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index 94e2ce2064ba868c847e6d98c11f96150072d3f0..fdc87fe2d23a1331666e672a0a32d3fa2c9ab912 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -119,7 +119,7 @@ def create_observation_subtask_specifications_from_observation_task_blueprint(ta
"angle2": task_spec["pointing"]["angle2"]}
# for the calibrator, the digital pointing is equal to the analog pointing
- subtask_spec['stations']['digital_pointings'] = [ {'name': 'calibrator', # there is no name for the calibrator pointing in the task spec
+ subtask_spec['stations']['digital_pointings'] = [ {'name': task_spec['name'],
'subbands': list(range(0,488)), # there are no subbands for the calibrator pointing in the task spec
'pointing': subtask_spec['stations']['analog_pointing'] } ]
# Use the Task Specification of the Target Observation
@@ -147,6 +147,7 @@ def create_observation_subtask_specifications_from_observation_task_blueprint(ta
for sap in task_spec.get("SAPs", []):
subtask_spec['stations']['digital_pointings'].append(
{"name": sap["name"],
+ "target": sap["target"],
"pointing": {"direction_type": sap["digital_pointing"]["direction_type"],
"angle1": sap["digital_pointing"]["angle1"],
"angle2": sap["digital_pointing"]["angle2"]},
@@ -742,7 +743,7 @@ def schedule_observation_subtask(observation_subtask: Subtask):
dataformat=Dataformat.objects.get(value="MeasurementSet"),
datatype=Datatype.objects.get(value="visibilities"), # todo: is this correct?
producer=subtask_output,
- specifications_doc={"sap": [sap_nr]}, # todo: set correct value. This will be provided by the RA somehow
+ specifications_doc={"sap": [str(sap_nr)]}, # todo: set correct value. This will be provided by the RA somehow
specifications_template=dataproduct_specifications_template,
feedback_doc=get_default_json_object_for_schema(dataproduct_feedback_template.schema),
feedback_template=dataproduct_feedback_template,
diff --git a/SAS/TMSS/test/t_scheduling.py b/SAS/TMSS/test/t_scheduling.py
index 8bdd2df6476ddb86c6b59629653875506c42f4cf..ae0c9036a3b1c07f93fb2478299a78716877f76b 100755
--- a/SAS/TMSS/test/t_scheduling.py
+++ b/SAS/TMSS/test/t_scheduling.py
@@ -268,16 +268,16 @@ class SubtaskInputOutputTest(unittest.TestCase):
# create connected pipeline subtask and inputs, specify input filtering
pipe_st = create_subtask_object_for_testing('pipeline', 'defined')
pipe_out = models.SubtaskOutput.objects.create(**SubtaskOutput_test_data(subtask=pipe_st)) # required by scheduling function
- pipe_in1 = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=pipe_st, producer=obs_out1, selection_doc={'sap': [0]}))
- pipe_in2 = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=pipe_st, producer=obs_out2, selection_doc={'sap': [1]}))
+ pipe_in1 = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=pipe_st, producer=obs_out1, selection_doc={'sap': ['target0']}))
+ pipe_in2 = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=pipe_st, producer=obs_out2, selection_doc={'sap': ['target1']}))
# create obs output dataproducts with specs we can filter on
- dp1_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': [0]}))
- dp1_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': [1]}))
- dp1_3 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': [0]}))
+ dp1_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': ['target0']}))
+ dp1_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': ['target1']}))
+ dp1_3 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out1, specifications_doc={'sap': ['target0']}))
- dp2_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': [0]}))
- dp2_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': [1]}))
+ dp2_1 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': ['target0']}))
+ dp2_2 = models.Dataproduct.objects.create(**Dataproduct_test_data(producer=obs_out2, specifications_doc={'sap': ['target1']}))
# trigger:
# schedule pipeline, which should attach the correct subset of dataproducts to the pipeline inputs
diff --git a/SAS/TMSS/test/t_subtasks.py b/SAS/TMSS/test/t_subtasks.py
index 7e024d6b143a6483c1bfcdaedceb428cd44ae281..cc14049e65a0bc1832812fbd0954b5fc42dd962e 100755
--- a/SAS/TMSS/test/t_subtasks.py
+++ b/SAS/TMSS/test/t_subtasks.py
@@ -228,6 +228,21 @@ class SubTasksCreationFromTaskBluePrint(unittest.TestCase):
subtasks = create_subtasks_from_task_blueprint(task_blueprint)
self.assertEqual(3, len(subtasks))
+ def test_create_subtasks_from_task_blueprint_translates_SAP_names(self):
+ task_blueprint = create_task_blueprint_object_for_testing('target observation')
+ task_blueprint.specifications_doc['SAPs'] = [{'name': 'target1', 'target': '', 'subbands': [],
+ 'digital_pointing': {'angle1': 0.1, 'angle2': 0.1, 'angle3': 0.1,
+ 'direction_type': 'J2000'}},
+ {'name': 'target2', 'target': '', 'subbands': [],
+ 'digital_pointing': {'angle1': 0.2, 'angle2': 0.2, 'angle3': 0.2,
+ 'direction_type': 'J2000'}}]
+ subtask = create_observation_control_subtask_from_task_blueprint(task_blueprint)
+ i = 0
+ for sap in task_blueprint.specifications_doc['SAPs']:
+ subtask_pointing = subtask.specifications_doc['stations']['digital_pointings'][i]
+ self.assertEqual(sap['name'], subtask_pointing['name'])
+ self.assertEqual(sap['digital_pointing']['angle1'], subtask_pointing['pointing']['angle1'])
+ i += 1
class SubTasksCreationFromTaskBluePrintCalibrator(unittest.TestCase):
@@ -283,34 +298,34 @@ class SubtaskInputSelectionFilteringTest(unittest.TestCase):
def test_specifications_doc_meets_selection_doc_returns_true_when_filter_applies(self):
# simple selection matches specs
- specs = {'sap': [0]}
- selection = {'sap': [0]}
+ specs = {'sap': ['target0']}
+ selection = {'sap': ['target0']}
self.assertTrue(specifications_doc_meets_selection_doc(specs, selection))
# extra specs are ignored
- specs = {'sap': [0], 'not': 'relevant'}
- selection = {'sap': [0]}
+ specs = {'sap': ['target0'], 'not': 'relevant'}
+ selection = {'sap': ['target0']}
self.assertTrue(specifications_doc_meets_selection_doc(specs, selection))
# complex selection matches specs; multiple keys and values
- specs = {'sap': [0], 'is_relevant': True}
- selection = {'sap': [0, 1], 'is_relevant': True}
+ specs = {'sap': ['target0'], 'is_relevant': True}
+ selection = {'sap': ['target0', 'target1'], 'is_relevant': True}
self.assertTrue(specifications_doc_meets_selection_doc(specs, selection))
def test_specifications_doc_meets_selection_doc_returns_true_when_filter_does_not_apply(self):
# selection mismatches specs
- specs = {'sap': [0]}
- selection = {'sap': [1]}
+ specs = {'sap': ['target0']}
+ selection = {'sap': ['target1']}
self.assertFalse(specifications_doc_meets_selection_doc(specs, selection))
# spec only partially selected
- specs = {'sap': [0, 1]}
- selection = {'sap': [1]}
+ specs = {'sap': ['target0', 'target1']}
+ selection = {'sap': ['target1']}
self.assertFalse(specifications_doc_meets_selection_doc(specs, selection))
# selection not in specs
- specs = {'sap': [0]}
- selection = {'sap': [0], 'is_relevant': True}
+ specs = {'sap': ['target0']}
+ selection = {'sap': ['target0'], 'is_relevant': True}
self.assertFalse(specifications_doc_meets_selection_doc(specs, selection))
diff --git a/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py b/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
index f5207337417bb58a4b825451ff3fe63437f9a9ea..d9f079f35a4ae89db4c160bc9c75c4b554cd5dfc 100755
--- a/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
+++ b/SAS/TMSS/test/t_tmssapp_scheduling_django_API.py
@@ -40,7 +40,7 @@ from lofar.sas.tmss.test.tmss_database_unittest_setup import *
from lofar.sas.tmss.test.tmss_test_data_django_models import *
from django.db.utils import IntegrityError
-
+from django.core.exceptions import ValidationError
class SubtaskTemplateTest(unittest.TestCase):
def test_SubtaskTemplate_gets_created_with_correct_creation_timestamp(self):
@@ -271,6 +271,19 @@ class SubtaskTest(unittest.TestCase):
self.assertEqual(set(), set(subtask4.successors.all()))
self.assertEqual(set((subtask6,)), set(subtask5.successors.all()))
+
+ def test_Subtask_raises_ValidationError_on_duplicate_pointing_names(self):
+ # setup
+ test_data = Subtask_test_data()
+ test_data['specifications_doc']['stations'] = {'digital_pointings': [{'name': 'popular_name'},
+ {'name': 'popular_name'},
+ {'name': 'not_so_popular'}]}
+ # assert
+ with self.assertRaises(ValidationError) as context:
+ models.Subtask = models.Subtask.objects.create(**test_data)
+ self.assertIn('popular_name', str(context.exception))
+
+
class DataproductTest(unittest.TestCase):
def test_Dataproduct_gets_created_with_correct_creation_timestamp(self):
diff --git a/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json b/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json
index 38dc23b9cc5e09253801bbce32c50273cc05b8af..ef80fbc15a59c4cd36b9569379e0b735b7d0bbff 100644
--- a/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json
+++ b/SAS/TMSS/test/testdata/example_UC1_scheduling_unit.json
@@ -223,7 +223,7 @@
"datatype": "visibilities"
},
"dataformat": "MeasurementSet",
- "selection_doc": {"sap":[0]},
+ "selection_doc": {"sap":["target0"]},
"selection_template": "SAP"
},
{
@@ -239,7 +239,7 @@
"datatype": "visibilities"
},
"dataformat": "MeasurementSet",
- "selection_doc": {"sap":[1]},
+ "selection_doc": {"sap":["target1"]},
"selection_template": "SAP"
}
],