diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index 81e15db2fa25ddd0d68245ed2d78273a47ccf76e..607273e7c9f438c01d81c5a90d077e7e79b3bd95 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 2.2.12 on 2020-08-04 12:35
+# Generated by Django 3.0.9 on 2020-08-19 13:24
from django.conf import settings
import django.contrib.postgres.fields
@@ -145,6 +145,9 @@ class Migration(migrations.Migration):
('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
+ options={
+ 'abstract': False,
+ },
),
migrations.CreateModel(
name='DataproductHash',
@@ -171,6 +174,9 @@ class Migration(migrations.Migration):
('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
+ options={
+ 'abstract': False,
+ },
),
migrations.CreateModel(
name='DataproductTransform',
@@ -309,6 +315,9 @@ class Migration(migrations.Migration):
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
('create_function', models.CharField(help_text='Python function to call to execute the generator.', max_length=128)),
],
+ options={
+ 'abstract': False,
+ },
),
migrations.CreateModel(
name='PeriodCategory',
@@ -450,6 +459,22 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='SchedulingUnitObservingStrategyTemplate',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
+ ('description', models.CharField(help_text='A longer description of this object.', max_length=255)),
+ ('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
+ ('template', django.contrib.postgres.fields.jsonb.JSONField(help_text='JSON-data compliant with the JSON-schema in the scheduling_unit_template. This observation strategy template like a predefined recipe with all the correct settings, and defines which parameters the user can alter.')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='SchedulingUnitTemplate',
fields=[
@@ -462,6 +487,9 @@ class Migration(migrations.Migration):
('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
+ options={
+ 'abstract': False,
+ },
),
migrations.CreateModel(
name='StationType',
@@ -550,6 +578,9 @@ class Migration(migrations.Migration):
('queue', models.BooleanField(default=False)),
('realtime', models.BooleanField(default=False)),
],
+ options={
+ 'abstract': False,
+ },
),
migrations.CreateModel(
name='SubtaskType',
@@ -649,28 +680,27 @@ class Migration(migrations.Migration):
('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
],
+ options={
+ 'abstract': False,
+ },
),
migrations.CreateModel(
- name='TaskSchedulingRelationBlueprint',
+ name='TaskType',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
- ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
- ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('time_offset', models.IntegerField(default=60, help_text='Time offset of start of second task with respect to start of first task.')),
+ ('value', models.CharField(max_length=128, primary_key=True, serialize=False, unique=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
- name='TaskSchedulingRelationDraft',
+ name='Setting',
fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('time_offset', models.IntegerField(default=60, help_text='Time offset of start of second task with respect to start of first task.')),
+ ('name', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, primary_key=True, serialize=False, to='tmssapp.Flag', unique=True)),
+ ('value', models.BooleanField()),
],
options={
'abstract': False,
@@ -688,73 +718,47 @@ class Migration(migrations.Migration):
('version', models.CharField(help_text='Version of this template (with respect to other templates of the same name).', max_length=128)),
('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
('validation_code_js', models.CharField(help_text='JavaScript code for additional (complex) validation.', max_length=128)),
+ ('type', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.TaskType')),
],
+ options={
+ 'abstract': False,
+ },
),
migrations.CreateModel(
- name='TaskType',
+ name='TaskSchedulingRelationDraft',
fields=[
- ('value', models.CharField(max_length=128, primary_key=True, serialize=False, unique=True)),
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('time_offset', models.IntegerField(default=60, help_text='Time offset of start of second task with respect to start of first task.')),
+ ('first', models.ForeignKey(help_text='First Task Draft to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='first_to_connect', to='tmssapp.TaskDraft')),
+ ('placement', models.ForeignKey(help_text='Task scheduling relation placement.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingRelationPlacement')),
+ ('second', models.ForeignKey(help_text='Second Task Draft to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='second_to_connect', to='tmssapp.TaskDraft')),
],
options={
'abstract': False,
},
),
-
migrations.CreateModel(
- name='Setting',
+ name='TaskSchedulingRelationBlueprint',
fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
- ('name', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, primary_key=True, serialize=False, to='tmssapp.Flag', unique=True)),
- ('value', models.BooleanField()),
+ ('time_offset', models.IntegerField(default=60, help_text='Time offset of start of second task with respect to start of first task.')),
+ ('first', models.ForeignKey(help_text='First Task Blueprint to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='first_to_connect', to='tmssapp.TaskBlueprint')),
+ ('placement', models.ForeignKey(default='after', help_text='Task scheduling relation placement.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingRelationPlacement')),
+ ('second', models.ForeignKey(help_text='Second Task Blueprint to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='second_to_connect', to='tmssapp.TaskBlueprint')),
],
options={
'abstract': False,
},
),
- migrations.AddConstraint(
- model_name='tasktemplate',
- constraint=models.UniqueConstraint(fields=('name', 'version'), name='TaskTemplate_unique_name_version'),
- ),
- migrations.AddField(
- model_name='tasktemplate',
- name='type',
- field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.TaskType'),
- ),
- migrations.AddField(
- model_name='taskschedulingrelationdraft',
- name='first',
- field=models.ForeignKey(help_text='First Task Draft to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='first_to_connect', to='tmssapp.TaskDraft'),
- ),
- migrations.AddField(
- model_name='taskschedulingrelationdraft',
- name='placement',
- field=models.ForeignKey(help_text='Task scheduling relation placement.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingRelationPlacement'),
- ),
- migrations.AddField(
- model_name='taskschedulingrelationdraft',
- name='second',
- field=models.ForeignKey(help_text='Second Task Draft to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='second_to_connect', to='tmssapp.TaskDraft'),
- ),
- migrations.AddField(
- model_name='taskschedulingrelationblueprint',
- name='first',
- field=models.ForeignKey(help_text='First Task Blueprint to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='first_to_connect', to='tmssapp.TaskBlueprint'),
- ),
- migrations.AddField(
- model_name='taskschedulingrelationblueprint',
- name='placement',
- field=models.ForeignKey(default='after', help_text='Task scheduling relation placement.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingRelationPlacement'),
- ),
- migrations.AddField(
- model_name='taskschedulingrelationblueprint',
- name='second',
- field=models.ForeignKey(help_text='Second Task Blueprint to connect.', on_delete=django.db.models.deletion.CASCADE, related_name='second_to_connect', to='tmssapp.TaskBlueprint'),
- ),
migrations.AddConstraint(
model_name='taskrelationselectiontemplate',
- constraint=models.UniqueConstraint(fields=('name', 'version'), name='TaskRelationSelectionTemplate_unique_name_version'),
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='taskrelationselectiontemplate_unique_name_version'),
),
migrations.AddField(
model_name='taskrelationdraft',
@@ -854,7 +858,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='taskconnectortype',
name='input_of',
- field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='inpput_connector_types', to='tmssapp.TaskTemplate'),
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='input_connector_types', to='tmssapp.TaskTemplate'),
),
migrations.AddField(
model_name='taskconnectortype',
@@ -968,7 +972,12 @@ class Migration(migrations.Migration):
),
migrations.AddConstraint(
model_name='schedulingunittemplate',
- constraint=models.UniqueConstraint(fields=('name', 'version'), name='SchedulingUnitTemplate_unique_name_version'),
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='schedulingunittemplate_unique_name_version'),
+ ),
+ migrations.AddField(
+ model_name='schedulingunitobservingstrategytemplate',
+ name='scheduling_unit_template',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingUnitTemplate'),
),
migrations.AddField(
model_name='schedulingunitdraft',
@@ -980,6 +989,11 @@ class Migration(migrations.Migration):
name='copy_reason',
field=models.ForeignKey(help_text='Reason why source was copied (NULLable).', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.CopyReason'),
),
+ migrations.AddField(
+ model_name='schedulingunitdraft',
+ name='observation_strategy_template',
+ field=models.ForeignKey(help_text='Observation Strategy Template used to create the requirements_doc.', null=True, on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingUnitObservingStrategyTemplate'),
+ ),
migrations.AddField(
model_name='schedulingunitdraft',
name='requirements_template',
@@ -1047,7 +1061,7 @@ class Migration(migrations.Migration):
),
migrations.AddConstraint(
model_name='generatortemplate',
- constraint=models.UniqueConstraint(fields=('name', 'version'), name='GeneratorTemplate_unique_name_version'),
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='generatortemplate_unique_name_version'),
),
migrations.AddField(
model_name='filesystem',
@@ -1096,7 +1110,7 @@ class Migration(migrations.Migration):
),
migrations.AddConstraint(
model_name='dataproductspecificationstemplate',
- constraint=models.UniqueConstraint(fields=('name', 'version'), name='DataproductSpecificationsTemplate_unique_name_version'),
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='dataproductspecificationstemplate_unique_name_version'),
),
migrations.AddField(
model_name='dataproducthash',
@@ -1110,7 +1124,7 @@ class Migration(migrations.Migration):
),
migrations.AddConstraint(
model_name='dataproductfeedbacktemplate',
- constraint=models.UniqueConstraint(fields=('name', 'version'), name='DataproductFeedbackTemplate_unique_name_version'),
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='dataproductfeedbacktemplate_unique_name_version'),
),
migrations.AddField(
model_name='dataproductarchiveinfo',
@@ -1152,6 +1166,10 @@ class Migration(migrations.Migration):
name='station_type',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.StationType'),
),
+ migrations.AddConstraint(
+ model_name='tasktemplate',
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='tasktemplate_unique_name_version'),
+ ),
migrations.AddIndex(
model_name='taskschedulingrelationdraft',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_tas_tags_d1e21f_gin'),
@@ -1174,7 +1192,7 @@ class Migration(migrations.Migration):
),
migrations.AddConstraint(
model_name='subtasktemplate',
- constraint=models.UniqueConstraint(fields=('name', 'version'), name='SubtaskTemplate_unique_name_version'),
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='subtasktemplate_unique_name_version'),
),
migrations.AddIndex(
model_name='subtaskstatelog',
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
index 0868f0e846e9baed309f476e779da82336ddf0b6..4ac8634bab3ccf3a644f423d5fca7330ef387a2f 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/scheduling.py
@@ -106,8 +106,6 @@ class SubtaskTemplate(Template):
queue = BooleanField(default=False)
realtime = BooleanField(default=False)
- class Meta:
- pass
class DefaultSubtaskTemplate(BasicCommon):
name = CharField(max_length=128, unique=True)
@@ -115,8 +113,7 @@ class DefaultSubtaskTemplate(BasicCommon):
class DataproductSpecificationsTemplate(Template):
- class Meta:
- pass
+ pass
class DefaultDataproductSpecificationsTemplate(BasicCommon):
@@ -125,8 +122,7 @@ class DefaultDataproductSpecificationsTemplate(BasicCommon):
class DataproductFeedbackTemplate(Template):
- class Meta:
- pass
+ pass
# todo: do we need to specify a default?
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
index c73951f9f2275285fca57e0b297b03de9916e0c1..f292c06a9e03d7a0e9a3d9e44626715c30daa714 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
@@ -232,18 +232,31 @@ class Template(NamedCommon):
class GeneratorTemplate(Template):
create_function = CharField(max_length=128, help_text='Python function to call to execute the generator.')
- class Meta:
- pass
-
class DefaultGeneratorTemplate(BasicCommon):
name = CharField(max_length=128, unique=True)
template = ForeignKey("GeneratorTemplate", on_delete=PROTECT)
+class SchedulingUnitObservingStrategyTemplate(NamedCommon):
+ '''
+ A SchedulingUnitObservingStrategyTemplate is a template in the sense that it serves as a template to fill in json data objects conform its referred scheduling_unit_template.
+ It is however not derived from the (abstract) Template super-class, because the Template super class is for JSON schemas, not JSON data objects.
+ '''
+ version = CharField(max_length=128, help_text='Version of this template (with respect to other templates of the same name).')
+ template = JSONField(null=False, help_text='JSON-data compliant with the JSON-schema in the scheduling_unit_template. '
+ 'This observation strategy template like a predefined recipe with all the correct settings, and defines which parameters the user can alter.')
+ scheduling_unit_template = ForeignKey("SchedulingUnitTemplate", on_delete=PROTECT, null=False, help_text="")
+
+ def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
+ if self.template and self.scheduling_unit_template_id and self.scheduling_unit_template.schema:
+ validate_json_against_schema(self.template, self.scheduling_unit_template.schema)
+
+ super().save(force_insert, force_update, using, update_fields)
+
+
class SchedulingUnitTemplate(Template):
- class Meta:
- pass
+ pass
class DefaultSchedulingUnitTemplate(BasicCommon):
@@ -255,8 +268,6 @@ class TaskTemplate(Template):
validation_code_js = CharField(max_length=128, help_text='JavaScript code for additional (complex) validation.')
type = ForeignKey('TaskType', null=False, on_delete=PROTECT)
- class Meta:
- pass
class DefaultTaskTemplate(BasicCommon):
name = CharField(max_length=128, unique=True)
@@ -264,8 +275,7 @@ class DefaultTaskTemplate(BasicCommon):
class TaskRelationSelectionTemplate(Template):
- class Meta:
- pass
+ pass
class DefaultTaskRelationSelectionTemplate(BasicCommon):
name = CharField(max_length=128, unique=True)
@@ -369,10 +379,17 @@ class SchedulingUnitDraft(NamedCommon):
generator_instance_doc = JSONField(null=True, help_text='Parameter value that generated this run draft (NULLable).')
scheduling_set = ForeignKey('SchedulingSet', related_name='scheduling_unit_drafts', on_delete=CASCADE, help_text='Set to which this scheduling unit draft belongs.')
requirements_template = ForeignKey('SchedulingUnitTemplate', on_delete=CASCADE, help_text='Schema used for requirements_doc.')
+ observation_strategy_template = ForeignKey('SchedulingUnitObservingStrategyTemplate', on_delete=PROTECT, null=True, help_text='Observation Strategy Template used to create the requirements_doc.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- if self.requirements_doc and self.requirements_template_id and self.requirements_template.schema:
- validate_json_against_schema(self.requirements_doc, self.requirements_template.schema)
+ if self.requirements_doc:
+ if self.requirements_template_id and self.requirements_template.schema:
+ # If this scheduling unit was created from an observation_strategy_template,
+ # then make sure that the observation_strategy_template validates against this unit's requirements_template.schema
+ if self.observation_strategy_template_id and self.observation_strategy_template.template:
+ validate_json_against_schema(self.observation_strategy_template.template, self.requirements_template.schema)
+
+ validate_json_against_schema(self.requirements_doc, self.requirements_template.schema)
super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index dbb041b5e1f659fc9129bb7571858a3076b559ea..91152d8f2c3e4526eb439e17448cb2ce60ee4034 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -47,6 +47,8 @@ def populate_settings(apps, schema_editor):
def populate_lofar_json_schemas(apps, schema_editor):
_populate_scheduling_unit_schema()
+ _populate_scheduling_unit_observation_strategry_schema()
+
# populate task schema's
_populate_preprocessing_schema()
_populate_observation_with_stations_schema()
@@ -81,124 +83,34 @@ def populate_test_data():
for set_nr in range(3):
scheduling_set_data = SchedulingSet_test_data(name="Test Scheduling Set UC1 example %s" % (set_nr,), project=tmss_project)
scheduling_set = models.SchedulingSet.objects.create(**scheduling_set_data)
- scheduling_set.tags = ["TEST"]
+ scheduling_set.tags = ["TEST", "UC1"]
scheduling_set.save()
- for unit_nr in range(3):
- # construct a scheduling_unit_doc, i.e.: a specification of interrelated tasks which conforms the scheduling unit schema
- # by default, this scheduling_unit_doc holds no tasks, so lets setup the UC1 sequence of tasks here, and add it to the scheduling_unit_doc
- scheduling_unit_template = models.SchedulingUnitTemplate.objects.get(name="scheduling unit schema")
- scheduling_unit_doc = get_default_json_object_for_schema(scheduling_unit_template.schema)
-
- # create and add a calibrator task spec
- # Change autoselect to False (or provide tile_beam pointings for Target Observation) to avoid Exception
- json_schema_calibrator = get_default_json_object_for_schema(models.TaskTemplate.objects.get(name="calibrator schema").schema)
- json_schema_calibrator['autoselect'] = False
- scheduling_unit_doc['tasks'].append({"name": "Calibrator Observation 1",
- "description": "Calibrator Observation for UC1 HBA scheduling unit",
- "specifications_doc": json_schema_calibrator,
- "specifications_template": "calibrator schema"})
-
- # create and add a calibrator preprocessing spec
- scheduling_unit_doc['tasks'].append({"name": "Pipeline Calibrator1",
- "description": "Preprocessing Pipeline for Calibrator Observation 1",
- "specifications_doc": get_default_json_object_for_schema(models.TaskTemplate.objects.get(name="preprocessing schema").schema),
- "specifications_template": "preprocessing schema"})
-
- # create and add a target obs spec
- scheduling_unit_doc['tasks'].append({"name": "Target Observation",
- "description": "Target Observation for UC1 HBA scheduling unit",
- "specifications_doc": get_default_json_object_for_schema(models.TaskTemplate.objects.get(name="observation schema").schema),
- "specifications_template": "observation schema"})
-
- # create and add a target pipeline spec for sap0
- scheduling_unit_doc['tasks'].append({"name": "Preprocessing Pipeline SAP0",
- "description": "Preprocessing Pipeline for Target Observation SAP0",
- "specifications_doc": get_default_json_object_for_schema(models.TaskTemplate.objects.get(name="preprocessing schema").schema),
- "specifications_template": "preprocessing schema"})
-
- # create and add a target pipeline spec for sap1
- scheduling_unit_doc['tasks'].append({"name": "Preprocessing Pipeline SAP1",
- "description": "Preprocessing Pipeline for Target Observation SAP1",
- "specifications_doc": get_default_json_object_for_schema(models.TaskTemplate.objects.get(name="preprocessing schema").schema),
- "specifications_template": "preprocessing schema"})
-
- # create and add a calibrator task spec
- scheduling_unit_doc['tasks'].append({"name": "Calibrator Observation 2",
- "description": "Calibrator Observation for UC1 HBA scheduling unit",
- "specifications_doc": json_schema_calibrator,
- "specifications_template": "calibrator schema"})
-
- # create and add a calibrator preprocessing spec
- scheduling_unit_doc['tasks'].append({"name": "Pipeline Calibrator2",
- "description": "Preprocessing Pipeline for Calibrator Observation 2",
- "specifications_doc": get_default_json_object_for_schema(models.TaskTemplate.objects.get(name="preprocessing schema").schema),
- "specifications_template": "preprocessing schema"})
-
- # ----- end of tasks
-
- # setup task_scheduling_relations between Target and Calibrator observations
- scheduling_unit_doc['task_scheduling_relations'].append({"first": "Calibrator Observation 1",
- "second": "Target Observation",
- "placement": "before",
- "time_offset": 60 })
- scheduling_unit_doc['task_scheduling_relations'].append({"first": "Calibrator Observation 2",
- "second": "Target Observation",
- "placement": "after",
- "time_offset": 60 })
-
- # ----- end of task_scheduling_relations
-
- #TODO: check various input/output datatypes and roles for each task_relation
- scheduling_unit_doc['task_relations'].append({"producer": "Calibrator Observation 1",
- "consumer": "Pipeline Calibrator1",
- "tags": [],
- "input": { "role": "input", "datatype": "visibilities" },
- "output": { "role": "correlator", "datatype": "visibilities" },
- "dataformat": "MeasurementSet",
- "selection_doc": {},
- "selection_template": "All" })
-
- scheduling_unit_doc['task_relations'].append({"producer": "Calibrator Observation 2",
- "consumer": "Pipeline Calibrator2",
- "tags": [],
- "input": { "role": "input", "datatype": "visibilities" },
- "output": { "role": "correlator", "datatype": "visibilities" },
- "dataformat": "MeasurementSet",
- "selection_doc": {},
- "selection_template": "All" })
-
- scheduling_unit_doc['task_relations'].append({"producer": "Target Observation",
- "consumer": "Preprocessing Pipeline SAP0",
- "tags": [],
- "input": { "role": "input", "datatype": "visibilities" },
- "output": { "role": "correlator", "datatype": "visibilities" },
- "dataformat": "MeasurementSet",
- "selection_doc": {"sap": [0]},
- "selection_template": "SAP" })
-
- scheduling_unit_doc['task_relations'].append({"producer": "Target Observation",
- "consumer": "Preprocessing Pipeline SAP1",
- "tags": [],
- "input": { "role": "input", "datatype": "visibilities" },
- "output": { "role": "correlator", "datatype": "visibilities" },
- "dataformat": "MeasurementSet",
- "selection_doc": {"sap": [1]},
- "selection_template": "SAP" })
-
- # finally... add the scheduling_unit_doc to a new SchedulingUnitDraft instance, and were ready to use it!
- scheduling_unit_data = SchedulingUnitDraft_test_data(name="Test Scheduling Unit UC1 example %s.%s" % (set_nr, unit_nr), scheduling_set=scheduling_set,
- template=scheduling_unit_template, requirements_doc=scheduling_unit_doc)
- scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(**scheduling_unit_data)
+ logger.info('created test scheduling_set: %s', scheduling_set.name)
+
+ for unit_nr in range(2):
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+
+
+ # the 'template' in the strategy_template is a predefined json-data blob which validates against the given scheduling_unit_template
+ # a user might 'upload' a partial json-data blob, so add all the known defaults
+ scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template, strategy_template.scheduling_unit_template.schema)
+
+ # add the scheduling_unit_doc to a new SchedulingUnitDraft instance, and were ready to use it!
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(name="UC1 test scheduling unit %s.%s" % (set_nr+1, unit_nr+1),
+ scheduling_set=scheduling_set,
+ requirements_template=strategy_template.scheduling_unit_template,
+ requirements_doc=scheduling_unit_spec,
+ observation_strategy_template=strategy_template)
+ scheduling_unit_draft.tags = ["TEST", "UC1"]
+ scheduling_unit_draft.save()
+
+ logger.info('created test scheduling_unit_draft: %s', scheduling_unit_draft.name)
try:
- if set_nr==0 and unit_nr==0:
- create_task_blueprints_and_subtasks_and_schedule_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
- else:
- create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+ create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
except TMSSException as e:
- logger.error(e)
-
+ logger.exception(e)
except ImportError:
pass
@@ -266,11 +178,25 @@ def _populate_scheduling_unit_schema():
scheduling_unit_template_data = {"name": "scheduling unit schema",
"description": 'Schema for scheduling unit',
"version": '0.1',
- "tags": ["UC1"],
+ "tags": [],
"schema": json_data}
SchedulingUnitTemplate.objects.create(**scheduling_unit_template_data)
+def _populate_scheduling_unit_observation_strategry_schema():
+ with open(os.path.join(working_dir, "schemas/UC1-scheduling-unit-observation-strategy.json")) as json_file:
+ json_data = json.loads(json_file.read())
+ scheduling_unit_template = models.SchedulingUnitTemplate.objects.get(name="scheduling unit schema")
+
+ template_data = {"name": "UC1 observation strategy template",
+ "description": 'UC1 observation strategy template',
+ "scheduling_unit_template": scheduling_unit_template,
+ "version": '0.1',
+ "tags": ["UC1"],
+ "template": json_data}
+ SchedulingUnitObservingStrategyTemplate.objects.create(**template_data)
+
+
def _populate_observation_with_stations_schema():
with open(os.path.join(working_dir, "schemas/task-observation-with-stations.json")) as json_file:
json_data = json.loads(json_file.read())
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt b/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
index 4fb2a448999fb6ad5988477c7ac5de4c037fd5f9..f192559794af5108cca56446981e32d39eb070da 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/CMakeLists.txt
@@ -3,6 +3,7 @@ include(PythonInstall)
set(_json_schema_files
scheduling-unit.json
+ UC1-scheduling-unit-observation-strategy.json
task-calibrator-addon.json
task-observation-with-stations.json
task-stations.json
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json b/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
new file mode 100644
index 0000000000000000000000000000000000000000..760f43b19e2d240272508892b4248cf515187768
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
@@ -0,0 +1,299 @@
+{
+ "tasks": {
+ "Calibrator Observation 1": {
+ "description": "Calibrator Observation for UC1 HBA scheduling unit",
+ "tags": [],
+ "specifications_doc": {
+ "duration": 600,
+ "autoselect": false,
+ "pointing": {
+ "direction_type": "J2000",
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0
+ }
+ },
+ "specifications_template": "calibrator schema"
+ },
+ "Pipeline 1": {
+ "description": "Preprocessing Pipeline for Calibrator Observation 1",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Target Observation": {
+ "description": "Target Observation for UC1 HBA scheduling unit",
+ "tags": [],
+ "specifications_doc": {
+ "QA": {
+ "plots": {
+ "enabled": true,
+ "autocorrelation": true,
+ "crosscorrelation": true
+ },
+ "file_conversion": {
+ "enabled": true,
+ "nr_of_subbands": -1,
+ "nr_of_timestamps": 256
+ }
+ },
+ "duration": 28800,
+ "correlator": {
+ "storage_cluster": "CEP4",
+ "integration_time": 1,
+ "channels_per_subband": 64
+ },
+ "antenna_set": "HBA_DUAL_INNER",
+ "filter": "HBA_110_190",
+ "stations": [
+ {
+ "group": "ALL",
+ "min_stations": 1
+ }
+ ],
+ "tile_beam": {
+ "direction_type": "J2000",
+ "angle1": 42,
+ "angle2": 42,
+ "angle3": 42
+ },
+ "SAPs": [
+ {
+ "name": "target0",
+ "digital_pointing": {
+ "direction_type": "J2000",
+ "angle1": 24,
+ "angle2": 24,
+ "angle3": 24
+ },
+ "subbands": [
+ 349,
+ 372
+ ]
+ },
+ {
+ "name": "target1",
+ "digital_pointing": {
+ "direction_type": "J2000",
+ "angle1": 24,
+ "angle2": 24,
+ "angle3": 24
+ },
+ "subbands": [
+ 349,
+ 372
+ ]
+ }
+ ]
+ },
+ "specifications_template": "observation schema"
+ },
+ "Pipeline SAP0": {
+ "description": "Preprocessing Pipeline for Target Observation SAP0",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Pipeline SAP1": {
+ "description": "Preprocessing Pipeline for Target Observation SAP1",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ },
+ "Calibrator Observation 2": {
+ "description": "Calibrator Observation for UC1 HBA scheduling unit",
+ "tags": [],
+ "specifications_doc": {
+ "duration": 600,
+ "autoselect": false,
+ "pointing": {
+ "direction_type": "J2000",
+ "angle1": 0,
+ "angle2": 0,
+ "angle3": 0
+ }
+ },
+ "specifications_template": "calibrator schema"
+ },
+ "Pipeline 2": {
+ "description": "Preprocessing Pipeline for Calibrator Observation 2",
+ "tags": [],
+ "specifications_doc": {
+ "flag": {
+ "rfi_strategy": "auto",
+ "outerchannels": true,
+ "autocorrelations": true
+ },
+ "demix": {
+ "sources": {},
+ "time_steps": 10,
+ "ignore_target": false,
+ "frequency_steps": 64
+ },
+ "average": {
+ "time_steps": 1,
+ "frequency_steps": 4
+ },
+ "storagemanager": "dysco"
+ },
+ "specifications_template": "preprocessing schema"
+ }
+ },
+ "task_relations": [
+ {
+ "producer": "Calibrator Observation 1",
+ "consumer": "Pipeline 1",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "producer": "Calibrator Observation 2",
+ "consumer": "Pipeline 2",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {},
+ "selection_template": "All"
+ },
+ {
+ "producer": "Target Observation",
+ "consumer": "Pipeline SAP0",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {
+ "sap": [
+ 0
+ ]
+ },
+ "selection_template": "SAP"
+ },
+ {
+ "producer": "Target Observation",
+ "consumer": "Pipeline SAP1",
+ "tags": [],
+ "input": {
+ "role": "input",
+ "datatype": "visibilities"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities"
+ },
+ "dataformat": "MeasurementSet",
+ "selection_doc": {
+ "sap": [
+ 1
+ ]
+ },
+ "selection_template": "SAP"
+ }
+ ],
+ "task_scheduling_relations": [
+ {
+ "first": "Calibrator Observation 1",
+ "second": "Target Observation",
+ "placement": "before",
+ "time_offset": 60
+ },
+ {
+ "first": "Calibrator Observation 2",
+ "second": "Target Observation",
+ "placement": "after",
+ "time_offset": 60
+ }
+ ],
+ "parameters": [
+ {
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/SAPs/0/digital_pointing"
+ ],
+ "name": "Target Pointing 0"
+ },{
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/SAPs/1/digital_pointing"
+ ],
+ "name": "Target Pointing 1"
+ },{
+ "refs": [
+ "#/tasks/Target Observation/specifications_doc/tile_beam"
+ ],
+ "name": "Tile Beam"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json b/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json
index ba879a079db4ee21158f0aa6363bc14e41ea5f29..d792ba7893922198058d75ff403561fe684e4a5c 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/scheduling-unit.json
@@ -7,6 +7,7 @@
"task_connector": {
"type": "object",
"additionalProperties": false,
+ "default": {},
"properties": {
"role": {
"type": "string",
@@ -26,21 +27,15 @@
"properties": {
"tasks": {
"title": "Tasks",
- "type": "array",
- "additionalItems": false,
+ "type": "object",
"uniqueItems": true,
- "default": [],
- "items": {
+ "default": {},
+ "additionalProperties": {
"type": "object",
"title": "Task",
"additionalProperties": false,
"default": {},
"properties": {
- "name": {
- "type": "string",
- "title": "Name (unique)",
- "default": "Default Task"
- },
"description": {
"type": "string",
"title": "Description",
@@ -64,12 +59,11 @@
},
"specifications_template": {
"type": "string",
- "title": "Name of Template for Specifications",
+ "title": "URI of Template for Specifications",
"default": ""
}
},
"required": [
- "name",
"specifications_doc",
"specifications_template"
]
@@ -78,9 +72,9 @@
"task_relations": {
"title": "Task Relations",
"type": "array",
+ "default": [],
"additionalItems": false,
"uniqueItems": true,
- "default": [],
"items": {
"type": "object",
"title": "Task Relation",
@@ -126,7 +120,7 @@
},
"selection_template": {
"type": "string",
- "title": "Name of Template for Selection"
+ "title": "URI of Template for Selection"
}
},
"required": [
@@ -141,9 +135,9 @@
"task_scheduling_relations": {
"title": "Task Scheduling Relations",
"type": "array",
+ "default": [],
"additionalItems": false,
"uniqueItems": true,
- "default": [],
"items": {
"type": "object",
"title": "Task Scheduling Relation",
@@ -181,7 +175,47 @@
"placement"
]
}
- }
- },
+ },
+ "parameters": {
+ "title": "Parameters",
+ "description": "Schema for instance-specific parameters",
+ "type": "array",
+ "additionalItems": false,
+ "uniqueItems": true,
+ "items": {
+ "type": "object",
+ "title": "Parameter",
+ "additionalProperties": false,
+ "properties": {
+ "refs": {
+ "title": "References",
+ "description": "JSON Pointers to locations within this schema that will hold this value",
+ "type": "array",
+ "additionalItems": false,
+ "uniqueItems": true,
+ "minItems": 1,
+ "items": {
+ "type": "string",
+ "title": "Reference",
+ "default": "#",
+ "description": "JSON Pointer to parameter location within this schema"
+ }
+ },
+ "name": {
+ "type": "string",
+ "title": "Name",
+ "description": "Name override"
+ },
+ "description": {
+ "type": "string",
+ "title": "Description",
+ "description": "Description override"
+ }
+ },
+ "required": [
+ "refs"
+ ]
+ },
"required": []
-}
\ No newline at end of file
+ }}
+}
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
index 0e173fff9865a917e28757e03e1bd8cb0ecd6e52..0a3584ed2c7a82e0415ce201a60c4d2e58151fe0 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
@@ -100,6 +100,12 @@ class DefaultGeneratorTemplateSerializer(RelationalHyperlinkedModelSerializer):
fields = '__all__'
+class SchedulingUnitObservingStrategyTemplateSerializer(RelationalHyperlinkedModelSerializer):
+ class Meta:
+ model = models.SchedulingUnitObservingStrategyTemplate
+ fields = '__all__'
+
+
class SchedulingUnitTemplateSerializer(RelationalHyperlinkedModelSerializer):
class Meta:
model = models.SchedulingUnitTemplate
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index 110a3c609b5e3bfc99f8f9439b5b04c793ee4e1c..9ea2b60535959f328fb44aafbbea754f4ed8302b 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -32,6 +32,17 @@ def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint) -> [Subta
'''Generic create-method for subtasks. Calls the appropriate create method based on the task_blueprint specifications_template name.'''
check_prerequities_for_subtask_creation(task_blueprint)
+ subtasks = []
+
+ # recurse over predecessors, so that all dependencies in predecessor subtasks can be met.
+ for predecessor in task_blueprint.predecessors.all():
+ subtasks.extend(create_subtasks_from_task_blueprint(predecessor))
+
+ if task_blueprint.subtasks.count() > 0:
+ logger.debug("skipping creation of subtasks because they already exist for task_blueprint id=%s, name='%s', task_template_name='%s'",
+ task_blueprint.id, task_blueprint.name, task_blueprint.specifications_template.name)
+ return subtasks
+
# fixed mapping from template name to generator functions which create the list of subtask(s) for this task_blueprint
generators_mapping = {'observation schema': [create_observation_control_subtask_from_task_blueprint,
create_qafile_subtask_from_task_blueprint,
@@ -42,7 +53,6 @@ def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint) -> [Subta
template_name = task_blueprint.specifications_template.name
if template_name in generators_mapping:
generators = generators_mapping[template_name]
- subtasks = []
for generator in generators:
try:
subtask = generator(task_blueprint)
@@ -700,15 +710,16 @@ def schedule_observation_subtask(observation_subtask: Subtask):
directory = "/data/%s/%s/L%s/uv" % ("projects" if isProductionEnvironment() else "test-projects",
observation_subtask.task_blueprint.scheduling_unit_blueprint.draft.scheduling_set.project.name,
observation_subtask.id)
- for sb_nr in specifications_doc['stations']['digital_pointings'][0]['subbands']:
- Dataproduct.objects.create(filename="L%d_SB%03d_uv.MS" % (observation_subtask.id, sb_nr),
- directory=directory,
- dataformat=Dataformat.objects.get(value="MeasurementSet"),
- producer=subtask_output,
- specifications_doc={"sap": [0]}, # todo: set correct value. This will be provided by the RA somehow
- specifications_template=dataproduct_specifications_template,
- feedback_doc="",
- feedback_template=dataproduct_feedback_template)
+ for sap_nr, pointing in enumerate(specifications_doc['stations']['digital_pointings']):
+ for sb_nr in pointing['subbands']:
+ Dataproduct.objects.create(filename="L%d_SAP%03d_SB%03d_uv.MS" % (observation_subtask.id, sap_nr, sb_nr),
+ directory=directory,
+ dataformat=Dataformat.objects.get(value="MeasurementSet"),
+ producer=subtask_output,
+ specifications_doc={"sap": [sap_nr]}, # todo: set correct value. This will be provided by the RA somehow
+ specifications_template=dataproduct_specifications_template,
+ feedback_doc="",
+ feedback_template=dataproduct_feedback_template)
# step 4: resource assigner (if possible)
_assign_resources(observation_subtask)
@@ -812,10 +823,6 @@ def schedule_independent_subtasks_in_task_blueprint(task_blueprint: TaskBlueprin
'''Convenience method: Schedule the subtasks in the task_blueprint that are not dependend on predecessors'''
subtasks = list(task_blueprint.subtasks.all())
- # sort them in 'data-flow'-order,
- # because successors can depend on predecessors, so the first tbp's need to be subtask'd first.
- subtasks.sort(key=cmp_to_key(lambda st_a, st_b: -1 if st_a in st_b.predecessors else 1 if st_b in st_a.predecessors else 0))
-
for subtask in subtasks:
if len(subtask.predecessors.all()) == len(subtask.predecessors.filter(state__value='finished').all()):
schedule_subtask(subtask)
diff --git a/SAS/TMSS/src/tmss/tmssapp/tasks.py b/SAS/TMSS/src/tmss/tmssapp/tasks.py
index 67b821e6d26b061032862045684e3d216b3c623e..0bc760ad2318aab6228232365d78d75f2ef3f9d3 100644
--- a/SAS/TMSS/src/tmss/tmssapp/tasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/tasks.py
@@ -6,6 +6,7 @@ from lofar.sas.tmss.tmss.tmssapp.models.specification import TaskBlueprint, Sche
from lofar.sas.tmss.tmss.tmssapp.subtasks import create_and_schedule_subtasks_from_task_blueprint, \
create_subtasks_from_task_blueprint, schedule_independent_subtasks_in_task_blueprint
from functools import cmp_to_key
+from lofar.common.json_utils import add_defaults_to_json_object_for_schema
import logging
logger = logging.getLogger(__name__)
@@ -37,29 +38,32 @@ def create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft: models.
"""
logger.debug("create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft.id=%s, name='%s') ...", scheduling_unit_draft.pk, scheduling_unit_draft.name)
- if len(scheduling_unit_draft.requirements_doc.get("tasks",[])) == 0:
+ if len(scheduling_unit_draft.requirements_doc.get("tasks", {})) == 0:
raise BlueprintCreationException("create_task_drafts_from_scheduling_unit_draft: scheduling_unit_draft.id=%s has no tasks defined in its requirements_doc" % (scheduling_unit_draft.pk,))
- for task_definition in scheduling_unit_draft.requirements_doc["tasks"]:
+ for task_name, task_definition in scheduling_unit_draft.requirements_doc["tasks"].items():
task_template_name = task_definition["specifications_template"]
task_template = models.TaskTemplate.objects.get(name=task_template_name)
- if scheduling_unit_draft.task_drafts.filter(name=task_definition["name"], specifications_template=task_template).count() > 0:
- logger.debug("skipping creation of task draft because it is already in the scheduling_unit... task_name='%s', task_template_name='%s'", task_definition["name"], task_template_name)
+ task_specifications_doc = task_definition["specifications_doc"]
+ task_specifications_doc = add_defaults_to_json_object_for_schema(task_specifications_doc, task_template.schema)
+
+ if scheduling_unit_draft.task_drafts.filter(name=task_name, specifications_template=task_template).count() > 0:
+ logger.debug("skipping creation of task draft because it is already in the scheduling_unit... task_name='%s', task_template_name='%s'", task_name, task_template_name)
continue
- logger.debug("creating task draft... task_name='%s', task_template_name='%s'", task_definition["name"], task_template_name)
+ logger.debug("creating task draft... task_name='%s', task_template_name='%s'", task_template_name, task_template_name)
- task_draft = models.TaskDraft.objects.create(name=task_definition["name"],
+ task_draft = models.TaskDraft.objects.create(name=task_name,
description=task_definition.get("description",""),
tags=task_definition.get("tags",[]),
- specifications_doc=task_definition["specifications_doc"],
+ specifications_doc=task_specifications_doc,
copy_reason=models.CopyReason.objects.get(value='template'),
copies=None,
scheduling_unit_draft=scheduling_unit_draft,
specifications_template=task_template)
- logger.info("created task draft id=%s task_name='%s', task_template_name='%s'", task_draft.pk, task_definition["name"], task_template_name)
+ logger.info("created task draft id=%s task_name='%s', task_template_name='%s'", task_draft.pk, task_name, task_template_name)
# Now create task relations
for task_relation_definition in scheduling_unit_draft.requirements_doc["task_relations"]:
@@ -77,7 +81,7 @@ def create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft: models.
output_role=output_role,
selection_template=selection_template,
selection_doc=task_relation_definition["selection_doc"]).count() > 0:
- logger.debug("skipping creation of task_relation between task draft '%s' and '%s' because it is already in the scheduling_unit...", task_relation_definition["producer"], task_relation_definition["consumer"])
+ logger.info("skipping creation of task_relation between task draft '%s' and '%s' because it is already in the scheduling_unit...", task_relation_definition["producer"], task_relation_definition["consumer"])
continue
task_relation = models.TaskRelationDraft.objects.create(tags=task_relation_definition.get("tags",[]),
@@ -103,7 +107,7 @@ def create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft: models.
time_offset=time_offset,
first=first_task_draft,
second=second_task_draft).count() > 0:
- logger.debug("skipping creation of task_scheduling_relation between task draft '%s' and '%s' because it is already in the scheduling_unit...",
+ logger.info("skipping creation of task_scheduling_relation between task draft '%s' and '%s' because it is already in the scheduling_unit...",
task_scheduling_relation_definition["first"], task_scheduling_relation_definition["second"])
continue
@@ -251,13 +255,7 @@ def create_task_blueprints_and_subtasks_from_scheduling_unit_blueprint(schedulin
'''Convenience method: Create the scheduling_unit_blueprint's task_blueprint(s), then create each task_blueprint's subtasks'''
scheduling_unit_blueprint = create_task_blueprints_from_scheduling_unit_blueprint(scheduling_unit_blueprint)
- task_blueprints = list(scheduling_unit_blueprint.task_blueprints.all())
-
- # sort task_blueprint(s) in 'data-flow'-order,
- # because successors can depend on predecessors, so the first tbp's need to be subtask'd first.
- task_blueprints.sort(key=cmp_to_key(lambda tbp_a, tbp_b: -1 if tbp_a in tbp_b.predecessors else 1 if tbp_b in tbp_a.predecessors else 0))
-
- for task_blueprint in task_blueprints:
+ for task_blueprint in scheduling_unit_blueprint.task_blueprints.all():
create_subtasks_from_task_blueprint(task_blueprint)
# refresh so all related fields are updated.
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index 4404f40d6265ef41461d8c6db5f2ee114c0e2f03..f7b2aeeafdd2d57ac65b7da5aa5d1df3e9b3fc2b 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -15,6 +15,7 @@ from rest_framework.permissions import IsAuthenticatedOrReadOnly, DjangoModelPer
from rest_framework.decorators import action
from drf_yasg.utils import swagger_auto_schema
+from drf_yasg.openapi import Parameter
from lofar.sas.tmss.tmss.tmssapp.viewsets.lofar_viewset import LOFARViewSet, LOFARNestedViewSet
from lofar.sas.tmss.tmss.tmssapp import models
@@ -50,6 +51,45 @@ class DefaultGeneratorTemplateViewSet(LOFARViewSet):
queryset = models.DefaultGeneratorTemplate.objects.all()
serializer_class = serializers.DefaultGeneratorTemplateSerializer
+
+class SchedulingUnitObservingStrategyTemplateViewSet(LOFARViewSet):
+ queryset = models.SchedulingUnitObservingStrategyTemplate.objects.all()
+ serializer_class = serializers.SchedulingUnitObservingStrategyTemplateSerializer
+
+ @swagger_auto_schema(responses={status.HTTP_201_CREATED: 'The newly created scheduling unit',
+ status.HTTP_403_FORBIDDEN: 'forbidden'},
+ operation_description="Create a new SchedulingUnit based on this SchedulingUnitObservingStrategyTemplate, with the given <name> and <description> and make it a child of the given <scheduling_set_id>",
+ manual_parameters=[Parameter(name='scheduling_set_id', required=True, type='integer', in_='query',
+ description="the id of the scheduling_set which will be the parent of the newly created scheduling_unit"),
+ Parameter(name='name', required=False, type='string', in_='query',
+ description="The name for the newly created scheduling_unit"),
+ Parameter(name='description', required=False, type='string', in_='query',
+ description="The description for the newly created scheduling_unit")])
+ @action(methods=['get'], detail=True)
+ def create_scheduling_unit(self, request, pk=None):
+ strategy_template = get_object_or_404(models.SchedulingUnitObservingStrategyTemplate, pk=pk)
+ spec = add_defaults_to_json_object_for_schema(strategy_template.template,
+ strategy_template.scheduling_unit_template.schema)
+
+ scheduling_set = get_object_or_404(models.SchedulingSet, pk=request.query_params['scheduling_set_id'])
+
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(name=request.query_params.get('name', "scheduling unit"),
+ description=request.query_params.get('description', ""),
+ requirements_doc=spec,
+ scheduling_set=scheduling_set,
+ requirements_template=strategy_template.scheduling_unit_template,
+ observation_strategy_template=strategy_template)
+
+ scheduling_unit_observation_strategy_template_path = request._request.path
+ base_path = scheduling_unit_observation_strategy_template_path[:scheduling_unit_observation_strategy_template_path.find('/scheduling_unit_observing_strategy_template')]
+ scheduling_unit_draft_path = '%s/scheduling_unit_draft/%s/' % (base_path, scheduling_unit_draft.id,)
+
+ # return a response with the new serialized SchedulingUnitDraft, and a Location to the new instance in the header
+ return Response(serializers.SchedulingUnitDraftSerializer(scheduling_unit_draft, context={'request':request}).data,
+ status=status.HTTP_201_CREATED,
+ headers={'Location': scheduling_unit_draft_path})
+
+
class SchedulingUnitTemplateFilter(filters.FilterSet):
class Meta:
model = models.SchedulingUnitTemplate
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index 5d831d9cf5d96fef8b1733a29786192f3ce82c42..53146045e08986f1cb8930e993b04129df909610 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -101,6 +101,7 @@ router.register(r'task_type', viewsets.TaskTypeViewSet)
# templates
router.register(r'generator_template', viewsets.GeneratorTemplateViewSet)
+router.register(r'scheduling_unit_observing_strategy_template', viewsets.SchedulingUnitObservingStrategyTemplateViewSet)
router.register(r'scheduling_unit_template', viewsets.SchedulingUnitTemplateViewSet)
router.register(r'task_template', viewsets.TaskTemplateViewSet)
router.register(r'task_relation_selection_template', viewsets.TaskRelationSelectionTemplateViewSet)
diff --git a/SAS/TMSS/test/t_scheduling.py b/SAS/TMSS/test/t_scheduling.py
index f4de89666fd4bb05b82009ccc46d11fd578cc769..1eee84c252de5e3a2a1a10cbabf19b56c4501d93 100755
--- a/SAS/TMSS/test/t_scheduling.py
+++ b/SAS/TMSS/test/t_scheduling.py
@@ -91,7 +91,8 @@ class SchedulingTest(unittest.TestCase):
subtask_data = test_data_creator.Subtask(specifications_template_url=subtask_template['url'],
specifications_doc=spec,
- cluster_url=cluster_url)
+ cluster_url=cluster_url,
+ task_blueprint_url=test_data_creator.post_data_and_get_url(test_data_creator.TaskBlueprint(), '/task_blueprint/'))
subtask = test_data_creator.post_data_and_get_response_as_json_object(subtask_data, '/subtask/')
subtask_id = subtask['id']
test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=subtask['url']), '/subtask_output/')
@@ -128,7 +129,8 @@ class SchedulingTest(unittest.TestCase):
subtask_data = test_data_creator.Subtask(specifications_template_url=subtask_template['url'],
specifications_doc=spec,
- cluster_url=cluster_url)
+ cluster_url=cluster_url,
+ task_blueprint_url=test_data_creator.post_data_and_get_url(test_data_creator.TaskBlueprint(), '/task_blueprint/'))
subtask = test_data_creator.post_data_and_get_response_as_json_object(subtask_data, '/subtask/')
subtask_id = subtask['id']
test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=subtask['url']), '/subtask_output/')
@@ -153,7 +155,8 @@ class SchedulingTest(unittest.TestCase):
obs_subtask_data = test_data_creator.Subtask(specifications_template_url=obs_subtask_template['url'],
specifications_doc=obs_spec,
- cluster_url=cluster_url)
+ cluster_url=cluster_url,
+ task_blueprint_url=test_data_creator.post_data_and_get_url(test_data_creator.TaskBlueprint(), '/task_blueprint/'))
obs_subtask = test_data_creator.post_data_and_get_response_as_json_object(obs_subtask_data, '/subtask/')
obs_subtask_output_url = test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=obs_subtask['url']), '/subtask_output/')
test_data_creator.post_data_and_get_url(test_data_creator.Dataproduct(filename="L%s_SB000.MS"%obs_subtask['id'],
@@ -194,14 +197,12 @@ class SchedulingTest(unittest.TestCase):
obs_task['QA']['plots']['enabled'] = False
obs_task['QA']['file_conversion']['enabled'] = False
obs_task['SAPs'][0]['subbands'] = [0,1]
- scheduling_unit_doc['tasks'].append({"name": "Observation",
- "specifications_doc": obs_task,
- "specifications_template": "observation schema"})
+ scheduling_unit_doc['tasks']["Observation"] = {"specifications_doc": obs_task,
+ "specifications_template": "observation schema"}
# define a pipeline
- scheduling_unit_doc['tasks'].append({"name": "Pipeline",
- "specifications_doc": get_default_json_object_for_schema(client.get_task_template(name="preprocessing schema")['schema']),
- "specifications_template": "preprocessing schema"})
+ scheduling_unit_doc['tasks']["Pipeline"] = { "specifications_doc": get_default_json_object_for_schema(client.get_task_template(name="preprocessing schema")['schema']),
+ "specifications_template": "preprocessing schema"}
# connect obs to pipeline
scheduling_unit_doc['task_relations'].append({"producer": "Observation",
diff --git a/SAS/TMSS/test/t_tasks.py b/SAS/TMSS/test/t_tasks.py
index d9f6c1b2a79eb78f03173fa38006b2c197bfde26..cc51eec0313d0ec53004e36e802bfbc8cb07495c 100755
--- a/SAS/TMSS/test/t_tasks.py
+++ b/SAS/TMSS/test/t_tasks.py
@@ -65,24 +65,24 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
6. create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft) -> [TaskDraft]:
3. create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft) -> models.SchedulingUnitBlueprint:
"""
- @staticmethod
- def create_scheduling_unit_draft_object(scheduling_unit_draft_name, requirements_doc=None):
- """
- Helper function to create a scheduling unit object for testing
- """
- scheduling_unit_draft_data = SchedulingUnitDraft_test_data(name=scheduling_unit_draft_name,
- requirements_doc=requirements_doc,
- template=models.SchedulingUnitTemplate.objects.get(name="scheduling unit schema"))
- draft_obj = models.SchedulingUnitDraft.objects.create(**scheduling_unit_draft_data)
- return draft_obj
-
def test_create_scheduling_unit_blueprint_from_scheduling_unit_draft(self):
"""
Create Scheduling Unit Draft
Check if the name draft (specified) is equal to name blueprint (created)
Check with REST-call if NO tasks are created
"""
- scheduling_unit_draft = self.create_scheduling_unit_draft_object("Test Scheduling Unit 1")
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template.template['tasks'] = {}
+
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
+ name="Test Scheduling Unit UC1",
+ requirements_doc=strategy_template.template,
+ requirements_template=strategy_template.scheduling_unit_template,
+ observation_strategy_template=strategy_template,
+ copy_reason=models.CopyReason.objects.get(value='template'),
+ generator_instance_doc="para",
+ copies=None,
+ scheduling_set=models.SchedulingSet.objects.create(**SchedulingSet_test_data()))
scheduling_unit_blueprint = create_scheduling_unit_blueprint_from_scheduling_unit_draft(scheduling_unit_draft)
self.assertEqual(scheduling_unit_draft.name, scheduling_unit_blueprint.draft.name)
@@ -94,7 +94,19 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Check if NO tasks are created
Check with REST-call if NO tasks are created
"""
- scheduling_unit_draft = self.create_scheduling_unit_draft_object("Test Scheduling Unit 2", requirements_doc={'tasks': []})
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template.template['tasks'] = {}
+
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
+ name="Test Scheduling Unit UC1",
+ requirements_doc=strategy_template.template,
+ requirements_template=strategy_template.scheduling_unit_template,
+ observation_strategy_template=strategy_template,
+ copy_reason=models.CopyReason.objects.get(value='template'),
+ generator_instance_doc="para",
+ copies=None,
+ scheduling_set=models.SchedulingSet.objects.create(**SchedulingSet_test_data()))
+
with self.assertRaises(BlueprintCreationException):
create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft)
@@ -109,14 +121,13 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Create Task Blueprints (only)
Check if tasks (7) are created
"""
- working_dir = os.path.dirname(os.path.abspath(__file__))
- with open(os.path.join(working_dir, "testdata/example_UC1_scheduling_unit.json")) as json_file:
- json_requirements_doc = json.loads(json_file.read())
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
name="Test Scheduling Unit UC1",
- requirements_doc=json_requirements_doc,
- requirements_template=models.SchedulingUnitTemplate.objects.get(name="scheduling unit schema"),
+ requirements_doc=strategy_template.template,
+ requirements_template=strategy_template.scheduling_unit_template,
+ observation_strategy_template=strategy_template,
copy_reason=models.CopyReason.objects.get(value='template'),
generator_instance_doc="para",
copies=None,
@@ -141,14 +152,13 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Every Pipeline Task: 1 subtasks (1 control)
makes 3x3 + 4x1 = 13
"""
- working_dir = os.path.dirname(os.path.abspath(__file__))
- with open(os.path.join(working_dir, "testdata/example_UC1_scheduling_unit.json")) as json_file:
- json_requirements_doc = json.loads(json_file.read())
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
name="Test Scheduling Unit UC1",
- requirements_doc=json_requirements_doc,
- requirements_template=models.SchedulingUnitTemplate.objects.get(name="scheduling unit schema"),
+ requirements_doc=strategy_template.template,
+ requirements_template=strategy_template.scheduling_unit_template,
+ observation_strategy_template=strategy_template,
copy_reason=models.CopyReason.objects.get(value='template'),
generator_instance_doc="para",
copies=None,
@@ -177,7 +187,18 @@ class CreationFromSchedulingUnitDraft(unittest.TestCase):
Check if the name draft (specified) is equal to name blueprint (created)
Check with REST-call if NO tasks are created
"""
- scheduling_unit_draft = self.create_scheduling_unit_draft_object("Test Scheduling Unit 3", {'tasks': []})
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 observation strategy template")
+ strategy_template.template['tasks'] = {}
+
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
+ name="Test Scheduling Unit UC1",
+ requirements_doc=strategy_template.template,
+ requirements_template=strategy_template.scheduling_unit_template,
+ observation_strategy_template=strategy_template,
+ copy_reason=models.CopyReason.objects.get(value='template'),
+ generator_instance_doc="para",
+ copies=None,
+ scheduling_set=models.SchedulingSet.objects.create(**SchedulingSet_test_data()))
with self.assertRaises(BlueprintCreationException):
create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
diff --git a/SAS/TMSS/test/t_tmssapp_specification_REST_API.py b/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
index 597ce1c04c433979c5e7b0ff0ada73720fb26686..6d922605dbb7a553227bd142e508d731bf620b47 100755
--- a/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
+++ b/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
@@ -1140,7 +1140,7 @@ class SchedulingUnitDraftTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, url, schedulingunitdraft_test_data)
test_patch = {"description": "This is a new and improved description",
- "requirements_doc": '{"para": "meter"}'}
+ "requirements_doc": '{"foo": "barbar"}'}
# PATCH item and verify
PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
diff --git a/SAS/TMSS/test/tmss_test_data_django_models.py b/SAS/TMSS/test/tmss_test_data_django_models.py
index 2e3c669605e640d380b5a82c77fcaaadde8456bf..dd093be160512794fd2c8a7025d4e8f6d0e2b5cf 100644
--- a/SAS/TMSS/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/test/tmss_test_data_django_models.py
@@ -50,14 +50,41 @@ def DefaultGeneratorTemplate_test_data(name=None, template=None) -> dict:
'template': template,
'tags':[]}
-def SchedulingUnitTemplate_test_data(name="my_SchedulingUnitTemplate", version:str=None) -> dict:
+def SchedulingUnitTemplate_test_data(name="my_SchedulingUnitTemplate", version:str=None, schema:dict=None) -> dict:
if version is None:
version = str(uuid.uuid4())
+ if schema is None:
+ schema = { "$schema": "https://json-schema.org/draft/2019-09/schema",
+ "type": "object",
+ "properties": { "foo" : { "type": "string", "default": "bar" } },
+ "required": ["foo"],
+ "default": {}
+ }
+
return {"name": name,
"description": 'My SchedulingUnitTemplate description',
"version": version,
- "schema": {"mykey": "my value"},
+ "schema": schema,
+ "tags": ["TMSS", "TESTING"]}
+
+def SchedulingUnitObservingStrategyTemplate_test_data(name="my_SchedulingUnitObservingStrategyTemplate", version:str=None,
+ scheduling_unit_template:models.SchedulingUnitTemplate=None,
+ template:dict=None) -> dict:
+ if version is None:
+ version = str(uuid.uuid4())
+
+ if scheduling_unit_template is None:
+ scheduling_unit_template = models.SchedulingUnitTemplate.objects.create(**SchedulingUnitTemplate_test_data())
+
+ if template is None:
+ template = get_default_json_object_for_schema(scheduling_unit_template.schema)
+
+ return {"name": name,
+ "description": 'My SchedulingUnitTemplate description',
+ "version": version,
+ "template": template,
+ "scheduling_unit_template": scheduling_unit_template,
"tags": ["TMSS", "TESTING"]}
def TaskTemplate_test_data(name="my TaskTemplate", version:str=None) -> dict:
@@ -135,7 +162,9 @@ def SchedulingSet_test_data(name="my_scheduling_set", project: models.Project=No
"generator_template": models.GeneratorTemplate.objects.create(**GeneratorTemplate_test_data()),
"generator_source": None}
-def SchedulingUnitDraft_test_data(name="my_scheduling_unit_draft", scheduling_set: models.SchedulingSet=None, template: models.SchedulingUnitTemplate=None, requirements_doc: dict=None) -> dict:
+def SchedulingUnitDraft_test_data(name="my_scheduling_unit_draft", scheduling_set: models.SchedulingSet=None,
+ template: models.SchedulingUnitTemplate=None, requirements_doc: dict=None,
+ observation_strategy_template: models.SchedulingUnitObservingStrategyTemplate=None) -> dict:
if scheduling_set is None:
scheduling_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data())
@@ -145,6 +174,9 @@ def SchedulingUnitDraft_test_data(name="my_scheduling_unit_draft", scheduling_se
if requirements_doc is None:
requirements_doc = get_default_json_object_for_schema(template.schema)
+ if observation_strategy_template is None:
+ observation_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.create(**SchedulingUnitObservingStrategyTemplate_test_data())
+
return {"name": name,
"description": "",
"tags": [],
@@ -153,7 +185,8 @@ def SchedulingUnitDraft_test_data(name="my_scheduling_unit_draft", scheduling_se
"generator_instance_doc": "para",
"copies": None,
"scheduling_set": scheduling_set,
- "requirements_template": template }
+ "requirements_template": template,
+ "observation_strategy_template": observation_strategy_template }
def TaskDraft_test_data(name: str="my_task_draft", specifications_template: models.TaskTemplate=None, specifications_doc: dict=None, scheduling_unit_draft: models.SchedulingUnitDraft=None) -> dict:
if specifications_template is None:
diff --git a/SAS/TMSS/test/tmss_test_data_rest.py b/SAS/TMSS/test/tmss_test_data_rest.py
index 64bf43f8744bd28b8add35d568597d95a53c42ba..d919fbbcc46cddd25b80ccc6e091b43802775c64 100644
--- a/SAS/TMSS/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/test/tmss_test_data_rest.py
@@ -26,6 +26,7 @@ import uuid
import requests
import json
from lofar.common.json_utils import get_default_json_object_for_schema
+from http import HTTPStatus
class TMSSRESTTestDataCreator():
def __init__(self, django_api_url: str, auth: requests.auth.HTTPBasicAuth):
@@ -43,7 +44,10 @@ class TMSSRESTTestDataCreator():
def post_data_and_get_response_as_json_object(self, data, url_postfix):
"""POST the given data the self.django_api_url+url_postfix, and return the response"""
- return json.loads(self.post_data_and_get_response(data, url_postfix).content.decode('utf-8'))
+ response = self.post_data_and_get_response(data, url_postfix)
+ if response.status_code == HTTPStatus.CREATED:
+ return json.loads(response.content.decode('utf-8'))
+ raise Exception("Error during POST request of '%s' status=%s content: %s" % (url_postfix, response.status_code, response.content.decode('utf-8')))
def post_data_and_get_url(self, data, url_postfix):
"""POST the given data the self.django_api_url+url_postfix, and return the response's url"""
@@ -72,17 +76,45 @@ class TMSSRESTTestDataCreator():
"create_function": 'Funky',
"tags": ["TMSS", "TESTING"]}
- def SchedulingUnitTemplate(self, name="schedulingunittemplate1", version:str=None) -> dict:
+ def SchedulingUnitTemplate(self, name="schedulingunittemplate1", version:str=None, schema:dict=None) -> dict:
if version is None:
version = str(uuid.uuid4())
+ if schema is None:
+ schema = {"$schema": "https://json-schema.org/draft/2019-09/schema",
+ "type": "object",
+ "properties": {"foo": {"type": "string", "default": "bar"}},
+ "required": ["foo"],
+ "default": {}
+ }
+
return { "name": name,
"description": 'My description',
"version": version,
- "schema": {"mykey": "my value"},
+ "schema": schema,
"tags": ["TMSS", "TESTING"]}
-
- def TaskTemplate(self, name="tasktemplate1", task_type_url:str=None, version:str=None) -> dict:
+
+ def SchedulingUnitObservingStrategyTemplate(self, name="my_SchedulingUnitObservingStrategyTemplate", version:str=None,
+ scheduling_unit_template_url=None,
+ template:dict=None) -> dict:
+ if version is None:
+ version = str(uuid.uuid4())
+
+ if scheduling_unit_template_url is None:
+ scheduling_unit_template_url = self.post_data_and_get_url(self.SchedulingUnitTemplate(), '/scheduling_unit_template/')
+
+ if template is None:
+ scheduling_unit_template = self.get_response_as_json_object(scheduling_unit_template_url)
+ template = get_default_json_object_for_schema(scheduling_unit_template['schema'])
+
+ return {"name": name,
+ "description": 'My SchedulingUnitTemplate description',
+ "version": version,
+ "template": template,
+ "scheduling_unit_template": scheduling_unit_template_url,
+ "tags": ["TMSS", "TESTING"]}
+
+ def TaskTemplate(self, name="tasktemplate1", task_type_url: str = None, version: str = None) -> dict:
if version is None:
version = str(uuid.uuid4())
@@ -185,7 +217,7 @@ class TMSSRESTTestDataCreator():
"generator_source": None,
"scheduling_unit_drafts": []}
- def SchedulingUnitDraft(self, name="my_scheduling_unit_draft", scheduling_set_url=None, template_url=None, requirements_doc=None):
+ def SchedulingUnitDraft(self, name="my_scheduling_unit_draft", scheduling_set_url=None, template_url=None, requirements_doc=None, observation_strategy_template_url=None):
if scheduling_set_url is None:
scheduling_set_url = self.post_data_and_get_url(self.SchedulingSet(), '/scheduling_set/')
@@ -196,6 +228,9 @@ class TMSSRESTTestDataCreator():
scheduling_unit_template = self.get_response_as_json_object(template_url)
requirements_doc = get_default_json_object_for_schema(scheduling_unit_template['schema'])
+ # if observation_strategy_template_url is None:
+ # observation_strategy_template_url = self.post_data_and_get_url(self.SchedulingUnitObservingStrategyTemplate(scheduling_unit_template_url=template_url), '/scheduling_unit_observing_strategy_template/')
+
return {"name": name,
"description": "This is my run draft",
"tags": [],
@@ -205,6 +240,7 @@ class TMSSRESTTestDataCreator():
"copies": None,
"scheduling_set": scheduling_set_url,
"requirements_template": template_url,
+ "observation_strategy_template": observation_strategy_template_url,
"scheduling_unit_blueprints": [],
"task_drafts": []}
@@ -256,17 +292,21 @@ class TMSSRESTTestDataCreator():
"selection_template": template_url,
'related_task_relation_blueprint': []}
- def SchedulingUnitBlueprint(self, name="my_scheduling_unit_blueprint", scheduling_unit_draft_url=None, template_url=None):
- if scheduling_unit_draft_url is None:
- scheduling_unit_draft_url = self.post_data_and_get_url(self.SchedulingUnitDraft(), '/scheduling_unit_draft/')
-
+ def SchedulingUnitBlueprint(self, name="my_scheduling_unit_blueprint", scheduling_unit_draft_url=None, template_url=None, requirements_doc:dict=None):
if template_url is None:
template_url = self.post_data_and_get_url(self.SchedulingUnitTemplate(), '/scheduling_unit_template/')
-
+
+ if scheduling_unit_draft_url is None:
+ scheduling_unit_draft_url = self.post_data_and_get_url(self.SchedulingUnitDraft(template_url=template_url), '/scheduling_unit_draft/')
+
+ if requirements_doc is None:
+ scheduling_unit_template = self.get_response_as_json_object(template_url)
+ requirements_doc = get_default_json_object_for_schema(scheduling_unit_template['schema'])
+
return {"name": name,
"description": "This is my run blueprint",
"tags": [],
- "requirements_doc": "{}",
+ "requirements_doc": requirements_doc,
"do_cancel": False,
"draft": scheduling_unit_draft_url,
"requirements_template": template_url,
@@ -410,9 +450,9 @@ class TMSSRESTTestDataCreator():
if cluster_url is None:
cluster_url = self.post_data_and_get_url(self.Cluster(), '/cluster/')
- if task_blueprint_url is None:
- task_blueprint = self.TaskBlueprint()
- task_blueprint_url = self.post_data_and_get_url(task_blueprint, '/task_blueprint/')
+ # if task_blueprint_url is None:
+ # task_blueprint = self.TaskBlueprint()
+ # task_blueprint_url = self.post_data_and_get_url(task_blueprint, '/task_blueprint/')
if specifications_template_url is None:
specifications_template_url = self.post_data_and_get_url(self.SubtaskTemplate(), '/subtask_template/')