diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py b/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py
index db6ba8d353b316f043d8b5bf7f64a77ef5fb4d94..810d42c38dfd98669d4d558b574559c19d232513 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py
@@ -1,3 +1,4 @@
+import copy
from lofar.sas.tmss.tmss.exceptions import *
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.subtasks import unschedule_subtasks_in_task_blueprint, cancel_subtask, mark_independent_subtasks_in_task_blueprint_as_unschedulable, mark_subtasks_in_task_blueprint_as_schedulable, set_task_blueprint_start_times, restart_on_hold_subtask, convert_task_station_groups_specification_to_station_list_without_used_and_or_reserved_stations
@@ -147,7 +148,7 @@ def create_scheduling_unit_blueprint_from_scheduling_unit_draft(scheduling_unit_
return scheduling_unit_blueprint
-def copy_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft, scheduling_set_dst: models.SchedulingSet) -> models.SchedulingUnitDraft:
+def copy_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft, scheduling_set_dst: models.SchedulingSet, remove_lofar1_stations=False, remove_lofar2_stations=False) -> models.SchedulingUnitDraft:
"""
Copy a SchedulingUnitDraft
:raises Exception if instantiate fails.
@@ -172,11 +173,11 @@ def copy_scheduling_unit_draft(scheduling_unit_draft: models.SchedulingUnitDraft
scheduling_unit_draft.pk, scheduling_unit_draft_copy.pk)
# instantiate a copy of the task graph
- scheduling_unit_draft_copy = update_task_graph_from_specifications_doc(scheduling_unit_draft_copy, scheduling_unit_draft.specifications_doc)
+ scheduling_unit_draft_copy = update_task_graph_from_specifications_doc(scheduling_unit_draft_copy, scheduling_unit_draft.specifications_doc, remove_lofar1_stations, remove_lofar2_stations)
return scheduling_unit_draft_copy
-def create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint: models.SchedulingUnitBlueprint) -> models.SchedulingUnitDraft:
+def create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint: models.SchedulingUnitBlueprint, remove_lofar1_stations=False, remove_lofar2_stations=False) -> models.SchedulingUnitDraft:
"""
Create a SchedulingUnitDraft from the SchedulingUnitBlueprint
:raises Exception if instantiate fails.
@@ -203,19 +204,22 @@ def create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_
scheduling_unit_blueprint.pk, scheduling_unit_draft_copy.pk)
# instantiate a copy of the tasks graph
- scheduling_unit_draft_copy = update_task_graph_from_specifications_doc(scheduling_unit_draft_copy, scheduling_unit_blueprint.specifications_doc)
+ scheduling_unit_draft_copy = update_task_graph_from_specifications_doc(scheduling_unit_draft_copy, scheduling_unit_blueprint.specifications_doc, remove_lofar1_stations, remove_lofar2_stations)
return scheduling_unit_draft_copy
-def copy_task_draft(task_draft: models.TaskDraft) -> models.TaskDraft:
+def copy_task_draft(task_draft: models.TaskDraft, remove_lofar1_stations=False, remove_lofar2_stations=False) -> models.TaskDraft:
'''create a copy of the given task. Also copy all the task_relation(s) and/or the task_scheduling_relation(s)'''
logger.debug("copy_task_draft(task_draft.id=%s) copying...", task_draft.pk)
with transaction.atomic():
+ specifications_doc = copy.deepcopy(task_draft.specifications_doc)
+ if remove_lofar1_stations or remove_lofar2_stations:
+ _remove_stations_from_task_specifications_doc(specifications_doc, remove_lofar1_stations, remove_lofar2_stations)
task_draft_copy = models.TaskDraft.objects.create(name="%s (Copy)" % (task_draft.name,),
description="%s (Copy from task_draft id=%s)" % (task_draft.description, task_draft.id),
short_description=task_draft.short_description,
- specifications_doc=task_draft.specifications_doc,
+ specifications_doc=specifications_doc,
scheduling_unit_draft=task_draft.scheduling_unit_draft,
specifications_template=task_draft.specifications_template)
@@ -237,15 +241,18 @@ def copy_task_draft(task_draft: models.TaskDraft) -> models.TaskDraft:
return task_draft_copy
-def copy_task_blueprint_to_task_draft(task_blueprint: models.TaskBlueprint) -> models.TaskDraft:
+def copy_task_blueprint_to_task_draft(task_blueprint: models.TaskBlueprint, remove_lofar1_stations=False, remove_lofar2_stations=False) -> models.TaskDraft:
'''create a (draft)copy of the given task_blueprint. Also copy the task_relation(s) and/or the task_scheduling_relation(s)'''
logger.debug("copy_task_blueprint_to_task_draft(task_blueprint.id=%s) copying...", task_blueprint.pk)
with transaction.atomic():
+ specifications_doc = copy.deepcopy(task_blueprint.specifications_doc)
+ if remove_lofar1_stations or remove_lofar2_stations:
+ _remove_stations_from_task_specifications_doc(specifications_doc, remove_lofar1_stations, remove_lofar2_stations)
task_draft_copy = models.TaskDraft.objects.create(name="%s (Copy)" % (task_blueprint.name,),
description="%s (Copy from task_blueprint id=%s)" % (task_blueprint.description, task_blueprint.id),
short_description=task_blueprint.short_description,
- specifications_doc=task_blueprint.specifications_doc,
+ specifications_doc=specifications_doc,
scheduling_unit_draft=task_blueprint.scheduling_unit_blueprint.draft,
specifications_template=task_blueprint.specifications_template)
@@ -267,22 +274,27 @@ def copy_task_blueprint_to_task_draft(task_blueprint: models.TaskBlueprint) -> m
return task_draft_copy
-def copy_task_blueprint_via_task_draft_to_new_task_blueprint(task_blueprint: models.TaskBlueprint) -> models.TaskBlueprint:
+def copy_task_blueprint_via_task_draft_to_new_task_blueprint(task_blueprint: models.TaskBlueprint, remove_lofar1_stations=False, remove_lofar2_stations=False) -> models.TaskBlueprint:
'''create a new (blueprint)copy of the given task_blueprint via a new (draft)copy. Also copies all the task_relation(s) and/or the task_scheduling_relation(s)'''
logger.debug("copy_task_blueprint_via_task_draft_to_new_task_blueprint(task_blueprint.id=%s)", task_blueprint.pk)
with transaction.atomic():
- task_draft_copy = copy_task_blueprint_to_task_draft(task_blueprint)
+ task_draft_copy = copy_task_blueprint_to_task_draft(task_blueprint, remove_lofar1_stations, remove_lofar2_stations)
task_blueprint_copy = create_task_blueprint_from_task_draft(task_draft_copy)
return task_blueprint_copy
-def update_task_graph_from_specifications_doc(scheduling_unit_draft: models.SchedulingUnitDraft, specifications_doc: dict) -> models.SchedulingUnitDraft:
+def update_task_graph_from_specifications_doc(scheduling_unit_draft: models.SchedulingUnitDraft, specifications_doc: dict, remove_lofar1_stations=False, remove_lofar2_stations=False) -> models.SchedulingUnitDraft:
"""
Update the current task graph in this scheduling_unit_draft according to the given specifications_doc: remove tasks from the graph that are not in the doc, update tasks with the same name, create tasks that are in the doc but not in the graph yet.
"""
logger.debug("update_task_graph_from_specifications_doc(scheduling_unit_draft.id=%s, name='%s') ...", scheduling_unit_draft.pk, scheduling_unit_draft.name)
+ # remove stations before validation
+ if remove_lofar1_stations or remove_lofar2_stations:
+ for task_definition in specifications_doc.get("tasks", {}).values():
+ _remove_stations_from_task_specifications_doc(task_definition.get("specifications_doc", {}), remove_lofar1_stations, remove_lofar2_stations)
+
# make sure the given specifications_doc validates
scheduling_unit_draft.specifications_template.validate_document(specifications_doc)
@@ -1112,3 +1124,20 @@ def enough_stations_available_for_scheduling_unit(scheduling_unit: SchedulingUni
'''Are there enough stations available if the given unavailable_stations are removed?
Checks all stations_groups of the spec, and checks if there are less stations missing than max_nr_missing'''
return all(enough_stations_available_for_task(obs_task, unavailable_stations) for obs_task in scheduling_unit.observation_tasks.all())
+
+
+def _remove_stations_from_task_specifications_doc(specifications_doc, remove_lofar1_stations=False, remove_lofar2_stations=False):
+ """
+ This function can be used to sanitize task specs from LOFAR 1 or LOFAR 2 stations, e.g. when copying older specs
+ that contains a set of stations that is now a mix of LOFAR 1 and 2, and the copy should only contain one type.
+ :param specifications_doc: The task specifications_doc to sanitize
+ """
+ from lofar.sas.tmss.tmss.tmssapp.conversions import get_lofar2_stations
+ lofar2_stations = set(get_lofar2_stations())
+ station_groups = specifications_doc.get('station_configuration', {}).get('station_groups', [])
+ for group in station_groups:
+ for station in group['stations']:
+ if station not in lofar2_stations and remove_lofar1_stations:
+ group['stations'].remove(station)
+ if station in lofar2_stations and remove_lofar2_stations:
+ group['stations'].remove(station)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
index b5fadea88dfb6e274725ba06fc985bea9ff1f047..1af8a62dcc55c68c8df7f815f7e948bc8976331e 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
@@ -1256,7 +1256,9 @@ class SchedulingUnitBlueprintViewSet(LOFARViewSet):
def copy_to_draft(self, request, pk=None):
scheduling_unit_blueprint = get_object_or_404(models.SchedulingUnitBlueprint, pk=pk)
- scheduling_unit_draft_copy = create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint)
+ remove_lofar1_stations = strtobool(request.query_params.get('remove_lofar1_stations', 'False'))
+ remove_lofar2_stations = strtobool(request.query_params.get('remove_lofar2_stations', 'False'))
+ scheduling_unit_draft_copy = create_scheduling_unit_draft_from_scheduling_unit_blueprint(scheduling_unit_blueprint, remove_lofar1_stations, remove_lofar2_stations)
# return a response with the new serialized SchedulingUnitBlueprintExtendedSerializer
return Response(serializers.SchedulingUnitDraftExtendedSerializer(scheduling_unit_draft_copy, context={'request':request}).data,
@@ -1513,7 +1515,11 @@ class TaskDraftViewSet(LOFARViewSet):
@action(methods=['post'], detail=True, url_name="copy", name="Create Copy")
def copy(self, request, pk=None):
task_draft = get_object_or_404(models.TaskDraft, pk=pk)
- task_draft_copy = copy_task_draft(task_draft)
+
+ remove_lofar1_stations = strtobool(request.query_params.get('remove_lofar1_stations', 'False'))
+ remove_lofar2_stations = strtobool(request.query_params.get('remove_lofar2_stations', 'False'))
+
+ task_draft_copy = copy_task_draft(task_draft, remove_lofar1_stations, remove_lofar2_stations)
return Response(serializers.TaskDraftSerializer(task_draft_copy, context={'request':request}).data,
status=status.HTTP_201_CREATED)
@@ -1632,7 +1638,11 @@ class TaskBlueprintViewSet(LOFARViewSet):
@action(methods=['post'], detail=True, url_name="copy_to_new_draft", name="Copy to new TaskDraft")
def copy_to_new_draft(self, request, pk=None):
task_blueprint = get_object_or_404(models.TaskBlueprint, pk=pk)
- task_draft_copy = copy_task_blueprint_to_task_draft(task_blueprint)
+
+ remove_lofar1_stations = strtobool(request.query_params.get('remove_lofar1_stations', 'False'))
+ remove_lofar2_stations = strtobool(request.query_params.get('remove_lofar2_stations', 'False'))
+
+ task_draft_copy = copy_task_blueprint_to_task_draft(task_blueprint, remove_lofar1_stations, remove_lofar2_stations)
return Response(serializers.TaskDraftSerializer(task_draft_copy, context={'request':request}).data,
status=status.HTTP_201_CREATED)
@@ -1645,7 +1655,11 @@ class TaskBlueprintViewSet(LOFARViewSet):
def copy_to_new_blueprint(self, request, pk=None):
with transaction.atomic():
task_blueprint = get_object_or_404(models.TaskBlueprint, pk=pk)
- task_blueprint_copy = copy_task_blueprint_via_task_draft_to_new_task_blueprint(task_blueprint)
+
+ remove_lofar1_stations = strtobool(request.query_params.get('remove_lofar1_stations', 'False'))
+ remove_lofar2_stations = strtobool(request.query_params.get('remove_lofar2_stations', 'False'))
+
+ task_blueprint_copy = copy_task_blueprint_via_task_draft_to_new_task_blueprint(task_blueprint, remove_lofar1_stations, remove_lofar2_stations)
create_or_update_subtasks_from_task_blueprint(task_blueprint_copy)
task_blueprint_copy.refresh_from_db()
diff --git a/SAS/TMSS/backend/test/t_scheduling_units.py b/SAS/TMSS/backend/test/t_scheduling_units.py
index 1f998c43e6a40205e9c22a293ef0fc7d5f2b0e73..1a6321b8712a48b109749197e79f9452a4d2e9ce 100644
--- a/SAS/TMSS/backend/test/t_scheduling_units.py
+++ b/SAS/TMSS/backend/test/t_scheduling_units.py
@@ -55,14 +55,15 @@ from lofar.sas.tmss.test.tmss_test_data_django_models import *
rest_data_creator = tmss_test_env.create_test_data_creator()
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.sas.tmss.tmss.exceptions import SchemaValidationException, ObsoleteTemplateException, ValidationException, ObsoleteValidationException
+from lofar.sas.tmss.tmss.exceptions import SchemaValidationException, ObsoleteTemplateException, ValidationException, ObsoleteValidationException, RuleValidationException
import requests
-from lofar.sas.tmss.tmss.tmssapp.tasks import create_scheduling_unit_blueprint_and_tasks_and_subtasks_from_scheduling_unit_draft, create_scheduling_unit_blueprint_from_scheduling_unit_draft, update_task_blueprint_graph_from_draft, update_task_graph_from_specifications_doc, create_scheduling_unit_draft_from_observing_strategy_template, mark_task_blueprint_as_obsolete, cancel_task_blueprint, schedule_independent_subtasks_in_task_blueprint
+from lofar.sas.tmss.tmss.tmssapp.tasks import create_scheduling_unit_blueprint_and_tasks_and_subtasks_from_scheduling_unit_draft, create_scheduling_unit_blueprint_from_scheduling_unit_draft, update_task_blueprint_graph_from_draft, update_task_graph_from_specifications_doc, create_scheduling_unit_draft_from_observing_strategy_template, mark_task_blueprint_as_obsolete, cancel_task_blueprint, schedule_independent_subtasks_in_task_blueprint, create_scheduling_unit_draft_from_scheduling_unit_blueprint
from lofar.sas.tmss.tmss.tmssapp.subtasks import schedule_subtask, cancel_subtask, wait_for_subtask_status
from lofar.sas.tmss.test.test_utils import set_subtask_state_following_allowed_transitions
from lofar.messaging.messagebus import BusListenerJanitor
+from lofar.sas.tmss.tmss.tmssapp.conversions import get_lofar2_stations
logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
@@ -495,11 +496,68 @@ class SchedulingUnitBlueprintStateTest(unittest.TestCase):
draft = create_scheduling_unit_draft_from_observing_strategy_template(strategy_template, scheduling_set=models.SchedulingSet.objects.create(**SchedulingSet_test_data()), specifications_doc_overrides=override_doc)
# the template contains international stations and a custom group by default, both including DE605.
- self.assertNotIn("DE605", draft.specifications_doc)
+ self.assertNotIn("DE605", str(draft.specifications_doc))
# assert the draft specs contain exactly the specified station groups:
self.assertEqual(draft.specifications_doc["tasks"]["Target Observation"]["specifications_doc"]["station_configuration"]["station_groups"], station_groups)
+ def test_remove_lofar2_station_while_create_scheduling_unit_draft_from_scheduling_unit_blueprint(self): # TMSS-2637
+ # create scheduling unit with several LOFAR1 stations
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.get_latest(name="IM HBA - 1 Beam")
+ station_groups = [{"stations": ["CS002","CS003","CS004","CS005","DE605"], "max_nr_missing": 1}]
+ override_doc = {"tasks": {"Target Observation": {"specifications_doc": {"station_configuration": {"station_groups": station_groups}}}}}
+ draft = create_scheduling_unit_draft_from_observing_strategy_template(strategy_template, scheduling_set=models.SchedulingSet.objects.create(**SchedulingSet_test_data()), specifications_doc_overrides=override_doc)
+ blueprint = create_scheduling_unit_blueprint_and_tasks_and_subtasks_from_scheduling_unit_draft(draft)
+
+ # check that the blueprint contains a particular station and all its tasks validate
+ self.assertIn("DE605", str(blueprint.specifications_doc))
+ for task in blueprint.task_blueprints.all():
+ task.validate_specifications_doc()
+
+ lofar2_stations = get_lofar2_stations()
+ self.assertNotIn('DE605', lofar2_stations)
+
+ # mark that station LOFAR2
+ station_schema_template = models.CommonSchemaTemplate.get_latest(name="stations")
+ groups = station_schema_template.schema['definitions']['station_group']['anyOf']
+ for group in groups:
+ if group['title'].upper() == 'LOFAR2':
+ group['properties']['stations']['enum'][0].append('DE605')
+ station_schema_template.save()
+ get_lofar2_stations.cache_clear()
+ lofar2_stations = get_lofar2_stations()
+ self.assertIn('DE605', lofar2_stations)
+
+ # make sure that the previously blueprinted tasks don't validate any more
+ with self.assertRaises(RuleValidationException):
+ for task in blueprint.task_blueprints.all():
+ task.validate_specifications_doc()
+
+ # start a subtask so that the unit is advance enough to skip rule-based validation on specs read access
+ set_subtask_state_following_allowed_transitions(blueprint.subtasks.first(), 'started')
+ blueprint.refresh_from_db()
+ self.assertEqual(blueprint.status.value, models.SchedulingUnitStatus.Choices.OBSERVING.value)
+
+ # try to create a draft copy, which should fail due to the invalid specs
+ with self.assertRaises(ValidationException):
+ create_scheduling_unit_draft_from_scheduling_unit_blueprint(blueprint)
+
+ # try again, but this time remove lofar2 stations when creating the draft so that it can be blueprinted
+ draft_copy = create_scheduling_unit_draft_from_scheduling_unit_blueprint(blueprint, remove_lofar2_stations=True)
+ blueprint_from_copy = create_scheduling_unit_blueprint_and_tasks_and_subtasks_from_scheduling_unit_draft(draft_copy)
+
+ # check that the new blueprint does not contain the LOFAR2 station and all its tasks validate
+ self.assertNotIn("DE605", str(blueprint_from_copy.specifications_doc))
+ for task in blueprint_from_copy.task_blueprints.all():
+ task.validate_specifications_doc()
+
+ # revert the list of LOFAR2 stations, so other tests still work
+ for group in groups:
+ if group['title'].upper() == 'LOFAR2':
+ group['properties']['stations']['enum'][0].remove('DE605')
+ station_schema_template.save()
+ get_lofar2_stations.cache_clear()
+
class TestFlatStations(unittest.TestCase):
"""