diff --git a/LCS/Messaging/python/messaging/messagebus.py b/LCS/Messaging/python/messaging/messagebus.py
index 7bc94c6719b6a51193969591d8b8e9c39a5deea9..1e0375f1fe91f5ec80380b0f74c16d19a1253cc3 100644
--- a/LCS/Messaging/python/messaging/messagebus.py
+++ b/LCS/Messaging/python/messaging/messagebus.py
@@ -942,7 +942,7 @@ class TemporaryExchange:
uuid.uuid4().hex[:8]))
logger.debug("Creating TemporaryExchange at %s ...", self.address)
create_exchange(name=self.address, broker=self.broker)
- logger.debug("Created TemporaryExchange at %s", self.address)
+ logger.info("Created TemporaryExchange at %s", self.address)
def close(self):
"""
@@ -954,7 +954,7 @@ class TemporaryExchange:
delete_exchange(self.address)
except Exception as e:
logger.error(e)
- logger.debug("Closed TemporaryExchange at %s", self.address)
+ logger.info("Closed TemporaryExchange at %s", self.address)
self.address = None
def __str__(self):
diff --git a/LCS/PyCommon/ring_coordinates.py b/LCS/PyCommon/ring_coordinates.py
index cc536c4ccf04445217d3f0073c5e5380d462f544..1113ff871821d7bd4c35abba6323f82a9df0d314 100755
--- a/LCS/PyCommon/ring_coordinates.py
+++ b/LCS/PyCommon/ring_coordinates.py
@@ -9,8 +9,8 @@ class RingCoordinates:
"""
This has been taken from RTCP/Conbalt test tRinGCoordinates.py
- Original RingCoordinates implementation (+ Vlad's fix). Taken from parset.py in
- RTCP\Run\src\LOFAR\parset
+ Original RingCoordinates implementation (+ Vlad's fix).
+ Taken from parset.py in RTCP\\Run\\src\\LOFAR\\parset
"""
def __init__(self, numrings, width, center, dirtype):
self.numrings = numrings
diff --git a/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingestjobmanagementserver.py b/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingestjobmanagementserver.py
index 78d2cd998044a685f9b5eba177d30f43648f72b8..7452f5bf6d1cad4b264b67d1280eba140db587fc 100644
--- a/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingestjobmanagementserver.py
+++ b/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingestjobmanagementserver.py
@@ -57,6 +57,16 @@ class IngestJobManager:
self._tobus = ToBus(exchange=exchange, broker=broker)
+ self._incoming_jobs_listener = BusListener(IngestIncomingJobsHandler, {'job_manager': self},
+ exchange=self._tobus.exchange, broker=self._tobus.broker,
+ routing_key="%s.#" % DEFAULT_INGEST_INCOMING_JOB_SUBJECT)
+
+ self._ingest_event_listener = IngestEventMesssageBusListener(IngestEventMessageHandlerForJobManager, {'job_manager': self},
+ exchange=self._tobus.exchange, broker=self._tobus.broker)
+
+ self._ingest_service = RPCService(DEFAULT_INGEST_SERVICENAME, IngestServiceMessageHandler, {'job_manager': self},
+ exchange=self._tobus.exchange, broker=self._tobus.broker, num_threads=4)
+
self.__running_jobs_log_timestamp = datetime.utcnow()
self.__last_putStalledJobsBackToToDo_timestamp = datetime.utcnow()
@@ -86,19 +96,8 @@ class IngestJobManager:
logger.info('starting listening for new jobs and notifications')
- incoming_jobs_listener = BusListener(IngestIncomingJobsHandler, {'job_manager': self},
- exchange=self._tobus.exchange, broker=self._tobus.broker,
- routing_key="%s.#" % DEFAULT_INGEST_INCOMING_JOB_SUBJECT)
-
- ingest_event_listener = IngestEventMesssageBusListener(IngestEventMessageHandlerForJobManager,
- {'job_manager': self},
- exchange=self._tobus.exchange, broker=self._tobus.broker)
-
- ingest_service = RPCService(DEFAULT_INGEST_SERVICENAME, IngestServiceMessageHandler, {'job_manager': self},
- exchange=self._tobus.exchange, broker=self._tobus.broker, num_threads=4)
-
# open exchange connections...
- with incoming_jobs_listener, ingest_event_listener, ingest_service, self._tobus:
+ with self._incoming_jobs_listener, self._ingest_event_listener, self._ingest_service, self._tobus:
with self.__lock:
# start with full jobs dir scan to retreive state from disk
self.scanJobsdir()
diff --git a/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingesttmssadapter.py b/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingesttmssadapter.py
index 3f89b769ebdcd86c9131ebf1da31f4ee648041e3..7fd829007bf08bc58122d8ba8b1ad33e8f62c1ff 100644
--- a/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingesttmssadapter.py
+++ b/LTA/LTAIngest/LTAIngestServer/LTAIngestAdminServer/lib/ingesttmssadapter.py
@@ -131,8 +131,12 @@ class TMSSEventMessageHandlerForIngestTMSSAdapter(UsingToBusMixin, TMSSEventMess
self.tmss_client.close()
def init_tobus(self, exchange, broker):
- logger.warning("FOR COMMISSIONING WE LET THE INGESTTMSSADAPTER SEND ITS INGEST JOBS TO THE PRODUCTION BROKER!")
- self._tobus = ToBus(exchange='lofar', broker='scu001.control.lofar')
+ from lofar.common import isDevelopmentEnvironment
+ if isDevelopmentEnvironment():
+ self._tobus = ToBus(exchange=exchange, broker=broker)
+ else:
+ logger.warning("FOR COMMISSIONING WE LET THE INGESTTMSSADAPTER SEND ITS INGEST JOBS TO THE PRODUCTION BROKER!")
+ self._tobus = ToBus(exchange='lofar', broker='scu001.control.lofar')
def onSubTaskStatusChanged(self, id: int, status: str):
super().onSubTaskStatusChanged(id, status)
@@ -162,7 +166,7 @@ class TMSSEventMessageHandlerForIngestTMSSAdapter(UsingToBusMixin, TMSSEventMess
obs_id=producing_subtask['id'], # the name 'obs_id' is somewhat misleading, but that's a legacy name 'forced' by MoM/OTDB. TODO: refactor when removing MoM/OTDB.
dataproduct_name=input_dp['filename'],
archive_id=dp_global_identifier['unique_identifier'],
- location=subtask['cluster_value']+':'+os.path.join(input_dp['directory'], input_dp['filename']),
+ location=subtask['cluster_name']+':'+os.path.join(input_dp['directory'], input_dp['filename']),
tmss_ingest_subtask_id=subtask['id'],
tmss_input_dataproduct_id=input_dp['id'])
@@ -180,13 +184,16 @@ class IngestTMSSAdapter:
It has two purpouses: 1) create and enqueue ingest jobs upon receiving an ingest-subtask scheduled event
and 2) track progress of the ingest-subtask (number of dataproducts transferred) and updating the (finished) state of the ingest-subtask'''
def __init__(self, tmss_creds: DBCredentials, exchange=DEFAULT_BUSNAME, broker=DEFAULT_BROKER):
+ from lofar.common import isDevelopmentEnvironment
self.ingest2tmss_adapter = IngestEventMesssageBusListener(handler_type=IngestEventMessageHandlerForIngestTMSSAdapter,
handler_kwargs={'tmss_creds': tmss_creds},
- exchange='lofar', broker='scu001.control.lofar') # TODO: replace hardcoded commissioning brokers by parameters
+ exchange=exchange if isDevelopmentEnvironment() else 'lofar', # TODO: replace hardcoded commissioning exchange by parameter
+ broker=broker if isDevelopmentEnvironment() else 'scu001.control.lofar') # TODO: replace hardcoded commissioning brokers by parameter
self.tmss2ingest_adapter = TMSSBusListener(handler_type=TMSSEventMessageHandlerForIngestTMSSAdapter,
handler_kwargs={'tmss_creds': tmss_creds},
- routing_key=TMSS_ALL_EVENTS_FILTER,
- exchange='test.lofar', broker='scu199.control.lofar') # TODO: replace hardcoded commissioning brokers by parameters
+ routing_key=TMSS_SUBTASK_STATUS_EVENT_PREFIX+'.#',
+ exchange=exchange if isDevelopmentEnvironment() else 'test.lofar', # TODO: replace hardcoded commissioning brokers by parameter
+ broker=broker if isDevelopmentEnvironment() else 'scu199.control.lofar') # TODO: replace hardcoded commissioning brokers by parameter
def open(self):
self.ingest2tmss_adapter.start_listening()
diff --git a/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/test/t_ingestpipeline.py b/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/test/t_ingestpipeline.py
index 61805d76f5117a298bdc2619c554d53a51a33384..a34af8533c65846efc631d61a363490661fed91a 100755
--- a/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/test/t_ingestpipeline.py
+++ b/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/test/t_ingestpipeline.py
@@ -293,6 +293,7 @@ with patch('lofar.lta.ingest.server.ltaclient.LTAClient', autospec=True) as Mock
os.removedirs(self.test_dir_path)
+ @unittest.skip("TODO: re-enable when merged with TMSS-261")
@integration_test
def test_directory_with_TMSS(self):
'''same test as test_directory (which tests against stubbed MoM), but now with TMSS'''
@@ -308,7 +309,7 @@ with patch('lofar.lta.ingest.server.ltaclient.LTAClient', autospec=True) as Mock
# assume the ingest_tmss_adapter works correctly. It is tested in t_ingesttmssadapter.
with TMSSTestEnvironment(exchange=self.tmp_exchange.address, populate_schemas=True) as tmss_test_env:
from lofar.lta.ingest.server.ingesttmssadapter import IngestTMSSAdapter
- with IngestTMSSAdapter(tmss_test_env.client_credentials.dbcreds, self.tmp_exchange.address):
+ with IngestTMSSAdapter(tmss_test_env.client_credentials.dbcreds, exchange=self.tmp_exchange.address, broker=self.tmp_exchange.broker):
from lofar.sas.tmss.test.tmss_test_data_django_models import SubtaskTemplate_test_data, Subtask_test_data, \
TaskBlueprint_test_data, TaskTemplate_test_data, Dataproduct_test_data, \
SubtaskOutput_test_data, SubtaskInput_test_data
@@ -334,7 +335,9 @@ with patch('lofar.lta.ingest.server.ltaclient.LTAClient', autospec=True) as Mock
ingest_task_template = models.TaskTemplate.objects.create(**TaskTemplate_test_data(task_type_value='ingest'))
ingest_task = models.TaskBlueprint.objects.create(**TaskBlueprint_test_data(scheduling_unit_blueprint=obs_subtask.task_blueprint.scheduling_unit_blueprint, specifications_template=ingest_task_template))
ingest_subtask_template = models.SubtaskTemplate.objects.create(**SubtaskTemplate_test_data(subtask_type_value='ingest'))
- ingest_subtask = models.Subtask.objects.create(**Subtask_test_data(subtask_template=ingest_subtask_template, task_blueprint=ingest_task))
+ ingest_subtask = models.Subtask.objects.create(**Subtask_test_data(subtask_template=ingest_subtask_template))
+ ingest_subtask.blueprints.set([ingest_task])
+ ingest_subtask.save()
ingest_subtask_input = models.SubtaskInput.objects.create(**SubtaskInput_test_data(subtask=ingest_subtask, producer=obs_subtask_output))
ingest_subtask_input.dataproducts.set([obs_dataproduct])
ingest_subtask_input.save()
diff --git a/LTA/LTAIngest/LTAIngestServer/test/CMakeLists.txt b/LTA/LTAIngest/LTAIngestServer/test/CMakeLists.txt
index 5e0c61f4f40ed78dd3a5c1147604321e9cab1e00..fd2961464fc6c66d5687648b5ea7890d57157ea2 100644
--- a/LTA/LTAIngest/LTAIngestServer/test/CMakeLists.txt
+++ b/LTA/LTAIngest/LTAIngestServer/test/CMakeLists.txt
@@ -1,6 +1,8 @@
include(LofarCTest)
-IF(BUILD_TESTING)
+IF(BUILD_TMSSBackend)
lofar_add_test(t_ingest_tmss_integration_test)
set_tests_properties(t_ingest_tmss_integration_test PROPERTIES TIMEOUT 600)
-ENDIF(BUILD_TESTING)
+ELSE()
+ message(WARNING "Skipping t_ingest_tmss_integration_test because it depends on the TMSSBackend package which is not included in the build")
+ENDIF(BUILD_TMSSBackend)
diff --git a/LTA/LTAIngest/LTAIngestServer/test/t_ingest_tmss_integration_test.py b/LTA/LTAIngest/LTAIngestServer/test/t_ingest_tmss_integration_test.py
index 82f9593d8d5d08fb318f6956488bcf69f2e0f69a..0300cb0df79de6dbf8aba4d8a25f3c70d4e8a47a 100755
--- a/LTA/LTAIngest/LTAIngestServer/test/t_ingest_tmss_integration_test.py
+++ b/LTA/LTAIngest/LTAIngestServer/test/t_ingest_tmss_integration_test.py
@@ -1,8 +1,5 @@
#!/usr/bin/env python3
-#TODO: Fix test
-exit(3)
-
import unittest
from unittest import mock
from random import randint
@@ -31,7 +28,7 @@ class TestIngestTMSSIntegration(unittest.TestCase):
def test(self):
with TemporaryExchange("TestIngestTMSSIntegration") as tmp_exchange:
- # override DEFAULT_BUSNAME
+ # override DEFAULT_BUSNAME (which is used in a call from TMSS to RA to schedule)
import lofar
lofar.messaging.config.DEFAULT_BUSNAME = tmp_exchange.address
@@ -87,49 +84,64 @@ class TestIngestTMSSIntegration(unittest.TestCase):
# mock throttling method
transfer_server.enoughResourcesAvailable = lambda: True
- strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
- scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template, strategy_template.scheduling_unit_template.schema)
- # limit the number of subbands
- for task_name, task in scheduling_unit_spec['tasks'].items():
- if 'SAPs' in task['specifications_doc']:
- SAPs = task['specifications_doc']['SAPs']
- for SAP in SAPs:
- SAP['subbands'] = [0]
- scheduling_unit_spec['tasks'][task_name]['specifications_doc']['SAPs'] = SAPs
-
- scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(**SchedulingUnitDraft_test_data(template=strategy_template.scheduling_unit_template, requirements_doc=scheduling_unit_spec))
- scheduling_unit = create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
- ingest_subtask = models.Subtask.objects.get(task_blueprint__scheduling_unit_blueprint__id=scheduling_unit.id,
- specifications_template__type__value=models.SubtaskType.Choices.INGEST.value)
- schedule_independent_subtasks_in_scheduling_unit_blueprint(scheduling_unit, datetime.utcnow())
-
- # make sure each dataproduct uses TEST_DATA_DIR as root
- for task in scheduling_unit.task_blueprints.all():
- for subtask in task.subtasks.all():
- if subtask.state.value == models.SubtaskState.Choices.SCHEDULED.value:
- for output_dp in subtask.output_dataproducts.all():
- output_dp.directory = output_dp.directory.replace('/data', TEST_DATA_DIR)
- output_dp.save()
-
- # start a simulator, forcing the scheduling_unit to "run" the observations and pipelines....
- # and let the ingest server act on the eventmessages.
- # as a result, the scheduling_unit should be finished at the end, and the dataproducts should be "archived" (not in the real LTA of course, because we faked the transfer)
- stop_event = threading.Event()
- with create_scheduling_unit_blueprint_simulator(scheduling_unit.id, stop_event,
- handle_ingest=False, handle_observations=True, handle_QA=True, handle_pipelines=True, create_output_dataproducts=True,
- delay=0, duration=0,
- exchange=tmp_exchange.address) as simulator:
- self.assertTrue(stop_event.wait(300))
-
- scheduling_unit.refresh_from_db()
- self.assertEqual("finished", scheduling_unit.status)
-
- ingest_subtask.refresh_from_db()
- self.assertGreater(ingest_subtask.output_dataproducts.count(), 0)
-
- for output_dp in ingest_subtask.output_dataproducts.all():
- self.assertEqual(1, models.DataproductArchiveInfo.objects.filter(dataproduct__id=output_dp.id).count())
-
+ # cleanup queues with janitor
+ with BusListenerJanitor(ingest_job_manager._incoming_jobs_listener), BusListenerJanitor(ingest_job_manager._ingest_event_listener), BusListenerJanitor(ingest_job_manager._ingest_service), \
+ BusListenerJanitor(ingest_tmss_adapter.ingest2tmss_adapter), BusListenerJanitor(ingest_tmss_adapter.tmss2ingest_adapter), BusListenerJanitor(transfer_server.incoming_jobs_listener):
+
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Short Test Observation - Pipeline - Ingest")
+ scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template, strategy_template.scheduling_unit_template.schema)
+ # limit the number of subbands, and disable QA subtasks, and cleanup task
+ for task_name, task in list(scheduling_unit_spec['tasks'].items()):
+ if 'SAPs' in task['specifications_doc']:
+ SAPs = task['specifications_doc']['SAPs']
+ for SAP in SAPs:
+ SAP['subbands'] = [0]
+ scheduling_unit_spec['tasks'][task_name]['specifications_doc']['SAPs'] = SAPs
+ if 'QA' in task['specifications_doc']:
+ task['specifications_doc']['QA']['plots']['enabled'] = False
+ task['specifications_doc']['QA']['file_conversion']['enabled'] = False
+ if task['specifications_template'] == 'cleanup':
+ # remove cleanup task and its relations
+ scheduling_unit_spec['tasks'].pop(task_name)
+ scheduling_unit_spec['task_relations'] = [task_rel for task_rel in scheduling_unit_spec['task_relations'] if task_rel['consumer'] != task_name]
+
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(**SchedulingUnitDraft_test_data(template=strategy_template.scheduling_unit_template, requirements_doc=scheduling_unit_spec))
+ scheduling_unit = create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+ ingest_subtask = models.Subtask.objects.get(task_blueprint__scheduling_unit_blueprint__id=scheduling_unit.id,
+ specifications_template__type__value=models.SubtaskType.Choices.INGEST.value)
+ schedule_independent_subtasks_in_scheduling_unit_blueprint(scheduling_unit, datetime.utcnow())
+
+ # make sure each dataproduct uses TEST_DATA_DIR as root
+ for task in scheduling_unit.task_blueprints.all():
+ for subtask in task.subtasks.all():
+ if subtask.state.value == models.SubtaskState.Choices.SCHEDULED.value:
+ for output_dp in subtask.output_dataproducts.all():
+ output_dp.directory = output_dp.directory.replace('/data', TEST_DATA_DIR)
+ output_dp.save()
+
+ # start a simulator, forcing the scheduling_unit to "run" the observations and pipelines....
+ # and let the ingest server act on the eventmessages.
+ # as a result, the scheduling_unit should be finished at the end, and the dataproducts should be "archived" (not in the real LTA of course, because we faked the transfer)
+ stop_event = threading.Event()
+ with create_scheduling_unit_blueprint_simulator(scheduling_unit.id, stop_event,
+ handle_ingest=False, handle_observations=True, handle_QA=True, handle_pipelines=True, create_output_dataproducts=True,
+ delay=0, duration=0, auto_grant_ingest_permission=True,
+ exchange=tmp_exchange.address) as simulator:
+
+ # wait until the observations/pipelines finished simulating
+ stop_event.wait(300)
+
+ # scheduling_unit (including ingest) should be finished
+ scheduling_unit.refresh_from_db()
+ self.assertEqual("finished", scheduling_unit.status)
+ ingest_subtask.refresh_from_db()
+ self.assertEqual("finished", ingest_subtask.state.value)
+
+ # check ingested dataproducts
+ self.assertGreater(ingest_subtask.output_dataproducts.count(), 0)
+ for output_dp in ingest_subtask.output_dataproducts.all():
+ self.assertTrue(output_dp.filepath.startswith("srm://"))
+ self.assertEqual(1, models.DataproductArchiveInfo.objects.filter(dataproduct__id=output_dp.id).count())
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
diff --git a/QA/QA_Service/lib/qa_service.py b/QA/QA_Service/lib/qa_service.py
index 992ddb000178fcf1fff3cb93cedc9b7d5a91ac25..1e6ece57a569369ebf75f844d9a56370e2b84cbc 100644
--- a/QA/QA_Service/lib/qa_service.py
+++ b/QA/QA_Service/lib/qa_service.py
@@ -76,6 +76,7 @@ class QAFilteringTMSSSubTaskBusListener(TMSSBusListener):
def _send_qa_command_message(self, subtask_id: int, command_subject: str):
with TMSSsession.create_from_dbcreds_for_ldap() as tmsssession:
tmsssession.set_subtask_status(subtask_id, 'queueing')
+ tmsssession.set_subtask_status(subtask_id, 'queued')
try:
content = {"subtask_id": subtask_id }
@@ -85,7 +86,6 @@ class QAFilteringTMSSSubTaskBusListener(TMSSBusListener):
except Exception as e:
logger.error('Could not send event message: %s', e)
- tmsssession.set_subtask_status(subtask_id, 'queued')
def onSubTaskStatusChanged(self, id: int, status:str):
if status == "scheduled":
diff --git a/QA/QA_Service/test/t_qa_service.py b/QA/QA_Service/test/t_qa_service.py
index 0b418a7a6885901801e665d9818cbba9667707c1..c139d3c7b8b5847e124b2969524535080f42beca 100755
--- a/QA/QA_Service/test/t_qa_service.py
+++ b/QA/QA_Service/test/t_qa_service.py
@@ -567,11 +567,10 @@ class TestQAService(unittest.TestCase):
obs_subtask = tdc.post_data_and_get_response_as_json_object(tdc.Subtask(specifications_template_url=obs_subtask_template['url']), '/subtask/')
obs_subtask_output = tdc.post_data_and_get_response_as_json_object(tdc.SubtaskOutput(subtask_url=obs_subtask['url']), '/subtask_output/')
uv_dataproduct = tdc.post_data_and_get_response_as_json_object(tdc.Dataproduct(filename="my_uv_dataprodcut.MS", directory=self.TEST_DIR, subtask_output_url=obs_subtask_output['url']), '/dataproduct/')
- for state in ('defined', 'scheduling', 'scheduled', 'queueing', 'queued', 'starting', 'started', 'finishing', 'finished'):
+ for state in ['defined', 'scheduling', 'scheduled', 'queueing', 'queued', 'starting', 'started', 'finishing', 'finished']:
tmss_client.set_subtask_status(obs_subtask['id'], state)
-
- qafile_subtask_template = tmss_client.get_subtask_template(name="QA file conversion")
+ qafile_subtask_template = tmss_client.get_subtask_template(name="QA file conversion")
qafile_subtask_spec_doc = tmss_client.get_subtask_template_default_specification(name="QA file conversion")
subtask = tdc.post_data_and_get_response_as_json_object(tdc.Subtask(specifications_template_url=qafile_subtask_template['url'],
diff --git a/SAS/DataManagement/Cleanup/CleanupClient/rpc.py b/SAS/DataManagement/Cleanup/CleanupClient/rpc.py
index cda86ffe8ec9efff420c28fbe61c51519b0dc50d..1247c5f4724881995ddde755631428bab5984e4e 100644
--- a/SAS/DataManagement/Cleanup/CleanupClient/rpc.py
+++ b/SAS/DataManagement/Cleanup/CleanupClient/rpc.py
@@ -25,17 +25,20 @@ class CleanupRPC(RPCClientContextManagerMixin):
def getPathForOTDBId(self, otdb_id):
return self._rpc_client.execute('GetPathForOTDBId', otdb_id=otdb_id)
+ def getPathForTMSSId(self, tmss_id):
+ return self._rpc_client.execute('GetPathForTMSSId', tmss_id=tmss_id)
+
def removePath(self, path):
return self._rpc_client.execute('RemovePath', path=path)
- def removeTaskData(self, otdb_id, delete_is=True, delete_cs=True, delete_uv=True, delete_im=True, delete_img=True, delete_pulp=True, delete_scratch=True, force=False):
- return self._rpc_client.execute('RemoveTaskData', otdb_id=otdb_id, delete_is=delete_is, delete_cs=delete_cs, delete_uv=delete_uv, delete_im=delete_im, delete_img=delete_img, delete_pulp=delete_pulp, delete_scratch=delete_scratch, force=force)
+ def removeTaskData(self, otdb_id=None, tmss_id=None, delete_is=True, delete_cs=True, delete_uv=True, delete_im=True, delete_img=True, delete_pulp=True, delete_scratch=True, force=False):
+ return self._rpc_client.execute('RemoveTaskData', otdb_id=otdb_id, tmss_id=tmss_id, delete_is=delete_is, delete_cs=delete_cs, delete_uv=delete_uv, delete_im=delete_im, delete_img=delete_img, delete_pulp=delete_pulp, delete_scratch=delete_scratch, force=force)
- def setTaskDataPinned(self, otdb_id, pinned=True):
- return self._rpc_client.execute('SetTaskDataPinned', otdb_id=otdb_id, pinned=pinned)
+ def setTaskDataPinned(self, otdb_id, tmss_id=None, pinned=True):
+ return self._rpc_client.execute('SetTaskDataPinned', otdb_id=otdb_id, tmss_id=tmss_id, pinned=pinned)
- def isTaskDataPinned(self, otdb_id):
- return convertStringDigitKeysToInt(self._rpc_client.execute('IsTaskDataPinned', otdb_id=otdb_id)).get(otdb_id, False)
+ def isTaskDataPinned(self, otdb_id=None, tmss_id=None):
+ return self._rpc_client.execute('IsTaskDataPinned', otdb_id=otdb_id, tmss_id=tmss_id)
def getPinnedStatuses(self):
return convertStringDigitKeysToInt(self._rpc_client.execute('GetPinnedStatuses'))
@@ -45,12 +48,12 @@ def main():
from optparse import OptionParser
# Check the invocation arguments
- parser = OptionParser('%prog [options] <otdb_id>',
+ parser = OptionParser('%prog [options] <otdb_id/tmss_id>',
description='do cleanup actions on cep4 from the commandline')
- parser.add_option('-d', '--delete', dest='delete', action='store_true', help='delete the data for the given otdb_id (see also --force option)')
+ parser.add_option('-d', '--delete', dest='delete', action='store_true', help='delete the data for the given otdb_id/tmss_id (see also --force option)')
parser.add_option('-f', '--force', dest='force', action='store_true', help='in combination with --delete, always delete the data even when safety checks block deletion. (But pinned data is still kept, even when this force flag is supplied.)')
- parser.add_option('-p', '--pin', dest='pin', action='store_true', help='pin the data for the given otdb_id')
- parser.add_option('-u', '--unpin', dest='unpin', action='store_true', help='unpin the data for the given otdb_id')
+ parser.add_option('-p', '--pin', dest='pin', action='store_true', help='pin the data for the given otdb_id/tmss_id')
+ parser.add_option('-u', '--unpin', dest='unpin', action='store_true', help='unpin the data for the given otdb_id/tmss_id')
parser.add_option('-b', '--broker', dest='broker', type='string', default=DEFAULT_BROKER,
help='Address of the broker, default: localhost')
parser.add_option('-e', '--exchange', dest='exchange', type='string', default=DEFAULT_BUSNAME,
@@ -66,29 +69,32 @@ def main():
level=logging.INFO if options.verbose else logging.WARN)
with CleanupRPC.create(exchange=options.exchange, broker=options.broker) as rpc:
- otdb_id = int(args[0])
+ # the cmdline given id is either an otdb_id or a tmss_id, based on the fact that tmss_id's start at 2000000
+ id = int(args[0])
+ otdb_id = id if id < 2000000 else None
+ tmss_id = id if id >= 2000000 else None
if options.pin or options.unpin:
- rpc.setTaskDataPinned(otdb_id, bool(options.pin))
+ rpc.setTaskDataPinned(otdb_id=otdb_id, tmss_id=tmss_id, pinned=bool(options.pin))
elif not options.delete:
- print('data for otdb_id %s is %spinned' % (otdb_id, '' if rpc.isTaskDataPinned(otdb_id) else 'not '))
+ print('data for otdb_id=%s tmss_id=%s is %spinned' % (otdb_id, tmss_id, '' if rpc.isTaskDataPinned(otdb_id=otdb_id, tmss_id=tmss_id) else 'not '))
if options.delete:
if options.pin:
print("You can't delete and pin data at the same time!")
exit(1)
- path_result = rpc.getPathForOTDBId(otdb_id)
+ path_result = rpc.getPathForOTDBId(otdb_id) if otdb_id is not None else rpc.getPathForTMSSId(tmss_id)
if path_result['found']:
path = path_result['path']
scratch_paths = path_result.get('scratch_paths', [])
paths = scratch_paths + [path]
print("This will delete everything in '%s'." % ', '.join(paths))
if input("Are you sure? (y/n) ") == 'y':
- result = rpc.removeTaskData(otdb_id, force=options.force)
+ result = rpc.removeTaskData(otdb_id=otdb_id, tmss_id=tmss_id, force=options.force)
print()
if not result['deleted']:
- print('Could not delete data for task with otdb_id=%s' % otdb_id)
+ print('Could not delete data for task with otdb_id=%s tmss_id=%s' % (otdb_id, tmss_id))
print(result['message'])
exit(0 if result['deleted'] else 1)
else:
diff --git a/SAS/DataManagement/Cleanup/CleanupService/CMakeLists.txt b/SAS/DataManagement/Cleanup/CleanupService/CMakeLists.txt
index e5455f56fc0c486e136a98037e12bace6930822f..0b3ab070dfb77de051931f3d05f659981c64d07c 100644
--- a/SAS/DataManagement/Cleanup/CleanupService/CMakeLists.txt
+++ b/SAS/DataManagement/Cleanup/CleanupService/CMakeLists.txt
@@ -1,6 +1,6 @@
# $Id$
-lofar_package(CleanupService 1.0 DEPENDS PyMessaging DataManagementCommon CleanupCommon)
+lofar_package(CleanupService 1.0 DEPENDS PyMessaging DataManagementCommon CleanupCommon CleanupClient TMSSClient)
lofar_find_package(Python 3.4 REQUIRED)
include(PythonInstall)
diff --git a/SAS/DataManagement/Cleanup/CleanupService/service.py b/SAS/DataManagement/Cleanup/CleanupService/service.py
index 243fc0a5c9679ad6932e619b2134316eeb2a0770..fbe3288b9ebcd923679bbdd42f109f464c1f3e98 100644
--- a/SAS/DataManagement/Cleanup/CleanupService/service.py
+++ b/SAS/DataManagement/Cleanup/CleanupService/service.py
@@ -22,6 +22,10 @@ from lofar.sas.datamanagement.cleanup.config import DEFAULT_CLEANUP_SERVICENAME
from lofar.sas.datamanagement.common.config import DEFAULT_DM_NOTIFICATION_PREFIX
from lofar.sas.datamanagement.storagequery.rpc import StorageQueryRPC
+from lofar.sas.datamanagement.cleanup.rpc import CleanupRPC
+
+from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
+from lofar.sas.tmss.client.tmssbuslistener import *
logger = logging.getLogger(__name__)
@@ -30,7 +34,7 @@ logger = logging.getLogger(__name__)
pinfile = os.path.join(os.environ.get('LOFARROOT', '.'), 'var', 'run', 'auto_cleanup_pinned_tasks.py')
#TODO: this local method is a temporary solution to store the pins in until it can be specified and stored for each task in mom/radb
-def _setTaskDataPinned(otdb_id, pinned=True):
+def _setOTDBTaskDataPinned(otdb_id, pinned=True):
try:
pins = {}
@@ -50,8 +54,8 @@ def _setTaskDataPinned(otdb_id, pinned=True):
logger.error(str(e))
return False
-#TODO: this local method is a temporary solution to store the pins in until it can be specified and stored for each task in mom/radb
-def _isTaskDataPinned(otdb_id):
+#TODO: this local method was a temporary solution to store the pins for otdb tasks. The method can be removed once we use TMSS only.
+def _isOTDBTaskDataPinned(otdb_id):
try:
if os.path.exists(pinfile):
with open(pinfile) as f:
@@ -62,8 +66,8 @@ def _isTaskDataPinned(otdb_id):
return False
-#TODO: this local method is a temporary solution to store the pins in until it can be specified and stored for each task in mom/radb
-def _getPinnedStatuses():
+#TODO: this local method was a temporary solution to store the pins for otdb tasks. The method can be removed once we use TMSS only.
+def _getOTDBPinnedStatuses():
try:
if os.path.exists(pinfile):
with open(pinfile) as f:
@@ -76,16 +80,18 @@ def _getPinnedStatuses():
class CleanupHandler(ServiceMessageHandler):
- def __init__(self, mountpoint=CEP4_DATA_MOUNTPOINT):
+ def __init__(self, mountpoint=CEP4_DATA_MOUNTPOINT, tmss_dbcreds_id: str=None):
super().__init__()
self.mountpoint = mountpoint
self.path_resolver = None
self._sqrpc = None
+ self._tmss_client = TMSSsession.create_from_dbcreds_for_ldap(tmss_dbcreds_id)
def init_service_handler(self, service_name: str):
super().init_service_handler(service_name)
self.register_service_method('GetPathForOTDBId', self.path_resolver.getPathForOTDBId)
+ self.register_service_method('GetPathForTMSSId', self.path_resolver.getPathForTMSSId)
self.register_service_method('RemovePath', self._removePath)
self.register_service_method('RemoveTaskData', self._removeTaskData)
self.register_service_method('SetTaskDataPinned', self._setTaskDataPinned)
@@ -99,26 +105,82 @@ class CleanupHandler(ServiceMessageHandler):
self._sqrpc = StorageQueryRPC.create(exchange=exchange, broker=broker)
def start_handling(self):
- super().start_handling()
+ self._tmss_client.open()
self.path_resolver.open()
self._sqrpc.open()
+ super().start_handling()
logger.info("%s started with projects_path=%s", self, self.path_resolver.projects_path)
def stop_handling(self):
+ super().stop_handling()
self.path_resolver.close()
self._sqrpc.close()
- super().stop_handling()
+ self._tmss_client.close()
- def _setTaskDataPinned(self, otdb_id, pinned=True):
- logger.info('setTaskDataPinned(otdb_id=%s, pinned=%s)', otdb_id, pinned)
- _setTaskDataPinned(otdb_id, pinned)
- self._sendNotification(subject='TaskDataPinned', content={ 'otdb_id':otdb_id, 'pinned': _isTaskDataPinned(otdb_id) })
+ def _setTaskDataPinned(self, otdb_id:int=None, tmss_id:int=None, pinned: bool=True):
+ logger.info('setTaskDataPinned(otdb_id=%s, tmss_id=%s, pinned=%s)', otdb_id, tmss_id, pinned)
+ if otdb_id is not None:
+ _setOTDBTaskDataPinned(otdb_id, pinned)
+ elif tmss_id is not None:
+ subtask = self._tmss_client.get_subtask(tmss_id)
+ self._tmss_client.session.patch(subtask['task_blueprint'], json={'output_pinned': pinned})
- def _isTaskDataPinned(self, otdb_id):
- return { str(otdb_id): _isTaskDataPinned(otdb_id) }
+ self._sendNotification(subject='TaskDataPinned', content={ 'otdb_id':otdb_id, 'tmss_id':tmss_id, 'pinned': self._isTaskDataPinned(otdb_id, tmss_id) })
+
+ def _isTaskDataPinned(self, otdb_id:int, tmss_id:int):
+ # TODO: otdb handling can be removed once we use TMSS only.
+ if otdb_id is not None:
+ return _isOTDBTaskDataPinned(otdb_id)
+
+ subtask = self._tmss_client.get_subtask(tmss_id)
+ task = self._tmss_client.get_url_as_json_object(subtask['task_blueprint'])
+ return task['output_pinned']
def _getPinnedStatuses(self):
- return _getPinnedStatuses()
+ # TODO: otdb handling can be removed once we use TMSS only.
+ # This method is currently only used in the web-scheduler for otdb/mom tasks. No need to TMSS-ify it.
+ return _getOTDBPinnedStatuses()
+
+ def _has_unfinished_non_cleanup_successors(self, otdb_id: int, tmss_id: int) -> bool:
+ # TODO: otdb handling can be removed once we use TMSS only.
+ if otdb_id is not None:
+ radbrpc = self.path_resolver.radbrpc
+ task = radbrpc.getTask(otdb_id=otdb_id)
+ if task:
+ suc_tasks = radbrpc.getTasks(task_ids=task['successor_ids'])
+ unfinished_suc_tasks = [t for t in suc_tasks if not (t['status'] == 'finished' or t['status'] == 'obsolete')]
+ return len(unfinished_suc_tasks)>0
+
+ successors = self._tmss_client.get_subtask_successors(tmss_id)
+ unfinished_successors = [x for x in successors
+ if x['state_value'] not in ('finished', 'cancelled')
+ and x['subtask_type'] != 'cleanup']
+ return len(unfinished_successors) > 0
+
+ def _has_uningested_output_dataproducts(self, otdb_id: int, tmss_id: int) -> bool:
+ # TODO: otdb/mom handling can be removed once we use TMSS only.
+ if otdb_id is not None:
+ radbrpc = self.path_resolver.radbrpc
+ task = radbrpc.getTask(otdb_id=otdb_id)
+ if task:
+ momrpc = self.path_resolver.momrpc
+ dataproducts = momrpc.getDataProducts(task['mom_id']).get(task['mom_id'])
+ ingestable_dataproducts = [dp for dp in dataproducts if dp['status'] not in [None, 'has_data', 'no_data', 'populated'] ]
+ ingested_dataproducts = [dp for dp in ingestable_dataproducts if dp['status'] == 'ingested']
+
+ if len(ingestable_dataproducts) > 0 and len(ingested_dataproducts) < len(ingestable_dataproducts):
+ uningested_dataproducts = [dp for dp in ingestable_dataproducts if dp['status'] != 'ingested']
+ return len(uningested_dataproducts) > 0
+ return False
+
+ subtask = self._tmss_client.get_subtask(tmss_id)
+ subtasks = self._tmss_client.get_subtasks_in_same_scheduling_unit(subtask)
+ for subtask in subtasks:
+ if subtask['subtask_type'] == 'ingest':
+ # TMSS keeps track per dataproduct if it's ingested or not, and translates that into a progress value 0.0 <= p <= 1.0
+ return self._tmss_client.get_subtask_progress(tmss_id)['id'] < 1.0
+
+ return False
def _sendNotification(self, subject, content):
try:
@@ -128,53 +190,49 @@ class CleanupHandler(ServiceMessageHandler):
except Exception as e:
logger.error(str(e))
- def _removeTaskData(self, otdb_id, delete_is=True, delete_cs=True, delete_uv=True, delete_im=True, delete_img=True, delete_pulp=True, delete_scratch=True, force=False):
- logger.info("Remove task data for otdb_id %s, force=%s" % (otdb_id, force))
+ def _removeTaskData(self, otdb_id, tmss_id, delete_is=True, delete_cs=True, delete_uv=True, delete_im=True, delete_img=True, delete_pulp=True, delete_scratch=True, force=False):
+ logger.info("Remove task data for otdb_id=%s, tmss_id=%s force=%s" % (otdb_id, tmss_id, force))
- if not isinstance(otdb_id, int):
+ if otdb_id is not None and not isinstance(otdb_id, int):
message = "Provided otdb_id is not an int"
logger.error(message)
return {'deleted': False, 'message': message}
- self._sendNotification(subject='TaskDeleting', content={ 'otdb_id': otdb_id })
+ if tmss_id is not None and not isinstance(tmss_id, int):
+ message = "Provided tmss_id is not an int"
+ logger.error(message)
+ return {'deleted': False, 'message': message}
- if _isTaskDataPinned(otdb_id):
- message = "Task otdb_id=%s is pinned. Not deleting data." % (otdb_id)
+ self._sendNotification(subject='TaskDeleting', content={ 'otdb_id': otdb_id, 'tmss_id': tmss_id })
+
+ if self._isTaskDataPinned(otdb_id, tmss_id):
+ message = "Task otdb_id=%s tmss_id=%s is pinned. Not deleting data." % (otdb_id, tmss_id)
logger.error(message)
self._sendNotification(subject='TaskDeleted', content={'deleted': False,
'otdb_id': otdb_id,
+ 'tmss_id': tmss_id,
'message': message})
return {'deleted': False, 'message': message}
- radbrpc = self.path_resolver.radbrpc
- task = radbrpc.getTask(otdb_id=otdb_id)
- if task:
- suc_tasks = radbrpc.getTasks(task_ids=task['successor_ids'])
- unfinished_scu_tasks = [t for t in suc_tasks if not (t['status'] == 'finished' or t['status'] == 'obsolete')]
- if unfinished_scu_tasks:
- message = "Task otdb_id=%s has unfinished successor tasks (otdb_ids: %s). Not deleting data." % (task['otdb_id'], [t['otdb_id'] for t in unfinished_scu_tasks])
- logger.error(message)
- self._sendNotification(subject='TaskDeleted', content={'deleted': False,
- 'otdb_id': otdb_id,
- 'message': message})
- return {'deleted': False, 'message': message}
+ if self._has_unfinished_non_cleanup_successors(otdb_id, tmss_id):
+ message = "Task otdb_id=%s tmss_id=%s has unfinished successor tasks. Not deleting data." % (otdb_id, tmss_id)
+ logger.error(message)
+ self._sendNotification(subject='TaskDeleted', content={'deleted': False,
+ 'otdb_id': otdb_id,
+ 'tmss_id': tmss_id,
+ 'message': message})
+ return {'deleted': False, 'message': message}
- momrpc = self.path_resolver.momrpc
- dataproducts = momrpc.getDataProducts(task['mom_id']).get(task['mom_id'])
- ingestable_dataproducts = [dp for dp in dataproducts if dp['status'] not in [None, 'has_data', 'no_data', 'populated'] ]
- ingested_dataproducts = [dp for dp in ingestable_dataproducts if dp['status'] == 'ingested']
+ if not force and self._has_uningested_output_dataproducts(otdb_id, tmss_id):
+ message = "Task otdb_id=%s tmss_id=%s has un-ingested dataproducts. Not deleting data." % (otdb_id, tmss_id)
+ logger.error(message)
+ self._sendNotification(subject='TaskDeleted', content={'deleted': False,
+ 'otdb_id': otdb_id,
+ 'tmss_id': tmss_id,
+ 'message': message})
+ return {'deleted': False, 'message': message}
- if not force:
- if len(ingestable_dataproducts) > 0 and len(ingested_dataproducts) < len(ingestable_dataproducts):
- uningested_dataproducts = [dp for dp in ingestable_dataproducts if dp['status'] != 'ingested']
- message = "Task otdb_id=%s has un-ingested dataproducts. Not deleting data." % (task['otdb_id'],)
- logger.error(message)
- self._sendNotification(subject='TaskDeleted', content={'deleted': False,
- 'otdb_id': otdb_id,
- 'message': message})
- return {'deleted': False, 'message': message}
-
- path_result = self.path_resolver.getPathForOTDBId(otdb_id)
+ path_result = self.path_resolver.getPathForOTDBId(otdb_id) if otdb_id is not None else self.path_resolver.getPathForTMSSId(tmss_id)
if path_result['found']:
rm_results = []
if delete_is and delete_cs and delete_uv and delete_im and delete_img and delete_pulp:
@@ -205,29 +263,37 @@ class CleanupHandler(ServiceMessageHandler):
combined_message = '\n'.join(x.get('message','') for x in rm_results)
if rm_result['deleted'] and not 'does not exist' in combined_message:
- task_type = task.get('type', 'task') if task else 'task'
- rm_result['message'] = 'Deleted %s of data from disk for %s with otdb_id %s\n' % (humanreadablesize(rm_result['size']), task_type, otdb_id)
+ task_type = path_result.get('task',{}).get('type', 'task') if otdb_id else self._tmss_client.get_subtask(tmss_id).get('subtask_type', 'task')
+ rm_result['message'] = 'Deleted %s of data from disk for %s with otdb_id=%s tmss_id=%s\n' % (humanreadablesize(rm_result['size']), task_type, otdb_id, tmss_id)
rm_result['message'] += combined_message
self._sendNotification(subject='TaskDeleted', content={'deleted':rm_result['deleted'],
'otdb_id':otdb_id,
+ 'tmss_id':tmss_id,
'paths': rm_result['paths'],
'message': rm_result['message'],
'size': rm_result['size'],
'size_readable': humanreadablesize(rm_result['size'])})
- self._endStorageResourceClaim(otdb_id)
+ if rm_result['deleted']:
+ self._endStorageResourceClaim(otdb_id=otdb_id, tmss_id=tmss_id)
+
+ if tmss_id is not None:
+ # annotate the dataproducts in tmss that they are deleted
+ dataprodutcs = self._tmss_client.get_subtask_output_dataproducts(tmss_id)
+ for dp in dataprodutcs:
+ self._tmss_client.session.patch(dp['url'], json={'deleted_since': datetime.utcnow().isoformat()})
return rm_result
return {'deleted': False, 'message': path_result['message']}
- def _endStorageResourceClaim(self, otdb_id):
+ def _endStorageResourceClaim(self, otdb_id=None, tmss_id=None):
try:
#check if all data has actually been removed,
#and adjust end time of claim on storage
- path_result = self.path_resolver.getPathForOTDBId(otdb_id)
+ path_result = self.path_resolver.getPathForTask(otdb_id=otdb_id, tmss_id=tmss_id)
if path_result['found']:
path = path_result['path']
@@ -237,7 +303,7 @@ class CleanupHandler(ServiceMessageHandler):
radbrpc = self.path_resolver.radbrpc
storage_resources = radbrpc.getResources(resource_types='storage')
cep4_storage_resource = next(x for x in storage_resources if 'CEP4' in x['name'])
- task = radbrpc.getTask(otdb_id=otdb_id)
+ task = radbrpc.getTask(otdb_id=otdb_id, tmss_id=tmss_id)
if task:
claims = radbrpc.getResourceClaims(task_ids=task['id'], resource_type='storage')
cep4_storage_claim_ids = [c['id'] for c in claims if c['resource_id'] == cep4_storage_resource['id']]
@@ -341,18 +407,161 @@ class CleanupHandler(ServiceMessageHandler):
'message': 'Failed to delete (part of) %s' % path,
'path': path }
-
-
-
-def createService(exchange=DEFAULT_BUSNAME, broker=DEFAULT_BROKER,
- mountpoint=CEP4_DATA_MOUNTPOINT):
+def create_rpc_service(exchange=DEFAULT_BUSNAME, broker=DEFAULT_BROKER,
+ mountpoint=CEP4_DATA_MOUNTPOINT,
+ tmss_dbcreds_id: str=None):
return RPCService(DEFAULT_CLEANUP_SERVICENAME,
handler_type=CleanupHandler,
- handler_kwargs={'mountpoint': mountpoint},
+ handler_kwargs={'mountpoint': mountpoint,
+ 'tmss_dbcreds_id': tmss_dbcreds_id},
exchange=exchange,
broker=broker,
num_threads=4)
+
+class TMSSEventMessageHandlerForCleanup(TMSSEventMessageHandler):
+ def __init__(self, tmss_dbcreds_id: str="TMSSClient", exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER):
+ super().__init__(log_event_messages=False)
+ self._tmss_client = TMSSsession.create_from_dbcreds_for_ldap(tmss_dbcreds_id)
+ self._cleanup_rpc = CleanupRPC.create(exchange=exchange, broker=broker)
+
+ def start_handling(self):
+ self._cleanup_rpc.open()
+ self._tmss_client.open()
+ super().start_handling()
+
+ def stop_handling(self):
+ super().start_handling()
+ self._tmss_client.close()
+ self._cleanup_rpc.close()
+
+ def onSubTaskStatusChanged(self, id: int, status: str):
+ logger.info("onSubTaskStatusChanged: id=%s status=%s", id, status)
+
+ if status in ('scheduled', 'queued', 'finished'):
+ subtask = self._tmss_client.get_subtask(id)
+
+ if subtask['subtask_type'] == 'cleanup':
+ if status == 'scheduled':
+ # a scheduled cleanup subtask should "just be startable",
+ # but we also need to check if the dataproducts are ingested.
+ # So, we change the state to queued,
+ # as a result this method onSubTaskStatusChanged will be called again for the queued status,
+ # and we can check the prerequisites before starting it
+ self.queue_cleanup_subtask_if_prerequisites_met(subtask)
+ elif status == 'queued':
+ self.start_cleanup_subtask_if_prerequisites_met(subtask)
+ elif status == 'started':
+ self.run_cleanup_subtask_if_prerequisites_met(subtask)
+
+ elif subtask['subtask_type'] == 'ingest':
+ if status == 'finished':
+ # when an ingest subtask finishes, then it is safe for the related cleanup subtask(s) to be started
+ subtasks = self._tmss_client.get_subtasks_in_same_scheduling_unit(subtask)
+ cleanup_subtasks = [s for s in subtasks if s['subtask_type'] == 'cleanup' and s['state_value']=='scheduled']
+ for cleanup_subtask in cleanup_subtasks:
+ self.queue_cleanup_subtask_if_prerequisites_met(cleanup_subtask)
+
+ def queue_cleanup_subtask_if_prerequisites_met(self, subtask: dict):
+ logger.debug("queue_cleanup_subtask_if_prerequisites_met: subtask id=%s type=%s status=%s", subtask['id'], subtask['subtask_type'], subtask['state_value'])
+
+ # check prerequisites
+ if subtask['subtask_type'] != 'cleanup':
+ # skip non-cleanup subtasks
+ return
+
+ if subtask['state_value'] != 'scheduled':
+ # skip cleanup subtasks which are not scheduled
+ return
+
+ # when an ingest subtask finishes, then it is safe for the related cleanup subtask(s) to be started
+ subtasks = self._tmss_client.get_subtasks_in_same_scheduling_unit(subtask)
+ ingest_subtasks = [s for s in subtasks if s['subtask_type'] == 'ingest']
+ unfinished_ingest_subtasks = [s for s in ingest_subtasks if s['state_value'] != 'finished']
+
+ if len(unfinished_ingest_subtasks) > 0:
+ logger.info("cleanup subtask id=%s is scheduled, but waiting for ingest id=%s to finish before queueing the cleanup subtask...",
+ subtask['id'], [s['id'] for s in unfinished_ingest_subtasks])
+ return
+
+ logger.info("cleanup subtask id=%s is scheduled, and all ingest subtasks id=%s are finished. queueing the cleanup subtask...",
+ subtask['id'], [s['id'] for s in ingest_subtasks])
+
+ self._tmss_client.set_subtask_status(subtask['id'], 'queueing')
+ self._tmss_client.set_subtask_status(subtask['id'], 'queued')
+ # as a result of setting the queued state, start_cleanup_subtask_if_prerequisites_met is called in onSubTaskStatusChanged
+
+
+ def start_cleanup_subtask_if_prerequisites_met(self, subtask: dict):
+ logger.debug("start_cleanup_subtask_if_prerequisites_met: subtask id=%s type=%s status=%s", subtask['id'], subtask['subtask_type'], subtask['state_value'])
+
+ # check prerequisites
+ if subtask['subtask_type'] != 'cleanup':
+ # skip non-cleanup subtasks
+ return
+
+ if subtask['state_value'] != 'queued':
+ # skip cleanup subtasks which are not queued
+ return
+
+ # prerequisites are met. Proceed.
+ logger.info("starting cleanup subtask id=%s...", subtask['id'])
+ self._tmss_client.set_subtask_status(subtask['id'], 'starting')
+ self._tmss_client.set_subtask_status(subtask['id'], 'started')
+
+ predecessors = self._tmss_client.get_subtask_predecessors(subtask['id'])
+ results = []
+
+ for predecessor in predecessors:
+ logger.info("cleanup subtask id=%s removing output data for subtask id=%s ...", subtask['id'], predecessor['id'])
+ result = self._cleanup_rpc.removeTaskData(tmss_id=predecessor['id'])
+ results.append(result)
+ logger.info("cleanup subtask id=%s: %s", subtask['id'], result.get('message',""))
+
+ if any([not r['deleted'] for r in results]):
+ self._tmss_client.set_subtask_status(subtask['id'], 'error')
+ else:
+ self._tmss_client.set_subtask_status(subtask['id'], 'finishing')
+ self._tmss_client.set_subtask_status(subtask['id'], 'finished')
+
+ def run_cleanup_subtask_if_prerequisites_met(self, subtask: dict):
+ logger.debug("run_cleanup_subtask_if_prerequisites_met: subtask id=%s type=%s status=%s", subtask['id'], subtask['subtask_type'], subtask['state_value'])
+
+ # check prerequisites
+ if subtask['subtask_type'] != 'cleanup':
+ # skip non-cleanup subtasks
+ return
+
+ if subtask['state_value'] != 'started':
+ # skip cleanup subtasks which are not queued
+ return
+
+ # prerequisites are met. Proceed.
+ logger.info("running cleanup subtask id=%s...", subtask['id'])
+
+ predecessors = self._tmss_client.get_subtask_predecessors(subtask['id'])
+ results = []
+
+ for predecessor in predecessors:
+ logger.info("cleanup subtask id=%s removing output data for subtask id=%s ...", subtask['id'], predecessor['id'])
+ result = self._cleanup_rpc.removeTaskData(tmss_id=predecessor['id'])
+ results.append(result)
+ logger.info("cleanup subtask id=%s: %s", subtask['id'], result.get('message',""))
+
+ if any([not r['deleted'] for r in results]):
+ self._tmss_client.set_subtask_status(subtask['id'], 'error')
+ else:
+ self._tmss_client.set_subtask_status(subtask['id'], 'finishing')
+ self._tmss_client.set_subtask_status(subtask['id'], 'finished')
+
+def create_tmss_buslistener(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER, tmss_dbcreds_id: str="TMSSClient"):
+ return TMSSBusListener(handler_type=TMSSEventMessageHandlerForCleanup,
+ handler_kwargs={'tmss_dbcreds_id': tmss_dbcreds_id,
+ 'exchange': exchange,
+ 'broker': broker},
+ exchange=exchange, broker=broker)
+
+
def main():
# make sure we run in UTC timezone
import os
@@ -366,15 +575,16 @@ def main():
parser.add_option("-e", "--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME,
help="Name of the bus exchange on the broker, [default: %default]")
parser.add_option("--mountpoint", dest="mountpoint", type="string", default=CEP4_DATA_MOUNTPOINT, help="path of local cep4 mount point, default: %default")
+ parser.add_option('-R', '--rest_credentials', dest='rest_credentials', type='string', default='TMSSClient', help='TMSS django REST API credentials name, default: %default')
parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
(options, args) = parser.parse_args()
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG if options.verbose else logging.INFO)
- with createService(exchange=options.exchange,
- broker=options.broker):
- waitForInterrupt()
+ with create_rpc_service(exchange=options.exchange, broker=options.broker, tmss_dbcreds_id=options.rest_credentials, mountpoint=options.mountpoint):
+ with create_tmss_buslistener(exchange=options.exchange, broker=options.broker, tmss_dbcreds_id=options.rest_credentials):
+ waitForInterrupt()
if __name__ == '__main__':
main()
diff --git a/SAS/DataManagement/Cleanup/CleanupService/test/CMakeLists.txt b/SAS/DataManagement/Cleanup/CleanupService/test/CMakeLists.txt
index 70fad49899886146924be3dc84ad2081fb47209b..f1124d403bd3dbe7c483dee67f49f41f9a7866a9 100644
--- a/SAS/DataManagement/Cleanup/CleanupService/test/CMakeLists.txt
+++ b/SAS/DataManagement/Cleanup/CleanupService/test/CMakeLists.txt
@@ -3,3 +3,8 @@ include(LofarCTest)
lofar_add_test(test_cleanup_service_and_rpc)
+IF(BUILD_TMSSBackend)
+ lofar_add_test(t_cleanup_tmss_integration_test)
+ELSE()
+ message(WARNING "Skipping t_cleanup_tmss_integration_test because it depends on the TMSSBackend package which is not included in the build")
+ENDIF(BUILD_TMSSBackend)
diff --git a/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.py b/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.py
new file mode 100755
index 0000000000000000000000000000000000000000..fc33cc56e106133760f46e89aa2a64374b6febe2
--- /dev/null
+++ b/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.py
@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+
+import unittest
+
+import logging
+logger = logging.getLogger('lofar.'+__name__)
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+from lofar.messaging.messagebus import TemporaryExchange, BusListenerJanitor
+from lofar.common.test_utils import integration_test
+
+from datetime import datetime
+from uuid import uuid4
+import threading
+import os
+from unittest import mock
+
+@integration_test
+class TestCleanupTMSSIntegration(unittest.TestCase):
+ def setUp(self) -> None:
+ self.TEST_DIR = '/tmp/cleanup_tmss_integration_test/' + str(uuid4())
+ os.makedirs(self.TEST_DIR)
+
+ # mockpatch the ssh calls which are issued from the cleanup subtask normally to cep4.
+ # in this test we just keep the original command without the ssh
+ ssh_cmd_list_patcher1 = mock.patch('lofar.common.ssh_utils.ssh_cmd_list')
+ self.addCleanup(ssh_cmd_list_patcher1.stop)
+ self.ssh_cmd_list_mock1 = ssh_cmd_list_patcher1.start()
+ self.ssh_cmd_list_mock1.side_effect = lambda host, user: []
+
+ def tearDown(self) -> None:
+ import shutil
+ shutil.rmtree(self.TEST_DIR, ignore_errors=True)
+
+ def test(self):
+ with TemporaryExchange("TestCleanupTMSSIntegration") as tmp_exchange:
+ # override DEFAULT_BUSNAME
+ import lofar
+ lofar.messaging.config.DEFAULT_BUSNAME = tmp_exchange.address
+
+ from lofar.sas.tmss.test.test_utils import TMSSTestEnvironment
+ with TMSSTestEnvironment(exchange=tmp_exchange.address,
+ populate_schemas=True, start_ra_test_environment=True, start_postgres_listener=True,
+ populate_test_data=False, enable_viewflow=False, start_dynamic_scheduler=False,
+ start_subtask_scheduler=True, start_workflow_service=False) as tmss_test_env:
+
+ from lofar.sas.tmss.tmss.tmssapp import models
+ from lofar.sas.tmss.tmss.tmssapp.tasks import create_task_blueprints_and_subtasks_from_scheduling_unit_draft, schedule_independent_subtasks_in_scheduling_unit_blueprint
+ from lofar.sas.tmss.test.test_utils import create_scheduling_unit_blueprint_simulator
+ from lofar.sas.tmss.test.tmss_test_data_django_models import SchedulingUnitDraft_test_data, SchedulingSet_test_data
+ from lofar.common.json_utils import add_defaults_to_json_object_for_schema
+ from lofar.sas.tmss.client.tmssbuslistener import TMSSEventMessageHandler, TMSSBusListener
+
+ scheduling_set = models.SchedulingSet.objects.create(**SchedulingSet_test_data())
+ scheduling_set.project.auto_ingest = False # for user granting permission (in this test the simulator does that for us)
+ scheduling_set.project.save()
+
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Short Test Observation - Pipeline - Ingest")
+ scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template, strategy_template.scheduling_unit_template.schema)
+ scheduling_unit_spec['tasks']['Observation']['specifications_doc']['SAPs'][0]['subbands'] = [0,1] #limit nr of subbands for readability
+ scheduling_unit_spec['tasks']['Observation']['specifications_doc']['QA']['plots']['enabled'] = False
+ scheduling_unit_spec['tasks']['Observation']['specifications_doc']['QA']['file_conversion']['enabled'] = False
+
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(**SchedulingUnitDraft_test_data(template=strategy_template.scheduling_unit_template,
+ requirements_doc=scheduling_unit_spec,
+ scheduling_set=scheduling_set))
+
+ scheduling_unit = create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+ schedule_independent_subtasks_in_scheduling_unit_blueprint(scheduling_unit, datetime.utcnow())
+
+ # make sure each dataproduct uses TEST_DATA_DIR as root
+ for task in scheduling_unit.task_blueprints.all():
+ for subtask in task.subtasks.all():
+ if subtask.state.value == models.SubtaskState.Choices.SCHEDULED.value:
+ for output_dp in subtask.output_dataproducts.all():
+ output_dp.directory = output_dp.directory.replace('/data', self.TEST_DIR)
+ output_dp.save()
+
+ # ensure/check the data dir is empty at the start
+ self.assertEqual([], os.listdir(self.TEST_DIR))
+
+ class TestEventhandler(TMSSEventMessageHandler):
+ """This test-TMSSEventMessageHandler tracks the interesing subtask status changes and determines
+ if the dataproducts were first written by the obs/pipeline and then deleted by the cleanuptask"""
+ def __init__(self, sync_object:{}):
+ self._sync_object = sync_object
+ super().__init__()
+
+ def onSubTaskStatusChanged(self, id: int, status: str):
+ if status=='starting':
+ subtask = models.Subtask.objects.get(id=id)
+ if subtask.specifications_template.type.value == models.SubtaskType.Choices.CLEANUP.value:
+ logger.info("subtask %s %s starting", id, subtask.specifications_template.type.value)
+
+ # from lofar.common.util import waitForInterrupt
+ # waitForInterrupt()
+
+ self._sync_object['cleanup_sees_written_files'] = subtask.input_dataproducts.count() > 0 and \
+ all(os.path.exists(dp.filepath) and os.path.getsize(dp.filepath) > 0
+ for dp in subtask.input_dataproducts.all())
+ elif status=='finished':
+ subtask = models.Subtask.objects.get(id=id)
+ logger.info("subtask %s %s finished", id, subtask.specifications_template.type.value)
+
+ subtask_did_write_files = all(os.path.exists(dp.filepath) and os.path.getsize(dp.filepath) > 0
+ for dp in subtask.output_dataproducts.all())
+
+ if subtask.specifications_template.type.value == models.SubtaskType.Choices.OBSERVATION.value:
+ self._sync_object['observation_did_write_files'] = subtask_did_write_files
+ elif subtask.specifications_template.type.value == models.SubtaskType.Choices.PIPELINE.value:
+ self._sync_object['pipeline_did_write_files'] = subtask_did_write_files
+ elif subtask.specifications_template.type.value == models.SubtaskType.Choices.CLEANUP.value:
+ self._sync_object['cleanup_deleted_written_files'] = not any(os.path.exists(dp.filepath) and os.path.getsize(dp.filepath) > 0
+ for dp in subtask.input_dataproducts.all())
+ # signal simulator and test-method that we are done
+ self._sync_object['stop_event'].set()
+
+ # helper object to communicate events/results
+ sync_object = {'observation_did_write_files': False,
+ 'pipeline_did_write_files': False,
+ 'cleanup_sees_written_files': False,
+ 'cleanup_deleted_written_files': False,
+ 'stop_event': threading.Event()}
+
+ with BusListenerJanitor(TMSSBusListener(handler_type=TestEventhandler, exchange=tmp_exchange.address, handler_kwargs={'sync_object': sync_object})):
+ # start a simulator, forcing the scheduling_unit to "run" the observations, pipelines, ingest....
+ # and let the cleanup server act on the eventmessages.
+ # as a result, the scheduling_unit should be finished at the end, and the dataproducts should be "cleaned up"
+
+ # check that the cleanup task is defined and ready to be used
+ cleanup_subtask = models.Subtask.objects.get(task_blueprint__scheduling_unit_blueprint__id=scheduling_unit.id, specifications_template__type__value=models.SubtaskType.Choices.CLEANUP.value)
+ self.assertEqual("defined", cleanup_subtask.state.value)
+
+ # check that the sync-results are in initial state.
+ # nobody wrote any files yet, and nothing was deleted yet.
+ self.assertFalse(sync_object['observation_did_write_files'])
+ self.assertFalse(sync_object['pipeline_did_write_files'])
+ self.assertFalse(sync_object['cleanup_sees_written_files'])
+ self.assertFalse(sync_object['cleanup_deleted_written_files'])
+
+ # start the objects-under-test: the cleanup service
+ # this service should respond to subtask events, and take care of the cleanup at the right moment.
+ from lofar.sas.datamanagement.cleanup.service import create_tmss_buslistener, create_rpc_service
+ with create_rpc_service(exchange=tmp_exchange.address, tmss_dbcreds_id=tmss_test_env.client_credentials.dbcreds_id, mountpoint=self.TEST_DIR):
+ with create_tmss_buslistener(exchange=tmp_exchange.address, tmss_dbcreds_id=tmss_test_env.client_credentials.dbcreds_id):
+ # simulate the obs/pipeline/ingest...
+ # allowing the cleanup service to handle the events and cleanup the obs/pipeline output
+ with create_scheduling_unit_blueprint_simulator(scheduling_unit.id,
+ sync_object['stop_event'],
+ handle_cleanup=False, handle_ingest=True,
+ handle_observations=True, handle_QA=True,
+ handle_pipelines=True,
+ create_output_dataproducts=True,
+ auto_grant_ingest_permission=True,
+ delay=0, duration=0,
+ exchange=tmp_exchange.address):
+
+ # wait until scheduling_unit including the cleanup task is done
+ # the actual tests are done in the TestEventhandler above, setting their results in the sync_object
+ self.assertTrue(sync_object['stop_event'].wait(300))
+
+ # check states
+ cleanup_subtask.refresh_from_db()
+ self.assertEqual("finished", cleanup_subtask.state.value)
+ scheduling_unit.refresh_from_db()
+ self.assertEqual("finished", scheduling_unit.status)
+
+ # check that the files were written and deleted
+ self.assertTrue(sync_object['observation_did_write_files'])
+ self.assertTrue(sync_object['pipeline_did_write_files'])
+ self.assertTrue(sync_object['cleanup_sees_written_files'])
+ self.assertTrue(sync_object['cleanup_deleted_written_files'])
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.run b/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.run
new file mode 100755
index 0000000000000000000000000000000000000000..909e0b819d34e37e6205d6369c8cb0df1107436d
--- /dev/null
+++ b/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.run
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+python3 t_cleanup_tmss_integration_test.py
+
diff --git a/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.sh b/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.sh
new file mode 100755
index 0000000000000000000000000000000000000000..413a9673c1dba3c644bc04b2badeac2f5e7c8094
--- /dev/null
+++ b/SAS/DataManagement/Cleanup/CleanupService/test/t_cleanup_tmss_integration_test.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_cleanup_tmss_integration_test
diff --git a/SAS/DataManagement/Cleanup/CleanupService/test/test_cleanup_service_and_rpc.py b/SAS/DataManagement/Cleanup/CleanupService/test/test_cleanup_service_and_rpc.py
index 5e28031cec8909f04098279fd098750f79c9a1c6..0142d880842209912b03eeb3f0c4f4fe850d1e67 100755
--- a/SAS/DataManagement/Cleanup/CleanupService/test/test_cleanup_service_and_rpc.py
+++ b/SAS/DataManagement/Cleanup/CleanupService/test/test_cleanup_service_and_rpc.py
@@ -41,7 +41,7 @@ class CleanupServiceRpcTest(unittest.TestCase):
#mock_momrpc.getObjectDetails.return_value = {'1000042': {'project_name': 'my_project'}}
## now that we have a mocked the external dependencies, import cleanupservice
- #from lofar.sas.datamanagement.cleanup.service import createService
+ #from lofar.sas.datamanagement.cleanup.service import create_rpc_service
#from lofar.sas.datamanagement.cleanup.rpc import CleanupRPC
#class TestCleanupServiceAndRPC(unittest.TestCase):
@@ -102,7 +102,7 @@ class CleanupServiceRpcTest(unittest.TestCase):
#self.assertTrue('Could not find task' in result['message'])
## create and run the service
- #with createService(busname=busname):
+ #with create_rpc_service(busname=busname):
## and run all tests
#unittest.main()
diff --git a/SAS/DataManagement/DataManagementCommon/CMakeLists.txt b/SAS/DataManagement/DataManagementCommon/CMakeLists.txt
index 5c160faa9b105d0325130a1f10e2f6ff86b433e5..5e0c0554e1ef45dcdab16bcbcda2d331c336a8b3 100644
--- a/SAS/DataManagement/DataManagementCommon/CMakeLists.txt
+++ b/SAS/DataManagement/DataManagementCommon/CMakeLists.txt
@@ -1,6 +1,6 @@
# $Id$
-lofar_package(DataManagementCommon 1.0 DEPENDS PyMessaging ResourceAssignmentService MoMQueryServiceClient)
+lofar_package(DataManagementCommon 1.0 DEPENDS PyMessaging ResourceAssignmentService MoMQueryServiceClient TMSSClient)
lofar_find_package(Python 3.4 REQUIRED)
include(PythonInstall)
diff --git a/SAS/DataManagement/DataManagementCommon/getPathForTask b/SAS/DataManagement/DataManagementCommon/getPathForTask
old mode 100644
new mode 100755
diff --git a/SAS/DataManagement/DataManagementCommon/path.py b/SAS/DataManagement/DataManagementCommon/path.py
index 36c15d93513d97b9ce8310cc47c5196370ad50a3..6bdcae38744c1420eaa9799a3a40ac6df0d13af8 100644
--- a/SAS/DataManagement/DataManagementCommon/path.py
+++ b/SAS/DataManagement/DataManagementCommon/path.py
@@ -17,6 +17,7 @@ from lofar.sas.datamanagement.common.config import CEP4_DATA_MOUNTPOINT
from lofar.sas.resourceassignment.resourceassignmentservice.rpc import RADBRPC
from lofar.mom.momqueryservice.momqueryrpc import MoMQueryRPC
+from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
logger = logging.getLogger(__name__)
@@ -24,7 +25,8 @@ class PathResolver:
def __init__(self,
mountpoint=CEP4_DATA_MOUNTPOINT,
exchange=DEFAULT_BUSNAME,
- broker=DEFAULT_BROKER):
+ broker=DEFAULT_BROKER,
+ tmss_dbcreds_id: str=None):
self.mountpoint = mountpoint
self.projects_path = os.path.join(self.mountpoint, 'projects' if isProductionEnvironment() else 'test-projects')
@@ -33,14 +35,17 @@ class PathResolver:
self.radbrpc = RADBRPC.create(exchange=exchange, broker=broker)
self.momrpc = MoMQueryRPC.create(exchange=exchange, broker=broker)
+ self._tmss_client = TMSSsession.create_from_dbcreds_for_ldap(tmss_dbcreds_id)
def open(self):
self.radbrpc.open()
self.momrpc.open()
+ self._tmss_client.open()
def close(self):
self.radbrpc.close()
self.momrpc.close()
+ self._tmss_client.close()
def __enter__(self):
self.open()
@@ -61,47 +66,76 @@ class PathResolver:
logger.debug("Get path for otdb_id %s" % (otdb_id,))
return self.getPathForTask(otdb_id=otdb_id)
- def getPathForTask(self, radb_id=None, mom_id=None, otdb_id=None, include_scratch_paths=True):
- logger.info("getPathForTask(radb_id=%s, mom_id=%s, otdb_id=%s)", radb_id, mom_id, otdb_id)
- '''get the path for a task for either the given radb_id, or for the given mom_id, or for the given otdb_id'''
- result = self._getProjectPathAndDetails(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id)
+ def getPathForTMSSId(self, tmss_id):
+ logger.debug("Get path for tmss_id %s" % (tmss_id,))
+ return self.getPathForTask(tmss_id=tmss_id)
+
+ def getPathForTask(self, radb_id=None, mom_id=None, otdb_id=None, tmss_id=None, include_scratch_paths=True):
+ logger.info("getPathForTask(radb_id=%s, mom_id=%s, otdb_id=%s, tmss_id=%s)", radb_id, mom_id, otdb_id, tmss_id)
+ '''get the path for a task for either the given radb_id, or for the given mom_id, or for the given otdb_id, or for the given tmss_id'''
+ result = self._getProjectPathAndDetails(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id, tmss_id=tmss_id)
if result['found']:
project_path = result['path']
- task = result['task']
- task_data_path = os.path.join(project_path, 'L%s' % task['otdb_id'])
- logger.info("constructed path '%s' for otdb_id=%s mom_id=%s radb_id=%s" % (task_data_path, task['otdb_id'], task['mom_id'], task['id']))
- path_result = {'found': True, 'message': '', 'path': task_data_path,
- 'radb_id': task.get('id'), 'mom_id': task.get('mom_id'), 'otdb_id': task.get('otdb_id')}
+ if 'task' in result:
+ task = result['task']
+ task_data_path = os.path.join(project_path, 'L%s' % task['otdb_id'])
+ elif tmss_id is not None:
+ task_data_path = os.path.join(project_path, 'L%s' % tmss_id)
+ else:
+ task_data_path = None
- if include_scratch_paths and task['type'] == 'pipeline':
- path_result['scratch_paths'] = []
+ path_result = {'found': task_data_path is not None, 'message': '', 'path': task_data_path,
+ 'radb_id': radb_id, 'mom_id': mom_id, 'otdb_id': otdb_id, 'tmss_id': tmss_id}
- scratch_path = os.path.join(self.scratch_path, 'Observation%s' % task['otdb_id'])
- share_path = os.path.join(self.share_path, 'Observation%s' % task['otdb_id'])
- logger.info("Checking scratch paths %s %s for otdb_id=%s mom_id=%s radb_id=%s" % (scratch_path, share_path, task['otdb_id'], task['mom_id'], task['id']))
+ logger.info("constructed path '%s' for otdb_id=%s mom_id=%s radb_id=%s tmss_id=%s" % (task_data_path, otdb_id, mom_id, radb_id, tmss_id))
+
+ if include_scratch_paths:
+ path_result['scratch_paths'] = []
- if self.pathExists(scratch_path):
- path_result['scratch_paths'].append(scratch_path)
+ if 'task' in result and task['type'] == 'pipeline':
+ task = result['task']
+ path_result['scratch_paths'].append(os.path.join(self.scratch_path, 'Observation%s' % task['otdb_id']))
+ path_result['scratch_paths'].append(os.path.join(self.share_path, 'Observation%s' % task['otdb_id']))
+ elif tmss_id is not None:
+ subtask = self._tmss_client.get_subtask(tmss_id)
+ if subtask['subtask_type'].lower() == 'pipeline':
+ path_result['scratch_paths'].append(os.path.join(self.scratch_path, 'Observation%s' % tmss_id))
+ path_result['scratch_paths'].append(os.path.join(self.share_path, 'Observation%s' % tmss_id))
- if self.pathExists(share_path):
- path_result['scratch_paths'].append(share_path)
+ logger.info("Checking scratch paths %s for otdb_id=%s mom_id=%s radb_id=%s tmss_id=%s" % (path_result['scratch_paths'], otdb_id, mom_id, radb_id, tmss_id))
+ path_result['scratch_paths'] = [path for path in path_result['scratch_paths'] if self.pathExists(path)]
- logger.info("result for getPathForTask(radb_id=%s, mom_id=%s, otdb_id=%s): %s", radb_id, mom_id, otdb_id, path_result)
+ logger.info("result for getPathForTask(radb_id=%s, mom_id=%s, otdb_id=%s, tmss_id=%s): %s", radb_id, mom_id, otdb_id, tmss_id, path_result)
return path_result
result = {'found': False, 'message': result.get('message', ''), 'path': '',
- 'radb_id': radb_id, 'mom_id': mom_id, 'otdb_id': otdb_id}
- logger.warn("result for getPathForTask(radb_id=%s, mom_id=%s, otdb_id=%s): %s", radb_id, mom_id, otdb_id, result)
+ 'radb_id': radb_id, 'mom_id': mom_id, 'otdb_id': otdb_id, 'tmss_id': tmss_id}
+ logger.warning("result for getPathForTask(radb_id=%s, mom_id=%s, otdb_id=%s, tmss_id=%s): %s", radb_id, mom_id, otdb_id, tmss_id, result)
return result
- def _getProjectPathAndDetails(self, radb_id=None, mom_id=None, otdb_id=None):
- '''get the project path and details of a task for either the given radb_id, or for the given mom_id, or for the given otdb_id'''
- ids = [radb_id, mom_id, otdb_id]
+ def _getProjectPathAndDetails(self, radb_id=None, mom_id=None, otdb_id=None, tmss_id=None):
+ '''get the project path and details of a task for either the given radb_id, or for the given mom_id, or for the given otdb_id, or for the given tmss_id'''
+ ids = [radb_id, mom_id, otdb_id, tmss_id]
validIds = [x for x in ids if x != None and isinstance(x, int)]
if len(validIds) != 1:
- raise KeyError("Provide one and only one id: radb_id=%s, mom_id=%s, otdb_id=%s" % (radb_id, mom_id, otdb_id))
+ raise KeyError("Provide one and only one id: radb_id=%s, mom_id=%s, otdb_id=%s, tmss_id=%s" % (radb_id, mom_id, otdb_id, tmss_id))
+
+ if tmss_id is not None:
+ output_dataproducts = self._tmss_client.get_subtask_output_dataproducts(tmss_id)
+ directories = set([dp['directory'] for dp in output_dataproducts])
+ subtask_dir_name = 'L%s' % (tmss_id,)
+ # extract the project path
+ project_paths = [dir[:dir.find(subtask_dir_name)] for dir in directories]
+
+ if len(project_paths) != 1:
+ message = "Could not determine project path for tmss_id=%s" % (tmss_id,)
+ logger.error(message)
+ return {'found': False, 'message': message, 'path': None}
+
+ project_path = project_paths[0]
+ return {'found': True, 'path': project_path}
task = self.radbrpc.getTask(id=radb_id, mom_id=mom_id, otdb_id=otdb_id)
@@ -125,22 +159,24 @@ class PathResolver:
project_path = os.path.join(self.projects_path, "_".join(project_name.split()))
return {'found': True, 'path': project_path, 'mom_details':mom_details, 'task':task}
- def getProjectPath(self, radb_id=None, mom_id=None, otdb_id=None):
- result = self._getProjectPathAndDetails(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id)
+ def getProjectPath(self, radb_id=None, mom_id=None, otdb_id=None, tmss_id=None):
+ result = self._getProjectPathAndDetails(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id, tmss_id=tmss_id)
if result['found']:
- del result['mom_details']
- del result['task']
+ if 'mom_details' in result:
+ del result['mom_details']
+ if 'task' in result:
+ del result['task']
return result
- def getProjectDirAndSubDirectories(self, radb_id=None, mom_id=None, otdb_id=None, project_name=None):
+ def getProjectDirAndSubDirectories(self, radb_id=None, mom_id=None, otdb_id=None, tmss_id=None, project_name=None):
'''get the project directory and its subdirectories of either the project_name, or the task's project for either the given radb_id, or for the given mom_id, or for the given otdb_id'''
if project_name:
project_path = os.path.join(self.projects_path, "_".join(project_name.split()))
return self.getSubDirectories(project_path)
- result = self.getProjectPath(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id)
+ result = self.getProjectPath(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id, tmss_id=tmss_id)
if result['found']:
return self.getSubDirectories(result['path'])
return result
@@ -154,8 +190,11 @@ class PathResolver:
def getSubDirectoriesForRADBId(self, radb_id):
return self.getSubDirectoriesForTask(radb_id=radb_id)
- def getSubDirectoriesForTask(self, radb_id=None, mom_id=None, otdb_id=None):
- result = self.getPathForTask(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id)
+ def getSubDirectoriesForTMSSId(self, tmss_id):
+ return self.getSubDirectoriesForTask(tmss_id=tmss_id)
+
+ def getSubDirectoriesForTask(self, radb_id=None, mom_id=None, otdb_id=None, tmss_id=None):
+ result = self.getPathForTask(radb_id=radb_id, mom_id=mom_id, otdb_id=otdb_id, tmss_id=tmss_id)
if result['found']:
return self.getSubDirectories(result['path'])
return result
@@ -214,13 +253,15 @@ def main():
parser.add_option('-o', '--otdb_id', dest='otdb_id', type='int', default=None, help='otdb_id of task to get the path for')
parser.add_option('-m', '--mom_id', dest='mom_id', type='int', default=None, help='mom_id of task to get the path for')
parser.add_option('-r', '--radb_id', dest='radb_id', type='int', default=None, help='radb_id of task to get the path for')
+ parser.add_option('-t', '--tmss_id', dest='tmss_id', type='int', default=None, help='tmss_id of the TMSS subtask to get the path for')
parser.add_option('-q', '--broker', dest='broker', type='string', default=DEFAULT_BROKER, help='Address of the broker, default: localhost')
parser.add_option("--mountpoint", dest="mountpoint", type="string", default=CEP4_DATA_MOUNTPOINT, help="path of local cep4 mount point, default: %default")
parser.add_option("--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME, help="Name of the exchange on which the services listen, default: %default")
+ parser.add_option('-R', '--rest_credentials', dest='rest_credentials', type='string', default='TMSSClient', help='TMSS django REST API credentials name, default: %default')
parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
(options, args) = parser.parse_args()
- if not (options.otdb_id or options.mom_id or options.radb_id):
+ if not (options.otdb_id or options.mom_id or options.radb_id or options.tmss_id):
parser.print_help()
exit(1)
@@ -230,7 +271,7 @@ def main():
with PathResolver(exchange=options.exchange, broker=options.broker) as path_resolver:
if options.path:
- result = path_resolver.getPathForTask(otdb_id=options.otdb_id, mom_id=options.mom_id, radb_id=options.radb_id)
+ result = path_resolver.getPathForTask(otdb_id=options.otdb_id, mom_id=options.mom_id, radb_id=options.radb_id, tmss_id=options.tmss_id)
if result['found']:
print("path: %s" % (result['path']))
else:
@@ -238,7 +279,7 @@ def main():
exit(1)
if options.project:
- result = path_resolver.getProjectDirAndSubDirectories(otdb_id=options.otdb_id, mom_id=options.mom_id, radb_id=options.radb_id)
+ result = path_resolver.getProjectDirAndSubDirectories(otdb_id=options.otdb_id, mom_id=options.mom_id, radb_id=options.radb_id, tmss_id=options.tmss_id)
if result['found']:
print("projectpath: %s" % (result['path']))
print("subdirectories: %s" % (' '.join(result['sub_directories'])))
@@ -247,7 +288,7 @@ def main():
exit(1)
if options.subdirs:
- result = path_resolver.getSubDirectoriesForTask(otdb_id=options.otdb_id, mom_id=options.mom_id, radb_id=options.radb_id)
+ result = path_resolver.getSubDirectoriesForTask(otdb_id=options.otdb_id, mom_id=options.mom_id, radb_id=options.radb_id, tmss_id=options.tmss_id)
if result['found']:
print("path: %s" % (result['path']))
print("subdirectories: %s" % (' '.join(result['sub_directories'])))
diff --git a/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py b/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py
index b8831d7759b9433108322e26254abd5b5586f317..85e452ae48330a0ca82348f8dddf3805ce34ae2f 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/constraints/__init__.py
@@ -87,7 +87,7 @@ def filter_scheduling_units_using_constraints(scheduling_units: [models.Scheduli
# or submit a feature request to implement constraint solvers for this new template.
logger.warning(e)
for subtask in models.Subtask.independent_subtasks().filter(task_blueprints__scheduling_unit_blueprint_id=scheduling_unit.id).all():
- subtask.status = models.SubtaskState.objects.get(value=models.SubtaskState.Choices.ERROR.value)
+ subtask.status = models.SubtaskState.objects.get(value=models.SubtaskState.Choices.UNSCHEDULABLE.value)
subtask.save()
# if we have schedulingunit(s) that can run exclusively in this time window (and not afterwards), then return only these.
@@ -152,7 +152,7 @@ def sort_scheduling_units_scored_by_constraints(scheduling_units: [models.Schedu
# or submit a feature request to implement constraint solvers for this new template.
logger.warning(e)
for subtask in models.Subtask.independent_subtasks().filter(task_blueprints__scheduling_unit_blueprint_id=scheduling_unit.id).all():
- subtask.status = models.SubtaskState.objects.get(value=models.SubtaskState.Choices.ERROR.value)
+ subtask.status = models.SubtaskState.objects.get(value=models.SubtaskState.Choices.UNSCHEDULABLE.value)
subtask.save()
return sorted(scored_scheduling_units, key=lambda x: x.weighted_score, reverse=True)
diff --git a/SAS/TMSS/backend/services/scheduling/lib/subtask_scheduling.py b/SAS/TMSS/backend/services/scheduling/lib/subtask_scheduling.py
index d6909516fe3c6bf2417c382ec7f1322923b8744c..4ca2887f4bc7ce9c82fa6068964db11081cb4e85 100644
--- a/SAS/TMSS/backend/services/scheduling/lib/subtask_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/subtask_scheduling.py
@@ -96,8 +96,9 @@ class TMSSSubTaskSchedulingEventMessageHandler(TMSSEventMessageHandler):
if subtask['state_value'] == 'defined':
subtask_template = self.tmss_client.get_url_as_json_object(subtask['specifications_template'])
if subtask_template['type_value'] == 'ingest':
- logger.info("trying to schedule ingest subtask id=%s for scheduling_unit_blueprint id=%s...", subtask['id'], id)
- self.tmss_client.schedule_subtask(subtask['id'])
+ if all(pred['state_value'] == 'finished' for pred in self.tmss_client.get_subtask_predecessors(subtask['id'])):
+ logger.info("trying to schedule ingest subtask id=%s for scheduling_unit_blueprint id=%s...", subtask['id'], id)
+ self.tmss_client.schedule_subtask(subtask['id'])
def create_subtask_scheduling_service(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER, tmss_client_credentials_id: str=None):
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py
index 570e8b9c7663a7c5878bdaf89f154235cf04e1a9..f647a9a9caada1b2c7b4e8a044ce5e15a6d22619 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/sip.py
@@ -193,7 +193,7 @@ def create_sip_representation_for_subtask(subtask: Subtask):
frequencyintegrationstep=spec['demixer']['frequency_steps'] if spec['demixer']['enabled'] else 1,
timeintegrationstep=spec['demixer']['time_steps'] if spec['demixer']['enabled'] else 1,
flagautocorrelations=spec['preflagger1']['enabled'] and spec['preflagger1']['corrtype'] == 'auto',
- demixing=spec['demixer']['enabled'] and (spec['demixer']['demix_always'] or spec['demixer']['demix_if_needed'])
+ demixing=spec['demixer']['enabled'] and (len(spec['demixer']['demix_always']) > 0 or len(spec['demixer']['demix_if_needed']) > 0)
)
# todo: distinguish and create other pipeline types. Probably most of these can be filled in over time as needed,
# but they are not required for UC1. Here are stubs to start from for the other types the LTA supports:
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
index 8cbbf14c8d50013c84e344d5459036911854d707..72d9c2a2c79fd4483bbc8201aa6561bb2848073e 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2021-04-14 07:47
+# Generated by Django 3.0.9 on 2021-04-08 14:57
from django.conf import settings
import django.contrib.postgres.fields
@@ -124,6 +124,7 @@ class Migration(migrations.Migration):
('size', models.BigIntegerField(help_text='Dataproduct size, in bytes. Used for accounting purposes. NULL if size is (yet) unknown (NULLable).', null=True)),
('feedback_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Dataproduct properties, as reported by the producing process.')),
],
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='DataproductArchiveInfo',
@@ -472,6 +473,7 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='ReservationStrategyTemplate',
@@ -539,6 +541,7 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='SAPTemplate',
@@ -595,6 +598,7 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='SchedulingUnitBlueprint',
@@ -609,7 +613,6 @@ class Migration(migrations.Migration):
('do_cancel', models.BooleanField()),
('ingest_permission_required', models.BooleanField(default=False, help_text='Explicit permission is needed before the task.')),
('ingest_permission_granted_since', models.DateTimeField(help_text='The moment when ingest permission was granted.', null=True)),
- ('output_data_allowed_to_be_ingested', models.BooleanField(default=False, help_text='boolean (default FALSE), which blocks Ingest Tasks from starting if OFF. When toggled ON, backend must scan for startable Ingest Tasks.')),
('output_pinned', models.BooleanField(default=False, help_text='boolean (default FALSE), which blocks deleting unpinned dataproducts. When toggled ON, backend must pick SUB up for deletion. It also must when dataproducts are unpinned.')),
('results_accepted', models.BooleanField(default=False, help_text='boolean (default NULL), which records whether the results were accepted, allowing the higher-level accounting to be adjusted.')),
('priority_rank', models.FloatField(default=0.0, help_text='Priority of this scheduling unit w.r.t. other scheduling units within the same queue and project.')),
@@ -617,7 +620,7 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
- bases=(lofar.sas.tmss.tmss.tmssapp.models.common.RefreshFromDbInvalidatesCachedPropertiesMixin, models.Model),
+ bases=(lofar.sas.tmss.tmss.tmssapp.models.common.RefreshFromDbInvalidatesCachedPropertiesMixin, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin, models.Model),
),
migrations.CreateModel(
name='SchedulingUnitDraft',
@@ -637,7 +640,7 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
- bases=(lofar.sas.tmss.tmss.tmssapp.models.common.RefreshFromDbInvalidatesCachedPropertiesMixin, models.Model),
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='SchedulingUnitObservingStrategyTemplate',
@@ -715,6 +718,7 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='SubtaskAllowedStateTransitions',
@@ -734,6 +738,7 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='SubtaskOutput',
@@ -826,7 +831,7 @@ class Migration(migrations.Migration):
('do_cancel', models.BooleanField(help_text='Cancel this task.')),
('output_pinned', models.BooleanField(default=False, help_text='True if the output of this task is pinned to disk, that is, forbidden to be removed.')),
],
- bases=(lofar.sas.tmss.tmss.tmssapp.models.common.RefreshFromDbInvalidatesCachedPropertiesMixin, models.Model),
+ bases=(lofar.sas.tmss.tmss.tmssapp.models.common.RefreshFromDbInvalidatesCachedPropertiesMixin, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin, models.Model),
),
migrations.CreateModel(
name='TaskConnectorType',
@@ -852,7 +857,7 @@ class Migration(migrations.Migration):
('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Specifications for this task.')),
('output_pinned', models.BooleanField(default=False, help_text='True if the output of this task is pinned to disk, that is, forbidden to be removed.')),
],
- bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.specification.ProjectPropertyMixin),
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.specification.ProjectPropertyMixin, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='TaskRelationBlueprint',
@@ -863,6 +868,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('selection_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Filter for selecting dataproducts from the output role.')),
],
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='TaskRelationDraft',
@@ -873,6 +879,7 @@ class Migration(migrations.Migration):
('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
('selection_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Filter for selecting dataproducts from the output role.')),
],
+ bases=(models.Model, lofar.sas.tmss.tmss.tmssapp.models.common.TemplateSchemaMixin),
),
migrations.CreateModel(
name='TaskRelationSelectionTemplate',
@@ -965,11 +972,6 @@ class Migration(migrations.Migration):
name='consumer',
field=models.ForeignKey(help_text='Task Draft that has the input connector.', on_delete=django.db.models.deletion.CASCADE, related_name='produced_by', to='tmssapp.TaskDraft'),
),
- migrations.AddField(
- model_name='taskrelationdraft',
- name='dataformat',
- field=models.ForeignKey(help_text='Selected data format to use. One of (MS, HDF5).', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Dataformat'),
- ),
migrations.AddField(
model_name='taskrelationdraft',
name='input_role',
@@ -995,11 +997,6 @@ class Migration(migrations.Migration):
name='consumer',
field=models.ForeignKey(help_text='Task Blueprint that has the input connector.', on_delete=django.db.models.deletion.CASCADE, related_name='produced_by', to='tmssapp.TaskBlueprint'),
),
- migrations.AddField(
- model_name='taskrelationblueprint',
- name='dataformat',
- field=models.ForeignKey(help_text='Selected data format to use.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Dataformat'),
- ),
migrations.AddField(
model_name='taskrelationblueprint',
name='draft',
@@ -1047,8 +1044,8 @@ class Migration(migrations.Migration):
),
migrations.AddField(
model_name='taskconnectortype',
- name='dataformats',
- field=models.ManyToManyField(blank=True, to='tmssapp.Dataformat'),
+ name='dataformat',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Dataformat'),
),
migrations.AddField(
model_name='taskconnectortype',
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py
index 80a9fb61594cbe8996f45fe0b0b35a1c842fe319..4eeeb68e1a42963aeabbd1111c7dcd509f0eb781 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/common.py
@@ -13,6 +13,7 @@ from lofar.sas.tmss.tmss.exceptions import SchemaValidationException
from django.urls import reverse as reverse_url
import json
import jsonschema
+from datetime import timedelta
class RefreshFromDbInvalidatesCachedPropertiesMixin():
"""Helper Mixin class which invalidates all 'cached_property' attributes on a model upon refreshing from the db"""
@@ -174,36 +175,41 @@ class Tags(Model):
description = CharField(max_length=255)
-# methods
-
-def annotate_validate_add_defaults_to_doc_using_template(model: Model, document_attr:str, template_attr:str) -> None:
- '''
- annotate, validate and add defaults to the JSON document in the model instance using the schema of the given template.
- '''
- try:
- # fetch the actual JSON document and template-model-instance
- document = getattr(model, document_attr)
- template = getattr(model, template_attr)
-
- if document is not None and template is not None:
- try:
- if isinstance(document, str):
- document = json.loads(document)
-
- # always annotate the json data document with a $schema URI to the schema that it is based on.
- # this enables all users using this document (inside or outside of TMSS) to do their own validation and usage of editors which use the schema as UI template
- document['$schema'] = template.schema['$id']
- except (KeyError, TypeError, AttributeError) as e:
- raise SchemaValidationException("Cannot set $schema in json_doc to the schema's $id.\nError: %s \njson_doc: %s\nschema: %s" % (str(e), document, template.schema))
-
- # add defaults for missing properies, and validate on the fly
- document = add_defaults_to_json_object_for_schema(document, template.schema)
-
- # update the model instance with the updated and validated document
- setattr(model, document_attr, document)
- except AttributeError:
- pass
- except json.JSONDecodeError as e:
- raise SchemaValidationException("Invalid JSON.\nError: %s \ndata: %s" % (str(e), document))
- except jsonschema.ValidationError as e:
- raise SchemaValidationException(str(e))
\ No newline at end of file
+class TemplateSchemaMixin():
+ '''The TemplateSchemaMixin class can be mixed in to models which do validate and add defaults to json documents given a json-schema.
+ It uses an internal cache with a max age to minimize the number of requests to schema's, subschema's or referenced (sub)schema's.'''
+ _schema_cache = {}
+ _MAX_SCHEMA_CACHE_AGE = timedelta(minutes=1)
+
+ def annotate_validate_add_defaults_to_doc_using_template(self, document_attr:str, template_attr:str) -> None:
+ '''
+ annotate, validate and add defaults to the JSON document in the model instance using the schema of the given template.
+ '''
+ try:
+ # fetch the actual JSON document and template-model-instance
+ document = getattr(self, document_attr)
+ template = getattr(self, template_attr)
+
+ if document is not None and template is not None:
+ try:
+ if isinstance(document, str):
+ document = json.loads(document)
+
+ # always annotate the json data document with a $schema URI to the schema that it is based on.
+ # this enables all users using this document (inside or outside of TMSS) to do their own validation and usage of editors which use the schema as UI template
+ document['$schema'] = template.schema['$id']
+ except (KeyError, TypeError, AttributeError) as e:
+ raise SchemaValidationException("Cannot set $schema in json_doc to the schema's $id.\nError: %s \njson_doc: %s\nschema: %s" % (str(e), document, template.schema))
+
+ # add defaults for missing properies, and validate on the fly
+ # use the class's _schema_cache
+ document = add_defaults_to_json_object_for_schema(document, template.schema, self._schema_cache)
+
+ # update the model instance with the updated and validated document
+ setattr(self, document_attr, document)
+ except AttributeError:
+ pass
+ except json.JSONDecodeError as e:
+ raise SchemaValidationException("Invalid JSON.\nError: %s \ndata: %s" % (str(e), document))
+ except jsonschema.ValidationError as e:
+ raise SchemaValidationException(str(e))
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
index 61e4e8e36356d2f37302ddb69a92c93f03f0c840..3fa4cc2134aa7b636f5a8809f0483fc749c2c229 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/scheduling.py
@@ -12,7 +12,7 @@ from django.db.models import Model, ForeignKey, OneToOneField, CharField, DateTi
ManyToManyField, CASCADE, SET_NULL, PROTECT, QuerySet, BigAutoField, UniqueConstraint
from django.contrib.postgres.fields import ArrayField, JSONField
from django.contrib.auth.models import User
-from .common import AbstractChoice, BasicCommon, Template, NamedCommon, annotate_validate_add_defaults_to_doc_using_template
+from .common import AbstractChoice, BasicCommon, Template, NamedCommon, TemplateSchemaMixin
from enum import Enum
from django.db.models.expressions import RawSQL
from django.core.exceptions import ValidationError
@@ -48,6 +48,7 @@ class SubtaskState(AbstractChoice):
CANCELLING = "cancelling"
CANCELLED = "cancelled"
ERROR = "error"
+ UNSCHEDULABLE = "unschedulable"
class SubtaskType(AbstractChoice):
@@ -61,7 +62,7 @@ class SubtaskType(AbstractChoice):
INSPECTION = "inspection"
QA_FILES = "qa_files" # task which creates "adder" QA h5 file(s) from a MeasurementSet of beamformed data
QA_PLOTS = "qa_plots" # task which creates "adder" QA plots from an "adder" QA h5 file h5
- DELETION = "deletion"
+ CLEANUP = "cleanup"
MANUAL = 'manual'
OTHER = 'other'
@@ -137,7 +138,7 @@ class SIPidentifier(Model):
#
# Instance Objects
#
-class Subtask(BasicCommon):
+class Subtask(BasicCommon, TemplateSchemaMixin):
"""
Represents a low-level task, which is an atomic unit of execution, such as running an observation, running
inspection plots on the observed data, etc. Each task has a specific configuration, will have resources allocated
@@ -276,7 +277,7 @@ class Subtask(BasicCommon):
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
creating = self._state.adding # True on create, False on update
- annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('specifications_doc', 'specifications_template')
SIPidentifier.assign_new_id_to_model(self)
# check for uniqueness of SAP names:
@@ -353,7 +354,7 @@ class SubtaskStateLog(BasicCommon):
new_state = ForeignKey('SubtaskState', null=False, editable=False, on_delete=PROTECT, related_name='is_new_state_of', help_text='Subtask state after update (see Subtask State Machine).')
-class SubtaskInput(BasicCommon):
+class SubtaskInput(BasicCommon, TemplateSchemaMixin):
subtask = ForeignKey('Subtask', null=False, on_delete=CASCADE, related_name='inputs', help_text='Subtask to which this input specification refers.')
task_relation_blueprint = ForeignKey('TaskRelationBlueprint', null=True, on_delete=SET_NULL, help_text='Task Relation Blueprint which this Subtask Input implements (NULLable).')
producer = ForeignKey('SubtaskOutput', on_delete=PROTECT, related_name='consumers', help_text='The SubtaskOutput producing the input dataproducts for this SubtaskInput.')
@@ -362,7 +363,7 @@ class SubtaskInput(BasicCommon):
selection_template = ForeignKey('TaskRelationSelectionTemplate', on_delete=PROTECT, help_text='Schema used for selection_doc.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'selection_doc', 'selection_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('selection_doc', 'selection_template')
super().save(force_insert, force_update, using, update_fields)
@@ -371,19 +372,19 @@ class SubtaskOutput(BasicCommon):
task_blueprint = ForeignKey('TaskBlueprint', null=False, on_delete=CASCADE, related_name='outputs', help_text='Task to which this output specification refers.')
-class SAP(BasicCommon):
+class SAP(BasicCommon, TemplateSchemaMixin):
specifications_doc = JSONField(help_text='SAP properties.')
specifications_template = ForeignKey('SAPTemplate', null=False, on_delete=CASCADE, help_text='Schema used for specifications_doc.')
global_identifier = OneToOneField('SIPidentifier', null=False, editable=False, on_delete=PROTECT, help_text='The global unique identifier for LTA SIP.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('specifications_doc', 'specifications_template')
SIPidentifier.assign_new_id_to_model(self)
super().save(force_insert, force_update, using, update_fields)
-class Dataproduct(BasicCommon):
+class Dataproduct(BasicCommon, TemplateSchemaMixin):
"""
A data product represents an atomic dataset, produced and consumed by subtasks. The consumed dataproducts are those
resulting from interpreting the Subtask Connector filters of the inputs. These links are explicitly saved, should
@@ -410,8 +411,8 @@ class Dataproduct(BasicCommon):
constraints = [UniqueConstraint(fields=['directory', 'filename'], name='%(class)s_unique_path')]
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
- annotate_validate_add_defaults_to_doc_using_template(self, 'feedback_doc', 'feedback_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('specifications_doc', 'specifications_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('feedback_doc', 'feedback_template')
SIPidentifier.assign_new_id_to_model(self)
super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
index 222653d1fe21c9d6b064ba5525b885efc0477155..b7e4c3b56074d6d6b21d6bec481ef603f562895b 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/models/specification.py
@@ -10,7 +10,7 @@ from django.contrib.postgres.fields import JSONField
from enum import Enum
from django.db.models.expressions import RawSQL
from django.db.models.deletion import ProtectedError
-from .common import AbstractChoice, BasicCommon, Template, NamedCommon, annotate_validate_add_defaults_to_doc_using_template, NamedCommonPK, RefreshFromDbInvalidatesCachedPropertiesMixin
+from .common import AbstractChoice, BasicCommon, Template, NamedCommon, TemplateSchemaMixin, NamedCommonPK, RefreshFromDbInvalidatesCachedPropertiesMixin
from lofar.common.json_utils import validate_json_against_schema, validate_json_against_its_schema, add_defaults_to_json_object_for_schema
from lofar.sas.tmss.tmss.exceptions import *
from django.core.exceptions import ValidationError
@@ -45,7 +45,8 @@ class ProjectPropertyMixin(RefreshFromDbInvalidatesCachedPropertiesMixin):
class Role(AbstractChoice):
"""Defines the model and predefined list of possible Role's for TaskConnectorType.
- The items in the Choises class below are automagically populated into the database via a data migration."""
+ The items in the Choises class below are automagically populated into the database via a data migration.
+ When changing/adding/removing items to the Choices, please update the common json schema for tasks as well."""
class Choices(Enum):
CORRELATOR = "correlator"
BEAMFORMER = "beamformer"
@@ -66,7 +67,8 @@ class IOType(AbstractChoice):
class Datatype(AbstractChoice):
"""Defines the model and predefined list of possible Datatype's for TaskConnectorType.
- The items in the Choises class below are automagically populated into the database via a data migration."""
+ The items in the Choises class below are automagically populated into the database via a data migration.
+ When changing/adding/removing items to the Choices, please update the common json schema for tasks as well."""
class Choices(Enum):
VISIBILITIES = "visibilities"
TIME_SERIES = "time series"
@@ -76,8 +78,9 @@ class Datatype(AbstractChoice):
class Dataformat(AbstractChoice):
- """Defines the model and predefined list of possible Dataformat's for TaskRelationDraft and TaskRelationBlueprint.
- The items in the Choises class below are automagically populated into the database via a data migration."""
+ """Defines the model and predefined list of possible Dataformat's for TaskConnectorType.
+ The items in the Choises class below are automagically populated into the database via a data migration.
+ When changing/adding/removing items to the Choices, please update the common json schema for tasks as well."""
class Choices(Enum):
MEASUREMENTSET = "MeasurementSet"
BEAMFORMED = "Beamformed"
@@ -152,6 +155,7 @@ class TaskType(AbstractChoice):
OBSERVATION = "observation"
PIPELINE = "pipeline"
INGEST = "ingest"
+ CLEANUP = 'cleanup'
MAINTENANCE = "maintenance"
OTHER = 'other'
@@ -177,7 +181,7 @@ class TaskConnectorType(BasicCommon):
TARGET roles.'''
role = ForeignKey('Role', null=False, on_delete=PROTECT)
datatype = ForeignKey('Datatype', null=False, on_delete=PROTECT)
- dataformats = ManyToManyField('Dataformat', blank=True)
+ dataformat = ForeignKey('Dataformat', null=False, on_delete=PROTECT)
task_template = ForeignKey("TaskTemplate", related_name='output_connector_types', null=False, on_delete=CASCADE)
iotype = ForeignKey('IOType', null=False, on_delete=PROTECT, help_text="Is this connector an input or output")
@@ -372,18 +376,18 @@ class ResourceType(NamedCommonPK):
quantity = ForeignKey('Quantity', null=False, on_delete=PROTECT, help_text='The quantity of this resource type.')
-class SchedulingSet(NamedCommon):
+class SchedulingSet(NamedCommon, TemplateSchemaMixin):
generator_doc = JSONField(null=True, help_text='Parameters for the generator (NULLable).')
generator_template = ForeignKey('GeneratorTemplate', on_delete=SET_NULL, null=True, help_text='Generator for the scheduling units in this set (NULLable).')
generator_source = ForeignKey('SchedulingUnitDraft', on_delete=SET_NULL, null=True, help_text='Reference for the generator to an existing collection of specifications (NULLable).')
project = ForeignKey('Project', related_name="scheduling_sets", on_delete=PROTECT, help_text='Project to which this scheduling set belongs.') # protected to avoid accidents
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'generator_doc', 'generator_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('generator_doc', 'generator_template')
super().save(force_insert, force_update, using, update_fields)
-class SchedulingUnitDraft(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
+class SchedulingUnitDraft(NamedCommon, TemplateSchemaMixin):
requirements_doc = JSONField(help_text='Scheduling and/or quality requirements for this run.')
copies = ForeignKey('SchedulingUnitDraft', related_name="copied_from", on_delete=SET_NULL, null=True, help_text='Source reference, if we are a copy (NULLable).')
copy_reason = ForeignKey('CopyReason', null=True, on_delete=PROTECT, help_text='Reason why source was copied (NULLable).')
@@ -409,11 +413,11 @@ class SchedulingUnitDraft(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCo
# This code only happens if the objects is not in the database yet. self._state.adding is True creating
if self._state.adding and hasattr(self, 'scheduling_set') and self.scheduling_set.project.auto_ingest is False:
- #When auto_ingest=False, the scheduling units will be created with ingest_permission_required = True
+ # When project.auto_ingest=False, the scheduling units will be created with ingest_permission_required = True
self.ingest_permission_required=True
- annotate_validate_add_defaults_to_doc_using_template(self, 'requirements_doc', 'requirements_template')
- annotate_validate_add_defaults_to_doc_using_template(self, 'scheduling_constraints_doc', 'scheduling_constraints_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('requirements_doc', 'requirements_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('scheduling_constraints_doc', 'scheduling_constraints_template')
super().save(force_insert, force_update, using, update_fields)
@cached_property
@@ -449,7 +453,7 @@ class SchedulingUnitDraft(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCo
return self.scheduling_set.project
-class SchedulingUnitBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
+class SchedulingUnitBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, TemplateSchemaMixin, NamedCommon):
class Status(Enum):
DEFINED = "defined"
FINISHED = "finished"
@@ -469,20 +473,17 @@ class SchedulingUnitBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, Nam
ingest_permission_granted_since = DateTimeField(auto_now_add=False, null=True, help_text='The moment when ingest permission was granted.')
requirements_template = ForeignKey('SchedulingUnitTemplate', on_delete=CASCADE, help_text='Schema used for requirements_doc (IMMUTABLE).')
draft = ForeignKey('SchedulingUnitDraft', related_name='scheduling_unit_blueprints', on_delete=PROTECT, help_text='Scheduling Unit Draft which this run instantiates.')
- output_data_allowed_to_be_ingested = BooleanField(default=False, help_text='boolean (default FALSE), which blocks Ingest Tasks from starting if OFF. When toggled ON, backend must scan for startable Ingest Tasks.')
output_pinned = BooleanField(default=False, help_text='boolean (default FALSE), which blocks deleting unpinned dataproducts. When toggled ON, backend must pick SUB up for deletion. It also must when dataproducts are unpinned.')
results_accepted = BooleanField(default=False, help_text='boolean (default NULL), which records whether the results were accepted, allowing the higher-level accounting to be adjusted.')
priority_rank = FloatField(null=False, default=0.0, help_text='Priority of this scheduling unit w.r.t. other scheduling units within the same queue and project.')
priority_queue = ForeignKey('PriorityQueueType', null=False, on_delete=PROTECT, default="A", help_text='Priority queue of this scheduling unit. Queues provide a strict ordering between scheduling units.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'requirements_doc', 'requirements_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('requirements_doc', 'requirements_template')
# This code only happens if the objects is not in the database yet. self._state.adding is True creating
- if self._state.adding and hasattr(self, 'draft') and self.draft.scheduling_set.project.auto_ingest is False:
- #When auto_ingest=False, the scheduling units will be created with ingest_permission_required = True
- self.ingest_permission_required=True
-
+ if self._state.adding and hasattr(self, 'draft'):
+ self.ingest_permission_required = self.draft.ingest_permission_required
super().save(force_insert, force_update, using, update_fields)
@@ -719,7 +720,7 @@ class ProjectPropertyMixin():
return obj
-class TaskDraft(NamedCommon, ProjectPropertyMixin):
+class TaskDraft(NamedCommon, ProjectPropertyMixin, TemplateSchemaMixin):
specifications_doc = JSONField(help_text='Specifications for this task.')
copies = ForeignKey('TaskDraft', related_name="copied_from", on_delete=SET_NULL, null=True, help_text='Source reference, if we are a copy (NULLable).')
copy_reason = ForeignKey('CopyReason', on_delete=PROTECT, null=True, help_text='Reason why source was copied (NULLable).')
@@ -733,7 +734,7 @@ class TaskDraft(NamedCommon, ProjectPropertyMixin):
constraints = [UniqueConstraint(fields=['name', 'scheduling_unit_draft'], name='TaskDraft_unique_name_in_scheduling_unit')]
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('specifications_doc', 'specifications_template')
if self._state.adding: # True on create, False on update, needs to be checked before super().save()
super().save(force_insert, force_update, using, update_fields)
if self.scheduling_unit_draft.scheduling_set.project.auto_pin:
@@ -850,7 +851,7 @@ class TaskDraft(NamedCommon, ProjectPropertyMixin):
# return None
-class TaskBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
+class TaskBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, TemplateSchemaMixin, NamedCommon):
specifications_doc = JSONField(help_text='Schedulings for this task (IMMUTABLE).')
do_cancel = BooleanField(help_text='Cancel this task.')
@@ -864,7 +865,7 @@ class TaskBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
constraints = [UniqueConstraint(fields=['name', 'scheduling_unit_blueprint'], name='TaskBlueprint_unique_name_in_scheduling_unit')]
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('specifications_doc', 'specifications_template')
super().save(force_insert, force_update, using, update_fields)
@cached_property
@@ -998,10 +999,9 @@ class TaskBlueprint(RefreshFromDbInvalidatesCachedPropertiesMixin, NamedCommon):
return "schedulable"
-class TaskRelationDraft(BasicCommon):
+class TaskRelationDraft(BasicCommon, TemplateSchemaMixin):
selection_doc = JSONField(help_text='Filter for selecting dataproducts from the output role.')
selection_template = ForeignKey('TaskRelationSelectionTemplate', on_delete=CASCADE, help_text='Schema used for selection_doc.')
- dataformat = ForeignKey('Dataformat', null=False, on_delete=PROTECT, help_text='Selected data format to use. One of (MS, HDF5).')
# caveat: it might look like producer has an incorrect related_name='consumed_by'. But it really is correct, denends on the way you look at it
producer = ForeignKey('TaskDraft', related_name='consumed_by', on_delete=CASCADE, help_text='Task Draft that has the output connector. NOTE: The producer does typically, but not necessarily, belong to the same Scheduling Unit (or even the same Project) as the consumer.')
@@ -1023,13 +1023,12 @@ class TaskRelationDraft(BasicCommon):
# output_role.output = True
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'selection_doc', 'selection_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('selection_doc', 'selection_template')
super().save(force_insert, force_update, using, update_fields)
-class TaskRelationBlueprint(BasicCommon):
+class TaskRelationBlueprint(BasicCommon, TemplateSchemaMixin):
selection_doc = JSONField(help_text='Filter for selecting dataproducts from the output role.')
- dataformat = ForeignKey('Dataformat', null=False, on_delete=PROTECT, help_text='Selected data format to use.')
# caveat: it might look like producer has an incorrect related_name='consumed_by'. But it really is correct, denends on the way you look at it
producer = ForeignKey('TaskBlueprint', related_name='consumed_by', on_delete=CASCADE, help_text='Task Blueprint that has the output connector.')
@@ -1046,7 +1045,7 @@ class TaskRelationBlueprint(BasicCommon):
constraints = [UniqueConstraint(fields=['producer', 'consumer', 'input_role', 'output_role'], name='TaskRelationBlueprint_unique_relation')]
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'selection_doc', 'selection_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('selection_doc', 'selection_template')
super().save(force_insert, force_update, using, update_fields)
@@ -1086,7 +1085,7 @@ class TaskSchedulingRelationDraft(BasicCommon):
super().save(force_insert, force_update, using, update_fields)
-class Reservation(NamedCommon):
+class Reservation(NamedCommon, TemplateSchemaMixin):
project = ForeignKey('Project', null=True, related_name='reservations', on_delete=CASCADE, help_text='Reservation will be accounted for this project.')
description = CharField(max_length=255, help_text='Short description for this reservation, used in overviews')
start_time = DateTimeField(help_text='Start of this reservation.')
@@ -1104,6 +1103,6 @@ class Reservation(NamedCommon):
return None
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
- annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ self.annotate_validate_add_defaults_to_doc_using_template('specifications_doc', 'specifications_template')
super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
index 205b1c262e6bcc27688322126e4b4959fbdd63d2..0b0581cb736b490b14ca5b14fc60b3a76167b9d3 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/populate.py
@@ -66,12 +66,14 @@ def populate_subtask_allowed_state_transitions(apps, schema_editor):
CANCELLING = SubtaskState.objects.get(value=SubtaskState.Choices.CANCELLING.value)
CANCELLED = SubtaskState.objects.get(value=SubtaskState.Choices.CANCELLED.value)
ERROR = SubtaskState.objects.get(value=SubtaskState.Choices.ERROR.value)
+ UNSCHEDULABLE = SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULABLE.value)
SubtaskAllowedStateTransitions.objects.bulk_create([
SubtaskAllowedStateTransitions(old_state=None, new_state=DEFINING),
SubtaskAllowedStateTransitions(old_state=DEFINING, new_state=DEFINED),
SubtaskAllowedStateTransitions(old_state=DEFINED, new_state=SCHEDULING),
SubtaskAllowedStateTransitions(old_state=SCHEDULING, new_state=SCHEDULED),
+ SubtaskAllowedStateTransitions(old_state=SCHEDULING, new_state=UNSCHEDULABLE),
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=STARTING), # this is an odd one, as most (all?) subtasks are queued before execution...
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=QUEUEING),
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=UNSCHEDULING),
@@ -95,9 +97,7 @@ def populate_subtask_allowed_state_transitions(apps, schema_editor):
SubtaskAllowedStateTransitions(old_state=DEFINED, new_state=CANCELLING),
SubtaskAllowedStateTransitions(old_state=SCHEDULED, new_state=CANCELLING),
SubtaskAllowedStateTransitions(old_state=QUEUED, new_state=CANCELLING),
- SubtaskAllowedStateTransitions(old_state=STARTED, new_state=CANCELLING),
- SubtaskAllowedStateTransitions(old_state=FINISHED, new_state=CANCELLING),
- SubtaskAllowedStateTransitions(old_state=ERROR, new_state=CANCELLING)
+ SubtaskAllowedStateTransitions(old_state=STARTED, new_state=CANCELLING)
])
def populate_settings(apps, schema_editor):
@@ -122,7 +122,8 @@ def populate_test_data():
constraints_spec = get_default_json_object_for_schema(constraints_template.schema)
uc1_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
- simple_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Simple Observation")
+ simple_obs_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Simple Observation")
+ short_obs_pl_ingest_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Short Test Observation - Pipeline - Ingest")
simple_beamforming_strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="Simple Beamforming Observation")
projects = models.Project.objects.order_by('-priority_rank').all()
@@ -145,7 +146,7 @@ def populate_test_data():
for scheduling_set in tmss_project.scheduling_sets.all():
for unit_nr in range(2):
- for strategy_template in [simple_beamforming_strategy_template, uc1_strategy_template, simple_strategy_template]:
+ for strategy_template in [short_obs_pl_ingest_strategy_template, simple_obs_strategy_template, simple_beamforming_strategy_template, uc1_strategy_template]:
# the 'template' in the strategy_template is a predefined json-data blob which validates against the given scheduling_unit_template
# a user might 'upload' a partial json-data blob, so add all the known defaults
scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template, strategy_template.scheduling_unit_template.schema)
@@ -155,7 +156,7 @@ def populate_test_data():
scheduling_unit_spec['tasks']['Calibrator Observation 1']['specifications_doc']['duration'] = 2*60
scheduling_unit_spec['tasks']['Target Observation']['specifications_doc']['duration'] = 2*3600
scheduling_unit_spec['tasks']['Calibrator Observation 2']['specifications_doc']['duration'] = 2*60
- elif strategy_template == simple_strategy_template:
+ elif strategy_template == simple_obs_strategy_template:
scheduling_unit_spec['tasks']['Observation']['specifications_doc']['duration'] = 5*60
# set some constraints, so the dynamic scheduler has something to chew on.
@@ -404,41 +405,40 @@ def populate_connectors():
# NOTE: This is an explicit list of each possible link between tasks. This model suffices
# until the number of connectors throw too large. By then, we could consider introducing
# wild cards, like output_of=NULL meaning "any".
- logger.info("POPULATING CONNECTORS")
+ logger.info("Populating TaskConnectorType's")
# calibrator observation
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.CORRELATOR.value),
datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
+ dataformat=Dataformat.objects.get(value=Dataformat.Choices.MEASUREMENTSET.value),
task_template=TaskTemplate.objects.get(name='calibrator observation'),
iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
# target observation
TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.CORRELATOR.value),
- datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
- task_template=TaskTemplate.objects.get(name='target observation'),
- iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
+ datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
+ dataformat=Dataformat.objects.get(value=Dataformat.Choices.MEASUREMENTSET.value),
+ task_template=TaskTemplate.objects.get(name='target observation'),
+ iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
# preprocessing pipeline
- TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
- datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
- task_template=TaskTemplate.objects.get(name='preprocessing pipeline'),
- iotype=IOType.objects.get(value=IOType.Choices.INPUT.value))
-
- TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
- datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
- task_template=TaskTemplate.objects.get(name='preprocessing pipeline'),
- iotype=IOType.objects.get(value=IOType.Choices.OUTPUT.value))
-
- # ingest
- TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
- datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
- task_template=TaskTemplate.objects.get(name='ingest'),
- iotype=IOType.objects.get(value=IOType.Choices.INPUT.value))
-
- TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
- datatype=Datatype.objects.get(value=Datatype.Choices.TIME_SERIES.value),
- task_template=TaskTemplate.objects.get(name='ingest'),
- iotype=IOType.objects.get(value=IOType.Choices.INPUT.value))
+ for iotype_value in (IOType.Choices.INPUT.value, IOType.Choices.OUTPUT.value):
+ TaskConnectorType.objects.create(role=Role.objects.get(value=Role.Choices.ANY.value),
+ datatype=Datatype.objects.get(value=Datatype.Choices.VISIBILITIES.value),
+ dataformat=Dataformat.objects.get(value=Dataformat.Choices.MEASUREMENTSET.value),
+ task_template=TaskTemplate.objects.get(name='preprocessing pipeline'),
+ iotype=IOType.objects.get(value=iotype_value))
+
+ # ingest and cleanup
+ for task_template_name in ('ingest', 'cleanup'):
+ for datatype_value in (Datatype.Choices.VISIBILITIES.value, Datatype.Choices.TIME_SERIES.value):
+ for dataformat_value in [choice.value for choice in Dataformat.Choices]:
+ for role_value in [choice.value for choice in Role.Choices]:
+ TaskConnectorType.objects.create(role=Role.objects.get(value=role_value),
+ datatype=Datatype.objects.get(value=datatype_value),
+ dataformat=Dataformat.objects.get(value=dataformat_value),
+ task_template=TaskTemplate.objects.get(name=task_template_name),
+ iotype=IOType.objects.get(value=IOType.Choices.INPUT.value))
def populate_permissions():
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
index fc409bf145881ef9dac3db69189dc2bce35f23b5..9a7a4fe7b836db4579a9111af512f2d31b6e4a9c 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/UC1-scheduling-unit-observation-strategy.json
@@ -200,15 +200,16 @@
"producer": "Calibrator Observation 1",
"consumer": "Pipeline 1",
"tags": [],
- "input": {
- "role": "any",
- "datatype": "visibilities"
- },
"output": {
"role": "correlator",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "input": {
+ "role": "any",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
},
@@ -216,15 +217,16 @@
"producer": "Calibrator Observation 2",
"consumer": "Pipeline 2",
"tags": [],
- "input": {
- "role": "any",
- "datatype": "visibilities"
- },
"output": {
"role": "correlator",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "input": {
+ "role": "any",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
},
@@ -232,15 +234,16 @@
"producer": "Target Observation",
"consumer": "Pipeline target1",
"tags": [],
- "input": {
- "role": "any",
- "datatype": "visibilities"
- },
"output": {
"role": "correlator",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "input": {
+ "role": "any",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {
"sap": [
"target1"
@@ -252,15 +255,16 @@
"producer": "Target Observation",
"consumer": "Pipeline target2",
"tags": [],
- "input": {
- "role": "any",
- "datatype": "visibilities"
- },
"output": {
"role": "correlator",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "input": {
+ "role": "any",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {
"sap": [
"target2"
@@ -272,15 +276,16 @@
"producer": "Pipeline 1",
"consumer": "Ingest",
"tags": [],
- "input": {
+ "output": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "output": {
+ "input": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
},
@@ -288,15 +293,16 @@
"producer": "Pipeline 2",
"consumer": "Ingest",
"tags": [],
- "input": {
+ "output": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "output": {
+ "input": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
},
@@ -304,15 +310,16 @@
"producer": "Pipeline target1",
"consumer": "Ingest",
"tags": [],
- "input": {
+ "output": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "output": {
+ "input": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
},
@@ -320,15 +327,16 @@
"producer": "Pipeline target2",
"consumer": "Ingest",
"tags": [],
- "input": {
+ "output": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "output": {
+ "input": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-tasks-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-tasks-1.json
index 01c7c91fdb8cccbc94aae63ac1539fb006d136e3..398542538b828ae57a2d392dffcb79e8259ac87e 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-tasks-1.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/common_schema_template-tasks-1.json
@@ -8,21 +8,33 @@
"definitions": {
"task_connector": {
"type": "object",
+ "description": "A task connector describes what a task can take as input and produces as output.",
"additionalProperties": false,
"default": {},
"properties": {
"role": {
"type": "string",
- "title": "Role"
+ "title": "Role",
+ "description": "The role of a task connector describes its intended use.",
+ "enum": ["correlator", "beamformer", "inspection plots", "calibrator", "target", "any"]
},
"datatype": {
"type": "string",
- "title": "Data Type"
+ "title": "Data Type",
+ "description": "The data type of a task connector describes its what kind of data is produced/consumed.",
+ "enum": ["visibilities", "time series", "instrument model", "image", "quality"]
+ },
+ "dataformat": {
+ "type": "string",
+ "title": "Data Format",
+ "description": "The data type of a task connector describes in which format the data is produced/consumed.",
+ "enum": ["MeasurementSet", "Beamformed", "QA_HDF5", "QA_Plots"]
}
},
"required": [
"role",
- "datatype"
+ "datatype",
+ "dataformat"
]
}
}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
index 00af272aa1318b9628e974edd49baed3be4ec25a..f92347892c9a0b3dcf67268e15f4b00ea85fe0c9 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/scheduling_unit_template-scheduling_unit-1.json
@@ -90,10 +90,6 @@
"$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/tasks/1/#/definitions/task_connector",
"default": {}
},
- "dataformat": {
- "type": "string",
- "title": "Data Format"
- },
"selection_doc": {
"type": "object",
"title": "Filter selection",
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json
index 0c5ba135fd763e1fa4f82633b7df6688e05ebbe9..6ae834740335d9474e7351d58c3739b1bf154a2f 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/short-observation-pipeline-ingest-scheduling-unit-observation-strategy.json
@@ -75,6 +75,12 @@
"tags": [],
"specifications_doc": {},
"specifications_template": "ingest"
+ },
+ "Cleanup": {
+ "description": "Cleanup all dataproducts from disk",
+ "tags": [],
+ "specifications_doc": {},
+ "specifications_template": "cleanup"
}
},
"task_relations": [
@@ -84,13 +90,14 @@
"tags": [],
"input": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
"output": {
"role": "correlator",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
},
@@ -100,13 +107,48 @@
"tags": [],
"input": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "output": {
+ "role": "any",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "selection_doc": {},
+ "selection_template": "all"
+ },
+ {
+ "producer": "Observation",
+ "consumer": "Cleanup",
+ "tags": [],
+ "input": {
+ "role": "any",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "output": {
+ "role": "correlator",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
+ },
+ "selection_doc": {},
+ "selection_template": "all"
+ },
+ {
+ "producer": "Pipeline",
+ "consumer": "Cleanup",
+ "tags": [],
+ "input": {
+ "role": "any",
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
"output": {
"role": "any",
- "datatype": "visibilities"
+ "datatype": "visibilities",
+ "dataformat": "MeasurementSet"
},
- "dataformat": "MeasurementSet",
"selection_doc": {},
"selection_template": "all"
}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/subtask_template-cleanup-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/subtask_template-cleanup-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0244ed9f921709d7a16176a3afe887e0b24d2a9
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/subtask_template-cleanup-1.json
@@ -0,0 +1,12 @@
+{
+ "$id":"http://tmss.lofar.org/api/schemas/subtasktemplate/cleanup/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title":"cleanup",
+ "description":"This schema defines the parameters to setup and control a dataproducts cleanup subtask.",
+ "version":1,
+ "type": "object",
+ "properties": {
+ },
+ "required": [
+ ]
+}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/task_template-cleanup-1.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/task_template-cleanup-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..993e48bf6386e887f9ead7cb9b448e72fe7bdace
--- /dev/null
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/task_template-cleanup-1.json
@@ -0,0 +1,12 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/tasktemplate/cleanup/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "cleanup",
+ "description": "This schema defines the parameters to setup a dataproduct(s) cleanup task.",
+ "version": 1,
+ "type": "object",
+ "properties": {
+ },
+ "required": [
+ ]
+}
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json
index e03990777545d78c5493574a707cbf328c369058..644405c2243aa00b45ea54d58bb696c767ebc1ac 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/schemas/templates.json
@@ -176,11 +176,21 @@
"template": "subtask_template",
"type": "ingest"
},
+ {
+ "file_name": "subtask_template-cleanup-1.json",
+ "template": "subtask_template",
+ "type": "cleanup"
+ },
{
"file_name": "task_template-ingest-1.json",
"template": "task_template",
"type": "ingest"
},
+ {
+ "file_name": "task_template-cleanup-1.json",
+ "template": "task_template",
+ "type": "cleanup"
+ },
{
"file_name": "reservation_template-reservation-1.json",
"template": "reservation_template"
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/scheduling.py b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/scheduling.py
index 0711865e0fcec308b7ae86bd170fc882fe105b0c..d0660311e04974f7bec4cc8c2f24b49c51d115e4 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/serializers/scheduling.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/serializers/scheduling.py
@@ -79,8 +79,8 @@ class DataproductFeedbackTemplateSerializer(AbstractTemplateSerializer):
class SubtaskSerializer(DynamicRelationalHyperlinkedModelSerializer):
# If this is OK then we can extend API with NO url ('flat' values) on more places if required
- cluster_value = serializers.StringRelatedField(source='cluster', label='cluster_value', read_only=True)
- subtask_type = serializers.StringRelatedField(source='specifications_template.type', label='subtask_type', read_only=True, help_text='The subtask type as defined in the specifications template.')
+ cluster_name = serializers.StringRelatedField(source='cluster', label='cluster_name', read_only=True, help_text='The cluster name as defined in the specifications template, provided here to safe an addition lookup.')
+ subtask_type = serializers.StringRelatedField(source='specifications_template.type', label='subtask_type', read_only=True, help_text='The subtask type as defined in the specifications template, provided here to safe an addition lookup.')
specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
duration = FloatDurationField(read_only=True)
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
index 7cea49f44baaba154f96a6003c6093c3c72c0a41..d69e7322100f0a494ba16d9a7c71d34905a48a50 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
@@ -63,7 +63,8 @@ def create_subtasks_from_task_blueprint(task_blueprint: TaskBlueprint) -> [Subta
create_qafile_subtask_from_task_blueprint,
create_qaplots_subtask_from_task_blueprint],
'preprocessing pipeline': [create_preprocessing_subtask_from_task_blueprint],
- 'ingest': [create_ingest_subtask_from_task_blueprint]}
+ 'ingest': [create_ingest_subtask_from_task_blueprint],
+ 'cleanup': [create_cleanup_subtask_from_task_blueprint]}
generators_mapping['calibrator observation'] = generators_mapping['target observation']
generators_mapping['beamforming observation'] = [create_observation_control_subtask_from_task_blueprint]
@@ -743,6 +744,49 @@ def create_ingest_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) ->
return subtask
+def create_cleanup_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) -> Subtask:
+ ''' Create a subtask for a cleanup job
+ This method implements "Instantiate subtasks" step from the "Specification Flow"
+ https://support.astron.nl/confluence/display/TMSS/Specification+Flow
+ '''
+ # step 0: check pre-requisites
+ check_prerequities_for_subtask_creation(task_blueprint)
+
+ # step 1: create subtask in defining state, with filled-in subtask_template
+ subtask_template = SubtaskTemplate.objects.get(name='cleanup')
+ subtask_specs = get_default_json_object_for_schema(subtask_template.schema)
+ cluster_name = task_blueprint.specifications_doc.get("storage_cluster", "CEP4")
+ subtask_data = {"start_time": None,
+ "stop_time": None,
+ "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value),
+ "task_blueprint": task_blueprint,
+ "specifications_template": subtask_template,
+ "specifications_doc": subtask_specs,
+ "cluster": Cluster.objects.get(name=cluster_name)}
+ subtask = Subtask.objects.create(**subtask_data)
+
+ # step 2: create and link subtask input
+ # for this cleanup subtask an 'input' seems a bit weird, but it actually makes sense!
+ # this cleanup subtask will cleanup the output data of all linked input predecessors.
+ for task_relation_blueprint in task_blueprint.produced_by.all():
+ producing_task_blueprint = task_relation_blueprint.producer
+
+ predecessor_subtasks = [st for st in producing_task_blueprint.subtasks.filter(specifications_template__type__value__in=(SubtaskType.Choices.OBSERVATION.value, SubtaskType.Choices.PIPELINE.value)).order_by('id').all()]
+ for predecessor_subtask in predecessor_subtasks:
+ for predecessor_subtask_output in predecessor_subtask.outputs.all():
+ SubtaskInput.objects.create(subtask=subtask,
+ producer=predecessor_subtask_output,
+ selection_doc=task_relation_blueprint.selection_doc,
+ selection_template=task_relation_blueprint.selection_template)
+
+ # step 3: set state to DEFINED
+ subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
+ subtask.save()
+
+ # done, now return the subtask, and allow the system to wait for the predecessors to be finished before we schedule this ingest
+ return subtask
+
+
# ==== various schedule* methods to schedule a Subtasks (if possible) ====
def schedule_subtask(subtask: Subtask) -> Subtask:
@@ -770,6 +814,9 @@ def schedule_subtask(subtask: Subtask) -> Subtask:
if subtask.specifications_template.type.value == SubtaskType.Choices.INGEST.value:
return schedule_ingest_subtask(subtask)
+ if subtask.specifications_template.type.value == SubtaskType.Choices.CLEANUP.value:
+ return schedule_cleanup_subtask(subtask)
+
if subtask.specifications_template.type.value == SubtaskType.Choices.COPY.value:
return schedule_copy_subtask(subtask)
@@ -780,12 +827,12 @@ def schedule_subtask(subtask: Subtask) -> Subtask:
logger.exception(e)
if isinstance(e, SubtaskSchedulingSpecificationException):
- # set the subtask to state 'ERROR' in case of a specification exception
- subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.ERROR.value)
+ # set the subtask to state 'UNSCHEDULABLE' in case of a specification exception
+ subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULABLE.value)
subtask.save()
- elif subtask.state == SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULING.value):
- # set the subtask back to state 'DEFINED' to allow the user/system to retry later
- subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
+ else:
+ # set the subtask to state 'ERROR'. TODO: we should annotate in the db what error occurred.
+ subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.ERROR.value)
subtask.save()
except Exception as e2:
logger.error(e2)
@@ -819,7 +866,7 @@ def unschedule_subtask(subtask: Subtask, post_state: SubtaskState=None) -> Subta
except Exception as e:
try:
# set the subtask to state 'ERROR'...
- subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.ERROR.value)
+ subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULABLE.value)
subtask.save()
except Exception as e2:
logger.error(e2)
@@ -1580,6 +1627,54 @@ def schedule_ingest_subtask(ingest_subtask: Subtask):
return ingest_subtask
+def schedule_cleanup_subtask(cleanup_subtask: Subtask):
+ ''' Schedule the given cleanup_subtask
+ This method should typically be called upon the event of an predecessor (pipeline or observation) subtask being finished.
+ This method implements "Scheduling subtasks" step from the "Specification Flow"
+ https://support.astron.nl/confluence/display/TMSS/Specification+Flow
+ '''
+ # step 0: check pre-requisites
+ check_prerequities_for_scheduling(cleanup_subtask)
+
+ if cleanup_subtask.specifications_template.type.value != SubtaskType.Choices.CLEANUP.value:
+ raise SubtaskSchedulingSpecificationException("Cannot schedule subtask id=%d type=%s but type should be %s" % (cleanup_subtask.pk,
+ cleanup_subtask.specifications_template.type,
+ SubtaskType.Choices.CLEANUP.value))
+
+ # step 1: set state to SCHEDULING
+ cleanup_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULING.value)
+ cleanup_subtask.save()
+
+ # step 1a: set start/stop times
+ # not very relevant for ingest subtasks, but it's nice for the user to see when the cleanup task was scheduled.
+ # please note that an cleanup subtask may idle for some time while it is in the cleanup queue.
+ # the actual start/stop times are set by the IngestTMSSAdapter when the subtask starts and stops.
+ cleanup_subtask.start_time = max([pred.stop_time for pred in cleanup_subtask.predecessors] + [datetime.utcnow()])
+ cleanup_subtask.stop_time = cleanup_subtask.start_time + timedelta(hours=6)
+
+ # step 2: link input dataproducts
+ if cleanup_subtask.inputs.count() == 0:
+ raise SubtaskSchedulingException("Cannot schedule subtask id=%d type=%s because it has no input(s)" % (cleanup_subtask.pk,
+ cleanup_subtask.specifications_template.type))
+
+ # iterate over all inputs
+ for cleanup_subtask_input in cleanup_subtask.inputs.all():
+ # select and set input dataproducts that meet the filter defined in selection_doc
+ input_dataproducts = [dataproduct for dataproduct in cleanup_subtask_input.producer.dataproducts.all()
+ if specifications_doc_meets_selection_doc(dataproduct.specifications_doc, cleanup_subtask_input.selection_doc)]
+ cleanup_subtask_input.dataproducts.set(input_dataproducts)
+
+ # cleanup has no outputs
+
+ # skip step 4: cleanup does not need to have resources assigned
+
+ # step 5: set state to SCHEDULED (resulting in the cleanup_service to pick this subtask up and run it)
+ cleanup_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULED.value)
+ cleanup_subtask.save()
+
+ return cleanup_subtask
+
+
def schedule_copy_subtask(copy_subtask: Subtask):
''' Schedule the given copy_subtask
This method should typically be called upon the event of an predecessor (pipeline or observation) subtask being finished.
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py b/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py
index 38276a789a370e374d26aada9f714da78d901763..a43d4d81c28c4cc5138f02645d1c9a0adbb066a2 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/tasks.py
@@ -1,13 +1,13 @@
from lofar.sas.tmss.tmss.exceptions import *
from lofar.sas.tmss.tmss.tmssapp import models
from lofar.sas.tmss.tmss.tmssapp.subtasks import unschedule_subtasks_in_task_blueprint, cancel_subtask
-from lofar.sas.tmss.tmss.tmssapp.models.specification import TaskBlueprint, SchedulingUnitBlueprint
+from lofar.sas.tmss.tmss.tmssapp.models.specification import TaskBlueprint, SchedulingUnitBlueprint, IOType, TaskTemplate, TaskType, TaskRelationSelectionTemplate
from lofar.sas.tmss.tmss.tmssapp.subtasks import create_and_schedule_subtasks_from_task_blueprint, create_subtasks_from_task_blueprint, schedule_independent_subtasks_in_task_blueprint, update_subtasks_start_times_for_scheduling_unit
from lofar.common.datetimeutils import round_to_minute_precision
from functools import cmp_to_key
import os
from copy import deepcopy
-from lofar.common.json_utils import add_defaults_to_json_object_for_schema
+from lofar.common.json_utils import add_defaults_to_json_object_for_schema, get_default_json_object_for_schema
import logging
from datetime import datetime, timedelta
from django.db.utils import IntegrityError
@@ -179,19 +179,25 @@ def create_task_drafts_from_scheduling_unit_draft(scheduling_unit_draft: models.
try:
producer_task_draft = scheduling_unit_draft.task_drafts.get(name=task_relation_definition["producer"])
consumer_task_draft = scheduling_unit_draft.task_drafts.get(name=task_relation_definition["consumer"])
- dataformat = models.Dataformat.objects.get(value=task_relation_definition["dataformat"])
- input_role = models.TaskConnectorType.objects.get(task_template=consumer_task_draft.specifications_template, role=task_relation_definition["input"]["role"], datatype=task_relation_definition["input"]["datatype"], iotype=models.IOType.objects.get(value=models.IOType.Choices.INPUT.value))
- output_role = models.TaskConnectorType.objects.get(task_template=producer_task_draft.specifications_template, role=task_relation_definition["output"]["role"], datatype=task_relation_definition["output"]["datatype"], iotype=models.IOType.objects.get(value=models.IOType.Choices.OUTPUT.value))
+ input_role = models.TaskConnectorType.objects.get(task_template=consumer_task_draft.specifications_template,
+ role=task_relation_definition["input"]["role"],
+ datatype=task_relation_definition["input"]["datatype"],
+ dataformat=task_relation_definition["input"]["dataformat"],
+ iotype=models.IOType.Choices.INPUT.value)
+ output_role = models.TaskConnectorType.objects.get(task_template=producer_task_draft.specifications_template,
+ role=task_relation_definition["output"]["role"],
+ datatype=task_relation_definition["output"]["datatype"],
+ dataformat=task_relation_definition["output"]["dataformat"],
+ iotype=models.IOType.Choices.OUTPUT.value)
selection_template = models.TaskRelationSelectionTemplate.objects.get(name=task_relation_definition["selection_template"])
except Exception as e:
- logger.error("Cannot create task_relation from spec '%s'. Error: %s", task_relation_definition, e)
+ logger.error("Could not determine Task Relations for %s. Error: %s", task_relation_definition, e)
raise
try:
with transaction.atomic():
task_relation = models.TaskRelationDraft.objects.create(tags=task_relation_definition.get("tags",[]),
selection_doc=task_relation_definition["selection_doc"],
- dataformat=dataformat,
producer=producer_task_draft,
consumer=consumer_task_draft,
input_role=input_role,
@@ -287,8 +293,7 @@ def create_task_blueprint_from_task_draft(task_draft: models.TaskDraft) -> model
producer=producing_task_blueprint,
consumer=consuming_task_blueprint,
selection_doc=task_relation_draft.selection_doc,
- selection_template=task_relation_draft.selection_template,
- dataformat=task_relation_draft.dataformat)
+ selection_template=task_relation_draft.selection_template)
logger.info("created task_relation_blueprint id=%s which connects task_blueprints producer_id=%s and consumer_id=%s",
task_relation_blueprint.pk, producing_task_blueprint.pk, consuming_task_blueprint.pk)
except IntegrityError as e:
@@ -438,3 +443,91 @@ def cancel_scheduling_unit_blueprint(scheduling_unit_blueprint: SchedulingUnitBl
cancel_task_blueprint(task_blueprint)
scheduling_unit_blueprint.refresh_from_db()
return scheduling_unit_blueprint
+
+def create_cleanuptask_for_scheduling_unit_blueprint(scheduling_unit_blueprint: SchedulingUnitBlueprint) -> models.SchedulingUnitBlueprint:
+ '''create a cleanuptask for the given scheduling_unit which will cleanup all output dataproducts from tasks in this scheduling_unit which aren't already cleaned up'''
+
+ # Rationale:
+ # adding a cleanup task(blueprint) to a scheduling_unit_blueprint adds a task to the graph (which breaks the immutable blueprint concept),
+ # but it does not modify observation/pipeline behaviour, hence we allow it.
+ # Regard this as a convenience function to allow users to simplify cleaning up after themselves if they forgot to specificy a cleanup task.
+ #
+ # Note: We do modify the graph (both in draft and blueprint),
+ # but we do NOT update the specifications_doc because that doc (blueprint) is immutable, and shows the user what was specified.
+ # The fact that the graph in the specifications_doc and in real instances are different (with an addded cleanup task) shows the users that cleanup
+ # was apparently forgotten at specification time, and added later, which is explainable.
+ #
+ # Maybe we want to split this function in the future into a "add cleanup to draft" and/or "add cleanup to blueprint"
+ # For now, we present it as a friendly convenience function to cleanup after yourself once the blueprint is already running / already ran with experimental scheduling units.
+ # In practice we will instantiate most scheduling units from properly defined observation_strategy_templates which include cleanup.
+
+ with transaction.atomic():
+ # create a cleanup task draft and blueprint....
+ cleanup_template = models.TaskTemplate.objects.get(name="cleanup")
+ cleanup_spec_doc = get_default_json_object_for_schema(cleanup_template.schema)
+
+ cleanup_task_draft = models.TaskDraft.objects.create(
+ name="Cleanup",
+ description="Cleaning up all output dataproducts for this scheduling unit",
+ scheduling_unit_draft=scheduling_unit_blueprint.draft,
+ specifications_doc=cleanup_spec_doc,
+ specifications_template=cleanup_template)
+
+ cleanup_task_blueprint = TaskBlueprint.objects.create(
+ description=cleanup_task_draft.description,
+ name=cleanup_task_draft.name,
+ do_cancel=False,
+ draft=cleanup_task_draft,
+ scheduling_unit_blueprint=scheduling_unit_blueprint,
+ specifications_doc=cleanup_task_draft.specifications_doc,
+ specifications_template=cleanup_task_draft.specifications_template,
+ output_pinned=False)
+
+ logger.info("Created Cleanup Task id=%d for scheduling_unit id=%s, adding the outputs of all producing tasks in the scheduling unit to the cleanup...", cleanup_task_blueprint.id, scheduling_unit_blueprint.id)
+
+ # ... and connect the outputs of the producing tasks to the cleanup, so the cleanup task knows what to remove.
+ selection_template = TaskRelationSelectionTemplate.objects.get(name="all")
+ selection_doc = get_default_json_object_for_schema(selection_template.schema)
+
+ for producer_task_blueprint in scheduling_unit_blueprint.task_blueprints.exclude(specifications_template__type=TaskType.Choices.CLEANUP).exclude(specifications_template__type=TaskType.Choices.INGEST).all():
+ for connector_type in producer_task_blueprint.specifications_template.output_connector_types.filter(iotype__value=IOType.Choices.OUTPUT.value).all():
+ # define what the producer_task_blueprint is producing
+ output_role = models.TaskConnectorType.objects.get(task_template=producer_task_blueprint.specifications_template,
+ role=connector_type.role,
+ datatype=connector_type.datatype,
+ iotype=models.IOType.objects.get(value=models.IOType.Choices.OUTPUT.value))
+
+ # define what the cleanup task accepts/consumes
+ input_role = models.TaskConnectorType.objects.filter(dataformat=connector_type.dataformat).get(task_template=cleanup_task_draft.specifications_template,
+ role=models.Role.objects.get(value=models.Role.Choices.ANY.value),
+ datatype=connector_type.datatype,
+ iotype=models.IOType.objects.get(value=models.IOType.Choices.INPUT.value))
+
+ # connect the two (in draft and blueprint)
+ task_relation_draft = models.TaskRelationDraft.objects.create(producer=producer_task_blueprint.draft,
+ consumer=cleanup_task_draft,
+ input_role=input_role,
+ output_role=output_role,
+ selection_doc=selection_doc,
+ selection_template=selection_template)
+
+ logger.info("created task_relation id=%s between task draft id=%s name='%s' and id=%s name='%s",
+ task_relation_draft.pk, task_relation_draft.producer.id, task_relation_draft.producer.name, task_relation_draft.consumer.id, task_relation_draft.consumer.name)
+
+ task_relation_blueprint = models.TaskRelationBlueprint.objects.create(draft=task_relation_draft,
+ producer=producer_task_blueprint,
+ consumer=cleanup_task_blueprint,
+ input_role=input_role,
+ output_role=output_role,
+ selection_doc=selection_doc,
+ selection_template=selection_template)
+
+ logger.info("created task_relation id=%s between task blueprint id=%s name='%s' and id=%s name='%s",
+ task_relation_blueprint.pk, task_relation_blueprint.producer.id, task_relation_blueprint.producer.name, task_relation_blueprint.consumer.id, task_relation_blueprint.consumer.name)
+
+ # and finally also create the executable subtask for the cleanup_task_blueprint, so it can actually run.
+ create_subtasks_from_task_blueprint(cleanup_task_blueprint)
+
+ # return the modified scheduling_unit
+ scheduling_unit_blueprint.refresh_from_db()
+ return scheduling_unit_blueprint
\ No newline at end of file
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/permissions.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/permissions.py
index 8e6b095a4fe79fd8680080065e5c82d7903f1325..291e602d5832032000e0db6a09771e2238e69d78 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/permissions.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/permissions.py
@@ -67,6 +67,8 @@ class IsProjectMember(drf_permissions.DjangoObjectPermissions):
# GET detail, PATCH, and DELETE
# we always have permission as superuser (e.g. in test environment, where a regular user is created to test permission specifically)
if request.user.is_superuser:
+ logger.info("IsProjectMember: User=%s is superuser. Not enforcing project permissions!" % request.user)
+ logger.info('### IsProjectMember.has_object_permission %s %s True' % (request._request, request.method))
return True
# todo: do we want to restrict access for that as well? Then we add it to the ProjectPermission model, but it seems cumbersome...?
@@ -98,9 +100,10 @@ class IsProjectMember(drf_permissions.DjangoObjectPermissions):
related_project = obj.project
if project_role['project'] == obj.project.name and \
models.ProjectRole.objects.get(value=project_role['role']) in permitted_project_roles:
+ logger.info('user=%s is permitted to access object=%s' % (request.user, obj))
+ logger.info('### IsProjectMember.has_object_permission %s %s True' % (request._request, request.method))
return True
else:
- related_project = None
logger.error("No project property on object %s, so cannot check project permission." % obj)
# todo: how to deal with objects that do not have a unique project associated to them?
# Do need users need the required role in all of them? Or just one?
@@ -110,6 +113,8 @@ class IsProjectMember(drf_permissions.DjangoObjectPermissions):
logger.warning("'%s' is a Template and action is '%s' so granting object access nonetheless." % (obj, view.action))
return True
+ logger.info('User=%s is not permitted to access object=%s with related project=%s since it requires one of project_roles=%s' % (request.user, obj, related_project, permitted_project_roles))
+ logger.info('### IsProjectMember.has_object_permission %s False' % (request._request))
return False
def has_permission(self, request, view):
diff --git a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
index c60756ca161044e086a0465722ba438b6c55ecaa..b87dc5fcdf41671d2766ea9bceeca78a86beac69 100644
--- a/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/viewsets/specification.py
@@ -839,6 +839,31 @@ class SchedulingUnitBlueprintViewSet(LOFARViewSet):
return RestResponse(serializer.data)
+ @swagger_auto_schema(responses={200: "All Subtasks in this SchedulingUnitBlueprint",
+ 403: 'forbidden'},
+ operation_description="Get all subtasks for this scheduling_unit")
+ @action(methods=['get'], detail=True, url_name="subtasks", name="all subtasks in this scheduling_unit")
+ def subtasks(self, request, pk=None):
+ subtasks = models.Subtask.objects.all().filter(task_blueprint__scheduling_unit_blueprint_id=pk). \
+ select_related('state', 'specifications_template', 'specifications_template__type', 'cluster', 'created_or_updated_by_user').all()
+
+ # return a response with the new serialized scheduling_unit_blueprint (with references to the created task_blueprint(s) and (scheduled) subtasks)
+ return Response(serializers.SubtaskSerializer(subtasks, many=True, context={'request':request}).data,
+ status=status.HTTP_200_OK)
+
+
+ @swagger_auto_schema(responses={201: "This SchedulingUnitBlueprint, with references to the created Cleanup TaskBlueprints.",
+ 403: 'forbidden'},
+ operation_description="Create a cleanup task for this scheduling unit.")
+ @action(methods=['get'], detail=True, url_name="create_cleanuptask", name="Create a cleanup task for this scheduling unit")
+ def create_cleanuptask_for_scheduling_unit_blueprint(self, request, pk=None):
+ scheduling_unit_blueprint = get_object_or_404(models.SchedulingUnitBlueprint, pk=pk)
+ scheduling_unit_blueprint = create_cleanuptask_for_scheduling_unit_blueprint(scheduling_unit_blueprint)
+
+ # return a response with the new serialized scheduling_unit_blueprint (with references to the created task_blueprint and subtask)
+ return Response(serializers.SchedulingUnitBlueprintSerializer(scheduling_unit_blueprint, context={'request':request}).data,
+ status=status.HTTP_201_CREATED)
+
class SchedulingUnitBlueprintExtendedViewSet(SchedulingUnitBlueprintViewSet):
serializer_class = serializers.SchedulingUnitBlueprintExtendedSerializer
diff --git a/SAS/TMSS/backend/src/tmss/workflowapp/flows/schedulingunitflow.py b/SAS/TMSS/backend/src/tmss/workflowapp/flows/schedulingunitflow.py
index 5ab934891e8e4358fbf19719cb1d82972de9587b..437d82c871b96c9492828f6505b13eba8d4f70ad 100644
--- a/SAS/TMSS/backend/src/tmss/workflowapp/flows/schedulingunitflow.py
+++ b/SAS/TMSS/backend/src/tmss/workflowapp/flows/schedulingunitflow.py
@@ -272,7 +272,7 @@ class SchedulingUnitFlow(Flow):
logger.info("granting ingest permission for scheduling unit blueprint id=%s", activation.process.su.id)
activation.process.su.ingest_permission_granted_since = round_to_second_precision(datetime.utcnow())
- activation.process.su.output_data_allowed_to_be_ingested = True
+ activation.process.su.ingest_permission_required = True
activation.process.su.save()
activation.process.save()
diff --git a/SAS/TMSS/backend/src/tmss/workflowapp/tests/t_workflow_qaworkflow.py b/SAS/TMSS/backend/src/tmss/workflowapp/tests/t_workflow_qaworkflow.py
index 744773b026fc4ea04d334a92ba4145c17785f024..67980972ba4c0f352cc187ca9309351dfce1d909 100755
--- a/SAS/TMSS/backend/src/tmss/workflowapp/tests/t_workflow_qaworkflow.py
+++ b/SAS/TMSS/backend/src/tmss/workflowapp/tests/t_workflow_qaworkflow.py
@@ -335,10 +335,6 @@ class SchedulingUnitFlowTest(unittest.TestCase):
ingest_permission_granted_since = models.SchedulingUnitBlueprint.objects.get(pk=scheduling_unit_process_id).ingest_permission_granted_since
self.assertEqual(True,isinstance(ingest_permission_granted_since, datetime))
- #verify that output_data_allowed_to_be_ingested is now True
- output_data_allowed_to_be_ingested = models.SchedulingUnitBlueprint.objects.get(pk=scheduling_unit_process_id).output_data_allowed_to_be_ingested
- self.assertEqual(True,output_data_allowed_to_be_ingested)
-
self.assertEqual(Task.objects.filter(process=scheduling_unit_process_id).order_by('id')[11].flow_task.name, 'ingest_done')
self.assertEqual(Task.objects.filter(process=scheduling_unit_process_id).order_by('id')[11].status, 'DONE')
@@ -1394,10 +1390,6 @@ class SchedulingUnitFlowTest(unittest.TestCase):
ingest_permission_granted_since = models.SchedulingUnitBlueprint.objects.get(pk=scheduling_unit_process_id).ingest_permission_granted_since
self.assertEqual(True,isinstance(ingest_permission_granted_since, datetime))
- #verify that output_data_allowed_to_be_ingested is now True
- output_data_allowed_to_be_ingested = models.SchedulingUnitBlueprint.objects.get(pk=scheduling_unit_process_id).output_data_allowed_to_be_ingested
- self.assertEqual(True,output_data_allowed_to_be_ingested)
-
self.assertEqual(Task.objects.filter(process=scheduling_unit_process_id).order_by('id')[11].flow_task.name, 'ingest_done')
self.assertEqual(Task.objects.filter(process=scheduling_unit_process_id).order_by('id')[11].status, 'DONE')
diff --git a/SAS/TMSS/backend/test/t_scheduling.py b/SAS/TMSS/backend/test/t_scheduling.py
index adeb59b790ee155bb8a1810efd8342d9769a808b..8fbad583b7a9edcc14053d35f79af2dfd1b9f1a2 100755
--- a/SAS/TMSS/backend/test/t_scheduling.py
+++ b/SAS/TMSS/backend/test/t_scheduling.py
@@ -231,7 +231,7 @@ class SchedulingTest(unittest.TestCase):
client.schedule_subtask(subtask_id)
subtask = client.get_subtask(subtask_id)
- self.assertEqual('error', subtask['state_value'])
+ self.assertEqual('unschedulable', subtask['state_value'])
self.assertEqual('conflict', tmss_test_env.ra_test_environment.radb.getTask(tmss_id=subtask_id)['status'])
def test_schedule_observation_subtask_with_blocking_reservations_failed(self):
@@ -254,7 +254,7 @@ class SchedulingTest(unittest.TestCase):
client.schedule_subtask(subtask_id)
subtask = client.get_subtask(subtask_id)
- self.assertEqual('error', subtask['state_value'])
+ self.assertEqual('unschedulable', subtask['state_value'])
ra_task = tmss_test_env.ra_test_environment.radb.getTask(tmss_id=subtask_id)
self.assertIsNotNone(ra_task)
self.assertEqual('conflict', ra_task['status'])
@@ -404,9 +404,8 @@ class SchedulingTest(unittest.TestCase):
# connect obs to pipeline
scheduling_unit_doc['task_relations'].append({"producer": "Observation",
"consumer": "Pipeline",
- "input": { "role": "any", "datatype": "visibilities" },
- "output": { "role": "correlator", "datatype": "visibilities" },
- "dataformat": "MeasurementSet",
+ "input": { "role": "any", "datatype": "visibilities", "dataformat": "MeasurementSet"},
+ "output": { "role": "correlator", "datatype": "visibilities", "dataformat": "MeasurementSet"},
"selection_doc": {},
"selection_template": "all" })
diff --git a/SAS/TMSS/backend/test/t_subtasks.py b/SAS/TMSS/backend/test/t_subtasks.py
index 993df8b8f548ae4abb10818120d5d02bffe22547..a59c19f89859090eac56e8b9d9d43425389eedeb 100755
--- a/SAS/TMSS/backend/test/t_subtasks.py
+++ b/SAS/TMSS/backend/test/t_subtasks.py
@@ -566,66 +566,68 @@ class SubTaskCreationFromTaskBlueprintBeamformer(unittest.TestCase):
class SubtaskAllowedStateTransitionsTest(unittest.TestCase):
def test_successful_path(self):
subtask = models.Subtask.objects.create(**Subtask_test_data())
- for state_value in (models.SubtaskState.Choices.DEFINING.value,
- models.SubtaskState.Choices.DEFINED.value,
- models.SubtaskState.Choices.SCHEDULING.value,
- models.SubtaskState.Choices.SCHEDULED.value,
- models.SubtaskState.Choices.QUEUEING.value,
- models.SubtaskState.Choices.QUEUED.value,
- models.SubtaskState.Choices.STARTING.value,
- models.SubtaskState.Choices.STARTED.value,
- models.SubtaskState.Choices.FINISHING.value,
- models.SubtaskState.Choices.FINISHED.value):
- subtask.state = models.SubtaskState.objects.get(value=state_value)
+ for state_value in (SubtaskState.Choices.DEFINING.value,
+ SubtaskState.Choices.DEFINED.value,
+ SubtaskState.Choices.SCHEDULING.value,
+ SubtaskState.Choices.SCHEDULED.value,
+ SubtaskState.Choices.QUEUEING.value,
+ SubtaskState.Choices.QUEUED.value,
+ SubtaskState.Choices.STARTING.value,
+ SubtaskState.Choices.STARTED.value,
+ SubtaskState.Choices.FINISHING.value,
+ SubtaskState.Choices.FINISHED.value):
+ subtask.state = SubtaskState.objects.get(value=state_value)
# no SubtaskIllegalStateTransitionException should be raised upon save. If it is raised, then test fails. No need for asserts.
subtask.save()
def test_helper_method_set_subtask_state_following_allowed_transitions_successful_path(self):
- for state_value in (models.SubtaskState.Choices.DEFINING.value,
- models.SubtaskState.Choices.DEFINED.value,
- models.SubtaskState.Choices.SCHEDULING.value,
- models.SubtaskState.Choices.SCHEDULED.value,
- models.SubtaskState.Choices.QUEUEING.value,
- models.SubtaskState.Choices.QUEUED.value,
- models.SubtaskState.Choices.STARTING.value,
- models.SubtaskState.Choices.STARTED.value,
- models.SubtaskState.Choices.FINISHING.value,
- models.SubtaskState.Choices.FINISHED.value):
+ for state_value in (SubtaskState.Choices.DEFINING.value,
+ SubtaskState.Choices.DEFINED.value,
+ SubtaskState.Choices.SCHEDULING.value,
+ SubtaskState.Choices.SCHEDULED.value,
+ SubtaskState.Choices.QUEUEING.value,
+ SubtaskState.Choices.QUEUED.value,
+ SubtaskState.Choices.STARTING.value,
+ SubtaskState.Choices.STARTED.value,
+ SubtaskState.Choices.FINISHING.value,
+ SubtaskState.Choices.FINISHED.value):
# start with subtask in defining state each time
- subtask = models.Subtask.objects.create(**Subtask_test_data(state=models.SubtaskState.objects.get(value=models.SubtaskState.Choices.DEFINING.value)))
- self.assertEqual(models.SubtaskState.Choices.DEFINING.value, subtask.state.value)
+ subtask = models.Subtask.objects.create(**Subtask_test_data(state=SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value)))
+ self.assertEqual(SubtaskState.Choices.DEFINING.value, subtask.state.value)
set_subtask_state_following_allowed_transitions(subtask, state_value)
self.assertEqual(state_value, subtask.state.value)
def test_helper_method_set_subtask_state_following_allowed_transitions_error_path(self):
- for state_value in (models.SubtaskState.Choices.DEFINING.value,
- models.SubtaskState.Choices.SCHEDULING.value,
- models.SubtaskState.Choices.QUEUEING.value,
- models.SubtaskState.Choices.STARTING.value,
- models.SubtaskState.Choices.STARTED.value,
- models.SubtaskState.Choices.FINISHING.value):
+ for intermediate_state_value in (SubtaskState.Choices.DEFINING.value,
+ SubtaskState.Choices.SCHEDULING.value,
+ SubtaskState.Choices.UNSCHEDULING.value,
+ SubtaskState.Choices.QUEUEING.value,
+ SubtaskState.Choices.STARTING.value,
+ SubtaskState.Choices.STARTED.value,
+ SubtaskState.Choices.FINISHING.value):
# start with subtask in defining state each time
- subtask = models.Subtask.objects.create(**Subtask_test_data(state=models.SubtaskState.objects.get(value=models.SubtaskState.Choices.DEFINING.value)))
- self.assertEqual(models.SubtaskState.Choices.DEFINING.value, subtask.state.value)
+ subtask = models.Subtask.objects.create(**Subtask_test_data(state=SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value)))
+ self.assertEqual(SubtaskState.Choices.DEFINING.value, subtask.state.value)
# then go to the requested intermediate state
- set_subtask_state_following_allowed_transitions(subtask, state_value)
- self.assertEqual(state_value, subtask.state.value)
+ set_subtask_state_following_allowed_transitions(subtask, intermediate_state_value)
+ self.assertEqual(intermediate_state_value, subtask.state.value)
# then go to the error state (should be allowed from any of these intermediate states)
- set_subtask_state_following_allowed_transitions(subtask, models.SubtaskState.Choices.ERROR.value)
- self.assertEqual(models.SubtaskState.Choices.ERROR.value, subtask.state.value)
+ subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.ERROR.value)
+ subtask.save()
+ self.assertEqual(SubtaskState.Choices.ERROR.value, subtask.state.value)
def test_helper_method_set_subtask_state_following_allowed_transitions_cancel_path(self):
- for desired_end_state_value in (models.SubtaskState.Choices.CANCELLING.value,models.SubtaskState.Choices.CANCELLED.value):
- for state_value in (models.SubtaskState.Choices.DEFINED.value,
- models.SubtaskState.Choices.SCHEDULED.value,
- models.SubtaskState.Choices.QUEUED.value,
- models.SubtaskState.Choices.STARTED.value):
+ for desired_end_state_value in (SubtaskState.Choices.CANCELLING.value,SubtaskState.Choices.CANCELLED.value):
+ for state_value in (SubtaskState.Choices.DEFINED.value,
+ SubtaskState.Choices.SCHEDULED.value,
+ SubtaskState.Choices.QUEUED.value,
+ SubtaskState.Choices.STARTED.value):
# start with subtask in defining state each time
- subtask = models.Subtask.objects.create(**Subtask_test_data(state=models.SubtaskState.objects.get(value=models.SubtaskState.Choices.DEFINING.value)))
- self.assertEqual(models.SubtaskState.Choices.DEFINING.value, subtask.state.value)
+ subtask = models.Subtask.objects.create(**Subtask_test_data(state=SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value)))
+ self.assertEqual(SubtaskState.Choices.DEFINING.value, subtask.state.value)
# then go to the requested intermediate state
set_subtask_state_following_allowed_transitions(subtask, state_value)
@@ -637,34 +639,44 @@ class SubtaskAllowedStateTransitionsTest(unittest.TestCase):
def test_helper_method_set_subtask_state_following_allowed_transitions_unscheduling_path(self):
# start with subtask in defining state
- subtask = models.Subtask.objects.create(**Subtask_test_data(state=models.SubtaskState.objects.get(value=models.SubtaskState.Choices.DEFINING.value)))
- self.assertEqual(models.SubtaskState.Choices.DEFINING.value, subtask.state.value)
+ subtask = models.Subtask.objects.create(**Subtask_test_data(state=SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value)))
+ self.assertEqual(SubtaskState.Choices.DEFINING.value, subtask.state.value)
# use helper method to follow the allowed path to 'unscheduling'
- set_subtask_state_following_allowed_transitions(subtask, models.SubtaskState.Choices.UNSCHEDULING.value)
- self.assertEqual(models.SubtaskState.Choices.UNSCHEDULING.value, subtask.state.value)
+ set_subtask_state_following_allowed_transitions(subtask, SubtaskState.Choices.UNSCHEDULING.value)
+ self.assertEqual(SubtaskState.Choices.UNSCHEDULING.value, subtask.state.value)
# check transition path
state_log = SubtaskStateLog.objects.filter(subtask=subtask).order_by('created_at').all()
- self.assertEqual(models.SubtaskState.Choices.DEFINING.value, state_log[0].new_state.value)
- self.assertEqual(models.SubtaskState.Choices.DEFINED.value, state_log[1].new_state.value)
- self.assertEqual(models.SubtaskState.Choices.SCHEDULING.value, state_log[2].new_state.value)
- self.assertEqual(models.SubtaskState.Choices.SCHEDULED.value, state_log[3].new_state.value)
- self.assertEqual(models.SubtaskState.Choices.UNSCHEDULING.value, state_log[4].new_state.value)
+ self.assertEqual(SubtaskState.Choices.DEFINING.value, state_log[0].new_state.value)
+ self.assertEqual(SubtaskState.Choices.DEFINED.value, state_log[1].new_state.value)
+ self.assertEqual(SubtaskState.Choices.SCHEDULING.value, state_log[2].new_state.value)
+ self.assertEqual(SubtaskState.Choices.SCHEDULED.value, state_log[3].new_state.value)
+ self.assertEqual(SubtaskState.Choices.UNSCHEDULING.value, state_log[4].new_state.value)
+
+ def test_end_states(self):
+ '''Check if the end states that we cannot get out of are according to the design'''
+ # there should be no state to go to from ERROR
+ self.assertEqual(0, SubtaskAllowedStateTransitions.objects.filter(old_state__value=SubtaskState.Choices.UNSCHEDULABLE.value).count())
+
+ # there should be no state to go to from FINISHED
+ self.assertEqual(0, SubtaskAllowedStateTransitions.objects.filter(old_state__value=SubtaskState.Choices.FINISHED.value).count())
+ # there should be no state to go to from CANCELLED
+ self.assertEqual(0, SubtaskAllowedStateTransitions.objects.filter(old_state__value=SubtaskState.Choices.CANCELLED.value).count())
def test_illegal_state_transitions(self):
- for state_value in [choice.value for choice in models.SubtaskState.Choices]:
+ for state_value in [choice.value for choice in SubtaskState.Choices]:
# assume helper method set_subtask_state_following_allowed_transitions is working (see other tests above)
# use it to create subtask in desired initial state
- subtask = models.Subtask.objects.create(**Subtask_test_data(state=models.SubtaskState.objects.get(value=models.SubtaskState.Choices.DEFINING.value)))
+ subtask = models.Subtask.objects.create(**Subtask_test_data(state=SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value)))
subtask = set_subtask_state_following_allowed_transitions(subtask, state_value)
self.assertEqual(state_value, subtask.state.value)
# derive the allowed and illegal state transitions states
allowed_new_states = SubtaskAllowedStateTransitions.allowed_new_states(subtask.state)
illegal_new_states = SubtaskAllowedStateTransitions.illegal_new_states(subtask.state)
- logger.info("test_illigal_state_transitions: old_state='%s' allowed_new_states=%s illegal_new_states=%s", state_value, [s.value for s in allowed_new_states], [s.value for s in illegal_new_states])
+ logger.info("test_illegal_state_transitions: old_state='%s' allowed_new_states=%s illegal_new_states=%s", state_value, [s.value for s in allowed_new_states], [s.value for s in illegal_new_states])
for illegal_new_state in illegal_new_states:
subtask.state = illegal_new_state
diff --git a/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py b/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py
index 7248ff73a2c58498048cec1df10da470800907d2..d7515c0afdd7169c391097f628cff0248a99bf1c 100755
--- a/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py
+++ b/SAS/TMSS/backend/test/t_tmssapp_specification_REST_API.py
@@ -615,9 +615,9 @@ class TaskConnectorTestCase(unittest.TestCase):
# POST a new item with invalid choice
test_data_invalid = dict(test_data_creator.TaskConnectorType(task_template_url=self.task_template_url))
- test_data_invalid['dataformats'] = [BASE_URL + '/dataformat/forbidden/']
+ test_data_invalid['dataformat'] = BASE_URL + '/dataformat/forbidden/'
r_dict = POST_and_assert_expected_response(self, BASE_URL + '/task_connector_type/', test_data_invalid, 400, {})
- self.assertTrue('Invalid hyperlink' in str(r_dict['dataformats']))
+ self.assertTrue('Invalid hyperlink' in str(r_dict['dataformat']))
def test_task_connector_POST_nonexistant_task_template_raises_error(self):
@@ -665,8 +665,7 @@ class TaskConnectorTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, url, tc_test_data)
test_patch = {"role": BASE_URL + '/role/calibrator',
- "dataformats": [BASE_URL + '/dataformat/Beamformed',
- BASE_URL + '/dataformat/MeasurementSet']}
+ "dataformat": BASE_URL + '/dataformat/Beamformed'}
# PATCH item and verify
PATCH_and_assert_expected_response(self, url, test_patch, 200, test_patch)
diff --git a/SAS/TMSS/backend/test/test_utils.py b/SAS/TMSS/backend/test/test_utils.py
index b19a5b544fa3270e5e3b060d575f6e3d81cdbc8a..cd815fcc66b5f907b2344c3cba6d775e7f863e30 100644
--- a/SAS/TMSS/backend/test/test_utils.py
+++ b/SAS/TMSS/backend/test/test_utils.py
@@ -83,17 +83,20 @@ def minimal_json_schema(title:str="my title", description:str="my description",
}
-def set_subtask_state_following_allowed_transitions(subtask: typing.Union[Subtask, int], state_value:str) -> Subtask:
- '''helper function to set subtask state following allowed transitions'''
+def set_subtask_state_following_allowed_transitions(subtask: typing.Union[Subtask, int], desired_state_value:str) -> Subtask:
+ '''helper function to set subtask state to the desired_state_value following allowed transitions
+ Please note that this function is meant to be used in unit/intgration tests only to "simulate" subtask going
+ from one state to the desired state, and thus preventing repetitive code like set_state A, set state B ... etc'''
if isinstance(subtask, int):
# the given subtask is an id. Fetch object.
subtask = Subtask.objects.get(id=subtask)
- while subtask.state.value != state_value and (subtask.state.value not in (SubtaskState.Choices.FINISHED.value,
- SubtaskState.Choices.ERROR.value,
- SubtaskState.Choices.CANCELLED.value)):
+ # end states that we cannot get out of accoring to the design
+ END_STATE_VALUES = (SubtaskState.Choices.FINISHED.value, SubtaskState.Choices.UNSCHEDULABLE.value, SubtaskState.Choices.CANCELLED.value)
+
+ while subtask.state.value != desired_state_value and (subtask.state.value not in END_STATE_VALUES):
# handle "unsuccessful path" to cancelled/canceling end state
- if state_value in (SubtaskState.Choices.CANCELLED.value, SubtaskState.Choices.CANCELLING.value) and \
+ if desired_state_value in (SubtaskState.Choices.CANCELLED.value, SubtaskState.Choices.CANCELLING.value) and \
subtask.state.value not in (SubtaskState.Choices.DEFINING.value,
SubtaskState.Choices.QUEUEING.value,
SubtaskState.Choices.STARTING.value,
@@ -102,15 +105,19 @@ def set_subtask_state_following_allowed_transitions(subtask: typing.Union[Subtas
subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.CANCELLING.value)
# handle "unsuccessful path" to error end state
- elif state_value == SubtaskState.Choices.ERROR.value and subtask.state.value in (SubtaskState.Choices.DEFINING.value,
- SubtaskState.Choices.QUEUEING.value,
- SubtaskState.Choices.STARTING.value,
- SubtaskState.Choices.FINISHING.value,
- SubtaskState.Choices.CANCELLING.value):
+ elif desired_state_value == SubtaskState.Choices.ERROR.value and subtask.state.value in (SubtaskState.Choices.DEFINING.value,
+ SubtaskState.Choices.QUEUEING.value,
+ SubtaskState.Choices.STARTING.value,
+ SubtaskState.Choices.FINISHING.value,
+ SubtaskState.Choices.CANCELLING.value):
subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.ERROR.value)
+ # handle "unsuccessful path" to unschedulable end state
+ elif desired_state_value == SubtaskState.Choices.UNSCHEDULABLE.value and subtask.state.value == SubtaskState.Choices.SCHEDULING.value:
+ subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULABLE.value)
+
# handle reverse path to unscheduling
- elif state_value == SubtaskState.Choices.UNSCHEDULING.value and subtask.state.value in (SubtaskState.Choices.SCHEDULED.value):
+ elif desired_state_value == SubtaskState.Choices.UNSCHEDULING.value and subtask.state.value in (SubtaskState.Choices.SCHEDULED.value):
subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.UNSCHEDULING.value)
else:
# handle "normal successful path"
@@ -138,4 +145,6 @@ def set_subtask_state_following_allowed_transitions(subtask: typing.Union[Subtas
subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.DEFINED.value)
subtask.save()
+ # loop, check in while statement at top if we reached the desired state already.
+
return subtask
diff --git a/SAS/TMSS/backend/test/tmss_test_data_django_models.py b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
index fb0d9a88c0b541baf613d8626b6d0f514536ceb5..9b7024f59cb7d6f0f06e429dc72ffb08fd231ef2 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_django_models.py
@@ -106,6 +106,7 @@ def TaskRelationSelectionTemplate_test_data(name="my_TaskRelationSelectionTempla
def TaskConnectorType_test_data() -> dict:
return {"role": models.Role.objects.get(value='calibrator'),
"datatype": models.Datatype.objects.get(value='instrument model'),
+ "dataformat": models.Dataformat.objects.get(value='Beamformed'),
"task_template": models.TaskTemplate.objects.create(**TaskTemplate_test_data()),
"iotype": models.IOType.objects.get(value=models.IOType.Choices.OUTPUT.value),
"tags": []}
@@ -243,7 +244,6 @@ def TaskRelationDraft_test_data(producer: models.TaskDraft = None, consumer: mod
return {"tags": [],
"selection_doc": {},
- "dataformat": models.Dataformat.objects.get(value='Beamformed'),
"producer": producer,
"consumer": consumer,
"input_role": models.TaskConnectorType.objects.create(**TaskConnectorType_test_data()),
@@ -307,7 +307,6 @@ def TaskRelationBlueprint_test_data(producer: models.TaskBlueprint = None, consu
return {"tags": [],
"selection_doc": {},
- "dataformat": models.Dataformat.objects.get(value='Beamformed'),
"input_role": models.TaskConnectorType.objects.create(**TaskConnectorType_test_data()),
"output_role": models.TaskConnectorType.objects.create(**TaskConnectorType_test_data()),
"draft": models.TaskRelationDraft.objects.create(**TaskRelationDraft_test_data()),
diff --git a/SAS/TMSS/backend/test/tmss_test_data_rest.py b/SAS/TMSS/backend/test/tmss_test_data_rest.py
index 3ac9952f3b0a98efd8caef3b36b1a90deff60e19..4b74a99f08e150ac3dd61c17157696fb048bf5c9 100644
--- a/SAS/TMSS/backend/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/backend/test/tmss_test_data_rest.py
@@ -227,7 +227,7 @@ class TMSSRESTTestDataCreator():
return {"role": self.django_api_url + '/role/%s'%role,
"datatype": self.django_api_url + '/datatype/image',
- "dataformats": [self.django_api_url + '/dataformat/Beamformed'],
+ "dataformat": self.django_api_url + '/dataformat/Beamformed',
"task_template": task_template_url,
"iotype": self.django_api_url + '/iotype/%s'%iotype,
"tags": []}
@@ -439,7 +439,6 @@ class TMSSRESTTestDataCreator():
return {"tags": [],
"selection_doc": selection_doc,
- "dataformat": self.django_api_url + "/dataformat/Beamformed",
"producer": producer_url,
"consumer": consumer_url,
"input_role": input_role_url,
@@ -539,7 +538,6 @@ class TMSSRESTTestDataCreator():
# test data
return {"tags": [],
"selection_doc": selection_doc,
- "dataformat": self.django_api_url + '/dataformat/MeasurementSet',
"input_role": input_role_url,
"output_role": output_role_url,
"draft": draft_url,
diff --git a/SAS/TMSS/client/lib/tmss_http_rest_client.py b/SAS/TMSS/client/lib/tmss_http_rest_client.py
index a4175393de18e50a6e0d4f19170fc27bcaf75780..8599dcf7e450ac081aaba34823ad713794744f07 100644
--- a/SAS/TMSS/client/lib/tmss_http_rest_client.py
+++ b/SAS/TMSS/client/lib/tmss_http_rest_client.py
@@ -127,12 +127,9 @@ class TMSSsession(object):
json_doc = {'state': "%s/subtask_state/%s/" % (self.api_url, status)}
if status == 'finishing' or status == 'cancelling':
json_doc['stop_time'] = datetime.utcnow().isoformat()
- if status == 'cancelling':
- json_doc['do_cancel'] = json_doc['stop_time']
-
+ logger.info("updating subtask id=%s status to '%s'", subtask_id, status)
response = self.session.patch(url='%s/subtask/%s/' % (self.api_url, subtask_id),
- json=json_doc,
- params={'format':'json'})
+ json=json_doc)
if response.status_code >= 200 and response.status_code < 300:
return json.loads(response.content.decode('utf-8'))
@@ -362,6 +359,14 @@ class TMSSsession(object):
returns a dict with the 'id' and 'progress', or raises."""
return self.get_path_as_json_object('subtask/%s/get_progress' % subtask_id)
+ def get_subtasks_in_same_scheduling_unit(self, subtask: dict) -> []:
+ """get all subtasks in the same scheduling_unit for the given subtask.
+ returns a list of subtask-dicts upon success, or raises."""
+ task_blueprint = self.get_url_as_json_object(subtask['task_blueprint'])
+ scheduling_unit_blueprint = self.get_url_as_json_object(task_blueprint['scheduling_unit_blueprint'])
+ subtasks = self.get_url_as_json_object(full_url=scheduling_unit_blueprint['url'].rstrip('/') + '/subtasks')
+ return subtasks
+
def get_setting(self, setting_name: str) -> {}:
"""get the value of a TMSS setting.
returns the setting value upon success, or raises."""
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.create.js
index e56f6d8a4d1cfc6dd273e13e744dfdbd2dd72018..3b2dc318d629e729a796321ae29ea1f9c18f0572 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.create.js
@@ -9,11 +9,11 @@ import { InputTextarea } from 'primereact/inputtextarea';
import { Button } from 'primereact/button';
import { Dialog } from 'primereact/components/dialog/Dialog';
import Flatpickr from "react-flatpickr";
-
import AppLoader from '../../layout/components/AppLoader';
import PageHeader from '../../layout/components/PageHeader';
import UIConstants from '../../utils/ui.constants';
import { CustomDialog } from '../../layout/components/CustomDialog';
+
import ProjectService from '../../services/project.service';
import ReservationService from '../../services/reservation.service';
import Jeditor from '../../components/JSONEditor/JEditor';
@@ -44,6 +44,9 @@ export class ReservationCreate extends Component {
stop_time: null,
project: (props.match?props.match.params.project:null) || null,
},
+ reservationStrategy: {
+ id: null,
+ },
errors: {}, // Validation Errors
validFields: {}, // For Validation
validForm: false, // To enable Save Button
@@ -51,6 +54,7 @@ export class ReservationCreate extends Component {
};
this.projects = []; // All projects to load project dropdown
this.reservationTemplates = [];
+ this.reservationStrategies = [];
// Validateion Rules
this.formRules = {
@@ -67,6 +71,8 @@ export class ReservationCreate extends Component {
this.checkIsDirty = this.checkIsDirty.bind(this);
this.close = this.close.bind(this);
this.initReservation = this.initReservation.bind(this);
+ this.changeStrategy = this.changeStrategy.bind(this);
+ this.setEditorFunction = this.setEditorFunction.bind(this);
}
async componentDidMount() {
@@ -74,19 +80,20 @@ export class ReservationCreate extends Component {
}
/**
- * Initialized the reservation template
+ * Initialize the reservation and relevant details
*/
async initReservation() {
const promises = [ ProjectService.getProjectList(),
ReservationService.getReservationTemplates(),
- UtilService.getUTC()
+ UtilService.getUTC(),
+ ReservationService.getReservationStrategyTemplates()
];
let emptyProjects = [{url: null, name: "Select Project"}];
- Promise.all(promises).then(responses => {
- let systemTime = moment.utc(responses[2]);
+ Promise.all(promises).then(responses => {
this.projects = emptyProjects.concat(responses[0]);
this.reservationTemplates = responses[1];
-
+ let systemTime = moment.utc(responses[2]);
+ this.reservationStrategies = responses[3];
let reservationTemplate = this.reservationTemplates.find(reason => reason.name === 'resource reservation');
let schema = {
properties: {}
@@ -98,12 +105,35 @@ export class ReservationCreate extends Component {
paramsSchema: schema,
isLoading: false,
reservationTemplate: reservationTemplate,
- systemTime: systemTime
+ systemTime: systemTime,
});
});
}
+ /**
+ *
+ * @param {Id} strategyId - id value of reservation strategy template
+ */
+ async changeStrategy(strategyId) {
+ this.setState({isLoading: true});
+ const reservationStrategy = _.find(this.reservationStrategies, {'id': strategyId});
+ let paramsOutput = {};
+ if(reservationStrategy.template.parameters) {
+ //if reservation strategy has parameter then prepare output parameter
+
+ } else {
+ paramsOutput = _.cloneDeep(reservationStrategy.template);
+ delete paramsOutput["$id"];
+ }
+ this.setState({
+ isLoading: false,
+ reservationStrategy: reservationStrategy,
+ paramsOutput: paramsOutput,
+ isDirty: true});
+ this.initReservation();
+ }
+
/**
* Function to set form values to the Reservation object
* @param {string} key
@@ -123,8 +153,6 @@ export class ReservationCreate extends Component {
[key]: true
}});
}
-
-
}
/**
@@ -253,9 +281,7 @@ export class ReservationCreate extends Component {
if (reservation && reservation.id){
const dialog = {header: 'Success', detail: 'Reservation is created successfully. Do you want to create another Reservation?'};
this.setState({ dialogVisible: true, dialog: dialog, paramsOutput: {}, showDialog: false, isDirty: false})
- }/* else {
- this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to save Reservation', showDialog: false, isDirty: false});
- }*/
+ }
}
/**
@@ -274,6 +300,9 @@ export class ReservationCreate extends Component {
dialog: { header: '', detail: ''},
errors: [],
reservation: tmpReservation,
+ reservationStrategy: {
+ id: null,
+ },
paramsSchema: null,
paramsOutput: null,
validEditor: false,
@@ -308,6 +337,14 @@ export class ReservationCreate extends Component {
this.setState({showDialog: false});
}
+ /**
+ * JEditor's function that to be called when parent wants to trigger change in the JSON Editor
+ * @param {Function} editorFunction
+ */
+ setEditorFunction(editorFunction) {
+ this.setState({editorFunction: editorFunction});
+ }
+
render() {
if (this.state.redirect) {
return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
@@ -437,6 +474,19 @@ export class ReservationCreate extends Component {
{(this.state.errors.project && this.state.touched.project) ? this.state.errors.project : "Select Project"}
</label>
</div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="strategy" className="col-lg-2 col-md-2 col-sm-12">Reservation Strategy</label>
+ <div className="col-lg-3 col-md-3 col-sm-12" data-testid="strategy" >
+ <Dropdown inputId="strategy" optionLabel="name" optionValue="id"
+ tooltip="Choose Reservation Strategy Template to set default values for create Reservation" tooltipOptions={this.tooltipOptions}
+ value={this.state.reservationStrategy.id}
+ options={this.reservationStrategies}
+ onChange={(e) => {this.changeStrategy(e.value)}}
+ placeholder="Select Strategy" />
+ <label className={(this.state.errors.reservationStrategy && this.state.touched.reservationStrategy) ?"error":"info"}>
+ {(this.state.errors.reservationStrategy && this.state.touched.reservationStrategy) ? this.state.errors.reservationStrategy : "Select Reservation Strategy Template"}
+ </label>
+ </div>
</div>
<div className="p-grid">
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.list.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.list.js
index 979508e47c8880a62d0e368adb8d356620c3d653..a1192925ef42f23a809a60da71488e4d3430e7a9 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.list.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Reservation/reservation.list.js
@@ -167,7 +167,7 @@ export class ReservationList extends Component{
mergeResourceWithReservation ( reservation, params) {
if( params ){
Object.keys(params).map((key, i) => (
- key !== 'description'? reservation[key]= params[key] : ''
+ ['name', 'description'].indexOf(key)<0? reservation[key]= params[key] : ''
));
}
return reservation;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/dataproduct.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/dataproduct.js
index c4ff0f2b0f63db9291702c4e72c206593119cd31..11ef48d543dcce7b4443e9eef226e42f8a50d810 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/dataproduct.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Task/dataproduct.js
@@ -64,7 +64,7 @@ export class DataProduct extends Component{
for(const id of subTaskIds){
let storageLocation = '';
await DataProductService.getSubtask(id).then( subtask =>{
- storageLocation = subtask.data.cluster_value;
+ storageLocation = subtask.data.cluster_name;
})
//Fetch data product for Input Subtask and Output Subtask
await DataProductService.getSubtaskInputDataproduct(id).then(async inputdata =>{
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/reservation.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/reservation.service.js
index d50476d3d0e5faeff74d5b39e2c6fe847a9a5b2b..5811e0e844a453b69b2903d4c3ab51c4927c1742 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/reservation.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/reservation.service.js
@@ -74,6 +74,15 @@ const ReservationService = {
console.error(error);
}
},
+ getReservationStrategyTemplates: async function () {
+ try {
+ const url = `/api/reservation_strategy_template/?ordering=id`;
+ const response = await axios.get(url);
+ return response.data.results;
+ } catch (error) {
+ console.error(error);
+ }
+ },
}
export default ReservationService;