diff --git a/SAS/TMSS/backend/services/CMakeLists.txt b/SAS/TMSS/backend/services/CMakeLists.txt index abf80747de02e923c2954237b09496c7ba3166a1..2fb200270ac09e02e2c2daeb8668bc68044f11be 100644 --- a/SAS/TMSS/backend/services/CMakeLists.txt +++ b/SAS/TMSS/backend/services/CMakeLists.txt @@ -5,4 +5,5 @@ lofar_add_package(TMSSFeedbackHandlingService feedback_handling) lofar_add_package(TMSSPostgresListenerService tmss_postgres_listener) lofar_add_package(TMSSWebSocketService websocket) lofar_add_package(TMSSWorkflowService workflow_service) +lofar_add_package(TMSSLTAAdapter tmss_lta_adapter) diff --git a/SAS/TMSS/backend/services/tmss_lta_adapter/CMakeLists.txt b/SAS/TMSS/backend/services/tmss_lta_adapter/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..c71d7b72950b64ea247b49f146e809cad35b3c72 --- /dev/null +++ b/SAS/TMSS/backend/services/tmss_lta_adapter/CMakeLists.txt @@ -0,0 +1,5 @@ +lofar_package(TMSSLTAAdapter 0.1 DEPENDS TMSSClient LTACatalogue) + +add_subdirectory(lib) +add_subdirectory(bin) + diff --git a/SAS/TMSS/backend/services/tmss_lta_adapter/bin/CMakeLists.txt b/SAS/TMSS/backend/services/tmss_lta_adapter/bin/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..54899820278f8568f3e11b4b1fb2aa394df41b41 --- /dev/null +++ b/SAS/TMSS/backend/services/tmss_lta_adapter/bin/CMakeLists.txt @@ -0,0 +1,4 @@ +lofar_add_bin_scripts(tmss_lta_adapter) + +# supervisord config files +lofar_add_sysconf_files(tmss_lta_adapter.ini DESTINATION supervisord.d) diff --git a/SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter b/SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter new file mode 100755 index 0000000000000000000000000000000000000000..851710c2991b632a530c925eaa2457f2679605d1 --- /dev/null +++ b/SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter @@ -0,0 +1,24 @@ +#!/usr/bin/python3 + +# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy) +# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands +# +# This file is part of the LOFAR software suite. +# The LOFAR software suite is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# The LOFAR software suite is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>. + + +from lofar.sas.tmss.services.tmss_lta_adapter import main + +if __name__ == "__main__": + main() diff --git a/SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter.ini b/SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter.ini new file mode 100644 index 0000000000000000000000000000000000000000..e90faa5b95e31d421adf102bf592abc42e74b2cf --- /dev/null +++ b/SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter.ini @@ -0,0 +1,9 @@ +[program:tmss_lta_adapter] +command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_lta_adapter' +user=lofarsys +stopsignal=INT ; KeyboardInterrupt +stopasgroup=true ; bash does not propagate signals +stdout_logfile=%(program_name)s.log +redirect_stderr=true +stderr_logfile=NONE +stdout_logfile_maxbytes=0 diff --git a/SAS/TMSS/backend/services/tmss_lta_adapter/lib/CMakeLists.txt b/SAS/TMSS/backend/services/tmss_lta_adapter/lib/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b7a0766f48f04acc02818d50411b6661a9251495 --- /dev/null +++ b/SAS/TMSS/backend/services/tmss_lta_adapter/lib/CMakeLists.txt @@ -0,0 +1,10 @@ +lofar_find_package(PythonInterp 3.4 REQUIRED) +include(PythonInstall) + +set(_py_files + tmss_lta_adapter.py + ) + +python_install(${_py_files} + DESTINATION lofar/sas/tmss/services) + diff --git a/SAS/TMSS/backend/services/tmss_lta_adapter/lib/tmss_lta_adapter.py b/SAS/TMSS/backend/services/tmss_lta_adapter/lib/tmss_lta_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..5dd7629547fcaab58a7f14132e8fb779504fbcdb --- /dev/null +++ b/SAS/TMSS/backend/services/tmss_lta_adapter/lib/tmss_lta_adapter.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +# subtask_scheduling.py +# +# Copyright (C) 2015 +# ASTRON (Netherlands Institute for Radio Astronomy) +# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands +# +# This file is part of the LOFAR software suite. +# The LOFAR software suite is free software: you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# The LOFAR software suite is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>. + + +import logging +import os +from optparse import OptionParser, OptionGroup + +logger = logging.getLogger(__name__) + +from lofar.common import dbcredentials +from lofar.sas.tmss.client.tmssbuslistener import * +from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession +from lofar.lta.lta_catalogue_db import LTACatalogueDatabaseConnection + +class TMSSEventMessageHandlerForLTASynchronization(TMSSEventMessageHandler): + def __init__(self, rest_client_creds_id: str="TMSSClient", lta_creds_id: str="LTACatalogue"): + super().__init__(log_event_messages=False) + self._tmss_client = TMSSsession.create_from_dbcreds_for_ldap(rest_client_creds_id) + self._lta_db = LTACatalogueDatabaseConnection(dbcredentials.DBCredentials().get(lta_creds_id)) + + def onProjectCreated(self, name: str): + with self._tmss_client, self._lta_db: + project = self._tmss_client.get_path_as_json_object('project/%s' % name) + self._lta_db.create_project(project_name=project['name'], description=project['description']) + + def onProjectUpdated(self, name: str): + logger.warning("TODO: implement synchronization to the LTA when a Project is updated") + + def onProjectDeleted(self, name: str): + logger.warning("TODO: implement synchronization to the LTA when a Project is deleted") + + def onProjectQuotaArchiveLocationCreated(self, id: int): + with self._tmss_client, self._lta_db: + project_quota_archive_location = self._tmss_client.get_path_as_json_object('project_quota_archive_location/%s' % id) + project_quota = self._tmss_client.get_url_as_json_object(project_quota_archive_location['project_quota']) + project = self._tmss_client.get_url_as_json_object(project_quota['project']) + + assert project_quota['resource_type_id'] == 'lta_storage' + + self._lta_db.add_project_storage_resource(project_name=project['name'], nr_of_bytes=project_quota['value'], uri=project_quota_archive_location['full_archive_uri']) + + def onProjectQuotaArchiveLocationUpdated(self, id: int): + logger.warning("TODO: implement synchronization to the LTA when a ProjectQuotaArchiveLocation is updated") + + def onProjectQuotaArchiveLocationDeleted(self, id: int): + logger.warning("TODO: implement synchronization to the LTA when a ProjectQuotaArchiveLocation is deleted") + + +def create_service(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER, rest_client_creds_id: str="TMSSClient", lta_creds_id: str="LTACatalogue"): + return TMSSBusListener(handler_type=TMSSEventMessageHandlerForLTASynchronization, + handler_kwargs={'rest_client_creds_id': rest_client_creds_id, + 'lta_creds_id': lta_creds_id}, + exchange=exchange, broker=broker) + + +def main(): + # make sure we run in UTC timezone + os.environ['TZ'] = 'UTC' + + logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) + + # Check the invocation arguments + parser = OptionParser('%prog [options]', + description='run the tmss_lta_adapter which continuously synchronizes TMSS projects and archive storage allocation to the LTA Catalogue.') + + group = OptionGroup(parser, 'Messaging options') + group.add_option('-b', '--broker', dest='broker', type='string', default=DEFAULT_BROKER, + help='Address of the message broker, default: %default') + group.add_option('-e', "--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME, + help="exchange where the TMSS event messages are published. [default: %default]") + parser.add_option_group(group) + + group = OptionGroup(parser, 'Django options') + parser.add_option_group(group) + group.add_option('-R', '--rest_credentials', dest='rest_credentials', type='string', default='TMSSClient', help='django REST API credentials name, default: %default') + + group = OptionGroup(parser, 'LTA options') + parser.add_option_group(group) + group.add_option('-L', '--lta_credentials', dest='lta_credentials', type='string', default='LTACatalogue', help='LTA Catalogue credentials name, default: %default') + + (options, args) = parser.parse_args() + + TMSSsession.check_connection_and_exit_on_error(options.rest_credentials) + + with create_service(options.exchange, options.broker, rest_client_creds_id=options.rest_credentials, lta_creds_id=options.lta_credentials): + waitForInterrupt() + +if __name__ == '__main__': + main() diff --git a/SubSystems/SCU/SCU.ini b/SubSystems/SCU/SCU.ini index e310cab9d7407c22413ebe03e7ca26fe5461181b..b039a9ee2cb707ec234630b7f493c7135cf5b316 100644 --- a/SubSystems/SCU/SCU.ini +++ b/SubSystems/SCU/SCU.ini @@ -21,4 +21,4 @@ programs=messagelogger programs=autocleanupservice,cleanupservice,storagequeryservice [group:TMSS] -programs=tmss,tmss_feedback_handling_service,tmss_postgres_listener_service,tmss_scheduling_service,tmss_websocket_service,tmss_workflow_service +programs=tmss,tmss_feedback_handling_service,tmss_postgres_listener_service,tmss_scheduling_service,tmss_websocket_service,tmss_workflow_service,tmss_lta_adapter