TMSS-418: use new LTACatalogueDatabaseConnection and TMSSEventMessageHandler...

TMSS-418: use new LTACatalogueDatabaseConnection and TMSSEventMessageHandler to sync new peojects and storage quota to the LTA automatically in a background service

SAS/TMSS/backend/services/CMakeLists.txt

@@ -5,4 +5,5 @@ lofar_add_package(TMSSFeedbackHandlingService feedback_handling)
 lofar_add_package(TMSSPostgresListenerService tmss_postgres_listener)
 lofar_add_package(TMSSWebSocketService websocket)
 lofar_add_package(TMSSWorkflowService workflow_service)
+lofar_add_package(TMSSLTAAdapter tmss_lta_adapter)
 

SAS/TMSS/backend/services/tmss_lta_adapter/CMakeLists.txt

+lofar_package(TMSSLTAAdapter 0.1 DEPENDS TMSSClient LTACatalogue)
+
+add_subdirectory(lib)
+add_subdirectory(bin)
+

SAS/TMSS/backend/services/tmss_lta_adapter/bin/CMakeLists.txt

+lofar_add_bin_scripts(tmss_lta_adapter)
+
+# supervisord config files
+lofar_add_sysconf_files(tmss_lta_adapter.ini DESTINATION supervisord.d)

SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter

+#!/usr/bin/python3
+
+# Copyright (C) 2012-2015  ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+
+from lofar.sas.tmss.services.tmss_lta_adapter import main
+
+if __name__ == "__main__":
+    main()

SAS/TMSS/backend/services/tmss_lta_adapter/bin/tmss_lta_adapter.ini

+[program:tmss_lta_adapter]
+command=docker run --rm --net=host -u 7149:7149 -v /opt/lofar/var/log:/opt/lofar/var/log -v /tmp/tmp -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro -v /localhome/lofarsys:/localhome/lofarsys -e HOME=/localhome/lofarsys -e USER=lofarsys nexus.cep4.control.lofar:18080/tmss_django:latest /bin/bash -c 'source ~/.lofar/.lofar_env;source $LOFARROOT/lofarinit.sh;exec tmss_lta_adapter'
+user=lofarsys
+stopsignal=INT ; KeyboardInterrupt
+stopasgroup=true ; bash does not propagate signals
+stdout_logfile=%(program_name)s.log
+redirect_stderr=true
+stderr_logfile=NONE
+stdout_logfile_maxbytes=0

SAS/TMSS/backend/services/tmss_lta_adapter/lib/CMakeLists.txt

+lofar_find_package(PythonInterp 3.4 REQUIRED)
+include(PythonInstall)
+
+set(_py_files
+    tmss_lta_adapter.py
+    )
+
+python_install(${_py_files}
+    DESTINATION lofar/sas/tmss/services)
+

SAS/TMSS/backend/services/tmss_lta_adapter/lib/tmss_lta_adapter.py

+#!/usr/bin/env python3
+
+# subtask_scheduling.py
+#
+# Copyright (C) 2015
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+import os
+from optparse import OptionParser, OptionGroup
+
+logger = logging.getLogger(__name__)
+
+from lofar.common import dbcredentials
+from lofar.sas.tmss.client.tmssbuslistener import *
+from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
+from lofar.lta.lta_catalogue_db import LTACatalogueDatabaseConnection
+
+class TMSSEventMessageHandlerForLTASynchronization(TMSSEventMessageHandler):
+    def __init__(self, rest_client_creds_id: str="TMSSClient", lta_creds_id: str="LTACatalogue"):
+        super().__init__(log_event_messages=False)
+        self._tmss_client = TMSSsession.create_from_dbcreds_for_ldap(rest_client_creds_id)
+        self._lta_db = LTACatalogueDatabaseConnection(dbcredentials.DBCredentials().get(lta_creds_id))
+
+    def onProjectCreated(self, name: str):
+        with self._tmss_client, self._lta_db:
+            project = self._tmss_client.get_path_as_json_object('project/%s' % name)
+            self._lta_db.create_project(project_name=project['name'], description=project['description'])
+
+    def onProjectUpdated(self, name: str):
+        logger.warning("TODO: implement synchronization to the LTA when a Project is updated")
+
+    def onProjectDeleted(self, name: str):
+        logger.warning("TODO: implement synchronization to the LTA when a Project is deleted")
+
+    def onProjectQuotaArchiveLocationCreated(self, id: int):
+        with self._tmss_client, self._lta_db:
+            project_quota_archive_location = self._tmss_client.get_path_as_json_object('project_quota_archive_location/%s' % id)
+            project_quota = self._tmss_client.get_url_as_json_object(project_quota_archive_location['project_quota'])
+            project = self._tmss_client.get_url_as_json_object(project_quota['project'])
+
+            assert project_quota['resource_type_id'] == 'lta_storage'
+
+            self._lta_db.add_project_storage_resource(project_name=project['name'], nr_of_bytes=project_quota['value'], uri=project_quota_archive_location['full_archive_uri'])
+
+    def onProjectQuotaArchiveLocationUpdated(self, id: int):
+        logger.warning("TODO: implement synchronization to the LTA when a ProjectQuotaArchiveLocation is updated")
+
+    def onProjectQuotaArchiveLocationDeleted(self, id: int):
+        logger.warning("TODO: implement synchronization to the LTA when a ProjectQuotaArchiveLocation is deleted")
+
+
+def create_service(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER, rest_client_creds_id: str="TMSSClient", lta_creds_id: str="LTACatalogue"):
+    return TMSSBusListener(handler_type=TMSSEventMessageHandlerForLTASynchronization,
+                           handler_kwargs={'rest_client_creds_id': rest_client_creds_id,
+                                           'lta_creds_id': lta_creds_id},
+                           exchange=exchange, broker=broker)
+
+
+def main():
+    # make sure we run in UTC timezone
+    os.environ['TZ'] = 'UTC'
+
+    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+    # Check the invocation arguments
+    parser = OptionParser('%prog [options]',
+                          description='run the tmss_lta_adapter which continuously synchronizes TMSS projects and archive storage allocation to the LTA Catalogue.')
+
+    group = OptionGroup(parser, 'Messaging options')
+    group.add_option('-b', '--broker', dest='broker', type='string', default=DEFAULT_BROKER,
+                     help='Address of the message broker, default: %default')
+    group.add_option('-e', "--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME,
+                     help="exchange where the TMSS event messages are published. [default: %default]")
+    parser.add_option_group(group)
+
+    group = OptionGroup(parser, 'Django options')
+    parser.add_option_group(group)
+    group.add_option('-R', '--rest_credentials', dest='rest_credentials', type='string', default='TMSSClient', help='django REST API credentials name, default: %default')
+
+    group = OptionGroup(parser, 'LTA options')
+    parser.add_option_group(group)
+    group.add_option('-L', '--lta_credentials', dest='lta_credentials', type='string', default='LTACatalogue', help='LTA Catalogue credentials name, default: %default')
+
+    (options, args) = parser.parse_args()
+
+    TMSSsession.check_connection_and_exit_on_error(options.rest_credentials)
+
+    with create_service(options.exchange, options.broker, rest_client_creds_id=options.rest_credentials, lta_creds_id=options.lta_credentials):
+        waitForInterrupt()
+
+if __name__ == '__main__':
+    main()

SubSystems/SCU/SCU.ini

@@ -21,4 +21,4 @@ programs=messagelogger
 programs=autocleanupservice,cleanupservice,storagequeryservice
 
 [group:TMSS]
-programs=tmss,tmss_feedback_handling_service,tmss_postgres_listener_service,tmss_scheduling_service,tmss_websocket_service,tmss_workflow_service
+programs=tmss,tmss_feedback_handling_service,tmss_postgres_listener_service,tmss_scheduling_service,tmss_websocket_service,tmss_workflow_service,tmss_lta_adapter