From 399e65545e347c9f4c31474f22039a221fff5dee Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Fri, 2 Jun 2023 11:18:20 +0200
Subject: [PATCH] TMSS-2565: removed tmss_ra_adapter now that MoM/OTDB is not
being used for observing anymore :)
---
CMake/LofarPackageList.cmake | 3 +-
Docker/lofar-ci/Dockerfile_ci_tmss | 5 +-
SAS/TMSS/backend/services/CMakeLists.txt | 1 -
.../services/ra_adapter/CMakeLists.txt | 5 -
.../services/ra_adapter/bin/CMakeLists.txt | 2 -
.../services/ra_adapter/bin/tmss_ra_adapter | 24 --
.../services/ra_adapter/lib/CMakeLists.txt | 10 -
.../services/ra_adapter/lib/ra_adapter.py | 208 ------------------
.../services/ra_adapter/requirements.txt | 1 -
SAS/TMSS/deploy/docker-compose.yml | 25 +--
10 files changed, 4 insertions(+), 280 deletions(-)
delete mode 100644 SAS/TMSS/backend/services/ra_adapter/CMakeLists.txt
delete mode 100644 SAS/TMSS/backend/services/ra_adapter/bin/CMakeLists.txt
delete mode 100755 SAS/TMSS/backend/services/ra_adapter/bin/tmss_ra_adapter
delete mode 100644 SAS/TMSS/backend/services/ra_adapter/lib/CMakeLists.txt
delete mode 100644 SAS/TMSS/backend/services/ra_adapter/lib/ra_adapter.py
delete mode 100644 SAS/TMSS/backend/services/ra_adapter/requirements.txt
diff --git a/CMake/LofarPackageList.cmake b/CMake/LofarPackageList.cmake
index eb719d222c8..185b233517f 100644
--- a/CMake/LofarPackageList.cmake
+++ b/CMake/LofarPackageList.cmake
@@ -1,5 +1,5 @@
# - Create for each LOFAR package a variable containing the absolute path to
-# its source directory.
+# its source directory.
#
# Generated by gen_LofarPackageList_cmake.sh at ma 5 dec 2022 15:15:08 CET
#
@@ -203,7 +203,6 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(TMSSLTAAdapter_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/backend/services/lta_adapter)
set(TMSSPostgresListenerService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/backend/services/postgres_listener)
set(TMSSPreCalculationsService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/backend/services/precalculations)
- set(TMSSRAAdapter_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/backend/services/ra_adapter)
set(TMSSSchedulingService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/backend/services/scheduling)
set(TMSSServices_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/backend/services)
set(TMSSSlackWebhookService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/backend/services/slack_webhook)
diff --git a/Docker/lofar-ci/Dockerfile_ci_tmss b/Docker/lofar-ci/Dockerfile_ci_tmss
index ee0d6989718..af84282eca9 100644
--- a/Docker/lofar-ci/Dockerfile_ci_tmss
+++ b/Docker/lofar-ci/Dockerfile_ci_tmss
@@ -8,7 +8,7 @@ FROM ci_base_ubuntu:$BASE_VERSION
RUN echo "Installing packages for TMSS..." && \
apt-get install -y liblog4cplus-2.0.5 liblog4cplus-dev libboost-all-dev libreadline-dev libreadline8 binutils gettext libldap-dev git default-jdk python3-twisted graphviz libaio1 libaio-dev unzip postgresql-client postgresql-server-dev-all postgresql-all libsasl2-dev
-
+
# Oracle decided in all its wisdom to not make use of rpm/deb
# So, we're forced to download the Oracle client packages, and configure the paths
# See: https://help.ubuntu.com/community/Oracle%20Instant%20Client
@@ -22,7 +22,6 @@ COPY SAS/TMSS/upper_constraints.txt tmss_constraints.txt
COPY SAS/TMSS/backend/services/lobster/requirements.txt tmss_lobster.txt
COPY SAS/TMSS/backend/services/ingest_tmss_adapter/requirements.txt tmss_ingest_tmss_adapter.txt
-COPY SAS/TMSS/backend/services/ra_adapter/requirements.txt tmss_ra_adapter.txt
COPY SAS/TMSS/backend/services/scheduling/requirements.txt tmss_scheduling.txt
COPY SAS/TMSS/backend/services/slack_webhook/requirements.txt tmss_slack_webhook.txt
COPY SAS/TMSS/backend/services/websocket/requirements.txt tmss_websocket.txt
@@ -36,7 +35,7 @@ RUN pip3 install astroplan cachetools comet coverage cx_Oracle cython django-aut
python-ldap-test python-qpid-proton pyxb==1.2.5 scipy SimpleWebSocketServer \
swagger-spec-validator testing.postgresql ujson websocket_client xmljson \
-r tmss_lobster.txt -r tmss_ingest_tmss_adapter.txt -r tmss_scheduling.txt \
- -r tmss_ra_adapter.txt -r tmss_slack_webhook.txt -r tmss_websocket.txt \
+ -r tmss_slack_webhook.txt -r tmss_websocket.txt \
-c tmss_constraints.txt
# Note: nodejs now comes with npm, do not install the npm package separately, since that will be taken from the ubuntu repo and is conflicting.
diff --git a/SAS/TMSS/backend/services/CMakeLists.txt b/SAS/TMSS/backend/services/CMakeLists.txt
index b1f9ca7de73..d28add1121a 100644
--- a/SAS/TMSS/backend/services/CMakeLists.txt
+++ b/SAS/TMSS/backend/services/CMakeLists.txt
@@ -6,7 +6,6 @@ lofar_add_package(TMSSLobster lobster)
lofar_add_package(TMSSLTAAdapter lta_adapter)
lofar_add_package(TMSSPostgresListenerService postgres_listener)
lofar_add_package(TMSSPreCalculationsService precalculations)
-lofar_add_package(TMSSRAAdapter ra_adapter)
lofar_add_package(TMSSSchedulingService scheduling)
lofar_add_package(TMSSSlackWebhookService slack_webhook)
lofar_add_package(TMSSWebSocketService websocket)
diff --git a/SAS/TMSS/backend/services/ra_adapter/CMakeLists.txt b/SAS/TMSS/backend/services/ra_adapter/CMakeLists.txt
deleted file mode 100644
index e438befc54e..00000000000
--- a/SAS/TMSS/backend/services/ra_adapter/CMakeLists.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-lofar_package(TMSSRAAdapter 0.1 DEPENDS TMSSClient RATaskSpecifiedService)
-
-add_subdirectory(lib)
-add_subdirectory(bin)
-
diff --git a/SAS/TMSS/backend/services/ra_adapter/bin/CMakeLists.txt b/SAS/TMSS/backend/services/ra_adapter/bin/CMakeLists.txt
deleted file mode 100644
index 31ee5977e19..00000000000
--- a/SAS/TMSS/backend/services/ra_adapter/bin/CMakeLists.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-lofar_add_bin_scripts(tmss_ra_adapter)
-
diff --git a/SAS/TMSS/backend/services/ra_adapter/bin/tmss_ra_adapter b/SAS/TMSS/backend/services/ra_adapter/bin/tmss_ra_adapter
deleted file mode 100755
index 2cc11a6a983..00000000000
--- a/SAS/TMSS/backend/services/ra_adapter/bin/tmss_ra_adapter
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/python3
-
-# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-
-from lofar.sas.tmss.services.ra_adapter import main
-
-if __name__ == "__main__":
- main()
diff --git a/SAS/TMSS/backend/services/ra_adapter/lib/CMakeLists.txt b/SAS/TMSS/backend/services/ra_adapter/lib/CMakeLists.txt
deleted file mode 100644
index 7d584bfa094..00000000000
--- a/SAS/TMSS/backend/services/ra_adapter/lib/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-lofar_find_package(PythonInterp 3.4 REQUIRED)
-include(PythonInstall)
-
-set(_py_files
- ra_adapter.py
-)
-
-python_install(${_py_files}
- DESTINATION lofar/sas/tmss/services
-)
diff --git a/SAS/TMSS/backend/services/ra_adapter/lib/ra_adapter.py b/SAS/TMSS/backend/services/ra_adapter/lib/ra_adapter.py
deleted file mode 100644
index eeadb94510f..00000000000
--- a/SAS/TMSS/backend/services/ra_adapter/lib/ra_adapter.py
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be
-# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-
-
-import logging
-import os
-from optparse import OptionParser, OptionGroup
-
-logger = logging.getLogger(__name__)
-
-import requests
-from dateutil import parser
-from datetime import timedelta
-import json
-from lofar.common import isProductionEnvironment, isTestEnvironment
-from lofar.common.util import waitForInterrupt
-from lofar.parameterset import parameterset
-from lofar.sas.tmss.client.tmssbuslistener import *
-from lofar.sas.tmss.client.tmss_http_rest_client import TMSSsession
-from lofar.sas.resourceassignment.resourceassigner.rabuslistener import RABusListener, RAEventMessageHandler
-from lofar.sas.resourceassignment.resourceassigner.rarpc import RARPC
-
-class TMSSEventMessageHandlerForRASynchronization(TMSSEventMessageHandler):
- '''Handle incoming TMSS subtask events, and propagate them into the RA as a reservation upon scheduling/unscheduling an observation subtask'''
- def __init__(self, rest_client_creds_id: str="TMSSClient"):
- super().__init__(log_event_messages=False)
- self._tmss_client = TMSSsession.create_from_dbcreds_for_ldap(rest_client_creds_id)
-
- def start_handling(self):
- self._tmss_client.open()
- super().start_handling()
-
- def stop_handling(self):
- super().stop_handling()
- self._tmss_client.close()
-
- def onSubTaskStatusChanged(self, id: int, status:str):
- if status in ('scheduled', 'unscheduling'):
- subtask = self._tmss_client.get_subtask(id)
- if subtask['subtask_type'] == 'observation':
- logger.info('Creating RA-reservation-specification for TMSS observation subtask: id=%s status=%s', id, status)
-
- # Lots of magic to create an RA task specification for a reservation.
- # The quick and dirty hack here is that we take the observation parset from TMSS
- # and just modify the processType/task_type into an RA reservation.
- # then just sumbit it to RA, and let it do the reservation-task creation/scheduling/unscheduling,
- # and making the claims on the used stations.
- parset_txt = self._tmss_client.get_subtask_parset(id)
- parset = parameterset.fromString(parset_txt)
- parset_dict = parset.dict()
- parset_dict['Observation.processType'] = 'RESERVATION' # for RA/OTDB this is a reservation
- parset_dict['Observation.processSubtype'] = 'project' # for RA/OTDB this is a reservation of the type project to denote that it is projects/observing relation (instead of maintenance)
-
- ra_spec = {'tmss_id': id,
- 'task_type': 'reservation', # for RA/OTDB this is a reservation
- 'task_subtype': 'project', # for RA/OTDB this is a reservation of the type project to denote that it is projects/observing relation (instead of maintenance)
- 'status': 'prescheduled' if status=='scheduled' else 'approved',
- 'starttime': parser.parse(subtask['scheduled_start_time'], ignoretz=True),
- 'endtime': parser.parse(subtask['scheduled_stop_time'], ignoretz=True),
- 'cluster': subtask['cluster_name'],
- 'station_requirements': [],
- 'specification': parset_dict}
-
- with RARPC.create() as rarpc:
- rarpc.do_assignment(ra_spec)
-
-
-class RAEventMessageHandlerForTMSSSynchronization(RAEventMessageHandler):
- '''Handle incoming RA task events, and propagate them into TMSS as a reservation upon scheduling/unscheduling an observation task'''
- def __init__(self, rest_client_creds_id: str="TMSSClient"):
- super().__init__()
- self._tmss_client = TMSSsession.create_from_dbcreds_for_ldap(rest_client_creds_id)
-
- def start_handling(self):
- self._tmss_client.open()
- super().start_handling()
-
- def stop_handling(self):
- super().stop_handling()
- self._tmss_client.close()
-
- def onTaskApproved(self, task_ids):
- self.onTaskApprovedOrScheduled(task_ids, 'approved')
-
- def onTaskScheduled(self, task_ids):
- self.onTaskApprovedOrScheduled(task_ids, 'scheduled')
-
- def onTaskApprovedOrScheduled(self, task_ids, status: str):
- otdb_id = task_ids.get('otdb_id')
- tmss_id = task_ids.get('tmss_id')
-
- if otdb_id is not None and tmss_id is None:
- if isProductionEnvironment():
- ra_url = 'http://scu001.control.lofar:7412'
- elif isTestEnvironment():
- ra_url = 'http://scu199.control.lofar:7412'
- else:
- ra_url = 'http://0.0.0.0:7412'
-
- ra_otdb_task_url = ra_url + '/rest/tasks/otdb/' + str(otdb_id)
- ra_task = json.loads(requests.get(ra_otdb_task_url).text).get('task')
-
- if ra_task.get('type') in ('observation', 'reservation'):
- reservation_name = "OTDB %s %s %s" % (ra_task['project_name'], ra_task.get('type'), ra_task['otdb_id'])
-
- if status in ('approved', 'prescheduled', 'scheduled'):
- # RA status 'approved' means that it is either new, or unscheduled.
- # RA status 'prescheduled'/'scheduled' means that it gets scheduled.
- # In both cases, check if we know any reservation for this otdb id.
- # If so, then the RA task was unscheduled, and we delete the corresponding TMSS reservation(s)
-
- # get exisiting reservations with this specific reservation name as key
- # this is hacky and not guaranteed unique, but hey, it's a temporary quick solution while both MoM/OTDB and TMSS are in use.
- # the name as key is unique enough until proven otherwise.
- existing_reservations = self._tmss_client.get_path_as_json_object('/reservation', {'name': reservation_name})
-
- for reservation in existing_reservations:
- logger.info('Deleting TMSS-reservation id=%s \'%s\' for unscheduled RA observation otdb_id=%s', reservation['id'], reservation_name, otdb_id)
- self._tmss_client.session.delete(url=reservation['url'])
-
-
- if status == 'scheduled':
- logger.info('Creating TMSS-reservation \'%s\' for RA %s otdb_id=%s', reservation_name, ra_task.get('type'), otdb_id)
-
- ra_claims_url = ra_url + '/rest/tasks/' + str(ra_task['id']) + '/resourceclaims'
- ra_claims = json.loads(requests.get(ra_claims_url).text).get('resourceclaims')
- station_claims = [c for c in ra_claims if c['resource_type_name']=='rcu']
- stations = [c['resource_name'].rstrip('rcu') for c in station_claims]
-
- # create a TMSS reservation
- # use the default reservation template, and fill in some details
- specifications_doc = self._tmss_client.get_path_as_json_object('/reservation_template/1/default')
- specifications_doc['activity']['subject'] = 'system'
- specifications_doc['activity']['type'] = 'other'
- specifications_doc['activity']['description'] = "OTDB observation %s %s" % (ra_task['project_name'], ra_task['otdb_id'])
- specifications_doc['schedulability']['dynamic'] = False
- specifications_doc['schedulability']['fixed_time'] = False
- specifications_doc['resources']['stations'] = stations
-
- reservation = {
- "name": "OTDB %s observation %s" % (ra_task['project_name'], ra_task['otdb_id']),
- "description": "A reservation while OTDB observation %s %s %s is observing" % (ra_task['project_name'], ra_task['otdb_id'], ra_task['description']),
- "specifications_doc": specifications_doc,
- "specifications_template": self._tmss_client.get_full_url_for_path('/reservation_template/1'),
- "start_time": (parser.parse(ra_task['starttime'], ignoretz=True) - timedelta(minutes=1)).isoformat(), # add one minute so that TMSS stops early enough before MoM starts
- "stop_time": (parser.parse(ra_task['endtime'], ignoretz=True) + timedelta(minutes=1)).isoformat() # add one minute so that TMSS does not start something else too soon after MoM finishes
- }
- self._tmss_client.post_to_path_and_get_result_as_json_object('/reservation', json_data=reservation)
-
-
-def create_tmss_buslistener(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER, rest_client_creds_id: str="TMSSClient"):
- return TMSSBusListener(handler_type=TMSSEventMessageHandlerForRASynchronization,
- handler_kwargs={'rest_client_creds_id': rest_client_creds_id},
- exchange=exchange, broker=broker)
-
-def create_ra_buslistener(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER, rest_client_creds_id: str="TMSSClient"):
- return RABusListener(handler_type=RAEventMessageHandlerForTMSSSynchronization,
- handler_kwargs={'rest_client_creds_id': rest_client_creds_id},
- exchange=exchange, broker=broker)
-
-def main():
- # make sure we run in UTC timezone
- os.environ['TZ'] = 'UTC'
-
- logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
-
- # Check the invocation arguments
- parser = OptionParser('%prog [options]',
- description='run the tmss_ra_adapter which continuously synchronizes TMSS and the ResourceAssigner. Each scheduled task in the one creates a reservation in the other.')
-
- group = OptionGroup(parser, 'Messaging options')
- group.add_option('-b', '--broker', dest='broker', type='string', default=DEFAULT_BROKER,
- help='Address of the message broker, default: %default')
- group.add_option('-e', "--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME,
- help="exchange where the RA & TMSS event messages are published. [default: %default]")
- parser.add_option_group(group)
-
- group = OptionGroup(parser, 'TMSS options')
- parser.add_option_group(group)
- group.add_option('-R', '--rest_credentials', dest='rest_credentials', type='string', default='TMSSClient', help='django REST API credentials name, default: %default')
-
- (options, args) = parser.parse_args()
-
- TMSSsession.check_connection_and_exit_on_error(options.rest_credentials)
-
- with create_tmss_buslistener(options.exchange, options.broker, rest_client_creds_id=options.rest_credentials):
- with create_ra_buslistener(options.exchange, options.broker, rest_client_creds_id=options.rest_credentials):
- waitForInterrupt()
-
-if __name__ == '__main__':
- main()
diff --git a/SAS/TMSS/backend/services/ra_adapter/requirements.txt b/SAS/TMSS/backend/services/ra_adapter/requirements.txt
deleted file mode 100644
index 12b55a18e76..00000000000
--- a/SAS/TMSS/backend/services/ra_adapter/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-requests # Apache 2
\ No newline at end of file
diff --git a/SAS/TMSS/deploy/docker-compose.yml b/SAS/TMSS/deploy/docker-compose.yml
index 9c57957a922..82017b243fa 100644
--- a/SAS/TMSS/deploy/docker-compose.yml
+++ b/SAS/TMSS/deploy/docker-compose.yml
@@ -135,29 +135,6 @@ services:
driver: journald
options:
tag: tmss_lta_adapter
- ra_adapter:
- container_name: tmss_ra_adapter
- image: tmss_ra_adapter
- build:
- context: ./app
- dockerfile: Dockerfile
- args:
- SOURCE_IMAGE: ${SOURCE_IMAGE}
- HOME: "/localhome/lofarsys"
- restart: unless-stopped
- env_file:
- - env
- environment:
- - USER=lofarsys
- - HOME=/localhome/lofarsys
- command: /bin/bash -c 'source /opt/lofar/lofarinit.sh; exec tmss_ra_adapter'
- depends_on:
- db_migrate:
- condition: service_completed_successfully
- logging:
- driver: journald
- options:
- tag: tmss_ra_adapter
ingest_adapter:
container_name: tmss_ingest_adapter
image: tmss_ingest_adapter
@@ -294,4 +271,4 @@ services:
logging:
driver: journald
options:
- tag: tmss_slack_webhook
\ No newline at end of file
+ tag: tmss_slack_webhook
--
GitLab