Commit 34876a7a authored by Jorrit Schaap's avatar Jorrit Schaap

SW-776: Merge branch 'SW-776' into 'LOFAR-Release-4_0'

Resolve SW-776

See merge request ro/lofar!11
parents 73cc8578 091b5008
# - Create for each LOFAR package a variable containing the absolute path to
# its source directory.
#
# Generated by gen_LofarPackageList_cmake.sh at do 27 jun 2019 14:25:48 CEST
# Generated by gen_LofarPackageList_cmake.sh at ma 22 jul 2019 15:53:29 CEST
#
# ---- DO NOT EDIT ----
#
......@@ -176,6 +176,8 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(CleanupClient_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/DataManagement/Cleanup/CleanupClient)
set(AutoCleanupService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/DataManagement/Cleanup/AutoCleanupService)
set(MoMQueryService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/MoM/MoMQueryService)
set(MoMSimpleAPIs_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/MoM/MoMSimpleAPIs)
set(MoMutils_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/MoM/MoMutils)
set(MoMQueryServiceCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/MoM/MoMQueryService/MoMQueryServiceCommon)
set(MoMQueryServiceClient_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/MoM/MoMQueryService/MoMQueryServiceClient)
set(MoMQueryServiceServer_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/MoM/MoMQueryService/MoMQueryServiceServer)
......@@ -209,6 +211,6 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(DataManagement_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/DataManagement)
set(Dragnet_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/Dragnet)
set(LTAIngest_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/LTAIngest)
set(Cobalt_validation_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/Online_Cobalt/validation)
set(LTAIngestTransfer_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/LTAIngestTransfer)
set(Cobalt_validation_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/Online_Cobalt/validation)
endif(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
......@@ -10,74 +10,20 @@ from lofar.lta.ingest.server.config import MOM_BASE_URL
from lofar.common import isProductionEnvironment
from lofar.lta.ingest.server.sip import *
from lofar.common.util import humanreadablesize
from lofar.mom.simpleapis.momhttpclient import BaseMomClient
import urllib3
urllib3.disable_warnings()
class MoMClient:
class MoMClient(BaseMomClient):
"""This is an HTTP client that knows how to use the Single Sign On of Mom2.
It is used instead of a SOAP client, because SOAPpy doesn't support
form handling and cookies."""
def __init__(self, user = None, password = None):
if user == None or password == None:
# (mis)use dbcredentials to read user/pass from disk
from lofar.common import dbcredentials
dbc = dbcredentials.DBCredentials()
creds = dbc.get('MoM_site' if isProductionEnvironment() else 'MoM_site_test')
user = creds.user
password = creds.password
self.__user = user
self.__password = password
self.__session = None
self.__momURLlogin = MOM_BASE_URL + 'useradministration/user/systemlogin.do'
self.__momUR_security_check = MOM_BASE_URL + 'useradministration/user/j_security_check'
self.__momURLgetSIP = MOM_BASE_URL + 'mom3/interface/importXML2.do'
self.__momURLsetStatus = MOM_BASE_URL + 'mom3/interface/service/setStatusDataProduct.do'
self.__momURLlogout = MOM_BASE_URL + 'useradministration/user/logout.do'
super().__init__(MOM_BASE_URL, user, password)
self.__momURLgetSIP = self.mom_base_url + 'mom3/interface/importXML2.do'
self.__momURLsetStatus = self.mom_base_url + 'mom3/interface/service/setStatusDataProduct.do'
self.MAX_MOM_RETRIES = 3
def login(self):
try:
if self.__session is not None:
self.logout()
logger.debug("logging in to MoM on url: %s", self.__momURLlogin)
session = requests.session()
r = session.get(self.__momURLlogin, verify=False)
if 200 != r.status_code:
raise Exception("Logging into MoM on %s failed: http return code = %s" % (self.__momURLlogin, r.status_code))
r = session.post(self.__momUR_security_check, data={'j_username': self.__user, 'j_password': self.__password}, verify=False)
if 200 != r.status_code:
raise Exception("Logging into MoM on %s failed: http return code = %s" % (self.__momUR_security_check, r.status_code))
logger.debug("logged in on MoM on url: %s", self.__momURLlogin)
self.__session = session
except Exception as e:
raise Exception("Logging into MoM on %s failed: %s" % (self.__momURLlogin, str(e)))
def logout(self):
try:
if self.__session is not None:
logger.debug("logging out of MoM on url: %s", self.__momURLlogout)
self.__session.get(self.__momURLlogout, verify=False)
self.__session.close()
self.__session = None
logger.debug("logged out of MoM on url: %s", self.__momURLlogout)
except Exception as e:
logger.warning("Logging out of MoM failed: " + str(e))
def __enter__(self):
self.login()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.logout()
def setStatus(self, export_id, status_id, message = None):
try:
# mom is quite reluctant in updating the status
......@@ -88,7 +34,7 @@ class MoMClient:
params = {"exportId" : export_id, "status" : status_id}
logger.info("updating MoM on url: %s with params: %s", self.__momURLsetStatus, params)
response = self.__session.get(self.__momURLsetStatus, params=params)
response = self.session.get(self.__momURLsetStatus, params=params)
reply = response.text.strip()
if reply == 'ok':
......@@ -105,7 +51,7 @@ class MoMClient:
params['message'] = message
logger.info("updating MoM (again to set the message) on url: %s with params: %s", self.__momURLsetStatus, params)
response = self.__session.get(self.__momURLsetStatus, params=params)
response = self.session.get(self.__momURLsetStatus, params=params)
reply = response.text.strip()
if reply == 'ok':
......@@ -177,7 +123,7 @@ class MoMClient:
xmlcontent = xmlcontent.replace(' ', ' ')
params = {"command": "get-sip-with-input", "xmlcontent" : xmlcontent}
response = self.__session.get(self.__momURLgetSIP, params=params)
response = self.session.get(self.__momURLgetSIP, params=params)
result = response.text
result = result.replace('<stationType>Europe</stationType>', '<stationType>International</stationType>')
......@@ -249,7 +195,7 @@ class MoMClient:
mom_id = archive_id - 1000000 # stupid mom one million archive_id offset
# logger.info('%s: GetSip call: %s %s', log_prefix, self.__momURLgetSIP, data)
response = self.__session.get(self.__momURLgetSIP, params={"command" : "GETSIP", "id" : mom_id})
response = self.session.get(self.__momURLgetSIP, params={"command" : "GETSIP", "id" : mom_id})
result = response.text
if 'DOCTYPE HTML PUBLIC' in result:
......
# $Id: CMakeLists.txt 32745 2015-11-01 20:17:08Z schoenmakers $
lofar_add_package(MoMQueryService)
lofar_add_package(MoMSimpleAPIs)
lofar_add_package(MoMutils)
# $Id$
lofar_package(MoMQueryServiceServer 1.0 DEPENDS PyMessaging MoMQueryServiceCommon MoMQueryServiceClient)
lofar_package(MoMQueryServiceServer 1.0 DEPENDS PyMessaging MoMQueryServiceCommon MoMQueryServiceClient MoMSimpleAPIs)
include(PythonInstall)
include(FindPythonModule)
find_python_module(mysql REQUIRED)
set(_py_files
momqueryservice.py
)
......
# $Id$
lofar_package(MoMSimpleAPIs 1.0)
include(PythonInstall)
include(FindPythonModule)
find_python_module(requests REQUIRED)
find_python_module(mysql REQUIRED)
set(_py_files
momhttpclient.py
momdbclient.py
)
python_install(${_py_files} DESTINATION lofar/mom/simpleapis)
This diff is collapsed.
#!/usr/bin/env python3
import requests
from time import sleep
import logging
logger = logging.getLogger()
from lofar.common import isProductionEnvironment
import urllib3
urllib3.disable_warnings()
class BaseMoMClient:
MOM_BASE_URL = 'https://lcs023.control.lofar:8443/' if isProductionEnvironment() else 'http://lofartest.control.lofar:8080/'
def __init__(self, mom_base_url, user = None, password = None):
if user == None or password == None:
# (mis)use dbcredentials to read user/pass from disk
from lofar.common import dbcredentials
dbc = dbcredentials.DBCredentials()
creds = dbc.get('MoM_site' if isProductionEnvironment() else 'MoM_site_test')
user = creds.user
password = creds.password
self.mom_base_url = mom_base_url
self.__user = user
self.__password = password
self.session = None
self.__momURLlogin = self.mom_base_url + 'useradministration/user/systemlogin.do'
self.__momUR_security_check = self.mom_base_url + 'useradministration/user/j_security_check'
self.__momURLlogout = self.mom_base_url + 'useradministration/user/logout.do'
def login(self):
try:
if self.session is not None:
self.logout()
logger.debug("logging in to MoM on url: %s", self.__momURLlogin)
session = requests.session()
r = session.get(self.__momURLlogin, verify=False)
if 200 != r.status_code:
raise Exception("Logging into MoM on %s failed: http return code = %s" % (self.__momURLlogin, r.status_code))
r = session.post(self.__momUR_security_check, data={'j_username': self.__user, 'j_password': self.__password}, verify=False)
if 200 != r.status_code:
raise Exception("Logging into MoM on %s failed: http return code = %s" % (self.__momUR_security_check, r.status_code))
logger.debug("logged in on MoM on url: %s", self.__momURLlogin)
self.session = session
except Exception as e:
raise Exception("Logging into MoM on %s failed: %s" % (self.__momURLlogin, str(e)))
def logout(self):
try:
if self.session is not None:
logger.debug("logging out of MoM on url: %s", self.__momURLlogout)
self.session.get(self.__momURLlogout, verify=False)
self.session.close()
self.session = None
logger.debug("logged out of MoM on url: %s", self.__momURLlogout)
except Exception as e:
logger.warning("Logging out of MoM failed: " + str(e))
def __enter__(self):
self.login()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.logout()
class SystemMoMClient(BaseMoMClient):
def __init__(self, user=None, password=None):
mom_base_url = 'https://lcs023.control.lofar:8443/' if isProductionEnvironment() else 'http://lofartest.control.lofar:8080/'
super().__init__(mom_base_url, user, password)
self.__momURLImportXML = self.mom_base_url + 'mom3/interface/importXML2.do'
def setPipelineStatus(self, mom2id: int, status: str):
try:
logger.info("MoMClient.setPipelineStatus mom2id:%s status:%s", mom2id, status)
self.login()
xmlcontent = """<?xml version="1.0" encoding="UTF-8"?>
<lofar:pipeline mom2Id="%s" xmlns:lofar="http://www.astron.nl/MoM2-Lofar" xmlns:mom2="http://www.astron.nl/MoM2">
<currentStatus>
<mom2:%sStatus/>
</currentStatus>
</lofar:pipeline>""" % (mom2id, status)
# sanitize, make compact
xmlcontent = xmlcontent.replace('\n', ' ')
while ' ' in xmlcontent:
xmlcontent = xmlcontent.replace(' ', ' ')
params = {"command": "importxml2", "xmlcontent": xmlcontent}
for i in range(3):
response = self.session.post(self.__momURLImportXML, params=params)
result = response.text
# sanitize, make compact
result = result.replace('\n', ' ')
while ' ' in result:
result = result.replace(' ', ' ')
if response.status_code == 200 and '<error>' not in result:
continue
else:
logger.error("MoMClient.setPipelineStatus mom2id:%s status:%s failed: %s", mom2id, status, result)
sleep(1)
except Exception as e:
self.logout()
raise Exception("MoMClient.setPipelineStatus mom2id:%s status:%s failed: %s" %(mom2id, status, e))
if response.status_code != 200 or '<error>' in result:
raise Exception("Could not set pipeline status mom2id:%s status:%s response: %s" % (mom2id, status, result))
if __name__ == '__main__':
mc = SystemMoMClient()
mc.setPipelineStatus(959395, 'opened')
\ No newline at end of file
# $Id$
lofar_package(MoMutils 1.0 DEPENDS MoMSimpleAPIs PyCommon)
include(PythonInstall)
include(FindPythonModule)
find_python_module(requests REQUIRED)
find_python_module(mysql REQUIRED)
set(_py_files
predecessor_connections.py
)
python_install(${_py_files} DESTINATION lofar/mom/utils)
lofar_add_bin_scripts(fix_mom_predecessor_connections)
\ No newline at end of file
#!/usr/bin/env python3
# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id$
if __name__ == '__main__':
from lofar.mom.utils.predecessor_connections import main
main()
#!/usr/bin/env python3
# Copyright (C) 2017 ASTRON (Netherlands Institute for Radio Astronomy)
# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
# $Id$
'''
'''
import logging
logger=logging.getLogger(__file__)
from time import sleep
from datetime import datetime, timedelta
from optparse import OptionParser
from lofar.common import dbcredentials
from lofar.mom.simpleapis.momhttpclient import SystemMoMClient
from lofar.mom.simpleapis.momdbclient import MoMDatabaseWrapper
def fix_predecessor_connection_if_needed(mom_id: int, dbcreds: dbcredentials.DBCredentials = None):
'''
check the predecessor topology string of the mom object given by mom_id, and fix it if needed.
:param mom_id: a mom2id of a mom object
:param dbcreds: the db credentials of the MoM database, defaults to 'MoM' credentials in ~/.lofar/dbcredentials.
'''
# little helper function to set the mom status via http (which is asynchronous)
# and which then waits for the status to be as set.
def setAndWaitForMomStatus(status):
with SystemMoMClient() as momhttp:
start_wait = datetime.utcnow()
momhttp.setPipelineStatus(mom_id, status)
object = momdb.getObjectDetails(mom_id)[mom_id]
while object['object_status'] != status and object['object_status_pending']:
logger.info('waiting for %s status change to %s. current status=%s pending=%s',
mom_id, status, object['object_status'], object['object_status_pending'])
if datetime.utcnow() - start_wait > timedelta(seconds=60):
raise TimeoutError("Timeout while waiting for mom_id=%s to get '%s' status" % (mom_id, status))
sleep(0.5)
with MoMDatabaseWrapper(dbcreds) as momdb:
object = momdb.getObjectDetails(mom_id)[mom_id]
object_predecessor_string = object.get('object_predecessor_string') or ''
object_predecessor_string_parts = [x.strip() for x in object_predecessor_string.split(',')]
logger.info("checking mom_id=%s status=%s name=%s predecessors=%s group_id=%s group_name=%s",
mom_id, object['object_status'], object['object_name'], object_predecessor_string,
object['object_group_id'], object['object_group_name'])
if len(object_predecessor_string_parts) == 0:
logger.info("mom_id=%s name=%s has no predecessors", mom_id, object['object_name'])
return
unconnected_object_predecessor_string_parts = [p for p in object_predecessor_string_parts if not p.startswith('M')]
if len(unconnected_object_predecessor_string_parts) == 0:
logger.info("all predecessors of mom_id=%s status=%s name=%s predecessors=%s group_id=%s group_name=%s have a proper mom id",
mom_id, object['object_status'], object['object_name'], object_predecessor_string,
object['object_group_id'], object['object_group_name'])
if object['object_status'] == 'opened':
# trigger MoM-server status change handling, which actually connects the object to its predecessors
setAndWaitForMomStatus('approved')
return
group_id = object['object_group_id']
group_tasks = momdb.getObjectDetailsOfObservationsAndPipelinesInGroup(group_id)
if len(group_tasks) == 0:
logger.warning("there are no other tasks in group %s %s that %s could connect to", group_id, object['object_group_name'], mom_id)
return
if object['object_status'] not in ['opened', 'approved', 'prescheduled', 'scheduled']:
logger.warning("Cannot connect object %s to its predecessors because its status is '%s'. predecessors: %s\nobject: %s",
mom_id, object['object_status'], object_predecessor_string, object)
return
# magic MoM-business-logic-like string parsing...
# MoM specs encode precesessors with so called 'topology' strings (or comma seperated strings for multiple predecessors)
# these topology strings are translated in mom2id's (prefixed with an M)
# Find unconnected_predecessors based on these topologies.
topopolgy_group_prefix = 'mom.G%d.' % (object['object_group_id'])
unconnected_object_predecessor_topologies = ["%s%s" % (topopolgy_group_prefix, p) for p in unconnected_object_predecessor_string_parts]
unconnected_predecessors = [t for t in group_tasks if t['object_topology'] in unconnected_object_predecessor_topologies]
if len(unconnected_predecessors) != len(unconnected_object_predecessor_string_parts):
logger.warning("could not find all unconnected predecessors for %s in group %s", mom_id, group_id)
if len(unconnected_predecessors):
if object['object_status'] != 'opened':
# make sure the status is opened
setAndWaitForMomStatus('opened')
object = momdb.getObjectDetails(mom_id)[mom_id]
# connect each known predecessor
for unconnected_predecessor in unconnected_predecessors:
momdb.connect_to_predecessor(mom_id, unconnected_predecessor['object_mom2id'])
if object['object_status'] == 'opened':
# trigger MoM-server status change handling, which actually connects the object to its predecessors
setAndWaitForMomStatus('approved')
def fix_predecessor_connections_in_group_if_needed(mom_group_id: int, dbcreds: dbcredentials.DBCredentials = None):
'''
check all the predecessor topology strings of all mom objects in the group given by mom_group_id, and fix them if needed.
:param mom_group_id: a mom2id of a mom group object
:param dbcreds: the db credentials of the MoM database, defaults to 'MoM' credentials in ~/.lofar/dbcredentials.
'''
with MoMDatabaseWrapper(dbcreds=dbcreds) as momdb:
# first, let's see if the lobos_group_id is for a folder of of sub-folders...
groups = momdb.getGroupsInGroup(mom_group_id)
if len(groups) == 0:
# not a folder of subfolders, so treat the given lobos_group_id as if its one of the subfolders...
groups = momdb.getObjectDetails(mom_group_id).values()
groups = sorted(groups, key=lambda g: g['object_mom2id'])
logger.info("checking and fixing the following groups: %s",
", ".join("%s %s" % (g['object_mom2id'], g['object_name']) for g in groups))
for group in groups:
group_id = group['object_mom2id']
group_tasks = momdb.getObjectDetailsOfObservationsAndPipelinesInGroup(group_id)
group_tasks = sorted(group_tasks, key=lambda t: t['object_mom2id'])
logger.info("checking and fixing the following tasks in group %s %s: %s",
group['object_mom2id'], group['object_name'],
", ".join("%s %s" % (t['object_mom2id'], t['object_name']) for t in group_tasks))
for task in group_tasks:
try:
if 'pipeline' in task['object_type'].lower():
fix_predecessor_connection_if_needed(task['object_mom2id'], dbcreds=dbcreds)
except Exception as e:
logger.error(e)
def main():
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
# Check the invocation arguments
parser = OptionParser('%prog <mom_id_of_LOBOS_group>', description='fix loose connections (unprocessed predecessor topologies) in the given LOBOS group and approved them.')
parser.add_option_group(dbcredentials.options_group(parser))
parser.set_defaults(dbcredentials="MoM")
(options, args) = parser.parse_args()
dbcreds = dbcredentials.parse_options(options)
print()
print("Using dbcreds:", dbcreds.stringWithHiddenPassword())
print("Using mom url:", SystemMoMClient().mom_base_url)
print()
if len(args) == 0:
parser.print_help()
exit()
group_id = int(args[0])
if input("Proceed with checking and fixing %s ? y/<n>: " % (group_id,)) == 'y':
fix_predecessor_connections_in_group_if_needed(group_id, dbcreds)
else:
print("exiting...")
if __name__ == '__main__':
main()
......@@ -28,7 +28,7 @@ import time
from optparse import OptionParser
from threading import Condition, Lock, current_thread, Thread
import _strptime
from datetime import datetime
from datetime import datetime, timedelta
from json import loads as json_loads
import time
import logging
......@@ -485,12 +485,7 @@ def putTask(task_id):
otdbrpc.taskSetStatus(task['otdb_id'], updatedTask['status'])
#we expect the status in otdb/radb to eventually become what we asked for...
expected_statuses = set([updatedTask['status']])
#except for the prescheduled status, because then the resource assigner tries
#to schedule the task, and it will end up in either 'scheduled', 'conflict', 'error' state.
if updatedTask['status'] == 'prescheduled':
expected_statuses = set(['scheduled', 'conflict', 'error'])
expected_status = updatedTask['status']
#block until radb and mom task status are equal to the expected_statuses (with timeout)
start_wait = datetime.utcnow()
......@@ -500,21 +495,24 @@ def putTask(task_id):
otdb_status = otdbrpc.taskGetStatus(task['otdb_id'])
logger.info('waiting for otdb/radb task status to be in [%s].... otdb:%s radb:%s',
', '.join(expected_statuses), otdb_status, task['status'])
expected_status, otdb_status, task['status'])
if (task['status'] in expected_statuses and
otdb_status in expected_statuses):
if (task['status'] == expected_status and otdb_status == expected_status):
logger.info('otdb/radb task status now has the expected status %s otdb:%s radb:%s',
expected_status, otdb_status, task['status'])
break
if datetime.utcnow() - start_wait > timedelta(seconds=10):
logger.warning('timeout while waiting for otdb/radb task status to get the expected status %s otdb:%s radb:%s',
expected_status, otdb_status, task['status'])
break
time.sleep(0.2)
time.sleep(0.1)
except RPCException as e:
if 'does not exist' in str(e):
# task does not exist (anymore) in otdb
#so remove it from radb as well (with cascading deletes on specification)
logger.warn('task with otdb_id %s does not exist anymore in OTDB. removing task radb_id %s from radb', task['otdb_id'], task['id'])
logger.warning('task with otdb_id %s does not exist anymore in OTDB. removing task radb_id %s from radb', task['otdb_id'], task['id'])
radb().deleteSpecification(task['specification_id'])
if 'data_pinned' in updatedTask:
......
......@@ -4,6 +4,7 @@
lofar_package(RAServices
DEPENDS MAC_Services
MoMQueryService
MoMutils
OTDBtoRATaskStatusPropagator
RATaskSpecifiedService
RAtoOTDBTaskSpecificationPropagator
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment