Skip to content
Snippets Groups Projects
Commit f886e90c authored by Jorrit Schaap's avatar Jorrit Schaap
Browse files

SW-516: fixed OTDB_Services tests for python3 and messagebus changes. Also...

SW-516: fixed OTDB_Services tests for python3 and messagebus changes. Also made the test test against a local testing postgres database. All contained in python, no hybrid .run/.py tests anymore.
parent bf53ac88
No related branches found
No related tags found
No related merge requests found
......@@ -4637,13 +4637,15 @@ SAS/OTDB_Services/doc/OTDB_services.md -text
SAS/OTDB_Services/doc/package.dox -text
SAS/OTDB_Services/otdbrpc.py -text
SAS/OTDB_Services/test/CMakeLists.txt -text
SAS/OTDB_Services/test/t_TreeService.in.unittest_db.dump.gz -text svneol=unset#application/x-gzip
SAS/OTDB_Services/test/t_TreeService.py -text
SAS/OTDB_Services/test/t_TreeService.run -text svneol=unset#application/x-shellscript
SAS/OTDB_Services/test/t_TreeService.sh -text svneol=unset#application/x-shellscript
SAS/OTDB_Services/test/t_TreeStatusEvents.py -text
SAS/OTDB_Services/test/t_TreeStatusEvents.run -text svneol=unset#application/x-shellscript
SAS/OTDB_Services/test/t_TreeStatusEvents.sh -text svneol=unset#application/x-shellscript
SAS/OTDB_Services/test/unittest_db.dump.gz -text svneol=unset#application/x-gzip
SAS/OTDB_Services/treeService -text
SAS/OTDB_Services/treeStatusEvents -text
SAS/QPIDInfrastructure/CMakeLists.txt -text
SAS/QPIDInfrastructure/README -text
SAS/QPIDInfrastructure/bin/CMakeLists.txt -text
......
......@@ -10,14 +10,16 @@ find_python_module(pg REQUIRED) # sudo aptitude install python3-pg
lofar_add_bin_scripts(
getOTDBParset
setOTDBTreeStatus
TreeService.py
TreeStatusEvents.py
treeService
treeStatusEvents
)
set(_py_files
config.py
otdbrpc.py
OTDBBusListener.py
TreeService.py
TreeStatusEvents.py
)
# supervisord config files
......
[program:TreeService]
command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec TreeService.py --dbcredentials=OTDB'
command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec treeService --dbcredentials=OTDB'
user=lofarsys
stopsignal=INT ; KeyboardInterrupt
stopasgroup=true ; bash does not propagate signals
......
......@@ -656,8 +656,17 @@ class PostgressMessageHandler(MessageHandlerInterface):
logger.info("_SetProject({0})".format(kwargs))
return SetProject(kwargs, self.connection)
if __name__ == "__main__":
def create_service(service_name, busname, dbcreds, broker=None):
return Service(service_name,
PostgressMessageHandler,
busname=busname,
use_service_methods=True,
numthreads=1,
handler_args={"dbcreds" : dbcreds},
broker=broker,
verbose=True)
def main():
from optparse import OptionParser
from lofar.common import dbcredentials
from lofar.common.util import waitForInterrupt
......@@ -685,15 +694,13 @@ if __name__ == "__main__":
dbcreds = dbcredentials.parse_options(options)
print("###dbcreds:", dbcreds)
with Service(options.servicename,
PostgressMessageHandler,
busname=options.busname,
use_service_methods=True,
numthreads=1,
handler_args={"dbcreds" : dbcreds},
broker=options.broker,
verbose=True):
with create_service(service_name=options.servicename,
busname=options.busname,
dbcreds=dbcreds,
broker=options.broker):
waitForInterrupt()
logger.info("Stopped the OTDB services")
if __name__ == "__main__":
main()
[program:TreeStatusEvents]
command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec TreeStatusEvents.py --dbcredentials=OTDB'
command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec treeStatusEvents --dbcredentials=OTDB'
user=lofarsys
stopsignal=INT ; KeyboardInterrupt
stopasgroup=true ; bash does not propagate signals
......
......@@ -78,7 +78,7 @@ def signal_handler(signum, frame):
alive = False
if __name__ == "__main__":
def main():
from optparse import OptionParser
from lofar.common import dbcredentials
import signal
......@@ -171,3 +171,5 @@ if __name__ == "__main__":
time.sleep(2)
if __name__ == "__main__":
main()
\ No newline at end of file
......@@ -3,18 +3,8 @@
include(LofarCTest)
lofar_find_package(Python 3.4 REQUIRED)
find_python_module(testing.postgresql)
set(_qpid_tests
t_TreeService
t_TreeStatusEvents)
lofar_add_test(t_TreeService)
lofar_add_test(t_TreeStatusEvents)
execute_process(COMMAND qpid-config RESULT_VARIABLE QPID_CONFIG_RESULT OUTPUT_QUIET ERROR_QUIET)
if(${QPID_CONFIG_RESULT} EQUAL 0)
foreach(_test ${_qpid_tests})
lofar_add_test(${_test})
endforeach()
else()
lofar_join_arguments(_qpid_tests)
message(WARNING "No running qpid daemon found. The following tests will not be run: ${_qpid_tests}")
endif()
......@@ -30,121 +30,162 @@ StatusUpdateCommand : finction to update the status of a tree.
import sys
import logging
import testing.postgresql
import psycopg2
import gzip
import subprocess
from lofar.messaging.messagebus import *
from lofar.messaging.RPC import *
from lofar.sas.otdb.config import DEFAULT_OTDB_SERVICENAME
from lofar.sas.otdb.TreeService import create_service
from lofar.common.dbcredentials import Credentials
logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)
def do_rpc_catch_exception(exc_text, rpc_instance, arg_dict):
try:
try:
postgresql = testing.postgresql.PostgresqlFactory()()
database_credentials = Credentials()
database_credentials.host = postgresql.dsn()['host']
database_credentials.database = postgresql.dsn()['database']
database_credentials.port = postgresql.dsn()['port']
# connect to test-db as root
conn = psycopg2.connect(**postgresql.dsn())
cursor = conn.cursor()
# set credentials to be used during tests
database_credentials.user = 'otdb_test_user'
database_credentials.password = 'otdb_test_password' # cannot be empty...
# create user role
query = "CREATE USER %s WITH SUPERUSER PASSWORD '%s'" % (database_credentials.user, database_credentials.password)
cursor.execute(query)
conn.commit()
conn.close()
cmd1 = ['gzip', '-dc', 't_TreeService.in.unittest_db.dump.gz']
cmd2 = ['psql', '-U', database_credentials.user, '-h', database_credentials.host,
'-p', str(database_credentials.port), database_credentials.database]
proc1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE)
proc2 = subprocess.Popen(cmd2, stdin=proc1.stdout)
proc1.wait(timeout=60)
proc2.wait(timeout=60)
def do_rpc_catch_exception(exc_text, rpc_instance, arg_dict):
try:
print("** Executing {0}({1})...".format(rpc_instance.ServiceName,arg_dict))
(data, status) = (rpc_instance)(**arg_dict)
raise Exception("Expected an exception {0}, didn't get any".format(exc_text))
except Exception:
print("Caught expected exception {0}".format(exc_text))
print("======")
def do_rpc(rpc_instance, arg_dict):
print("** Executing {0}({1})...".format(rpc_instance.ServiceName,arg_dict))
(data, status) = (rpc_instance)(**arg_dict)
raise Exception("Expected an exception {0}, didn't get any".format(exc_text))
except Exception:
print("Caught expected exception {0}".format(exc_text))
print("======")
def do_rpc(rpc_instance, arg_dict):
print("** Executing {0}({1})...".format(rpc_instance.ServiceName,arg_dict))
(data, status) = (rpc_instance)(**arg_dict)
if status != "OK":
raise Exception("Status returned is {0}".format(status))
# if isinstance(data, dict):
# for key in sorted(data):
# print "%s ==> %s" % (key, data[key])
# else:
print("result =", data)
print("======")
return data
if __name__ == "__main__":
if status != "OK":
raise Exception("Status returned is {0}".format(status))
# if isinstance(data, dict):
# for key in sorted(data):
# print "%s ==> %s" % (key, data[key])
# else:
print("result =", data)
print("======")
return data
with TemporaryQueue(__name__) as tmp_queue:
busname = tmp_queue.address
with RPC("OTDBService.TaskGetIDs", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
# Existing: otdb_id:1099268, mom_id:353713
do_rpc (otdbRPC, {'OtdbID': 1099268, 'MomID': 353713 })
do_rpc (otdbRPC, {'OtdbID': 1099268, 'MomID': 5 })
do_rpc (otdbRPC, {'OtdbID': 1099268, 'MomID': None })
do_rpc (otdbRPC, {'OtdbID': 5, 'MomID': 353713 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': 5, 'MomID': 5 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': 5, 'MomID': None })
do_rpc (otdbRPC, {'OtdbID': None, 'MomID': 353713 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': None, 'MomID': 5 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': None, 'MomID': None })
with RPC("OTDBService.GetDefaultTemplates", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC,{})
with RPC("OTDBService.SetProject", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC,{'name':"Taka Tuka Land", "title":"Adventure movie", "pi":"Pippi", "co_i":"Mr.Nelson", "contact":"Witje"})
with RPC("OTDBService.TaskCreate", ForwardExceptions=True, busname=busname, timeout=10) as task_create:
do_rpc(task_create, {'OtdbID':1099268, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
do_rpc(task_create, {'MomID':353713, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
do_rpc_catch_exception('on non-exsisting campaign', task_create,
{'MomID':998877, 'TemplateName':'BeamObservation',
'CampaignName':'No such campaign', 'Specification': {'state':'finished'}})
do_rpc(task_create, {'MomID':998877, 'TemplateName':'BeamObservation',
'CampaignName':'Taka Tuka Land', 'Specification': {'state':'finished'}})
data = do_rpc(task_create, {'MomID':12345, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
new_tree1 = data['MomID']
data = do_rpc(task_create, {'MomID':54321, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
new_tree2= data['MomID']
with RPC("OTDBService.TaskPrepareForScheduling", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC, {'MomID':new_tree1}) # template
do_rpc(otdbRPC, {'MomID':new_tree1}) # now a VIC tree
do_rpc(otdbRPC, {'MomID':new_tree1, 'StartTime':'2016-03-01 12:00:00', 'StopTime':'2016-03-01 12:34:56'})
do_rpc_catch_exception("on invalid stoptime", otdbRPC,
{'MomID':new_tree1, 'StartTime':'2016-03-01 12:00:00', 'StopTime':'2016'})
with RPC("OTDBService.TaskDelete", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC, {'MomID':new_tree2})
with RPC("OTDBService.TaskGetSpecification", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC, {'OtdbID':1099269}) # PIC
do_rpc(otdbRPC, {'OtdbID':1099238}) # Template
do_rpc(otdbRPC, {'OtdbID':1099266}) # VIC
do_rpc_catch_exception('on non-existing treeID', otdbRPC, {'OtdbID':5}) # Non existing
with RPC("OTDBService.TaskSetStatus", ForwardExceptions=True, busname=busname, timeout=5) as status_update_command:
# PIC
do_rpc(status_update_command, {'OtdbID':1099269, 'NewStatus':'finished', 'UpdateTimestamps':True})
# Template
do_rpc(status_update_command, {'OtdbID':1099238, 'NewStatus':'finished', 'UpdateTimestamps':True})
# VIC
do_rpc(status_update_command, {'OtdbID':1099266, 'NewStatus':'finished', 'UpdateTimestamps':True})
# Nonexisting tree
do_rpc_catch_exception('on invalid treeID',
status_update_command, {'OtdbID':10, 'NewStatus':'finished', 'UpdateTimestamps':True})
# VIC tree: invalid status
do_rpc_catch_exception('on invalid status',
status_update_command, {'OtdbID':1099266, 'NewStatus':'what_happend', 'UpdateTimestamps':True})
# Set PIC back to active...
do_rpc(status_update_command, {'OtdbID':1099269, 'NewStatus':'active', 'UpdateTimestamps':True})
with RPC("OTDBService.GetStations", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC,{})
with RPC("OTDBService.TaskSetSpecification", ForwardExceptions=True, busname=busname, timeout=5) as key_update:
# VIC tree: valid
do_rpc(key_update, {'OtdbID':1099266,
'Specification':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.pythonHost':'NameOfTestHost'}})
# Template tree: not supported yet
do_rpc(key_update, {'OtdbID':1099238,
'Specification':{'LOFAR.ObsSW.Observation.Scheduler.priority':'0.1'}})
# PIC tree: not supported yet
do_rpc_catch_exception('on invalid treetype (PIC)', key_update,
{'OtdbID':1099269, 'Specification':{'LOFAR.PIC.Core.CS001.status_state':'50'}})
# Non exsisting tree
do_rpc_catch_exception('on invalid treeID', key_update, {'OtdbID':10,
'Specification':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.pythonHost':'NameOfTestHost'}})
# VIC tree: wrong key
do_rpc_catch_exception('on invalid key', key_update, {'OtdbID':1099266,
'Specification':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.NoSuchKey':'NameOfTestHost'}})
with create_service(DEFAULT_OTDB_SERVICENAME, busname, database_credentials):
with RPC("OTDBService.TaskGetIDs", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
# Existing: otdb_id:1099268, mom_id:353713
do_rpc (otdbRPC, {'OtdbID': 1099268, 'MomID': 353713 })
do_rpc (otdbRPC, {'OtdbID': 1099268, 'MomID': 5 })
do_rpc (otdbRPC, {'OtdbID': 1099268, 'MomID': None })
do_rpc (otdbRPC, {'OtdbID': 5, 'MomID': 353713 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': 5, 'MomID': 5 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': 5, 'MomID': None })
do_rpc (otdbRPC, {'OtdbID': None, 'MomID': 353713 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': None, 'MomID': 5 })
do_rpc_catch_exception('', otdbRPC, {'OtdbID': None, 'MomID': None })
with RPC("OTDBService.GetDefaultTemplates", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC,{})
with RPC("OTDBService.SetProject", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC,{'name':"Taka Tuka Land", "title":"Adventure movie", "pi":"Pippi", "co_i":"Mr.Nelson", "contact":"Witje"})
with RPC("OTDBService.TaskCreate", ForwardExceptions=True, busname=busname, timeout=10) as task_create:
do_rpc(task_create, {'OtdbID':1099268, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
do_rpc(task_create, {'MomID':353713, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
do_rpc_catch_exception('on non-exsisting campaign', task_create,
{'MomID':998877, 'TemplateName':'BeamObservation',
'CampaignName':'No such campaign', 'Specification': {'state':'finished'}})
do_rpc(task_create, {'MomID':998877, 'TemplateName':'BeamObservation',
'CampaignName':'Taka Tuka Land', 'Specification': {'state':'finished'}})
data = do_rpc(task_create, {'MomID':12345, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
new_tree1 = data['MomID']
data = do_rpc(task_create, {'MomID':54321, 'TemplateName':'BeamObservation', 'Specification': {'state':'finished'}})
new_tree2= data['MomID']
with RPC("OTDBService.TaskPrepareForScheduling", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC, {'MomID':new_tree1}) # template
do_rpc(otdbRPC, {'MomID':new_tree1}) # now a VIC tree
do_rpc(otdbRPC, {'MomID':new_tree1, 'StartTime':'2016-03-01 12:00:00', 'StopTime':'2016-03-01 12:34:56'})
do_rpc_catch_exception("on invalid stoptime", otdbRPC,
{'MomID':new_tree1, 'StartTime':'2016-03-01 12:00:00', 'StopTime':'2016'})
with RPC("OTDBService.TaskDelete", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC, {'MomID':new_tree2})
with RPC("OTDBService.TaskGetSpecification", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC, {'OtdbID':1099269}) # PIC
do_rpc(otdbRPC, {'OtdbID':1099238}) # Template
do_rpc(otdbRPC, {'OtdbID':1099266}) # VIC
do_rpc_catch_exception('on non-existing treeID', otdbRPC, {'OtdbID':5}) # Non existing
with RPC("OTDBService.TaskSetStatus", ForwardExceptions=True, busname=busname, timeout=5) as status_update_command:
# PIC
do_rpc(status_update_command, {'OtdbID':1099269, 'NewStatus':'finished', 'UpdateTimestamps':True})
# Template
do_rpc(status_update_command, {'OtdbID':1099238, 'NewStatus':'finished', 'UpdateTimestamps':True})
# VIC
do_rpc(status_update_command, {'OtdbID':1099266, 'NewStatus':'finished', 'UpdateTimestamps':True})
# Nonexisting tree
do_rpc_catch_exception('on invalid treeID',
status_update_command, {'OtdbID':10, 'NewStatus':'finished', 'UpdateTimestamps':True})
# VIC tree: invalid status
do_rpc_catch_exception('on invalid status',
status_update_command, {'OtdbID':1099266, 'NewStatus':'what_happend', 'UpdateTimestamps':True})
# Set PIC back to active...
do_rpc(status_update_command, {'OtdbID':1099269, 'NewStatus':'active', 'UpdateTimestamps':True})
with RPC("OTDBService.GetStations", ForwardExceptions=True, busname=busname, timeout=10) as otdbRPC:
do_rpc(otdbRPC,{})
with RPC("OTDBService.TaskSetSpecification", ForwardExceptions=True, busname=busname, timeout=5) as key_update:
# VIC tree: valid
do_rpc(key_update, {'OtdbID':1099266,
'Specification':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.pythonHost':'NameOfTestHost'}})
# Template tree: not supported yet
do_rpc(key_update, {'OtdbID':1099238,
'Specification':{'LOFAR.ObsSW.Observation.Scheduler.priority':'0.1'}})
# PIC tree: not supported yet
do_rpc_catch_exception('on invalid treetype (PIC)', key_update,
{'OtdbID':1099269, 'Specification':{'LOFAR.PIC.Core.CS001.status_state':'50'}})
# Non exsisting tree
do_rpc_catch_exception('on invalid treeID', key_update, {'OtdbID':10,
'Specification':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.pythonHost':'NameOfTestHost'}})
# VIC tree: wrong key
do_rpc_catch_exception('on invalid key', key_update, {'OtdbID':1099266,
'Specification':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.NoSuchKey':'NameOfTestHost'}})
finally:
postgresql.stop()
#!/bin/bash -x
# constants
DBHOST=sasdbtest.control.lofar
#cleanup on normal exit and on SIGHUP, SIGINT, SIGQUIT, and SIGTERM
trap 'kill ${SERVICE_PID} ; dropdb -U postgres -h ${DBHOST} ${DBNAME}' 0 1 2 3 15
# Generate randome queue name
DBNAME=unittest_$queue
# Setup a clean database with predefined content
createdb -U postgres -h ${DBHOST} ${DBNAME}
gzip -dc $srcdir/unittest_db.dump.gz | psql -U postgres -h ${DBHOST} ${DBNAME} -f -
TreeService.py -D ${DBNAME} -H ${DBHOST} -U postgres &
SERVICE_PID=$!
# Starting up takes a while
sleep 3
# Run the unit test
source python-coverage.sh
python_coverage_test "Messaging/python" t_TreeService.py
python_coverage_test "*TreeService*" t_TreeService.py
#!/usr/bin/env python3
#coding: iso-8859-15
#
# Copyright (C) 2015
# ASTRON (Netherlands Institute for Radio Astronomy)
# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
#
from lofar.sas.otdb.TreeService import main
if __name__ == "__main__":
main()
#!/usr/bin/env python3
#coding: iso-8859-15
#
# Copyright (C) 2015
# ASTRON (Netherlands Institute for Radio Astronomy)
# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This file is part of the LOFAR software suite.
# The LOFAR software suite is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# The LOFAR software suite is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
#
from lofar.sas.otdb.TreeStatusEvents import main
if __name__ == "__main__":
main()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment