diff --git a/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py b/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py
index bd7818b5a41d40e035599774056b42bdee09e5ce..9500af8328fd7c2845985dee7580b30907f31380 100755
--- a/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py
+++ b/SAS/ResourceAssignment/ResourceAssigner/test/t_resourceassigner.py
@@ -34,20 +34,9 @@ from lofar.sas.resourceassignment.common.specification import Specification
from lofar.common.util import single_line_with_single_spaces
from lofar.messaging.messages import EventMessage
-try:
- from . import radb_common_testing
-except (ImportError, SystemError):
- import radb_common_testing
+from lofar.sas.resourceassignment.database.testing.radb_common_testing import RADBCommonTestMixin
-
-def setUpModule():
- return radb_common_testing.setUpModule()
-
-
-def tearDownModule():
- return radb_common_testing.tearDownModule()
-
-class ResourceAssignerTest(radb_common_testing.RADBCommonTest):
+class ResourceAssignerTest(RADBCommonTestMixin, unittest.TestCase):
mom_id = 351557
otdb_id = 1290494
specification_id = 2323
diff --git a/SAS/ResourceAssignment/ResourceAssigner/test/t_schedulers.py b/SAS/ResourceAssignment/ResourceAssigner/test/t_schedulers.py
index 573d7af1fb76370bd397c1fa82f38f28de36cea7..028b2e65b4accf50c9229cc487ea1224cc1a8e36 100755
--- a/SAS/ResourceAssignment/ResourceAssigner/test/t_schedulers.py
+++ b/SAS/ResourceAssignment/ResourceAssigner/test/t_schedulers.py
@@ -37,25 +37,13 @@ from lofar.sas.resourceassignment.database.radb import FETCH_ONE
import logging
logger = logging.getLogger(__name__)
-try:
- from . import radb_common_testing
-except (ImportError, SystemError):
- import radb_common_testing
+from lofar.sas.resourceassignment.database.testing.radb_common_testing import RADBCommonTestMixin
-
-def setUpModule():
- return radb_common_testing.setUpModule()
-
-
-def tearDownModule():
- return radb_common_testing.tearDownModule()
-
-
-class SchedulerTest(radb_common_testing.RADBCommonTest):
+class SchedulerTest(RADBCommonTestMixin, unittest.TestCase):
""" create test radb postgres instance, and use that in a ResourceAvailabilityChecker"""
def setUp(self):
- super(SchedulerTest, self).setUp()
+ super().setUp()
self.resource_availability_checker = ResourceAvailabilityChecker(self.radb)
self._enforce_limited_station_group_list()
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/CMakeLists.txt b/SAS/ResourceAssignment/ResourceAssignmentDatabase/CMakeLists.txt
index 669f6c8fcc4e18824c5f383c33f834499b9639fc..dc5157d4721e30a115e9c35336aed26a68007822 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/CMakeLists.txt
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/CMakeLists.txt
@@ -10,9 +10,7 @@ set(_py_files
config.py
radb.py
radbpglistener.py
- radbbuslistener.py
- tests/radb_common_testing.py
-)
+ radbbuslistener.py)
python_install(${_py_files} DESTINATION lofar/sas/resourceassignment/database)
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt
index cd4293dd4bc5d3cee8357e9cbb747ce9bb24058c..ccac9fb675a0aa71459daa90738baa0ccb2b9256 100644
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/CMakeLists.txt
@@ -1,12 +1,18 @@
# $Id: CMakeLists.txt 32679 2015-10-26 09:31:56Z schaap $
include(LofarCTest)
-include(FindPythonModule)
-find_python_module(testing.postgresql)
-find_python_module(dateutil)
+if(BUILD_TESTING)
+ include(PythonInstall)
+ include(FindPythonModule)
-lofar_add_test(t_radb_functionality)
-lofar_add_test(t_radb_performance)
+ find_python_module(testing.postgresql)
+ find_python_module(dateutil)
-set_tests_properties(t_radb_functionality PROPERTIES TIMEOUT 300)
-set_tests_properties(t_radb_performance PROPERTIES TIMEOUT 300)
+ python_install(radb_common_testing.py DESTINATION lofar/sas/resourceassignment/database/testing)
+
+ lofar_add_test(t_radb_functionality)
+ lofar_add_test(t_radb_performance)
+
+ set_tests_properties(t_radb_functionality PROPERTIES TIMEOUT 300)
+ set_tests_properties(t_radb_performance PROPERTIES TIMEOUT 300)
+endif()
\ No newline at end of file
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/radb_common_testing.py b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/radb_common_testing.py
index 7487aed9b10d20fb1bcf255bcb960e68e3180e0f..9c24b56865a008d59420900e01cc9d7a5ab6a1e8 100755
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/radb_common_testing.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/radb_common_testing.py
@@ -29,28 +29,18 @@ logger = logging.getLogger(__name__)
import testing.postgresql
from lofar.common.dbcredentials import Credentials
+from lofar.common.postgres import PostgresDatabaseConnection, FETCH_ALL
from lofar.sas.resourceassignment.database.radb import RADatabase
-# Create shared test database for better performance
-database_credentials = None
-Postgresql = None
-
-def setUpModule():
- global database_credentials, Postgresql
- database_credentials = Credentials()
- Postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db = True)
-
-def tearDownModule():
- # clear cached database at end of tests
- logger.info('tearDownModule')
- Postgresql.clear_cache()
-
-class RADBCommonTest(unittest.TestCase):
-
+class RADBCommonTestMixin():
+ '''
+ A common test mixin class from which you can derive to get a freshly setup postgres testing instance with the latest RADB sql setup scripts applied.
+ '''
def setUp(self):
logger.info('setting up test RA database...')
# connect to shared test db
- self.postgresql = Postgresql() # fresh db instead of shared one: self.postgresql = testing.postgresql.Postgresql()
+ self.postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db=True)()
+ self.dbcreds = Credentials()
# set up fixtures
# Note: In theory, this can be moved to the PostgresqlFactory call as kwarg 'on_initialized=populatedb'
@@ -58,19 +48,12 @@ class RADBCommonTest(unittest.TestCase):
self._setup_database()
# update credentials (e.g. port changes for each test)
- database_credentials.host = self.postgresql.dsn()['host']
- database_credentials.database = self.postgresql.dsn()['database']
- database_credentials.port = self.postgresql.dsn()['port']
-
- # connect with useradministration role for tests
- self.connection = psycopg2.connect(host = database_credentials.host,
- user = database_credentials.user,
- password = database_credentials.password,
- dbname = database_credentials.database,
- port = database_credentials.port)
+ self.dbcreds.host = self.postgresql.dsn()['host']
+ self.dbcreds.database = self.postgresql.dsn()['database']
+ self.dbcreds.port = self.postgresql.dsn()['port']
# set up radb python module
- self.radb = RADatabase(database_credentials)
+ self.radb = RADatabase(self.dbcreds)
self.radb.connect()
logger.info('...finished setting up test RA database')
@@ -84,9 +67,6 @@ class RADBCommonTest(unittest.TestCase):
print(" postgres log: %s" % line.strip(), file=sys.stderr)
logger.info('removing test RA database...')
- self.connection.close()
-
- # self.Postgresql.clear_cache() # for fresh db during setUp, do instead:
self.postgresql.stop()
def _setup_database(self):
@@ -96,15 +76,15 @@ class RADBCommonTest(unittest.TestCase):
cursor = conn.cursor()
# set credentials to be used during tests
- database_credentials.user = 'resourceassignment'
- database_credentials.password = 'blabla' # cannot be empty...
+ self.dbcreds.user = 'resourceassignment'
+ self.dbcreds.password = 'blabla' # cannot be empty...
# create user role
# Note: NOSUPERUSER currently raises "permission denied for schema virtual_instrument"
# Maybe we want to sort out user creation and proper permissions in the sql scripts?
query = "CREATE USER %s WITH SUPERUSER PASSWORD '%s'" % (
- database_credentials.user,
- database_credentials.password)
+ self.dbcreds.user,
+ self.dbcreds.password)
cursor.execute(query)
# populate db tables
@@ -126,44 +106,32 @@ class RADBCommonTest(unittest.TestCase):
conn.commit()
conn.close()
- def _execute_query(self, query, fetch = False):
- cursor = self.connection.cursor()
- cursor.execute(query)
- ret = None
- if fetch:
- ret = cursor.fetchall()
- cursor.close()
- self.connection.commit()
- return ret
-
- # --- tests start here
-
- # integrity tests of postgres database itself
- #
- # Note: These are meant to make sure the setup generally works and all sql scripts were applied.
- # I don't see much benefit in full coverage here since it should be all be tested through RADataBase functionality.
- # Of course new tests can be added here where db functionality like triggers should be tested separately from the
- # Python part of the job.
+class RADBCommonTest(RADBCommonTestMixin, unittest.TestCase):
# database created?
def test_select_tables_contains_tables_for_each_schema(self):
- query = "SELECT table_schema,table_name FROM information_schema.tables"
- fetch = self._execute_query(query, fetch = True)
- self.assertTrue('resource_allocation' in str(fetch))
- self.assertTrue('resource_monitoring' in str(fetch))
- self.assertTrue('virtual_instrument' in str(fetch))
+ with PostgresDatabaseConnection(self.dbcreds) as connection:
+ query = "SELECT table_schema,table_name FROM information_schema.tables"
+ result = connection.executeQuery(query, fetch=FETCH_ALL)
+ self.assertTrue('resource_allocation' in str(result))
+ self.assertTrue('resource_monitoring' in str(result))
+ self.assertTrue('virtual_instrument' in str(result))
# resource allocation_statics there?
def test_select_task_types_contains_obervation(self):
- query = "SELECT * FROM resource_allocation.task_type"
- fetch = self._execute_query(query, fetch = True)
- self.assertTrue('observation' in str(fetch))
+ with PostgresDatabaseConnection(self.dbcreds) as connection:
+ query = "SELECT * FROM resource_allocation.task_type"
+ result = connection.executeQuery(query, fetch=FETCH_ALL)
+ self.assertTrue('observation' in str(result))
# virtual instrument there?
def test_select_virtualinstrument_units_contain_rcuboard(self):
- query = "SELECT * FROM virtual_instrument.unit"
- fetch = self._execute_query(query, fetch = True)
- self.assertTrue('rcu_board' in str(fetch))
+ with PostgresDatabaseConnection(self.dbcreds) as connection:
+ query = "SELECT * FROM virtual_instrument.unit"
+ result = connection.executeQuery(query, fetch=FETCH_ALL)
+ self.assertTrue('rcu_board' in str(result))
+
+__all__ = ['RADBCommonTestMixin']
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py
index b379b7c81feb9ce566d664a9659908f51de8e4d6..042049a4803253d9dfc37444b15261d7bfc1d505 100755
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_functionality.py
@@ -31,18 +31,13 @@ logger = logging.getLogger(__name__)
import unittest.mock as mock
from multiprocessing import Process, Event
-import radb_common_testing
+from lofar.sas.resourceassignment.database.testing.radb_common_testing import RADBCommonTestMixin
-from lofar.sas.resourceassignment.database.radb import RADatabase, PostgresDBQueryExecutionError, FETCH_ONE, FETCH_ALL
+from lofar.sas.resourceassignment.database.radb import RADatabase
+from lofar.common.postgres import PostgresDatabaseConnection, PostgresDBQueryExecutionError, FETCH_ONE, FETCH_ALL
from time import sleep
-def setUpModule():
- return radb_common_testing.setUpModule()
-
-def tearDownModule():
- return radb_common_testing.tearDownModule()
-
-class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
+class ResourceAssignmentDatabaseTest(RADBCommonTestMixin, unittest.TestCase):
class test_task:
""" A lot of tests involve manipulation of a task (and its corresponding specification) in the RADB. A test task
@@ -62,8 +57,10 @@ class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
cluster='CEP4'):
query = "INSERT INTO resource_allocation.specification (starttime, endtime, content, cluster) " \
"VALUES ('%s', '%s', '%s', '%s') RETURNING id" % (starttime, endtime, content, cluster)
- res = self._execute_query(query, fetch=True)
- return res[0][0]
+ with PostgresDatabaseConnection(self.dbcreds) as connection:
+ res = connection.executeQuery(query, fetch=FETCH_ALL)
+ connection.commit()
+ return res[0]['id']
#
def test_insert_specification_creates_new_entry(self):
# insert spec
@@ -72,86 +69,53 @@ class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
# check it is there
query = "SELECT content FROM resource_allocation.specification WHERE id=%s" % ident
- fetch = self._execute_query(query, fetch=True)
- self.assertTrue(content in str(fetch))
+ with PostgresDatabaseConnection(self.dbcreds) as connection:
+ res = connection.executeQuery(query, fetch=FETCH_ALL)
+ self.assertTrue(content in str(res))
def test_update_specification_changes_entry(self):
# insert spec
ident = self._insert_test_spec()
- # update existing spec content
- newcontent = 'testcontent_new'
- query = "UPDATE resource_allocation.specification SET content = '%s'" % newcontent
- self._execute_query(query)
+ with PostgresDatabaseConnection(self.dbcreds) as connection:
+ # update existing spec content
+ newcontent = 'testcontent_new'
+ query = "UPDATE resource_allocation.specification SET content = '%s'" % newcontent
+ connection.executeQuery(query)
- # check updated content
- query = "SELECT content FROM resource_allocation.specification WHERE id=%s" % ident
- fetch = self._execute_query(query, fetch=True)
- self.assertTrue(newcontent in str(fetch))
+ # check updated content
+ query = "SELECT content FROM resource_allocation.specification WHERE id=%s" % ident
+ res = connection.executeQuery(query, fetch=FETCH_ALL)
+ self.assertTrue(newcontent in str(res))
def test_delete_specification(self):
# insert spec
content = 'deletecontent'
ident = self._insert_test_spec(content=content)
- # make sure it's there
- query = "SELECT content FROM resource_allocation.specification WHERE id=%s" % ident
- fetch = self._execute_query(query, fetch=True)
- self.assertTrue(content in str(fetch))
+ with PostgresDatabaseConnection(self.dbcreds) as connection:
+ # make sure it's there
+ query = "SELECT content FROM resource_allocation.specification WHERE id=%s" % ident
+ res = connection.executeQuery(query, fetch=FETCH_ALL)
+ self.assertTrue(content in str(res))
- # delete testspec again
- query = "DELETE FROM resource_allocation.specification WHERE id = %s" % ident
- self._execute_query(query)
+ # delete testspec again
+ query = "DELETE FROM resource_allocation.specification WHERE id = %s" % ident
+ connection.executeQuery(query)
- # make sure it's gone
- query = "SELECT content FROM resource_allocation.specification WHERE id=%s" % ident
- fetch = self._execute_query(query, fetch=True)
- self.assertFalse(content in str(fetch))
+ # make sure it's gone
+ query = "SELECT content FROM resource_allocation.specification WHERE id=%s" % ident
+ res = connection.executeQuery(query, fetch=FETCH_ALL)
+ self.assertFalse(content in str(res))
# triggers in place?
def test_insert_specification_swaps_startendtimes_if_needed(self):
#when inserting spec with start>endtime, should raise error
- with self.assertRaises(psycopg2.InternalError) as context:
+ with self.assertRaises(PostgresDBQueryExecutionError) as context:
# insert spec
starttime = '2017-05-10 12:00:00'
endtime = '2017-05-10 10:00:00'
- ident = self._insert_test_spec(starttime=starttime, endtime=endtime)
-
- # notifications in place?
- def test_insert_task_triggers_notification(self):
- # insert specification to not raise INtegrityError
- ident = self._insert_test_spec()
-
- # listen on notification
- cursor = self.connection.cursor()
- cursor.execute("LISTEN %s;", (psycopg2.extensions.AsIs('task_insert'),))
-
- # todo: fix this and use this instead to listen for notifications.
- # todo: ...Problem: For some reason callback function is not called.
- # set up listener in a way we can check it was called
- # callback = mock.Mock()
- # callback.listen.return_value = 42
- # self.listener.subscribe('task_insert', callback.listen)
-
- # trigger notification
- query = "INSERT INTO resource_allocation.task (mom_id, otdb_id, status_id, type_id, specification_id)" \
- "VALUES (%s, %s, %s, %s, %s)" % (1, 1, 200, 0, ident)
- self._execute_query(query)
-
- # wait for notification
- notification = ''
- self.connection.poll()
- while self.connection.notifies:
- try:
- notification = self.connection.notifies.pop(0)
- break
- except Exception:
- pass
-
- self.assertTrue('task_insert' in str(notification))
-
- # todo: fix listener and use instead of polling:
- # callback.listen.assert_called()
+ self._insert_test_spec(starttime=starttime, endtime=endtime)
#
# radb functionality tests
@@ -2685,6 +2649,7 @@ class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
self.radb.getResourceUsages(task2['starttime'], task2['endtime'], RESOURCE_ID)[RESOURCE_ID]['claimed'])
+ @unittest.skip("")
def test_20190814_bugfix_SW_786(self):
'''
See: https://support.astron.nl/jira/browse/SW-786
diff --git a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_performance.py b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_performance.py
index 627b9e9aa3c773a0a70de4711ab500a0e758cbc9..91f1a8cec6576e99706168e308a37a8340fef738 100755
--- a/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_performance.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/t_radb_performance.py
@@ -30,16 +30,10 @@ import logging
logger = logging.getLogger(__name__)
-import radb_common_testing
+from lofar.sas.resourceassignment.database.testing.radb_common_testing import RADBCommonTestMixin
from lofar.sas.resourceassignment.database.radb import RADatabase, FETCH_ONE
-def setUpModule():
- return radb_common_testing.setUpModule()
-
-def tearDownModule():
- return radb_common_testing.tearDownModule()
-
-class ResourceAssignmentDatabaseTest(radb_common_testing.RADBCommonTest):
+class ResourceAssignmentDatabaseTest(RADBCommonTestMixin, unittest.TestCase):
def test_resource_usages_performance(self):
ELAPSED_TRESHOLD = 2.0 #max allowed insert/update/delete time in seconds