SW-828: abstracted RADBCommonTestMixin into even more common...

SW-828: abstracted RADBCommonTestMixin into even more common PostgresTestMixin. adapted t_postgres.py and radb_common_testing.py to new PostgresTestMixin.

LCS/PyCommon/test/CMakeLists.txt

 # $Id$
 
-include(LofarCTest)
-
-file(COPY
-  ${CMAKE_CURRENT_SOURCE_DIR}/python-coverage.sh
-  DESTINATION ${CMAKE_BINARY_DIR}/bin)
-
-lofar_add_test(t_cache)
-lofar_add_test(t_dbcredentials)
-lofar_add_test(t_defaultmailaddresses)
-lofar_add_test(t_methodtrigger)
-lofar_add_test(t_util)
-lofar_add_test(t_test_utils)
-lofar_add_test(t_cep4_utils)
-lofar_add_test(t_postgres)
+IF(BUILD_TESTING)
+    lofar_find_package(Python 3.4 REQUIRED)
+
+    include(PythonInstall)
+
+    set(_py_files
+      postgres.py)
+
+    python_install(${_py_files} DESTINATION lofar/common/testing)
+
+    include(FindPythonModule)
+    find_python_module(testing.postgresql)
+
+    include(LofarCTest)
+
+    file(COPY
+      ${CMAKE_CURRENT_SOURCE_DIR}/python-coverage.sh
+      DESTINATION ${CMAKE_BINARY_DIR}/bin)
+
+    lofar_add_test(t_cache)
+    lofar_add_test(t_dbcredentials)
+    lofar_add_test(t_defaultmailaddresses)
+    lofar_add_test(t_methodtrigger)
+    lofar_add_test(t_util)
+    lofar_add_test(t_test_utils)
+    lofar_add_test(t_cep4_utils)
+    lofar_add_test(t_postgres)
+ENDIF()
\ No newline at end of file

LCS/PyCommon/test/postgres.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2012-2015    ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+# $Id$
+import psycopg2
+import os, sys
+import logging
+
+logger = logging.getLogger(__name__)
+
+import testing.postgresql
+from lofar.common.dbcredentials import Credentials
+from lofar.common.postgres import PostgresDatabaseConnection
+
+class PostgresTestMixin():
+    '''
+    A common test mixin class from which you can derive to get a freshly setup postgres testing instance with the latest RADB sql setup scripts applied.
+    It implements the unittest setUpClass/tearDownClass methods and uses them as a template method pattern to do all the testing-database setup/teardown work for you.
+    '''
+
+    @classmethod
+    def setUpClass(cls):
+        logger.info('setting up test-database instance...')
+        # connect to shared test db
+        cls._postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db=True)()
+        cls.dbcreds    = Credentials()
+
+        # update credentials (e.g. port changes for each test)
+        cls.dbcreds.host = cls._postgresql.dsn()['host']
+        cls.dbcreds.database = cls._postgresql.dsn()['database']
+        cls.dbcreds.port = cls._postgresql.dsn()['port']
+
+        try:
+            # connect to db as root
+            conn = psycopg2.connect(**cls._postgresql.dsn())
+            cursor = conn.cursor()
+
+            # set credentials to be used during tests
+            cls.dbcreds.user = cls.get_username()
+            cls.dbcreds.password = 'testing_password'    # cannot be empty...
+
+            # create user role
+            query = "CREATE USER %s WITH SUPERUSER PASSWORD '%s'" % (cls.dbcreds.user, cls.dbcreds.password)
+            cursor.execute(query)
+
+            logger.info('Finished setting up test-database instance. It is avaiblable at: %s', cls.dbcreds.stringWithHiddenPassword())
+        finally:
+            cursor.close()
+            conn.commit()
+            conn.close()
+
+        logger.info('Applying test-database schema...')
+        cls.apply_database_schema()
+
+        logger.info('Creating PostgresDatabaseConnection to test-database...')
+        cls.db = cls.create_database_connection()
+        logger.info('PostgresDatabaseConnection to test-database %s is ready to be used.', cls.db)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.close_database_connection()
+
+        cls.print_database_instance_log()
+
+        try:
+            logger.info('removing test-database instance at %s', cls.dbcreds.stringWithHiddenPassword())
+            cls._postgresql.stop()
+            logger.info('test-database instance removed')
+        except Exception as e:
+            logger.info('error while removing test-database instance at %s: %s', cls.dbcreds.stringWithHiddenPassword(), e)
+
+    @classmethod
+    def get_username(cls) -> str:
+        '''return a username for the testing-database who will get SUPERUSER powers'''
+        return 'testing_user'
+
+    @classmethod
+    def apply_database_schema(cls):
+        ''' Override and implement this method. Open a connection to the database specified by cls.dbcreds, and apply your database's sql schema.'''
+        raise NotImplementedError("Please override PostgresTestMixin.apply_database_schema and setup your database with an sql schema.")
+
+    @classmethod
+    def create_database_connection(cls) -> PostgresDatabaseConnection:
+        ''' Override and implement this method if you want to use your PostgresDatabaseConnection-subclass, connect it using the given cls.dbcreds, and return it.'''
+        db = PostgresDatabaseConnection(cls.dbcreds)
+        db.connect()
+        return db
+
+    @classmethod
+    def close_database_connection(cls):
+        ''' close the database connection created in create_database_connection'''
+        try:
+            logger.info('Closing PostgresDatabaseConnection to test-database %s...', cls.db)
+            cls.db.disconnect()
+        except Exception as e:
+            logger.error("Error while closing PostgresDatabaseConnection to test-database: %s", e)
+
+    @classmethod
+    def print_database_instance_log(cls):
+        '''print the log of the testing-database instance (can help debugging sql statements)'''
+        try:
+            db_log_file_name = os.path.join(cls._postgresql.base_dir, '%s.log' % cls._postgresql.name)
+            logger.info('Printing test-postgress-database server log for reference: %s', db_log_file_name)
+            with open(db_log_file_name, 'r') as db_log_file:
+                for line in db_log_file.readlines():
+                    print("  postgres log: %s" % line.strip(), file=sys.stderr)
+        except Exception as e:
+            logger.error("Error while printing test-postgress-database server log: %s", e)

LCS/PyCommon/test/t_postgres.py

@@ -3,65 +3,37 @@
 import unittest
 from unittest import mock
 from lofar.common.postgres import *
-import testing.postgresql
+from lofar.common.testing.postgres import PostgresTestMixin
 import psycopg2
 import signal
-from lofar.common.dbcredentials import Credentials
+from copy import deepcopy
 
 import logging
 logger = logging.getLogger(__name__)
 
-def setUpModule():
-    pass
-
-def tearDownModule():
-    pass
-
-class TestPostgres(unittest.TestCase):
-    def setUp(self):
-        logger.debug("starting test-postgres-database-instance...")
-        self.pgfactory = testing.postgresql.PostgresqlFactory()
-        self.pginstance = self.pgfactory()
-
-        self.dbcreds = Credentials()
-        self.dbcreds.user = 'test_pg_user'
-        self.dbcreds.password = 'test_pg_password' # cannot be empty...
-
-        dsn = self.pginstance.dsn()
-
-        # update credentials from current testing pginstance (e.g. port changes for each test)
-        self.dbcreds.host = dsn['host']
-        self.dbcreds.database = dsn['database']
-        self.dbcreds.port = dsn['port']
-
+class TestPostgres(PostgresTestMixin, unittest.TestCase):
+    @classmethod
+    def apply_database_schema(cls):
         # use 'normal' psycopg2 API to connect and setup the database,
         # not the PostgresDatabaseConnection class, because that's the object-under-test.
-        logger.debug("connecting to %s test-postgres-database-instance...", self.dbcreds.stringWithHiddenPassword())
-        with psycopg2.connect(**dsn) as connection:
+        logger.debug("connecting to %s test-postgres-database-instance...", cls.dbcreds.stringWithHiddenPassword())
+        with psycopg2.connect(**cls.dbcreds.psycopg2_connect_options()) as connection:
             with connection.cursor() as cursor:
-                logger.debug("creating database user and tables in %s test-postgres-database-instance...", self.dbcreds.stringWithHiddenPassword())
-                cursor.execute("CREATE USER %s WITH SUPERUSER PASSWORD '%s';" % (self.dbcreds.user,self.dbcreds.password))
+                logger.debug("creating database user and tables in %s test-postgres-database-instance...", cls.dbcreds.stringWithHiddenPassword())
                 cursor.execute("CREATE TABLE IF NOT EXISTS foo (id serial NOT NULL, bar text NOT NULL, PRIMARY KEY (id));")
                 connection.commit()
 
-        logger.info("created and started %s test-postgres-database-instance", self.dbcreds.stringWithHiddenPassword())
-
-    def tearDown(self):
-        logger.debug("stopping %s test-postgres-database-instance...", self.dbcreds.stringWithHiddenPassword())
-        self.pginstance.stop()
-        logger.info("stopped %s test-postgres-database-instance", self.dbcreds.stringWithHiddenPassword())
-
-    def test_connection_error_on_stopped_pginstance(self):
-        # force to pginstance to stop so we cannot connect to it.
-        self.pginstance.stop()
-        logger.info("stopped %s test-postgres-database-instance", self.dbcreds.stringWithHiddenPassword())
+    def test_connection_error_with_incorrect_dbcreds(self):
+        #connect to incorrect port -> should result in PostgresDBConnectionError
+        incorrect_dbcreds = deepcopy(self.dbcreds)
+        incorrect_dbcreds.port += 1
 
         # test if connecting fails
         with mock.patch('lofar.common.postgres.logger') as mocked_logger:
             with self.assertRaises(PostgresDBConnectionError):
                 NUM_CONNECT_RETRIES = 2
-                db = PostgresDatabaseConnection(dbcreds=self.dbcreds, connect_retry_interval=0.1, num_connect_retries=NUM_CONNECT_RETRIES)
-                db.connect()
+                with PostgresDatabaseConnection(dbcreds=incorrect_dbcreds, connect_retry_interval=0.1, num_connect_retries=NUM_CONNECT_RETRIES) as db:
+                    pass
 
             # check logging
             self.assertEqual(NUM_CONNECT_RETRIES, len([ca for ca in mocked_logger.info.call_args_list if 'retrying to connect' in ca[0][0]]))
@@ -92,7 +64,7 @@ class TestPostgres(unittest.TestCase):
                         self.pginstance.start()
                         logger.info("restarted test-postgres-database-instance")
 
-        with HelperPostgresDatabaseConnection(dbcreds=self.dbcreds, pginstance=self.pginstance) as db:
+        with HelperPostgresDatabaseConnection(dbcreds=self.dbcreds, pginstance=self._postgresql) as db:
             # insert some test data
             db.executeQuery("INSERT INTO foo (bar) VALUES ('my_value');")
             db.commit()
@@ -102,7 +74,8 @@ class TestPostgres(unittest.TestCase):
             self.assertEqual([{'id':1, 'bar': 'my_value'}], result)
 
             # terminate the pginstance (simulating a production database malfunction)
-            self.pginstance.terminate(signal.SIGTERM)
+            logger.info("terminating %s test-postgres-database-instance...", self.dbcreds.stringWithHiddenPassword())
+            self._postgresql.terminate(signal.SIGTERM)
             logger.info("terminated %s test-postgres-database-instance", self.dbcreds.stringWithHiddenPassword())
 
             # prove that the database is down by trying to connect which results in a PostgresDBConnectionError
@@ -131,11 +104,6 @@ class TestPostgres(unittest.TestCase):
                 db.executeQuery("SELECT * FROM error_func();")
 
 
-
-
-
-
-
 logging.basicConfig(format='%(asctime)s %(process)s %(threadName)s %(levelname)s %(message)s', level=logging.DEBUG)
 
 if __name__ == "__main__":

SAS/ResourceAssignment/ResourceAssignmentDatabase/tests/radb_common_testing.py

@@ -27,93 +27,54 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-import testing.postgresql
-from lofar.common.dbcredentials import Credentials
 from lofar.common.postgres import PostgresDatabaseConnection, FETCH_ALL
+from lofar.common.testing.postgres import PostgresTestMixin
 from lofar.sas.resourceassignment.database.radb import RADatabase
 
-class RADBCommonTestMixin():
+class RADBCommonTestMixin(PostgresTestMixin):
     '''
     A common test mixin class from which you can derive to get a freshly setup postgres testing instance with the latest RADB sql setup scripts applied.
     '''
+    def setUp(self):
+        # wipe all tables by truncating specification which cascades into the rest.
+        logger.debug("setUp: Wiping radb tables for each unittest.")
+        self.db.executeQuery("TRUNCATE TABLE resource_allocation.specification CASCADE;")
+        self.db.executeQuery("TRUNCATE TABLE resource_allocation.resource_usage CASCADE;")
+        self.db.executeQuery("TRUNCATE TABLE resource_allocation.resource_usage_delta CASCADE;")
+        self.db.commit()
 
     @classmethod
-    def setUpClass(cls):
-        logger.info('setting up test database instance...')
-        # connect to shared test db
-        cls.postgresql = testing.postgresql.PostgresqlFactory(cache_initialized_db=True)()
-        cls.dbcreds    = Credentials()
-
-        # update credentials (e.g. port changes for each test)
-        cls.dbcreds.host = cls.postgresql.dsn()['host']
-        cls.dbcreds.database = cls.postgresql.dsn()['database']
-        cls.dbcreds.port = cls.postgresql.dsn()['port']
-
-        # connect to db as root
-        conn = psycopg2.connect(**cls.postgresql.dsn())
-        cursor = conn.cursor()
-
-        # set credentials to be used during tests
-        cls.dbcreds.user = 'resourceassignment'
-        cls.dbcreds.password = 'secret'    # cannot be empty...
-
-        # create user role
-        # Note: NOSUPERUSER currently raises "permission denied for schema virtual_instrument"
-        # Maybe we want to sort out user creation and proper permissions in the sql scripts?
-        query = "CREATE USER %s WITH SUPERUSER PASSWORD '%s'" % (cls.dbcreds.user, cls.dbcreds.password)
-        cursor.execute(query)
-
-        cursor.close()
-        conn.commit()
-        conn.close()
+    def get_username(cls) -> str:
+        '''return a username for the testing-database who will get SUPERUSER powers'''
+        return 'resourceassignment'
 
-        logger.info('Finished setting up test database instance. It is avaiblable at: %s', cls.dbcreds.stringWithHiddenPassword())
+    @classmethod
+    def apply_database_schema(cls):
+        logger.info('applying RADB sql schema to %s', cls.dbcreds)
+
+        with PostgresDatabaseConnection(cls.dbcreds) as db:
+            # populate db tables
+            # These are applied in given order to set up test db
+            # Note: cannot use create_and_populate_database.sql since '\i' is not understood by cursor.execute()
+            sql_basepath = os.environ['LOFARROOT'] + "/share/radb/sql/"
+            sql_createdb_paths = [sql_basepath + "create_database.sql",
+                                  sql_basepath + "/add_resource_allocation_statics.sql",
+                                  sql_basepath + "/add_virtual_instrument.sql",
+                                  sql_basepath + "/add_notifications.sql",
+                                  sql_basepath + "/add_functions_and_triggers.sql"]
+
+            for sql_path in sql_createdb_paths:
+                logger.debug("setting up database. applying sql file: %s", sql_path)
+                with open(sql_path) as sql:
+                    db.executeQuery(sql.read())
+                    db.commit()
 
+    @classmethod
+    def create_database_connection(cls) -> PostgresDatabaseConnection:
         cls.radb = RADatabase(cls.dbcreds)
         cls.radb.connect()
+        return cls.radb
 
-        # set up a fresh copy of the RADB sql schema
-        cls._setup_database(cls.radb)
-
-    def setUp(self):
-        # wipe all tables by truncating specification which cascades into the rest.
-        logger.debug("setUp: Wiping radb tables for each unittest.")
-        self.radb.executeQuery("TRUNCATE TABLE resource_allocation.specification CASCADE;")
-        self.radb.commit()
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.radb.disconnect()
-
-        db_log_file_name = os.path.join(cls.postgresql.base_dir, '%s.log' % cls.postgresql.name)
-        logger.info('Printing test-postgress-database server log: %s', db_log_file_name)
-        with open(db_log_file_name, 'r') as db_log_file:
-            for line in db_log_file.readlines():
-                print("  postgres log: %s" % line.strip(), file=sys.stderr)
-
-        logger.info('removing test RA database at %s', cls.dbcreds.stringWithHiddenPassword())
-        cls.postgresql.stop()
-        logger.info('test RA removed')
-
-    @staticmethod
-    def _setup_database(db: PostgresDatabaseConnection):
-        logger.info('applying RADB sql schema to %s', db)
-
-        # populate db tables
-        # These are applied in given order to set up test db
-        # Note: cannot use create_and_populate_database.sql since '\i' is not understood by cursor.execute()
-        sql_basepath = os.environ['LOFARROOT'] + "/share/radb/sql/"
-        sql_createdb_paths = [sql_basepath + "create_database.sql",
-                              sql_basepath + "/add_resource_allocation_statics.sql",
-                              sql_basepath + "/add_virtual_instrument.sql",
-                              sql_basepath + "/add_notifications.sql",
-                              sql_basepath + "/add_functions_and_triggers.sql"]
-
-        for sql_path in sql_createdb_paths:
-            logger.debug("setting up database. applying sql file: %s", sql_path)
-            with open(sql_path) as sql:
-                db.executeQuery(sql.read())
-                db.commit()
 
 class RADBCommonTest(RADBCommonTestMixin, unittest.TestCase):
     # database created?