diff --git a/LCS/PyCommon/test/postgres.py b/LCS/PyCommon/test/postgres.py index 6ff9dc07e9f2bfdd4a811547e956cac24de87baf..f98092c3d7fd2a42f745c20ff59b2703bdb6cb43 100755 --- a/LCS/PyCommon/test/postgres.py +++ b/LCS/PyCommon/test/postgres.py @@ -30,12 +30,16 @@ from lofar.common.dbcredentials import Credentials from lofar.common.postgres import PostgresDatabaseConnection from lofar.common.testing.dbcredentials import TemporaryCredentials from lofar.common.util import find_free_port +from datetime import datetime, timedelta + +from lofar.common.locking import NamedAtomicLock class PostgresTestDatabaseInstance(): ''' A helper class which instantiates a running postgres server (not interfering with any other test/production postgres servers) Best used in a 'with'-context so the server is destroyed automagically. Derive your own sub-class and implement apply_database_schema with your own sql schema to setup your type of database. ''' + _named_lock = NamedAtomicLock('PostgresTestDatabaseInstance') def __init__(self, user: str = 'test_user', preferred_port: int=5444) -> None: self._postgresql = None @@ -68,22 +72,30 @@ class PostgresTestDatabaseInstance(): '''instantiate the isolated postgres server''' logger.info('creating test-database instance...') - factory = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) - factory.settings['port'] = find_free_port(self.tmp_creds.dbcreds.port) - self._postgresql = factory() - - # fill credentials with the dynamically created postgress instance (e.g. port changes for each time) - dsn = self._postgresql.dsn() - self.tmp_creds.dbcreds.host = dsn['host'] - self.tmp_creds.dbcreds.database = dsn['database'] - self.tmp_creds.dbcreds.port = dsn['port'] - self.tmp_creds.create() - - # make the user known in the new test database - self._create_superuser(dsn) - - logger.info('Applying test-database schema...') - self.apply_database_schema() + with self._named_lock: + start_time = datetime.utcnow() + while datetime.utcnow()-start_time < timedelta(minutes=1): + try: + factory = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) + factory.settings['port'] = find_free_port(self.tmp_creds.dbcreds.port) + self._postgresql = factory() + + # fill credentials with the dynamically created postgress instance (e.g. port changes for each time) + dsn = self._postgresql.dsn() + self.tmp_creds.dbcreds.host = dsn['host'] + self.tmp_creds.dbcreds.database = dsn['database'] + self.tmp_creds.dbcreds.port = dsn['port'] + self.tmp_creds.create() + + # make the user known in the new test database + self._create_superuser(dsn) + + logger.info('Applying test-database schema...') + self.apply_database_schema() + return + except Exception as e: + logger.warning("%s could not be started, retrying with next free port. Error: %s %s", self.__class__.__name__, e.__class__.__name__, e) + raise TimeoutError("%s could not be started within 60 seconds. bailing out..." % self.__class__.__name__) def _create_superuser(self, dsn): try: diff --git a/SAS/TMSS/test/ldap_test_service.py b/SAS/TMSS/test/ldap_test_service.py index 59eb0b60c041495333804830b90075ed0d472baa..6cb6921e83745cedcff267a5e139b6669963a7a7 100644 --- a/SAS/TMSS/test/ldap_test_service.py +++ b/SAS/TMSS/test/ldap_test_service.py @@ -5,6 +5,9 @@ logger = logging.getLogger(__name__) logging_already_configured = len(logging.root.handlers)>0 from ldap_test import LdapServer +from ldap_test.server import DEFAULT_GATEWAY_PORT, DEFAULT_PYTHON_PROXY_PORT +from py4j.java_gateway import Py4JNetworkError +from datetime import datetime, timedelta if not logging_already_configured: # the 3rd party ldap_test module erroneously does a logging.basicConfig upon module import... @@ -18,11 +21,13 @@ from optparse import OptionParser from lofar.common.util import waitForInterrupt, find_free_port from lofar.common.testing.dbcredentials import TemporaryCredentials +from lofar.common.locking import NamedAtomicLock class TestLDAPServer(): ''' A helper class which instantiates a running LDAP server (not interfering with any other test/production LDAP servers) Best used in a 'with'-context so the server is stoped automagically. ''' + _named_lock = NamedAtomicLock('TestLDAPServer') def __init__(self, user: str = 'test', password: str = 'test') -> None: self._tmp_creds = TemporaryCredentials(user=user, password=password) @@ -55,61 +60,71 @@ class TestLDAPServer(): '''instantiate the isolated postgres server''' logger.info('creating test-LDAP instance...') - self._tmp_creds.dbcreds.type = 'LDAP' - self._tmp_creds.dbcreds.host = '127.0.0.1' - self._tmp_creds.dbcreds.port = find_free_port() - self._tmp_creds.create() - - logger.info("Using dbcreds '%s' to start and configure LDAP server: %s", - self.dbcreds_id, self.dbcreds.stringWithHiddenPassword()) - - self._server = LdapServer({'port': self.dbcreds.port, - 'base': {'objectclass': ['domain'], - 'dn': 'o=lofar,c=eu', - 'attributes': {'o': 'lofar'}}, - 'entries': [ - {'objectclass': 'organizationUnit', - 'dn': 'ou=Users,o=lofar,c=eu', - 'attributes': {'ou': 'Users'}}, - {'objectclass': 'lofarPerson', - 'dn': 'cn=paulus,ou=users,o=lofar,c=eu', - 'attributes': {'cn': 'paulus', - 'userPassword': 'pauluspass', - 'mail': 'paulus@boskabouter.nl', - 'givenName': 'Paulus', - 'sn': 'Boskabouter', - 'lofarPersonSystemrole': 'cn=support,ou=Roles,o=lofar,c=eu'}}, - {'objectclass': 'lofarPerson', - 'dn': 'cn=paula,ou=users,o=lofar,c=eu', - 'attributes': {'cn': 'paula', - 'userPassword': 'paulapass', - 'mail': 'paulus@boskabouter.nl', - 'givenName': 'Paulus', - 'sn': 'Boskabouter', - 'lofarPersonSystemrole': 'cn=user,ou=Roles,o=lofar,c=eu'}}, - {'objectclass': 'lofarPerson', - 'dn': 'cn=%s,ou=users,o=lofar,c=eu' % self.dbcreds.user, - 'attributes': {'cn': self.dbcreds.user, - 'userPassword': self.dbcreds.password, - 'mail': '%s@lofar.test' % self.dbcreds.user, - 'givenName': self.dbcreds.user, - 'sn': 'lofar_test'}}, - {'objectclass': 'organizationUnit', - 'dn': 'ou=Roles,o=lofar,c=eu', - 'attributes': {'ou': 'Roles'}}, - {'objectclass': 'lofarSystemrole', - 'dn': 'cn=user,ou=roles,o=lofar,c=eu', - 'attributes': {'cn': 'user'}}, - {'objectclass': 'lofarSystemrole', - 'dn': 'cn=support,ou=roles,o=lofar,c=eu', - 'attributes': {'cn': 'support'}}, - ] - }) - - self._server.start() - os.environ["TMSS_LDAPCREDENTIALS"] = self.dbcreds_id - logger.info('LDAP server running and listening on port %s...', self.dbcreds.port) - logger.info('LDAP test user/pass: %s %s...', self.dbcreds.user, self.dbcreds.password) + with self._named_lock: + self._tmp_creds.dbcreds.type = 'LDAP' + self._tmp_creds.dbcreds.host = '127.0.0.1' + self._tmp_creds.dbcreds.port = find_free_port() + self._tmp_creds.create() + + logger.info("Using dbcreds '%s' to start and configure LDAP server: %s", + self.dbcreds_id, self.dbcreds.stringWithHiddenPassword()) + + start_time = datetime.utcnow() + while datetime.utcnow()-start_time < timedelta(minutes=1): + try: + self._server = LdapServer(java_gateway_port=find_free_port(DEFAULT_GATEWAY_PORT), + python_proxy_port=find_free_port(DEFAULT_PYTHON_PROXY_PORT), + config={'port': self.dbcreds.port, + 'base': {'objectclass': ['domain'], + 'dn': 'o=lofar,c=eu', + 'attributes': {'o': 'lofar'}}, + 'entries': [ + {'objectclass': 'organizationUnit', + 'dn': 'ou=Users,o=lofar,c=eu', + 'attributes': {'ou': 'Users'}}, + {'objectclass': 'lofarPerson', + 'dn': 'cn=paulus,ou=users,o=lofar,c=eu', + 'attributes': {'cn': 'paulus', + 'userPassword': 'pauluspass', + 'mail': 'paulus@boskabouter.nl', + 'givenName': 'Paulus', + 'sn': 'Boskabouter', + 'lofarPersonSystemrole': 'cn=support,ou=Roles,o=lofar,c=eu'}}, + {'objectclass': 'lofarPerson', + 'dn': 'cn=paula,ou=users,o=lofar,c=eu', + 'attributes': {'cn': 'paula', + 'userPassword': 'paulapass', + 'mail': 'paulus@boskabouter.nl', + 'givenName': 'Paulus', + 'sn': 'Boskabouter', + 'lofarPersonSystemrole': 'cn=user,ou=Roles,o=lofar,c=eu'}}, + {'objectclass': 'lofarPerson', + 'dn': 'cn=%s,ou=users,o=lofar,c=eu' % self.dbcreds.user, + 'attributes': {'cn': self.dbcreds.user, + 'userPassword': self.dbcreds.password, + 'mail': '%s@lofar.test' % self.dbcreds.user, + 'givenName': self.dbcreds.user, + 'sn': 'lofar_test'}}, + {'objectclass': 'organizationUnit', + 'dn': 'ou=Roles,o=lofar,c=eu', + 'attributes': {'ou': 'Roles'}}, + {'objectclass': 'lofarSystemrole', + 'dn': 'cn=user,ou=roles,o=lofar,c=eu', + 'attributes': {'cn': 'user'}}, + {'objectclass': 'lofarSystemrole', + 'dn': 'cn=support,ou=roles,o=lofar,c=eu', + 'attributes': {'cn': 'support'}}, + ] + }) + + self._server.start() + os.environ["TMSS_LDAPCREDENTIALS"] = self.dbcreds_id + logger.info('LDAP server running and listening on port %s...', self.dbcreds.port) + logger.info('LDAP test user/pass: %s %s...', self.dbcreds.user, self.dbcreds.password) + return + except Py4JNetworkError as e: + logger.warning("TestLDAPServer could not be started, retrying with next free port. Error: %s", e) + raise TimeoutError("%s could not be started within 60 seconds. bailing out..." % self.__class__.__name__) def stop(self): '''stop the running postgres server'''