TMSS-60: use inter-process namedlock to start test-ldap/-postgres servers on...

TMSS-60: use inter-process namedlock to start test-ldap/-postgres servers on their own free ports. This enables running multiple tests servers in parallel, for example for running unit/integration tests in parallel

LCS/PyCommon/test/postgres.py

@@ -30,12 +30,16 @@ from lofar.common.dbcredentials import Credentials
 from lofar.common.postgres import PostgresDatabaseConnection
 from lofar.common.testing.dbcredentials import TemporaryCredentials
 from lofar.common.util import find_free_port
+from datetime import datetime, timedelta
+
+from lofar.common.locking import NamedAtomicLock
 
 class PostgresTestDatabaseInstance():
     ''' A helper class which instantiates a running postgres server (not interfering with any other test/production postgres servers)
     Best used in a 'with'-context so the server is destroyed automagically.
     Derive your own sub-class and implement apply_database_schema with your own sql schema to setup your type of database.
     '''
+    _named_lock = NamedAtomicLock('PostgresTestDatabaseInstance')
 
     def __init__(self, user: str = 'test_user', preferred_port: int=5444) -> None:
         self._postgresql = None
@@ -68,6 +72,10 @@ class PostgresTestDatabaseInstance():
         '''instantiate the isolated postgres server'''
         logger.info('creating test-database instance...')
 
+        with self._named_lock:
+            start_time = datetime.utcnow()
+            while datetime.utcnow()-start_time < timedelta(minutes=1):
+                try:
                     factory = testing.postgresql.PostgresqlFactory(cache_initialized_db=True)
                     factory.settings['port'] = find_free_port(self.tmp_creds.dbcreds.port)
                     self._postgresql = factory()
@@ -84,6 +92,10 @@ class PostgresTestDatabaseInstance():
 
                     logger.info('Applying test-database schema...')
                     self.apply_database_schema()
+                    return
+                except Exception as e:
+                    logger.warning("%s could not be started, retrying with next free port. Error: %s %s", self.__class__.__name__, e.__class__.__name__, e)
+            raise TimeoutError("%s could not be started within 60 seconds. bailing out..." % self.__class__.__name__)
 
     def _create_superuser(self, dsn):
         try:

SAS/TMSS/test/ldap_test_service.py

@@ -5,6 +5,9 @@ logger = logging.getLogger(__name__)
 logging_already_configured = len(logging.root.handlers)>0
 
 from ldap_test import LdapServer
+from ldap_test.server import DEFAULT_GATEWAY_PORT, DEFAULT_PYTHON_PROXY_PORT
+from py4j.java_gateway import Py4JNetworkError
+from datetime import datetime, timedelta
 
 if not logging_already_configured:
     # the 3rd party ldap_test module erroneously does a logging.basicConfig upon module import...
@@ -18,11 +21,13 @@ from optparse import OptionParser
 
 from lofar.common.util import waitForInterrupt, find_free_port
 from lofar.common.testing.dbcredentials import TemporaryCredentials
+from lofar.common.locking import NamedAtomicLock
 
 class TestLDAPServer():
     ''' A helper class which instantiates a running LDAP server (not interfering with any other test/production LDAP servers)
     Best used in a 'with'-context so the server is stoped automagically.
     '''
+    _named_lock = NamedAtomicLock('TestLDAPServer')
 
     def __init__(self, user: str = 'test', password: str = 'test') -> None:
         self._tmp_creds = TemporaryCredentials(user=user, password=password)
@@ -55,6 +60,7 @@ class TestLDAPServer():
         '''instantiate the isolated postgres server'''
         logger.info('creating test-LDAP instance...')
 
+        with self._named_lock:
             self._tmp_creds.dbcreds.type = 'LDAP'
             self._tmp_creds.dbcreds.host = '127.0.0.1'
             self._tmp_creds.dbcreds.port = find_free_port()
@@ -63,7 +69,12 @@ class TestLDAPServer():
             logger.info("Using dbcreds '%s' to start and configure LDAP server: %s",
                         self.dbcreds_id, self.dbcreds.stringWithHiddenPassword())
 
-        self._server = LdapServer({'port': self.dbcreds.port,
+            start_time = datetime.utcnow()
+            while datetime.utcnow()-start_time < timedelta(minutes=1):
+                try:
+                    self._server = LdapServer(java_gateway_port=find_free_port(DEFAULT_GATEWAY_PORT),
+                                              python_proxy_port=find_free_port(DEFAULT_PYTHON_PROXY_PORT),
+                                              config={'port': self.dbcreds.port,
                                                'base': {'objectclass': ['domain'],
                                                         'dn': 'o=lofar,c=eu',
                                                         'attributes': {'o': 'lofar'}},
@@ -110,6 +121,10 @@ class TestLDAPServer():
                     os.environ["TMSS_LDAPCREDENTIALS"] = self.dbcreds_id
                     logger.info('LDAP server running and listening on port %s...', self.dbcreds.port)
                     logger.info('LDAP test user/pass: %s %s...', self.dbcreds.user, self.dbcreds.password)
+                    return
+                except Py4JNetworkError as e:
+                    logger.warning("TestLDAPServer could not be started, retrying with next free port. Error: %s", e)
+            raise TimeoutError("%s could not be started within 60 seconds. bailing out..." % self.__class__.__name__)
 
     def stop(self):
         '''stop the running postgres server'''