diff --git a/.gitattributes b/.gitattributes
index fe2f299fa150f3fcdcba8518d10f075886f06f98..045dc8ec87e73a538079e68f551e6d9354992441 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2648,6 +2648,9 @@ LCS/PyCommon/CMakeLists.txt -text
LCS/PyCommon/__init__.py -text
LCS/PyCommon/datetimeutils.py -text
LCS/PyCommon/factory.py -text
+LCS/PyCommon/test/python-coverage.sh eol=lf
+LCS/PyCommon/test/t_dbcredentials.run eol=lf
+LCS/PyCommon/test/t_dbcredentials.sh eol=lf
LCS/PyCommon/util.py -text
LCS/Tools/src/checkcomp.py -text
LCS/Tools/src/countalllines -text
diff --git a/LCS/Messaging/python/messaging/messagebus.py b/LCS/Messaging/python/messaging/messagebus.py
index b8065dd1f210d7ade83942f0b1ec06dbbebecfa6..a367294f7a43cc41e6fd15c3be99ca90f7a2e123 100644
--- a/LCS/Messaging/python/messaging/messagebus.py
+++ b/LCS/Messaging/python/messaging/messagebus.py
@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
# Default settings for often used parameters.
DEFAULT_ADDRESS_OPTIONS = {'create': 'never'}
DEFAULT_BROKER = "localhost:5672"
-DEFAULT_BROKER_OPTIONS = {}
+DEFAULT_BROKER_OPTIONS = {'reconnect': True}
DEFAULT_RECEIVER_CAPACITY = 1
DEFAULT_TIMEOUT = 5
@@ -58,19 +58,20 @@ class FromBus(object):
but that of __new__().
"""
- def __init__(self, address, options=None, broker=None):
+ def __init__(self, address, options=None, broker=None, broker_options=None):
"""
Initializer.
:param address: valid Qpid address
:param options: valid Qpid address options, e.g. {'create': 'never'}
:param broker: valid Qpid broker URL, e.g. "localhost:5672"
+ :param broker_options: valid Qpid broker options, e.g. {'reconnect': True}
"""
self.address = address
self.options = options if options else DEFAULT_ADDRESS_OPTIONS
self.broker = broker if broker else DEFAULT_BROKER
+ self.broker_options = broker_options if broker_options else DEFAULT_BROKER_OPTIONS
- self.connection = qpid.messaging.Connection(self.broker,
- **DEFAULT_BROKER_OPTIONS)
+ self.connection = qpid.messaging.Connection(self.broker, **self.broker_options)
self.session = None
self.opened=0
@@ -246,19 +247,20 @@ class ToBus(object):
but that of __new__().
"""
- def __init__(self, address, options=None, broker=None):
+ def __init__(self, address, options=None, broker=None, broker_options=None):
"""
Initializer.
:param address: valid Qpid address
:param options: valid Qpid address options, e.g. {'create': 'never'}
:param broker: valid Qpid broker URL, e.g. "localhost:5672"
+ :param broker_options: valid Qpid broker options, e.g. {'reconnect': True}
"""
self.address = address
self.options = options if options else DEFAULT_ADDRESS_OPTIONS
self.broker = broker if broker else DEFAULT_BROKER
+ self.broker_options = broker_options if broker_options else DEFAULT_BROKER_OPTIONS
- self.connection = qpid.messaging.Connection(self.broker,
- **DEFAULT_BROKER_OPTIONS)
+ self.connection = qpid.messaging.Connection(self.broker, **self.broker_options)
self.session = None
self.opened = 0
diff --git a/LCS/Messaging/python/messaging/test/t_RPC.run b/LCS/Messaging/python/messaging/test/t_RPC.run
index ff5140cce0cdcbf9b608287e46524efacebd793a..78025a096a250aae7211dab6c133d46597065e22 100755
--- a/LCS/Messaging/python/messaging/test/t_RPC.run
+++ b/LCS/Messaging/python/messaging/test/t_RPC.run
@@ -10,29 +10,5 @@ queue=$(< /dev/urandom tr -dc [:alnum:] | head -c16)
qpid-config add exchange topic $queue
# Run the unit test
-# either with or without code coverage measurements,
-# depending wheter coverage has been installed
-
-if type "coverage" > /dev/null; then
- #run test using python coverage tool
-
- #erase previous results
- coverage erase
-
- #setup coverage config file
- printf "[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" > .coveragerc
-
- coverage run --branch --include=*Messaging/python* t_RPC.py $queue
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code coverage results *** "
- coverage report -m
- echo " *** End coverage results *** "
- fi
- exit $RESULT
-else
- #coverage not available
- echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- python t_RPC.py $queue
-fi
+source python-coverage.sh
+python_coverage_test "Messaging/python" t_RPC.py $queue
diff --git a/LCS/Messaging/python/messaging/test/t_messagebus.py b/LCS/Messaging/python/messaging/test/t_messagebus.py
index 68a454f9cfc404ecbb92ed2390a358c10678155a..c4a70a0d66dc1df2b6f4930824e659dcf8320333 100644
--- a/LCS/Messaging/python/messaging/test/t_messagebus.py
+++ b/LCS/Messaging/python/messaging/test/t_messagebus.py
@@ -53,7 +53,7 @@ class FromBusInitFailed(unittest.TestCase):
regexp = re.escape(self.error)
regexp += '.*' + 'No address associated with hostname'
with self.assertRaisesRegexp(MessageBusError, regexp):
- with FromBus(QUEUE, broker="foo.bar"):
+ with FromBus(QUEUE, broker="foo.bar", broker_options={'reconnect': False}):
pass
def test_connection_refused(self):
@@ -62,7 +62,7 @@ class FromBusInitFailed(unittest.TestCase):
"""
regexp = re.escape(self.error) + '.*' + 'Connection refused'
with self.assertRaisesRegexp(MessageBusError, regexp):
- with FromBus("fake" + QUEUE, broker="localhost:4"):
+ with FromBus("fake" + QUEUE, broker="localhost:4", broker_options={'reconnect': False}):
pass
@@ -164,7 +164,7 @@ class ToBusInitFailed(unittest.TestCase):
regexp = re.escape(self.error)
regexp += '.*' + 'No address associated with hostname'
with self.assertRaisesRegexp(MessageBusError, regexp):
- with ToBus(QUEUE, broker="foo.bar"):
+ with ToBus(QUEUE, broker="foo.bar", broker_options={'reconnect': False}):
pass
def test_connection_refused(self):
@@ -173,7 +173,7 @@ class ToBusInitFailed(unittest.TestCase):
"""
regexp = re.escape(self.error) + '.*' + 'Connection refused'
with self.assertRaisesRegexp(MessageBusError, regexp):
- with ToBus(QUEUE, broker="localhost:4"):
+ with ToBus(QUEUE, broker="localhost:4", broker_options={'reconnect': False}):
pass
diff --git a/LCS/Messaging/python/messaging/test/t_messagebus.run b/LCS/Messaging/python/messaging/test/t_messagebus.run
index 4d4b2a3d7fe5e11397e7b087c275402acbf068d1..a64fd56ccd2c11c40b771f6cfb90ebda604483ab 100755
--- a/LCS/Messaging/python/messaging/test/t_messagebus.run
+++ b/LCS/Messaging/python/messaging/test/t_messagebus.run
@@ -10,29 +10,5 @@ queue=$(< /dev/urandom tr -dc [:alnum:] | head -c16)
qpid-config add queue $queue
# Run the unit test
-# either with or without code coverage measurements,
-# depending wheter coverage has been installed
-
-if type "coverage" > /dev/null; then
- #run test using python coverage tool
-
- #erase previous results
- coverage erase
-
- #setup coverage config file
- printf "[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" > .coveragerc
-
- coverage run --branch --include=*Messaging/python* t_messagebus.py $queue
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code coverage results *** "
- coverage report -m
- echo " *** End coverage results *** "
- fi
- exit $RESULT
-else
- #coverage not available
- echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- python t_messagebus.py $queue
-fi
+source python-coverage.sh
+python_coverage_test "Messaging/python" t_messagebus.py $queue
diff --git a/LCS/Messaging/python/messaging/test/t_messages.run b/LCS/Messaging/python/messaging/test/t_messages.run
index 1cc5f37320d40620e07e8b02f9d9592068850014..2d82773f1b7a6a80b89bbd455a43c7757aeb2fec 100755
--- a/LCS/Messaging/python/messaging/test/t_messages.run
+++ b/LCS/Messaging/python/messaging/test/t_messages.run
@@ -1,29 +1,5 @@
#!/bin/bash -e
# Run the unit test
-# either with or without code coverage measurements,
-# depending wheter coverage has been installed
-
-if type "coverage" > /dev/null; then
- #run test using python coverage tool
-
- #erase previous results
- coverage erase
-
- #setup coverage config file
- printf "[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" > .coveragerc
-
- coverage run --branch --include=*Messaging/python* t_messages.py
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code coverage results *** "
- coverage report -m
- echo " *** End coverage results *** "
- fi
- exit $RESULT
-else
- #coverage not available
- echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- python t_messages.py
-fi
+source python-coverage.sh
+python_coverage_test "Messaging/python" t_messages.py
diff --git a/LCS/Messaging/python/messaging/test/t_service_message_handler.run b/LCS/Messaging/python/messaging/test/t_service_message_handler.run
index 67ad6c38702db4a0169108c309572f22434f4109..0cbe002228ef9b283cc734f9b6f373cb62635de2 100755
--- a/LCS/Messaging/python/messaging/test/t_service_message_handler.run
+++ b/LCS/Messaging/python/messaging/test/t_service_message_handler.run
@@ -10,29 +10,5 @@ queue=$(< /dev/urandom tr -dc [:alnum:] | head -c16)
qpid-config add exchange topic $queue
# Run the unit test
-# either with or without code coverage measurements,
-# depending wheter coverage has been installed
-
-if type "coverage" > /dev/null; then
- #run test using python coverage tool
-
- #erase previous results
- coverage erase
-
- #setup coverage config file
- printf "[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" > .coveragerc
-
- coverage run --branch --include=*Messaging/python* t_service_message_handler.py $queue
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code coverage results *** "
- coverage report -m
- echo " *** End coverage results *** "
- fi
- exit $RESULT
-else
- #coverage not available
- echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- python t_service_message_handler.py $queue
-fi
+source python-coverage.sh
+python_coverage_test "Messaging/python" t_service_message_handler.py $queue
diff --git a/LCS/PyCommon/CMakeLists.txt b/LCS/PyCommon/CMakeLists.txt
index d7f694b23785b590d5b3fa529eadf0eb48ff3b6b..d8c88f0a3a0a9b6a42efab6b3e76e33c7d6ac3bf 100644
--- a/LCS/PyCommon/CMakeLists.txt
+++ b/LCS/PyCommon/CMakeLists.txt
@@ -5,9 +5,10 @@ lofar_package(PyCommon 1.0)
lofar_find_package(Python 2.6 REQUIRED)
include(PythonInstall)
-#add_subdirectory(test)
+add_subdirectory(test)
set(_py_files
+ dbcredentials.py
factory.py
util.py
datetimeutils.py)
diff --git a/LCS/PyCommon/dbcredentials.py b/LCS/PyCommon/dbcredentials.py
new file mode 100644
index 0000000000000000000000000000000000000000..666a0879b1a2e7e401f3b5169986594511a97c87
--- /dev/null
+++ b/LCS/PyCommon/dbcredentials.py
@@ -0,0 +1,257 @@
+#!/usr/bin/python
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+# $Id$
+
+from glob import glob
+import os
+import pwd
+from ConfigParser import SafeConfigParser, NoSectionError, DuplicateSectionError
+from optparse import OptionGroup
+
+__all__ = ["Credentials", "DBCredentials", "options_group", "parse_options"]
+
+# obtain the environment, and add USER and HOME if needed (since supervisord does not)
+environ = os.environ
+user_info = pwd.getpwuid(os.getuid())
+environ.setdefault("HOME", user_info.pw_dir)
+environ.setdefault("USER", user_info.pw_name)
+
+
+def findfiles(pattern):
+ """ Returns a list of files matched by `pattern'.
+ The pattern can include environment variables using the
+ {VAR} notation.
+ """
+ try:
+ return glob(pattern.format(**environ))
+ except KeyError:
+ return []
+
+
+class Credentials:
+ def __init__(self):
+ # Flavour of database (postgres, mysql, oracle, sqlite)
+ self.type = "postgres"
+
+ # Connection information (port 0 = use default)
+ self.host = "localhost"
+ self.port = 0
+
+ # Authentication
+ self.user = environ["USER"]
+ self.password = ""
+
+ # Database selection
+ self.database = ""
+
+ def __str__(self):
+ return "type={type} addr={host}:{port} auth={user}:{password} db={database}".format(**self.__dict__)
+
+ def pg_connect_options(self):
+ """
+ Returns a dict of options to provide to PyGreSQL's pg.connect function. Use:
+
+ conn = pg.connect(**dbcreds.pg_connect_options())
+ """
+ return {
+ "host": self.host,
+ "port": self.port or -1,
+
+ "user": self.user,
+ "passwd": self.password,
+
+ "dbname": self.database,
+ }
+
+
+class DBCredentials:
+ def __init__(self, filepatterns=None):
+ """
+ Read database credentials from all configuration files matched by any of the patterns.
+
+ By default, the following files are read:
+
+ $LOFARROOT/etc/dbcredentials/*.ini
+ ~/.lofar/dbcredentials/*.ini
+
+ The configuration files allow for any number of database sections:
+
+ [database:OTDB]
+ type = postgres # postgres, mysql, oracle, sqlite
+ host = localhost
+ port = 0 # 0 = use default port
+ user = paulus
+ password = boskabouter
+ database = LOFAR_4
+
+ These database credentials can subsequently be queried under their
+ symbolic name ("OTDB" in the example).
+ """
+ if filepatterns is None:
+ filepatterns = [
+ "{LOFARROOT}/etc/dbcredentials/*.ini",
+ "{HOME}/.lofar/dbcredentials/*.ini",
+ ]
+
+ self.config = SafeConfigParser()
+
+ self.files = sum([findfiles(p) for p in filepatterns],[])
+ self.config.read(self.files)
+
+
+ def get(self, database):
+ """
+ Return credentials for a given database.
+ """
+ # create default credentials
+ creds = Credentials()
+
+ # read configuration
+ try:
+ d = dict(self.config.items(self._section(database)))
+ except NoSectionError:
+ return creds
+
+ # parse and convert config information
+ if "host" in d: creds.host = d["host"]
+ if "port" in d: creds.port = int(d["port"] or 0)
+
+ if "user" in d: creds.user = d["user"]
+ if "password" in d: creds.password = d["password"]
+
+ if "database" in d: creds.database = d["database"]
+
+ return creds
+
+
+ def set(self, database, credentials):
+ """
+ Add or overwrite credentials for a given database.
+ """
+ section = self._section(database)
+
+ # create section if needed
+ try:
+ self.config.add_section(section)
+ except DuplicateSectionError:
+ pass
+
+ # set or override credentials
+ self.config.set(section, "type", credentials.type)
+ self.config.set(section, "host", credentials.host)
+ self.config.set(section, "port", str(credentials.port))
+ self.config.set(section, "user", credentials.user)
+ self.config.set(section, "password", credentials.password)
+ self.config.set(section, "database", credentials.database)
+
+ def list(self):
+ """
+ Return a list of databases for which credentials are available.
+ """
+ sections = self.config.sections()
+ return [s[9:] for s in sections if s.startswith("database:")]
+
+
+ def _section(self, database):
+ return "database:%s" % (database,)
+
+
+def options_group(parser):
+ """
+ Return an optparse.OptionGroup containing command-line parameters
+ for database connections and authentication.
+ """
+ group = OptionGroup(parser, "Database Credentials")
+ group.add_option("-D", "--database", dest="dbName", type="string", default="",
+ help="Name of the database")
+ group.add_option("-H", "--host", dest="dbHost", type="string", default="",
+ help="Hostname of the database server")
+ group.add_option("-p", "--port", dest="dbPort", type="string", default="",
+ help="Port number of the database server")
+ group.add_option("-U", "--user", dest="dbUser", type="string", default="",
+ help="User of the database server")
+ group.add_option("-P", "--password", dest="dbPassword", type="string", default="",
+ help="Password of the database server")
+ group.add_option("-C", "--dbcredentials", dest="dbcredentials", type="string", default="",
+ help="Name of database credential set to use")
+
+ return group
+
+
+def parse_options(options, filepatterns=None):
+ """
+ Parses command-line parameters provided through options_group()
+ and returns a credentials dictionary.
+
+ `filepatterns' can be used to override the patterns used to find configuration
+ files.
+ """
+
+ dbc = DBCredentials(filepatterns)
+
+ # get default values
+ creds = dbc.get(options.dbcredentials)
+
+ # process supplied overrides
+ if options.dbHost: creds.host = options.dbHost
+ if options.dbPort: creds.port = options.dbPort
+ if options.dbUser: creds.user = options.dbUser
+ if options.dbPassword: creds.password = options.dbPassword
+ if options.dbName: creds.database = options.dbName
+
+ return creds
+
+
+if __name__ == "__main__":
+ import sys
+ from optparse import OptionParser
+
+ parser = OptionParser("%prog [options]")
+ parser.add_option("-D", "--database", dest="database", type="string", default="",
+ help="Name of the database")
+ parser.add_option("-L", "--list", dest="list", action="store_true", default=False,
+ help="List known databases")
+ parser.add_option("-F", "--files", dest="files", action="store_true", default=False,
+ help="List names of parsed configuration files")
+ (options, args) = parser.parse_args()
+
+ if not options.database and not options.list and not options.files:
+ print "Missing database name"
+ parser.print_help()
+ sys.exit(1)
+
+ dbc = DBCredentials()
+
+ if options.files:
+ """ Print list of configuration files that we've read. """
+ if dbc.files:
+ print "\n".join(dbc.files)
+ sys.exit(0)
+
+ if options.list:
+ """ Print list of databases. """
+ databases = dbc.list()
+ if databases:
+ print "\n".join(databases)
+ sys.exit(0)
+
+ """ Print credentials of a specific database. """
+ print str(dbc.get(options.database))
+
diff --git a/LCS/PyCommon/test/CMakeLists.txt b/LCS/PyCommon/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a2abf73a98a57ed76555c5beb9d870804ff264b8
--- /dev/null
+++ b/LCS/PyCommon/test/CMakeLists.txt
@@ -0,0 +1,9 @@
+# $Id$
+
+include(LofarCTest)
+
+file(COPY
+ ${CMAKE_CURRENT_SOURCE_DIR}/python-coverage.sh
+ DESTINATION ${CMAKE_BINARY_DIR}/bin)
+
+lofar_add_test(t_dbcredentials)
diff --git a/LCS/PyCommon/test/python-coverage.sh b/LCS/PyCommon/test/python-coverage.sh
new file mode 100755
index 0000000000000000000000000000000000000000..87452ffe0b30e7ae7f362e12f8f6f2523107ef41
--- /dev/null
+++ b/LCS/PyCommon/test/python-coverage.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Default lines to exclude in python-coverage
+COVERAGE_EXCLUDE_LINES="[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n"
+
+# Determine python-coverage executable
+if type "coverage" >& /dev/null; then
+ COVERAGE=coverage
+elif type "python-coverage" >& /dev/null; then
+ COVERAGE=python-coverage
+else
+ COVERAGE=""
+fi
+
+#
+# Run a python test under python-coverage (if available).
+#
+# Usage:
+#
+# python_coverage_test module mytest.py [testarg1 testarg2 ...]
+#
+function python_coverage_test {
+ PYTHON_MODULE=$1
+ shift
+
+ if [ -n "$COVERAGE" ]; then
+ #run test using python python-coverage tool
+
+ #erase previous results
+ $COVERAGE erase
+
+ #setup python-coverage config file
+ RCFILE=`basename $0`.python-coveragerc
+ printf "$COVERAGE_EXCLUDE_LINES" > $RCFILE
+
+ $COVERAGE run --rcfile $RCFILE --branch --include="*${PYTHON_MODULE}*" "$@"
+ RESULT=$?
+ if [ $RESULT -eq 0 ]; then
+ echo " *** Code python-coverage results *** "
+ $COVERAGE report -m
+ echo " *** End python-coverage results *** "
+ fi
+ exit $RESULT
+ else
+ #python-coverage not available
+ echo "Please run: 'pip install python-coverage' to enable code coverage reporting of the unit tests"
+ #run plain test script
+ python "$@"
+ fi
+}
+
diff --git a/LCS/PyCommon/test/t_dbcredentials.py b/LCS/PyCommon/test/t_dbcredentials.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5e8db69c457c5cabdec33f79c5e48b486ec74d3
--- /dev/null
+++ b/LCS/PyCommon/test/t_dbcredentials.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+import unittest
+import tempfile
+from lofar.common.dbcredentials import *
+
+def setUpModule():
+ pass
+
+def tearDownModule():
+ pass
+
+class TestCredentials(unittest.TestCase):
+ def test_default_values(self):
+ c = Credentials()
+
+ self.assertEqual(c.type, "postgres")
+ self.assertEqual(c.host, "localhost")
+ self.assertEqual(c.port, 0)
+ #self.assertEqual(c.user, "")
+ self.assertEqual(c.password, "")
+ self.assertEqual(c.database, "")
+
+ def test_pg_connect_options(self):
+ c = Credentials()
+
+ self.assertEqual(
+ c.pg_connect_options(),
+ { "host": "localhost",
+ "port": -1,
+ "user": c.user,
+ "passwd": "",
+ "dbname": "",
+ })
+
+
+class TestDBCredentials(unittest.TestCase):
+ def test_set_get(self):
+ dbc = DBCredentials(filepatterns=[])
+
+ c_in = Credentials()
+ c_in.host = "example.com"
+ c_in.port = 1234
+ c_in.user = "root"
+ c_in.password = "secret"
+ c_in.database = "mydb"
+
+ dbc.set("DATABASE", c_in)
+ c_out = dbc.get("DATABASE")
+
+ self.assertEqual(str(c_out), str(c_in))
+
+ def test_list(self):
+ dbc = DBCredentials(filepatterns=[])
+
+ c = Credentials()
+ c.host = "foo"
+ dbc.set("FOO", c)
+
+ c = Credentials()
+ c.host = "bar"
+ dbc.set("BAR", c)
+
+ self.assertEqual(sorted(dbc.list()), ["BAR", "FOO"])
+
+
+ def test_config(self):
+ f = tempfile.NamedTemporaryFile()
+ f.write("""
+[database:DATABASE]
+type = postgres
+host = example.com
+port = 1234
+user = root
+password = secret
+database = mydb
+""")
+ f.flush() # don't close since that will delete the TemporaryFile
+
+ # test if DATABASE is there
+ dbc = DBCredentials(filepatterns=[f.name])
+ self.assertEqual(dbc.list(), ["DATABASE"])
+
+ # test if credentials match with what we've written
+ c_in = Credentials()
+ c_in.host = "example.com"
+ c_in.port = 1234
+ c_in.user = "root"
+ c_in.password = "secret"
+ c_in.database = "mydb"
+
+ c_out = dbc.get("DATABASE")
+
+ self.assertEqual(str(c_out), str(c_in))
+
+
+def main(argv):
+ unittest.main(verbosity=2)
+
+if __name__ == "__main__":
+ # run all tests
+ import sys
+ main(sys.argv[1:])
diff --git a/LCS/PyCommon/test/t_dbcredentials.run b/LCS/PyCommon/test/t_dbcredentials.run
new file mode 100755
index 0000000000000000000000000000000000000000..9497e5890832722cbf5692f586e5aac25c6aac1d
--- /dev/null
+++ b/LCS/PyCommon/test/t_dbcredentials.run
@@ -0,0 +1,4 @@
+#!/bin/bash
+source python-coverage.sh
+
+python_coverage_test dbcredentials t_dbcredentials.py
diff --git a/LCS/PyCommon/test/t_dbcredentials.sh b/LCS/PyCommon/test/t_dbcredentials.sh
new file mode 100755
index 0000000000000000000000000000000000000000..16dd2c87db63a6b4c0a82cb045679cc6926c99ca
--- /dev/null
+++ b/LCS/PyCommon/test/t_dbcredentials.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+./runctest.sh t_dbcredentials
diff --git a/SAS/OTDB_Services/TreeService.ini b/SAS/OTDB_Services/TreeService.ini
index a116e6494e582c2424c7bab6e106cdacb5de94a0..b44765f1d908f5663a2c4b891428c94eb502f1f0 100644
--- a/SAS/OTDB_Services/TreeService.ini
+++ b/SAS/OTDB_Services/TreeService.ini
@@ -1,6 +1,6 @@
[program:TreeService]
-command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;TreeService.py --hostname=sasdb --database=LOFAR_4'
+command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;TreeService.py --dbcredentials=OTDB --busname=lofar.otdb.specification'
user=lofarsys
stopsignal=INT ; KeyboardInterrupt
-stdout_logfile=%{program_name)s.stdout
-stderr_logfile=%{program_name)s.stderr
+stdout_logfile=%(program_name)s.stdout
+stderr_logfile=%(program_name)s.stderr
diff --git a/SAS/OTDB_Services/TreeService.py b/SAS/OTDB_Services/TreeService.py
index 39489ced5f2745302a6770af35e0f0af0b4303dd..33cd9ba53ef7b30220171638df8b519d13124094 100755
--- a/SAS/OTDB_Services/TreeService.py
+++ b/SAS/OTDB_Services/TreeService.py
@@ -32,7 +32,6 @@ StatusUpdateCommand : finction to update the status of a tree.
import sys, time, pg
import logging
-from optparse import OptionParser
from lofar.messaging.Service import *
QUERY_EXCEPTIONS = (TypeError, ValueError, MemoryError, pg.ProgrammingError, pg.InternalError)
@@ -213,9 +212,7 @@ class PostgressMessageHandlerInterface(MessageHandlerInterface):
"""
def __init__(self, **kwargs):
super(PostgressMessageHandlerInterface, self).__init__()
- self.database = kwargs.pop("database")
- self.db_user = kwargs.pop("db_user", "postgres")
- self.db_host = kwargs.pop("db_host", "localhost")
+ self.dbcreds = kwargs.pop("dbcreds")
if len(kwargs):
raise AttributeError("Unknown keys in arguments of 'DatabaseTiedMessageHandler: %s" % kwargs)
self.connection = None
@@ -227,13 +224,12 @@ class PostgressMessageHandlerInterface(MessageHandlerInterface):
self.connected = (self.connection and self.connection.status == 1)
while not self.connected:
try:
- self.connection = pg.connect(user=self.db_user, host=self.db_host, dbname=self.database)
+ self.connection = pg.connect(**self.dbcreds.pg_connect_options())
self.connected = True
- logger.info("Connected to database %s on host %s" % (self.database, self.db_host))
- except (TypeError, SyntaxError, pg.InternalError):
+ logger.info("Connected to database %s" % (self.dbcreds,))
+ except (TypeError, SyntaxError, pg.InternalError), e:
self.connected = False
- logger.error("Not connected to database %s on host %s (anymore), retry in 5 seconds"
- % (self.database, self.db_host))
+ logger.error("Not connected to database %s, retry in 5 seconds: %s" % (self.dbcreds, e))
time.sleep(5)
class PostgressTaskSpecificationRequest(PostgressMessageHandlerInterface):
@@ -273,39 +269,37 @@ class PostgressKeyUpdateCommand(PostgressMessageHandlerInterface):
if __name__ == "__main__":
+ from optparse import OptionParser
+ from lofar.common import dbcredentials
+ from lofar.common.util import waitForInterrupt
+
# Check the invocation arguments
parser = OptionParser("%prog [options]")
- parser.add_option("-D", "--database", dest="dbName", type="string", default="",
- help="Name of the database")
- parser.add_option("-H", "--hostname", dest="dbHost", type="string", default="sasdb",
- help="Hostname of database server")
+ parser.add_option("-B", "--busname", dest="busname", type="string", default="testbus",
+ help="Busname or queue-name on which RPC commands are received")
+ parser.add_option_group(dbcredentials.options_group(parser))
(options, args) = parser.parse_args()
- if not options.dbName:
- print "Missing database name"
- parser.print_help()
- sys.exit(0)
+ dbcreds = dbcredentials.parse_options(options)
- if not options.dbHost:
- print "Missing database server name"
+ if not options.busname:
+ print "Missing busname"
parser.print_help()
- sys.exit(0)
-
- busname = sys.argv[1] if len(sys.argv) > 1 else "simpletest"
+ sys.exit(1)
serv1 = Service("TaskSpecification", PostgressTaskSpecificationRequest,
- busname=busname, numthreads=1,
- handler_args = {"database" : options.dbName, "db_host" : options.dbHost})
+ busname=options.busname, numthreads=1,
+ handler_args = {"dbcreds" : dbcreds})
serv2 = Service("StatusUpdateCmd", PostgressStatusUpdateCommand,
- busname=busname, numthreads=1,
- handler_args = {"database" : options.dbName, "db_host" : options.dbHost})
+ busname=options.busname, numthreads=1,
+ handler_args = {"dbcreds" : dbcreds})
serv3 = Service("KeyUpdateCmd", PostgressKeyUpdateCommand,
- busname=busname, numthreads=1,
- handler_args = {"database" : options.dbName, "db_host" : options.dbHost})
+ busname=options.busname, numthreads=1,
+ handler_args = {"dbcreds" : dbcreds})
with serv1, serv2, serv3:
logger.info("Started the OTDB services")
- serv3.wait_for_interrupt()
+ waitForInterrupt()
logger.info("Stopped the OTDB services")
diff --git a/SAS/OTDB_Services/TreeStatusEvents.ini b/SAS/OTDB_Services/TreeStatusEvents.ini
index 3eb140b7361ef6dba04d87a0f3651cfb33d3196b..fd32c77dd843f0fa4dca62f3cede7dc05163e75b 100644
--- a/SAS/OTDB_Services/TreeStatusEvents.ini
+++ b/SAS/OTDB_Services/TreeStatusEvents.ini
@@ -1,6 +1,6 @@
[program:TreeStatusEvents]
-command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;TreeStatusEvents.py --hostname=sasdb --database=LOFAR_4 --busname lofar.otdb.status'
+command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;TreeStatusEvents.py --dbcredentials=OTDB --busname lofar.otdb.status'
user=lofarsys
stopsignal=INT ; KeyboardInterrupt
-stdout_logfile=%{program_name)s.stdout
-stderr_logfile=%{program_name)s.stderr
+stdout_logfile=%(program_name)s.stdout
+stderr_logfile=%(program_name)s.stderr
diff --git a/SAS/OTDB_Services/TreeStatusEvents.py b/SAS/OTDB_Services/TreeStatusEvents.py
index a9264f7b5780df6148c16cfcf22a09626a3416a3..47e4b19dd20c78be07f72fdd0a987c7a2105a353 100755
--- a/SAS/OTDB_Services/TreeStatusEvents.py
+++ b/SAS/OTDB_Services/TreeStatusEvents.py
@@ -24,13 +24,16 @@
Daemon that watches the OTDB database for status changes of trees and publishes those on the messagebus.
"""
-import os, sys, time, pg, signal
-from optparse import OptionParser
+import sys, time, pg
+import logging
from lofar.messaging import EventMessage, ToBus
QUERY_EXCEPTIONS = (TypeError, ValueError, MemoryError, pg.ProgrammingError, pg.InternalError)
alive = False
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger(__name__)
+
# Define our own exceptions
class FunctionError(Exception):
"Something when wrong during the execution of the function"
@@ -68,36 +71,29 @@ def PollForStatusChanges(start_time, end_time, otdb_connection):
def signal_handler(signum, frame):
"Signal redirection to stop the daemon in a neat way."
- print "Stopping program"
+ logger.info("Stopping program")
global alive
alive = False
if __name__ == "__main__":
+ from optparse import OptionParser
+ from lofar.common import dbcredentials
+ import signal
+
# Check the invocation arguments
parser = OptionParser("%prog [options]")
- parser.add_option("-D", "--database", dest="dbName", type="string", default="",
- help="Name of the database")
- parser.add_option("-H", "--hostname", dest="dbHost", type="string", default="sasdb",
- help="Hostname of database server")
parser.add_option("-B", "--busname", dest="busname", type="string", default="",
help="Busname or queue-name the status changes are published on")
+ parser.add_option_group(dbcredentials.options_group(parser))
(options, args) = parser.parse_args()
- if not options.dbName:
- print "Missing database name"
- parser.print_help()
- sys.exit(0)
-
- if not options.dbHost:
- print "Missing database server name"
- parser.print_help()
- sys.exit(0)
+ dbcreds = dbcredentials.parse_options(options)
if not options.busname:
print "Missing busname"
parser.print_help()
- sys.exit(0)
+ sys.exit(1)
# Set signalhandler to stop the program in a neat way.
signal.signal(signal.SIGINT, signal_handler)
@@ -110,15 +106,17 @@ if __name__ == "__main__":
while alive and not connected:
# Connect to the database
try:
- otdb_connection = pg.connect(user="postgres", host=options.dbHost, dbname=options.dbName)
+ otdb_connection = pg.connect(**dbcreds.pg_connect_options())
connected = True
+ logger.info("Connected to database %s" % (dbcreds,))
+
# Get list of allowed tree states
allowed_states = {}
for (state_nr, name) in otdb_connection.query("select id,name from treestate").getresult():
allowed_states[state_nr] = name
- except (TypeError, SyntaxError, pg.InternalError):
+ except (TypeError, SyntaxError, pg.InternalError), e:
connected = False
- print "DatabaseError: Connection to database could not be made, reconnect attempt in 5 seconds"
+ logger.error("Not connected to database %s, retry in 5 seconds: %s" % (dbcreds, e))
time.sleep(5)
# When we are connected we can poll the database
@@ -129,22 +127,23 @@ if __name__ == "__main__":
start_time = otdb_connection.query("select treestatusevent from otdb_admin").getresult()[0][0]
except IndexError, QUERY_EXCEPTIONS:
start_time = "2015-01-01 00:00:00.00"
- print "start_time=", start_time
try:
+ logger.info("start_time=%s, polling database" % (start_time,))
record_list = PollForStatusChanges(start_time, "now", otdb_connection)
except FunctionError, exc_info:
- print exc_info
+ logger.error(exc_info)
else:
for (treeid, state, modtime, creation) in record_list:
content = { "treeID" : treeid, "state" : allowed_states.get(state, "unknwon_state"),
"time_of_change" : modtime }
msg = EventMessage(context="otdb.treestatus", content=content)
- print treeid, allowed_states.get(state, "unknwon_state"), modtime, creation
+ logger.info("sending message treeid %s state %s modtime %s" % (treeid, allowed_states.get(state, "unknwon_state"), modtime))
send_bus.send(msg)
- otdb_connection.query("update otdb_admin set treestatusevent = '%s'" % start_time)
+
start_time = creation
- print "==="
+ logger.info("start_time:=%s" % (start_time,))
+ otdb_connection.query("update otdb_admin set treestatusevent = '%s'" % start_time)
# Redetermine the database status.
connected = (otdb_connection and otdb_connection.status == 1)
diff --git a/SAS/OTDB_Services/test/t_TreeService.run b/SAS/OTDB_Services/test/t_TreeService.run
index 14cc7a3968eb5a1f300a595e24721b6109db21a5..b1abae81cd982d867afd4de30a1e00b27306730b 100755
--- a/SAS/OTDB_Services/test/t_TreeService.run
+++ b/SAS/OTDB_Services/test/t_TreeService.run
@@ -14,36 +14,11 @@ qpid-config add exchange topic $queue
# Setup a clean database with predefined content
dropdb -U postgres -h ${DBHOST} unittest_db
gzip -dc $srcdir/unittest_db.dump.gz | psql -U postgres -h ${DBHOST} -f -
-TreeService.py $queue -D unittest_db -H ${DBHOST} &
+TreeService.py -B $queue -D unittest_db -H ${DBHOST} -U postgres &
SERVER_PID=$!
# Starting up takes a while
sleep 3
# Run the unit test
-# either with or without code coverage measurements,
-# depending wheter coverage has been installed
-
-if type "coverage" > /dev/null; then
- #run test using python coverage tool
-
- #erase previous results
- coverage erase
-
- #setup coverage config file
- printf "[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" > .coveragerc
-
- coverage run --branch --include=*Messaging/python* t_TreeService.py $queue
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code coverage results *** "
- coverage report -m
- echo " *** End coverage results *** "
- fi
- exit $RESULT
-else
- #coverage not available
- echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- python t_TreeService.py $queue
-fi
-
+source python-coverage.sh
+python_coverage_test "Messaging/python" t_TreeService.py $queue
diff --git a/SAS/OTDB_Services/test/t_TreeStatusEvents.run b/SAS/OTDB_Services/test/t_TreeStatusEvents.run
index 22148b99324eeabc1c49ec8f3b56edb082a8df64..3284bfe30000241b37e44914f97be5eebcd4af9e 100755
--- a/SAS/OTDB_Services/test/t_TreeStatusEvents.run
+++ b/SAS/OTDB_Services/test/t_TreeStatusEvents.run
@@ -14,36 +14,11 @@ qpid-config add exchange topic $queue
# Setup a clean database with predefined content
dropdb -U postgres -h ${DBHOST} unittest_db
gzip -dc $srcdir/unittest_db.dump.gz | psql -U postgres -h ${DBHOST} -f -
-TreeStatusEvents.py -B $queue -D unittest_db -H ${DBHOST} &
+TreeStatusEvents.py -B $queue -D unittest_db -H ${DBHOST} -U postgres &
SERVICE_PID=$!
# Starting up takes a while
sleep 3
# Run the unit test
-# either with or without code coverage measurements,
-# depending wheter coverage has been installed
-
-if type "coverage" > /dev/null; then
- #run test using python coverage tool
-
- #erase previous results
- coverage erase
-
- #setup coverage config file
- printf "[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" > .coveragerc
-
- coverage run --branch --include=*Messaging/python* t_TreeStatusEvents.py -D unittest_db -H ${DBHOST} -B $queue
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code coverage results *** "
- coverage report -m
- echo " *** End coverage results *** "
- fi
- exit $RESULT
-else
- #coverage not available
- echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- python t_TreeStatusEvents.py -D unittest_db -H ${DBHOST} -B $queue
-fi
-
+source python-coverage.sh
+python_coverage_test "Messaging/python" t_TreeStatusEvents.py -D unittest_db -H ${DBHOST} -B $queue
diff --git a/SAS/ResourceAssignment/ResourceAssignmentEditor/test/test_webservice.run b/SAS/ResourceAssignment/ResourceAssignmentEditor/test/test_webservice.run
index 3919f00767c19e31b32dee056b20f286e5440180..e929c40a7d057f9338482bf221b9506e15039c7e 100755
--- a/SAS/ResourceAssignment/ResourceAssignmentEditor/test/test_webservice.run
+++ b/SAS/ResourceAssignment/ResourceAssignmentEditor/test/test_webservice.run
@@ -1,25 +1,3 @@
#!/bin/bash
-
-if type "coverage" > /dev/null; then
- #run test using python coverage tool
-
- #erase previous results
- coverage erase
-
- #setup coverage config file
- printf "[report]\nexclude_lines = \n if __name__ == .__main__.\n def main\n" > .coveragerc
-
- coverage run --branch --include=*webservice* test_webservice.py
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- echo " *** Code coverage results *** "
- coverage report -m
- echo " *** End coverage results *** "
- fi
- exit $RESULT
-else
- #coverage not available
- echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
- #run plain test script
- python test_webservice.py
-fi
+source python-coverage.sh
+python_coverage_test "webservice" test_webservice.py