diff --git a/.gitattributes b/.gitattributes
index 65491dbe12c9bd96e4cb43db929505908bca7251..b29d576159d02102eab2bcc15f6a86eaa41d69e7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -822,6 +822,31 @@ CEP/Pipeline/docs/sphinx/source/user/usage/running.rst -text
CEP/Pipeline/docs/sphinx/source/user/usage/tasks.rst -text
CEP/Pipeline/mac/src/Echo_Protocol.prot -text
CEP/Pipeline/mac/test/Observation6129 -text
+CEP/Pipeline/recipes/sip/master/__init__.py -text
+CEP/Pipeline/recipes/sip/nodes/__init__.py -text
+CEP/Pipeline/recipes/sip/nodes/imager_create_dbs.py -text
+CEP/Pipeline/test/CMakeLists.txt -text
+CEP/Pipeline/test/__init__.py -text
+CEP/Pipeline/test/cuisine/__init__.py -text
+CEP/Pipeline/test/cuisine/lofaringredient.py -text
+CEP/Pipeline/test/pipeline_test.run -text
+CEP/Pipeline/test/pipeline_test.sh -text
+CEP/Pipeline/test/recipes/__init__.py -text
+CEP/Pipeline/test/recipes/master/__init__.py -text
+CEP/Pipeline/test/recipes/nodes/__init__.py -text
+CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py -text
+CEP/Pipeline/test/support/__init__.py -text
+CEP/Pipeline/test/test_framework/CMakeLists.txt -text
+CEP/Pipeline/test/test_framework/__init__.py -text
+CEP/Pipeline/test/test_framework/fixture/__init__.py -text
+CEP/Pipeline/test/test_framework/fixture/gsmutils.py -text
+CEP/Pipeline/test/test_framework/fixture/logger.py -text
+CEP/Pipeline/test/test_framework/fixture/monetdb/__init__.py -text
+CEP/Pipeline/test/test_framework/fixture/monetdb/sql.py -text
+CEP/Pipeline/test/test_framework/fixture/pyrap/__init__.py -text
+CEP/Pipeline/test/test_framework/fixture/pyrap/tables.py -text
+CEP/Pipeline/test/test_framework/fixture/read.me -text
+CEP/Pipeline/test/test_framework/unittest_runner.py -text
CEP/pyparmdb/src/gsmutils.py -text
CEP/pyparmdb/src/lsm.py -text
CMake/FindAskapSoft.cmake -text
diff --git a/CEP/Pipeline/CMakeLists.txt b/CEP/Pipeline/CMakeLists.txt
index 3f840d30676f232eb592425808da27fd9e10c6eb..5b1dcbff6a83b2d264f2262bc8dd33e20643b7eb 100644
--- a/CEP/Pipeline/CMakeLists.txt
+++ b/CEP/Pipeline/CMakeLists.txt
@@ -1,5 +1,7 @@
# $Id$
lofar_add_package(Pipeline-Framework framework)
-#lofar_add_package(Pipeline-MAC mac)
lofar_add_package(Pipeline-Recipes recipes)
+#lofar_add_package(Pipeline-MAC mac)
+
+add_subdirectory(test)
diff --git a/CEP/Pipeline/recipes/sip/CMakeLists.txt b/CEP/Pipeline/recipes/sip/CMakeLists.txt
index 211866f1bd937a0c7e4eab293001eb8d6fcb29f8..79d78ab5d51b3cb75976c1141e7ff3b41b71511f 100644
--- a/CEP/Pipeline/recipes/sip/CMakeLists.txt
+++ b/CEP/Pipeline/recipes/sip/CMakeLists.txt
@@ -24,6 +24,7 @@ python_install(
master/target_pipeline.py
master/vdsmaker.py
master/vdsreader.py
+ master/__init__.py
nodes/bbs.py
nodes/cimager.py
nodes/count_timesteps.py
@@ -43,6 +44,8 @@ python_install(
nodes/demix/shiftphasecenter.py
nodes/demix/smoothdemix.py
nodes/demix/subtract_from_averaged.py
+ nodes/imager_create_dbs.py
+ nodes/__init__.py
DESTINATION lofarpipe/recipes)
install(FILES
diff --git a/CEP/Pipeline/framework/lofarpipe/tests/__init__.py b/CEP/Pipeline/recipes/sip/master/__init__.py
similarity index 100%
rename from CEP/Pipeline/framework/lofarpipe/tests/__init__.py
rename to CEP/Pipeline/recipes/sip/master/__init__.py
diff --git a/CEP/Pipeline/recipes/sip/nodes/__init__.py b/CEP/Pipeline/recipes/sip/nodes/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/recipes/sip/nodes/imager_create_dbs.py b/CEP/Pipeline/recipes/sip/nodes/imager_create_dbs.py
new file mode 100644
index 0000000000000000000000000000000000000000..6126c350f20fd5cfb54ec8df2df031394fdb9fdc
--- /dev/null
+++ b/CEP/Pipeline/recipes/sip/nodes/imager_create_dbs.py
@@ -0,0 +1,258 @@
+## LOFAR AWIMAGER RECIPE
+## Wouter Klijn 2012
+## klijn@astron.nl
+## -----------------------------------------------------------------------------
+#
+from __future__ import with_statement
+import sys
+import subprocess
+import math
+
+import pyrap.tables as pt #@UnresolvedImport
+import monetdb.sql as db #@UnresolvedImport
+import gsmutils as gsm #@UnresolvedImport
+
+from lofarpipe.support.lofarnode import LOFARnodeTCP
+from lofarpipe.support.pipelinelogging import log_process_output
+#from lofar.parameterset import parameterset #@UnresolvedImport #marcel new im
+
+#TODO: de template moet mischien toch ergens anders vandaan worden gehaalt.
+template_parmdb = """
+create tablename="{0}"
+adddef Gain:0:0:Ampl values=1.0
+adddef Gain:1:1:Ampl values=1.0
+adddef Gain:0:0:Real values=1.0
+adddef Gain:1:1:Real values=1.0
+adddef DirectionalGain:0:0:Ampl values=1.0
+adddef DirectionalGain:1:1:Ampl values=1.0
+adddef DirectionalGain:0:0:Real values=1.0
+adddef DirectionalGain:1:1:Real values=1.0
+adddef AntennaOrientation values=5.497787144
+quit
+"""
+
+
+class imager_create_dbs(LOFARnodeTCP):
+ def run(self, concatenated_measurement_set, sourcedb_target_path,
+ monet_db_hostname, monet_db_port, monet_db_name, monet_db_user,
+ monet_db_password, assoc_theta, parmdb_executable, slice_paths,
+ parmdb_suffix):
+ """
+ run() creates a source db for the concatenated measurement set. For each
+ of the individual time slices a paramdb is created.
+ All inputs are path wise, and no information regarding the number of
+ slices is needed
+ """
+ self.logger.info("Starting imager_create_dbs Node")
+
+ # create the source db
+ # returns zero on succes
+ if self._create_bbs_sky_model(concatenated_measurement_set,
+ sourcedb_target_path, monet_db_hostname, monet_db_port,
+ monet_db_name, monet_db_user, monet_db_password, assoc_theta):
+ return 1
+
+ # for each slice_set, create a parmdb in the target path
+ # slice_paths is string representation of am array, convert
+ slice_paths = eval(slice_paths)
+ if self._create_parmdb_for_timeslices(parmdb_executable, slice_paths,
+ parmdb_suffix):
+ return 1
+
+ return 0
+
+
+ def _field_of_view(self, measurement_set, alpha_one = None):
+ """
+ _field_of_view calculates the fov, which is dependend on the
+ station type, location and mode:
+ For details see:
+ (1) http://www.astron.nl/radio-observatory/astronomers/lofar-imaging-capabilities-sensitivity/lofar-imaging-capabilities/lofa
+
+ """
+ # Open the ms
+ t = pt.table(measurement_set)
+
+ # Get antenna name and observation mode
+ antenna = pt.table(t.getkeyword("ANTENNA"))
+ antenna_name = antenna.getcell('NAME', 0)
+ antenna.close()
+
+ observation = pt.table(t.getkeyword("OBSERVATION"))
+ antenna_set = observation.getcell('LOFAR_ANTENNA_SET', 0)
+ observation.close
+
+ #static parameters for the station diameters ref (1)
+ hba_core_diameter = 30.8
+ hba_remote_diameter = 41.1
+ lba_inner = 32.3
+ lba_outer = 81.3
+
+ #use measurement set information to assertain antenna diameter
+ station_diameter = None
+ if antenna_name.count('HBA'):
+ if antenna_name.count('CS'):
+ station_diameter = hba_core_diameter
+ elif antenna_name.count('RS'):
+ station_diameter = hba_remote_diameter
+ elif antenna_name.count('LBA'):
+ if antenna_set.count('INNER'):
+ station_diameter = lba_inner
+ elif antenna_set.count('OUTER'):
+ station_diameter = lba_outer
+
+ #raise exception if the antenna is not of a supported type
+ if station_diameter == None:
+ self.logger.error('Unknown antenna type for antenna: {0} , {1}'.format(\
+ antenna_name, antenna_set))
+ raise Exception("Unknown antenna type encountered in Measurement set")
+
+ #Get the wavelength
+ spectral_window_table = pt.table(t.getkeyword("SPECTRAL_WINDOW"))
+ freq = float(spectral_window_table.getcell("REF_FREQUENCY", 0))
+ wave_length = pt.taql('CALC C()') / freq
+
+ # Now calculate the FOV see ref (1)
+ # alpha_one is a magic parameter: The value 1.3 is representative for a
+ # WSRT dish, where it depends on the dish illumination
+ alpha_one = 1.3
+
+ #alpha_one is in radians so transform to degrees for output
+ fwhm = alpha_one * (wave_length / station_diameter) * (180 / math.pi)
+ fov = fwhm / 2.0
+ t.close()
+
+ return fov
+
+
+ def _create_parmdb(self, parmdb_executable, target_dir_path):
+ """
+ _create_parmdb, creates a parmdb_executable at the target_dir_path using the
+ suplied executable. Does not test for existence of target parent dir
+ returns 1 if parmdb_executable failed 0 otherwise
+ """
+ # Format the template string by inserting the target dir
+ formatted_template = template_parmdb.format(target_dir_path)
+ try:
+ # Spawn a subprocess and connect the pipelines
+ parmdbm_process = subprocess.Popen(
+ parmdb_executable,
+ stdin = subprocess.PIPE,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.PIPE
+ )
+ # Send formatted template on stdin
+ sout, serr = parmdbm_process.communicate(formatted_template)
+
+ # Log the output
+ log_process_output("parmdbm", sout, serr, self.logger)
+ except OSError, e:
+ self.logger.error("Failed to spawn parmdbm: {0}".format(str(e)))
+ return 1
+
+ return 0
+
+
+ def _create_parmdb_for_timeslices(self, parmdb_executable, slice_paths,
+ suffix):
+ """
+ _create_parmdb_for_timeslices creates a paramdb for each of the
+ supplied time slices. The paramdb path = input path + suffix.
+ returns 0 on succes 1 on failure:
+ """
+ for slice_path in slice_paths:
+ #Create the paths based on the 'source ms'
+ ms_parmdb_path = slice_path + suffix
+ #call parmdb return failure if a single create failed
+ if self._create_parmdb(parmdb_executable, ms_parmdb_path) != 0:
+ return 1
+
+ return 0
+
+
+ def _create_monet_db_connection(self, hostname, database, username, password,
+ port):
+ """
+ Create and return a monat db connection. Return None if the creation
+ failed and log the error. Returns the connection if succeed.
+ """
+ try:
+ conn = db.connect(hostname = hostname, database = database,
+ username = username, password = password,
+ port = port)
+ except db.Error, e:
+ self.logger.error("failed to create a monetDB connection: "
+ "{0}".format(str(e)))
+ return None
+
+ return conn
+
+
+ def _get_ra_and_decl_from_ms(self, measurement_set):
+ """
+ This function uses pyrap to read the ra and declanation from a
+ measurement set (used by exprected_fluxes_in_fov). This is a position
+ in the sky. These values are stored in the field.phase_dir in the first
+ row. All exceptions thrown are caught and logged, return None if reading
+ failed
+ """
+ try:
+ # open the ms, get the phase direction
+ t = pt.table(measurement_set)
+ t1 = pt.table(t.getkeyword("FIELD"))
+ ra_and_decl = t1.getcell("PHASE_DIR", 0)
+ t1.close()
+ t.close()
+ except Exception, e:
+ #catch all exceptions and log
+ self.logger.error("Error loading FIELD/PHASE_DIR from "
+ "measurementset {0} : {1}".format(measurement_set,
+ str(e)))
+ return None
+
+ # Return the ra and decl
+ if len(ra_and_decl) != 2:
+ self.logger.error("returned PHASE_DIR data did not contain two values")
+ return None
+ return (ra_and_decl[0], ra_and_decl[1])
+
+
+ def _create_bbs_sky_model(self, measurement_set, path_output_skymap,
+ monet_db_host, monet_db_port, monet_db_name,
+ monet_db_user, monet_db_password,
+ assoc_theta = None):
+ """
+ Create a bbs sky model. Based on the measurement (set) suplied
+ The skymap is created at the path_output_skymap
+ """
+
+ # Create monetdb connection
+ conn = self._create_monet_db_connection(monet_db_host, monet_db_name,
+ monet_db_user, monet_db_password, monet_db_port)
+
+ # get position of the target in the sky
+ (ra_c, decl_c) = self._get_ra_and_decl_from_ms(measurement_set)
+
+ # Get the Fov: sources in this fov should be included in the skumodel
+ fov_radius = self._field_of_view(measurement_set)
+
+ # !!magic constant!! This value is calculated based on communications with Bart Sheers
+ if assoc_theta == None:
+ assoc_theta = 90.0 / 3600
+
+ try:
+ gsm.expected_fluxes_in_fov(conn, ra_c, decl_c, fov_radius, assoc_theta,
+ path_output_skymap, storespectraplots = False)
+ except Exception, e:
+ self.logger.error("expected_fluxes_in_fov raise exception: " +
+ str(e))
+ return 1
+
+ return 0
+
+
+if __name__ == "__main__":
+ # args contain information regarding to the logging server
+ jobid, jobhost, jobport = sys.argv[1:4]
+ sys.exit(imager_create_dbs(jobid, jobhost, jobport).run_with_stored_arguments())
+
diff --git a/CEP/Pipeline/test/CMakeLists.txt b/CEP/Pipeline/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9c0697c51d251e7eb8e90d99f1235165692b567a
--- /dev/null
+++ b/CEP/Pipeline/test/CMakeLists.txt
@@ -0,0 +1,6 @@
+# $Id: CMakeLists.txt 18904 2011-09-28 12:15:43Z diepen $
+include(LofarCTest)
+
+lofar_add_test(pipeline_test)
+
+
diff --git a/CEP/Pipeline/test/__init__.py b/CEP/Pipeline/test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/cuisine/__init__.py b/CEP/Pipeline/test/cuisine/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/framework/lofarpipe/tests/lofaringredient.py b/CEP/Pipeline/test/cuisine/lofaringredient.py
similarity index 95%
rename from CEP/Pipeline/framework/lofarpipe/tests/lofaringredient.py
rename to CEP/Pipeline/test/cuisine/lofaringredient.py
index 0a41c79ed0ff8c0f98a2914ed82d0eb94efb7df1..7540bb86cf788ecb424398e783b1d37902dc7e75 100644
--- a/CEP/Pipeline/framework/lofarpipe/tests/lofaringredient.py
+++ b/CEP/Pipeline/test/cuisine/lofaringredient.py
@@ -1,217 +1,217 @@
-# LOFAR PIPELINE FRAMEWORK
-#
-# Tests: ingredients
-# John Swinbank, 2010
-# swinbank@transientskp.org
-# ------------------------------------------------------------------------------
-import xmlrunner
-import unittest
-import os
-
-class StringFieldTest(unittest.TestCase):
- """
- Tests for :class:`lofarpipe.support.lofaringredient.StringField`
- """
- def setUp(self):
- from lofarpipe.support.lofaringredient import StringField
- self.stringfield = StringField(default="a string")
-
- def test_validator(self):
- """
- Check that strings are correctly regarded as valid, and integers
- aren't.
- """
- self.assertFalse(self.stringfield.is_valid(1))
- self.assertTrue(self.stringfield.is_valid("1"))
-
- def test_default(self):
- """
- Check that default is correctly set.
- """
- self.assertEqual(self.stringfield.default, "a string")
-
-class IntFieldTest(unittest.TestCase):
- """
- Tests for :class:`lofarpipe.support.lofaringredient.IntField`
- """
- def setUp(self):
- from lofarpipe.support.lofaringredient import IntField
- self.intfield = IntField(default=1)
-
- def test_validator(self):
- """
- Check that integers are correctly regarded as valid, and strings
- aren't.
- """
- self.assertFalse(self.intfield.is_valid("1"))
- self.assertTrue(self.intfield.is_valid(1))
-
- def test_default(self):
- """
- Check that default is correctly set.
- """
- self.assertEqual(self.intfield.default, 1)
-
- def test_coerce(self):
- """
- Check that a string is correctly coerced to an integer.
- """
- self.assertEqual(self.intfield.coerce("1"), 1)
-
-class FloatFieldTest(unittest.TestCase):
- """
- Tests for :class:`lofarpipe.support.lofaringredient.FloatField`
- """
- def setUp(self):
- from lofarpipe.support.lofaringredient import FloatField
- self.floatfield = FloatField(default=1.0)
-
- def test_validator(self):
- """
- Check that floats are correctly regarded as valid, and strings
- aren't.
- """
- self.assertFalse(self.floatfield.is_valid("1"))
- self.assertTrue(self.floatfield.is_valid(1.0))
-
- def test_default(self):
- """
- Check that default is correctly set.
- """
- self.assertEqual(self.floatfield.default, 1.0)
-
- def test_coerce(self):
- """
- Check that a string is correctly coerced to an float.
- """
- self.assertEqual(self.floatfield.coerce("1"), 1.0)
-
-class FileFieldTest(unittest.TestCase):
- """
- Tests for :class:`lofarpipe.support.lofaringredient.FileField`
- """
- def setUp(self):
- from lofarpipe.support.lofaringredient import FileField
- self.filefield = FileField(default='/')
-
- def test_validator(self):
- """
- Integers are not valid as filenames, and certainly don't exist on
- disk.
-
- ``/`` should, though.
- """
- self.assertFalse(self.filefield.is_valid(1))
- self.assertTrue(self.filefield.is_valid("/"))
-
- def test_default(self):
- """
- Check that default is correctly set.
- """
- self.assertEqual(self.filefield.default, "/")
-
-class ExecFieldTest(unittest.TestCase):
- """
- Tests for :class:`lofarpipe.support.lofaringredient.ExecField`
- """
- def setUp(self):
- from lofarpipe.support.lofaringredient import ExecField
- self.execfield = ExecField(default='/bin/ls')
-
- def test_validator(self):
- """
- ``/etc/passwd`` should always exist as a file on disk, but not be
- executable.
-
- ``/bin/ls`` should always exist, and must be executable.
- """
- self.assertFalse(self.execfield.is_valid("/etc/passwd"))
- self.assertTrue(self.execfield.is_valid("/bin/ls"))
-
- def test_default(self):
- """
- Check that default is correctly set.
- """
- self.assertEqual(self.execfield.default, "/bin/ls")
-
-class DirectoryFieldTest(unittest.TestCase):
- """
- Tests for :class:`lofarpipe.support.lofaringredient.DirectoryField`
- """
- def setUp(self):
- from lofarpipe.support.lofaringredient import DirectoryField
- self.directoryfield = DirectoryField(default='/tmp')
-
- def test_validator(self):
- """
- An integer is not a valid directory.
-
- ``/tmp`` should always be valid.
- """
- self.assertFalse(self.directoryfield.is_valid(1))
- self.assertTrue(self.directoryfield.is_valid("/tmp"))
-
- def test_default(self):
- """
- Check that default is correctly set.
- """
- self.assertEqual(self.directoryfield.default, "/tmp")
-
- def test_coerce(self):
- """
- Coercing a create-able directory name should cause it to exist. We
- should always be able to write in ``/tmp``.
- """
- self.directoryfield.coerce("/tmp/foo")
- self.assertTrue(os.path.exists("/tmp/foo"))
-
-class LOFARIngredientTest(unittest.TestCase):
- """
- Tests for :class:`lofarpipe.support.lofaringredient.LOFARingredient`
- """
- def setUp(self):
- """
- An instance of
- :class:`~lofarpipe.support.lofaringredient.LOFARingredient` is defined
- which contains three instances of
- :class:`~lofarpipe.support.lofaringredient.StringField`.
- """
- from lofarpipe.support.lofaringredient import StringField
- from lofarpipe.support.lofaringredient import LOFARingredient
- f = StringField(default="foo")
- g = StringField()
- h = StringField(default=1)
- self.lofaringredient = LOFARingredient({"f": f, "g": g, "h": h})
-
- def test_keys(self):
- """
- ``self.lofaringredient`` should contain keys for the two fields
- which have default parameters, but not for the one which is unset.
- """
- self.assertEqual(len(self.lofaringredient.keys()), 2)
- self.assertRaises(KeyError, lambda: self.lofaringredient['g'])
-
- def test_values(self):
- """
- Prior to setting, the value of the fields should be equal to
- the default value.
- """
- self.assertEqual(self.lofaringredient['f'], "foo")
-
- def test_set(self):
- """
- When set, the value of the fields should be equal to the new value.
- """
- self.lofaringredient['g'] = "bar"
- self.assertEqual(self.lofaringredient['g'], "bar")
-
- def test_bad_values(self):
- """
- Unacceptable values should raise an exception.
- """
- self.assertRaises(TypeError, lambda: self.lofaringredient['h'])
- self.lofaringredient['h'] = "bar"
- self.assertEqual(self.lofaringredient['h'], "bar")
-
-if __name__ == "__main__":
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports.xml'))
+# LOFAR PIPELINE FRAMEWORK
+#
+# Tests: ingredients
+# John Swinbank, 2010
+# swinbank@transientskp.org
+# ------------------------------------------------------------------------------
+import unittest
+import os
+
+class StringFieldTest(unittest.TestCase):
+ """
+ Tests for :class:`lofarpipe.support.lofaringredient.StringField`
+ """
+ def setUp(self):
+ from lofarpipe.support.lofaringredient import StringField
+ self.stringfield = StringField(default="a string")
+
+ def test_validator(self):
+ """
+ Check that strings are correctly regarded as valid, and integers
+ aren't.
+ """
+ self.assertFalse(self.stringfield.is_valid(1))
+ self.assertTrue(self.stringfield.is_valid("1"))
+
+ def test_default(self):
+ """
+ Check that default is correctly set.
+ """
+ self.assertEqual(self.stringfield.default, "a string")
+
+class IntFieldTest(unittest.TestCase):
+ """
+ Tests for :class:`lofarpipe.support.lofaringredient.IntField`
+ """
+ def setUp(self):
+ from lofarpipe.support.lofaringredient import IntField
+ self.intfield = IntField(default=1)
+
+ def test_validator(self):
+ """
+ Check that integers are correctly regarded as valid, and strings
+ aren't.
+ """
+ self.assertFalse(self.intfield.is_valid("1"))
+ self.assertTrue(self.intfield.is_valid(1))
+
+ def test_default(self):
+ """
+ Check that default is correctly set.
+ """
+ self.assertEqual(self.intfield.default, 1)
+
+ def test_coerce(self):
+ """
+ Check that a string is correctly coerced to an integer.
+ """
+ self.assertEqual(self.intfield.coerce("1"), 1)
+
+class FloatFieldTest(unittest.TestCase):
+ """
+ Tests for :class:`lofarpipe.support.lofaringredient.FloatField`
+ """
+ def setUp(self):
+ from lofarpipe.support.lofaringredient import FloatField
+ self.floatfield = FloatField(default=1.0)
+
+ def test_validator(self):
+ """
+ Check that floats are correctly regarded as valid, and strings
+ aren't.
+ """
+ self.assertFalse(self.floatfield.is_valid("1"))
+ self.assertTrue(self.floatfield.is_valid(1.0))
+
+ def test_default(self):
+ """
+ Check that default is correctly set.
+ """
+ self.assertEqual(self.floatfield.default, 1.0)
+
+ def test_coerce(self):
+ """
+ Check that a string is correctly coerced to an float.
+ """
+ self.assertEqual(self.floatfield.coerce("1"), 1.0)
+
+class FileFieldTest(unittest.TestCase):
+ """
+ Tests for :class:`lofarpipe.support.lofaringredient.FileField`
+ """
+ def setUp(self):
+ from lofarpipe.support.lofaringredient import FileField
+ self.filefield = FileField(default='/')
+
+ def test_validator(self):
+ """
+ Integers are not valid as filenames, and certainly don't exist on
+ disk.
+
+ ``/`` should, though.
+ """
+ self.assertFalse(self.filefield.is_valid(1))
+ self.assertTrue(self.filefield.is_valid("/"))
+
+ def test_default(self):
+ """
+ Check that default is correctly set.
+ """
+ self.assertEqual(self.filefield.default, "/")
+
+class ExecFieldTest(unittest.TestCase):
+ """
+ Tests for :class:`lofarpipe.support.lofaringredient.ExecField`
+ """
+ def setUp(self):
+ from lofarpipe.support.lofaringredient import ExecField
+ self.execfield = ExecField(default='/bin/ls')
+
+ def test_validator(self):
+ """
+ ``/etc/passwd`` should always exist as a file on disk, but not be
+ executable.
+
+ ``/bin/ls`` should always exist, and must be executable.
+ """
+ self.assertFalse(self.execfield.is_valid("/etc/passwd"))
+ self.assertTrue(self.execfield.is_valid("/bin/ls"))
+
+ def test_default(self):
+ """
+ Check that default is correctly set.
+ """
+ self.assertEqual(self.execfield.default, "/bin/ls")
+
+class DirectoryFieldTest(unittest.TestCase):
+ """
+ Tests for :class:`lofarpipe.support.lofaringredient.DirectoryField`
+ """
+ def setUp(self):
+ from lofarpipe.support.lofaringredient import DirectoryField
+ self.directoryfield = DirectoryField(default='/tmp')
+
+ def test_validator(self):
+ """
+ An integer is not a valid directory.
+
+ ``/tmp`` should always be valid.
+ """
+ self.assertFalse(self.directoryfield.is_valid(1))
+ self.assertTrue(self.directoryfield.is_valid("/tmp"))
+
+ def test_default(self):
+ """
+ Check that default is correctly set.
+ """
+ self.assertEqual(self.directoryfield.default, "/tmp")
+
+ def test_coerce(self):
+ """
+ Coercing a create-able directory name should cause it to exist. We
+ should always be able to write in ``/tmp``.
+ """
+ self.directoryfield.coerce("/tmp/foo")
+ self.assertTrue(os.path.exists("/tmp/foo"))
+
+class LOFARIngredientTest(unittest.TestCase):
+ """
+ Tests for :class:`lofarpipe.support.lofaringredient.LOFARingredient`
+ """
+ def setUp(self):
+ """
+ An instance of
+ :class:`~lofarpipe.support.lofaringredient.LOFARingredient` is defined
+ which contains three instances of
+ :class:`~lofarpipe.support.lofaringredient.StringField`.
+ """
+ from lofarpipe.support.lofaringredient import StringField
+ from lofarpipe.support.lofaringredient import LOFARingredient
+ f = StringField(default="foo")
+ g = StringField()
+ h = StringField(default=1)
+ self.lofaringredient = LOFARingredient({"f": f, "g": g, "h": h})
+
+ def test_keys(self):
+ """
+ ``self.lofaringredient`` should contain keys for the two fields
+ which have default parameters, but not for the one which is unset.
+ """
+ self.assertEqual(len(self.lofaringredient.keys()), 2)
+ self.assertRaises(KeyError, lambda: self.lofaringredient['g'])
+
+ def test_values(self):
+ """
+ Prior to setting, the value of the fields should be equal to
+ the default value.
+ """
+ self.assertEqual(self.lofaringredient['f'], "foo")
+
+ def test_set(self):
+ """
+ When set, the value of the fields should be equal to the new value.
+ """
+ self.lofaringredient['g'] = "bar"
+ self.assertEqual(self.lofaringredient['g'], "bar")
+
+ def test_bad_values(self):
+ """
+ Unacceptable values should raise an exception.
+ """
+ self.assertRaises(TypeError, lambda: self.lofaringredient['h'])
+ self.lofaringredient['h'] = "bar"
+ self.assertEqual(self.lofaringredient['h'], "bar")
+
+if __name__ == "__main__":
+ import xmlrunner
+ unittest.main(testRunner=xmlrunner.XMLTestRunner(output='result.xml'))
diff --git a/CEP/Pipeline/test/pipeline_test.run b/CEP/Pipeline/test/pipeline_test.run
new file mode 100755
index 0000000000000000000000000000000000000000..7afad961b81abe0b40919b7641574134888235ff
--- /dev/null
+++ b/CEP/Pipeline/test/pipeline_test.run
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+#add correct path for lofarPipe python files
+#**WORKAROUND** for incorrect path setting in testframework code
+# For standalone unittest the path should only contain the code to be tested and
+# the testframework containing fake externals
+
+#run the actual python script performing the tests
+# Run all test collected in the current directory (including possible self tests!)
+
+PYTHONPATH=${srcdir}/test_framework/fixture:${PYTHONPATH}
+export PYTHONPATH
+
+python ${srcdir}/test_framework/unittest_runner.py -x results.xml -p ${srcdir}
diff --git a/CEP/Pipeline/test/pipeline_test.sh b/CEP/Pipeline/test/pipeline_test.sh
new file mode 100755
index 0000000000000000000000000000000000000000..861679c24a454f27073f4fe17be0697dce9ce56e
--- /dev/null
+++ b/CEP/Pipeline/test/pipeline_test.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh pipeline_test > pipeline_test.log 2>&1
diff --git a/CEP/Pipeline/test/recipes/__init__.py b/CEP/Pipeline/test/recipes/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/recipes/master/__init__.py b/CEP/Pipeline/test/recipes/master/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/recipes/nodes/__init__.py b/CEP/Pipeline/test/recipes/nodes/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py b/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..1982c3f5483a97a48d31873db09b3c33c58dad99
--- /dev/null
+++ b/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
@@ -0,0 +1,369 @@
+from __future__ import with_statement
+import os
+import errno
+import unittest
+import shutil
+import numpy
+import tempfile
+
+
+import pyrap.tables as tb #@UnresolvedImport
+from lofarpipe.support.utilities import create_directory #@UnresolvedImport
+from lofarpipe.recipes.nodes.imager_create_dbs import imager_create_dbs #@UnresolvedImport
+from logger import logger
+
+
+class ImagerCreateDBsTestWrapper(imager_create_dbs):
+ """
+ Wrapper for the imager_create_dbs allows overwriting of
+ """
+ def __init__(self):
+ """
+ Overloaded __init__ function, hiding the original __init__ on
+ LOFARnodeTCP.
+ """
+ self.logger = logger()
+
+class ImagerCreateDBsTest(unittest.TestCase):
+ """
+ Tests for ImagerCreateDBs class
+ """
+ test_path = "/data/scratch/python_unittest"
+
+ def __init__(self, arg): #todo deze moet toch in de setUp
+ super(ImagerCreateDBsTest, self).__init__(arg)
+
+ def setUp(self):
+ self.imager_create_dbs = ImagerCreateDBsTestWrapper()
+ create_directory(self.test_path)
+
+ def tearDown(self):
+ shutil.rmtree(self.test_path)
+
+
+ def test_field_of_view_HBA_120_CS(self):
+ """
+ Test the calcultaion of the FOV for lowest freq on a hba core station
+ """
+ variable_dictionary = {'NAME':["CS--HBA--"],
+ 'REF_FREQUENCY':["120E6"]}
+ tb.table.variable_dictionary = variable_dictionary
+ fov = self.imager_create_dbs._field_of_view("MS_name")
+ self.assertAlmostEqual(fov, 3.02, 2, "Incorrect FOV Value")
+
+
+ def test_field_of_view_HBA_240_RS(self):
+ """
+ Test the calcultaion of the FOV for lowest freq on a hba core station
+ """
+ variable_dictionary = {'NAME':["RS--HBA--"],
+ 'REF_FREQUENCY':["240E6"]}
+ tb.table.variable_dictionary = variable_dictionary
+ fov = self.imager_create_dbs._field_of_view("MS_name")
+ self.assertAlmostEqual(fov, 1.13, 2, "Incorrect FOV Value")
+
+ def test_field_of_view_LBA_15_INNER(self):
+ """
+ Test the calcultaion of the FOV for lowest freq on a hba core station
+ """
+ variable_dictionary = {'NAME':["--LBA--"],
+ 'REF_FREQUENCY':["15E6"],
+ 'LOFAR_ANTENNA_SET':["--INNER--"]}
+ tb.table.variable_dictionary = variable_dictionary
+ fov = self.imager_create_dbs._field_of_view("MS_name")
+ self.assertAlmostEqual(fov, 23.04, 2, "Incorrect FOV Value")
+
+
+ def test_field_of_view_LBA_75_OUTER(self):
+ """
+ Test the calcultaion of the FOV for lowest freq on a hba core station
+ """
+ variable_dictionary = {'NAME':["--LBA--"],
+ 'REF_FREQUENCY':["75E6"],
+ 'LOFAR_ANTENNA_SET':["--OUTER--"]}
+ tb.table.variable_dictionary = variable_dictionary
+ fov = self.imager_create_dbs._field_of_view("MS_name")
+ self.assertAlmostEqual(fov, 1.83, 2, "Incorrect FOV Value")
+
+
+ def test_field_of_view_incorrect_antenna_name(self):
+ """
+ When the measurement set is from an antenna with a name NOT
+ containing either LBA or HBA en Exception should be trown
+ """
+ variable_dictionary = {'NAME':["--error--"]}
+ tb.table.variable_dictionary = variable_dictionary
+ self.assertRaises(Exception, self.imager_create_dbs._field_of_view, "MS_name")
+
+ def test__create_parmdb(self):
+ """
+ Test the correct functioning of the create parmdbs function
+ 1. test if dir is created
+ 2. test if dir contains files (content tests omitted: thats a parmdbs
+ unit test.
+ 3. correct return value
+
+ """
+ path_to_create = os.path.join(self.test_path, "testParmdb")
+ create_directory(path_to_create)
+
+ parmdb_output = os.path.join(path_to_create, "parmdbs")
+ parmdb_executable = "/opt/cep/LofIm/daily/lofar/bin/parmdbm" #TODO: static
+ self.assertTrue(0 == self.imager_create_dbs._create_parmdb(parmdb_executable,
+ parmdb_output),
+ self.imager_create_dbs.logger._log[-1])
+
+ self.assertTrue(os.path.exists(parmdb_output), "targer dir to be"
+ "created by parmdb does not exist")
+ table_data_file_path = os.path.join(parmdb_output, "table.dat")
+ self.assertTrue(os.path.exists(table_data_file_path),
+ "Creation of table.dat failed")
+
+
+ shutil.rmtree(path_to_create)
+
+ def test__create_parmdb_missing_exec(self):
+ """
+ Test the correct functioning of the create parmdbs function
+
+ """
+ path_to_create = os.path.join(self.test_path, "testParmdb")
+ create_directory(path_to_create)
+
+ parmdb_output = os.path.join(path_to_create, "parmdbs")
+ parmdb_executable = "/opt/cep/LofIm/daily/lofar/bin/incorrectExecutable"
+ self.assertTrue(1 == self.imager_create_dbs._create_parmdb(parmdb_executable,
+ parmdb_output),
+ self.imager_create_dbs.logger.last())
+
+
+ self.assertFalse(os.path.exists(parmdb_output), "target dir to be"
+ "created by parmdb does exist, while it should not")
+
+ shutil.rmtree(path_to_create)
+
+ def test__create_parmdb_for_timeslices(self):
+ """
+ Test the correct functioning of the _create_parmdb_for_timeslices
+ Creating paramdbs for multiple measurement sets
+ """
+ path_to_create = os.path.join(self.test_path, "testParmdb")
+ parmdb_ms_output = os.path.join(path_to_create, "parmdbs")
+ create_directory(parmdb_ms_output)
+ parmdb_executable = "/opt/cep/LofIm/daily/lofar/bin/parmdbm"
+
+ #Create a number of paths to supply to the create function
+ ms_paths = []
+ for idx in range(5):
+ ms_paths.append(os.path.join(parmdb_ms_output, str(idx)))
+
+
+ #test output
+ self.assertTrue(
+ 0 == self.imager_create_dbs._create_parmdb_for_timeslices(parmdb_executable,
+ ms_paths, ".parmdb"),
+ self.imager_create_dbs.logger.last())
+
+ #test creation of parmdb
+ final_ms_path = os.path.join(parmdb_ms_output, "4.parmdb")
+ self.assertTrue(os.path.exists(final_ms_path))
+ final_ms_table = os.path.join(final_ms_path, "table.dat")
+ self.assertTrue(os.path.exists(final_ms_table))
+
+ def test__create_parmdb_for_timeslices_except(self):
+ """
+ Test the errorous functioning of the _create_parmdb_for_timeslices
+ with missing executable should return 1 and no created directories
+ """
+ path_to_create = os.path.join(self.test_path, "testParmdb")
+ parmdb_ms_output = os.path.join(path_to_create, "parmdbs")
+ create_directory(parmdb_ms_output)
+ parmdb_executable = "/opt/cep/LofIm/daily/lofar/bin/missingExcecutable"
+
+ #Create a number of paths to supply to the create function
+ ms_paths = []
+ for idx in range(5):
+ ms_paths.append(os.path.join(parmdb_ms_output, str(idx)))
+
+
+ self.assertTrue(
+ 1 == self.imager_create_dbs._create_parmdb_for_timeslices(parmdb_executable,
+ ms_paths, ".parmdb"),
+ self.imager_create_dbs.logger.last())
+ final_ms_path = os.path.join(parmdb_ms_output, "time_slice_8.dppp.ms.parmdb")
+ self.assertFalse(os.path.exists(final_ms_path))
+
+ def test__create_monet_db_connection(self):
+ """
+ Tests the correct creation of a monetdb connection
+ Monat db is mucked!! and returns: "connection"
+ """
+ db_host = "hostname"
+ db_dbase = "spam"
+ db_user = "spam"
+ db_passwd = "spam"
+ db_port = 1
+ self.assertTrue("connection" ==
+ self.imager_create_dbs._create_monet_db_connection(db_host,
+ db_dbase, db_user, db_passwd, db_port),
+ "_create_monat_db_connection() did not return the"
+ " string 'connection'")
+
+
+ def test__create_monet_db_connection_fail(self):
+ """
+ Tests the monetdb connection that failse: Altough internally db.Error
+ exceptions could be trown these should be caught the function return None
+ When no conenction could be established
+
+ """
+ db_host = "except"
+ db_dbase = "spam"
+ db_user = "spam"
+ db_passwd = "spam"
+ db_port = 1
+ self.assertTrue(None ==
+ self.imager_create_dbs._create_monet_db_connection(db_host,
+ db_dbase, db_user, db_passwd, db_port),
+ "_create_monat_db_connection() did not return the"
+ " string 'connection'")
+
+
+ def test__get_ra_and_decl_from_ms(self):
+ """
+ Test the extraction of the beam direction from the measurement set
+ 1. insert temp values is muck db
+ 2. call extract function
+ """
+ ra = 123
+ decl = 456
+ variable_dictionary = {'PHASE_DIR':[numpy.array([ra, decl])]}
+ tb.table.variable_dictionary = variable_dictionary
+
+ ret_ra, ret_decl = \
+ self.imager_create_dbs._get_ra_and_decl_from_ms("measurementset")
+
+ self.assertTrue((ra == ret_ra) and (decl == ret_decl) ,
+ "_get_ra_and_decl_from_ms dir not return the expected"
+ " values for the re and decl")
+
+
+ def test__get_ra_and_decl_from_ms_pyrap_raised_except(self):
+ """
+ Test correct raising of exceptions
+ """
+ error_message = "test__get_ra_and_decl_from_ms_pyrap_raised_except"
+ variable_dictionary = {'FIELD': error_message}
+ tb.table.variable_dictionary = variable_dictionary
+
+ self.assertTrue(
+ None == self.imager_create_dbs._get_ra_and_decl_from_ms('except'),
+ "_get_ra_and_decl_from_ms should return None when exception"
+ "thrown in pyrap")
+ self.assertTrue(self.imager_create_dbs.logger.last()[1].count(error_message) > 0,
+ "The last logged message is incorrect")
+
+
+ def test__get_ra_and_decl_from_ms_pyrap_incorrect_data(self):
+ """
+ Test correct return value on non correct values (but none exceptionaly)
+ """
+ error_message = "returned PHASE_DIR data did not contain two values"
+ variable_dictionary = {'PHASE_DIR': [numpy.array([1])]}
+ tb.table.variable_dictionary = variable_dictionary
+
+ self.assertTrue(
+ None == self.imager_create_dbs._get_ra_and_decl_from_ms('ms'),
+ "_get_ra_and_decl_from_ms should return None when retreived"
+ "data has not 2 entries")
+ self.assertTrue(self.imager_create_dbs.logger.last()[1].count(error_message) > 0,
+ "The last logged message is incorrect")
+
+
+ def test__create_bbs_sky_model_no_theta(self):
+ """
+ Test correct functioning of _create_bbs_sky_model.
+ The inner workings of nested funtions is not tested
+ """
+
+ # create the muck db with location
+ ra = 123
+ decl = 456
+
+ variable_dictionary = {'NAME':["--LBA--"],
+ 'REF_FREQUENCY':["75E6"],
+ 'LOFAR_ANTENNA_SET':["--OUTER--"],
+ 'PHASE_DIR':[numpy.array([ra, decl])]}
+ tb.table.variable_dictionary = variable_dictionary
+
+ #Create temp location to save the output!!
+ tempdir = tempfile.mkdtemp()
+ output_skymodel_name = "bbs.skymodel.test"
+ test_skymodel_path = os.path.join(tempdir, output_skymodel_name)
+
+
+ #test correct return value
+ self.assertTrue(0 == self.imager_create_dbs._create_bbs_sky_model(
+ "measurement_set", test_skymodel_path, "host", "db_port", "db_name",
+ "db_user", "db_password"))
+
+ #assert creation of output file
+ self.assertTrue(os.path.exists(test_skymodel_path))
+
+ #assert correct creation of theta
+ fp = open(test_skymodel_path)
+ theta = fp.readline()
+ self.assertTrue(theta == "0.025\n")
+ #clean up the created file and dir
+ try:
+ os.remove(test_skymodel_path)
+ except:
+ pass
+
+ os.rmdir(tempdir)
+
+ def test__create_bbs_sky_model_theta(self):
+ """
+ Test correct functioning of _create_bbs_sky_model.
+ The inner workings of nested funtions is not tested
+ """
+ theta = "20"
+
+ # create the muck db with location
+ ra = 123
+ decl = 456
+ variable_dictionary = {'NAME':["--LBA--"],
+ 'REF_FREQUENCY':["75E6"],
+ 'LOFAR_ANTENNA_SET':["--OUTER--"],
+ 'PHASE_DIR':[numpy.array([ra, decl])]}
+ tb.table.variable_dictionary = variable_dictionary
+
+ #Create temp location to save the output!!
+ tempdir = tempfile.mkdtemp()
+ output_skymodel_name = "bbs.skymodel.test"
+ test_skymodel_path = os.path.join(tempdir, output_skymodel_name)
+
+
+ #test correct return value
+ self.assertTrue(0 == self.imager_create_dbs._create_bbs_sky_model(
+ "measurement_set", test_skymodel_path, "host", "db_port",
+ "db_name", "db_user", "db_password", theta))
+
+ #assert creation of output file
+ self.assertTrue(os.path.exists(test_skymodel_path))
+
+ #assert correct creation of theta
+ fp = open(test_skymodel_path)
+ theta_red = fp.readline()
+ self.assertTrue(theta_red == theta + "\n")
+ #clean up the created file and dir
+ try:
+ os.remove(test_skymodel_path)
+ except:
+ pass
+
+ os.rmdir(tempdir)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/CEP/Pipeline/test/support/__init__.py b/CEP/Pipeline/test/support/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/test_framework/CMakeLists.txt b/CEP/Pipeline/test/test_framework/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a141ed21c5de46216c274f91623faad520e0ba88
--- /dev/null
+++ b/CEP/Pipeline/test/test_framework/CMakeLists.txt
@@ -0,0 +1,3 @@
+# $Id: CMakeLists.txt 18904 2011-09-28 12:15:43Z diepen $
+
+
diff --git a/CEP/Pipeline/test/test_framework/__init__.py b/CEP/Pipeline/test/test_framework/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/test_framework/fixture/__init__.py b/CEP/Pipeline/test/test_framework/fixture/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/test_framework/fixture/gsmutils.py b/CEP/Pipeline/test/test_framework/fixture/gsmutils.py
new file mode 100644
index 0000000000000000000000000000000000000000..4df8c7d64bad64d9c7f76a9fd06c3f7b7f85f62b
--- /dev/null
+++ b/CEP/Pipeline/test/test_framework/fixture/gsmutils.py
@@ -0,0 +1,11 @@
+
+def expected_fluxes_in_fov(conn, ra_c, decl_c, fov_radius, assoc_theta,
+ output_path, storespectraplots = False):
+ """
+ Muck skymodel creation: Only creates the file on output_path.
+ The file is filled with the supplied assoc_theta
+ """
+ fp = open(output_path, "w")
+ fp.write(str(assoc_theta))
+ fp.write("\nbbs.skymodel.test")
+ fp.close()
diff --git a/CEP/Pipeline/test/test_framework/fixture/logger.py b/CEP/Pipeline/test/test_framework/fixture/logger.py
new file mode 100644
index 0000000000000000000000000000000000000000..851b77d8cabae2e437d6b483eb85d1de4c34e46d
--- /dev/null
+++ b/CEP/Pipeline/test/test_framework/fixture/logger.py
@@ -0,0 +1,28 @@
+#test fixture for pipeline framework
+
+class logger():
+ """
+ muck logger object, allows logging of info, debug and error function
+ new entries are appended to a list with the type of logger error and
+ the suplied input
+ """
+ def __init__(self):
+ self._log = []
+
+ def info(self, input_string):
+ self._log.append(("info", input_string))
+
+ def debug(self, input_string):
+ self._log.append(("debug", input_string))
+
+ def error(self, input_string):
+ self._log.append(("error", input_string))
+
+ def warn(self, input_string):
+ self._log.append(("warn", input_string))
+
+ def last(self):
+ """
+ return that last error
+ """
+ return self._log[-1]
diff --git a/CEP/Pipeline/test/test_framework/fixture/monetdb/__init__.py b/CEP/Pipeline/test/test_framework/fixture/monetdb/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/test_framework/fixture/monetdb/sql.py b/CEP/Pipeline/test/test_framework/fixture/monetdb/sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0f847c465bba0a04e2e29c712b675407aba477c
--- /dev/null
+++ b/CEP/Pipeline/test/test_framework/fixture/monetdb/sql.py
@@ -0,0 +1,20 @@
+class Error(Exception):
+ """
+ Exception returned by sql.connect()
+ """
+ def __str__(self):
+ return "Error"
+
+def connect(hostname, database, username, password, port):
+ """
+ Muck sql.connect: returns the string connection upon a call of connect with
+ an int as port and the string 'hostname' as hostname.
+ Will return an Error of the type Exception when the hostname is 'except'.
+ Otherwise the function return the string 'connection Failed'
+ """
+ if hostname == "hostname" and isinstance(port, int):
+ return "connection"
+ if hostname == "except":
+ raise Error()
+
+ return "connection Failed"
diff --git a/CEP/Pipeline/test/test_framework/fixture/pyrap/__init__.py b/CEP/Pipeline/test/test_framework/fixture/pyrap/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/CEP/Pipeline/test/test_framework/fixture/pyrap/tables.py b/CEP/Pipeline/test/test_framework/fixture/pyrap/tables.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a9d4337eeec4d11985c641b69b3a9b2c8d3b99f
--- /dev/null
+++ b/CEP/Pipeline/test/test_framework/fixture/pyrap/tables.py
@@ -0,0 +1,68 @@
+from numpy.ma import extras
+
+class table():
+ """
+ Muck table object.
+ A minimal implementation of the pyrapl.table object:
+ It allows getting of keywords and cells from a previously initialed static
+ dictionary of keyword -> value pairs. Currently a global dictionary is used
+
+ """
+ # TODO: global dict prevent parallen running of unit tests (fragile code smell)
+ variable_dictionary = {}
+
+ def __init__(self, measurement_set):
+ """
+ Initialize the table, give it a name. If the name is except all calls
+ to getters will return an exception
+ """
+ if measurement_set == 'except':
+ self.exception = True
+ else:
+ self.exception = False
+ self.measurement_set = measurement_set
+
+ def getkeyword(self, keyword, default = None):
+ """
+ Returns the value found in the variable dict.
+ Returns exception if set
+ """
+ if self.exception:
+ raise Exception(table.variable_dictionary[keyword])
+ if table.variable_dictionary.has_key(keyword):
+ return table.variable_dictionary[keyword]
+
+ return default
+
+ def getcell(self, keyword, idx, default = None):
+ """
+ Returns the value found in the variable dict
+ Returns exception if set
+ """
+ if self.exception:
+ raise Exception(table.variable_dictionary[keyword])
+ if table.variable_dictionary.has_key(keyword):
+ cell_data = table.variable_dictionary[keyword]
+ if idx < len(cell_data):
+ return cell_data[idx]
+
+ return default
+
+ def set_variable_dictionary(self, variable_dictionary):
+ """
+ Sets the class member variable_dictionary.
+ """
+ table.variable_dictionary = variable_dictionary
+
+ def close(self):
+ return
+
+def taql(input_string):
+ """
+ Return taql commands:
+ 1. 'CALC C()' -> return c (float)
+ """
+ if input_string.strip() == 'CALC C()':
+ return 299792458.0
+ raise Exception("command not know in this muck implementation!")
+
diff --git a/CEP/Pipeline/test/test_framework/fixture/read.me b/CEP/Pipeline/test/test_framework/fixture/read.me
new file mode 100644
index 0000000000000000000000000000000000000000..f9804a32cb9d7b0aa8f3f034f11118b20f8b7090
--- /dev/null
+++ b/CEP/Pipeline/test/test_framework/fixture/read.me
@@ -0,0 +1,7 @@
+The files in this directory are MUCK implementations:
+They mimic limited framework functionality in a reproducible manner allowing for unit testing.
+
+The file are NOT functional.
+
+2012
+klijn@astron.nl
\ No newline at end of file
diff --git a/CEP/Pipeline/test/test_framework/unittest_runner.py b/CEP/Pipeline/test/test_framework/unittest_runner.py
new file mode 100644
index 0000000000000000000000000000000000000000..2abbf275e0d36b3ef8b7968518a582624c701d45
--- /dev/null
+++ b/CEP/Pipeline/test/test_framework/unittest_runner.py
@@ -0,0 +1,167 @@
+import os
+import unittest
+import getopt
+import string
+import inspect
+import sys
+import re
+
+class Discover():
+ """
+ Discover class collects all unit test case in <path> and recursive directories
+ Collects them in a single large suite.
+ Start at supplied <path> an add all tests in files matching the supplied expression and
+ all individual tests matching the expression
+ """
+ #TODO: ordering of suites is not controlled atm.
+ #TODO: maybe it should be expanded with multi path input??
+ suite = unittest.TestSuite()
+
+ def __init__(self,path,pattern):
+
+ #match with all (used for a filename matches with expression: all individual test must be loaded
+ allMatcher = re.compile(".*")
+ #matcher for the expression
+ patternMatcher = re.compile(pattern)
+ #matcher for hidden dirs
+ hiddenMatcher = re.compile(".*/\..*")
+
+ for root, dirs, files in os.walk(path):
+ #skip hidden directories
+ if hiddenMatcher.match(root):
+ continue
+
+ dirSuite = unittest.TestSuite()
+ for name in files:
+ fileNameParts = name.split('.')
+ #assert correct file extention
+ if (len(fileNameParts) == 1) or (fileNameParts[1] != 'py'):
+ continue
+
+ #try loading as a module
+ try:
+ module = self.import_path(root, fileNameParts[0]) #use import including path
+ except BaseException:
+ continue
+
+ #the expression mechanism
+ testMatcher = None
+ if patternMatcher.match(name):
+ testMatcher = allMatcher #if current dir matches with expression include all tests
+ else:
+ testMatcher = patternMatcher
+ #create a test suite
+ fileSuite = unittest.TestSuite()
+ testnames = dir(module)
+
+ #add all cases ending with test and match the regexp search string
+ for testName in testnames:
+ if testName.endswith('Test') or testName.endswith('test'):
+ try:
+ testClass = getattr(module, testName) #load attribute
+ if inspect.isclass(testClass): #if class
+ if not testMatcher.match(testName): #Continue of current testname does not match supplied expression
+ continue
+ fileSuite.addTest(unittest.makeSuite(testClass))
+ except:
+ pass
+
+ #if tests found add the file suite to the directory suite
+ if fileSuite.countTestCases() != 0:
+ dirSuite.addTest(fileSuite)
+
+ #add to top level suite
+ if dirSuite.countTestCases() != 0:
+ self.suite.addTest(dirSuite)
+
+
+ def import_path(self, path, filename):
+ """
+ Import a file with full path specification. Allows one to
+ import from anywhere, something __import__ does not do.
+ """
+ filename, ext = os.path.splitext(filename)
+ sys.path.append(path)
+ module = __import__(filename)
+ reload(module) # Might be out of date
+ del sys.path[-1]
+ return module
+
+
+class UnitTesterTest(unittest.TestCase):
+ """
+ Self test for the UnitTester
+ """
+ #TODO: Add propper test suite, creating come files and try laoding it (multiple directories and depths)
+ def setUp(self):
+ self.tester = "A test string"
+
+ def test_validator(self):
+ """
+ Check that current testClass is loaded and performed
+ """
+ self.assertTrue(self.tester == "A test string")
+
+
+def usage():
+ """
+ Display a short overview of available arguments
+ """
+ usage = r"""Usage: python UnitTester [-p <path = '.'> -e <expression = *> -h -x]
+ Recursively look in path for unit test classes matching expression.
+ Collect in a single suite and run them
+ -p, --path <path> to start looking. Default is '.'
+ -e, --exp <expresion> to match with found classes to perform a subset of the tests
+ or
+ -m, --matchword match with found classes to perform a subset of tests (shorthand for .*arg.* expression
+ -h, --help Display this usage
+ -x, --xml <filename> Export resuls to xml (results are overwritten)
+ """
+ print usage
+
+if __name__ == "__main__":
+
+ #Default parameters settings
+ path = '.'
+ expression = '.*'
+ xml = ""
+
+ #parse command lines and set parameters for Discover function
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], "p:e:hx:m:", ["path=", "exp=", "help", "xml=","matchword="])
+ except getopt.GetoptError:
+ usage()
+ sys.exit(2)
+ for opt, arg in opts:
+ if opt in ("-h", "--help"):
+ usage()
+ sys.exit()
+ elif opt in ( "-p", "--path"):
+ path = arg
+ elif opt in ("-e", "--exp"):
+ expression = arg
+ elif opt in ("-x", "--xml"):
+ xml = arg
+ elif opt in ("-m","--matchword"):
+ expression = ".*{0}.*".format(arg)
+
+ #Collect tests from files and paths
+ test = Discover(path, expression)
+
+ #decide on unit testrunner to use, run it and save the results
+ if xml:
+ import xmlrunner
+ result = xmlrunner.XMLTestRunner(output=xml).run(test.suite)
+ else:
+ result = unittest.TextTestRunner(verbosity=2).run(test.suite)
+
+ #collect the numeric results using expressions
+ FailedTestMatcher = re.compile(".*run=(\d+).*errors=(\d+).*failures=(\d+)")
+ matches = FailedTestMatcher.match(str(result))
+ runErrorFailures = matches.groups(0)
+
+ #add to get the total number of not succesfull tests
+ failingTests = int(runErrorFailures[1]) + int(runErrorFailures[2])
+
+ #provide number of failing tests as exit value
+ sys.exit(failingTests)