diff --git a/.gitattributes b/.gitattributes
index 4e01e398cc3e9364bba2d7c7362028800f5858b2..c422d346b03a37aa3563266cc19ea420cfa13d6d 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1679,16 +1679,20 @@ LCS/Messaging/test/tTimeOut.cc -text
LCS/PyCommon/CMakeLists.txt -text
LCS/PyCommon/__init__.py -text
LCS/PyCommon/cache.py -text
+LCS/PyCommon/cep4_utils.py -text
LCS/PyCommon/datetimeutils.py -text
LCS/PyCommon/defaultmailaddresses.py -text
LCS/PyCommon/factory.py -text
LCS/PyCommon/flask_utils.py -text
LCS/PyCommon/math.py -text
LCS/PyCommon/postgres.py -text
-LCS/PyCommon/subprocess.py -text
+LCS/PyCommon/subprocess_utils.py -text
LCS/PyCommon/test/python-coverage.sh eol=lf
LCS/PyCommon/test/t_cache.py -text
LCS/PyCommon/test/t_cache.sh -text
+LCS/PyCommon/test/t_cep4_utils.py -text
+LCS/PyCommon/test/t_cep4_utils.run -text
+LCS/PyCommon/test/t_cep4_utils.sh -text
LCS/PyCommon/test/t_dbcredentials.run eol=lf
LCS/PyCommon/test/t_dbcredentials.sh eol=lf
LCS/PyCommon/test/t_defaultmailaddresses.py -text
@@ -3938,15 +3942,11 @@ QA/QA_Common/bin/find_hdf5 -text
QA/QA_Common/bin/show_hdf5_info -text
QA/QA_Common/lib/CMakeLists.txt -text
QA/QA_Common/lib/__init__.py -text
-QA/QA_Common/lib/cep4_utils.py -text
QA/QA_Common/lib/geoconversions.py -text
QA/QA_Common/lib/hdf5_io.py -text
QA/QA_Common/lib/utils.py -text
QA/QA_Common/test/CMakeLists.txt -text
QA/QA_Common/test/create_test_hypercube -text
-QA/QA_Common/test/t_cep4_utils.py -text
-QA/QA_Common/test/t_cep4_utils.run -text
-QA/QA_Common/test/t_cep4_utils.sh -text
QA/QA_Common/test/t_hdf5_io.py -text
QA/QA_Common/test/t_hdf5_io.run -text
QA/QA_Common/test/t_hdf5_io.sh -text
diff --git a/LCS/PyCommon/CMakeLists.txt b/LCS/PyCommon/CMakeLists.txt
index 5d6cdb28631646683d940fdf23bfa56ec986dedf..8a8e55d488fe805f469f702e948e6b19038074e9 100644
--- a/LCS/PyCommon/CMakeLists.txt
+++ b/LCS/PyCommon/CMakeLists.txt
@@ -10,6 +10,7 @@ find_python_module(psycopg2)
set(_py_files
__init__.py
+ cep4_utils.py
cache.py
dbcredentials.py
defaultmailaddresses.py
diff --git a/QA/QA_Common/lib/cep4_utils.py b/LCS/PyCommon/cep4_utils.py
similarity index 100%
rename from QA/QA_Common/lib/cep4_utils.py
rename to LCS/PyCommon/cep4_utils.py
diff --git a/LCS/PyCommon/subprocess.py b/LCS/PyCommon/subprocess_utils.py
similarity index 100%
rename from LCS/PyCommon/subprocess.py
rename to LCS/PyCommon/subprocess_utils.py
diff --git a/LCS/PyCommon/test/CMakeLists.txt b/LCS/PyCommon/test/CMakeLists.txt
index 48f4f4d47f53903b1d6b735e9f44f2c969b5ccb4..6309f036769b6cb3b76377b36c9c9df46fdf24e1 100644
--- a/LCS/PyCommon/test/CMakeLists.txt
+++ b/LCS/PyCommon/test/CMakeLists.txt
@@ -12,3 +12,4 @@ lofar_add_test(t_defaultmailaddresses)
lofar_add_test(t_methodtrigger)
lofar_add_test(t_util)
lofar_add_test(t_test_utils)
+lofar_add_test(t_cep4_utils)
diff --git a/QA/QA_Common/test/t_cep4_utils.py b/LCS/PyCommon/test/t_cep4_utils.py
similarity index 99%
rename from QA/QA_Common/test/t_cep4_utils.py
rename to LCS/PyCommon/test/t_cep4_utils.py
index 8aa8f90fa73dbb69fde98a7c168a93b7db89ea04..12ad67dce79cfb5054c1d7cc42e6d0def0a45b80 100755
--- a/QA/QA_Common/test/t_cep4_utils.py
+++ b/LCS/PyCommon/test/t_cep4_utils.py
@@ -21,7 +21,7 @@ import unittest
from subprocess import call
import logging
-from lofar.qa.cep4_utils import *
+from lofar.common.cep4_utils import *
logger = logging.getLogger(__name__)
diff --git a/QA/QA_Common/test/t_cep4_utils.run b/LCS/PyCommon/test/t_cep4_utils.run
similarity index 94%
rename from QA/QA_Common/test/t_cep4_utils.run
rename to LCS/PyCommon/test/t_cep4_utils.run
index b8d03e94c099ffa993719a2665c2ee89cf7b027d..dbbadd78378910833774ca467f8a7008a126b7ee 100755
--- a/QA/QA_Common/test/t_cep4_utils.run
+++ b/LCS/PyCommon/test/t_cep4_utils.run
@@ -19,5 +19,5 @@
# Run the unit test
source python-coverage.sh
-python_coverage_test "*QA*" t_cep4_utils.py
+python_coverage_test "*cep4_utils*" t_cep4_utils.py
diff --git a/QA/QA_Common/test/t_cep4_utils.sh b/LCS/PyCommon/test/t_cep4_utils.sh
similarity index 100%
rename from QA/QA_Common/test/t_cep4_utils.sh
rename to LCS/PyCommon/test/t_cep4_utils.sh
diff --git a/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/lib/ltacp.py b/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/lib/ltacp.py
index 5c847da68f3fc9527b7d30563f246f4259c13a75..0c1090e918bbaf3ec9eef238fde45ff5e91e70e1 100755
--- a/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/lib/ltacp.py
+++ b/LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/lib/ltacp.py
@@ -20,7 +20,7 @@ import atexit
from datetime import datetime, timedelta
from lofar.common.util import humanreadablesize
from lofar.common.datetimeutils import totalSeconds
-from lofar.common.subprocess import PipeReader
+from lofar.common.subprocess_utils import PipeReader
from lofar.lta.ingest.common.config import hostnameToIp
from lofar.lta.ingest.server.config import GLOBUS_TIMEOUT
from lofar.lta.ingest.common.srm import *
diff --git a/QA/QA_Common/lib/CMakeLists.txt b/QA/QA_Common/lib/CMakeLists.txt
index 7ab0697539811de6e20f4e31a5724617640cf4f2..b72e082de94987bf0a4d19d95d5ed7490fc54040 100644
--- a/QA/QA_Common/lib/CMakeLists.txt
+++ b/QA/QA_Common/lib/CMakeLists.txt
@@ -19,7 +19,6 @@
python_install(
__init__.py
- cep4_utils.py
hdf5_io.py
geoconversions.py
utils.py
diff --git a/QA/QA_Common/test/CMakeLists.txt b/QA/QA_Common/test/CMakeLists.txt
index 82f42c7156d3787a4fbe8d4f165436ddacadaec3..805b871beb4b8d2d05ee2172e6e3029bb915b496 100644
--- a/QA/QA_Common/test/CMakeLists.txt
+++ b/QA/QA_Common/test/CMakeLists.txt
@@ -18,7 +18,6 @@
# $Id$
include(LofarCTest)
-lofar_add_test(t_cep4_utils)
lofar_add_test(t_hdf5_io)
diff --git a/QA/QA_Common/test/t_hdf5_io.py b/QA/QA_Common/test/t_hdf5_io.py
index 3d52ff2cd11aae6e37b80e81dbaaa43252913433..fcd97e8c1011eb7028a766412d81591e99570b3e 100755
--- a/QA/QA_Common/test/t_hdf5_io.py
+++ b/QA/QA_Common/test/t_hdf5_io.py
@@ -41,43 +41,55 @@ class TestHdf5_IO(unittest.TestCase):
import h5py
from multiprocessing import Process, Event
path = tempfile.mkstemp()[1]
- # make sure there is no test file in the way
- os.remove(path)
+ try:
+ # make sure there is no test file in the way
+ os.remove(path)
+ except OSError:
+ pass
try:
# for testing synchronization/concurrency
- event = Event()
+ event_created = Event()
+ event_closed = Event()
# define helper method to open h5 file in other process
def create_file_and_keep_open_for_a_while():
logger.info("creating h5 file %s", path)
with h5py.File(path, 'a') as file1:
+ event_created.set()
logger.info("created/openend h5 file %s, writing contents", path)
file1['foo'] = 'bar'
logger.info("keeping h5 file %s open for 2 seconds", path)
sleep(2)
logger.info("closed h5 file %s", path)
- event.set()
+ event_closed.set()
- # test if the event isn't set yet (as we did not start the other process yet...)
- self.assertFalse(event.is_set())
+ # test if the events aren't set yet (as we did not start the other process yet...)
+ self.assertFalse(event_created.is_set())
+ self.assertFalse(event_closed.is_set())
# start process, which opens and keeps open the h5 file...
proc = Process(target=create_file_and_keep_open_for_a_while)
proc.start()
+ # wait until file was created
+ event_created.wait(5)
+
# open SharedH5File (it should wait until create_file_and_keep_open_for_a_while is finished)
with SharedH5File(path, 'r+') as file2:
# Now that the SharedH5File is open, this tells us that the other process should be finished and the event is set.
- self.assertTrue(event.is_set())
+ self.assertTrue(event_closed.is_set())
# was foo=bar really written?
self.assertEqual('bar', file2['foo'].value)
# proc should already be finished, but let's wait anyway
proc.join()
finally:
- logger.info('removing test file: %s', path)
- os.remove(path)
+ try:
+ logger.info('removing test file: %s', path)
+ os.remove(path)
+ except OSError:
+ pass
def test_write_and_read_again(self):
'''tests writing and reading an hdf5 file, and checks all parameters except for the visibility data.
diff --git a/QA/QA_Service/CMakeLists.txt b/QA/QA_Service/CMakeLists.txt
index 686f55135647ed35cfea0e55514686a58d1f895e..3da6a2d95811cc7bc01fe5147727e1a6edf4d9c0 100644
--- a/QA/QA_Service/CMakeLists.txt
+++ b/QA/QA_Service/CMakeLists.txt
@@ -17,14 +17,7 @@
# $Id$
-IF(BUILD_TESTING)
- # add a dependency to InspectionPlots only if we do build testing.
- # the t_qa_service test depends on plot_hdf5_dynamic_spectra,
- # but the actual qa_service doesn't because all commands are called to cep4 docker images.
- lofar_package(QA_Service 1.0 DEPENDS QA_Common PyMessaging OTDB_Services pyparameterset InspectionPlots)
-ELSE()
- lofar_package(QA_Service 1.0 DEPENDS QA_Common PyMessaging OTDB_Services pyparameterset)
-ENDIF(BUILD_TESTING)
+lofar_package(QA_Service 1.0 DEPENDS QA_Common PyMessaging OTDB_Services pyparameterset)
add_subdirectory(lib)
add_subdirectory(bin)
diff --git a/QA/QA_Service/lib/qa_service.py b/QA/QA_Service/lib/qa_service.py
index 526db5eba321d31ad809b678590cad49dfb24011..2f1d064f52336996936617c5fb5039070e6ddeb4 100644
--- a/QA/QA_Service/lib/qa_service.py
+++ b/QA/QA_Service/lib/qa_service.py
@@ -29,7 +29,7 @@ from lofar.sas.otdb.config import DEFAULT_OTDB_NOTIFICATION_SUBJECT
from lofar.messaging.messagebus import ToBus
from lofar.messaging.messages import EventMessage
from lofar.qa.service.config import DEFAULT_QA_NOTIFICATION_BUSNAME, DEFAULT_QA_NOTIFICATION_SUBJECT_PREFIX
-from lofar.qa.cep4_utils import *
+from lofar.common.cep4_utils import *
logger = logging.getLogger(__name__)
diff --git a/QA/QA_Service/test/t_qa_service.py b/QA/QA_Service/test/t_qa_service.py
index c1cf167781f25a4e22f7140f23f7620ee9a235f4..c9e9a095885cb2520c24d6fba32e8c775c5ac187 100755
--- a/QA/QA_Service/test/t_qa_service.py
+++ b/QA/QA_Service/test/t_qa_service.py
@@ -126,7 +126,7 @@ class TestQAService(unittest.TestCase):
# we need to patch the wrap_command_in_cep4_head_node_ssh_call function from module lofar.qa.service.qa_service,
# because that's were it's imported and used.
- # (and not the original lofar.qa.cep4_utils.wrap_command_for_docker)
+ # (and not the original lofar.common.cep4_utils.wrap_command_for_docker)
wrap_command_for_docker_patcher = mock.patch('lofar.qa.service.qa_service.wrap_command_for_docker')
self.addCleanup(wrap_command_for_docker_patcher.stop)
self.wrap_command_for_docker_mock = wrap_command_for_docker_patcher.start()
@@ -137,12 +137,12 @@ class TestQAService(unittest.TestCase):
self.wrap_command_in_cep4_head_node_ssh_call_mock = wrap_command_in_cep4_head_node_ssh_call_patcher.start()
self.wrap_command_in_cep4_head_node_ssh_call_mock.side_effect = mocked_wrap_command_in_cep4_head_node_ssh_call
- wrap_command_in_cep4_cpu_node_ssh_call_patcher = mock.patch('lofar.qa.cep4_utils.wrap_command_in_cep4_cpu_node_ssh_call')
+ wrap_command_in_cep4_cpu_node_ssh_call_patcher = mock.patch('lofar.common.cep4_utils.wrap_command_in_cep4_cpu_node_ssh_call')
self.addCleanup(wrap_command_in_cep4_cpu_node_ssh_call_patcher.stop)
self.wrap_command_in_cep4_cpu_node_ssh_call_mock = wrap_command_in_cep4_cpu_node_ssh_call_patcher.start()
self.wrap_command_in_cep4_cpu_node_ssh_call_mock.side_effect = mocked_wrap_command_in_cep4_cpu_node_ssh_call
- get_cep4_available_cpu_nodes_patcher = mock.patch('lofar.qa.cep4_utils.get_cep4_available_cpu_nodes')
+ get_cep4_available_cpu_nodes_patcher = mock.patch('lofar.common.cep4_utils.get_cep4_available_cpu_nodes')
self.addCleanup(get_cep4_available_cpu_nodes_patcher.stop)
self.get_cep4_available_cpu_nodes_mock = get_cep4_available_cpu_nodes_patcher.start()
self.get_cep4_available_cpu_nodes_mock.side_effect = mocked_get_cep4_available_cpu_nodes
@@ -152,7 +152,7 @@ class TestQAService(unittest.TestCase):
# So, in principle it should not be needed to mock it,
# but when there is some error in the code/test/mock we would like to prevent
# an accidental ssh call to cep4
- ssh_cmd_list_patcher = mock.patch('lofar.qa.cep4_utils.ssh_cmd_list')
+ ssh_cmd_list_patcher = mock.patch('lofar.common.cep4_utils.ssh_cmd_list')
self.addCleanup(ssh_cmd_list_patcher.stop)
self.ssh_cmd_list_mock = ssh_cmd_list_patcher.start()