diff --git a/.gitattributes b/.gitattributes
index 22fdef416baa5eb45ab667231b40b8edb199cb62..d6f9072fcda479d0ff0720e28b0abea69c3b6ce3 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2239,6 +2239,7 @@ CMake/variants/variants.dop282 -text
CMake/variants/variants.fs0 -text
CMake/variants/variants.gpu01 -text
CMake/variants/variants.gpu1 -text
+CMake/variants/variants.lhn002 -text
CMake/variants/variants.node501 -text
CMake/variants/variants.node502 -text
CMake/variants/variants.node503 -text
@@ -2444,6 +2445,27 @@ LCS/MSLofar/test/tBeamTables.in_during_filled -text
LCS/MSLofar/test/tBeamTables.in_hd/CS001-iHBADeltas.conf -text
LCS/MSLofar/test/tBeamTables.in_hd/DE601-iHBADeltas.conf -text
LCS/MSLofar/test/tBeamTables.in_hd/RS106-iHBADeltas.conf -text
+LCS/MessageBus/data/feedback_dp_44833.txt -text
+LCS/MessageBus/data/feedback_dp_empty.txt -text
+LCS/MessageBus/data/feedback_dp_no_ids.txt -text
+LCS/MessageBus/data/feedback_dp_xml_error.txt -text
+LCS/MessageBus/data/task_spec_system_78958.txt -text
+LCS/MessageBus/data/task_spec_system_empty.txt -text
+LCS/MessageBus/data/task_spec_system_no_ids.txt -text
+LCS/MessageBus/data/task_spec_system_xml_error.txt -text
+LCS/MessageBus/include/MessageBus/Protocols/TaskSpecificationSystem.h -text
+LCS/MessageBus/src/LofarMsgTemplate.txt -text
+LCS/MessageBus/src/Message.cc -text
+LCS/MessageBus/src/__init__.py -text
+LCS/MessageBus/src/basemsgbus.py -text
+LCS/MessageBus/src/msgbus.py -text
+LCS/MessageBus/test/tMessage.cc -text
+LCS/MessageBus/test/tMsgBus.cc -text
+LCS/MessageBus/test/tMsgBus.run eol=lf
+LCS/MessageBus/test/tMsgBus.sh eol=lf
+LCS/MessageDaemons/src/MessageRouter -text
+LCS/MessageDaemons/src/MessageRouter.conf.ccu001 -text
+LCS/MessageDaemons/src/MessageRouter.conf.ccu099 -text
LCS/Tools/src/checkcomp.py -text
LCS/Tools/src/countalllines -text
LCS/Tools/src/countlines -text
@@ -4136,7 +4158,9 @@ RTCP/Cobalt/GPUProc/test/Kernels/tKernelPerformance.run eol=lf
RTCP/Cobalt/GPUProc/test/Kernels/tKernelPerformance.sh eol=lf
RTCP/Cobalt/GPUProc/test/Kernels/visualizeBeamformer.py eol=lf
RTCP/Cobalt/GPUProc/test/Pipelines/tCorrelatorPipelineProcessObs.sh eol=lf
+RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.queue -text
RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run eol=lf
+RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run2.in eol=lf
RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.sh eol=lf
RTCP/Cobalt/GPUProc/test/SubbandProcs/plot_arrays.py eol=lf
RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandProcProcessSb.parset -text
@@ -4249,6 +4273,14 @@ RTCP/Cobalt/clAmdFft/samples/statisticalTimer.h -text
RTCP/Cobalt/clAmdFft/samples/stdafx.cpp -text
RTCP/Cobalt/clAmdFft/samples/stdafx.h -text
RTCP/Cobalt/clAmdFft/samples/targetver.h -text
+SAS/Feedback_Service/CMakeLists.txt -text
+SAS/Feedback_Service/package.dox -text
+SAS/Feedback_Service/src/CMakeLists.txt -text
+SAS/Feedback_Service/src/Feedback.cc -text
+SAS/Feedback_Service/src/Feedback.h -text
+SAS/Feedback_Service/src/FeedbackMain.cc -text
+SAS/Feedback_Service/src/FeedbackService.conf -text
+SAS/Feedback_Service/src/fb_data_44883.txt -text
SAS/OTB/OTB-distribution/assembly.xml -text
SAS/OTB/OTB-distribution/pom.xml -text
SAS/OTB/OTB/assembly.xml -text
@@ -4571,6 +4603,7 @@ SubSystems/LAPS_CEP/test/tExampleTest.sh eol=lf
SubSystems/Online_Cobalt/install/install_DAL.sh eol=lf
SubSystems/Online_Cobalt/install/install_IERS.sh eol=lf
SubSystems/Online_Cobalt/install/install_casacore.sh eol=lf
+SubSystems/Online_Cobalt/install/install_qpid.sh eol=lf
SubSystems/Online_Cobalt/install/lofarsys/bash_profile -text
SubSystems/Online_Cobalt/install/lofarsys/bashrc -text
SubSystems/Online_Cobalt/install/postinstall.sh eol=lf
@@ -4598,6 +4631,8 @@ SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.outp
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/SB2.cfloat.raw -text
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/SB3.cfloat.raw -text
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/SB4.cfloat.raw -text
+SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.processing -text
+SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.state -text
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.parset -text
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.run eol=lf
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.sh eol=lf
@@ -4617,17 +4652,17 @@ SubSystems/Online_Cobalt/test/Correlator/tCorrelate_3sec_2st_5sb.output/SB4.cflo
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_3sec_2st_5sb.parset -text
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_3sec_2st_5sb.run eol=lf
SubSystems/Online_Cobalt/test/Correlator/tCorrelate_3sec_2st_5sb.sh eol=lf
-SubSystems/Online_Cobalt/test/MockOnlineControl.sh eol=lf
-SubSystems/Online_Cobalt/test/tMACfeedback.in_parset_failure_1 -text
-SubSystems/Online_Cobalt/test/tMACfeedback.in_parset_success_1 -text
-SubSystems/Online_Cobalt/test/tMACfeedback.run eol=lf
-SubSystems/Online_Cobalt/test/tMACfeedback.sh eol=lf
SubSystems/Online_Cobalt/test/tProductionParsets.in_parsets/CorrProc_173014.param -text
SubSystems/Online_Cobalt/test/tProductionParsets.in_parsets/CorrProc_196344-CygAf009-225SAPS.cleaned.param -text
SubSystems/Online_Cobalt/test/tProductionParsets.run eol=lf
SubSystems/Online_Cobalt/test/tProductionParsets.sh eol=lf
SubSystems/Online_Cobalt/test/tRTmetadataToFile.run eol=lf
SubSystems/Online_Cobalt/test/tRTmetadataToFile.sh eol=lf
+SubSystems/Online_Cobalt/test/tStatusFeedback.in_parset_failure_1 -text
+SubSystems/Online_Cobalt/test/tStatusFeedback.in_parset_success_1 -text
+SubSystems/Online_Cobalt/test/tStatusFeedback.run eol=lf
+SubSystems/Online_Cobalt/test/tStatusFeedback.run2.in eol=lf
+SubSystems/Online_Cobalt/test/tStatusFeedback.sh eol=lf
SubSystems/Online_Cobalt/test/tgenerateStationStreams.run eol=lf
SubSystems/Online_Cobalt/test/tgenerateStationStreams.sh eol=lf
SubSystems/Online_Cobalt/test/tstartBGL.in_parset -text
diff --git a/CEP/Pipeline/framework/lofarpipe/CMakeLists.txt b/CEP/Pipeline/framework/lofarpipe/CMakeLists.txt
index e5a6089bcab31fcf053d6ef9c623afae58f8353b..6ecc2a1b27e09bb7958fe47506ac0453f8c557b4 100644
--- a/CEP/Pipeline/framework/lofarpipe/CMakeLists.txt
+++ b/CEP/Pipeline/framework/lofarpipe/CMakeLists.txt
@@ -26,7 +26,6 @@ python_install(
support/lofarnode.py
support/loggingdecorators.py
support/mac.py
- support/mac_feedback.py
support/parset.py
support/pipelinelogging.py
support/remotecommand.py
diff --git a/CEP/Pipeline/framework/lofarpipe/support/baserecipe.py b/CEP/Pipeline/framework/lofarpipe/support/baserecipe.py
index c72f155f50936a2ad3ec1109093566697edffddb..2c34a2fad60206e3a2bc423fd4237900ba4b1e4c 100644
--- a/CEP/Pipeline/framework/lofarpipe/support/baserecipe.py
+++ b/CEP/Pipeline/framework/lofarpipe/support/baserecipe.py
@@ -45,7 +45,7 @@ class BaseRecipe(RecipeIngredients, WSRTrecipe):
# Environment variables we like to pass on to the node script.
self.environment = dict(
(k, v) for (k, v) in os.environ.iteritems()
- if k.endswith('PATH') or k.endswith('ROOT')
+ if k.endswith('PATH') or k.endswith('ROOT') or k == 'QUEUE_PREFIX'
)
@property
diff --git a/CEP/Pipeline/framework/lofarpipe/support/control.py b/CEP/Pipeline/framework/lofarpipe/support/control.py
index 2d85dc179f7f20dba54e881a69533a797e8b2bae..ce89b9ea70992c51b7157fe43ee1e1a8bf401cf2 100644
--- a/CEP/Pipeline/framework/lofarpipe/support/control.py
+++ b/CEP/Pipeline/framework/lofarpipe/support/control.py
@@ -6,6 +6,7 @@
# Marcel Loose, 2012
# loose@astron.nl
# ------------------------------------------------------------------------------
+import os
import sys
import re
import traceback
@@ -13,7 +14,11 @@ import traceback
from lofarpipe.support.stateful import StatefulRecipe
from lofarpipe.support.lofarexceptions import PipelineException
from lofarpipe.support.xmllogging import get_active_stack
-import lofarpipe.support.mac_feedback as mac_feedback
+from lofar.parameterset import parameterset
+import lofar.messagebus.msgbus
+from lofar.messagebus.protocols.taskfeedbackdataproducts import TaskFeedbackDataproducts
+from lofar.messagebus.protocols.taskfeedbackprocessing import TaskFeedbackProcessing
+from lofar.messagebus.protocols.taskfeedbackstate import TaskFeedbackState
# Standalone Pipeline Control System
# ------------------------------------------------------------------------------
@@ -22,7 +27,7 @@ class control(StatefulRecipe):
"""
Basic pipeline control framework.
- Define a pipeline by subclassing and provding a body for the
+ Define a pipeline by subclassing and providing a body for the
:meth:`pipeline_logic`.
This class provides little, but can be specialised to eg provide a
@@ -30,46 +35,77 @@ class control(StatefulRecipe):
"""
inputs = {}
- def _send_mac_feedback(self, status):
+ def __init__(self):
+ super(control, self).__init__()
+
+ self.parset = parameterset()
+ self.momID = 0
+ self.sasID = 0
+
+ def usage(self):
+ """
+ Display usage information
+ """
+ print >> sys.stderr, "Usage: %s <parset-file> [options]" % sys.argv[0]
+ return 1
+
+ def send_feedback_processing(self, feedback):
+ """
+ Send processing feedback information back to LOFAR.
+
+ `feedback` must be a parameterset
+ """
+
+ if self.feedback_method == "messagebus":
+ bus = lofar.messagebus.msgbus.ToBus("lofar.task.feedback.processing")
+ msg = TaskFeedbackProcessing(
+ "lofarpipe.support.control",
+ "",
+ "Processing feedback from the pipeline framework",
+ self.momID,
+ self.sasID,
+ feedback)
+
+ bus.send(msg)
+
+ def send_feedback_dataproducts(self, feedback):
"""
- Send status information back to MAC, but only if the Python controller
- host to send this information to was given as input argument.
+ Send dataproduct feedback information back to LOFAR.
+
+ `feedback` must be a parameterset
+ """
+
+ if self.feedback_method == "messagebus":
+ bus = lofar.messagebus.msgbus.ToBus("lofar.task.feedback.dataproducts")
+ msg = TaskFeedbackDataproducts(
+ "lofarpipe.support.control",
+ "",
+ "Dataproduct feedback from the pipeline framework",
+ self.momID,
+ self.sasID,
+ feedback)
+
+ bus.send(msg)
+
+ def _send_feedback_status(self, status):
+ """
+ Send status information back to LOFAR.
+
`status` must be an integer; 0 indicates success, any other value
indicates failure.
- The port number is calculated as 22000 + observationNr%1000.
- We need to extract this number from the job-name, which should be equal
- to "Observation" + str(observationNr).
"""
- try:
- host = self.inputs['args'][1]
- except IndexError:
- self.logger.warn(
- "No MAC Python controller host specified. "
- "Not sending status feedback to MAC"
- )
- return
- # Determine port number to use.
- match = re.findall(r'^Observation(\d+)$', self.inputs['job_name'])
- if match:
- port = 25000 + int(match[0]) % 7000
- self.logger.info(
- "Sending status feedback to MAC [%s:%s] (status: %s)" %
- (host, port, status)
- )
- else:
- self.logger.warn(
- r"Job-name does not match with pattern '^Observation(\d+)$'. "
- "Not sending status feedback to MAC"
- )
- return
- # Send feedback information
- try:
- mac_feedback.send_status(host, port, status)
- except IOError, error:
- self.logger.warn(
- "Failed to send status feedback to MAC [%s:%s]: %s" %
- (host, port, error)
- )
+
+ if self.feedback_method == "messagebus":
+ bus = lofar.messagebus.msgbus.ToBus("lofar.task.feedback.state")
+ msg = TaskFeedbackState(
+ "lofarpipe.support.control",
+ "",
+ "Status feedback from the pipeline framework",
+ self.momID,
+ self.sasID,
+ status == 0)
+
+ bus.send(msg)
def pipeline_logic(self):
"""
@@ -78,8 +114,40 @@ class control(StatefulRecipe):
raise NotImplementedError
def go(self):
+ # Read the parset-file that was given as input argument
+ try:
+ parset_file = os.path.abspath(self.inputs['args'][0])
+ except IndexError:
+ return self.usage()
+ self.parset.adoptFile(parset_file)
+ # Set job-name to basename of parset-file w/o extension, if it's not
+ # set on the command-line with '-j' or '--job-name'
+ if not 'job_name' in self.inputs:
+ self.inputs['job_name'] = (
+ os.path.splitext(os.path.basename(parset_file))[0]
+ )
+
+ # we can call our parent now that we have a job_name
super(control, self).go()
+
+ # we now have a self.config -- read our settings
+ try:
+ self.feedback_method = self.config.get('feedback', 'method')
+ except:
+ self.feedback_method = "messagebus"
+
+ if self.feedback_method == "messagebus" and not lofar.messagebus.msgbus.enabled:
+ self.logger.error("Feedback over messagebus requested, but messagebus support is not enabled or functional")
+ return 1
+
+ # Pull several parameters from the parset
+ self.momID = self.parset.getString("ObsSW.Observation.momID", "") # Note: 0 if obs was copied in Scheduler
+ self.sasID = self.parset.getString("ObsSW.Observation.otdbID", "") # SAS ID
+
+ # Start the pipeline
self.logger.info("LOFAR Pipeline (%s) starting." % self.name)
+ self.logger.info("SASID = %s, MOMID = %s, Feedback method = %s" % (self.sasID, self.momID, self.feedback_method))
+
try:
self.pipeline_logic()
except Exception, message:
@@ -98,10 +166,12 @@ class control(StatefulRecipe):
self.logger.error("*******************************************")
- self._send_mac_feedback(1)
+ # Emit process status
+ self._send_feedback_status(1)
return 1
else:
- self._send_mac_feedback(0)
+ # Emit process status
+ self._send_feedback_status(0)
return 0
finally:
# always print a xml stats file
diff --git a/CEP/Pipeline/framework/lofarpipe/support/lofaringredient.py b/CEP/Pipeline/framework/lofarpipe/support/lofaringredient.py
index df9ca7e230b961d5bc195211f8613c8106c06771..e6f6006242c8920664c82e7ffadcf37c0e6b16b5 100644
--- a/CEP/Pipeline/framework/lofarpipe/support/lofaringredient.py
+++ b/CEP/Pipeline/framework/lofarpipe/support/lofaringredient.py
@@ -11,6 +11,7 @@ from UserDict import DictMixin
from lofarpipe.cuisine.ingredient import WSRTingredient
from lofarpipe.support.utilities import string_to_list, is_iterable
+from lofar.parameterset import parameterset
# These are currently only used by lofarrecipe.run_task to provide default
# input and output dicts based on copying metadata from the parent.
@@ -194,6 +195,13 @@ class DictField(Field):
def is_valid(self, value):
return isinstance(value, dict)
+class ParsetField(Field):
+ """
+ A Field which accepts a parameterset object.
+ """
+ def is_valid(self, value):
+ return isinstance(value, parameterset)
+
class FileList(ListField):
"""
A Field which accepts a list of extant filenames.
diff --git a/CEP/Pipeline/framework/lofarpipe/support/mac_feedback.py b/CEP/Pipeline/framework/lofarpipe/support/mac_feedback.py
deleted file mode 100644
index baa2561991687ac2da905590a2871bd8ae0b3e7f..0000000000000000000000000000000000000000
--- a/CEP/Pipeline/framework/lofarpipe/support/mac_feedback.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# LOFAR IMAGING PIPELINE
-#
-# MAC Status Feedback QuickFix
-# Marcel Loose, 2012
-# loose@astron.nl
-# ------------------------------------------------------------------------------
-
-"""
-This module implements the quick fix (Redmine issue #3633) for the pipeline
-status feedback to MAC.
-"""
-
-import random
-import socket
-import time
-
-def __try_connect(host, port, tries=5, min_timeout=1.0, max_timeout=5.0):
- """
- Try to connect to `host`:`port` up time `tries` times, using a random
- timeout interval ([`min_timeout` .. `max_timeout`) seconds ) between
- retries.
- Return a socket object.
- Raises `socket.error` if all connect tries fail.
- """
- # Create a socket (SOCK_STREAM means a TCP socket)
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- while True:
- tries -= 1
- try:
- sock.connect((host, port))
- except socket.error, err:
- print("Could not connect to %s:%s (got %s)" %
- (host, str(port), str(err)))
- if tries > 0:
- timeout = random.uniform(min_timeout, max_timeout)
- print("Retrying in %f seconds (%d more %s)." %
- (timeout, tries, "try" if tries==1 else "tries"))
- time.sleep(timeout)
- else:
- raise IOError(err)
- else:
- return sock
-
-
-def send_status(host, port, status):
- """
- Send the pipeline status to a TCP listener at `host`:`port`. If `status` is
- non-zero, send the string 'ABORT'; otherwise send the string 'FINISHED'.
- """
- message = "FINISHED" if status == 0 else "ABORT"
- sock = __try_connect(host, port)
- sock.sendall(message)
- sock.close()
-
-
-if __name__ == "__main__":
- """
- Simple command line test.
- Usage: python mac_feedback.py [<host>] [<port>] [<status>]
- """
- import sys
- host = str(sys.argv[1]) if len(sys.argv) > 1 else "localhost"
- port = int(sys.argv[2]) if len(sys.argv) > 2 else 9999
- stat = int(sys.argv[3]) if len(sys.argv) > 3 else 0
- send_status(host, port, stat)
-
-
diff --git a/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py b/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py
index d2ce444a39cddd741a0132b3e7c24276e82a3d53..963ee22660ab255a8d1badfc0f5f6df68a733b1f 100644
--- a/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py
+++ b/CEP/Pipeline/framework/lofarpipe/support/remotecommand.py
@@ -252,7 +252,8 @@ class ComputeJob(object):
"PATH": os.environ.get('PATH'),
"PYTHONPATH": os.environ.get('PYTHONPATH'),
"LD_LIBRARY_PATH": os.environ.get('LD_LIBRARY_PATH'),
- "LOFARROOT" : os.environ.get('LOFARROOT')
+ "LOFARROOT" : os.environ.get('LOFARROOT'),
+ "QUEUE_PREFIX" : os.environ.get('QUEUE_PREFIX','')
},
arguments = [id, jobhost, jobport]
)
diff --git a/CEP/Pipeline/recipes/sip/bin/calibration_pipeline.py b/CEP/Pipeline/recipes/sip/bin/calibration_pipeline.py
index a6d35a7bd03fe2cdb010e3ec5f0645b515b58885..733c5dd90cb6417cc2f0f0ee8c934f531927e56e 100755
--- a/CEP/Pipeline/recipes/sip/bin/calibration_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/calibration_pipeline.py
@@ -31,23 +31,13 @@ class calibration_pipeline(control):
4. Create a sourcedb from the user-supplied sky model, and an empty parmdb.
5. Run BBS to calibrate the data.
6. Copy the MS's to their final output destination.
- 7. Create feedback file for further processing by the LOFAR framework (MAC)
+ 7. Create feedback for further processing by the LOFAR framework
"""
def __init__(self):
super(calibration_pipeline, self).__init__()
- self.parset = parameterset()
self.input_data = {}
self.output_data = {}
- self.parset_feedback_file = None
-
-
- def usage(self):
- """
- Display usage
- """
- print >> sys.stderr, "Usage: %s [options] <parset-file>" % sys.argv[0]
- return 1
def _get_io_product_specs(self):
@@ -98,28 +88,6 @@ class calibration_pipeline(control):
)
- def go(self):
- """
- Read the parset-file that was given as input argument, and set the
- jobname before calling the base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
-
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not self.inputs.has_key('job_name'):
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0])
-
- # Call the base-class's `go()` method.
- return super(calibration_pipeline, self).go()
-
-
@mail_log_on_exception
def pipeline_logic(self):
"""
@@ -294,31 +262,27 @@ class calibration_pipeline(control):
)
# *********************************************************************
- # 7. Create feedback file for further processing by the LOFAR framework
+ # 7. Create feedback for further processing by the LOFAR framework
# a. get metadata of the measurement sets
# b. get metadata of the instrument models
- # c. join the two files and write the final feedback file
- correlated_metadata = os.path.join(parset_dir, "correlated.metadata")
- instrument_metadata = os.path.join(parset_dir, "instrument.metadata")
+ # c. join the two and write the final feedback
with duration(self, "get_metadata"):
- self.run_task("get_metadata", output_correlated_mapfile,
- parset_file=correlated_metadata,
+ correlated_metadata = self.run_task("get_metadata", output_correlated_mapfile,
parset_prefix=(
self.parset.getString('prefix') +
self.parset.fullModuleName('DataProducts')),
- product_type="Correlated")
+ product_type="Correlated")["metadata"]
with duration(self, "get_metadata"):
- self.run_task("get_metadata", output_instrument_mapfile,
- parset_file=instrument_metadata,
+ instrument_metadata = self.run_task("get_metadata", output_instrument_mapfile,
parset_prefix=(
self.parset.getString('prefix') +
self.parset.fullModuleName('DataProducts')),
- product_type="InstrumentModel")
+ product_type="InstrumentModel")["metadata"]
- parset = parameterset(correlated_metadata)
- parset.adoptFile(instrument_metadata)
- parset.writeFile(self.parset_feedback_file)
+ self.send_feedback_processing(parameterset())
+ self.send_feedback_dataproducts(correlated_metadata)
+ self.send_feedback_dataproducts(instrument_metadata)
return 0
diff --git a/CEP/Pipeline/recipes/sip/bin/imaging_pipeline.py b/CEP/Pipeline/recipes/sip/bin/imaging_pipeline.py
index fa2f6e071981bfae11fb21ac6e4932c6995d753f..269bea6a1072d68f2db48ab4b3e10882205536dc 100755
--- a/CEP/Pipeline/recipes/sip/bin/imaging_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/imaging_pipeline.py
@@ -81,7 +81,7 @@ class imaging_pipeline(control):
and results are collected an added to the casa image. The images created
are converted from casa to HDF5 and copied to the correct output
location.
- 7. Export meta data: An outputfile with meta data is generated ready for
+ 7. Export meta data: meta data is generated ready for
consumption by the LTA and/or the LOFAR framework.
@@ -97,41 +97,13 @@ class imaging_pipeline(control):
Initialize member variables and call superclass init function
"""
control.__init__(self)
- self.parset = parameterset()
self.input_data = DataMap()
self.target_data = DataMap()
self.output_data = DataMap()
self.scratch_directory = None
- self.parset_feedback_file = None
self.parset_dir = None
self.mapfile_dir = None
- def usage(self):
- """
- Display usage information
- """
- print >> sys.stderr, "Usage: %s <parset-file> [options]" % sys.argv[0]
- return 1
-
- def go(self):
- """
- Read the parset-file that was given as input argument, and set the
- jobname before calling the base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not 'job_name' in self.inputs:
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0]
- )
- return super(imaging_pipeline, self).go()
-
@mail_log_on_exception
def pipeline_logic(self):
"""
@@ -231,14 +203,16 @@ class imaging_pipeline(control):
# *********************************************************************
# (7) Get metadata
- # Create a parset-file containing the metadata for MAC/SAS
- self.run_task("get_metadata", placed_data_image_map,
- parset_file = self.parset_feedback_file,
+ # Create a parset containing the metadata for MAC/SAS
+ metadata = self.run_task("get_metadata", placed_data_image_map,
parset_prefix = (
full_parset.getString('prefix') +
full_parset.fullModuleName('DataProducts')
),
- product_type = "SkyImage")
+ product_type = "SkyImage")["metadata"]
+
+ self.send_feedback_processing(parameterset())
+ self.send_feedback_dataproducts(metadata)
return 0
diff --git a/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py b/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py
index 8ccd77e892d4c817047dbe0ebc8ee22e78110ce7..7dcc5600358151311284a430f495c7dc13ceb465 100644
--- a/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/long_baseline_pipeline.py
@@ -52,7 +52,7 @@ class msss_imager_pipeline(control):
single large measurement set and perform flagging, RFI and bad station
exclusion.
- 2. Generate meta information feedback files based on dataproduct information
+ 2. Generate meta information feedback based on dataproduct information
and parset/configuration data
**Per subband-group, the following output products will be delivered:**
@@ -64,40 +64,13 @@ class msss_imager_pipeline(control):
Initialize member variables and call superclass init function
"""
control.__init__(self)
- self.parset = parameterset()
self.input_data = DataMap()
self.target_data = DataMap()
self.output_data = DataMap()
self.scratch_directory = None
- self.parset_feedback_file = None
self.parset_dir = None
self.mapfile_dir = None
- def usage(self):
- """
- Display usage information
- """
- print >> sys.stderr, "Usage: %s <parset-file> [options]" % sys.argv[0]
- return 1
-
- def go(self):
- """
- Read the parset-file that was given as input argument, and set the
- jobname before calling the base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not 'job_name' in self.inputs:
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0]
- )
- return super(msss_imager_pipeline, self).go()
@mail_log_on_exception
def pipeline_logic(self):
@@ -178,30 +151,17 @@ class msss_imager_pipeline(control):
str(subbands_per_subbandgroup))
toplevel_meta_data.replace("subbandGroupsPerMS",
str(subbandgroups_per_ms))
-
- toplevel_meta_data_path = os.path.join(
- self.parset_dir, "toplevel_meta_data.parset")
-
- try:
- toplevel_meta_data.writeFile(toplevel_meta_data_path)
- self.logger.info("Wrote meta data to: " +
- toplevel_meta_data_path)
- except RuntimeError, err:
- self.logger.error(
- "Failed to write toplevel meta information parset: %s" % str(
- toplevel_meta_data_path))
- return 1
-
# Create a parset-file containing the metadata for MAC/SAS at nodes
- self.run_task("get_metadata", output_ms_mapfile,
- parset_file = self.parset_feedback_file,
+ metadata = self.run_task("get_metadata", output_ms_mapfile,
parset_prefix = (
full_parset.getString('prefix') +
full_parset.fullModuleName('DataProducts')
),
- toplevel_meta_data_path=toplevel_meta_data_path,
- product_type = "Correlated")
+ product_type = "Correlated")["metadata"]
+
+ self.send_feedback_processing(toplevel_meta_data)
+ self.send_feedback_dataproducts(metadata)
return 0
diff --git a/CEP/Pipeline/recipes/sip/bin/msss_calibrator_pipeline.py b/CEP/Pipeline/recipes/sip/bin/msss_calibrator_pipeline.py
index 15d9632db40dc08ab1454e498d88a3c59c7cb2fa..893c16ff40237d413ba6d4b9a8cd3c1b95f23d2b 100755
--- a/CEP/Pipeline/recipes/sip/bin/msss_calibrator_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/msss_calibrator_pipeline.py
@@ -37,7 +37,7 @@ class msss_calibrator_pipeline(control):
parset, and the sourcedb made earlier
5. Perform gain correction on the created instrument table
6. Copy corrected MS's to their final output destination
- 7. Create output for consumption by the LOFAR framework
+ 7. Create metadata for consumption by the LOFAR framework
**Per subband-group, the following output products will be delivered:**
@@ -48,18 +48,8 @@ class msss_calibrator_pipeline(control):
def __init__(self):
control.__init__(self)
- self.parset = parameterset()
self.input_data = {}
self.output_data = {}
- self.parset_feedback_file = None
-
-
- def usage(self):
- """
- Display usage
- """
- print >> sys.stderr, "Usage: %s [options] <parset-file>" % sys.argv[0]
- return 1
def _get_io_product_specs(self):
@@ -110,28 +100,6 @@ class msss_calibrator_pipeline(control):
)
- def go(self):
- """
- Read the parset-file that was given as input argument, and set the
- jobname before calling the base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
-
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not self.inputs.has_key('job_name'):
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0])
-
- # Call the base-class's `go()` method.
- return super(msss_calibrator_pipeline, self).go()
-
-
@mail_log_on_exception
def pipeline_logic(self):
"""
@@ -309,31 +277,27 @@ class msss_calibrator_pipeline(control):
)
# *********************************************************************
- # 7. Create feedback file for further processing by the LOFAR framework
+ # 7. Create feedback for further processing by the LOFAR framework
# a. get metadata of the measurement sets
# b. get metadata of the instrument models
- # c. join the two files and write the final feedback file
- correlated_metadata = os.path.join(parset_dir, "correlated.metadata")
- instrument_metadata = os.path.join(parset_dir, "instrument.metadata")
+ # c. join the two and write the final feedback
with duration(self, "get_metadata"):
- self.run_task("get_metadata", output_correlated_mapfile,
- parset_file=correlated_metadata,
+ correlated_metadata = self.run_task("get_metadata", output_correlated_mapfile,
parset_prefix=(
self.parset.getString('prefix') +
self.parset.fullModuleName('DataProducts')),
- product_type="Correlated")
+ product_type="Correlated")["metadata"]
with duration(self, "get_metadata"):
- self.run_task("get_metadata", output_instrument_mapfile,
- parset_file=instrument_metadata,
+ instrument_metadata = self.run_task("get_metadata", output_instrument_mapfile,
parset_prefix=(
self.parset.getString('prefix') +
self.parset.fullModuleName('DataProducts')),
- product_type="InstrumentModel")
+ product_type="InstrumentModel")["metadata"]
- parset = parameterset(correlated_metadata)
- parset.adoptFile(instrument_metadata)
- parset.writeFile(self.parset_feedback_file)
+ self.send_feedback_processing(parameterset())
+ self.send_feedback_dataproducts(correlated_metadata)
+ self.send_feedback_dataproducts(instrument_metadata)
return 0
diff --git a/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py b/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py
index 42c6a13db83c7ebdcef935928ccf85699fb2d4bd..29d05237a0ff5d5954c9168b100734d73ebb2194 100755
--- a/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/msss_imager_pipeline.py
@@ -77,7 +77,7 @@ class msss_imager_pipeline(control):
and results are collected an added to the casa image. The images created
are converted from casa to HDF5 and copied to the correct output
location.
- 7. Export meta data: An outputfile with meta data is generated ready for
+ 7. Export meta data: meta data is generated ready for
consumption by the LTA and/or the LOFAR framework.
@@ -93,40 +93,13 @@ class msss_imager_pipeline(control):
Initialize member variables and call superclass init function
"""
control.__init__(self)
- self.parset = parameterset()
self.input_data = DataMap()
self.target_data = DataMap()
self.output_data = DataMap()
self.scratch_directory = None
- self.parset_feedback_file = None
self.parset_dir = None
self.mapfile_dir = None
- def usage(self):
- """
- Display usage information
- """
- print >> sys.stderr, "Usage: %s <parset-file> [options]" % sys.argv[0]
- return 1
-
- def go(self):
- """
- Read the parset-file that was given as input argument, and set the
- jobname before calling the base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not 'job_name' in self.inputs:
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0]
- )
- return super(msss_imager_pipeline, self).go()
@mail_log_on_exception
def pipeline_logic(self):
@@ -239,29 +212,17 @@ class msss_imager_pipeline(control):
toplevel_meta_data = parameterset()
toplevel_meta_data.replace("numberOfMajorCycles",
str(number_of_major_cycles))
- toplevel_meta_data_path = os.path.join(
- self.parset_dir, "toplevel_meta_data.parset")
- try:
- toplevel_meta_data.writeFile(toplevel_meta_data_path)
- self.logger.info("Wrote meta data to: " +
- toplevel_meta_data_path)
- except RuntimeError, err:
- self.logger.error(
- "Failed to write toplevel meta information parset: %s" % str(
- toplevel_meta_data_path))
- return 1
-
-
- # Create a parset-file containing the metadata for MAC/SAS at nodes
- self.run_task("get_metadata", placed_data_image_map,
- parset_file = self.parset_feedback_file,
+ # Create a parset containing the metadata for MAC/SAS at nodes
+ metadata = self.run_task("get_metadata", placed_data_image_map,
parset_prefix = (
full_parset.getString('prefix') +
full_parset.fullModuleName('DataProducts')
),
- toplevel_meta_data_path=toplevel_meta_data_path,
- product_type = "SkyImage")
+ product_type = "SkyImage")["metadata"]
+
+ self.send_feedback_processing(toplevel_meta_data)
+ self.send_feedback_dataproducts(metadata)
return 0
diff --git a/CEP/Pipeline/recipes/sip/bin/msss_target_pipeline.py b/CEP/Pipeline/recipes/sip/bin/msss_target_pipeline.py
index 59c853ad7843cddbc1823d0e2b4e5cce98a1da4f..2cf67ccc934ba7b4c950beeba79ae8d455487f3e 100755
--- a/CEP/Pipeline/recipes/sip/bin/msss_target_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/msss_target_pipeline.py
@@ -38,7 +38,7 @@ class msss_target_pipeline(control):
5. Run BBS using the instrument file from the target observation, to
correct for instrumental effects
6. Copy the MS's to their final output destination.
- 7. Create feedback file for further processing by the LOFAR framework (MAC)
+ 7. Create feedback for further processing by the LOFAR framework
**Per subband-group, the following output products will be delivered:**
@@ -48,18 +48,8 @@ class msss_target_pipeline(control):
def __init__(self):
control.__init__(self)
- self.parset = parameterset()
self.input_data = {}
self.output_data = {}
- self.parset_feedback_file = None
-
-
- def usage(self):
- """
- Display usage information
- """
- print >> sys.stderr, "Usage: %s [options] <parset-file>" % sys.argv[0]
- return 1
def _get_io_product_specs(self):
@@ -171,29 +161,6 @@ class msss_target_pipeline(control):
)
- def go(self):
- """
- Read the parset-file that was given as input argument, and set the
- jobname before calling the base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
-
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not self.inputs.has_key('job_name'):
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0]
- )
-
- # Call the base-class's `go()` method.
- return super(msss_target_pipeline, self).go()
-
-
@mail_log_on_exception
def pipeline_logic(self):
"""
@@ -345,17 +312,17 @@ class msss_target_pipeline(control):
)
# *********************************************************************
- # 7. Create feedback file for further processing by the LOFAR framework
- # (MAC)
- # Create a parset-file containing the metadata for MAC/SAS
+ # 7. Create feedback for further processing by the LOFAR framework
with duration(self, "get_metadata"):
- self.run_task("get_metadata", corrected_mapfile,
- parset_file=self.parset_feedback_file,
+ metadata = self.run_task("get_metadata", corrected_mapfile,
parset_prefix=(
self.parset.getString('prefix') +
self.parset.fullModuleName('DataProducts')
),
- product_type="Correlated")
+ product_type="Correlated")["metadata"]
+
+ self.send_feedback_processing(parameterset())
+ self.send_feedback_dataproducts(metadata)
return 0
diff --git a/CEP/Pipeline/recipes/sip/bin/preprocessing_pipeline.py b/CEP/Pipeline/recipes/sip/bin/preprocessing_pipeline.py
index 6039d9211d34f4a2baa6de011af3f909a57be235..a5f8a75db9e26f429aa64c055a1fc834da0a1fbc 100755
--- a/CEP/Pipeline/recipes/sip/bin/preprocessing_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/preprocessing_pipeline.py
@@ -30,19 +30,9 @@ class preprocessing_pipeline(control):
def __init__(self):
super(preprocessing_pipeline, self).__init__()
- self.parset = parameterset()
self.input_data = []
self.output_data = []
self.io_data_mask = []
- self.parset_feedback_file = None
-
-
- def usage(self):
- """
- Display usage
- """
- print >> sys.stderr, "Usage: %s [options] <parset-file>" % sys.argv[0]
- return 1
def _get_io_product_specs(self):
@@ -109,28 +99,6 @@ class preprocessing_pipeline(control):
# )
# )
- def go(self):
- """
- Read the parset-file that was given as input argument;
- set jobname, and input/output data products before calling the
- base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
-
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not self.inputs.has_key('job_name'):
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0])
-
- # Call the base-class's `go()` method.
- return super(preprocessing_pipeline, self).go()
-
@mail_log_on_exception
def pipeline_logic(self):
@@ -232,15 +200,15 @@ class preprocessing_pipeline(control):
# *********************************************************************
# 6. Create feedback file for further processing by the LOFAR framework
- # (MAC)
- # Create a parset-file containing the metadata for MAC/SAS
+ # Create a parset containing the metadata
with duration(self, "get_metadata"):
- self.run_task("get_metadata", output_data_mapfile,
- parset_file=self.parset_feedback_file,
+ metadata = self.run_task("get_metadata", output_data_mapfile,
parset_prefix=(
self.parset.getString('prefix') +
self.parset.fullModuleName('DataProducts')),
- product_type="Correlated")
+ product_type="Correlated")["metadata"]
+
+ self.send_feedback_dataproducts(metadata)
return 0
diff --git a/CEP/Pipeline/recipes/sip/bin/pulsar_pipeline.py b/CEP/Pipeline/recipes/sip/bin/pulsar_pipeline.py
index 6b54ec10b66b0de8d2fbc95cad197564c7e8246f..d70a1b8cac550aac6518ffe3bb2b4e104818c1f9 100755
--- a/CEP/Pipeline/recipes/sip/bin/pulsar_pipeline.py
+++ b/CEP/Pipeline/recipes/sip/bin/pulsar_pipeline.py
@@ -34,33 +34,8 @@ class pulsar_pipeline(control):
def __init__(self):
super(pulsar_pipeline, self).__init__()
- self.parset = parameterset()
self.input_data = {}
self.output_data = {}
- self.parset_feedback_file = None
-
-
- def go(self):
- """
- Read the parset-file that was given as input argument;
- set jobname, and input/output data products before calling the
- base-class's `go()` method.
- """
- try:
- parset_file = os.path.abspath(self.inputs['args'][0])
- except IndexError:
- return self.usage()
- self.parset.adoptFile(parset_file)
- self.parset_feedback_file = parset_file + "_feedback"
-
- # Set job-name to basename of parset-file w/o extension, if it's not
- # set on the command-line with '-j' or '--job-name'
- if not self.inputs.has_key('job_name'):
- self.inputs['job_name'] = (
- os.path.splitext(os.path.basename(parset_file))[0])
-
- # Call the base-class's `go()` method.
- return super(pulsar_pipeline, self).go()
def _get_io_product_specs(self):
@@ -120,6 +95,17 @@ class pulsar_pipeline(control):
"""
# *********************************************************************
# 1. Prepare phase, collect data from parset and input mapfiles.
+ #
+ # Note that PULP will read many of these fields directly. That makes
+ # the following fields, and possibly others, part of the API towards
+ # PULP:
+ #
+ # self.config
+ # self.logger
+ # self.input_data
+ # self.output_data
+ # self.parset_feedback_file
+ # self.job_dir
# Get input/output-data products specifications.
self._get_io_product_specs()
@@ -164,11 +150,29 @@ class pulsar_pipeline(control):
if (not self.incoherentStokesEnabled):
sys.argv.append("--noIS")
+
+ # Tell PULP where to write the feedback to
+ self.parset_feedback_file = "%s_feedback" % (self.parset_file,)
# Run the pulsar pipeline
self.logger.debug("Starting pulp with: " + join(sys.argv))
- p = pulp.pulp(self)
- return p.go()
+ p = pulp.pulp(self) # TODO: MUCK self to capture the API
+
+ # NOTE: PULP returns 0 on SUCCESS!!
+ if p.go():
+ self.logger.error("PULP did not succeed. Bailing out!")
+ return 0
+
+ # Read and forward the feedback
+ try:
+ metadata = parameterset(self.parset_feedback_file)
+ except IOError, e:
+ self.logger.error("Could not read feedback from %s: %s" % (metadata_file,e))
+ return 0
+
+ self.send_feedback_dataproducts(metadata)
+
+ return 1
if __name__ == '__main__':
diff --git a/CEP/Pipeline/recipes/sip/bin/startPython.sh b/CEP/Pipeline/recipes/sip/bin/startPython.sh
index dd1fb545237af6c2f1fad935094c38b6bd4e9b45..0aed7151de359ced6ca908e7c1f0829bc8a48d2c 100755
--- a/CEP/Pipeline/recipes/sip/bin/startPython.sh
+++ b/CEP/Pipeline/recipes/sip/bin/startPython.sh
@@ -27,8 +27,7 @@ logFile=/opt/lofar/var/log/startPython.log
usage()
{
- echo "Usage: $0 <pythonProgram> <parsetname> <MAC-Python-control-host> \\"
- echo " <MAC-Python-control-listener> <MAC-Python-control-server>"
+ echo "Usage: $0 <pythonProgram> <parsetname>"
exit 1
}
@@ -41,10 +40,9 @@ usage()
pythonProgram="${1}"
parsetFile="${2}"
-controlHost="${3}"
echo "**** $(date) ****" >> ${logFile}
-echo "Executing: $0 ${pythonProgram} ${parsetFile} ${controlHost}" >> ${logFile}
+echo "Executing: $0 ${pythonProgram} ${parsetFile}" >> ${logFile}
use_pulp="$(getparsetvalue $parsetFile "ObsSW.Observation.processSubtype")"
if [ "${use_pulp}" == "Pulsar Pipeline" ]; then
@@ -53,6 +51,9 @@ if [ "${use_pulp}" == "Pulsar Pipeline" ]; then
fi
echo "Initializing Lofar" >> ${logFile}
use Lofar
+echo "Initializing QPID" >> ${logFile}
+# TODO: implement 'use Qpid'
+source /data/qpid/.profile || source /opt/cep/qpid/.profile || echo "Could NOT load qpid .profile" >> ${logFile}
# Try to reset the environment based on a parset software version value
@@ -81,14 +82,14 @@ if [ -n "$debug" ]; then
echo "PATH=${PATH}" >> ${logFile}
echo "PYTHONPATH=${PYTHONPATH}" >> ${logFile}
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> ${logFile}
- echo "${pythonProgram} ${programOptions} ${parsetFile} ${controlHost}" \
+ echo "${pythonProgram} ${programOptions} ${parsetFile}" \
>> ${logFile}
fi
# Start the Python program in the background.
# This script should return ASAP so that MAC can set the task to ACTIVE.
# STDERR will be redirected to the log-file.
-${pythonProgram} ${programOptions} ${parsetFile} ${controlHost} \
+${pythonProgram} ${programOptions} ${parsetFile} \
1> /dev/null 2>> ${logFile} &
# Check if the Python program died early. If so, this indicates an error.
diff --git a/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py b/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py
index d1f4a665e4adf7aa7fb7b48f065c1f4f1fb3cbb6..491d7c0860d8b9bf9c6ade44a1a23ff021f1121b 100644
--- a/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py
+++ b/CEP/Pipeline/recipes/sip/master/deprecated/bbs.py
@@ -196,7 +196,7 @@ class bbs(BaseRecipe):
# already available on disk.
# ------------------------------------------------------------------
self.logger.debug("Building VDS file describing data for BBS run")
- vds_dir = tempfile.mkdtemp()
+ vds_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
vds_file = os.path.join(vds_dir, "bbs.gvds")
combineproc = utilities.spawn_process(
[
@@ -344,7 +344,7 @@ class bbs(BaseRecipe):
"""
env = utilities.read_initscript(self.logger, self.inputs['initscript'])
self.logger.info("Running BBS GlobalControl")
- working_dir = tempfile.mkdtemp()
+ working_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(
working_dir,
self.logger.name + ".GlobalControl",
diff --git a/CEP/Pipeline/recipes/sip/master/get_metadata.py b/CEP/Pipeline/recipes/sip/master/get_metadata.py
index e34b94e0fef44d725d7a0f97daf1881352bc72b0..8367f97d54f085013864e8636b025c914d41d3c6 100644
--- a/CEP/Pipeline/recipes/sip/master/get_metadata.py
+++ b/CEP/Pipeline/recipes/sip/master/get_metadata.py
@@ -25,7 +25,7 @@ class get_metadata(BaseRecipe, RemoteCommandRecipeMixIn):
2. Load mapfiles
3. call node side of the recipe
4. validate performance
- 5. Create the parset-file and write it to disk.
+ 5. Create the parset and return it.
**Command line arguments**
@@ -36,23 +36,17 @@ class get_metadata(BaseRecipe, RemoteCommandRecipeMixIn):
'--product-type',
help="Data product type",
),
- 'parset_file': ingredient.StringField(
- '--parset-file',
- help="Path to the output parset file"
- ),
'parset_prefix': ingredient.StringField(
'--parset-prefix',
help="Prefix for each key in the output parset file",
default=''
),
- 'toplevel_meta_data_path': ingredient.StringField(
- '--toplevel-meta-data',
- help="Path to parset with toplevel meta information, default = ''",
- default=''
- )
}
outputs = {
+ 'metadata': ingredient.ParsetField(
+ help="parset containing obtained metadata"
+ )
}
# List of valid data product types.
@@ -117,17 +111,11 @@ class get_metadata(BaseRecipe, RemoteCommandRecipeMixIn):
data.save(args[0])
# ********************************************************************
- # 5. Create the parset-file and write it to disk.
+ # 5. Create the parset-file and return it to the caller
parset = parameterset()
prefix = "Output_%s_" % product_type
parset.replace('%snrOf%s' % (global_prefix, prefix), str(len(jobs)))
- # If there is meta data to add from the toplevel script
- pipeline_meta_parset_path = self.inputs['toplevel_meta_data_path']
- if pipeline_meta_parset_path != "":
- pipeline_meta_parset = parameterset(pipeline_meta_parset_path)
- parset.adoptCollection(pipeline_meta_parset)
-
prefix = global_prefix + prefix
for idx, job in enumerate(jobs):
self.logger.debug("job[%d].results = %s" % (idx, job.results))
@@ -140,16 +128,8 @@ class get_metadata(BaseRecipe, RemoteCommandRecipeMixIn):
parset.adoptCollection(meta_data_parset,
'%s[%d].' % (prefix, idx))
- try:
-
- create_directory(os.path.dirname(self.inputs['parset_file']))
- parset.writeFile(self.inputs['parset_file'])
- self.logger.info("Wrote meta data to: " +
- self.inputs['parset_file'])
- except RuntimeError, err:
- self.logger.error("Failed to write meta-data: %s" % str(err))
- return 1
-
+ # Return result to caller
+ self.outputs["metadata"] = parset
return 0
diff --git a/CEP/Pipeline/recipes/sip/master/new_bbs.py b/CEP/Pipeline/recipes/sip/master/new_bbs.py
index 02b99cb80765a3535edf758280e1dd32c93f1710..acdaa5f073b70c4b9840c4a0a87eeca60b34c0ec 100644
--- a/CEP/Pipeline/recipes/sip/master/new_bbs.py
+++ b/CEP/Pipeline/recipes/sip/master/new_bbs.py
@@ -355,7 +355,7 @@ class new_bbs(BaseRecipe):
code.
"""
self.logger.info("Running BBS GlobalControl")
- working_dir = tempfile.mkdtemp()
+ working_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(
working_dir,
self.logger.name + ".GlobalControl",
diff --git a/CEP/Pipeline/recipes/sip/master/setupparmdb.py b/CEP/Pipeline/recipes/sip/master/setupparmdb.py
index e887d33b1ddcdd3db8a9e5a7c51c0a5d1361a4cd..ad54b38277b6376bfa188372f50da38107aeaecb 100644
--- a/CEP/Pipeline/recipes/sip/master/setupparmdb.py
+++ b/CEP/Pipeline/recipes/sip/master/setupparmdb.py
@@ -90,7 +90,8 @@ class setupparmdb(BaseRecipe, RemoteCommandRecipeMixIn):
# generate a temp dir
pdbdir = tempfile.mkdtemp(
- dir=self.config.get("layout", "job_directory")
+ dir=self.config.get("layout", "job_directory"),
+ suffix=".%s" % (os.path.basename(__file__),)
)
pdbfile = os.path.join(pdbdir, self.inputs['suffix'])
diff --git a/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py b/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py
index d327ae04c4d29a9f4b9e747632f8e7ad15940914..64e3bb9054fd2b2b5d92c631760c023a43f2706c 100644
--- a/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py
+++ b/CEP/Pipeline/recipes/sip/nodes/bbs_reducer.py
@@ -46,7 +46,7 @@ class bbs_reducer(LOFARnodeTCP):
return 1
# Run bbs-reducer. Catch log output from bbs-reducer and stdout.
- scratch_dir = mkdtemp()
+ scratch_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
try:
cmd = [executable,
"--parmdb=%s" % parmdb,
diff --git a/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py b/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py
index 6fbaca14092d505d086a556fd5dd9f5309a35f34..8ab25875631e5686dbf826d0e5e6b0b753c9c2b1 100644
--- a/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py
+++ b/CEP/Pipeline/recipes/sip/nodes/deprecated/bbs.py
@@ -66,7 +66,7 @@ class bbs(LOFARnodeTCP):
# Run the kernel
# Catch & log output from the kernel logger and stdout
# ------------------------------------------------------------------
- working_dir = mkdtemp()
+ working_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
env = read_initscript(self.logger, initscript)
try:
cmd = [executable, parset_filename, "0"]
diff --git a/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py b/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py
index 4b1d715c61f61c18b801399586793372f8e85181..278d47336c6a92541392e1bfbcb4ef7f3a6c99a3 100644
--- a/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py
+++ b/CEP/Pipeline/recipes/sip/nodes/deprecated/cimager.py
@@ -47,7 +47,7 @@ class cimager(LOFARnodeTCP):
self.logger.info("Image already exists: aborting.")
return 0
try:
- working_dir = mkdtemp()
+ working_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
# If a time range has been specified, copy that section of the
# input MS and only image that.
diff --git a/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py b/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py
index c72a5cf9e8f49c8e1276964bd4f436f35884556a..cbd49e5a281b14cc6f0c89d7532f042d6ab3c8e4 100644
--- a/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py
+++ b/CEP/Pipeline/recipes/sip/nodes/deprecated/demixing.py
@@ -54,7 +54,7 @@ class demixing(LOFARnodeTCP):
def _execute(self, cmd):
try:
- temp_dir = tempfile.mkdtemp()
+ temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(temp_dir,
self.logger.name,
os.path.basename(cmd[0])
diff --git a/CEP/Pipeline/recipes/sip/nodes/dppp.py b/CEP/Pipeline/recipes/sip/nodes/dppp.py
index de369323e889d343b29f7e794720b3d590ef3079..f77761729c52edc1e978753bec84419e063d34b1 100644
--- a/CEP/Pipeline/recipes/sip/nodes/dppp.py
+++ b/CEP/Pipeline/recipes/sip/nodes/dppp.py
@@ -142,7 +142,7 @@ class dppp(LOFARnodeTCP):
temp_parset_filename
)
try:
- working_dir = tempfile.mkdtemp()
+ working_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
# ****************************************************************
# 6. Run ndppp
cmd = [executable, temp_parset_filename, '1']
diff --git a/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py b/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py
index 03ee5128f5900c996aa9e8756d7a13d8b08793f3..db263daa8594159aae108df39faa2d69b087b1bf 100644
--- a/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py
+++ b/CEP/Pipeline/recipes/sip/nodes/gainoutliercorrection.py
@@ -77,7 +77,7 @@ class gainoutliercorrection(LOFARnodeTCP):
self.logger.info(
"Using the gainoutlier correction based on parmexportcal")
try:
- temp_dir = tempfile.mkdtemp()
+ temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(
temp_dir,
self.logger.name + '.' + os.path.basename(infile),
diff --git a/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py b/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py
index 81cc87d795acaf5b07bfafa27684f3b6e1430cbe..a06849c3dbdadb36aba4ca6d5c901b9c40eeb9cd 100644
--- a/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py
+++ b/CEP/Pipeline/recipes/sip/nodes/imager_finalize.py
@@ -116,7 +116,7 @@ class imager_finalize(LOFARnodeTCP):
os.unlink(fits_output)
try:
- temp_dir = tempfile.mkdtemp()
+ temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
with CatchLog4CPlus(temp_dir,
self.logger.name + '.' + os.path.basename(awimager_output),
"image2fits") as logger:
diff --git a/CEP/Pipeline/recipes/sip/nodes/new_bbs.py b/CEP/Pipeline/recipes/sip/nodes/new_bbs.py
index a0ee543182e7e52ddf4abd582fa7b5c0df8b5755..0123140135f7cb5329e7b693017b1ef69088fcd6 100644
--- a/CEP/Pipeline/recipes/sip/nodes/new_bbs.py
+++ b/CEP/Pipeline/recipes/sip/nodes/new_bbs.py
@@ -75,7 +75,7 @@ class new_bbs(LOFARnodeTCP):
# Run the kernel
# Catch & log output from the kernel logger and stdout
# ------------------------------------------------------------------
- working_dir = mkdtemp()
+ working_dir = mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
try:
self.logger.info("******** {0}".format(open(parset_file).read()))
cmd = [executable, parset_file, "0"]
diff --git a/CEP/Pipeline/recipes/sip/nodes/rficonsole.py b/CEP/Pipeline/recipes/sip/nodes/rficonsole.py
index d9869abff65bbdb1049ff4244dc07d70d82163a0..ceecf728f5941a97ab1e0089ee4735ff201569bb 100644
--- a/CEP/Pipeline/recipes/sip/nodes/rficonsole.py
+++ b/CEP/Pipeline/recipes/sip/nodes/rficonsole.py
@@ -27,7 +27,7 @@ class rficonsole(LOFARnodeTCP):
if not os.access(executable, os.X_OK):
raise ExecutableMissing(executable)
- working_dir = tempfile.mkdtemp(dir=wd)
+ working_dir = tempfile.mkdtemp(dir=wd,suffix=".%s" % (os.path.basename(__file__),))
cmd = [executable, "-j", str(nthreads)]
if strategy:
if os.path.exists(strategy):
diff --git a/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py b/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py
index ee35999e2d938e85f86acfab0991718ec62c5e32..af1212d13c4fae1f88da87dc5bf6e9b3668b3b1b 100644
--- a/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py
+++ b/CEP/Pipeline/recipes/sip/nodes/setupsourcedb.py
@@ -49,7 +49,7 @@ class setupsourcedb(LOFARnodeTCP):
shutil.rmtree(skydb, ignore_errors=True)
self.logger.info("Creating skymodel: %s" % (skydb))
- scratch_dir = tempfile.mkdtemp()
+ scratch_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
try:
cmd = [executable,
"in=%s" % catalogue,
diff --git a/CEP/Pipeline/recipes/sip/pipeline.cfg.in b/CEP/Pipeline/recipes/sip/pipeline.cfg.in
index 8732c7a30820b5cd5c51e3140dc170f37a663c5d..2b311047dd96f45bd74332b997240099414d4d28 100644
--- a/CEP/Pipeline/recipes/sip/pipeline.cfg.in
+++ b/CEP/Pipeline/recipes/sip/pipeline.cfg.in
@@ -23,3 +23,10 @@ engine_lpath = %(lofarroot)s/lib:%(casaroot)s/lib:%(pyraproot)s/lib:%(hdf5root)s
[logging]
log_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/pipeline.log
xml_stat_file = %(runtime_directory)s/%(job_name)s/logs/%(start_time)s/statistics.xml
+
+[feedback]
+# Method of providing feedback to LOFAR.
+# Valid options:
+# messagebus Send feedback and status using LCS/MessageBus
+# none Do NOT send feedback and status
+method = messagebus
diff --git a/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py b/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py
index adc4651d8490ebe63b5b0abf7419c4f1f0cc1b57..6da1b92ed44e36fb3e52fb68db66df1fc959a8e9 100644
--- a/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py
+++ b/CEP/Pipeline/test/recipes/helpers/WritableParmDB_test.py
@@ -28,7 +28,7 @@ class writableParmdbTest(unittest.TestCase):
super(writableParmdbTest, self).__init__(arg)
def setUp(self):
- self.tempDir = tempfile.mkdtemp()
+ self.tempDir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.tempParmDB = os.path.join(self.tempDir, "parmDB")
self.writable_parmdb = writableParmdbWrapper(self.tempParmDB)
diff --git a/CEP/Pipeline/test/recipes/master/copier_test.py b/CEP/Pipeline/test/recipes/master/copier_test.py
index 518445c1a1ea456b6b4c612b63352c2d8fc409d4..b94b35ce06cf6f1e68eb92cfe4eb8d3b639a78b6 100644
--- a/CEP/Pipeline/test/recipes/master/copier_test.py
+++ b/CEP/Pipeline/test/recipes/master/copier_test.py
@@ -33,10 +33,10 @@ class copierTest(unittest.TestCase):
def setUp(self):
self.imager_create_dbs = copierWrapper()
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_validate_mapfiles_norename(self):
@@ -120,10 +120,10 @@ class MasterNodeInterfaceTest(unittest.TestCase):
super(MasterNodeInterfaceTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test__init__raise_exception(self):
diff --git a/CEP/Pipeline/test/recipes/master/imager_bbs_test.py b/CEP/Pipeline/test/recipes/master/imager_bbs_test.py
index 5bc5e2edea0b8304e6aeaf7db602c59efd408da3..d85e0d7c74b8a25c52361f5af8778e28dc1c0f64 100644
--- a/CEP/Pipeline/test/recipes/master/imager_bbs_test.py
+++ b/CEP/Pipeline/test/recipes/master/imager_bbs_test.py
@@ -35,10 +35,10 @@ class imager_bbsTest(unittest.TestCase):
super(imager_bbsTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_constructor(self):
diff --git a/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py b/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py
index af56368efd08f4a05146e517ce0e93fe6ca8c623..ca5b75912416ef0123b1dfaf485658d36a620877 100644
--- a/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py
+++ b/CEP/Pipeline/test/recipes/master/imager_create_dbs_test.py
@@ -35,10 +35,10 @@ class imager_create_dbsTest(unittest.TestCase):
super(imager_create_dbsTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_validate_input_data(self):
diff --git a/CEP/Pipeline/test/recipes/master/imager_prepare_test.py b/CEP/Pipeline/test/recipes/master/imager_prepare_test.py
index a4504406a2d623a0294d0f62cc036bc78aed9c76..5eb2c34067c9ba0f3b49ac51667d794e370fb669 100644
--- a/CEP/Pipeline/test/recipes/master/imager_prepare_test.py
+++ b/CEP/Pipeline/test/recipes/master/imager_prepare_test.py
@@ -33,10 +33,10 @@ class imager_prepareTest(unittest.TestCase):
super(imager_prepareTest, self).__init__(arg)
def setUp(self):
- self.test_path = temp_path = tempfile.mkdtemp()
+ self.test_path = temp_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
pass
def test_create_input_map_for_sbgroup_single_ms(self):
diff --git a/CEP/Pipeline/test/recipes/nodes/copier_test.py b/CEP/Pipeline/test/recipes/nodes/copier_test.py
index d72935df9584d0461aec7cc1de44f25b591d7fdd..fe67f3444fb0a67c210b07748c0d52463b9c9f2f 100644
--- a/CEP/Pipeline/test/recipes/nodes/copier_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/copier_test.py
@@ -38,7 +38,7 @@ class copierTest(unittest.TestCase):
def tearDown(self):
pass
- #shutil.rmtree(self.test_path)
+ shutil.rmtree(self.test_path)
# # refactor of copier means this test is not needed anymore. Comment out for
# # now
@@ -49,7 +49,7 @@ class copierTest(unittest.TestCase):
# rsync return value
#
# """
-# temp_dir = tempfile.mkdtemp()
+# temp_dir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
# path_to_unowned_dir = "/home/klijntest/testdir" #
# file_to_copy = open(os.path.join(temp_dir, "test.txt"), 'w')
# file_to_copy.close()
diff --git a/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py b/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py
index 60b446fd50913463b85b5820b0cfd66d37f19a96..014fce027a85e125bf6d2cb80be6ce8704dedd84 100644
--- a/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/gainoutliercorrection_test.py
@@ -30,7 +30,7 @@ class GainOutlierDetectionTest(unittest.TestCase):
super(GainOutlierDetectionTest, self).__init__(arg)
def setUp(self):
- self.tempDir = tempfile.mkdtemp()
+ self.tempDir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
diff --git a/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py b/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py
index 9a17e1faf8b8dde5b0baea98b62b6831bef3602b..cb67c88998a4d59e4933e6c08ec88e735f321111 100644
--- a/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/imager_bbs_test.py
@@ -33,9 +33,10 @@ class imager_bbsTest(unittest.TestCase):
super(imager_bbsTest, self).__init__(arg)
def setUp(self):
- self.test_path = tempfile.mkdtemp()
+ self.test_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
+ shutil.rmtree(self.test_path)
pass
def test_constructor(self):
diff --git a/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py b/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
index bce1838dae1be5b2a20c6acb58003b1e5b1651ef..004cd81676eb1ad04fda512aec92d10cd995b22c 100644
--- a/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/imager_create_dbs_test.py
@@ -46,9 +46,10 @@ class ImagerCreateDBsTest(unittest.TestCase):
def setUp(self):
self.imager_create_dbs = ImagerCreateDBsTestWrapper()
- self.test_path = tempfile.mkdtemp()
+ self.test_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
+ shutil.rmtree(self.test_path)
pass
# New version of gsm: Quick fix to allow tests to succeed
diff --git a/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py b/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py
index 85cc51057fc6a076ac197c9f24be3297c16fc756..35e61a823acbc5d50635faae89d9228840efb14b 100644
--- a/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py
+++ b/CEP/Pipeline/test/recipes/nodes/imager_prepare_test.py
@@ -40,9 +40,10 @@ class ImagerPrepareTest(unittest.TestCase):
def setUp(self):
self.ImagerPrepareTestWrapper = ImagerPrepareTestWrapper()
- self.test_path = tempfile.mkdtemp()
+ self.test_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
def tearDown(self):
+ shutil.rmtree(self.test_path)
pass
def test__copy_input_files_multi_file(self):
@@ -97,7 +98,7 @@ class ImagerPrepareTest(unittest.TestCase):
"""
working_dir = ""
- time_slice_dir_path = tempfile.mkdtemp()
+ time_slice_dir_path = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
slices_per_image = 2
input_map = [("lce072", "test_file_path1"),
("lce072", "test_file_path2"),
@@ -145,6 +146,8 @@ class ImagerPrepareTest(unittest.TestCase):
"time_slice_1.dppp.ms.ndppp.par")).read())
self.assertTrue(parset_2_output == parset_2_content_expected,
"\n{0} != \n{1}".format(parset_2_output, parset_2_content_expected))
+
+ shutil.rmtree(time_slice_dir_path)
if __name__ == "__main__":
diff --git a/CEP/Pipeline/test/regression_tests/regression_test_runner.sh b/CEP/Pipeline/test/regression_tests/regression_test_runner.sh
index 5f439bfbe9cf73553688830a5423f6bd6e4723c1..6bd86a79813ddcbb5a3ce39ca575f0a4f40f5948 100755
--- a/CEP/Pipeline/test/regression_tests/regression_test_runner.sh
+++ b/CEP/Pipeline/test/regression_tests/regression_test_runner.sh
@@ -88,6 +88,7 @@ mkdir -p $"$WORKSPACE/installed/var/run/pipeline"
use Lofar # this is a weak point in the script we should be able to run without
use Pythonlibs
. $"$WORKSPACE/lofarinit.sh"
+. /data/qpid/.profile
# *****************************************************
# 3) Clear old data:
@@ -150,6 +151,12 @@ sed -i $"s|input_path2_placeholder|$WORKING_DIR/input_data|g" $"$WORKING_DIR/$P
sed -i $"s|output_path1_placeholder|$WORKING_DIR/output_data|g" $"$WORKING_DIR/$PIPELINE.parset"
sed -i $"s|output_path2_placeholder|$WORKING_DIR/output_data|g" $"$WORKING_DIR/$PIPELINE.parset"
+# setup the qpid environment (is a no-op if qpid is not installed)
+source $WORKSPACE/bin/MessageFuncs.sh
+create_queue lofar.task.feedback.state
+create_queue lofar.task.feedback.dataproducts
+create_queue lofar.task.feedback.processing
+
# *********************************************************************
# 5) Run the pipeline
echo "Run the pipeline"
diff --git a/CEP/Pipeline/test/support/data_map_test.py b/CEP/Pipeline/test/support/data_map_test.py
index 17ad4c607ebb537455828066eae6b26a19135a75..5321e766e946f50cf085f57133a76e5b8c808cc0 100644
--- a/CEP/Pipeline/test/support/data_map_test.py
+++ b/CEP/Pipeline/test/support/data_map_test.py
@@ -33,7 +33,7 @@ class DataMapTest(unittest.TestCase):
"""
Create scratch directory and create required input files in there.
"""
- self.tmpdir = tempfile.mkdtemp()
+ self.tmpdir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.old_style_map_file = self._create_old_style_map_file()
self.new_style_map_file = self._create_new_style_map_file()
self.syntax_error_map_file = self._create_syntax_error_map_file()
@@ -155,7 +155,7 @@ class MultiDataMapTest(unittest.TestCase):
"""
Create scratch directory and create required input files in there.
"""
- self.tmpdir = tempfile.mkdtemp()
+ self.tmpdir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.old_style_map_file = self._create_old_style_map_file()
self.new_style_map_file = self._create_new_style_map_file()
self.syntax_error_map_file = self._create_syntax_error_map_file()
@@ -290,7 +290,7 @@ class HelperFunctionDataMapTest(unittest.TestCase):
"""
Create scratch directory and create required input files in there.
"""
- self.tmpdir = tempfile.mkdtemp()
+ self.tmpdir = tempfile.mkdtemp(suffix=".%s" % (os.path.basename(__file__),))
self.old_style_map_file = self._create_old_style_map_file()
self.new_style_map_file = self._create_new_style_map_file()
self.syntax_error_map_file = self._create_syntax_error_map_file()
diff --git a/CMake/FindQPID.cmake b/CMake/FindQPID.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..75343f7a49d055cc3b6cd043e8873ae307532a98
--- /dev/null
+++ b/CMake/FindQPID.cmake
@@ -0,0 +1,63 @@
+# - Try to find QPID: Apache's implementation of the AMPQ protocol
+# Variables used by this module:
+# QPID_ROOT_DIR - QPID root directory
+# Variables defined by this module:
+# QPID_FOUND - system has QPID
+# QPID_INCLUDE_DIR - the QPID include directory (cached)
+# QPID_INCLUDE_DIRS - the QPID include directories
+# (identical to QPID_INCLUDE_DIR)
+# QPID_LIBRARY - the QPID library (cached)
+# QPID_LIBRARIES - the QPID libraries
+# (identical to QPID_LIBRARY)
+# QPID_RECEIVE_EXECUTABLE - the full path of qpid-receive
+# QPID_SEND_EXECUTABLE - the full path of qpid-send
+# QPID_STAT_EXECUTABLE - the full path of qpid-stat
+
+# Copyright (C) 2015
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# $Id$
+
+if(NOT QPID_FOUND)
+
+ find_path(QPID_INCLUDE_DIR qpid/messaging/Connection.h
+ HINTS ${QPID_ROOT_DIR} PATH_SUFFIXES include)
+ find_library(QPID_MESSAGING_LIBRARY qpidmessaging
+ HINTS ${QPID_ROOT_DIR} PATH_SUFFIXES lib)
+ find_library(QPID_TYPES_LIBRARY qpidtypes
+ HINTS ${QPID_ROOT_DIR} PATH_SUFFIXES lib)
+
+ find_program(QPID_RECEIVE_EXECUTABLE qpid-receive
+ HINTS ${QPID_ROOT_DIR} PATH_SUFFIXES bin)
+ find_program(QPID_SEND_EXECUTABLE qpid-send
+ HINTS ${QPID_ROOT_DIR} PATH_SUFFIXES bin)
+ find_program(QPID_STAT_EXECUTABLE qpid-stat
+ HINTS ${QPID_ROOT_DIR} PATH_SUFFIXES bin)
+
+ set(QPID_LIBRARY ${QPID_MESSAGING_LIBRARY} ${QPID_TYPES_LIBRARY})
+
+ mark_as_advanced(QPID_INCLUDE_DIR QPID_LIBRARY)
+
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(QPID DEFAULT_MSG
+ QPID_LIBRARY QPID_INCLUDE_DIR)
+
+ set(QPID_INCLUDE_DIRS ${QPID_INCLUDE_DIR})
+ set(QPID_LIBRARIES ${QPID_LIBRARY})
+
+endif(NOT QPID_FOUND)
diff --git a/CMake/FindUUID.cmake b/CMake/FindUUID.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..7fdac87a2aae3c3e6ce19e92ae5e3b67db80426f
--- /dev/null
+++ b/CMake/FindUUID.cmake
@@ -0,0 +1,49 @@
+# - Try to find libuuid: A library to generate UUIDs
+# Variables used by this module:
+# UUID_ROOT_DIR - QPID root directory
+# Variables defined by this module:
+# UUID_FOUND - system has UUID
+# UUID_INCLUDE_DIR - the UUID include directory (cached)
+# UUID_INCLUDE_DIRS - the UUID include directories
+# (identical to UUID_INCLUDE_DIR)
+# UUID_LIBRARY - the UUID library (cached)
+# UUID_LIBRARIES - the UUID libraries
+# (identical to UUID_LIBRARY)
+
+# Copyright (C) 2015
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# $Id$
+
+if(NOT UUID_FOUND)
+
+ find_path(UUID_INCLUDE_DIR uuid/uuid.h
+ HINTS ${UUID_ROOT_DIR} PATH_SUFFIXES include)
+ find_library(UUID_LIBRARY uuid
+ HINTS ${UUID_ROOT_DIR} PATH_SUFFIXES lib)
+
+ mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY)
+
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(UUID DEFAULT_MSG
+ UUID_LIBRARY UUID_INCLUDE_DIR)
+
+ set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR})
+ set(UUID_LIBRARIES ${UUID_LIBRARY})
+
+endif(NOT UUID_FOUND)
diff --git a/CMake/LofarPackageList.cmake b/CMake/LofarPackageList.cmake
index 309fc1ec0154caf538f729a922cc901665504d6d..ef3efc6d6b27c56120e115193dc0ed89140db356 100644
--- a/CMake/LofarPackageList.cmake
+++ b/CMake/LofarPackageList.cmake
@@ -1,7 +1,7 @@
# - Create for each LOFAR package a variable containing the absolute path to
# its source directory.
#
-# Generated by gen_LofarPackageList_cmake.sh at Tue Nov 4 09:18:08 CET 2014
+# Generated by gen_LofarPackageList_cmake.sh at Tue Jan 27 14:42:15 UTC 2015
#
# ---- DO NOT EDIT ----
#
@@ -14,14 +14,6 @@
#
if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(LOFAR_PACKAGE_LIST_INCLUDED TRUE)
- set(BBSKernel_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/BBSKernel)
- set(BBSControl_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/BBSControl)
- set(ExpIon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/ExpIon)
- set(pystationresponse_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/pystationresponse)
- set(pyselfcal_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/pyselfcal)
- set(BBSTools_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/BBSTools)
- set(ElementResponse_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/ElementResponse)
- set(StationResponse_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/StationResponse)
set(Calibration_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration)
set(DP3_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3)
set(GSM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/GSM)
@@ -33,6 +25,14 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(PyBDSM_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/PyBDSM)
set(pyparmdb_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/pyparmdb)
set(LAPS_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/LAPS)
+ set(BBSKernel_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/BBSKernel)
+ set(BBSControl_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/BBSControl)
+ set(ExpIon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/ExpIon)
+ set(pystationresponse_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/pystationresponse)
+ set(pyselfcal_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/pyselfcal)
+ set(BBSTools_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/BBSTools)
+ set(ElementResponse_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/ElementResponse)
+ set(StationResponse_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/Calibration/StationResponse)
set(DPPP_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3/DPPP)
set(TestDynDPPP_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3/TestDynDPPP)
set(PythonDPPP_SOURCE_DIR ${CMAKE_SOURCE_DIR}/CEP/DP3/PythonDPPP)
@@ -62,6 +62,8 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(ApplCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/ApplCommon)
set(Blob_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/Blob)
set(Common_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/Common)
+ set(MessageBus_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/MessageBus)
+ set(MessageDaemons_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/MessageDaemons)
set(MSLofar_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/MSLofar)
set(pyparameterset_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/pyparameterset)
set(pytools_SOURCE_DIR ${CMAKE_SOURCE_DIR}/LCS/pytools)
@@ -131,6 +133,7 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(Scheduler_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/Scheduler)
set(jOTDB3_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/OTB/jOTDB3)
set(OTB-Java_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/OTB/OTB)
+ set(SAS_Feedback_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/Feedback_Service)
set(CCU_MAC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/CCU_MAC)
set(LCU_MAC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/LCU_MAC)
set(MCU_MAC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/MCU_MAC)
diff --git a/CMake/PythonInstall.cmake b/CMake/PythonInstall.cmake
index b62fff3eaeb9e95fbbb98cac8a867add35615a35..ee464b09ab63ec17eb438588391b1c45b8ce7e32 100644
--- a/CMake/PythonInstall.cmake
+++ b/CMake/PythonInstall.cmake
@@ -83,14 +83,15 @@ macro(python_install)
install(FILES ${_py} DESTINATION ${_inst_dir}/${_py_path})
if(USE_PYTHON_COMPILATION)
set(_py_code
- "import py_compile"
- "print('-- Byte-compiling: ${_inst_dir}/${_py}')"
- "py_compile.compile('${_inst_dir}/${_py}', doraise=True)")
+ "import py_compile, os"
+ "destdir = os.environ.get('DESTDIR','')"
+ "print('-- Byte-compiling: %s${_inst_dir}/${_py}' % destdir)"
+ "py_compile.compile('%s${DESTDIR}${_inst_dir}/${_py}' % destdir, doraise=True)")
install(CODE
"execute_process(COMMAND ${PYTHON_EXECUTABLE} -c \"${_py_code}\"
RESULT_VARIABLE _result)
if(NOT _result EQUAL 0)
- message(FATAL_ERROR \"Byte-compilation FAILED: ${_inst_dir}/${_py}\")
+ message(FATAL_ERROR \"Byte-compilation FAILED: \$ENV{DESTDIR}${_inst_dir}/${_py}\")
endif(NOT _result EQUAL 0)")
endif(USE_PYTHON_COMPILATION)
endforeach(_py ${_py_files})
@@ -104,7 +105,7 @@ macro(python_install)
"${PYTHON_BUILD_DIR}${_init_dir}/__init__.py")
install(CODE
"execute_process(COMMAND ${CMAKE_COMMAND} -E touch
- \"${PYTHON_INSTALL_DIR}${_init_dir}/__init__.py\")")
+ \"\$ENV{DESTDIR}${PYTHON_INSTALL_DIR}${_init_dir}/__init__.py\")")
endforeach(_dir ${_dir_list})
endmacro(python_install)
diff --git a/CMake/TODO b/CMake/TODO
index 166c32bb70051053fb57bdd3a027622ed56c7107..79d64c794158a0b26c3d8fa1e091eb2ca9e188b0 100644
--- a/CMake/TODO
+++ b/CMake/TODO
@@ -30,9 +30,6 @@
Currently, python sources are compiled during install. What's worse,
compilation failures are not reported, and no non-zero exit status is set.
-* Python byte-code compilation fails during install when using DESTDIR option
- to specify a different install directory.
-
* Add option DEPENDS to lofar_add_library() and lofar_add_executable(), which
can be used to add additional dependencies. This is mainly useful for
custom targets, because "normal" target dependencies are already handled by
diff --git a/CMake/gen_LofarPackageList_cmake.sh b/CMake/gen_LofarPackageList_cmake.sh
index ea5ceb395c6c1985e2dec9b838d28dda25edc42e..e1eed1e579f6cd0fc4681df91fcdedeae9f3a60a 100755
--- a/CMake/gen_LofarPackageList_cmake.sh
+++ b/CMake/gen_LofarPackageList_cmake.sh
@@ -28,13 +28,13 @@
# Get the LOFAR source directory root
script_dir=$(cd $(dirname $0) && pwd)
-lofar_root=$(echo $script_dir | sed -e "s|\(.*/LOFAR\)/.*|\1|")
+lofar_root=$script_dir/..
# Just a safety net; this script must be inside the LOFAR tree.
-if test "$script_dir" = "$lofar_root"; then
- echo "ERROR: $(basename $0) MUST be inside the LOFAR source tree!"
- exit 1
-fi
+#if test "$script_dir" = "$lofar_root"; then
+# echo "ERROR: $(basename $0) MUST be inside the LOFAR source tree!"
+# exit 1
+#fi
# Open the output file
exec 3> $script_dir/LofarPackageList.cmake
diff --git a/CMake/variants/variants.RS005C b/CMake/variants/variants.RS005C
index 34cff54b9059065e815d1923e096fa3b89e75007..d655bb6d000a5d7c7756acc6cfaaafc6ff60df66 100644
--- a/CMake/variants/variants.RS005C
+++ b/CMake/variants/variants.RS005C
@@ -20,6 +20,9 @@ set(MATLAB_ROOT_DIR /usr/local/matlab-2008a)
# Path to JAVA
set(ENV{JAVA_HOME} /usr/local/jdk1.6.0_29)
+# Path to Qpid
+set(QPID_ROOT_DIR /opt/qpid)
+
# Ignore warnings matching the following pattern(s)
set(CTEST_CUSTOM_WARNING_EXCEPTION
"/boost/date_time/time_facet.hpp:[0-9]+: warning: unused parameter"
diff --git a/CMake/variants/variants.cbt001 b/CMake/variants/variants.cbt001
index 979c48fb3ed81fd935f06dba833783b9c37f85ab..f79a4cd2a4a46f82fc50a0dfb8ada617fe50aab8 100644
--- a/CMake/variants/variants.cbt001
+++ b/CMake/variants/variants.cbt001
@@ -14,6 +14,7 @@ option(USE_CUDA "Use CUDA" ON)
set(CASACORE_ROOT_DIR /localhome/lofar/casacore-1.7.0)
set(MPI_ROOT_DIR /opt/openmpi)
set(DAL_ROOT_DIR /localhome/lofar/DAL)
+set(QPID_ROOT_DIR /localhome/lofar/qpid)
#set(VALGRIND_ROOT_DIR /globalhome/lofarsystem/cobalt-root)
#set(GNU_C /globalhome/amesfoort/cbtroot/bin/gcc-4.8)
diff --git a/CMake/variants/variants.kis001 b/CMake/variants/variants.kis001
index be8b6c767181ceda9beb9baacefdaaeee0ddecfc..2c6781578ce25f71252d0cf83084750dbe151db0 100644
--- a/CMake/variants/variants.kis001
+++ b/CMake/variants/variants.kis001
@@ -5,6 +5,7 @@ set(ENV{JAVA_HOME} /usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0)
#set(PVSS_ROOT_DIR /opt/pvss/pvss2_v3.7)
set(PVSS_ROOT_DIR /opt/WinCC_OA/3.10)
set(LOG4CPLUS_ROOT_DIR "/usr/local/log4cplus")
+set(QPID_ROOT_DIR /opt/qpid)
set(CTEST_CUSTOM_WARNING_EXCEPTION
"/boost/date_time/time_facet.hpp:[0-9]+: warning: unused parameter"
diff --git a/CMake/variants/variants.lhn001 b/CMake/variants/variants.lhn001
index 1b2617ece3ef679fa088e55693233bd4b90b93f7..ab8bf36cc2f9b427241c1be2ba46d61b16093faf 100644
--- a/CMake/variants/variants.lhn001
+++ b/CMake/variants/variants.lhn001
@@ -4,6 +4,8 @@ set(DAL_ROOT_DIR /opt/cep/dal/current)
set(PYRAP_ROOT_DIR /opt/cep/pyrap)
+set(QPID_ROOT_DIR /opt/cep/qpid)
+
set(WCSLIB_ROOT_DIR /opt/cep/wcslib)
set(LOG4CXX_ROOT_DIR /opt/cep/LofIm/external/log4cxx)
diff --git a/CMake/variants/variants.lhn002 b/CMake/variants/variants.lhn002
deleted file mode 100644
index 1b2617ece3ef679fa088e55693233bd4b90b93f7..0000000000000000000000000000000000000000
--- a/CMake/variants/variants.lhn002
+++ /dev/null
@@ -1,21 +0,0 @@
-set(CASACORE_ROOT_DIR /opt/cep/casacore)
-set(CASAREST_ROOT_DIR /opt/cep/casarest)
-set(DAL_ROOT_DIR /opt/cep/dal/current)
-
-set(PYRAP_ROOT_DIR /opt/cep/pyrap)
-
-set(WCSLIB_ROOT_DIR /opt/cep/wcslib)
-
-set(LOG4CXX_ROOT_DIR /opt/cep/LofIm/external/log4cxx)
-set(LOG4CPLUS_ROOT_DIR /opt/cep/lofar/external/log4cplus)
-
-#set(ENV{JAVA_HOME} /usr/lib/jvm/java-1.5.0-sun)
-
-set(CTEST_CUSTOM_WARNING_EXCEPTION
- "/log4cxx/helpers/objectptr.h:[0-9]+: warning: base class"
- "/log4cxx/helpers/objectptr.h:[0-9]+: warning: dereferencing type-punned pointer"
-)
-
-# Drop the now default C++ debug compile flag -D_GLIBCXX_DEBUG, because Python
-# (or Boost.Python) barfs on it (glibc detected: python free(): invalid pointer)
-set(GNU_CXX_FLAGS_DEBUG "-g")
diff --git a/CMake/variants/variants.lhn002 b/CMake/variants/variants.lhn002
new file mode 120000
index 0000000000000000000000000000000000000000..92890f596ce0939cfafa22bdd2337914338bdd0a
--- /dev/null
+++ b/CMake/variants/variants.lhn002
@@ -0,0 +1 @@
+variants.lhn001
\ No newline at end of file
diff --git a/CMake/variants/variants.locus102 b/CMake/variants/variants.locus102
index ec7471769b4689da707dea90afe625bd4ea0a0a6..7f28ed3d162f7f7f5f0e11e47a37b0d221f63dac 100644
--- a/CMake/variants/variants.locus102
+++ b/CMake/variants/variants.locus102
@@ -3,6 +3,7 @@ option(BUILD_SHARED_LIBS "Build shared libraries" ON)
set(CASACORE_ROOT_DIR /opt/cep/casacore)
set(CASAREST_ROOT_DIR /opt/cep/casarest)
set(DAL_ROOT_DIR /opt/cep/dal/current)
+set(QPID_ROOT_DIR /data/qpid)
set(PYRAP_ROOT_DIR /opt/cep/pyrap)
diff --git a/CMake/variants/variants.sas099 b/CMake/variants/variants.sas099
index b3e304975595f76dd4e1cb386490bcd6f4f20c4a..15a7c9289aa6fea7fb9e334aaa1f4a9eed6690a1 100644
--- a/CMake/variants/variants.sas099
+++ b/CMake/variants/variants.sas099
@@ -1,5 +1,6 @@
set(ENV{JAVA_HOME} /usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0)
set(PVSS_ROOT_DIR /opt/pvss/pvss2_v3.7)
+set(QPID_ROOT_DIR /opt/qpid)
set(CTEST_CUSTOM_WARNING_EXCEPTION
"/boost/date_time/time_facet.hpp:[0-9]+: warning: unused parameter"
diff --git a/LCS/CMakeLists.txt b/LCS/CMakeLists.txt
index a973e43d1cdc30d44029f40a0cf0ca93e39fe254..eaf721a214020ba8ed6cfc3118664179f15592c1 100644
--- a/LCS/CMakeLists.txt
+++ b/LCS/CMakeLists.txt
@@ -5,6 +5,8 @@ lofar_add_package(AMC) # Astronomical Measures Conversions
lofar_add_package(ApplCommon) # Application common stuff
lofar_add_package(Blob) # Binary Large Objects
lofar_add_package(Common) # Common stuff
+lofar_add_package(MessageBus) # Support for QPID message exchange
+lofar_add_package(MessageDaemons) # Daemons to do message routing and handling
lofar_add_package(MSLofar) # MS for LOFAR based on ICD
lofar_add_package(pyparameterset) # Python ParameterSet bindings
lofar_add_package(pytools) # Python tools
diff --git a/LCS/MessageBus/CMakeLists.txt b/LCS/MessageBus/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3c6af3f13d0d9e7ee2e1d315057ee705657dfd80
--- /dev/null
+++ b/LCS/MessageBus/CMakeLists.txt
@@ -0,0 +1,10 @@
+# $Id$
+
+lofar_package(MessageBus 1.0 DEPENDS Common)
+
+include(LofarFindPackage)
+lofar_find_package(QPID)
+
+add_subdirectory(include/MessageBus)
+add_subdirectory(src)
+add_subdirectory(test)
diff --git a/LCS/MessageBus/data/feedback_dp_44833.txt b/LCS/MessageBus/data/feedback_dp_44833.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e6a7627720e1a4fcd441bdf2695137b23c75118f
--- /dev/null
+++ b/LCS/MessageBus/data/feedback_dp_44833.txt
@@ -0,0 +1,40 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>lofar.observation.start</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>mySubSystem</name>
+ <user>user</user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>some test message</summary>
+ </source>
+ <ids>
+ <momid>12345</momid>
+ <sasid>44883</sasid>
+ </ids>
+ </header>
+ <payload>
+Observation.Correlator.channelWidth=3051.7578125
+Observation.Correlator.channelsPerSubband=64
+Observation.Correlator.integrationInterval=1.00139008
+Observation.DataProducts.Output_Correlated_[0].SAP=0
+Observation.DataProducts.Output_Correlated_[0].centralFrequency=115039062.500000
+Observation.DataProducts.Output_Correlated_[0].channelWidth=3051.757812
+Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=64
+Observation.DataProducts.Output_Correlated_[0].duration=119.165420
+Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+Observation.DataProducts.Output_Correlated_[0].filename=L257915_SAP000_SB000_uv.MS
+Observation.DataProducts.Output_Correlated_[0].integrationInterval=1.001390
+Observation.DataProducts.Output_Correlated_[0].location=locus001:/data/L257915/
+Observation.DataProducts.Output_Correlated_[0].percentageWritten=100
+Observation.DataProducts.Output_Correlated_[0].size=268083200
+Observation.DataProducts.Output_Correlated_[0].startTime=2015-01-15 09:32:09
+Observation.DataProducts.Output_Correlated_[0].stationSubband=77
+Observation.DataProducts.Output_Correlated_[0].subband=0
+ </payload>
+</message>
diff --git a/LCS/MessageBus/data/feedback_dp_empty.txt b/LCS/MessageBus/data/feedback_dp_empty.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9c8bb9a44d8a26a87fea2ea595b24daeec526bc1
--- /dev/null
+++ b/LCS/MessageBus/data/feedback_dp_empty.txt
@@ -0,0 +1,23 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>lofar.observation.start</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>mySubSystem</name>
+ <user>user</user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>some test message</summary>
+ </source>
+ <ids>
+ <momid>12345</momid>
+ <sasid>44883</sasid>
+ </ids>
+ </header>
+ <payload>
+ </payload>
+</message>
diff --git a/LCS/MessageBus/data/feedback_dp_no_ids.txt b/LCS/MessageBus/data/feedback_dp_no_ids.txt
new file mode 100644
index 0000000000000000000000000000000000000000..101d55bcf0b4bc087b7e4cb04a5b1d330ece29ab
--- /dev/null
+++ b/LCS/MessageBus/data/feedback_dp_no_ids.txt
@@ -0,0 +1,40 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>lofar.observation.start</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>mySubSystem</name>
+ <user>user</user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>some test message</summary>
+ </source>
+ <ids>
+ <momid></momid>
+ <sasid></sasid>
+ </ids>
+ </header>
+ <payload>
+Observation.Correlator.channelWidth=3051.7578125
+Observation.Correlator.channelsPerSubband=64
+Observation.Correlator.integrationInterval=1.00139008
+Observation.DataProducts.Output_Correlated_[0].SAP=0
+Observation.DataProducts.Output_Correlated_[0].centralFrequency=115039062.500000
+Observation.DataProducts.Output_Correlated_[0].channelWidth=3051.757812
+Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=64
+Observation.DataProducts.Output_Correlated_[0].duration=119.165420
+Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+Observation.DataProducts.Output_Correlated_[0].filename=L257915_SAP000_SB000_uv.MS
+Observation.DataProducts.Output_Correlated_[0].integrationInterval=1.001390
+Observation.DataProducts.Output_Correlated_[0].location=locus001:/data/L257915/
+Observation.DataProducts.Output_Correlated_[0].percentageWritten=100
+Observation.DataProducts.Output_Correlated_[0].size=268083200
+Observation.DataProducts.Output_Correlated_[0].startTime=2015-01-15 09:32:09
+Observation.DataProducts.Output_Correlated_[0].stationSubband=77
+Observation.DataProducts.Output_Correlated_[0].subband=0
+ </payload>
+</message>
diff --git a/LCS/MessageBus/data/feedback_dp_xml_error.txt b/LCS/MessageBus/data/feedback_dp_xml_error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0e7ebf05ed491bfb68e6db14b3476647223bca9c
--- /dev/null
+++ b/LCS/MessageBus/data/feedback_dp_xml_error.txt
@@ -0,0 +1,40 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>lofar.observation.start</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>mySubSystem</name>
+ <user>user</user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>some test message</summary>
+ </source>
+ <ids>
+ <momid>12345</momid>
+ <sasid>44883</sasid>
+ </ids>
+ </header>
+ <payload_illegal_tag>
+Observation.Correlator.channelWidth=3051.7578125
+Observation.Correlator.channelsPerSubband=64
+Observation.Correlator.integrationInterval=1.00139008
+Observation.DataProducts.Output_Correlated_[0].SAP=0
+Observation.DataProducts.Output_Correlated_[0].centralFrequency=115039062.500000
+Observation.DataProducts.Output_Correlated_[0].channelWidth=3051.757812
+Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=64
+Observation.DataProducts.Output_Correlated_[0].duration=119.165420
+Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+Observation.DataProducts.Output_Correlated_[0].filename=L257915_SAP000_SB000_uv.MS
+Observation.DataProducts.Output_Correlated_[0].integrationInterval=1.001390
+Observation.DataProducts.Output_Correlated_[0].location=locus001:/data/L257915/
+Observation.DataProducts.Output_Correlated_[0].percentageWritten=100
+Observation.DataProducts.Output_Correlated_[0].size=268083200
+Observation.DataProducts.Output_Correlated_[0].startTime=2015-01-15 09:32:09
+Observation.DataProducts.Output_Correlated_[0].stationSubband=77
+Observation.DataProducts.Output_Correlated_[0].subband=0
+ </payload>
+</message>
diff --git a/LCS/MessageBus/data/task_spec_system_78958.txt b/LCS/MessageBus/data/task_spec_system_78958.txt
new file mode 100644
index 0000000000000000000000000000000000000000..04fa392015cd5eb81a5d8dbe1f4fc5107505ec0c
--- /dev/null
+++ b/LCS/MessageBus/data/task_spec_system_78958.txt
@@ -0,0 +1,308 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>task.specification.system</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>LOFAR.MACScheduler</name>
+ <user></user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary></summary>
+ </source>
+ <ids>
+ <momid>180785</momid>
+ <sasid>78958</sasid>
+ </ids>
+ </header>
+ <payload>
+Clock160.channelWidth=610.3515625
+Clock160.samplesPerSecond=155648
+Clock160.subbandWidth=156.250
+Clock160.systemClock=160
+Clock200.channelWidth=762.939453125
+Clock200.samplesPerSecond=196608
+Clock200.subbandWidth=195.3125
+Clock200.systemClock=200
+ObsSW.Observation.AnaBeam[0].angle1=0.5141401557033334
+ObsSW.Observation.AnaBeam[0].angle2=1.085720371180154
+ObsSW.Observation.AnaBeam[0].directionType=J2000
+ObsSW.Observation.AnaBeam[0].duration=1920
+ObsSW.Observation.AnaBeam[0].rank=1
+ObsSW.Observation.AnaBeam[0].startTime=
+ObsSW.Observation.AnaBeam[0].target=
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].absoluteAngle1=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].absoluteAngle2=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].angle1=0.514140156
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].angle2=1.085720371
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].coherent=true
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].directionType=J2000
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].dispersionMeasure=1.23456789
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].absoluteAngle1=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].absoluteAngle2=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].angle1=0.403039045
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].angle2=1.07461926
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].coherent=true
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].directionType=J2000
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].dispersionMeasure=1.1223344
+ObsSW.Observation.Beam[0].angle1=0.5141401557033334
+ObsSW.Observation.Beam[0].angle2=1.085720371180154
+ObsSW.Observation.Beam[0].directionType=J2000
+ObsSW.Observation.Beam[0].duration=1920
+ObsSW.Observation.Beam[0].momID=180786
+ObsSW.Observation.Beam[0].nrTabRings=3
+ObsSW.Observation.Beam[0].nrTiedArrayBeams=2
+ObsSW.Observation.Beam[0].startTime=
+ObsSW.Observation.Beam[0].subbandList=[200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443]
+ObsSW.Observation.Beam[0].tabRingSize=4.0
+ObsSW.Observation.Beam[0].target=BF Target
+ObsSW.Observation.Campaign.CO_I="Vermaas-Scientist, Nico"
+ObsSW.Observation.Campaign.PI="Verhoef, Ir. Bastiaan"
+ObsSW.Observation.Campaign.contact="Verhoef, Ir. Bastiaan"
+ObsSW.Observation.Campaign.name="test-lofar"
+ObsSW.Observation.Campaign.title="test-lofar"
+ObsSW.Observation.DataProducts.Input_CoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Input_CoherentStokes.enabled=false
+ObsSW.Observation.DataProducts.Input_CoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Input_CoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Input_Correlated.dirmask=
+ObsSW.Observation.DataProducts.Input_Correlated.enabled=false
+ObsSW.Observation.DataProducts.Input_Correlated.filenames=[]
+ObsSW.Observation.DataProducts.Input_Correlated.identifications=[]
+ObsSW.Observation.DataProducts.Input_Correlated.locations=[]
+ObsSW.Observation.DataProducts.Input_Correlated.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_Correlated.namemask=
+ObsSW.Observation.DataProducts.Input_Correlated.skip=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.enabled=false
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.dirmask=
+ObsSW.Observation.DataProducts.Input_InstrumentModel.enabled=false
+ObsSW.Observation.DataProducts.Input_InstrumentModel.filenames=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.identifications=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.locations=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.namemask=
+ObsSW.Observation.DataProducts.Input_InstrumentModel.skip=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.dirmask=
+ObsSW.Observation.DataProducts.Input_SkyImage.enabled=false
+ObsSW.Observation.DataProducts.Input_SkyImage.filenames=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.identifications=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.locations=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.namemask=
+ObsSW.Observation.DataProducts.Input_SkyImage.skip=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.archived=false
+ObsSW.Observation.DataProducts.Output_CoherentStokes.deleted=false
+ObsSW.Observation.DataProducts.Output_CoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Output_CoherentStokes.enabled=true
+ObsSW.Observation.DataProducts.Output_CoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Output_CoherentStokes.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.retentiontime=14
+ObsSW.Observation.DataProducts.Output_CoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Output_Correlated.archived=false
+ObsSW.Observation.DataProducts.Output_Correlated.deleted=false
+ObsSW.Observation.DataProducts.Output_Correlated.dirmask=
+ObsSW.Observation.DataProducts.Output_Correlated.enabled=false
+ObsSW.Observation.DataProducts.Output_Correlated.filenames=[]
+ObsSW.Observation.DataProducts.Output_Correlated.identifications=[]
+ObsSW.Observation.DataProducts.Output_Correlated.locations=[]
+ObsSW.Observation.DataProducts.Output_Correlated.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_Correlated.namemask=
+ObsSW.Observation.DataProducts.Output_Correlated.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_Correlated.retentiontime=14
+ObsSW.Observation.DataProducts.Output_Correlated.skip=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.archived=false
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.deleted=false
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.enabled=true
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.retentiontime=14
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.archived=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.deleted=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.dirmask=
+ObsSW.Observation.DataProducts.Output_InstrumentModel.enabled=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.filenames=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.identifications=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.locations=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.namemask=
+ObsSW.Observation.DataProducts.Output_InstrumentModel.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.retentiontime=14
+ObsSW.Observation.DataProducts.Output_InstrumentModel.skip=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.archived=false
+ObsSW.Observation.DataProducts.Output_Pulsar.deleted=false
+ObsSW.Observation.DataProducts.Output_Pulsar.dirmask=
+ObsSW.Observation.DataProducts.Output_Pulsar.enabled=false
+ObsSW.Observation.DataProducts.Output_Pulsar.filenames=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.identifications=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.locations=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.namemask=
+ObsSW.Observation.DataProducts.Output_Pulsar.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.retentiontime=14
+ObsSW.Observation.DataProducts.Output_Pulsar.skip=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.archived=false
+ObsSW.Observation.DataProducts.Output_SkyImage.deleted=false
+ObsSW.Observation.DataProducts.Output_SkyImage.dirmask=
+ObsSW.Observation.DataProducts.Output_SkyImage.enabled=false
+ObsSW.Observation.DataProducts.Output_SkyImage.filenames=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.identifications=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.locations=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.namemask=
+ObsSW.Observation.DataProducts.Output_SkyImage.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.retentiontime=14
+ObsSW.Observation.DataProducts.Output_SkyImage.skip=[]
+ObsSW.Observation.Dataslots.DataslotInfo.DataslotList=[]
+ObsSW.Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=0
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=243
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=15
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.which=I
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=2
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=241
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=17
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.which=IQUV
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.coherentDedisperseChannels=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.integrationTime=1.0
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrBlocksPerIntegration=1
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrChannelsPerSubband=16
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.blockSize=1
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.correctBandPass=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.correctClocks=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.delayCompensation=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.realTime=true
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._executable=CN_Processing
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._hostname=bgstfen
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._nodes=[]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._startstopType=bgl
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc.workingdir=/opt/lofar/bin/
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl._hostname=bgstfen
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.extraInfo=["OLAP","PIC"]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.procesOrder=[]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.processes=["CorrProc"]
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.channelsPerSubband=0
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.subbandsPerFile=243
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.timeIntegrationFactor=15
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.which=I
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.channelsPerSubband=2
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.subbandsPerFile=241
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.timeIntegrationFactor=17
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.which=IQUV
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.integrationTime=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.nrBlocksPerIntegration=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.nrChannelsPerSubband=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.PencilInfo.flysEye=false
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.coherentDedisperseChannels=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.correctBandPass=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.delayCompensation=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.realTime=true
+ObsSW.Observation.ObservationControl.OnlineControl._hostname=CCU001
+ObsSW.Observation.ObservationControl.OnlineControl.applOrder=["CorrAppl"]
+ObsSW.Observation.ObservationControl.OnlineControl.applications=["CorrAppl"]
+ObsSW.Observation.ObservationControl.OnlineControl.inspectionHost=lhn001.cep2.lofar
+ObsSW.Observation.ObservationControl.OnlineControl.inspectionProgram=/opt/cep/pyautoplot/bin/launch-msplots.sh
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.CoincidenceTime=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.DoDirectionFit=none
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.MaxFitVariance=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.MinElevation=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.NoCoincChann=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.ParamExtension=
+ObsSW.Observation.ObservationControl.StationControl._hostname=[DE602]
+ObsSW.Observation.ObservationControl.StationControl.tbbPiggybackAllowed=true
+ObsSW.Observation.ObservationControl._hostname=MCU001
+ObsSW.Observation.ObservationControl.heartbeatInterval=10
+ObsSW.Observation.Scheduler.contactEmail=
+ObsSW.Observation.Scheduler.contactName=
+ObsSW.Observation.Scheduler.contactPhone=
+ObsSW.Observation.Scheduler.firstPossibleDay=0
+ObsSW.Observation.Scheduler.fixedDay=false
+ObsSW.Observation.Scheduler.fixedTime=false
+ObsSW.Observation.Scheduler.lastPossibleDay=0
+ObsSW.Observation.Scheduler.late=false
+ObsSW.Observation.Scheduler.nightTimeWeightFactor=0
+ObsSW.Observation.Scheduler.predMaxTimeDif=
+ObsSW.Observation.Scheduler.predMinTimeDif=
+ObsSW.Observation.Scheduler.predecessors=[]
+ObsSW.Observation.Scheduler.priority=0.0
+ObsSW.Observation.Scheduler.reason=
+ObsSW.Observation.Scheduler.referenceFrame=0
+ObsSW.Observation.Scheduler.reservation=0
+ObsSW.Observation.Scheduler.storageSelectionMode=1
+ObsSW.Observation.Scheduler.taskDuration=1920
+ObsSW.Observation.Scheduler.taskID=0
+ObsSW.Observation.Scheduler.taskName=My Trigger Obs
+ObsSW.Observation.Scheduler.taskType=0
+ObsSW.Observation.Scheduler.windowMaximumTime=
+ObsSW.Observation.Scheduler.windowMinimumTime=
+ObsSW.Observation.TBB.TBBsetting.RCUs=[]
+ObsSW.Observation.TBB.TBBsetting.baselevel=127
+ObsSW.Observation.TBB.TBBsetting.filter=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff0=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff1=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff2=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff3=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff0=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff1=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff2=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff3=0
+ObsSW.Observation.TBB.TBBsetting.operatingMode=1
+ObsSW.Observation.TBB.TBBsetting.startlevel=7
+ObsSW.Observation.TBB.TBBsetting.stoplevel=7
+ObsSW.Observation.TBB.TBBsetting.subbandList=[]
+ObsSW.Observation.TBB.TBBsetting.triggerMode=1
+ObsSW.Observation.TBB.TBBsetting.window=1M
+ObsSW.Observation.VirtualInstrument.minimalNrStations=1
+ObsSW.Observation.VirtualInstrument.partitionList=["R00"]
+ObsSW.Observation.VirtualInstrument.stationList=[DE602]
+ObsSW.Observation.VirtualInstrument.stationSet=Custom
+ObsSW.Observation.antennaArray=HBA
+ObsSW.Observation.antennaSet=HBA_ZERO
+ObsSW.Observation.bandFilter=HBA_110_190
+ObsSW.Observation.channelsPerSubband=16
+ObsSW.Observation.claimPeriod=25
+ObsSW.Observation.clockMode=<<Clock200
+ObsSW.Observation.momID=180785
+ObsSW.Observation.nrAnaBeams=1
+ObsSW.Observation.nrBeams=1
+ObsSW.Observation.nrBitsPerSample=16
+ObsSW.Observation.nrTBBSettings=0
+ObsSW.Observation.originID=78795
+ObsSW.Observation.otdbID=78958
+ObsSW.Observation.preparePeriod=25
+ObsSW.Observation.processSubtype=Beam Observation
+ObsSW.Observation.processType=Observation
+ObsSW.Observation.sampleClock=200
+ObsSW.Observation.startTime=2015-02-04 11:27:00
+ObsSW.Observation.stopTime=2015-02-04 12:12:12
+ObsSW.Observation.strategy=default
+ObsSW.Observation.topologyID=
+Version.number=28583
+_DPname=LOFAR_ObsSW_TempObs0001
+prefix=LOFAR.
+
+ </payload>
+</message>
\ No newline at end of file
diff --git a/LCS/MessageBus/data/task_spec_system_empty.txt b/LCS/MessageBus/data/task_spec_system_empty.txt
new file mode 100644
index 0000000000000000000000000000000000000000..03504e63c63421be3960efdd089cb666558a4a62
--- /dev/null
+++ b/LCS/MessageBus/data/task_spec_system_empty.txt
@@ -0,0 +1,23 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>task.specification.system</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>LOFAR.MACScheduler</name>
+ <user></user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary></summary>
+ </source>
+ <ids>
+ <momid>180785</momid>
+ <sasid>78958</sasid>
+ </ids>
+ </header>
+ <payload>
+ </payload>
+</message>
diff --git a/LCS/MessageBus/data/task_spec_system_no_ids.txt b/LCS/MessageBus/data/task_spec_system_no_ids.txt
new file mode 100644
index 0000000000000000000000000000000000000000..edf31e7e7e59af0994dcc96293eea33e63ddbb96
--- /dev/null
+++ b/LCS/MessageBus/data/task_spec_system_no_ids.txt
@@ -0,0 +1,308 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>task.specification.system</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>LOFAR.MACScheduler</name>
+ <user></user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary></summary>
+ </source>
+ <ids>
+ <momid></momid>
+ <sasid></sasid>
+ </ids>
+ </header>
+ <payload>
+Clock160.channelWidth=610.3515625
+Clock160.samplesPerSecond=155648
+Clock160.subbandWidth=156.250
+Clock160.systemClock=160
+Clock200.channelWidth=762.939453125
+Clock200.samplesPerSecond=196608
+Clock200.subbandWidth=195.3125
+Clock200.systemClock=200
+ObsSW.Observation.AnaBeam[0].angle1=0.5141401557033334
+ObsSW.Observation.AnaBeam[0].angle2=1.085720371180154
+ObsSW.Observation.AnaBeam[0].directionType=J2000
+ObsSW.Observation.AnaBeam[0].duration=1920
+ObsSW.Observation.AnaBeam[0].rank=1
+ObsSW.Observation.AnaBeam[0].startTime=
+ObsSW.Observation.AnaBeam[0].target=
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].absoluteAngle1=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].absoluteAngle2=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].angle1=0.514140156
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].angle2=1.085720371
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].coherent=true
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].directionType=J2000
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].dispersionMeasure=1.23456789
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].absoluteAngle1=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].absoluteAngle2=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].angle1=0.403039045
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].angle2=1.07461926
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].coherent=true
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].directionType=J2000
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].dispersionMeasure=1.1223344
+ObsSW.Observation.Beam[0].angle1=0.5141401557033334
+ObsSW.Observation.Beam[0].angle2=1.085720371180154
+ObsSW.Observation.Beam[0].directionType=J2000
+ObsSW.Observation.Beam[0].duration=1920
+ObsSW.Observation.Beam[0].momID=180786
+ObsSW.Observation.Beam[0].nrTabRings=3
+ObsSW.Observation.Beam[0].nrTiedArrayBeams=2
+ObsSW.Observation.Beam[0].startTime=
+ObsSW.Observation.Beam[0].subbandList=[200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443]
+ObsSW.Observation.Beam[0].tabRingSize=4.0
+ObsSW.Observation.Beam[0].target=BF Target
+ObsSW.Observation.Campaign.CO_I="Vermaas-Scientist, Nico"
+ObsSW.Observation.Campaign.PI="Verhoef, Ir. Bastiaan"
+ObsSW.Observation.Campaign.contact="Verhoef, Ir. Bastiaan"
+ObsSW.Observation.Campaign.name="test-lofar"
+ObsSW.Observation.Campaign.title="test-lofar"
+ObsSW.Observation.DataProducts.Input_CoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Input_CoherentStokes.enabled=false
+ObsSW.Observation.DataProducts.Input_CoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Input_CoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Input_Correlated.dirmask=
+ObsSW.Observation.DataProducts.Input_Correlated.enabled=false
+ObsSW.Observation.DataProducts.Input_Correlated.filenames=[]
+ObsSW.Observation.DataProducts.Input_Correlated.identifications=[]
+ObsSW.Observation.DataProducts.Input_Correlated.locations=[]
+ObsSW.Observation.DataProducts.Input_Correlated.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_Correlated.namemask=
+ObsSW.Observation.DataProducts.Input_Correlated.skip=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.enabled=false
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.dirmask=
+ObsSW.Observation.DataProducts.Input_InstrumentModel.enabled=false
+ObsSW.Observation.DataProducts.Input_InstrumentModel.filenames=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.identifications=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.locations=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.namemask=
+ObsSW.Observation.DataProducts.Input_InstrumentModel.skip=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.dirmask=
+ObsSW.Observation.DataProducts.Input_SkyImage.enabled=false
+ObsSW.Observation.DataProducts.Input_SkyImage.filenames=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.identifications=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.locations=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.namemask=
+ObsSW.Observation.DataProducts.Input_SkyImage.skip=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.archived=false
+ObsSW.Observation.DataProducts.Output_CoherentStokes.deleted=false
+ObsSW.Observation.DataProducts.Output_CoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Output_CoherentStokes.enabled=true
+ObsSW.Observation.DataProducts.Output_CoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Output_CoherentStokes.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.retentiontime=14
+ObsSW.Observation.DataProducts.Output_CoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Output_Correlated.archived=false
+ObsSW.Observation.DataProducts.Output_Correlated.deleted=false
+ObsSW.Observation.DataProducts.Output_Correlated.dirmask=
+ObsSW.Observation.DataProducts.Output_Correlated.enabled=false
+ObsSW.Observation.DataProducts.Output_Correlated.filenames=[]
+ObsSW.Observation.DataProducts.Output_Correlated.identifications=[]
+ObsSW.Observation.DataProducts.Output_Correlated.locations=[]
+ObsSW.Observation.DataProducts.Output_Correlated.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_Correlated.namemask=
+ObsSW.Observation.DataProducts.Output_Correlated.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_Correlated.retentiontime=14
+ObsSW.Observation.DataProducts.Output_Correlated.skip=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.archived=false
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.deleted=false
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.enabled=true
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.retentiontime=14
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.archived=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.deleted=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.dirmask=
+ObsSW.Observation.DataProducts.Output_InstrumentModel.enabled=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.filenames=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.identifications=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.locations=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.namemask=
+ObsSW.Observation.DataProducts.Output_InstrumentModel.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.retentiontime=14
+ObsSW.Observation.DataProducts.Output_InstrumentModel.skip=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.archived=false
+ObsSW.Observation.DataProducts.Output_Pulsar.deleted=false
+ObsSW.Observation.DataProducts.Output_Pulsar.dirmask=
+ObsSW.Observation.DataProducts.Output_Pulsar.enabled=false
+ObsSW.Observation.DataProducts.Output_Pulsar.filenames=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.identifications=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.locations=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.namemask=
+ObsSW.Observation.DataProducts.Output_Pulsar.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.retentiontime=14
+ObsSW.Observation.DataProducts.Output_Pulsar.skip=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.archived=false
+ObsSW.Observation.DataProducts.Output_SkyImage.deleted=false
+ObsSW.Observation.DataProducts.Output_SkyImage.dirmask=
+ObsSW.Observation.DataProducts.Output_SkyImage.enabled=false
+ObsSW.Observation.DataProducts.Output_SkyImage.filenames=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.identifications=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.locations=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.namemask=
+ObsSW.Observation.DataProducts.Output_SkyImage.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.retentiontime=14
+ObsSW.Observation.DataProducts.Output_SkyImage.skip=[]
+ObsSW.Observation.Dataslots.DataslotInfo.DataslotList=[]
+ObsSW.Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=0
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=243
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=15
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.which=I
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=2
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=241
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=17
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.which=IQUV
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.coherentDedisperseChannels=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.integrationTime=1.0
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrBlocksPerIntegration=1
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrChannelsPerSubband=16
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.blockSize=1
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.correctBandPass=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.correctClocks=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.delayCompensation=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.realTime=true
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._executable=CN_Processing
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._hostname=bgstfen
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._nodes=[]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._startstopType=bgl
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc.workingdir=/opt/lofar/bin/
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl._hostname=bgstfen
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.extraInfo=["OLAP","PIC"]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.procesOrder=[]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.processes=["CorrProc"]
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.channelsPerSubband=0
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.subbandsPerFile=243
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.timeIntegrationFactor=15
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.which=I
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.channelsPerSubband=2
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.subbandsPerFile=241
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.timeIntegrationFactor=17
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.which=IQUV
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.integrationTime=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.nrBlocksPerIntegration=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.nrChannelsPerSubband=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.PencilInfo.flysEye=false
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.coherentDedisperseChannels=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.correctBandPass=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.delayCompensation=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.realTime=true
+ObsSW.Observation.ObservationControl.OnlineControl._hostname=CCU001
+ObsSW.Observation.ObservationControl.OnlineControl.applOrder=["CorrAppl"]
+ObsSW.Observation.ObservationControl.OnlineControl.applications=["CorrAppl"]
+ObsSW.Observation.ObservationControl.OnlineControl.inspectionHost=lhn001.cep2.lofar
+ObsSW.Observation.ObservationControl.OnlineControl.inspectionProgram=/opt/cep/pyautoplot/bin/launch-msplots.sh
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.CoincidenceTime=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.DoDirectionFit=none
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.MaxFitVariance=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.MinElevation=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.NoCoincChann=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.ParamExtension=
+ObsSW.Observation.ObservationControl.StationControl._hostname=[DE602]
+ObsSW.Observation.ObservationControl.StationControl.tbbPiggybackAllowed=true
+ObsSW.Observation.ObservationControl._hostname=MCU001
+ObsSW.Observation.ObservationControl.heartbeatInterval=10
+ObsSW.Observation.Scheduler.contactEmail=
+ObsSW.Observation.Scheduler.contactName=
+ObsSW.Observation.Scheduler.contactPhone=
+ObsSW.Observation.Scheduler.firstPossibleDay=0
+ObsSW.Observation.Scheduler.fixedDay=false
+ObsSW.Observation.Scheduler.fixedTime=false
+ObsSW.Observation.Scheduler.lastPossibleDay=0
+ObsSW.Observation.Scheduler.late=false
+ObsSW.Observation.Scheduler.nightTimeWeightFactor=0
+ObsSW.Observation.Scheduler.predMaxTimeDif=
+ObsSW.Observation.Scheduler.predMinTimeDif=
+ObsSW.Observation.Scheduler.predecessors=[]
+ObsSW.Observation.Scheduler.priority=0.0
+ObsSW.Observation.Scheduler.reason=
+ObsSW.Observation.Scheduler.referenceFrame=0
+ObsSW.Observation.Scheduler.reservation=0
+ObsSW.Observation.Scheduler.storageSelectionMode=1
+ObsSW.Observation.Scheduler.taskDuration=1920
+ObsSW.Observation.Scheduler.taskID=0
+ObsSW.Observation.Scheduler.taskName=My Trigger Obs
+ObsSW.Observation.Scheduler.taskType=0
+ObsSW.Observation.Scheduler.windowMaximumTime=
+ObsSW.Observation.Scheduler.windowMinimumTime=
+ObsSW.Observation.TBB.TBBsetting.RCUs=[]
+ObsSW.Observation.TBB.TBBsetting.baselevel=127
+ObsSW.Observation.TBB.TBBsetting.filter=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff0=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff1=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff2=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff3=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff0=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff1=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff2=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff3=0
+ObsSW.Observation.TBB.TBBsetting.operatingMode=1
+ObsSW.Observation.TBB.TBBsetting.startlevel=7
+ObsSW.Observation.TBB.TBBsetting.stoplevel=7
+ObsSW.Observation.TBB.TBBsetting.subbandList=[]
+ObsSW.Observation.TBB.TBBsetting.triggerMode=1
+ObsSW.Observation.TBB.TBBsetting.window=1M
+ObsSW.Observation.VirtualInstrument.minimalNrStations=1
+ObsSW.Observation.VirtualInstrument.partitionList=["R00"]
+ObsSW.Observation.VirtualInstrument.stationList=[DE602]
+ObsSW.Observation.VirtualInstrument.stationSet=Custom
+ObsSW.Observation.antennaArray=HBA
+ObsSW.Observation.antennaSet=HBA_ZERO
+ObsSW.Observation.bandFilter=HBA_110_190
+ObsSW.Observation.channelsPerSubband=16
+ObsSW.Observation.claimPeriod=25
+ObsSW.Observation.clockMode=<<Clock200
+ObsSW.Observation.momID=180785
+ObsSW.Observation.nrAnaBeams=1
+ObsSW.Observation.nrBeams=1
+ObsSW.Observation.nrBitsPerSample=16
+ObsSW.Observation.nrTBBSettings=0
+ObsSW.Observation.originID=78795
+ObsSW.Observation.otdbID=78958
+ObsSW.Observation.preparePeriod=25
+ObsSW.Observation.processSubtype=Beam Observation
+ObsSW.Observation.processType=Observation
+ObsSW.Observation.sampleClock=200
+ObsSW.Observation.startTime=2015-02-04 11:27:00
+ObsSW.Observation.stopTime=2015-02-04 12:12:12
+ObsSW.Observation.strategy=default
+ObsSW.Observation.topologyID=
+Version.number=28583
+_DPname=LOFAR_ObsSW_TempObs0001
+prefix=LOFAR.
+
+ </payload>
+</message>
diff --git a/LCS/MessageBus/data/task_spec_system_xml_error.txt b/LCS/MessageBus/data/task_spec_system_xml_error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7db2d5a631ae5c68d2c3ffbc85bdc97a4f3846ef
--- /dev/null
+++ b/LCS/MessageBus/data/task_spec_system_xml_error.txt
@@ -0,0 +1,308 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>task.specification.system</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>LOFAR.MACScheduler</name>
+ <user></user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary></summary>
+ </source>
+ <ids>
+ <momid>180785</momid>
+ <sasid>78958</sasid>
+ </ids>
+ </header>
+ <wrong_payload_tag>
+Clock160.channelWidth=610.3515625
+Clock160.samplesPerSecond=155648
+Clock160.subbandWidth=156.250
+Clock160.systemClock=160
+Clock200.channelWidth=762.939453125
+Clock200.samplesPerSecond=196608
+Clock200.subbandWidth=195.3125
+Clock200.systemClock=200
+ObsSW.Observation.AnaBeam[0].angle1=0.5141401557033334
+ObsSW.Observation.AnaBeam[0].angle2=1.085720371180154
+ObsSW.Observation.AnaBeam[0].directionType=J2000
+ObsSW.Observation.AnaBeam[0].duration=1920
+ObsSW.Observation.AnaBeam[0].rank=1
+ObsSW.Observation.AnaBeam[0].startTime=
+ObsSW.Observation.AnaBeam[0].target=
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].absoluteAngle1=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].absoluteAngle2=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].angle1=0.514140156
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].angle2=1.085720371
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].coherent=true
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].directionType=J2000
+ObsSW.Observation.Beam[0].TiedArrayBeam[0].dispersionMeasure=1.23456789
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].absoluteAngle1=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].absoluteAngle2=0
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].angle1=0.403039045
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].angle2=1.07461926
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].coherent=true
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].directionType=J2000
+ObsSW.Observation.Beam[0].TiedArrayBeam[1].dispersionMeasure=1.1223344
+ObsSW.Observation.Beam[0].angle1=0.5141401557033334
+ObsSW.Observation.Beam[0].angle2=1.085720371180154
+ObsSW.Observation.Beam[0].directionType=J2000
+ObsSW.Observation.Beam[0].duration=1920
+ObsSW.Observation.Beam[0].momID=180786
+ObsSW.Observation.Beam[0].nrTabRings=3
+ObsSW.Observation.Beam[0].nrTiedArrayBeams=2
+ObsSW.Observation.Beam[0].startTime=
+ObsSW.Observation.Beam[0].subbandList=[200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443]
+ObsSW.Observation.Beam[0].tabRingSize=4.0
+ObsSW.Observation.Beam[0].target=BF Target
+ObsSW.Observation.Campaign.CO_I="Vermaas-Scientist, Nico"
+ObsSW.Observation.Campaign.PI="Verhoef, Ir. Bastiaan"
+ObsSW.Observation.Campaign.contact="Verhoef, Ir. Bastiaan"
+ObsSW.Observation.Campaign.name="test-lofar"
+ObsSW.Observation.Campaign.title="test-lofar"
+ObsSW.Observation.DataProducts.Input_CoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Input_CoherentStokes.enabled=false
+ObsSW.Observation.DataProducts.Input_CoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_CoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Input_CoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Input_Correlated.dirmask=
+ObsSW.Observation.DataProducts.Input_Correlated.enabled=false
+ObsSW.Observation.DataProducts.Input_Correlated.filenames=[]
+ObsSW.Observation.DataProducts.Input_Correlated.identifications=[]
+ObsSW.Observation.DataProducts.Input_Correlated.locations=[]
+ObsSW.Observation.DataProducts.Input_Correlated.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_Correlated.namemask=
+ObsSW.Observation.DataProducts.Input_Correlated.skip=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.enabled=false
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Input_IncoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.dirmask=
+ObsSW.Observation.DataProducts.Input_InstrumentModel.enabled=false
+ObsSW.Observation.DataProducts.Input_InstrumentModel.filenames=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.identifications=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.locations=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_InstrumentModel.namemask=
+ObsSW.Observation.DataProducts.Input_InstrumentModel.skip=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.dirmask=
+ObsSW.Observation.DataProducts.Input_SkyImage.enabled=false
+ObsSW.Observation.DataProducts.Input_SkyImage.filenames=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.identifications=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.locations=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.mountpoints=[]
+ObsSW.Observation.DataProducts.Input_SkyImage.namemask=
+ObsSW.Observation.DataProducts.Input_SkyImage.skip=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.archived=false
+ObsSW.Observation.DataProducts.Output_CoherentStokes.deleted=false
+ObsSW.Observation.DataProducts.Output_CoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Output_CoherentStokes.enabled=true
+ObsSW.Observation.DataProducts.Output_CoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Output_CoherentStokes.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_CoherentStokes.retentiontime=14
+ObsSW.Observation.DataProducts.Output_CoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Output_Correlated.archived=false
+ObsSW.Observation.DataProducts.Output_Correlated.deleted=false
+ObsSW.Observation.DataProducts.Output_Correlated.dirmask=
+ObsSW.Observation.DataProducts.Output_Correlated.enabled=false
+ObsSW.Observation.DataProducts.Output_Correlated.filenames=[]
+ObsSW.Observation.DataProducts.Output_Correlated.identifications=[]
+ObsSW.Observation.DataProducts.Output_Correlated.locations=[]
+ObsSW.Observation.DataProducts.Output_Correlated.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_Correlated.namemask=
+ObsSW.Observation.DataProducts.Output_Correlated.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_Correlated.retentiontime=14
+ObsSW.Observation.DataProducts.Output_Correlated.skip=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.archived=false
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.deleted=false
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.dirmask=
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.enabled=true
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.filenames=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.identifications=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.locations=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.namemask=
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.retentiontime=14
+ObsSW.Observation.DataProducts.Output_IncoherentStokes.skip=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.archived=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.deleted=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.dirmask=
+ObsSW.Observation.DataProducts.Output_InstrumentModel.enabled=false
+ObsSW.Observation.DataProducts.Output_InstrumentModel.filenames=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.identifications=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.locations=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.namemask=
+ObsSW.Observation.DataProducts.Output_InstrumentModel.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_InstrumentModel.retentiontime=14
+ObsSW.Observation.DataProducts.Output_InstrumentModel.skip=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.archived=false
+ObsSW.Observation.DataProducts.Output_Pulsar.deleted=false
+ObsSW.Observation.DataProducts.Output_Pulsar.dirmask=
+ObsSW.Observation.DataProducts.Output_Pulsar.enabled=false
+ObsSW.Observation.DataProducts.Output_Pulsar.filenames=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.identifications=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.locations=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.namemask=
+ObsSW.Observation.DataProducts.Output_Pulsar.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_Pulsar.retentiontime=14
+ObsSW.Observation.DataProducts.Output_Pulsar.skip=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.archived=false
+ObsSW.Observation.DataProducts.Output_SkyImage.deleted=false
+ObsSW.Observation.DataProducts.Output_SkyImage.dirmask=
+ObsSW.Observation.DataProducts.Output_SkyImage.enabled=false
+ObsSW.Observation.DataProducts.Output_SkyImage.filenames=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.identifications=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.locations=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.mountpoints=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.namemask=
+ObsSW.Observation.DataProducts.Output_SkyImage.percentageWritten=[]
+ObsSW.Observation.DataProducts.Output_SkyImage.retentiontime=14
+ObsSW.Observation.DataProducts.Output_SkyImage.skip=[]
+ObsSW.Observation.Dataslots.DataslotInfo.DataslotList=[]
+ObsSW.Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=0
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=243
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=15
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.which=I
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=2
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=241
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=17
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.which=IQUV
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.coherentDedisperseChannels=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.integrationTime=1.0
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrBlocksPerIntegration=1
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrChannelsPerSubband=16
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.blockSize=1
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.correctBandPass=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.correctClocks=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.delayCompensation=true
+ObsSW.Observation.ObservationControl.OnlineControl.Cobalt.realTime=true
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._executable=CN_Processing
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._hostname=bgstfen
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._nodes=[]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc._startstopType=bgl
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.CorrProc.workingdir=/opt/lofar/bin/
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl._hostname=bgstfen
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.extraInfo=["OLAP","PIC"]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.procesOrder=[]
+ObsSW.Observation.ObservationControl.OnlineControl.CorrAppl.processes=["CorrProc"]
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.channelsPerSubband=0
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.subbandsPerFile=243
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.timeIntegrationFactor=15
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_CoherentStokes.which=I
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.channelsPerSubband=2
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.subbandsPerFile=241
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.timeIntegrationFactor=17
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.CNProc_IncoherentStokes.which=IQUV
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.integrationTime=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.nrBlocksPerIntegration=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.Correlator.nrChannelsPerSubband=1
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.PencilInfo.flysEye=false
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.coherentDedisperseChannels=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.correctBandPass=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.delayCompensation=true
+ObsSW.Observation.ObservationControl.OnlineControl.OLAP.realTime=true
+ObsSW.Observation.ObservationControl.OnlineControl._hostname=CCU001
+ObsSW.Observation.ObservationControl.OnlineControl.applOrder=["CorrAppl"]
+ObsSW.Observation.ObservationControl.OnlineControl.applications=["CorrAppl"]
+ObsSW.Observation.ObservationControl.OnlineControl.inspectionHost=lhn001.cep2.lofar
+ObsSW.Observation.ObservationControl.OnlineControl.inspectionProgram=/opt/cep/pyautoplot/bin/launch-msplots.sh
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.CoincidenceTime=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.DoDirectionFit=none
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.MaxFitVariance=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.MinElevation=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.NoCoincChann=0
+ObsSW.Observation.ObservationControl.StationControl.TBBControl.ParamExtension=
+ObsSW.Observation.ObservationControl.StationControl._hostname=[DE602]
+ObsSW.Observation.ObservationControl.StationControl.tbbPiggybackAllowed=true
+ObsSW.Observation.ObservationControl._hostname=MCU001
+ObsSW.Observation.ObservationControl.heartbeatInterval=10
+ObsSW.Observation.Scheduler.contactEmail=
+ObsSW.Observation.Scheduler.contactName=
+ObsSW.Observation.Scheduler.contactPhone=
+ObsSW.Observation.Scheduler.firstPossibleDay=0
+ObsSW.Observation.Scheduler.fixedDay=false
+ObsSW.Observation.Scheduler.fixedTime=false
+ObsSW.Observation.Scheduler.lastPossibleDay=0
+ObsSW.Observation.Scheduler.late=false
+ObsSW.Observation.Scheduler.nightTimeWeightFactor=0
+ObsSW.Observation.Scheduler.predMaxTimeDif=
+ObsSW.Observation.Scheduler.predMinTimeDif=
+ObsSW.Observation.Scheduler.predecessors=[]
+ObsSW.Observation.Scheduler.priority=0.0
+ObsSW.Observation.Scheduler.reason=
+ObsSW.Observation.Scheduler.referenceFrame=0
+ObsSW.Observation.Scheduler.reservation=0
+ObsSW.Observation.Scheduler.storageSelectionMode=1
+ObsSW.Observation.Scheduler.taskDuration=1920
+ObsSW.Observation.Scheduler.taskID=0
+ObsSW.Observation.Scheduler.taskName=My Trigger Obs
+ObsSW.Observation.Scheduler.taskType=0
+ObsSW.Observation.Scheduler.windowMaximumTime=
+ObsSW.Observation.Scheduler.windowMinimumTime=
+ObsSW.Observation.TBB.TBBsetting.RCUs=[]
+ObsSW.Observation.TBB.TBBsetting.baselevel=127
+ObsSW.Observation.TBB.TBBsetting.filter=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff0=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff1=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff2=0
+ObsSW.Observation.TBB.TBBsetting.filter0_coeff3=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff0=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff1=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff2=0
+ObsSW.Observation.TBB.TBBsetting.filter1_coeff3=0
+ObsSW.Observation.TBB.TBBsetting.operatingMode=1
+ObsSW.Observation.TBB.TBBsetting.startlevel=7
+ObsSW.Observation.TBB.TBBsetting.stoplevel=7
+ObsSW.Observation.TBB.TBBsetting.subbandList=[]
+ObsSW.Observation.TBB.TBBsetting.triggerMode=1
+ObsSW.Observation.TBB.TBBsetting.window=1M
+ObsSW.Observation.VirtualInstrument.minimalNrStations=1
+ObsSW.Observation.VirtualInstrument.partitionList=["R00"]
+ObsSW.Observation.VirtualInstrument.stationList=[DE602]
+ObsSW.Observation.VirtualInstrument.stationSet=Custom
+ObsSW.Observation.antennaArray=HBA
+ObsSW.Observation.antennaSet=HBA_ZERO
+ObsSW.Observation.bandFilter=HBA_110_190
+ObsSW.Observation.channelsPerSubband=16
+ObsSW.Observation.claimPeriod=25
+ObsSW.Observation.clockMode=<<Clock200
+ObsSW.Observation.momID=180785
+ObsSW.Observation.nrAnaBeams=1
+ObsSW.Observation.nrBeams=1
+ObsSW.Observation.nrBitsPerSample=16
+ObsSW.Observation.nrTBBSettings=0
+ObsSW.Observation.originID=78795
+ObsSW.Observation.otdbID=78958
+ObsSW.Observation.preparePeriod=25
+ObsSW.Observation.processSubtype=Beam Observation
+ObsSW.Observation.processType=Observation
+ObsSW.Observation.sampleClock=200
+ObsSW.Observation.startTime=2015-02-04 11:27:00
+ObsSW.Observation.stopTime=2015-02-04 12:12:12
+ObsSW.Observation.strategy=default
+ObsSW.Observation.topologyID=
+Version.number=28583
+_DPname=LOFAR_ObsSW_TempObs0001
+prefix=LOFAR.
+
+ </payload>
+</message>
diff --git a/LCS/MessageBus/include/MessageBus/CMakeLists.txt b/LCS/MessageBus/include/MessageBus/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1e71faa1a0f684b4f4911acf23d82d5c5f708723
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/CMakeLists.txt
@@ -0,0 +1,13 @@
+# $Id$
+
+# Create symbolic link to include directory.
+execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
+ ${CMAKE_CURRENT_SOURCE_DIR}
+ ${CMAKE_BINARY_DIR}/include/${PACKAGE_NAME})
+
+# Install header files.
+install(FILES
+ MsgBus.h
+ DESTINATION include/${PACKAGE_NAME})
+
+add_subdirectory(Protocols)
diff --git a/LCS/MessageBus/include/MessageBus/Message.h b/LCS/MessageBus/include/MessageBus/Message.h
new file mode 100644
index 0000000000000000000000000000000000000000..1117ac5545c76d88e1bcbc1d10339ac3ab214120
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/Message.h
@@ -0,0 +1,130 @@
+//# Message.h: Wrapper for messages to be exchanged using QPID.
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_MESSAGEBUS_MESSAGE_H
+#define LOFAR_MESSAGEBUS_MESSAGE_H
+
+#ifdef HAVE_QPID
+#include <qpid/messaging/Message.h>
+#include <qpid/types/Variant.h>
+
+#include <string>
+#include <ostream>
+
+namespace LOFAR {
+
+// Name of the system sending these messages
+static const std::string system = "LOFAR";
+
+// Version of the header we write
+static const std::string headerVersion = "1.0.0";
+
+class Message
+{
+public:
+ // Construct a message
+ Message() {};
+
+ // With header info
+ Message(
+ // Name of the service or process producing this message
+ const std::string &from,
+
+ // End-user responsible for this request (if applicable)
+ const std::string &forUser,
+
+ // Human-readable summary describing this request
+ const std::string &summary,
+
+ // Protocol of the contents of the message
+ const std::string &protocol,
+
+ // Version of the protocol we're using to describe the payload
+ const std::string &protocolVersion,
+
+ // momID of the information (if available)
+ const std::string &momID,
+
+ // sasID of the information (if available)
+ const std::string &sasID
+ );
+
+ // Parse a message
+ Message(const qpid::messaging::Message qpidMsg) : itsQpidMsg(qpidMsg) {};
+
+ // Read a message from disk (header + payload)
+ Message(const std::string &rawContent);
+
+ // Set the payload, supporting various types
+ void setXMLPayload (const std::string &payload);
+ void setTXTPayload (const std::string &payload);
+ void setMapPayload (const qpid::types::Variant::Map &payload);
+ void setListPayload(const qpid::types::Variant::List &payload);
+
+ virtual ~Message();
+
+ // Return properties of the constructed or received message
+ std::string system() const { return (getXMLvalue("message.header.system")); }
+ std::string headerVersion() const { return (getXMLvalue("message.header.version")); }
+ std::string protocol() const { return (getXMLvalue("message.header.protocol.name")); }
+ std::string protocolVersion() const { return (getXMLvalue("message.header.protocol.version")); }
+ std::string from() const { return (getXMLvalue("message.header.source.name")); }
+ std::string forUser() const { return (getXMLvalue("message.header.source.user")); }
+ std::string uuid() const { return (getXMLvalue("message.header.source.uuid")); }
+ std::string summary() const { return (getXMLvalue("message.header.source.summary")); }
+ std::string timestamp() const { return (getXMLvalue("message.header.source.timestamp")); }
+ std::string momid() const { return (getXMLvalue("message.header.ids.momid")); }
+ std::string sasid() const { return (getXMLvalue("message.header.ids.sasid")); }
+ std::string payload() const { return (getXMLvalue("message.payload")); }
+ std::string header() const { return (getXMLvalue("message.header")); }
+
+ // Construct the given fields as a QPID message
+ qpid::messaging::Message& qpidMsg() { return (itsQpidMsg); }
+
+ // Return the raw message (header + payload)
+ std::string rawContent() const { return (itsQpidMsg.getContent()); }
+
+ // Return a short (one line) description of the message
+ std::string short_desc() const;
+
+ // function for printing
+ std::ostream& print (std::ostream& os) const;
+
+ // Very simple XML parser to get a key from the XML content.
+ std::string getXMLvalue(const std::string& key) const;
+
+private:
+ // -- datamembers --
+ qpid::messaging::Message itsQpidMsg;
+};
+
+inline std::ostream &operator<<(std::ostream &os, const Message &msg)
+{
+ return (msg.print(os));
+}
+
+} // namespace LOFAR
+
+#endif
+
+#endif
+
diff --git a/LCS/MessageBus/include/MessageBus/MsgBus.h b/LCS/MessageBus/include/MessageBus/MsgBus.h
new file mode 100644
index 0000000000000000000000000000000000000000..849c43bad7fe060c9db19ebf47ae8dd56835cca3
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/MsgBus.h
@@ -0,0 +1,90 @@
+//# MsgBus.h: Wrapper for QPID clients to send and receive AMQP messages.
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_MESSAGEBUS_MSGBUS_H
+#define LOFAR_MESSAGEBUS_MSGBUS_H
+
+#ifdef HAVE_QPID
+#include <qpid/messaging/Connection.h>
+#include <qpid/messaging/Message.h>
+#include <qpid/messaging/Receiver.h>
+#include <qpid/messaging/Sender.h>
+#include <qpid/messaging/Session.h>
+#include <qpid/messaging/Address.h>
+
+#include <Common/Exception.h>
+#include <MessageBus/Message.h>
+
+#include <map>
+#include <string>
+
+namespace LOFAR {
+
+EXCEPTION_CLASS(MessageBusException, LOFAR::Exception);
+
+namespace MessageBus {
+ // Generic initialisation of the Messaging framework
+ void init();
+}
+
+class FromBus
+{
+public:
+ FromBus(const std::string &address="testqueue" , const std::string &options="; {create: never}", const std::string &broker = "amqp:tcp:127.0.0.1:5672") ;
+ ~FromBus(void);
+ bool addQueue(const std::string &address="testqueue", const std::string &options="; {create: never}");
+
+ bool getMessage(LOFAR::Message &msg, double timeout = 0.0); // timeout 0.0 means blocking
+
+ void ack (LOFAR::Message &msg);
+ void nack (LOFAR::Message &msg);
+ void reject(LOFAR::Message &msg);
+
+private:
+ qpid::messaging::Connection itsConnection;
+ qpid::messaging::Session itsSession;
+
+ int itsNrMissingACKs;
+};
+
+class ToBus
+{
+public:
+ ToBus(const std::string &address="testqueue" , const std::string &options="; {create: never}", const std::string &broker = "amqp:tcp:127.0.0.1:5672") ;
+ ~ToBus(void);
+
+ void send(const std::string &msg);
+ void send(LOFAR::Message &msg);
+
+private:
+ // -- datamambers --
+ qpid::messaging::Connection itsConnection;
+ qpid::messaging::Session itsSession;
+ qpid::messaging::Sender itsSender;
+};
+
+} // namespace LOFAR
+
+#endif
+
+#endif
+
diff --git a/LCS/MessageBus/include/MessageBus/Protocols/CMakeLists.txt b/LCS/MessageBus/include/MessageBus/Protocols/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..09c3af9f8cb3681828a27571c6429bd60acccf2c
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/Protocols/CMakeLists.txt
@@ -0,0 +1,14 @@
+# $Id$
+
+# Create symbolic link to include directory.
+execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
+ ${CMAKE_CURRENT_SOURCE_DIR}
+ ${CMAKE_BINARY_DIR}/include/${PACKAGE_NAME}/Protocols)
+
+# Install header files.
+install(FILES
+ TaskFeedbackDataproducts.h
+ TaskFeedbackProcessing.h
+ TaskFeedbackState.h
+ TaskSpecificationSystem.h
+ DESTINATION include/${PACKAGE_NAME}/Protocols)
diff --git a/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackDataproducts.h b/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackDataproducts.h
new file mode 100644
index 0000000000000000000000000000000000000000..0c43c1ae6892b44a1300a6745c69f8bac6b26ee3
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackDataproducts.h
@@ -0,0 +1,98 @@
+//# TaskFeedbackDataproducts.h: Protocol for emission of dataproducts feedback information
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_MESSAGEBUS_TASK_FEEDBACK_DATAPRODUCTS_H
+#define LOFAR_MESSAGEBUS_TASK_FEEDBACK_DATAPRODUCTS_H
+
+#ifdef HAVE_QPID
+#include <MessageBus/Message.h>
+#include <Common/ParameterSet.h>
+
+namespace LOFAR {
+
+namespace Protocols {
+
+class TaskFeedbackDataproducts: public Message
+{
+public:
+ TaskFeedbackDataproducts(
+ // Name of the service or process producing this message
+ const std::string &from,
+
+ // End-user responsible for this request (if applicable)
+ const std::string &forUser,
+
+ // Human-readable summary describing this request
+ const std::string &summary,
+
+ // Identifiers for the context of this message
+ const std::string &momID,
+ const std::string &sasID,
+
+ // Payload: a parset containing the generated feedback
+ const ParameterSet &feedback
+ ):
+ Message(
+ from,
+ forUser,
+ summary,
+ "task.feedback.dataproducts",
+ "1.0.0",
+ momID,
+ sasID)
+ {
+ std::string buffer;
+
+ feedback.writeBuffer(buffer);
+ setTXTPayload(buffer);
+ }
+
+ // Parse a message
+ TaskFeedbackDataproducts(const qpid::messaging::Message qpidMsg)
+ :
+ Message(qpidMsg)
+ {
+ }
+
+ // Read a message from disk (header + payload)
+ TaskFeedbackDataproducts(const std::string &rawContent)
+ :
+ Message(rawContent)
+ {
+ }
+
+ ParameterSet feedback() const {
+ ParameterSet result;
+ result.adoptBuffer(payload());
+
+ return result;
+ }
+};
+
+} // namespace Protocols
+
+} // namespace LOFAR
+
+#endif
+
+#endif
+
diff --git a/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackProcessing.h b/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackProcessing.h
new file mode 100644
index 0000000000000000000000000000000000000000..4fcec26201f88f26089f41653516944c66ec04a4
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackProcessing.h
@@ -0,0 +1,98 @@
+//# TaskFeedbackProcessing.h: Protocol for emission of processing feedback information
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_MESSAGEBUS_TASK_FEEDBACK_PROCESSING_H
+#define LOFAR_MESSAGEBUS_TASK_FEEDBACK_PROCESSING_H
+
+#ifdef HAVE_QPID
+#include <MessageBus/Message.h>
+#include <Common/ParameterSet.h>
+
+namespace LOFAR {
+
+namespace Protocols {
+
+class TaskFeedbackProcessing: public Message
+{
+public:
+ TaskFeedbackProcessing(
+ // Name of the service or process producing this message
+ const std::string &from,
+
+ // End-user responsible for this request (if applicable)
+ const std::string &forUser,
+
+ // Human-readable summary describing this request
+ const std::string &summary,
+
+ // Identifiers for the context of this message
+ const std::string &momID,
+ const std::string &sasID,
+
+ // Payload: a parset containing the generated feedback
+ const ParameterSet &feedback
+ ):
+ Message(
+ from,
+ forUser,
+ summary,
+ "task.feedback.processing",
+ "1.0.0",
+ momID,
+ sasID)
+ {
+ std::string buffer;
+
+ feedback.writeBuffer(buffer);
+ setTXTPayload(buffer);
+ }
+
+ // Parse a message
+ TaskFeedbackProcessing(const qpid::messaging::Message qpidMsg)
+ :
+ Message(qpidMsg)
+ {
+ }
+
+ // Read a message from disk (header + payload)
+ TaskFeedbackProcessing(const std::string &rawContent)
+ :
+ Message(rawContent)
+ {
+ }
+
+ ParameterSet feedback() const {
+ ParameterSet result;
+ result.adoptBuffer(payload());
+
+ return result;
+ }
+};
+
+} // namespace Protocols
+
+} // namespace LOFAR
+
+#endif
+
+#endif
+
diff --git a/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackState.h b/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackState.h
new file mode 100644
index 0000000000000000000000000000000000000000..5777f4f6fd82dbc40ca3a3adae27f6360f21727e
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/Protocols/TaskFeedbackState.h
@@ -0,0 +1,101 @@
+//# TaskFeedbackState.h: Protocol for emission of status feedback information
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_MESSAGEBUS_TASK_FEEDBACK_STATUS_H
+#define LOFAR_MESSAGEBUS_TASK_FEEDBACK_STATUS_H
+
+#ifdef HAVE_QPID
+#include <MessageBus/Message.h>
+#include <Common/ParameterSet.h>
+#include <Common/StringUtil.h>
+
+namespace LOFAR {
+
+namespace Protocols {
+
+class TaskFeedbackState: public Message
+{
+public:
+ TaskFeedbackState(
+ // Name of the service or process producing this message
+ const std::string &from,
+
+ // End-user responsible for this request (if applicable)
+ const std::string &forUser,
+
+ // Human-readable summary describing this request
+ const std::string &summary,
+
+ // Identifiers for the context of this message
+ const std::string &momID,
+ const std::string &sasID,
+
+ // Payload: a boolean indicating success
+ bool success
+ ):
+ Message(
+ from,
+ forUser,
+ summary,
+ "task.feedback.state",
+ "1.0.0",
+ momID,
+ sasID)
+ {
+ setXMLPayload(formatString(
+ "<task>\n"
+ " <type>observation</type>\n"
+ " <state>%s</state>\n"
+ "</task>",
+ success ? "finished" : "aborted"));
+ }
+
+ // Parse a message
+ TaskFeedbackState(const qpid::messaging::Message qpidMsg)
+ :
+ Message(qpidMsg)
+ {
+ }
+
+ // Read a message from disk (header + payload)
+ TaskFeedbackState(const std::string &rawContent)
+ :
+ Message(rawContent)
+ {
+ }
+
+ ParameterSet feedback() const {
+ ParameterSet result;
+ result.adoptBuffer(payload());
+
+ return result;
+ }
+};
+
+} // namespace Protocols
+
+} // namespace LOFAR
+
+#endif
+
+#endif
+
diff --git a/LCS/MessageBus/include/MessageBus/Protocols/TaskSpecificationSystem.h b/LCS/MessageBus/include/MessageBus/Protocols/TaskSpecificationSystem.h
new file mode 100644
index 0000000000000000000000000000000000000000..c67b9c8fc4efd2e211bd74a60432bdba2eddeec6
--- /dev/null
+++ b/LCS/MessageBus/include/MessageBus/Protocols/TaskSpecificationSystem.h
@@ -0,0 +1,84 @@
+//# TaskSpecificationSystem.h: Protocol for emission of status feedback information
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id: TaskSpecificationSystem.h 30864 2015-02-03 16:07:56Z mol $
+
+#ifndef LOFAR_MESSAGEBUS_TASK_SPECIFICATION_SYSTEM_H
+#define LOFAR_MESSAGEBUS_TASK_SPECIFICATION_SYSTEM_H
+
+#ifdef HAVE_QPID
+#include <MessageBus/Message.h>
+#include <Common/ParameterSet.h>
+#include <Common/StringUtil.h>
+
+namespace LOFAR {
+namespace Protocols {
+
+class TaskSpecificationSystem: public Message
+{
+public:
+ TaskSpecificationSystem(
+ // Name of the service or process producing this message
+ const std::string &from,
+
+ // End-user responsible for this request (if applicable)
+ const std::string &forUser,
+
+ // Human-readable summary describing this request
+ const std::string &summary,
+
+ // Identifiers for the context of this message
+ const std::string &momID,
+ const std::string &sasID,
+
+ // Payload: a parset containing the generated feedback
+ const ParameterSet &feedback
+ ):
+ Message( from, forUser, summary, "task.specification.system", "1.0.0", momID, sasID)
+ {
+ std::string buffer;
+ feedback.writeBuffer(buffer);
+ setTXTPayload(buffer);
+ }
+
+ // Parse a message
+ TaskSpecificationSystem(const qpid::messaging::Message qpidMsg) :
+ Message(qpidMsg)
+ { }
+
+ // Read a message from disk (header + payload)
+ TaskSpecificationSystem(const std::string &rawContent) :
+ Message(rawContent)
+ { }
+
+ ParameterSet specfications() const {
+ ParameterSet result;
+ result.adoptBuffer(payload());
+ return result;
+ }
+};
+
+ } // namespace Protocols
+} // namespace LOFAR
+
+#endif
+
+#endif
+
diff --git a/LCS/MessageBus/src/CMakeLists.txt b/LCS/MessageBus/src/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e8062864899dc5048adb5fcc42620b8d8d9ea9fc
--- /dev/null
+++ b/LCS/MessageBus/src/CMakeLists.txt
@@ -0,0 +1,28 @@
+# $Id$
+
+include(LofarPackageVersion)
+include(PythonInstall)
+
+set(messagebus_LIB_SRCS
+ Package__Version.cc
+ LogSink.cc
+ MsgBus.cc
+ Message.cc)
+
+set(messagebus_PROGRAMS
+)
+
+lofar_add_library(messagebus ${messagebus_LIB_SRCS})
+
+foreach(prog ${messagebus_PROGRAMS})
+ lofar_add_bin_program(${prog} ${prog}.cc)
+endforeach(prog ${messagebus_PROGRAMS})
+
+python_install(
+ __init__.py
+ msgbus.py
+ basemsgbus.py
+ message.py
+ DESTINATION lofar/messagebus)
+
+add_subdirectory(protocols)
diff --git a/LCS/MessageBus/src/LofarMsgTemplate.txt b/LCS/MessageBus/src/LofarMsgTemplate.txt
new file mode 100644
index 0000000000000000000000000000000000000000..796df813bad773dca7782f7fc60567ca0bb34e2d
--- /dev/null
+++ b/LCS/MessageBus/src/LofarMsgTemplate.txt
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>%s</name>
+ <version>%s</version>
+ </protocol>
+ <source>
+ <name>%s</name>
+ <user>%s</user>
+ <uuid>%s</uuid>
+ <timestamp>%s</timestamp>
+ <summary>%s</summary>
+ </source>
+ <ids>
+ <momid>%s</momid>
+ <sasid>%s</sasid>
+ </ids>
+ </header>
+ <payload>
+%s
+ </payload>
+</message>
diff --git a/LCS/MessageBus/src/LogSink.cc b/LCS/MessageBus/src/LogSink.cc
new file mode 100644
index 0000000000000000000000000000000000000000..9e698119ebd12d8dd313617a98ef5067d92c2c80
--- /dev/null
+++ b/LCS/MessageBus/src/LogSink.cc
@@ -0,0 +1,90 @@
+#include "lofar_config.h"
+
+#include "LogSink.h"
+#include <Common/LofarLogger.h>
+
+#include <Common/SystemUtil.h>
+#include <Common/StringUtil.h>
+#include <string>
+#include <sstream>
+
+#ifdef HAVE_QPID
+#include <qpid/messaging/Logger.h>
+
+using namespace qpid::messaging;
+
+namespace LOFAR {
+
+ class QpidLogSink: public LoggerOutput {
+ public:
+ QpidLogSink() {
+ }
+
+ virtual void log(Level level, bool user, const char* file, int line, const char* function, const std::string& message) {
+ (void)user;
+
+ // shorten filename to reduce spam
+ std::string filename = basename(file);
+
+ // remove trailing \n from message (and other white space while we're at it)
+ std::string trimmed_message = message;
+ rtrim(trimmed_message, " \t\n");
+
+ // construct log message
+ std::stringstream msg;
+ msg << function << "(): " << trimmed_message << " (" << filename << ":" << line << ")";
+
+ // emit at the right level
+ switch(level) {
+ case trace:
+ case debug:
+ LOG_DEBUG(msg.str());
+ break;
+
+ case info:
+ case notice:
+ LOG_INFO(msg.str());
+ break;
+
+ case warning:
+ LOG_WARN(msg.str());
+ break;
+
+ case error:
+ LOG_ERROR(msg.str());
+ break;
+
+ case critical:
+ LOG_FATAL(msg.str());
+ break;
+ }
+ }
+ };
+
+ static QpidLogSink qpidLogSink;
+
+ void qpidlogsink_init() {
+ const char *argv[] = {
+ // Name of program (dummy)
+ "a.out",
+
+ // QPID logger configuration parameters (see Logger::configure)
+
+ // Disable all default QPID sinks
+ "--log-to-stdout", "off",
+ "--log-to-stderr", "off",
+
+ // Trim the log messages
+ "--log-time", "off"
+ };
+ int argc = sizeof argv / sizeof argv[0];
+
+ Logger::configure(argc, argv);
+
+ // Add the LOFAR logger as a sink for QPID
+ Logger::setOutput(qpidLogSink);
+ }
+
+} // namespace LOFAR
+
+#endif
diff --git a/LCS/MessageBus/src/LogSink.h b/LCS/MessageBus/src/LogSink.h
new file mode 100644
index 0000000000000000000000000000000000000000..ddddee4c8dcb620abdcf2ef4506b87888ed9e50f
--- /dev/null
+++ b/LCS/MessageBus/src/LogSink.h
@@ -0,0 +1,37 @@
+//# LogSink.h: Wrapper to initialise the QPID library
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_MESSAGEBUS_LOGSINK_H
+#define LOFAR_MESSAGEBUS_LOGSINK_H
+
+namespace LOFAR {
+
+#ifdef HAVE_QPID
+
+void qpidlogsink_init();
+
+#endif
+
+} // namespace LOFAR
+
+#endif
+
diff --git a/LCS/MessageBus/src/Message.cc b/LCS/MessageBus/src/Message.cc
new file mode 100644
index 0000000000000000000000000000000000000000..2993229754a543729f77c8335b4bc87873a755bb
--- /dev/null
+++ b/LCS/MessageBus/src/Message.cc
@@ -0,0 +1,194 @@
+//# Message.cc: one_line_description
+//#
+//# Copyright (C) 2002-2004
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id: $
+
+//# Always #include <lofar_config.h> first!
+#include <lofar_config.h>
+
+//# Includes
+#include <Common/LofarLogger.h>
+#include <Common/lofar_string.h>
+#include <Common/StringUtil.h>
+#include <MessageBus/Message.h>
+
+#include <qpid/types/Uuid.h>
+
+#include <time.h>
+
+
+namespace LOFAR {
+ using namespace StringUtil;
+
+const string LOFAR_MSG_TEMPLATE = "\
+<message>\n\
+ <header>\n\
+ <system>LOFAR</system>\n\
+ <version>1.0.0</version>\n\
+ <protocol>\n\
+ <name>%s</name>\n\
+ <version>%s</version>\n\
+ </protocol>\n\
+ <source>\n\
+ <name>%s</name>\n\
+ <user>%s</user>\n\
+ <uuid>%s</uuid>\n\
+ <timestamp>%s</timestamp>\n\
+ <summary>%s</summary>\n\
+ </source>\n\
+ <ids>\n\
+ <momid>%s</momid>\n\
+ <sasid>%s</sasid>\n\
+ </ids>\n\
+ </header>\n\
+ <payload>\n\
+%s\n\
+ </payload>\n\
+</message>";
+
+static string _timestamp() {
+ // Get now (in seconds since epoch)
+ time_t now = time(NULL);
+
+ // Dissect into components (year, month, etc)
+ struct tm now_tm;
+ gmtime_r(&now, &now_tm);
+
+ // Convert to string
+ char buffer[64];
+ if (strftime(buffer, sizeof buffer, "%FT%T", &now_tm) == 0)
+ buffer[0] = 0;
+
+ return buffer;
+}
+
+static string _uuid() {
+ qpid::types::Uuid uuid(true);
+ return uuid.str();
+}
+
+Message::Message(const std::string &from,
+ const std::string &forUser,
+ const std::string &summary,
+ const std::string &protocol,
+ const std::string &protocolVersion,
+ const std::string &momid,
+ const std::string &sasid)
+{
+ itsQpidMsg.setContent(formatString(LOFAR_MSG_TEMPLATE.c_str(), protocol.c_str(), protocolVersion.c_str(),
+ from.c_str(), forUser.c_str(), _uuid().c_str(), _timestamp().c_str(), summary.c_str(),
+ momid.c_str(), sasid.c_str(), "%s"));
+ itsQpidMsg.setContentType("text/plain");
+ itsQpidMsg.setDurable(true);
+}
+
+// Read a message from disk (header + payload)
+Message::Message(const std::string &rawContent)
+{
+ itsQpidMsg.setContent(rawContent);
+}
+
+Message::~Message()
+{}
+
+void Message::setXMLPayload (const std::string &payload)
+{
+ itsQpidMsg.setContent(formatlString(itsQpidMsg.getContent().c_str(), payload.c_str()));
+}
+
+void Message::setTXTPayload (const std::string &payload)
+{
+ itsQpidMsg.setContent(formatlString(itsQpidMsg.getContent().c_str(), payload.c_str()));
+}
+
+void Message::setMapPayload (const qpid::types::Variant::Map &payload)
+{
+
+}
+
+void Message::setListPayload(const qpid::types::Variant::List &payload)
+{
+
+}
+
+std::string Message::short_desc() const
+{
+ return formatString("[%s] %s", uuid().c_str(), summary().c_str());
+}
+
+//
+// print
+//
+std::ostream& Message::print (std::ostream& os) const
+{
+ os << "system : " << system() << endl;
+ os << "systemversion : " << headerVersion() << endl;
+ os << "protocolName : " << protocol() << endl;
+ os << "protocolVersion: " << protocolVersion() << endl;
+ os << "summary : " << summary() << endl;
+ os << "timestamp : " << timestamp() << endl;
+ os << "source : " << from() << endl;
+ os << "user : " << forUser() << endl;
+ os << "uuid : " << uuid() << endl;
+ os << "momid : " << momid() << endl;
+ os << "sasid : " << sasid() << endl;
+ os << "payload : " << payload() << endl;
+ os << "BEGIN FULL PACKET" << endl;
+ os << itsQpidMsg.getContent() << endl;
+ os << "END FULL PACKET" << endl;
+ return (os);
+}
+
+//
+// getXMLvalue(tag)
+//
+string Message::getXMLvalue(const string& key) const
+{
+ // get copy of content
+ vector<string> labels = split(key, '.');
+ string content(itsQpidMsg.getContent());
+
+ // loop over subkeys
+ string::size_type offset = 0;
+ string::size_type begin = string::npos;
+ string::size_type end = string::npos;
+ string startTag;
+ for (size_t i = 0; i < labels.size(); ++i) {
+ // define tags to find
+ startTag = string("<"+labels[i]+">");
+ // search begin tag
+ begin = content.find(startTag, offset);
+ if (begin == string::npos) {
+ return ("???");
+ }
+ offset = begin;
+ }
+ // search end tag
+ string stopTag ("</"+labels[labels.size()-1]+">");
+ begin+=startTag.size();
+ end = content.find(stopTag, begin);
+ if (end == string::npos) {
+ return ("???");
+ }
+ return (content.substr(begin, end - begin));
+}
+
+
+} // namespace LOFAR
diff --git a/LCS/MessageBus/src/MsgBus.cc b/LCS/MessageBus/src/MsgBus.cc
new file mode 100644
index 0000000000000000000000000000000000000000..e40584cb3cee3a477a324811060f3411d77b1463
--- /dev/null
+++ b/LCS/MessageBus/src/MsgBus.cc
@@ -0,0 +1,177 @@
+#include "lofar_config.h"
+
+#include <MessageBus/MsgBus.h>
+#include "LogSink.h"
+#include <Common/LofarLogger.h>
+
+#ifdef HAVE_QPID
+#include <qpid/types/Exception.h>
+#include <qpid/messaging/exceptions.h>
+#include <stdlib.h>
+
+using namespace qpid::messaging;
+
+namespace LOFAR {
+
+ static Duration TimeOutDuration(double secs)
+ {
+ if (secs > 0.0)
+ return (Duration)(1000.0 * secs);
+
+ return Duration::FOREVER;
+ }
+
+ static std::string queue_prefix()
+ {
+ char *prefix = getenv("QUEUE_PREFIX");
+
+ if (!prefix)
+ return "";
+
+ return prefix;
+ }
+
+ namespace MessageBus {
+ void init() {
+ qpidlogsink_init();
+ }
+ }
+
+ FromBus::FromBus(const std::string &address, const std::string &options, const std::string &broker)
+ try:
+ itsConnection(broker,"{reconnect:true}"),
+ itsNrMissingACKs(0)
+ {
+ LOG_DEBUG_STR("[FromBus] Connecting to broker " << broker);
+ itsConnection.open();
+ LOG_INFO_STR("[FromBus] Connected to broker " << itsConnection.getUrl());
+
+ itsSession = itsConnection.createSession();
+
+ addQueue(address, options);
+ } catch(const qpid::types::Exception &ex) {
+ THROW(MessageBusException, ex.what());
+ }
+
+
+ FromBus::~FromBus(void)
+ {
+ if (itsNrMissingACKs) {
+ LOG_ERROR_STR("[FromBus] " << itsNrMissingACKs << " messages not ACK'ed");
+ }
+
+ try {
+ // Make sure all requests are finished
+
+ // connection.close() closes all senders, receivers, and sessions as well.
+ itsConnection.close();
+ } catch(const qpid::types::Exception &ex) {
+ LOG_FATAL_STR("Exception in destructor, cannot guarantee message delivery: " << ex.what());
+ }
+ }
+
+ bool FromBus::getMessage(LOFAR::Message &msg, double timeout) // timeout 0.0 means blocking
+ {
+ Receiver next;
+ qpid::messaging::Message qmsg;
+
+ LOG_DEBUG_STR("[FromBus] Waiting for message");
+ if (itsSession.nextReceiver(next,TimeOutDuration(timeout))) {
+ LOG_DEBUG_STR("[FromBus] Message available on queue " << next.getName());
+ itsNrMissingACKs++;
+ if (next.get(qmsg)) {
+ msg = LOFAR::Message(qmsg);
+ LOG_INFO_STR("[FromBus] Message received on queue " << next.getName() << ": " << msg.short_desc());
+ return true;
+ } else {
+ LOG_ERROR_STR("[FromBus] Could not retrieve available message on queue " << next.getName());
+ }
+ }
+ return false;
+ }
+
+ void FromBus::ack(LOFAR::Message &msg)
+ {
+ itsSession.acknowledge(msg.qpidMsg());
+ itsNrMissingACKs --;
+
+ LOG_INFO_STR("[FromBus] Message ACK'ed: " << msg.short_desc());
+ }
+
+ void FromBus::nack(LOFAR::Message &msg)
+ {
+ itsSession.release(msg.qpidMsg());
+ itsNrMissingACKs--;
+
+ LOG_ERROR_STR("[FromBus] Message NACK'ed: " << msg.short_desc());
+ }
+
+ void FromBus::reject(LOFAR::Message &msg)
+ {
+ itsSession.reject(msg.qpidMsg());
+ itsNrMissingACKs --;
+
+ LOG_ERROR_STR("[FromBus] Message rejected: " << msg.short_desc());
+ }
+
+ bool FromBus::addQueue(const std::string &address, const std::string &options)
+ {
+ try
+ {
+ Address addr(queue_prefix()+address+options);
+ Receiver receiver = itsSession.createReceiver(addr);
+ receiver.setCapacity(1);
+
+ LOG_INFO_STR("[FromBus] Receiver started at queue " << receiver.getName());
+ } catch(const qpid::types::Exception &ex) {
+ //THROW(MessageBusException, ex.what());
+ return false;
+ }
+ return true;
+ }
+
+ ToBus::ToBus(const std::string &address, const std::string &options, const std::string &broker)
+ try:
+ itsConnection(broker,"{reconnect:true}")
+ {
+ LOG_DEBUG_STR("[ToBus] Connecting to broker " << broker);
+ itsConnection.open();
+ LOG_INFO_STR("[ToBus] Connected to broker " << itsConnection.getUrl());
+
+ itsSession = itsConnection.createSession();
+ Address addr(queue_prefix()+address+options);
+ itsSender = itsSession.createSender(addr);
+ LOG_INFO_STR("[ToBus] Sender created at queue " << itsSender.getName());
+ } catch(const qpid::types::Exception &ex) {
+ THROW(MessageBusException, ex.what());
+ }
+
+ ToBus::~ToBus(void)
+ {
+
+ try {
+ // Make sure all requests are finished
+
+ // connection.close() closes all senders, receivers, and sessions as well.
+ itsConnection.close();
+ } catch(const qpid::types::Exception &ex) {
+ LOG_FATAL_STR("Exception in destructor, cannot guarantee message delivery: " << ex.what());
+ }
+ }
+
+ void ToBus::send(const std::string &msg)
+ {
+ LOFAR::Message tosend(msg);
+ itsSender.send(tosend.qpidMsg(), true);
+ LOG_INFO_STR("[ToBus] Message sent to queue " << itsSender.getName() << ": " << tosend.short_desc());
+ }
+
+ void ToBus::send(LOFAR::Message& msg)
+ {
+ itsSender.send(msg.qpidMsg(), true);
+ LOG_INFO_STR("[ToBus] Message sent to queue " << itsSender.getName() << ": " << msg.short_desc());
+ }
+
+} // namespace LOFAR
+
+#endif
diff --git a/LCS/MessageBus/src/__init__.py b/LCS/MessageBus/src/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3bcbe73ef576c7f93974459df18e6da942628cd6
--- /dev/null
+++ b/LCS/MessageBus/src/__init__.py
@@ -0,0 +1,18 @@
+# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# $Id: __init__.py 29911 2014-08-12 14:45:14Z klijn $
diff --git a/LCS/MessageBus/src/basemsgbus.py b/LCS/MessageBus/src/basemsgbus.py
new file mode 100644
index 0000000000000000000000000000000000000000..d51c21f3b4747cc9dfeb90842dd81271f1375576
--- /dev/null
+++ b/LCS/MessageBus/src/basemsgbus.py
@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# id.. TDB
+from qpid.messaging import *
+
+# Candidate for a config file
+broker="127.0.0.1"
+address=""
+options=""
+ # "create:always, node: { type: queue, durable: True}"
+
+class ToBus:
+ def __init__(self, address, broker=broker, options=options):
+ self.connection = Connection(broker)
+ self.connection.reconnect = True
+
+ try:
+ self.connection.open()
+ self.session = self.connection.session()
+ #self.receiver = self.session.receiver("%s;{%s}" %(address,options))
+ #self.receiver.capacity = 32
+ self.sender = self.session.sender(address)
+
+ except MessagingError,m:
+ print " OMG!!"
+ print m
+
+ def sendstr(self,messagetxt,subject="",reply_to=""):
+ msg = Message(messgaetxt)
+ msg.subject=subject
+ msg.durable=True
+ if (reply_to != ""):
+ msg.reply_to=reply_to
+ self.sender.send(msg)
+
+ def send(self,msg):
+ if msg is None:
+ print( " attempt to send None object!")
+ else:
+ msg.durable=True # force durable messages
+ self.sender.send(msg)
+
+
+class FromBus:
+ def __init__(self, address=address, broker=broker, options=options):
+ self.connection = Connection(broker)
+ self.connection.reconnect = True
+
+ try:
+ self.connection.open()
+ self.session = self.connection.session()
+ self.receiver = self.session.receiver("%s;{%s}" %(address,options))
+ #self.address = self.receiver.address
+ self.receiver.capacity = 1
+
+ except MessagingError,m:
+ print " OMG!!"
+ print m
+
+ def get(self,timeout=None):
+ try:
+ self.msg= self.receiver.fetch(timeout)
+ except exceptions.Empty, e:
+ self.msg = None
+ return self.msg
+ #.content, self.msg.subject
+
+ def ack(self,msg):
+ if msg is None:
+ print(" Ack on None object!")
+ else:
+ self.session.acknowledge(msg)
+
+class BusReply:
+ def __init__(self, address="Reply", broker=broker, options=options):
+ self.address = "%s%s ; { %s }" %(str(uuid4()),address,options)
+ self.connection = Connection(broker)
+ self.connection.reconnect = True
+
+ try:
+ self.connection.open()
+ self.session = self.connection.session()
+ self.receiver = self.session.receiver(self.address)
+ self.address = self.receiver.source
+ #self.receiver.capacity = 32
+
+ except MessagingError,m:
+ print " OMG!!"
+ print m
+
+ def get(self,timeout=None):
+ try:
+ self.msg= self.receiver.fetch(timeout)
+ except exceptions.Empty, e:
+ return "None" , "None"
+ return self.msg.content, self.msg.subject
+
+ def ack(self):
+ self.session.acknowledge()
+
+class MultiReceiveBus:
+ def __init__(self, handler, address=address, broker=broker, options=options):
+ self.connection = Connection(broker)
+ self.connection.reconnect = True
+ self.handlers={}
+ try:
+ self.connection.open()
+ self.session = self.connection.session()
+ receiver = self.session.receiver("%s;{%s}" %(address,options))
+ receiver.capacity = 1 #32
+ self.handlers[receiver] = handler
+
+ except MessagingError,m:
+ print " OMG!!"
+ print m
+
+ def add(self,handler,address,options=options):
+ try:
+ receiver=self.session.receiver("%s;{%s}" %(address,options))
+ receiver.capacity = 1 #32
+ self.handlers[receiver]=handler
+ except MessagingError,m:
+ print "Error adding receiver"
+ print m
+
+ def HandleMessages(self):
+ while True:
+ print "waiting for messages"
+ receiver = self.session.next_receiver()
+ print "got incoming message"
+ handler = self.handlers[receiver]
+ msg = receiver.fetch()
+ handler(self,msg.content,msg.subject)
+
+ def ack(self,msg):
+ self.session.acknowledge(msg)
+
diff --git a/LCS/MessageBus/src/message.py b/LCS/MessageBus/src/message.py
new file mode 100644
index 0000000000000000000000000000000000000000..793f256b8e3bb300dcad37c919ddb265b96009ea
--- /dev/null
+++ b/LCS/MessageBus/src/message.py
@@ -0,0 +1,214 @@
+#!/usr/bin/python
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+import qpid.messaging
+import xml.dom.minidom as xml
+import xml.parsers.expat as expat
+import datetime
+
+LOFAR_MSG_TEMPLATE = """
+<message>
+ <header>
+ <system/>
+ <version/>
+ <protocol>
+ <name/>
+ <version/>
+ </protocol>
+ <source>
+ <name/>
+ <user/>
+ <uuid/>
+ <timestamp/>
+ <summary/>
+ </source>
+ <ids>
+ <momid/>
+ <sasid/>
+ </ids>
+ </header>
+ <payload>
+ </payload>
+</message>"""
+
+def _timestamp():
+ """
+ Return the current time as YYYY-MM-DDTHH:MM:SS
+ """
+ now = datetime.datetime.now()
+ return now.strftime("%FT%T")
+
+def _uuid():
+ """
+ Return an UUID
+ """
+ return str(qpid.messaging.uuid4())
+
+class MessageException(Exception):
+ pass
+
+class Message(object):
+ def __init__(self, from_="", forUser="", summary="", protocol="", protocolVersion="", momid="", sasid="", qpidMsg=None):
+
+ for name, element in self._property_list().iteritems():
+ self._add_property(name, element)
+
+ if qpidMsg is None:
+ self.document = xml.parseString(LOFAR_MSG_TEMPLATE)
+ self._qpidMsg = qpid.messaging.Message(content_type="text/plain", durable=True)
+ self.dirty = True
+
+ # Set properties provided by constructor
+ self.system = "LOFAR"
+ self.headerVersion = "1.0.0"
+ self.protocol = protocol
+ self.protocolVersion = protocolVersion
+ self.from_ = from_
+ self.forUser = forUser
+ self.summary = summary
+ self.uuid = _uuid()
+ self.timestamp = _timestamp()
+ self.momid = momid
+ self.sasid = sasid
+ else:
+ self._qpidMsg = qpidMsg
+ self.dirty = False
+
+ # Set properties by provided qpidMsg
+ try:
+ # Replace literal << in the content, which is occasionally inserted by the C++
+ # code as part of the Parset ("Observation.Clock=<<Clock200")
+ self.document = xml.parseString(qpidMsg.content.replace("<<","<<"))
+ except expat.ExpatError, e:
+ print "Could not parse XML message content: ", e, msg.content
+ raise MessageException(e)
+
+ def _add_property(self, name, element):
+ def getter(self):
+ return self._getXMLdata(element)
+ def setter(self, value):
+ self._setXMLdata(element, str(value))
+
+ setattr(self.__class__, name, property(getter, setter))
+
+ def _property_list(self):
+ """ List of XML elements that are exposed as properties. """
+ return {
+ "system": "message.header.system",
+ "headerVersion": "message.header.version",
+ "protocol": "message.header.protocol.name",
+ "protocolVersion": "message.header.protocol.version",
+ "from_": "message.header.source.name",
+ "forUser": "message.header.source.user",
+ "uuid": "message.header.source.uuid",
+ "summary": "message.header.source.summary",
+ "timestamp": "message.header.source.timestamp",
+ "momid": "message.header.ids.momid",
+ "sasid": "message.header.ids.sasid",
+ "payload": "message.payload",
+ "header": "message.header",
+ }
+
+ """ API (apart from properties). """
+
+ def __repr__(self):
+ return "Message(%s %s)" % (self.protocol, self.protocolVersion)
+
+ def __str__(self):
+ return ("system : %s\n" % (self.system,) +
+ "systemversion : %s\n" % (self.headerVersion,) +
+ "protocol : %s\n" % (self.protocol,) +
+ "protocolVersion: %s\n" % (self.protocolVersion,) +
+ "summary : %s\n" % (self.summary,) +
+ "timestamp : %s\n" % (self.timestamp,) +
+ "source : %s\n" % (self.from_,) +
+ "user : %s\n" % (self.forUser,) +
+ "uuid : %s\n" % (self.uuid,) +
+ "momid : %s\n" % (self.momid,) +
+ "sasid : %s\n" % (self.sasid,) +
+ "payload : %s\n" % (self.payload,)
+ )
+
+ def qpidMsg(self):
+ """ Construct the QPID message content. """
+
+ if self.dirty:
+ self._qpidMsg.content = self.document.toxml()
+
+ return self._qpidMsg
+
+ """ XML support functions. See also lofarpipe/support/xmllogging.py. """
+
+ def _get_child(self, node, name):
+ """ Return a specific child node. """
+
+ for child in node.childNodes:
+ if child.nodeName == name and child.nodeType == child.ELEMENT_NODE:
+ return child
+
+ return None
+
+ def _get_data(self, node):
+ """ Return the textual content of a node. """
+
+ data = []
+ for child in node.childNodes:
+ if child.nodeType == child.TEXT_NODE:
+ data.append(child.data)
+
+ return ''.join(data)
+
+ def _set_data(self, node, data):
+ """ Set the textual content of a node. """
+
+ newchild = self.document.createTextNode(data)
+
+ for child in node.childNodes:
+ if child.nodeType == child.TEXT_NODE:
+ node.replaceChild(newchild, child)
+ break;
+ else:
+ node.appendChild(newchild)
+
+ self.dirty = True
+
+ def _getXMLnode(self, name):
+ """ Return a node given by its dot-separated path name.
+ So a.b.c returns the inner node of <a><b><c></c></b></a>. """
+ parts = name.split(".")
+ node = self.document
+
+ for p in parts:
+ node = self._get_child(node,p)
+
+ if node is None:
+ return None
+
+ return node
+
+ def _getXMLdata(self, name):
+ return self._get_data(self._getXMLnode(name))
+
+ def _setXMLdata(self, name, data):
+ return self._set_data(self._getXMLnode(name), data)
+
+if __name__ == "__main__":
+ m = Message("FROM", "FORUSER", "SUMMARY", "PROTOCOL", "1.2.3", "11111", "22222")
+ print str(m)
+ print m.document.toxml()
+
diff --git a/LCS/MessageBus/src/msgbus.py b/LCS/MessageBus/src/msgbus.py
new file mode 100644
index 0000000000000000000000000000000000000000..a17c2ac2bde0927fafa46cf0e7075994e1957333
--- /dev/null
+++ b/LCS/MessageBus/src/msgbus.py
@@ -0,0 +1,114 @@
+#!/usr/bin/python
+# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# id.. TDB
+
+try:
+ import qpid.messaging
+ enabled = True
+except ImportError:
+ enabled = False
+
+import os
+import signal
+import lofar.messagebus.message as message
+
+# Candidate for a config file
+broker="127.0.0.1"
+options="create:never"
+
+class BusException(Exception):
+ pass
+
+class Session:
+ def __init__(self, broker):
+ self.connection = qpid.messaging.Connection(broker)
+ self.connection.reconnect = True
+
+ try:
+ self.connection.open()
+ self.session = self.connection.session()
+ except qpid.messaging.MessagingError, m:
+ raise BusException(m)
+
+ def __del__(self):
+ # NOTE: session.close() freezes under certain error conditions,
+ # f.e. opening a receiver on a non-existing queue.
+ # This seems to happen whenever a Python exception was thrown
+ # by the qpid wrapper.
+ #
+ # We set a timeout to prevent freezing, which obviously leads
+ # to data loss if the stall was legit.
+ try:
+ self.connection.close(5.0)
+ except qpid.messaging.exceptions.Timeout, t:
+ raise BusException(t)
+
+ def address(self, queue, options):
+ return "%s%s; {%s}" % (self._queue_prefix(), queue, options)
+
+ def _queue_prefix(self):
+ return os.environ.get("QUEUE_PREFIX", "")
+
+class ToBus(Session):
+ def __init__(self, queue, options=options, broker=broker):
+ Session.__init__(self, broker)
+
+ try:
+ self.sender = self.session.sender(self.address(queue, options))
+ except qpid.messaging.MessagingError, m:
+ raise BusException(m)
+
+ def send(self, msg):
+ try:
+ self.sender.send(msg.qpidMsg())
+ except qpid.messaging.SessionError, m:
+ raise BusException(m)
+
+class FromBus(Session):
+ def __init__(self, queue, options=options, broker=broker):
+ Session.__init__(self, broker)
+
+ self.add_queue(queue, options)
+
+ def add_queue(self, queue, options=options):
+ try:
+ receiver = self.session.receiver(self.address(queue, options))
+
+ # Need capacity >=1 for 'self.session.next_receiver' to function across multiple queues
+ receiver.capacity = 1 #32
+ except qpid.messaging.MessagingError, m:
+ raise BusException(m)
+
+ def get(self, timeout=None):
+ msg = None
+
+ try:
+ receiver = self.session.next_receiver(timeout)
+ if receiver != None: msg = receiver.fetch() # receiver.get() is better, but requires qpid 0.31+
+ except qpid.messaging.exceptions.Empty, e:
+ return None
+
+ if msg is None:
+ return None
+ else:
+ return message.Message(qpidMsg=msg)
+
+ def ack(self, msg):
+ self.session.acknowledge(msg.qpidMsg)
+
diff --git a/LCS/MessageBus/src/protocols/CMakeLists.txt b/LCS/MessageBus/src/protocols/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ee3c47119038645b810a0c166a29b07b37cb376d
--- /dev/null
+++ b/LCS/MessageBus/src/protocols/CMakeLists.txt
@@ -0,0 +1,10 @@
+# $Id$
+
+include(PythonInstall)
+
+python_install(
+ __init__.py
+ taskfeedbackdataproducts.py
+ taskfeedbackprocessing.py
+ taskfeedbackstate.py
+ DESTINATION lofar/messagebus/protocols)
diff --git a/LCS/MessageBus/src/protocols/__init__.py b/LCS/MessageBus/src/protocols/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..779271f4bae8ff25f9c6e5fd683317634f899945
--- /dev/null
+++ b/LCS/MessageBus/src/protocols/__init__.py
@@ -0,0 +1,18 @@
+# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# $Id$
diff --git a/LCS/MessageBus/src/protocols/taskfeedbackdataproducts.py b/LCS/MessageBus/src/protocols/taskfeedbackdataproducts.py
new file mode 100644
index 0000000000000000000000000000000000000000..da9e06a1eee0b03d77de1a9440533e10378a88fa
--- /dev/null
+++ b/LCS/MessageBus/src/protocols/taskfeedbackdataproducts.py
@@ -0,0 +1,33 @@
+
+#!/usr/bin/python
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+import lofar.messagebus.message
+
+class TaskFeedbackDataproducts(lofar.messagebus.message.Message):
+ def __init__(self, from_, forUser, summary, momID, sasID, feedback):
+ super(TaskFeedbackDataproducts, self).__init__(
+ from_,
+ forUser,
+ summary,
+ "task.feedback.dataproducts",
+ "1.0.0",
+ momID,
+ sasID)
+
+ self.payload = str(feedback)
diff --git a/LCS/MessageBus/src/protocols/taskfeedbackprocessing.py b/LCS/MessageBus/src/protocols/taskfeedbackprocessing.py
new file mode 100644
index 0000000000000000000000000000000000000000..bacd6a7491e2c185a37a4414204cbb6ce3c1a72e
--- /dev/null
+++ b/LCS/MessageBus/src/protocols/taskfeedbackprocessing.py
@@ -0,0 +1,33 @@
+
+#!/usr/bin/python
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+import lofar.messagebus.message
+
+class TaskFeedbackProcessing(lofar.messagebus.message.Message):
+ def __init__(self, from_, forUser, summary, momID, sasID, feedback):
+ super(TaskFeedbackProcessing, self).__init__(
+ from_,
+ forUser,
+ summary,
+ "task.feedback.processing",
+ "1.0.0",
+ momID,
+ sasID)
+
+ self.payload = str(feedback)
diff --git a/LCS/MessageBus/src/protocols/taskfeedbackstate.py b/LCS/MessageBus/src/protocols/taskfeedbackstate.py
new file mode 100644
index 0000000000000000000000000000000000000000..c9d3c766ac2b830096998a8e8411772d4a407037
--- /dev/null
+++ b/LCS/MessageBus/src/protocols/taskfeedbackstate.py
@@ -0,0 +1,64 @@
+
+#!/usr/bin/python
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+#import lofar.messagebus.Message
+import lofar.messagebus.message
+import xml.dom.minidom as xml
+
+LOFAR_STATUS_MSG_TEMPLATE = """
+<task>
+ <type/>
+ <state/>
+</task>"""
+
+class TaskFeedbackState(lofar.messagebus.message.Message):
+ def __init__(self, from_, forUser, summary, momID, sasID, status):
+ super(TaskFeedbackState, self).__init__(
+ from_,
+ forUser,
+ summary,
+ "task.feedback.state",
+ "1.0.0",
+ momID,
+ sasID)
+
+ payload_document = xml.parseString(LOFAR_STATUS_MSG_TEMPLATE)
+
+ self._getXMLnode("message.payload").appendChild(payload_document.firstChild)
+
+ self.type_ = "pipeline"
+ if status:
+ self.state = "finished"
+ else:
+ self.state = "aborted"
+
+ def _property_list(self):
+ properties = super(TaskFeedbackState, self)._property_list()
+
+ properties.update( {
+ "type_": "message.payload.task.type",
+ "state": "message.payload.task.state",
+ } )
+
+ return properties
+
+if __name__ == "__main__":
+ msg = TaskFeedbackState("FROM", "FORUSER", "SUMMARY", "11111", "22222", True)
+ print msg.document.toxml()
+
diff --git a/LCS/MessageBus/test/CMakeLists.txt b/LCS/MessageBus/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0f8f56a78780c14db8f6efddfbe82d300753b60c
--- /dev/null
+++ b/LCS/MessageBus/test/CMakeLists.txt
@@ -0,0 +1,10 @@
+# $Id$
+
+include(LofarCTest)
+
+configure_file(
+ ${CMAKE_CURRENT_SOURCE_DIR}/MessageFuncs.sh.in
+ ${CMAKE_BINARY_DIR}/bin/MessageFuncs.sh @ONLY)
+
+lofar_add_test(tMsgBus tMsgBus.cc)
+lofar_add_test(tMessage tMessage.cc)
diff --git a/LCS/MessageBus/test/MessageFuncs.sh.in b/LCS/MessageBus/test/MessageFuncs.sh.in
new file mode 100644
index 0000000000000000000000000000000000000000..79a521e8f5089f78fc36fda73e987d40947b5be9
--- /dev/null
+++ b/LCS/MessageBus/test/MessageFuncs.sh.in
@@ -0,0 +1,182 @@
+#!/bin/bash
+#
+# Usage: source MessageFuncs.sh
+#
+
+function _on_exit() {
+ # Calls $1 on exit, while preserving existing EXIT traps
+
+ # Get existing EXIT traps
+ TRAPLINE=`trap -p EXIT`
+ ONEXITS=`echo $TRAPLINE | perl -ne 'print "$1\n" if /trap -- .(.*). EXIT/;'`
+
+ # Expand traps
+ trap -- "$1;$ONEXITS" EXIT
+}
+
+function _generate_prefix() {
+ # Generate a unique prefix for this test
+ NOW=`date +"%FT%T.%N"`
+
+ echo test-$NOW.
+}
+
+# Generate an unique prefix for all queues in this test
+export QUEUE_PREFIX=`_generate_prefix`
+echo "MessageBus: QUEUE_PREFIX=$QUEUE_PREFIX"
+
+function have_qpid() {
+ [ "@HAVE_QPID@" == "TRUE" ]
+}
+
+# A list of all queues we created
+CREATED_QUEUES=""
+
+function _get_host() {
+ # input = host:queue -> output = host
+ # input = queue -> output = ''
+ HOSTQUEUE="$1"
+
+ echo "$HOSTQUEUE" | perl -ne 'print $1 if /(.*):/;'
+}
+function _get_queue() {
+ # input = host:queue -> output = queue
+ # input = queue -> output = queue
+ HOSTQUEUE="$1"
+
+ echo "$HOSTQUEUE" | perl -ne 'print "$1$2" if /:(.*)|^([^:\n]*)$/;'
+}
+
+function _qpid_receive() {
+ HOSTQUEUE="$1"
+ OPTIONS="$2"
+ shift 2
+
+ QUEUE="`_get_queue "$HOSTQUEUE"`"
+ HOST="`_get_host "$HOSTQUEUE"`"
+
+ if [ "$HOST" == "" ]; then
+ HOST=127.0.0.1
+ fi
+
+ if have_qpid; then
+ @QPID_RECEIVE_EXECUTABLE@ \
+ -b "$HOST" \
+ -a "$QUEUE_PREFIX$QUEUE$OPTIONS" "$@"
+ fi
+}
+
+function _qpid_send() {
+ HOSTQUEUE="$1"
+ OPTIONS="$2"
+ shift 2
+
+ QUEUE="`_get_queue "$HOSTQUEUE"`"
+ HOST="`_get_host "$HOSTQUEUE"`"
+
+ if [ "$HOST" == "" ]; then
+ HOST=127.0.0.1
+ fi
+
+ if have_qpid; then
+ @QPID_SEND_EXECUTABLE@ \
+ -b "$HOST" \
+ -a "$QUEUE_PREFIX$QUEUE$OPTIONS" "$@"
+ fi
+}
+
+
+function create_queue() {
+ # Creates an empty queue
+ #
+ # Usage:
+ # create_queue [host:]queue
+ HOSTQUEUE="$1"
+
+ echo "MessageBus: Creating queue $HOSTQUEUE"
+
+ _qpid_receive "$HOSTQUEUE" "; { create: always }" --print-content no --ignore-reply-to
+
+ # Update the list of queues we created
+ CREATED_QUEUES="$CREATED_QUEUES $HOSTQUEUE"
+}
+
+function delete_queue() {
+ # Empties and deletes a queue
+ #
+ # Usage:
+ # delete_queue [host:]queue
+ #
+ # Will not remove used queues
+ HOSTQUEUE="$1"
+
+ echo "MessageBus: Deleting queue $HOSTQUEUE"
+
+ _qpid_receive "$HOSTQUEUE" "; { delete: always }" --print-content no --ignore-reply-to
+}
+
+function delete_all_queues() {
+ # Empties and deletes all queues created by create_queue
+ #
+ # Usage:
+ # delete_all_queues
+
+ for QUEUE_NAME in $CREATED_QUEUES; do
+ delete_queue "$QUEUE_NAME"
+ done
+}
+
+# Automatically delete all unused queues (created by create_queue) on EXIT
+_on_exit delete_all_queues
+
+function recv_msg() {
+ # Retrieves one message from a queue
+ #
+ # Usage:
+ # recv_msg [host:]queue > message
+ #
+ # Returns an empty message if none was available
+ HOSTQUEUE="$1"
+
+ _qpid_receive "$HOSTQUEUE" "" --ignore-reply-to -m 1
+}
+
+function recv_all_msgs() {
+ # Retrieves all messages from a queue
+ #
+ # Usage:
+ # recv_all_msgs [host:]queue > messages
+ #
+ # Returns an empty message if none was available
+ HOSTQUEUE="$1"
+
+ _qpid_receive "$HOSTQUEUE" "" --ignore-reply-to -m 0
+}
+
+function send_msg() {
+ # Send one message to a queue
+ #
+ # Usage:
+ # send_msg [host:]queue message
+ HOSTQUEUE="$1"
+ MESSAGE="$2"
+
+ _qpid_send "$HOSTQUEUE" "" --content-string "$MESSAGE"
+}
+
+function compare_msg() {
+ # Compare two messages, ignoring uncontrollable fields (UUID, timestamp).
+ # Both messages are expected to be in files (or filedescriptors).
+ #
+ # Usage:
+ # compare_msg reference_message.txt generated_message.txt
+ REFERENCE="$1"
+ GENERATED="$2"
+
+ function compare_msg_filter() {
+ fgrep -v '<uuid>' | fgrep -v '<timestamp>'
+ }
+
+ diff -w <(<"$REFERENCE" compare_msg_filter) <(<"$GENERATED" compare_msg_filter)
+}
+
diff --git a/LCS/MessageBus/test/tMessage.cc b/LCS/MessageBus/test/tMessage.cc
new file mode 100644
index 0000000000000000000000000000000000000000..cd0c4a8e49da767afa1262c6b7662d1dc2960acb
--- /dev/null
+++ b/LCS/MessageBus/test/tMessage.cc
@@ -0,0 +1,85 @@
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id: $
+
+//# Always #include <lofar_config.h> first!
+#include <lofar_config.h>
+
+//# Includes
+#include <Common/LofarLogger.h>
+#include <MessageBus/Message.h>
+
+//using namespace qpid::messaging;
+using namespace LOFAR;
+
+#if 0
+void showMessage(Message& msg)
+{
+ cout << "Message ID : " << msg.getMessageId() << endl;
+ cout << "User ID : " << msg.getUserId() << endl;
+ cout << "Correlation ID: " << msg.getCorrelationId() << endl;
+ cout << "Subject : " << msg.getSubject() << endl;
+ cout << "Reply to : " << msg.getReplyTo() << endl;
+ cout << "Content type : " << msg.getContentType() << endl;
+ cout << "Priority : " << msg.getPriority() << endl;
+// cout << "TTL : " << msg.getTtl() << endl;
+ cout << "Durable : " << (msg.getDurable() ? "Yes" : "No") << endl;
+ cout << "Redelivered : " << (msg.getRedelivered() ? "Yes" : "No") << endl;
+ cout << "Properties : " << msg.getProperties() << endl;
+ cout << "Content size : " << msg.getContentSize() << endl;
+ cout << "Content : " << msg.getContent() << endl;
+}
+
+void compareMessages(Message& lhm, Message& rhm)
+{
+ ASSERTSTR(lhm.getMessageId() == rhm.getMessageId(), "messageIDs differ");
+ ASSERTSTR(lhm.getUserId() == rhm.getUserId(), "UserIDs differ");
+ ASSERTSTR(lhm.getCorrelationId() == rhm.getCorrelationId(), "CorrelationIDs differ");
+ ASSERTSTR(lhm.getSubject() == rhm.getSubject(), "Subjects differ");
+ ASSERTSTR(lhm.getReplyTo() == rhm.getReplyTo(), "ReplyTos differ");
+ ASSERTSTR(lhm.getContentType() == rhm.getContentType(), "ContentTypes differ");
+ ASSERTSTR(lhm.getPriority() == rhm.getPriority(), "Priorities differ");
+ ASSERTSTR(lhm.getTtl() == rhm.getTtl(), "TTLs differ");
+ ASSERTSTR(lhm.getDurable() == rhm.getDurable(), "Durability differs");
+ ASSERTSTR(lhm.getRedelivered() == rhm.getRedelivered(), "Redelivered differs");
+// ASSERTSTR(lhm.getProperties() == rhm.getProperties(), "Properties differ");
+ ASSERTSTR(lhm.getContentSize() == rhm.getContentSize(), "ContentSize differs");
+ ASSERTSTR(lhm.getContent() == rhm.getContent(), "Content differs");
+}
+#endif
+
+int main(int argc, char* argv[]) {
+ if (argc != 1) {
+ cout << "Syntax: " << argv[0] << endl;
+ return (1);
+ }
+
+ Message msg1("mySubSystem", "user", "some test message", "lofar.observation.start", "1.0", "12345", "54321");
+
+ qpid::messaging::Message qpMsg("Qpid message");
+ Message msg2(qpMsg);
+
+ string KVmapje("abc=[aap,noot,mies]\nmyInteger=5\nmyDouble=3.14");
+ msg1.setTXTPayload(KVmapje);
+
+ cout << msg1;
+
+ return (0);
+}
+
diff --git a/LCS/MessageBus/test/tMsgBus.cc b/LCS/MessageBus/test/tMsgBus.cc
new file mode 100644
index 0000000000000000000000000000000000000000..14688926fa777573978e0b36f474feaab3270829
--- /dev/null
+++ b/LCS/MessageBus/test/tMsgBus.cc
@@ -0,0 +1,117 @@
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id: $
+
+//# Always #include <lofar_config.h> first!
+#include <lofar_config.h>
+
+//# Includes
+#include <Common/LofarLogger.h>
+#include <MessageBus/MsgBus.h>
+
+using namespace qpid::messaging;
+using namespace LOFAR;
+
+
+void showMessage(qpid::messaging::Message& msg)
+{
+ cout << "Message ID : " << msg.getMessageId() << endl;
+ cout << "User ID : " << msg.getUserId() << endl;
+ cout << "Correlation ID: " << msg.getCorrelationId() << endl;
+ cout << "Subject : " << msg.getSubject() << endl;
+ cout << "Reply to : " << msg.getReplyTo() << endl;
+ cout << "Content type : " << msg.getContentType() << endl;
+ cout << "Priority : " << msg.getPriority() << endl;
+// cout << "TTL : " << msg.getTtl() << endl;
+ cout << "Durable : " << (msg.getDurable() ? "Yes" : "No") << endl;
+ cout << "Redelivered : " << (msg.getRedelivered() ? "Yes" : "No") << endl;
+ cout << "Properties : " << msg.getProperties() << endl;
+ cout << "Content size : " << msg.getContentSize() << endl;
+ cout << "Content : " << msg.getContent() << endl;
+}
+
+void compareMessages(qpid::messaging::Message& lhm, qpid::messaging::Message& rhm)
+{
+ ASSERTSTR(lhm.getMessageId() == rhm.getMessageId(), "messageIDs differ");
+ ASSERTSTR(lhm.getUserId() == rhm.getUserId(), "UserIDs differ");
+ ASSERTSTR(lhm.getCorrelationId() == rhm.getCorrelationId(), "CorrelationIDs differ");
+ ASSERTSTR(lhm.getSubject() == rhm.getSubject(), "Subjects differ");
+ ASSERTSTR(lhm.getReplyTo() == rhm.getReplyTo(), "ReplyTos differ");
+ ASSERTSTR(lhm.getContentType() == rhm.getContentType(), "ContentTypes differ");
+ ASSERTSTR(lhm.getPriority() == rhm.getPriority(), "Priorities differ");
+ ASSERTSTR(lhm.getTtl() == rhm.getTtl(), "TTLs differ");
+ ASSERTSTR(lhm.getDurable() == rhm.getDurable(), "Durability differs");
+ ASSERTSTR(lhm.getRedelivered() == rhm.getRedelivered(), "Redelivered differs");
+// ASSERTSTR(lhm.getProperties() == rhm.getProperties(), "Properties differ");
+ ASSERTSTR(lhm.getContentSize() == rhm.getContentSize(), "ContentSize differs");
+ ASSERTSTR(lhm.getContent() == rhm.getContent(), "Content differs");
+}
+
+
+int main(int argc, char* argv[]) {
+ INIT_LOGGER("tMsgBus");
+
+ MessageBus::init();
+
+ std::string queue(argc == 2 ? argv[1] : "tMsgBus");
+
+ cout << "Using queue " << queue << " (Syntax: " << argv[0] << " messagebus)" << endl;
+
+ cout << "--- Drain the queue ---" << endl;
+ FromBus fb(queue);
+ LOFAR::Message receivedMsg;
+ while (fb.getMessage(receivedMsg, 0.01)) {
+ fb.ack(receivedMsg);
+ }
+
+ cout << "--- TEST 1: create a message from a string, send it, receive it, print it. --- " << endl;
+ ToBus tb(queue);
+ string someText("An example message constructed from text");
+ tb.send(someText);
+ fb.getMessage(receivedMsg);
+ fb.ack(receivedMsg);
+ showMessage(receivedMsg.qpidMsg());
+
+ cout << "--- TEST 2: create a message by hand, send it, receive it, print it, compare them. --- " << endl;
+ LOFAR::Message msg2Send;
+ msg2Send.setTXTPayload("Manually constructed message");
+ tb.send(msg2Send);
+ fb.getMessage(receivedMsg);
+ fb.ack(receivedMsg);
+ showMessage(receivedMsg.qpidMsg());
+ compareMessages(msg2Send.qpidMsg(), receivedMsg.qpidMsg());
+
+ cout << "--- TEST 3: add an extra queue, send messages to both queues, receive them. --- " << endl;
+ ToBus tbExtra("tMsgBus-extraTestQ");
+ tbExtra.send("Message send to extra queue");
+ tb.send("Message send to original queue");
+
+ fb.addQueue("tMsgBus-extraTestQ");
+ fb.getMessage(receivedMsg);
+ fb.ack(receivedMsg);
+ showMessage(receivedMsg.qpidMsg());
+ fb.getMessage(receivedMsg);
+ fb.ack(receivedMsg);
+ showMessage(receivedMsg.qpidMsg());
+
+ cout << "--- All test successful! ---" << endl;
+
+ return (0);
+}
+
diff --git a/LCS/MessageBus/test/tMsgBus.run b/LCS/MessageBus/test/tMsgBus.run
new file mode 100755
index 0000000000000000000000000000000000000000..52872de3543ef87e064670700f9bcd882843fb17
--- /dev/null
+++ b/LCS/MessageBus/test/tMsgBus.run
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+source MessageFuncs.sh
+
+create_queue tMsgBus
+create_queue tMsgBus-extraTestQ
+
+./tMsgBus
diff --git a/LCS/MessageBus/test/tMsgBus.sh b/LCS/MessageBus/test/tMsgBus.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5dbd572389fbed059f2b93fe10ca70f52458f7f5
--- /dev/null
+++ b/LCS/MessageBus/test/tMsgBus.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+./runctest.sh tMsgBus
diff --git a/LCS/MessageDaemons/CMakeLists.txt b/LCS/MessageDaemons/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e7f292a5c6dff46d05688ea7b0f53f7874483476
--- /dev/null
+++ b/LCS/MessageDaemons/CMakeLists.txt
@@ -0,0 +1,5 @@
+# $Id$
+
+lofar_package(MessageDaemons 1.0 DEPENDS MessageBus)
+
+add_subdirectory(src)
diff --git a/LCS/MessageDaemons/src/CMakeLists.txt b/LCS/MessageDaemons/src/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..89821db1ee3b2a69c5c724524c9ea48865fec2db
--- /dev/null
+++ b/LCS/MessageDaemons/src/CMakeLists.txt
@@ -0,0 +1,20 @@
+# $Id$
+
+include(LofarPackageVersion)
+
+set(messagedaemons_LIB_SRCS
+ Package__Version.cc)
+
+lofar_add_library(messagedaemons ${messagedaemons_LIB_SRCS})
+
+install(FILES
+ MessageRouter
+ DESTINATION bin)
+
+file(GLOB MessageRouterConfs
+ MessageRouter.conf*
+ )
+
+install(FILES
+ ${MessageRouterConfs}
+ DESTINATION etc)
diff --git a/LCS/MessageDaemons/src/MessageRouter b/LCS/MessageDaemons/src/MessageRouter
new file mode 100644
index 0000000000000000000000000000000000000000..873663507e07f35050cffa6ff2fa0279aac8fdb7
--- /dev/null
+++ b/LCS/MessageDaemons/src/MessageRouter
@@ -0,0 +1,146 @@
+#!/usr/bin/python
+# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+# id.. TDB
+"""Very basic messagebus router that routs the messages according to the settings in Router.conf
+"""
+
+import lofar.messagebus.basemsgbus as msgbus
+#import lofar.messagebus.message as message
+
+import threading
+from ConfigParser import SafeConfigParser
+import os.path
+from datetime import datetime
+
+def log(level, msg):
+ print "%s %-5s %s" % (datetime.now(), level, msg)
+
+class BusMulticast(threading.Thread):
+ """
+ Sets up the router from one inbus to several outbusses
+ """
+ def __init__(self, source, destlist):
+ threading.Thread.__init__(self)
+ self.source = source
+ self.destlist = destlist
+
+ def run(self):
+ inbus = msgbus.FromBus(self.source)
+ outbusses = [msgbus.ToBus(dest) for dest in self.destlist]
+
+ log("INFO","Forwarding %s -> %s" % (self.source,self.destlist))
+
+ while True:
+ # TODO: Use a transaction (not supported yet in qpid 0.30)
+
+ msg = inbus.get()
+ if msg is None:
+ log("ERROR","Received 'None' message from %s" % (self.source,))
+ continue
+
+ log("INFO","Msg received from %s" % (self.source))
+
+ for outbus in outbusses:
+ outbus.send(msg)
+ inbus.ack(msg)
+
+ log("INFO","Forwarded %s -> %s" % (self.source,self.destlist))
+
+class RouterConfig(SafeConfigParser):
+ """
+ Router configuration. Example:
+
+ [multicast]
+ source-queue-1: dest-queue-1, dest-queue2
+ source-queue-2: dest-queue-3
+
+ """
+ def __init__(self, filename=None):
+ SafeConfigParser.__init__(self)
+
+ # set defaults
+ self.add_section('multicast')
+
+ # read configuration
+ if filename is not None:
+ self.read(filename)
+
+ def read(self, filename):
+ log("INFO","[RouterConfig] Considering reading %s" % (filename,))
+ if not os.path.isfile(filename):
+ return False
+
+ log("INFO","[RouterConfig] Reading %s" % (filename,))
+ SafeConfigParser.read(self, filename)
+ return True
+
+ def sources(self):
+ return self.options('multicast')
+
+ def destinations(self, source):
+ return [field.strip() for field in self.get('multicast', source).split(',')]
+
+if __name__ == "__main__":
+ """
+ Apply the routing specified in router.conf and router.conf.`hostname`;
+ both configration files are found in $LOFARROOT/etc, or . if $LOFARROOT is
+ not set.
+
+ Application runs forever, regardless of the number of routes. Also runs
+ forever if no routing is required, to keep behaviour consistent across
+ nodes.
+ """
+ import os
+ import platform
+ import time
+
+ if "LOFARROOT" in os.environ:
+ path = os.path.expandvars("$LOFARROOT/etc")
+ else:
+ path = "."
+
+ # read default config file
+ config = RouterConfig('%s/MessageRouter.conf' % path)
+
+ # read host-specific config file
+ my_configfile = '%s/MessageRouter.conf.%s' % (path, platform.node().lower()) # = hostname
+ config.read(my_configfile)
+
+ threadlist = []
+
+ # set up router
+ for source in config.sources():
+ destlist = config.destinations(source)
+
+ t = BusMulticast(source, destlist)
+ t.start()
+ threadlist.append(t)
+
+ log("INFO","[main] Running %s threads" % (len(threadlist),))
+
+ if not threadlist:
+ # stall forever (future: allow rereading config from config file? command queue?)
+ log("INFO","[main] Nothing to do. Sleeping forever.");
+ while True:
+ time.sleep(60)
+ else:
+ # wait for join (forever)
+ for t in threadlist:
+ t.join()
+
diff --git a/LCS/MessageDaemons/src/MessageRouter.conf.ccu001 b/LCS/MessageDaemons/src/MessageRouter.conf.ccu001
new file mode 120000
index 0000000000000000000000000000000000000000..e63542263b7d53a99b30f12d189c9e516bcfebf4
--- /dev/null
+++ b/LCS/MessageDaemons/src/MessageRouter.conf.ccu001
@@ -0,0 +1 @@
+MessageRouter.conf.ccu099
\ No newline at end of file
diff --git a/LCS/MessageDaemons/src/MessageRouter.conf.ccu099 b/LCS/MessageDaemons/src/MessageRouter.conf.ccu099
new file mode 100644
index 0000000000000000000000000000000000000000..c5249afdb23e4b517cfdf3f7ff242af74ddfb8f9
--- /dev/null
+++ b/LCS/MessageDaemons/src/MessageRouter.conf.ccu099
@@ -0,0 +1,10 @@
+[multicast]
+#
+# Routing tables
+#
+# source destinations
+lofar.task.feedback.dataproducts: otdb.task.feedback.dataproducts, local.copy.dataproducts
+lofar.task.feedback.processing: otdb.task.feedback.processing, local.copy.processing
+lofar.task.feedback.state: mac.task.feedback.state, local.copy.state
+lofar.task.specification.system: local.copy.specification.system
+
diff --git a/LCS/pyparameterset/src/pyparameterset.cc b/LCS/pyparameterset/src/pyparameterset.cc
index 20aad23021b650694a8d5a656f5b99d2911aec58..22a54009774401654e555cf47617ef26cf45918a 100644
--- a/LCS/pyparameterset/src/pyparameterset.cc
+++ b/LCS/pyparameterset/src/pyparameterset.cc
@@ -87,6 +87,15 @@ namespace LOFAR {
void adoptCollection (const PyParameterSet& parset, const string& prefix)
{ return ParameterSet::adoptCollection (parset, prefix); }
+ string __str__() const
+ { string buffer; writeBuffer(buffer); return buffer; }
+
+ string __getstate__() const
+ { return __str__(); }
+
+ void __setstate__(const string& state)
+ { adoptBuffer(state); }
+
PyParameterValue get (const string& key) const
{ return ParameterSet::get (key); }
@@ -174,6 +183,58 @@ namespace LOFAR {
vector<string> (ParameterSet::*fgetvecstring2)(const string&, const vector<string>&, bool) const =
&ParameterSet::getStringVector;
+ // Define pickling/unpickling information
+ struct pyparameterset_pickle_suite : boost::python::pickle_suite
+ {
+ /*
+ static
+ boost::python::tuple
+ gtinitargs(const world& w)
+ {
+ return boost::python::make_tuple(w.get_country());
+ }
+ */
+
+ static
+ boost::python::tuple
+ getstate(boost::python::object obj)
+ {
+ PyParameterSet const& ps = boost::python::extract<PyParameterSet const&>(obj)();
+
+ return boost::python::make_tuple(
+ obj.attr("__dict__"),
+ ps.__str__());
+ }
+
+ static
+ void
+ setstate(boost::python::object obj, boost::python::tuple state)
+ {
+ using namespace boost::python;
+ PyParameterSet& ps = extract<PyParameterSet&>(obj)();
+
+ if (len(state) != 2)
+ {
+ PyErr_SetObject(PyExc_ValueError,
+ ("expected 2-item tuple in call to __setstate__; got %s"
+ % state).ptr()
+ );
+ throw_error_already_set();
+ }
+
+ // restore the object's __dict__
+ dict d = extract<dict>(obj.attr("__dict__"))();
+ d.update(state[0]);
+
+ // restore the internal state of the C++ object
+ string parsetStr = extract<string>(state[1]);
+ ps.adoptBuffer(parsetStr);
+ }
+
+ static bool getstate_manages_dict() { return true; }
+ };
+
+
// Define the python interface to ParameterValue.
void pyparametervalue()
@@ -226,6 +287,7 @@ namespace LOFAR {
.def (init<PyParameterSet>())
.def (init<bool, int, int>())
.def (init<std::string, bool>())
+ .def_pickle(pyparameterset_pickle_suite())
.def ("version", &PyParameterSet::version,
(boost::python::arg("type")="other"),
"Get the software version.")
@@ -252,6 +314,8 @@ namespace LOFAR {
(boost::python::arg("filename"),
boost::python::arg("append")=false),
"Write the parameterset into a parset file with the given name.")
+ .def ("__str__", &PyParameterSet::__str__,
+ "Return the full parset as a string")
.def ("add", fadd,
(boost::python::arg("key"),
boost::python::arg("value")),
diff --git a/LCS/pyparameterset/test/tpyparameterset.py b/LCS/pyparameterset/test/tpyparameterset.py
index be0dd2cd79b183950a98d4add4bb4b9c1bea2f22..272d166282c85d056e6886cec71f34d5b7d6eff6 100644
--- a/LCS/pyparameterset/test/tpyparameterset.py
+++ b/LCS/pyparameterset/test/tpyparameterset.py
@@ -67,7 +67,7 @@ print ps.version("top")
print ps.version()
print "<<<"
ps.add ("a.b", "7")
-ps.add ("a.b.lange_naam", "dit is nu een andere naam geworden met extra spaties aan het einde ")
+ps.add ("a.b.lange_naam", "dit is nu een andere naam geworden zonder extra spaties aan het einde want die gaan verloren bij wegschrijven + teruglezen")
ps.add ("a.b.c", "5")
ps.add ("a.b.double", "3.1415926")
ps.add ("a.b.bool", "true")
@@ -88,3 +88,12 @@ print pss.makeSubset('b.', 'aa.bb.').keys()
print pss.makeSubset('b.').size()
print pss.makeSubset('cc').keys() # should be empty
print len(pss.makeSubset('cc'))
+
+# Check str()
+print str(ps)
+
+# Check picking/unpickling
+import cPickle as pickle
+s = pickle.dumps(ps)
+ps2 = pickle.loads(s)
+assert str(ps) == str(ps2)
diff --git a/LCS/pyparameterset/test/tpyparameterset.stdout b/LCS/pyparameterset/test/tpyparameterset.stdout
index d11dc795152badf639efae6bf85e38e2a36b202b..287a9a8ebc8cd221ce48bf3b902f9b97dd3b6275 100644
--- a/LCS/pyparameterset/test/tpyparameterset.stdout
+++ b/LCS/pyparameterset/test/tpyparameterset.stdout
@@ -38,6 +38,50 @@ True
[1, 2, 3, 10, 10, 10, 10, 10]
[5, 6, 7, 8, 9, 10]
+>>>
+parameterset: version = LOFAR-Task7336 (in CMakeLists.txt: 1.0)
+ overall revision = 30929
+ package revision = 30927 (last change in package)
+ built on cbt009 by mol at Feb 10 2015 11:04:26
+ 2 files were different from the repository
+ tree of packages used:
+ pyparameterset-LOFAR-Task7336 (rev.30929)
+ pytools-LOFAR-Task7336 (rev.30929)
+ Common-LOFAR-Task7336 (rev.30929)
+
+parameterset: version = LOFAR-Task7336 (in CMakeLists.txt: 1.0)
+ overall revision = 30929
+ package revision = 30927 (last change in package)
+ built on cbt009 by mol at Feb 10 2015 11:04:26
+ 2 files were different from the repository
+ packages used:
+
+ pytools: version = LOFAR-Task7336 (in CMakeLists.txt: 1.0)
+ overall revision = 30929
+ package revision = 30536 (last change in package)
+ built on cbt009 by mol at Feb 10 2015 10:37:11
+
+ Common: version = LOFAR-Task7336 (in CMakeLists.txt: 3.3)
+ overall revision = 30929
+ package revision = 30651 (last change in package)
+ built on cbt009 by mol at Feb 10 2015 10:37:07
+
+parameterset: version = LOFAR-Task7336 (in CMakeLists.txt: 1.0)
+ overall revision = 30929
+ package revision = 30927 (last change in package)
+ built on cbt009 by mol at Feb 10 2015 11:04:26
+ 2 files were different from the repository
+
+parameterset: version = LOFAR-Task7336 (in CMakeLists.txt: 1.0)
+ overall revision = 30929
+ package revision = 30927 (last change in package)
+ built on cbt009 by mol at Feb 10 2015 11:04:26
+ 2 files were different from the repository
+ packages used:
+ pytools-LOFAR-Task7336 (rev.30929)
+ Common-LOFAR-Task7336 (rev.30929)
+
+<<<
['a.b', 'a.b.bool', 'a.b.c', 'a.b.double', 'a.b.lange_naam', 'e.g', 'egg', 'vec', 'vecbool', 'vecexp', 'vecnest']
False
True
@@ -45,7 +89,7 @@ True
7
7
aa
-dit is nu een andere naam geworden met extra spaties aan het einde
+dit is nu een andere naam geworden zonder extra spaties aan het einde want die gaan verloren bij wegschrijven + teruglezen
True
True
False
@@ -84,3 +128,15 @@ b.c = 5
4
[]
0
+a.b=7
+a.b.bool=true
+a.b.c=5
+a.b.double=3.1415926
+a.b.lange_naam=dit is nu een andere naam geworden zonder extra spaties aan het einde want die gaan verloren bij wegschrijven + teruglezen
+e.g=een voorbeeld
+egg=een ei
+vec=[1,2,3]
+vecbool=[true,false,true]
+vecexp=[1..3,5..10]
+vecnest=[[1..3,5*10],[5..10]]
+
diff --git a/MAC/APL/APLCommon/src/swlevel b/MAC/APL/APLCommon/src/swlevel
index 7d02ef091231df0f81f5b2855be015ba05ee7ca3..fd40de6370890cad8883f05be135e2116a1e4ef9 100644
--- a/MAC/APL/APLCommon/src/swlevel
+++ b/MAC/APL/APLCommon/src/swlevel
@@ -175,7 +175,7 @@ start_prog()
fi
# Check if program is already running
- /sbin/pidof ${prog} 1>/dev/null 2>&1
+ /sbin/pidof -x ${prog} 1>/dev/null 2>&1
if [ $? -ne 0 ]; then
curdate=`date +%Y%m%dT%H%M%S`
# PVSS needs special treatment
@@ -248,7 +248,7 @@ stop_prog()
fi
# get processlist
- /sbin/pidof ${prog} 1>/dev/null 2>&1
+ /sbin/pidof -x ${prog} 1>/dev/null 2>&1
if [ $? -ne 0 ]; then
return
fi
@@ -267,7 +267,7 @@ stop_prog()
fi
# first try normal kill
- for pid in `/sbin/pidof ${prog}`
+ for pid in `/sbin/pidof -x ${prog}`
do
echo "Softly killing ${prog}(${pid})"
$asroot kill $pid 1>/dev/null 2>&1
@@ -275,7 +275,7 @@ stop_prog()
done
# when normal kill did not work, kill is with -9
- for pid in `/sbin/pidof ${prog}`
+ for pid in `/sbin/pidof -x ${prog}`
do
echo "Hard killing ${prog}(${pid})"
$asroot kill -9 $pid 1>/dev/null 2>&1
@@ -283,7 +283,7 @@ stop_prog()
done
# if user0 or lofarsys, try normal kill as root
- for pid in `/sbin/pidof ${prog}`
+ for pid in `/sbin/pidof -x ${prog}`
do
if [ "$user" == "user0" -o "$user" == "lofarsys" ]; then
sudo kill $pid 1>/dev/null 2>&1
@@ -292,7 +292,7 @@ stop_prog()
done
# if user0 or lofarsys, try hard kill as root
- for pid in `/sbin/pidof ${prog}`
+ for pid in `/sbin/pidof -x ${prog}`
do
if [ "$user" == "user0" -o "$user" == "lofarsys" ]; then
sudo kill -9 $pid 1>/dev/null 2>&1
@@ -301,7 +301,7 @@ stop_prog()
done
# if still alive, write a message
- for pid in `/sbin/pidof ${prog}`
+ for pid in `/sbin/pidof -x ${prog}`
do
echo -n "Could not kill ${prog}(${pid}); "
if [ "$user" == "user0" -o "$user" == "lofarsys" ]; then
@@ -350,9 +350,9 @@ status_prog()
# find out the processID of the possibly (running) process
obsid=()
pid_user=()
- /sbin/pidof ${prog} 1>/dev/null 2>&1
+ /sbin/pidof -x ${prog} 1>/dev/null 2>&1
if [ $? -eq 0 ]; then
- pid=( `/sbin/pidof ${prog}` )
+ pid=( `/sbin/pidof -x ${prog}` )
i=0
for apid in ${pid[@]}
do
diff --git a/MAC/APL/APLCommon/src/swlevel.conf b/MAC/APL/APLCommon/src/swlevel.conf
index 6abc540fac70337cac2e0b516d55819dde4d5d63..b9e9922bc3e9c9f8c98324c6576b1c8a59d9cf75 100644
--- a/MAC/APL/APLCommon/src/swlevel.conf
+++ b/MAC/APL/APLCommon/src/swlevel.conf
@@ -4,6 +4,7 @@
# Table to manage the progrma that should be started and stopped
# level : up : down : root : mpi : program
#
+1:u:d:::MessageRouter
1:u:d:::PVSS00pmon
1:u:d:::SoftwareMonitor
1:u:d:::LogProcessor
@@ -11,6 +12,7 @@
1:u:d:::ServiceBroker
1:u:d:::SASGateway
1:u:d:::PVSSGateway
+1:u:d:::FeedbackService
#
2:u:d:r::_EPAStub
2:u:d:r::RSPDriver
diff --git a/MAC/APL/CEPCU/CMakeLists.txt b/MAC/APL/CEPCU/CMakeLists.txt
index f7b97cd31136c339dbcd2294693f547b144465fd..8013b1c814bc7c99ebd974c861b1539933205631 100644
--- a/MAC/APL/CEPCU/CMakeLists.txt
+++ b/MAC/APL/CEPCU/CMakeLists.txt
@@ -1,7 +1,7 @@
# $Id$
# Do not split the following line, otherwise makeversion will fail!
-lofar_package(CEPCU 1.0 DEPENDS Common ApplCommon MACIO GCFTM GCFRTDB APLCommon RTDBCommon OTDB Stream)
+lofar_package(CEPCU 1.0 DEPENDS Common ApplCommon MACIO GCFTM GCFRTDB APLCommon RTDBCommon OTDB Stream MessageBus)
include(LofarFindPackage)
lofar_find_package(Boost REQUIRED COMPONENTS date_time)
diff --git a/MAC/APL/CEPCU/src/OnlineControl/CMakeLists.txt b/MAC/APL/CEPCU/src/OnlineControl/CMakeLists.txt
index 04c131069de8e6f5572de6bf0185a3c8d60aacdc..753e334b009b96c1ba092e218d5ccf65202568dd 100644
--- a/MAC/APL/CEPCU/src/OnlineControl/CMakeLists.txt
+++ b/MAC/APL/CEPCU/src/OnlineControl/CMakeLists.txt
@@ -12,3 +12,7 @@ lofar_add_bin_program(OnlineControl
forkexec.cc)
lofar_add_executable(tPVSSMapping tPVSSMapping.cc)
+
+install(FILES
+ OnlineControl.conf
+ DESTINATION etc)
diff --git a/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.cc b/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.cc
index 8fd5d8e1b2fa0a6d5803ac070aeeac8a68bcaeb2..6afb4bb872440cdccfe704581360c215772750f7 100644
--- a/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.cc
+++ b/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.cc
@@ -31,6 +31,7 @@
#include <Common/Exceptions.h>
#include <Common/SystemUtil.h>
#include <Common/hexdump.h>
+#include <MessageBus/Message.h>
#include <ApplCommon/StationInfo.h>
#include <ApplCommon/Observation.h>
#include <ApplCommon/LofarDirs.h>
@@ -65,6 +66,8 @@ namespace LOFAR {
// static pointer to this object for signal handler
static OnlineControl* thisOnlineControl = 0;
+const double QUEUE_POLL_TIMEOUT = 1.0;
+
//
// OnlineControl()
//
@@ -80,9 +83,9 @@ OnlineControl::OnlineControl(const string& cntlrName) :
itsForcedQuitTimer (0),
itsLogControlPort (0),
itsState (CTState::NOSTATE),
- itsFeedbackListener (0), // QUICK FIX #4022
- itsFeedbackPort (0), // QUICK FIX #4022
- itsFeedbackResult (CT_RESULT_NO_ERROR), // QUICK FIX #4022
+ itsMsgQueue (0),
+ itsQueueTimer (0),
+ itsFeedbackResult (CT_RESULT_NO_ERROR),
itsTreePrefix (""),
itsInstanceNr (0),
itsStartTime (),
@@ -110,14 +113,13 @@ OnlineControl::OnlineControl(const string& cntlrName) :
// need port for timers.
itsTimerPort = new GCFTimerPort(*this, "TimerPort");
ASSERTSTR(itsTimerPort, "Can't allocate the timer!");
+ itsQueueTimer = new GCFTimerPort(*this, "MsgQTimer");
+ ASSERTSTR(itsQueueTimer, "Cannot allocate queue timer");
// Controlport to logprocessor
itsLogControlPort = new GCFTCPPort(*this, MAC_SVCMASK_CEPLOGCONTROL, GCFPortInterface::SAP, CONTROLLER_PROTOCOL);
ASSERTSTR(itsLogControlPort, "Can't allocate the logControlPort");
- // QUICK FIX #4022
- itsFeedbackListener = new GCFTCPPort (*this, "Feedbacklistener", GCFPortInterface::MSPP, CONTROLLER_PROTOCOL);
- ASSERTSTR(itsFeedbackListener, "Cannot allocate TCP port for feedback");
itsForcedQuitTimer = new GCFTimerPort(*this, "EmergencyTimer");
ASSERTSTR(itsForcedQuitTimer, "Can't allocate the emergency timer!");
itsForceTimeout = globalParameterSet()->getTime("emergencyTimeout", 3600);
@@ -141,14 +143,9 @@ OnlineControl::~OnlineControl()
delete itsLogControlPort;
}
- if (itsTimerPort) {
- delete itsTimerPort;
- }
-
- if (itsFeedbackListener) {
- itsFeedbackListener->close();
- delete itsFeedbackListener;
- }
+ delete itsTimerPort;
+ delete itsQueueTimer;
+ delete itsMsgQueue;
}
//
@@ -268,11 +265,12 @@ GCFEvent::TResult OnlineControl::initial_state(GCFEvent& event, GCFPortInterface
itsParentPort = itsParentControl->registerTask(this);
// results in CONTROL_CONNECT
- // QUICK FIX #4022
- uint32 obsID = globalParameterSet()->getUint32("Observation.ObsID");
- LOG_INFO_STR("Openening feedback port for OLAP: " << MAC_ONLINE_FEEDBACK_QF + obsID%1000);
- itsFeedbackListener->setPortNumber(MAC_ONLINE_FEEDBACK_QF + obsID%1000);
- itsFeedbackListener->open(); // will result in F_CONN
+ // open connection with messagebus
+ if (!itsMsgQueue) {
+ string queueName = globalParameterSet()->getString("TaskStateQueue");
+ itsMsgQueue = new FromBus(queueName);
+ LOG_INFO_STR("Starting to listen on " << queueName);
+ }
LOG_DEBUG ("Going to operational state");
TRAN(OnlineControl::active_state); // go to next state.
@@ -310,11 +308,7 @@ GCFEvent::TResult OnlineControl::active_state(GCFEvent& event, GCFPortInterface&
// update PVSS
itsPropertySet->setValue(PN_FSM_CURRENT_ACTION, GCFPVString("active"));
itsPropertySet->setValue(PN_FSM_ERROR, GCFPVString(""));
- } break;
-
- // QUICKFIX #4022
- case F_ACCEPT_REQ: {
- _handleAcceptRequest(port);
+ itsQueueTimer->setTimer(QUEUE_POLL_TIMEOUT);
} break;
case F_CONNECTED: {
@@ -326,18 +320,35 @@ GCFEvent::TResult OnlineControl::active_state(GCFEvent& event, GCFPortInterface&
_handleDisconnect(port);
} break;
- // QUICKFIX #4022
- case F_DATAIN: {
- _handleDataIn(port);
- } break;
-
case DP_CHANGED:
_databaseEventHandler(event);
break;
case F_TIMER: {
GCFTimerEvent& timerEvent=static_cast<GCFTimerEvent&>(event);
- if (timerEvent.id == itsStopTimerID) {
+ if (&port == itsQueueTimer) {
+ Message msg;
+ if (itsMsgQueue->getMessage(msg, 0.1)) {
+ string obsIDstr = msg.getXMLvalue("message.header.ids.sasid");
+ LOG_INFO_STR("Received message from task " << obsIDstr);
+ if (atoi(obsIDstr.c_str()) == itsObsID) {
+ string result = msg.getXMLvalue("message.payload.task.state");
+ if (result == "aborted") {
+ itsFeedbackResult = CT_RESULT_PIPELINE_FAILED;
+ }
+ else if (result != "finished") {
+ LOG_FATAL_STR("Unknown result received from correlator: " << result << " assuming failure!");
+ itsFeedbackResult = CT_RESULT_PIPELINE_FAILED;
+ }
+ LOG_INFO_STR("Received '" << result << "' on the messagebus");
+ itsMsgQueue->ack(msg);
+ TRAN(OnlineControl::finishing_state);
+ break;
+ } // my obsid
+ } // getMsg
+ itsQueueTimer->setTimer(QUEUE_POLL_TIMEOUT);
+ }
+ else if (timerEvent.id == itsStopTimerID) {
LOG_DEBUG("StopTimer expired, starting QUIT sequence");
itsStopTimerID = 0;
_setState(CTState::QUIT);
@@ -360,9 +371,10 @@ GCFEvent::TResult OnlineControl::active_state(GCFEvent& event, GCFPortInterface&
CONTROLConnectEvent msg(event);
LOG_DEBUG_STR("Received CONNECT(" << msg.cntlrName << ")");
itsMyName = msg.cntlrName;
+ itsObsID = getObservationNr(msg.cntlrName);
// first inform CEPlogProcessor
CONTROLAnnounceEvent announce;
- announce.observationID = toString(getObservationNr(msg.cntlrName));
+ announce.observationID = toString(itsObsID);
itsLogControlPort->send(announce);
// execute this state
_setState(CTState::CONNECT);
@@ -442,45 +454,6 @@ GCFEvent::TResult OnlineControl::active_state(GCFEvent& event, GCFPortInterface&
return (status);
}
-//
-// completing_state(event, port)
-//
-//
-GCFEvent::TResult OnlineControl::completing_state(GCFEvent& event, GCFPortInterface& port)
-{
- LOG_INFO_STR ("completing:" << eventName(event) << "@" << port.getName());
-
- switch (event.signal) {
- case F_ENTRY: {
- // update PVSS
- itsPropertySet->setValue(PN_FSM_CURRENT_ACTION, GCFPVString("completing"));
- itsPropertySet->setValue(PN_FSM_ERROR, GCFPVString(""));
- itsInFinishState = true;
-
- _passMetadatToOTDB();
-
- TRAN(OnlineControl::finishing_state);
- } break;
-
- case F_TIMER:
- break;
-
- case F_DISCONNECTED:
- _handleDisconnect(port);
- break;
-
- case F_DATAIN:
- _handleDataIn(port);
- break;
-
- default:
- LOG_DEBUG("completing state default");
- return (GCFEvent::NOT_HANDLED);
- }
-
- return (GCFEvent::HANDLED);
-}
-
//
// finishing_state(event, port)
//
@@ -511,10 +484,6 @@ GCFEvent::TResult OnlineControl::finishing_state(GCFEvent& event, GCFPortInterfa
_handleDisconnect(port);
break;
- case F_DATAIN:
- _handleDataIn(port);
- break;
-
default:
LOG_DEBUG("finishing_state default");
return (GCFEvent::NOT_HANDLED);
@@ -798,91 +767,6 @@ void OnlineControl::_clearCobaltDatapoints()
delete myDPservice;
}
-//
-// _passMetadatToOTDB();
-// THIS ROUTINE IS A MODIFIED COPY FROM PYTHONCONTROL.CC
-//
-void OnlineControl::_passMetadatToOTDB()
-{
- // No name specified?
- bool metadataFileAvailable (true);
- uint32 obsID(globalParameterSet()->getUint32("Observation.ObsID", 0));
- string feedbackFile = observationParset(obsID)+"_feedback";
- LOG_INFO_STR ("Expecting metadata to be in file " << feedbackFile);
- if (feedbackFile.empty()) {
- metadataFileAvailable = false;
- }
-
- // read parameterset
- // Try to setup the connection with the database
- string confFile = globalParameterSet()->getString("OTDBconfFile", "SASGateway.conf");
- ConfigLocator CL;
- string filename = CL.locate(confFile);
- LOG_INFO_STR("Trying to read database information from file " << filename);
- ParameterSet otdbconf;
- otdbconf.adoptFile(filename);
- string database = otdbconf.getString("SASGateway.OTDBdatabase");
- string dbhost = otdbconf.getString("SASGateway.OTDBhostname");
- OTDBconnection conn("paulus", "boskabouter", database, dbhost);
- if (!conn.connect()) {
- LOG_FATAL_STR("Cannot connect to database " << database << " on machine " << dbhost);
- // WE DO HAVE A PROBLEM HERE BECAUSE THIS PIPELINE CANNOT BE SET TO FINISHED IN SAS.
- return;
- }
- LOG_INFO_STR("Connected to database " << database << " on machine " << dbhost);
-
- if (metadataFileAvailable) {
- try {
- TreeValue tv(&conn, getObservationNr(getName()));
- ParameterSet metadata;
- metadata.adoptFile(feedbackFile);
- // Loop over the parameterset and send the information to the KVTlogger.
- // During the transition phase from parameter-based to record-based storage in OTDB the
- // nodenames ending in '_' are implemented both as parameter and as record.
- ParameterSet::iterator iter = metadata.begin();
- ParameterSet::iterator end = metadata.end();
- while (iter != end) {
- string key(iter->first); // make destoyable copy
- rtrim(key, "[]0123456789");
- // bool doubleStorage(key[key.size()-1] == '_');
- bool isRecord(iter->second.isRecord());
- // isRecord doubleStorage
- // --------------------------------------------------------------
- // Y Y store as record and as parameters
- // Y N store as parameters
- // N * store parameter
- if (!isRecord) {
- LOG_DEBUG_STR("BASIC: " << iter->first << " = " << iter->second);
- tv.addKVT(iter->first, iter->second, ptime(microsec_clock::local_time()));
- }
- else {
- // if (doubleStorage) {
- // LOG_DEBUG_STR("RECORD: " << iter->first << " = " << iter->second);
- // tv.addKVT(iter->first, iter->second, ptime(microsec_clock::local_time()));
- // }
- // to store is a node/param values the last _ should be stipped of
- key = iter->first; // destroyable copy
- // string::size_type pos = key.find_last_of('_');
- // key.erase(pos,1);
- ParameterRecord pr(iter->second.getRecord());
- ParameterRecord::const_iterator prIter = pr.begin();
- ParameterRecord::const_iterator prEnd = pr.end();
- while (prIter != prEnd) {
- LOG_DEBUG_STR("ELEMENT: " << key+"."+prIter->first << " = " << prIter->second);
- tv.addKVT(key+"."+prIter->first, prIter->second, ptime(microsec_clock::local_time()));
- prIter++;
- }
- }
- iter++;
- }
- LOG_INFO_STR(metadata.size() << " metadata values send to SAS");
- }
- catch (APSException &e) {
- // Parameterfile not found
- LOG_FATAL(e.text());
- }
- }
-}
// -------------------- Application-order administration --------------------
//
@@ -891,55 +775,6 @@ void OnlineControl::_passMetadatToOTDB()
void OnlineControl::_handleDisconnect(GCFPortInterface& port)
{
port.close();
- // QUICKFIX #4022
- if (&port == itsFeedbackPort) {
- LOG_INFO_STR("Lost connection with Feedback of OLAP.");
- delete itsFeedbackPort;
- itsFeedbackPort = 0;
- }
-}
-
-//
-// _handleAcceptRequest(port)
-//
-void OnlineControl::_handleAcceptRequest(GCFPortInterface& port)
-{
- ASSERTSTR(&port == itsFeedbackListener, "Incoming connection on main listener iso feedbackListener");
- itsFeedbackPort = new GCFTCPPort();
- itsFeedbackPort->init(*this, "feedback", GCFPortInterface::SPP, 0, true); // raw port
- if (!itsFeedbackListener->accept(*itsFeedbackPort)) {
- delete itsFeedbackPort;
- itsFeedbackPort = 0;
- LOG_ERROR("Connection with Python feedback FAILED");
- }
- else {
- LOG_INFO("Connection made on feedback port, accepting commands");
- }
-}
-
-//
-// _handleDataIn(port)
-//
-void OnlineControl::_handleDataIn(GCFPortInterface& port)
-{
- ASSERTSTR(&port == itsFeedbackPort, "Didn't expect raw data on port " << port.getName());
- char buf[1024];
- ssize_t btsRead = port.recv((void*)&buf[0], 1023);
- buf[btsRead] = '\0';
- string s;
- hexdump(s, buf, btsRead);
- LOG_INFO_STR("Received command on feedback port: " << s);
-
- if (!strcmp(buf, "ABORT")) {
- itsFeedbackResult = CT_RESULT_PIPELINE_FAILED;
- TRAN(OnlineControl::completing_state); // pass metadata
- }
- else if (!strcmp(buf, "FINISHED")) {
- TRAN(OnlineControl::completing_state);
- }
- else {
- LOG_FATAL_STR("Received command on feedback port unrecognized");
- }
}
}; // CEPCU
diff --git a/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.conf b/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.conf
index 628c55fa5cf86fa0ba6c8a83c675aadf9c7c06d4..c0c088bf99f5a4a1f058c99265d2713b63ed7249 100644
--- a/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.conf
+++ b/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.conf
@@ -5,5 +5,8 @@
# Host on which the KVTLogger runs (normally localhost)
OTDBconfFile = SASGateway.conf
+# Queue to listen at.
+TaskStateQueue = mac.task.feedback.state
+
# Max time to wait for response from Python Framework when quiting
emergencyTimeout = 1h
diff --git a/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.h b/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.h
index a3fe3a4fc85b6f86a9dbab5d1b33ff89e580557c..d6daf80a70afdcf578ef0293ade9c40a139757e5 100644
--- a/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.h
+++ b/MAC/APL/CEPCU/src/OnlineControl/OnlineControl.h
@@ -27,10 +27,11 @@
#include <Common/lofar_string.h>
#include <Common/lofar_vector.h>
#include <Common/LofarLogger.h>
-
-//# ACC Includes
#include <Common/ParameterSet.h>
+//# MsgBus Includes
+#include <MessageBus/MsgBus.h>
+
//# GCF Includes
#include <GCF/TM/GCF_Control.h>
#include <GCF/PVSS/PVSSservice.h>
@@ -90,15 +91,13 @@ private:
void _stopApplications();
void _finishController (uint16_t result);
void _handleDisconnect (GCFPortInterface& port);
- void _handleAcceptRequest (GCFPortInterface& port);
- void _handleDataIn (GCFPortInterface& port);
void _setState (CTState::CTstateNr newState);
void _databaseEventHandler(GCFEvent& event);
- void _passMetadatToOTDB ();
void _clearCobaltDatapoints();
// ----- datamembers -----
string itsMyName;
+ int itsObsID;
RTDBPropertySet* itsPropertySet;
bool itsPropertySetInitialized;
PVSSservice* itsPVSSService;
@@ -115,9 +114,8 @@ private:
CTState::CTstateNr itsState;
- // QUICK FIX #4022
- GCFTCPPort* itsFeedbackListener;
- GCFTCPPort* itsFeedbackPort;
+ FromBus* itsMsgQueue;
+ GCFTimerPort* itsQueueTimer;
int itsFeedbackResult;
// ParameterSet variables
diff --git a/MAC/APL/CEPCU/src/OnlineControl/OnlineControlMain.cc b/MAC/APL/CEPCU/src/OnlineControl/OnlineControlMain.cc
index e3356ed014303d8897d4e7eb7954bbd60a614e93..9963bec3477a7bdec1d347369d15d89a7cfc0c5c 100644
--- a/MAC/APL/CEPCU/src/OnlineControl/OnlineControlMain.cc
+++ b/MAC/APL/CEPCU/src/OnlineControl/OnlineControlMain.cc
@@ -23,6 +23,7 @@
#include <lofar_config.h>
#include <Common/LofarLogger.h>
#include <Common/Exception.h>
+#include <MessageBus/MsgBus.h>
#include "OnlineControl.h"
@@ -44,6 +45,8 @@ int main(int argc, char* argv[])
GCFScheduler::instance()->init(argc, argv, argv[1]);
+ MessageBus::init();
+
ParentControl* pc = ParentControl::instance();
pc->start(); // make initial transition
diff --git a/MAC/APL/CEPCU/src/PythonControl/PythonControl.cc b/MAC/APL/CEPCU/src/PythonControl/PythonControl.cc
index 3bd66084edb679c58e5ab8bda4ac3ed4f94fa646..f6e1d880f10618d39615a14a1683a4943186bba7 100644
--- a/MAC/APL/CEPCU/src/PythonControl/PythonControl.cc
+++ b/MAC/APL/CEPCU/src/PythonControl/PythonControl.cc
@@ -50,6 +50,7 @@
using namespace std;
using namespace boost::posix_time;
+using qpid::messaging::Message;
namespace LOFAR {
using namespace APLCommon;
@@ -62,6 +63,7 @@ namespace LOFAR {
// static pointer to this object for signal handler
static PythonControl* thisPythonControl = 0;
+const double QUEUE_POLL_TIMEOUT = 1.0;
//
// PythonControl()
//
@@ -72,14 +74,13 @@ PythonControl::PythonControl(const string& cntlrName) :
itsParentControl (0),
itsParentPort (0),
itsTimerPort (0),
+ itsQueueTimer (0),
itsForcedQuitTimer (0),
itsListener (0),
- itsFeedbackListener (0), // QUICK FIX #3633
- itsFeedbackPort (0), // QUICK FIX #3633
- itsFeedbackResult (CT_RESULT_NO_ERROR), // QUICK FIX #3633
+ itsMsgQueue (0),
+ itsFeedbackResult (CT_RESULT_NO_ERROR),
itsPythonPort (0),
itsState (CTState::NOSTATE),
- itsFeedbackFile (""),
itsForceTimeout (3600.0)
{
LOG_TRACE_OBJ_STR (cntlrName << " construction");
@@ -97,9 +98,6 @@ PythonControl::PythonControl(const string& cntlrName) :
itsListener = new GCFTCPPort (*this, "listener", GCFPortInterface::MSPP, CONTROLLER_PROTOCOL);
ASSERTSTR(itsListener, "Cannot allocate TCP port for server port");
- // QUICK FIX #3633
- itsFeedbackListener = new GCFTCPPort (*this, "Feedbacklistener", GCFPortInterface::MSPP, CONTROLLER_PROTOCOL);
- ASSERTSTR(itsFeedbackListener, "Cannot allocate TCP port for feedback");
itsForcedQuitTimer = new GCFTimerPort(*this, "ForcedQuitTimer");
ASSERTSTR(itsForcedQuitTimer, "Cannot allocate emergency quit timer");
itsForceTimeout = globalParameterSet()->getTime("emergencyTimeout", 3600);
@@ -107,6 +105,8 @@ PythonControl::PythonControl(const string& cntlrName) :
// need port for timers.
itsTimerPort = new GCFTimerPort(*this, "TimerPort");
ASSERTSTR(itsTimerPort, "Cannot allocate the timer");
+ itsQueueTimer = new GCFTimerPort(*this, "MsgQTimer");
+ ASSERTSTR(itsQueueTimer, "Cannot allocate queue timer");
// for debugging purposes
registerProtocol (CONTROLLER_PROTOCOL, CONTROLLER_PROTOCOL_STRINGS);
@@ -125,10 +125,9 @@ PythonControl::~PythonControl()
delete itsListener;
}
- if (itsFeedbackListener) { // QUICK FIX #3633
- itsFeedbackListener->close();
- delete itsFeedbackListener;
- }
+ delete itsTimerPort;
+ delete itsQueueTimer;
+ delete itsMsgQueue;
}
//
@@ -203,9 +202,6 @@ bool PythonControl::_startPython(const string& pythonProg,
}
}
- // Readin parameters from the obsercationfile.
- itsFeedbackFile = observationParset(obsID)+"_feedback";
- LOG_INFO_STR ("Expect metadata to be in file " << itsFeedbackFile);
// construct system command
string startCmd;
@@ -307,11 +303,6 @@ GCFEvent::TResult PythonControl::initial_state(GCFEvent& event, GCFPortInterface
switch (event.signal) {
case F_ENTRY: {
itsListener->open(); // will result in F_CONN
- // QUICK FIX #3633
- unsigned portNr = MAC_PYTHON_FEEDBACK_QF + getObservationNr(getName()) % 7000;
- LOG_INFO_STR("Listening for feedback on port " << portNr);
- itsFeedbackListener->setPortNumber(portNr);
- itsFeedbackListener->open(); // will result in F_CONN
}
break;
@@ -353,6 +344,7 @@ GCFEvent::TResult PythonControl::initial_state(GCFEvent& event, GCFPortInterface
case CONTROL_CONNECT: {
CONTROLConnectEvent msg(event);
itsMyName = msg.cntlrName;
+ itsObsID = getObservationNr(getName());
// request from parent task to start up the child side.
ParameterSet* thePS = globalParameterSet(); // shortcut to global PS.
@@ -361,9 +353,8 @@ GCFEvent::TResult PythonControl::initial_state(GCFEvent& event, GCFPortInterface
string pythonHost (thePS->getString(myPrefix+"pythonHost", "@pythonHost@"));
itsChildCanCommunicate = thePS->getBool (myPrefix+"canCommunicate", true);
// START PYTHON
- // QUICK FIX #3633: if-else nesting was different
if (itsChildCanCommunicate) {
- bool startOK = _startPython(pythonProg, getObservationNr(getName()), realHostname(pythonHost),
+ bool startOK = _startPython(pythonProg, itsObsID, realHostname(pythonHost),
itsListener->makeServiceName());
if (!startOK) {
LOG_ERROR("Failed to start the Python environment.");
@@ -421,12 +412,18 @@ GCFEvent::TResult PythonControl::waitForConnection_state(GCFEvent& event, GCFPor
break;
case F_TIMER: {
- LOG_FATAL("Python environment does not respond! QUITING!");
- CONTROLConnectedEvent answer;
- answer.cntlrName = itsMyName;
- answer.result = CONTROL_LOST_CONN_ERR;
- itsParentPort->send(answer);
- finish(CT_RESULT_PIPELINE_FAILED);
+ if (&port == itsForcedQuitTimer) {
+ LOG_WARN("Aborting program on emergency timer!");
+ finish(CT_RESULT_EMERGENCY_TIMEOUT);
+ }
+ if (&port == itsTimerPort) {
+ LOG_FATAL("Python environment does not respond! QUITING!");
+ CONTROLConnectedEvent answer;
+ answer.cntlrName = itsMyName;
+ answer.result = CONTROL_LOST_CONN_ERR;
+ itsParentPort->send(answer);
+ finish(CT_RESULT_PIPELINE_FAILED);
+ }
} break;
case F_EXIT:
@@ -504,22 +501,12 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
itsPropertySet->setValue(PN_FSM_CURRENT_ACTION, GCFPVString("active"));
itsPropertySet->setValue(PN_FSM_ERROR, GCFPVString(""));
#endif
- }
- break;
-
- // QUICK FIX #3633
- case F_ACCEPT_REQ: {
- ASSERTSTR(&port == itsFeedbackListener, "Incoming connection on main listener iso feedbackListener");
- itsFeedbackPort = new GCFTCPPort();
- itsFeedbackPort->init(*this, "feedback", GCFPortInterface::SPP, 0, true); // raw port
- if (!itsFeedbackListener->accept(*itsFeedbackPort)) {
- delete itsFeedbackPort;
- itsFeedbackPort = 0;
- LOG_ERROR("Connection with Python feedback FAILED");
- }
- else {
- LOG_INFO("Connection made on feedback port, accepting commands");
+ if (!itsMsgQueue) {
+ string queueName = globalParameterSet()->getString("TaskStateQueue");
+ itsMsgQueue = new FromBus(queueName);
+ LOG_INFO_STR("Starting to listen on " << queueName);
}
+ itsQueueTimer->setTimer(QUEUE_POLL_TIMEOUT);
} break;
case F_DISCONNECTED: {
@@ -528,33 +515,37 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
LOG_FATAL_STR("Lost connection with Python, going to wait for a new connection");
TRAN(PythonControl::waitForConnection_state);
}
- // QUICK FIX #3633
- if (&port == itsFeedbackPort) {
- LOG_FATAL_STR("Lost connection with Feedback of PythonFramework.");
- delete itsFeedbackPort;
- itsFeedbackPort = 0;
+ else {
+ LOG_FATAL_STR("Lost connection with unknown port! (" << port.getName() << ")");
}
} break;
- // QUICK FIX #3633
- case F_DATAIN: {
- ASSERTSTR(&port == itsFeedbackPort, "Didn't expect raw data on port " << port.getName());
- char buf[1024];
- ssize_t btsRead = port.recv((void*)&buf[0], 1023);
- buf[btsRead] = '\0';
- string s;
- hexdump(s, buf, btsRead);
- LOG_INFO_STR("Received command on feedback port: " << s);
-
- if (!strcmp(buf, "ABORT")) {
- itsFeedbackResult = CT_RESULT_PIPELINE_FAILED;
- TRAN(PythonControl::completing_state);
- }
- else if (!strcmp(buf, "FINISHED")) {
- TRAN(PythonControl::completing_state);
+ case F_TIMER: {
+ if (&port == itsForcedQuitTimer) {
+ LOG_WARN("Aborting program on emergency timer!");
+ finish(CT_RESULT_EMERGENCY_TIMEOUT);
}
- else {
- LOG_ERROR_STR("Received command on feedback port unrecognized");
+ if (&port == itsQueueTimer) {
+ Message msg;
+ if (itsMsgQueue->getMessage(msg, 0.1)) {
+ string obsIDstr = msg.getXMLvalue("message.header.ids.sasid");
+ LOG_INFO_STR("Received message from task " << obsIDstr);
+ if (atoi(obsIDstr.c_str()) == itsObsID) {
+ string result = msg.getXMLvalue("message.payload.task.state");
+ if (result == "aborted") {
+ itsFeedbackResult = CT_RESULT_PIPELINE_FAILED;
+ }
+ else if (result != "finished") {
+ LOG_FATAL_STR("Unknown result received from pipeline: " << result << " assuming failure!");
+ itsFeedbackResult = CT_RESULT_PIPELINE_FAILED;
+ }
+ LOG_INFO_STR("Received '" << result << "' on the messagebus");
+ itsMsgQueue->ack(msg);
+ TRAN(PythonControl::finishing_state);
+ break;
+ } // ID matches?
+ } // getMsg
+ itsQueueTimer->setTimer(QUEUE_POLL_TIMEOUT);
}
} break;
@@ -562,13 +553,6 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
_databaseEventHandler(event);
break;
- case F_TIMER: {
- if (&port == itsForcedQuitTimer) {
- LOG_WARN("Aborting program on emergency timer!");
- finish(CT_RESULT_EMERGENCY_TIMEOUT);
- }
- } break;
-
// -------------------- EVENTS RECEIVED FROM PARENT CONTROL --------------------
case CONTROL_CONNECT: {
CONTROLConnectEvent msg(event);
@@ -616,13 +600,12 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
itsPythonPort->send(msg);
}
else {
- // QUICK FIX #3633
LOG_INFO("Trying to start the Python environment");
ParameterSet* thePS = globalParameterSet(); // shortcut to global PS.
string myPrefix (thePS->locateModule("PythonControl")+"PythonControl.");
string pythonProg (thePS->getString(myPrefix+"pythonProgram", "@pythonProgram@"));
string pythonHost (thePS->getString(myPrefix+"pythonHost", "@pythonHost@"));
- bool startOK = _startPython(pythonProg, getObservationNr(getName()), realHostname(pythonHost),
+ bool startOK = _startPython(pythonProg, itsObsID, realHostname(pythonHost),
itsListener->makeServiceName());
if (!startOK) {
LOG_ERROR("Failed to start the Python environment, ABORTING.");
@@ -633,7 +616,6 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
finish(CT_RESULT_PIPELINE_FAILED);
break;
}
- // QUICK FIX #3633 END
LOG_WARN("Start of Python environment looks OK, sending FAKE Resume response");
sendControlResult(*itsParentPort, event.signal, itsMyName, CT_RESULT_NO_ERROR);
}
@@ -678,7 +660,7 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
ParameterSet* thePS = globalParameterSet(); // shortcut to global PS.
string myPrefix (thePS->locateModule("PythonControl")+"PythonControl.");
string pythonHost(thePS->getString(myPrefix+"pythonHost","@pythonHost@"));
- bool stopOK = _stopPython(getObservationNr(getName()), realHostname(pythonHost));
+ bool stopOK = _stopPython(itsObsID, realHostname(pythonHost));
if (!stopOK) {
LOG_ERROR("Failed to stop the Python environment.");
finish(CT_RESULT_PIPELINE_FAILED);
@@ -742,7 +724,7 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
// msg.result = CT_RESULT_NO_ERROR;
// itsParentPort->send(msg);
LOG_INFO("Python environment has quited, quiting too.");
- TRAN(PythonControl::completing_state);
+ TRAN(PythonControl::finishing_state);
break;
}
@@ -755,39 +737,6 @@ GCFEvent::TResult PythonControl::operational_state(GCFEvent& event, GCFPortInter
return (status);
}
-//
-// completing_state(event, port)
-//
-// Pickup Metadata feedbackfile is any and pass it to SAS
-//
-GCFEvent::TResult PythonControl::completing_state(GCFEvent& event, GCFPortInterface& port)
-{
- LOG_DEBUG_STR ("completing:" << eventName(event) << "@" << port.getName());
-
- switch (event.signal) {
- case F_ENTRY: {
- // update PVSS
-#ifdef USE_PVSS_DATABASE
- itsPropertySet->setValue(PN_FSM_CURRENT_ACTION, GCFPVString("completing"));
- itsPropertySet->setValue(PN_FSM_ERROR, GCFPVString(""));
-#endif
- _passMetadatToOTDB();
-
- TRAN(PythonControl::finishing_state);
- } break;
-
- case F_DISCONNECTED:
- port.close();
- break;
-
- default:
- LOG_DEBUG("completing_state, default");
- return (GCFEvent::NOT_HANDLED);
- }
-
- return (GCFEvent::HANDLED);
-}
-
//
// finishing_state(event, port)
//
@@ -809,6 +758,8 @@ GCFEvent::TResult PythonControl::finishing_state(GCFEvent& event, GCFPortInterfa
msg.cntlrName = itsMyName;
msg.result = itsFeedbackResult;
itsParentPort->send(msg);
+ itsQueueTimer->cancelAllTimers();
+ itsForcedQuitTimer->cancelAllTimers();
itsTimerPort->cancelAllTimers();
itsTimerPort->setTimer(1.0); // give parent task time to process the message
} break;
@@ -829,100 +780,5 @@ GCFEvent::TResult PythonControl::finishing_state(GCFEvent& event, GCFPortInterfa
return (GCFEvent::HANDLED);
}
-//
-// _passMetadatToOTDB();
-//
-void PythonControl::_passMetadatToOTDB()
-{
- bool metadataFileAvailable (true);
-
- // No name specified?
- if (itsFeedbackFile.empty()) {
- metadataFileAvailable = false;
- }
- else {
- // Copy file from remote system to localsystem
- ParameterSet* thePS = globalParameterSet(); // shortcut to global PS.
- string myPrefix (thePS->locateModule("PythonControl")+"PythonControl.");
- string pythonHost(thePS->getString(myPrefix+"pythonHost","@pythonHost@"));
- try {
- if (copyFromRemote(realHostname(pythonHost), itsFeedbackFile, itsFeedbackFile) != 0) {
- LOG_ERROR_STR("Failed to copy metadatafile " << itsFeedbackFile << " from host " << realHostname(pythonHost));
- metadataFileAvailable = false;
- }
- }
- catch (...) {
- metadataFileAvailable = false;
- }
- }
-
- // Try to setup the connection with the database
- string confFile = globalParameterSet()->getString("OTDBconfFile", "SASGateway.conf");
- ConfigLocator CL;
- string filename = CL.locate(confFile);
- LOG_DEBUG_STR("Trying to read database information from file " << filename);
- ParameterSet otdbconf;
- otdbconf.adoptFile(filename);
- string database = otdbconf.getString("SASGateway.OTDBdatabase");
- string dbhost = otdbconf.getString("SASGateway.OTDBhostname");
- OTDBconnection conn("paulus", "boskabouter", database, dbhost);
- if (!conn.connect()) {
- LOG_FATAL_STR("Cannot connect to database " << database << " on machine " << dbhost);
- // WE DO HAVE A PROBLEM HERE BECAUSE THIS PIPELINE CANNOT BE SET TO FINISHED IN SAS.
- return;
- }
- LOG_INFO_STR("Connected to database " << database << " on machine " << dbhost);
-
- int obsID(getObservationNr(getName()));
- TreeValue tv(&conn, obsID);
-
- if (metadataFileAvailable) {
- // read parameterset
- ParameterSet metadata;
- metadata.adoptFile(itsFeedbackFile);
-
- // Loop over the parameterset and send the information to the SAS database
- // During the transition phase from parameter-based to record-based storage in OTDB the
- // nodenames ending in '_' are implemented both as parameter and as record.
- ParameterSet::iterator iter = metadata.begin();
- ParameterSet::iterator end = metadata.end();
- while (iter != end) {
- string key(iter->first); // make destoyable copy
- rtrim(key, "[]0123456789");
- // bool doubleStorage(key[key.size()-1] == '_');
- bool isRecord(iter->second.isRecord());
- // isRecord doubleStorage
- // --------------------------------------------------------------
- // Y Y store as record and as parameters
- // Y N store as parameters
- // N * store parameter
- if (!isRecord) {
- LOG_DEBUG_STR("BASIC: " << iter->first << " = " << iter->second);
- tv.addKVT(iter->first, iter->second, ptime(microsec_clock::local_time()));
- }
- else {
- // if (doubleStorage) {
- // LOG_DEBUG_STR("RECORD: " << iter->first << " = " << iter->second);
- // tv.addKVT(iter->first, iter->second, ptime(microsec_clock::local_time()));
- // }
- // to store is a node/param values the last _ should be stipped of
- key = iter->first; // destroyable copy
- // string::size_type pos = key.find_last_of('_');
- // key.erase(pos,1);
- ParameterRecord pr(iter->second.getRecord());
- ParameterRecord::const_iterator prIter = pr.begin();
- ParameterRecord::const_iterator prEnd = pr.end();
- while (prIter != prEnd) {
- LOG_DEBUG_STR("ELEMENT: " << key+"."+prIter->first << " = " << prIter->second);
- tv.addKVT(key+"."+prIter->first, prIter->second, ptime(microsec_clock::local_time()));
- prIter++;
- }
- }
- iter++;
- }
- LOG_INFO_STR(metadata.size() << " metadata values send to SAS");
- }
-}
-
}; // CEPCU
}; // LOFAR
diff --git a/MAC/APL/CEPCU/src/PythonControl/PythonControl.conf b/MAC/APL/CEPCU/src/PythonControl/PythonControl.conf
index 0cffc595c807e87cc5ce4e4c3f7d20268bd3a0d4..674e9fc88ab3e58d75bb4fa78b32b8b3dc28ea40 100644
--- a/MAC/APL/CEPCU/src/PythonControl/PythonControl.conf
+++ b/MAC/APL/CEPCU/src/PythonControl/PythonControl.conf
@@ -2,8 +2,8 @@
# PythonControl.conf
#
-# Host on which the KVTLogger runs (normally localhost)
-OTDBconfFile = SASGateway.conf
+# Queue to listen at.
+TaskStateQueue = mac.task.feedback.state
# Max time to wait for response from Python Framework when quiting
emergencyTimeout = 1h
diff --git a/MAC/APL/CEPCU/src/PythonControl/PythonControl.h b/MAC/APL/CEPCU/src/PythonControl/PythonControl.h
index 7d3460d2382608e9e4086b39824c4af9bf0591a5..25f277d81a85c81a90729c95b9be9e3b4d2c8790 100644
--- a/MAC/APL/CEPCU/src/PythonControl/PythonControl.h
+++ b/MAC/APL/CEPCU/src/PythonControl/PythonControl.h
@@ -27,10 +27,11 @@
#include <Common/lofar_string.h>
#include <Common/lofar_vector.h>
#include <Common/LofarLogger.h>
-
-//# ACC Includes
#include <Common/ParameterSet.h>
+//# MsgBus Includes
+#include <MessageBus/MsgBus.h>
+
//# GCF Includes
#include <GCF/TM/GCF_Control.h>
#include <GCF/RTDB/RTDB_PropertySet.h>
@@ -67,8 +68,6 @@ public:
GCFEvent::TResult waitForConnection_state(GCFEvent& event, GCFPortInterface& port);
// Normal control mode.
GCFEvent::TResult operational_state (GCFEvent& event, GCFPortInterface& port);
- // Completing mode.
- GCFEvent::TResult completing_state (GCFEvent& event, GCFPortInterface& port);
// Finishing mode.
GCFEvent::TResult finishing_state (GCFEvent& event, GCFPortInterface& port);
@@ -89,26 +88,25 @@ private:
bool _stopPython ( int obsID,
const string& pythonHost);
void _databaseEventHandler(GCFEvent& event);
- void _passMetadatToOTDB();
// ----- datamembers -----
RTDBPropertySet* itsPropertySet;
bool itsPropertySetInitialized;
string itsMyName;
+ int itsObsID;
// pointer to parent control task
ParentControl* itsParentControl;
GCFITCPort* itsParentPort;
GCFTimerPort* itsTimerPort;
+ GCFTimerPort* itsQueueTimer;
GCFTimerPort* itsForcedQuitTimer;
GCFTCPPort* itsListener;
- // QUICK FIX #3633
- GCFTCPPort* itsFeedbackListener;
- GCFTCPPort* itsFeedbackPort;
+ FromBus* itsMsgQueue;
int itsFeedbackResult;
GCFTCPPort* itsPythonPort;
@@ -119,7 +117,6 @@ private:
CTState::CTstateNr itsState;
// conf-file variables
- string itsFeedbackFile;
double itsForceTimeout;
};
diff --git a/MAC/APL/CEPCU/src/PythonControl/PythonControlMain.cc b/MAC/APL/CEPCU/src/PythonControl/PythonControlMain.cc
index 565ab9bd97fac9b5cd81cc3ef5c4d7e8cc7f5505..6c05cfd3e151cf4423595bf0836e48e11478a19c 100644
--- a/MAC/APL/CEPCU/src/PythonControl/PythonControlMain.cc
+++ b/MAC/APL/CEPCU/src/PythonControl/PythonControlMain.cc
@@ -31,7 +31,7 @@ using namespace LOFAR::CEPCU;
using namespace LOFAR;
// Use a terminate handler that can produce a backtrace.
-Exception::TerminateHandler t(Exception::terminate);
+LOFAR::Exception::TerminateHandler t(Exception::terminate);
int main(int argc, char* argv[])
{
@@ -57,7 +57,7 @@ int main(int argc, char* argv[])
pc->quit(); // let tasks quit nicely.
pyc.quit();
GCFScheduler::instance()->run(0.3); // let tasks die.
- } catch( Exception &ex ) {
+ } catch( LOFAR::Exception &ex ) {
LOG_FATAL_STR("Caught exception: " << ex);
return 1;
}
diff --git a/MAC/APL/CUDaemons/CT_StartDaemon/CTStartDaemon.cc b/MAC/APL/CUDaemons/CT_StartDaemon/CTStartDaemon.cc
index b1103e70473e52d9348b813a1b2dbaf44e219264..9ef212f531f1db668a36e2f97f90e461ad0e9c4f 100644
--- a/MAC/APL/CUDaemons/CT_StartDaemon/CTStartDaemon.cc
+++ b/MAC/APL/CUDaemons/CT_StartDaemon/CTStartDaemon.cc
@@ -264,10 +264,10 @@ int32 CTStartDaemon::startController(uint16 cntlrType,
parentService.c_str());
LOG_INFO_STR("About to start: " << startCmd);
- int32 result = system (startCmd.c_str());
+ int result = system (startCmd.c_str());
LOG_INFO_STR ("Result of start = " << result);
- if (result == -1) {
+ if (result != 0) {
return (SD_RESULT_START_FAILED);
}
diff --git a/MAC/APL/MainCU/CMakeLists.txt b/MAC/APL/MainCU/CMakeLists.txt
index 8ee48d7734a84388d07d04cb40dc07b5f029d7c4..772dc46fbda9608ed28b552f5863e09857fe02b7 100644
--- a/MAC/APL/MainCU/CMakeLists.txt
+++ b/MAC/APL/MainCU/CMakeLists.txt
@@ -1,7 +1,7 @@
# $Id$
# Do not split the following line, otherwise makeversion will fail!
-lofar_package(MainCU 1.0 DEPENDS Common OTDB MACIO GCFTM GCFPVSS GCFRTDB APLCommon RTDBCommon ApplCommon CR_Protocol)
+lofar_package(MainCU 1.0 DEPENDS Common MessageBus OTDB MACIO GCFTM GCFPVSS GCFRTDB APLCommon RTDBCommon ApplCommon CR_Protocol)
include(LofarFindPackage)
lofar_find_package(Boost REQUIRED COMPONENTS date_time)
diff --git a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc
index a812099ed1b3ea17ae8aa17f1716282cbf22ba3e..5464388d30bb33ac1d421301d7fe5b8765d52124 100644
--- a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc
+++ b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.cc
@@ -28,6 +28,8 @@
#include <Common/ParameterSet.h>
#include <GCF/TM/GCF_Protocols.h>
#include <MACIO/MACServiceInfo.h>
+#include <MessageBus/MsgBus.h>
+#include <MessageBus/Protocols/TaskSpecificationSystem.h>
#include <GCF/PVSS/GCF_PVTypes.h>
#include <APL/APLCommon/APL_Defines.h>
#include <APL/APLCommon/ControllerDefines.h>
@@ -47,6 +49,7 @@ using namespace LOFAR::GCF::PVSS;
using namespace LOFAR::GCF::TM;
using namespace LOFAR::GCF::RTDB;
using namespace LOFAR::OTDB;
+using namespace LOFAR::Protocols;
using namespace boost::posix_time;
using namespace std;
@@ -77,7 +80,8 @@ MACScheduler::MACScheduler() :
itsNextFinishedTime (0),
itsNrPlanned (0),
itsNrActive (0),
- itsOTDBconnection (0)
+ itsOTDBconnection (0),
+ itsMsgQueue (0)
{
LOG_TRACE_OBJ ("MACscheduler construction");
@@ -93,7 +97,8 @@ MACScheduler::MACScheduler() :
itsMaxPlanned = globalParameterSet()->getTime("maxPlannedList", 30);
itsMaxFinished = globalParameterSet()->getTime("maxFinishedList", 40);
- ASSERTSTR(itsMaxPlanned + itsMaxFinished < MAX_CONCURRENT_OBSERVATIONS, "maxPlannedList + maxFinishedList should be less than " << MAX_CONCURRENT_OBSERVATIONS);
+ ASSERTSTR(itsMaxPlanned + itsMaxFinished < MAX_CONCURRENT_OBSERVATIONS,
+ "maxPlannedList + maxFinishedList should be less than " << MAX_CONCURRENT_OBSERVATIONS);
// Read the schedule periods for starting observations.
itsQueuePeriod = globalParameterSet()->getTime("QueuePeriod");
@@ -101,20 +106,23 @@ MACScheduler::MACScheduler() :
// attach to child control task
itsChildControl = ChildControl::instance();
- itsChildPort = new GCFITCPort (*this, *itsChildControl, "childITCport",
- GCFPortInterface::SAP, CONTROLLER_PROTOCOL);
+ itsChildPort = new GCFITCPort (*this, *itsChildControl, "childITCport", GCFPortInterface::SAP, CONTROLLER_PROTOCOL);
ASSERTSTR(itsChildPort, "Cannot allocate ITCport for childcontrol");
itsChildPort->open(); // will result in F_CONNECTED
// create an PVSSprepare Task
itsClaimerTask = new ObsClaimer(this);
ASSERTSTR(itsClaimerTask, "Cannot construct a ObsClaimerTask");
- itsClaimerPort = new GCFITCPort (*this, *itsClaimerTask, "ObsClaimerPort",
- GCFPortInterface::SAP, CM_PROTOCOL);
+ itsClaimerPort = new GCFITCPort (*this, *itsClaimerTask, "ObsClaimerPort", GCFPortInterface::SAP, CM_PROTOCOL);
// need port for timers
itsTimerPort = new GCFTimerPort(*this, "Timerport");
+ // setup MsgQueue
+ string queueName = globalParameterSet()->getString("ParsetQueuename");
+ ASSERTSTR(!queueName.empty(), "Queuename for distributing parameterSets not specified");
+ itsMsgQueue = new ToBus(queueName);
+
registerProtocol(CONTROLLER_PROTOCOL, CONTROLLER_PROTOCOL_STRINGS);
registerProtocol(DP_PROTOCOL, DP_PROTOCOL_STRINGS);
}
@@ -134,6 +142,10 @@ MACScheduler::~MACScheduler()
if (itsOTDBconnection) {
delete itsOTDBconnection;
}
+
+ if (itsMsgQueue) {
+ delete itsMsgQueue;
+ }
}
//
@@ -696,6 +708,10 @@ void MACScheduler::_updatePlannedList()
// add controller to our 'monitor' administration
itsControllerMap[cntlrName] = obsID;
LOG_DEBUG_STR("itsControllerMap[" << cntlrName << "]=" << obsID);
+ if (!itsPreparedObs[obsID].parsetDistributed) {
+ _setParsetOnMsgBus(observationParset(obsID));
+ itsPreparedObs[obsID].parsetDistributed = true;
+ }
}
else {
LOG_DEBUG_STR("Observation " << obsID << " is already (being) started");
@@ -809,6 +825,29 @@ void MACScheduler::_updateFinishedList()
}
}
+//
+// _setParsetOnMsgBus(parsetFile)
+//
+void MACScheduler::_setParsetOnMsgBus(const string& filename) const
+{
+ // open file
+ ParameterSet obsSpecs(filename);
+ string obsPrefix = obsSpecs.fullModuleName("Observation");
+ string momID = obsSpecs.getString(obsPrefix + ".momID");
+ string sasID = obsSpecs.getString(obsPrefix + ".otdbID");
+
+ // from, forUser, summary, protocol, protocolVersion, momID, sasID
+#if 1
+ TaskSpecificationSystem outMsg("LOFAR.MACScheduler", "", "", momID, sasID, obsSpecs);
+#else
+ Message outMsg("LOFAR.MACScheduler", "", "", "task.specification.system", "1.0", momID, sasID);
+ stringstream ss;
+ obsSpecs.writeStream(ss);
+ outMsg.setTXTPayload(ss.str());
+#endif
+ cout << outMsg << endl;
+ itsMsgQueue->send(outMsg);
+}
//
// _connectedHandler(port)
diff --git a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.conf.in b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.conf.in
index e4ac723afce39d2d10bc94678d610d2ca78644cd..911f75463162324556fcb8d03141d9aece3381b6 100644
--- a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.conf.in
+++ b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.conf.in
@@ -29,3 +29,4 @@ maxFinishedList = 40 # Never show more finished observations
#ChildControl.StartupRetryInterval = 10s
#ChildControl.MaxStartupRetry = 5
+ParsetQueuename = lofar.task.specification.system
diff --git a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.h b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.h
index 65f6f754792c1f0dd0583284090f913c96528fdb..83b8069877267ab001b02fe9002c2544c9a7b9cc 100644
--- a/MAC/APL/MainCU/src/MACScheduler/MACScheduler.h
+++ b/MAC/APL/MainCU/src/MACScheduler/MACScheduler.h
@@ -39,6 +39,7 @@
#include <Common/lofar_vector.h>
#include <Common/LofarLogger.h>
#include <ApplCommon/Observation.h>
+#include <MessageBus/MsgBus.h>
//# ACC Includes
#include <OTDB/OTDBconnection.h>
@@ -107,6 +108,7 @@ private:
void _updatePlannedList();
void _updateActiveList();
void _updateFinishedList();
+ void _setParsetOnMsgBus(const string& filename) const;
// ----- DATA MEMBERS -----
// Our own propertySet in PVSS to inform the operator
@@ -130,8 +132,9 @@ private:
public:
ptime modTime;
bool prepReady;
- schedInfo(ptime t, bool p) : modTime(t), prepReady(p) {};
- schedInfo() : modTime(min_date_time), prepReady(false) {};
+ bool parsetDistributed;
+ schedInfo(ptime t, bool p) : modTime(t), prepReady(p), parsetDistributed(false) {};
+ schedInfo() : modTime(min_date_time), prepReady(false), parsetDistributed(false) {};
};
typedef map<int /*obsID*/, schedInfo /*prepReady*/> ObsList;
typedef map<int ,schedInfo>::iterator OLiter;
@@ -163,8 +166,10 @@ private:
uint32 itsQueuePeriod; // period between queueing and start
// OTDB related variables.
- OTDB::OTDBconnection* itsOTDBconnection; // connection to the database
+ OTDB::OTDBconnection* itsOTDBconnection; // connection to the database
+ // Messagebus related variables
+ ToBus* itsMsgQueue; // Bus used for sending
};
};//MainCU
diff --git a/MAC/APL/MainCU/src/MACScheduler/MACSchedulerMain.cc b/MAC/APL/MainCU/src/MACScheduler/MACSchedulerMain.cc
index 14e7f54e380a8abd101892423c5a971882119276..2b472d7c01a3741d039d35a98eca861b32520e1f 100644
--- a/MAC/APL/MainCU/src/MACScheduler/MACSchedulerMain.cc
+++ b/MAC/APL/MainCU/src/MACScheduler/MACSchedulerMain.cc
@@ -23,6 +23,7 @@
#include <lofar_config.h>
#include <Common/LofarLogger.h>
#include <Common/Exception.h>
+#include <MessageBus/MsgBus.h>
#include "MACScheduler.h"
@@ -38,6 +39,8 @@ int main(int argc, char* argv[])
{
GCFScheduler::instance()->init(argc, argv, "MACScheduler");
+ MessageBus::init();
+
ChildControl* cc = ChildControl::instance();
cc->start(); // make initial transition
diff --git a/MAC/APL/MainCU/src/ObservationControl/ObservationControl.cc b/MAC/APL/MainCU/src/ObservationControl/ObservationControl.cc
index ac46c6ce423a37b285c7216194d532b9186d7474..62a099b482fd5716d5919cdd7d252c23b250c01c 100644
--- a/MAC/APL/MainCU/src/ObservationControl/ObservationControl.cc
+++ b/MAC/APL/MainCU/src/ObservationControl/ObservationControl.cc
@@ -107,10 +107,11 @@ ObservationControl::ObservationControl(const string& cntlrName) :
itsProcessType = globalParameterSet()->getString("Observation.processType", "Observation");
// Values from my conf file
- itsLateLimit = globalParameterSet()->getTime ("ObservationControl.lateLimit", 15);
- itsFailedLimit = globalParameterSet()->getTime ("ObservationControl.failedLimit", 30);
- itsHeartBeatItv = globalParameterSet()->getTime ("ObservationControl.heartbeatInterval", 10);
- string reportType= globalParameterSet()->getString("ObservationControl.reportType", "Full");
+ itsLateLimit = globalParameterSet()->getTime ("ObservationControl.lateLimit", 15);
+ itsFailedLimit = globalParameterSet()->getTime ("ObservationControl.failedLimit", 30);
+ itsHeartBeatItv = globalParameterSet()->getTime ("ObservationControl.heartbeatInterval", 10);
+ itsFinalStateDelay = globalParameterSet()->getTime ("ObservationControl.finalStateDelay", 10);
+ string reportType = globalParameterSet()->getString("ObservationControl.reportType", "Full");
if (reportType == "Full") itsFullReport = true;
else if (reportType == "Changes") itsChangeReport = true;
@@ -655,6 +656,8 @@ GCFEvent::TResult ObservationControl::finishing_state(GCFEvent& event,
setState(CTState::QUITED);
// inform MACScheduler we are going down
+ LOG_INFO_STR("Waiting " << itsFinalStateDelay << " seconds before reporting final state...");
+ sleep (itsFinalStateDelay);
CONTROLQuitedEvent msg;
msg.cntlrName = getName();
msg.result = itsQuitReason;
diff --git a/MAC/APL/MainCU/src/ObservationControl/ObservationControl.conf.in b/MAC/APL/MainCU/src/ObservationControl/ObservationControl.conf.in
index 2a43f6f73b9846150e972423742cf3e3c7ffbeab..554b7fa92589e077632e060b9b6eb0d1160d5f47 100644
--- a/MAC/APL/MainCU/src/ObservationControl/ObservationControl.conf.in
+++ b/MAC/APL/MainCU/src/ObservationControl/ObservationControl.conf.in
@@ -28,3 +28,7 @@ ObservationControl.reportType = Full
#
ObservationControl.emergencyTimeout = 1h
+#
+# Delay before reporting FINAL state to MACScheduler
+#
+ObservationControl.finalStateDelay = 10s
diff --git a/MAC/APL/MainCU/src/ObservationControl/ObservationControl.h b/MAC/APL/MainCU/src/ObservationControl/ObservationControl.h
index 79dd634ae6c1cd85caa43315580881f0a9ccf5b9..cb8f86738577f6faf9241566ce5ff77ac4cc677b 100644
--- a/MAC/APL/MainCU/src/ObservationControl/ObservationControl.h
+++ b/MAC/APL/MainCU/src/ObservationControl/ObservationControl.h
@@ -171,6 +171,7 @@ private:
uint32 itsTreeID;
uint32 itsHeartBeatItv;
uint32 itsForcedQuitDelay;
+ uint32 itsFinalStateDelay;
uint32 itsClaimPeriod;
uint32 itsPreparePeriod;
int32 itsLateLimit; // after how many seconds a requested state should have been reached.
diff --git a/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc b/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc
index 8928df3d4e4e513b45f52b5e072bf82107ff95da..7396924d513ac9e7122b7269aaa00f3f6103e251 100644
--- a/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc
+++ b/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc
@@ -192,7 +192,7 @@ namespace LOFAR
}
- ParameterSet LTAFeedback::allFeedback() const
+ ParameterSet LTAFeedback::processingFeedback() const
{
ParameterSet ps;
@@ -209,11 +209,6 @@ namespace LOFAR
str(format("%u") % settings.correlator.nrChannels));
ps.add("Observation.Correlator.channelWidth",
str(format("%.16g") % (settings.subbandWidth() / settings.correlator.nrChannels)));
-
- // add the feedback for the individual files
- for (size_t i = 0; i < settings.correlator.files.size(); ++i) {
- ps.adoptCollection(correlatedFeedback(i));
- }
}
ps.add("Observation.DataProducts.nrOfOutput_Beamformed_",
@@ -314,9 +309,25 @@ namespace LOFAR
ps.add("Observation.IncoherentStokes.stationList",
settings.rawStationList);
}
+
+ return ps;
+ }
+
+
+ ParameterSet LTAFeedback::allFeedback() const
+ {
+ ParameterSet ps;
+
+ ps.adoptCollection(processingFeedback());
+
+ // add the feedback for the individual files
+ if (settings.correlator.enabled) {
+ for (size_t i = 0; i < settings.correlator.files.size(); ++i) {
+ ps.adoptCollection(correlatedFeedback(i));
+ }
+ }
if (settings.beamFormer.enabled) {
- // add the feedback for the individual files
for (size_t i = 0; i < settings.beamFormer.files.size(); ++i) {
ps.adoptCollection(beamFormedFeedback(i));
}
diff --git a/RTCP/Cobalt/CoInterface/src/LTAFeedback.h b/RTCP/Cobalt/CoInterface/src/LTAFeedback.h
index 3fbe662591407bc996a41a0bea3f388f58bc1a80..e8be4dd8d389581c12d45bc4eb991ddd247f6ab6 100644
--- a/RTCP/Cobalt/CoInterface/src/LTAFeedback.h
+++ b/RTCP/Cobalt/CoInterface/src/LTAFeedback.h
@@ -45,6 +45,14 @@ namespace LOFAR
ParameterSet correlatedFeedback(size_t fileno) const;
ParameterSet beamFormedFeedback(size_t fileno) const;
+ // Return the generic processing LTA feedback parameters.
+ // (Non data-product specific).
+ // \note Details about the meaning of the different meta-data parameters
+ // can be found in the XSD that describes the Submission Information
+ // Package (sip) for the LTA.
+ // \see http://proposal.astron.nl/schemas/LTA-SIP.xsd
+ ParameterSet processingFeedback() const;
+
// Return the LTA feedback parameters.
// \note Details about the meaning of the different meta-data parameters
// can be found in the XSD that describes the Submission Information
diff --git a/RTCP/Cobalt/CoInterface/src/OMPThread.h b/RTCP/Cobalt/CoInterface/src/OMPThread.h
index 6d83f05a6438a47a07bf2bd6811f785a27a71c30..2914a0fe5bfc9d0162077558c5ea379c57e5a2c1 100644
--- a/RTCP/Cobalt/CoInterface/src/OMPThread.h
+++ b/RTCP/Cobalt/CoInterface/src/OMPThread.h
@@ -162,6 +162,8 @@ namespace LOFAR
if ((retval = pthread_setname_np(pthread_self(), name.substr(0,15).c_str())) != 0)
throw SystemCallException("pthread_setname_np", retval, THROW_ARGS);
+#else
+ (void)name;
#endif
}
diff --git a/RTCP/Cobalt/CoInterface/src/Parset.cc b/RTCP/Cobalt/CoInterface/src/Parset.cc
index 911a06981a7e495902a0bc0f3ae980d34156ae1d..316a4a32e0db490331c4e8746cde53c264ae6f66 100644
--- a/RTCP/Cobalt/CoInterface/src/Parset.cc
+++ b/RTCP/Cobalt/CoInterface/src/Parset.cc
@@ -321,6 +321,7 @@ namespace LOFAR
// Generic information
settings.realTime = getBool("Cobalt.realTime", false);
settings.observationID = getUint32("Observation.ObsID", 0);
+ settings.momID = getUint32("Observation.momID", 0);
settings.commandStream = getString("Cobalt.commandStream", "null:");
settings.startTime = getTime("Observation.startTime", "2013-01-01 00:00:00");
settings.stopTime = getTime("Observation.stopTime", "2013-01-01 00:01:00");
diff --git a/RTCP/Cobalt/CoInterface/src/Parset.h b/RTCP/Cobalt/CoInterface/src/Parset.h
index 8c9361fdbab97aa3692fa2c82e8080210ef0c854..5c99148bee65924870d3fdfcb6fe06eca3264f00 100644
--- a/RTCP/Cobalt/CoInterface/src/Parset.h
+++ b/RTCP/Cobalt/CoInterface/src/Parset.h
@@ -70,6 +70,11 @@ namespace LOFAR
// key: Observation.ObsID
unsigned observationID;
+ // The MoM observation number
+ //
+ // key: Observation.momID
+ unsigned momID;
+
// Command stream, or null: if not used
//
// key: Cobalt.commandStream
diff --git a/RTCP/Cobalt/CoInterface/test/tParset.cc b/RTCP/Cobalt/CoInterface/test/tParset.cc
index 1b80fa02e6c2c7783b2268aa9bdde8f97a0203c1..a07e2c33f430ea9da88058e8c0b71e9dc8849b78 100644
--- a/RTCP/Cobalt/CoInterface/test/tParset.cc
+++ b/RTCP/Cobalt/CoInterface/test/tParset.cc
@@ -96,6 +96,12 @@ TEST(observationID) {
CHECK_EQUAL(12345U, ps.settings.observationID);
}
+TEST(momID) {
+ Parset ps = makeDefaultTestParset("Observation.momID", "12345");
+
+ CHECK_EQUAL(12345U, ps.settings.momID);
+}
+
TEST(startTime) {
Parset ps = makeDefaultTestParset("Observation.startTime", "2013-03-17 10:55:08");
diff --git a/RTCP/Cobalt/GPUProc/CMakeLists.txt b/RTCP/Cobalt/GPUProc/CMakeLists.txt
index 6b420ab50d288c5be17f15fd2a571887f6e4e4a1..3a8a7c8ad69c4f4ce5cf3b79a5c3502cdaccdadb 100644
--- a/RTCP/Cobalt/GPUProc/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/CMakeLists.txt
@@ -2,7 +2,7 @@
# Handle options USE_CUDA and USE_OPENCL.
if(USE_CUDA AND NOT USE_OPENCL)
- set(_gpuproc_deps Common Stream ApplCommon CoInterface InputProc MACIO BrokenAntennaInfo)
+ set(_gpuproc_deps Common Stream ApplCommon CoInterface InputProc MACIO BrokenAntennaInfo MessageBus)
lofar_find_package(CUDA 4.1 REQUIRED)
lofar_find_package(CUDADriver REQUIRED)
@@ -34,7 +34,7 @@ if(USE_CUDA AND NOT USE_OPENCL)
endif()
elseif(USE_OPENCL AND NOT USE_CUDA)
- set(_gpuproc_deps Common Stream ApplCommon CoInterface InputProc MACIO BrokenAntennaInfo OpenCL_FFT)
+ set(_gpuproc_deps Common Stream ApplCommon CoInterface InputProc MACIO BrokenAntennaInfo OpenCL_FFT MessageBus)
lofar_find_package(OpenCL REQUIRED)
add_definitions(-DUSE_OPENCL)
else()
diff --git a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/MAC-feedback.parset b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/MAC-feedback.parset
deleted file mode 100644
index 14e6a2cedc84950869b3e4829e7c993c85a2e2c9..0000000000000000000000000000000000000000
--- a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/MAC-feedback.parset
+++ /dev/null
@@ -1,15 +0,0 @@
-# $Id$
-
-# The host that is running OnlineControl
-# The feedback port is derived through
-# 21000 + (Observation.ObsID % 1000)
-Cobalt.Feedback.host = ccu001
-
-# The remote directory in which the LTA feedback needs to be stored
-Cobalt.Feedback.remotePath = /opt/lofar/var/run
-
-# The host that is running PVSSGateway.
-# If empty, data points are never sent.
-# One can also start a PVSSGateway_Stub
-# on localhost, which writes to a file.
-Cobalt.PVSSGateway.host = ccu001
diff --git a/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/PVSS-feedback.parset b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/PVSS-feedback.parset
new file mode 100644
index 0000000000000000000000000000000000000000..4e062233e380cd0bc57b388f0ecec42bf1ccfe33
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/etc/parset-additions.d/default/PVSS-feedback.parset
@@ -0,0 +1,7 @@
+# $Id: MAC-feedback.parset 30525 2014-11-28 12:21:15Z mol $
+
+# The host that is running PVSSGateway.
+# If empty, data points are never sent.
+# One can also start a PVSSGateway_Stub
+# on localhost, which writes to a file.
+Cobalt.PVSSGateway.host = ccu001
diff --git a/RTCP/Cobalt/GPUProc/src/CMakeLists.txt b/RTCP/Cobalt/GPUProc/src/CMakeLists.txt
index b511d42b9b92e42c54929bff6b07a00bb52308a6..da6eeee66c2931fee22f2d14c7bb88f4fbeb8a60 100644
--- a/RTCP/Cobalt/GPUProc/src/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/src/CMakeLists.txt
@@ -119,6 +119,7 @@ endif()
lofar_add_bin_program(mpi_node_list Station/mpi_node_list.cc)
lofar_add_bin_program(station_stream Station/station_stream.cc)
+lofar_add_bin_program(send_state send_state.cc)
# install scripts used to run an observation under bin
install(PROGRAMS
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
index 929197ed7a0f925bb1c02e723676c0872e7dfcc0..2a12316f9fe2d18c02d3bbf0d77de5ec4b21b98b 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
@@ -558,12 +558,16 @@ namespace LOFAR {
if (ps.settings.realTime) {
#pragma omp parallel for num_threads(nrBoards)
for(size_t board = 0; board < nrBoards; board++) {
- OMPThreadSet::ScopedRun sr(packetReaderThreads);
- OMPThread::ScopedName sn(str(format("%s rd %u") % ps.settings.antennaFields.at(stationIdx).name % board));
+ try {
+ OMPThreadSet::ScopedRun sr(packetReaderThreads);
+ OMPThread::ScopedName sn(str(format("%s rd %u") % ps.settings.antennaFields.at(stationIdx).name % board));
- Thread::ScopedPriority sp(SCHED_FIFO, 10);
+ Thread::ScopedPriority sp(SCHED_FIFO, 10);
- readRSPRealTime(board, mdLogger, mdKeyPrefix);
+ readRSPRealTime(board, mdLogger, mdKeyPrefix);
+ } catch(OMPThreadSet::CannotStartException &ex) {
+ LOG_INFO_STR( logPrefix << "Stopped");
+ }
}
} else {
readRSPNonRealTime(mdLogger, mdKeyPrefix);
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc
index 5b602f491460b70ca486630f8d6e87c5529bb75d..d381e8075c34e47c397f4bb190f002f26eb30185 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc
@@ -28,8 +28,11 @@
#include <Common/LofarLogger.h>
#include <Common/Thread/Thread.h>
+#include <MessageBus/MsgBus.h>
+#include <MessageBus/Protocols/TaskFeedbackDataproducts.h>
#include <Stream/PortBroker.h>
#include <CoInterface/Stream.h>
+#include <CoInterface/LTAFeedback.h>
namespace LOFAR
{
@@ -45,7 +48,9 @@ namespace LOFAR
itsParset(parset),
itsLogPrefix(str(boost::format("%s [StorageWriter rank %2d host %s] ") % logPrefix % rank % hostname)),
itsRank(rank),
- itsHostname(hostname)
+ itsHostname(hostname),
+ itsSentFeedback(false),
+ itsSuccessful(false)
{
}
@@ -57,6 +62,46 @@ namespace LOFAR
// stop immediately
struct timespec immediately = { 0, 0 };
stop(immediately);
+
+ if (!itsSentFeedback) {
+ // send default LTA feedback for this host
+ ToBus bus("lofar.task.feedback.dataproducts");
+
+ const std::string myName = "Cobalt/GPUProc/Storage/StorageProcess";
+
+ LTAFeedback feedback(itsParset.settings);
+
+ if (itsParset.settings.correlator.enabled)
+ for (size_t i = 0; i < itsParset.settings.correlator.files.size(); ++i) {
+ LOG_INFO_STR(itsParset.settings.correlator.files[i].location.host << " == " << itsHostname);
+
+ if (itsParset.settings.correlator.files[i].location.host == itsHostname) {
+ Protocols::TaskFeedbackDataproducts msg(
+ myName,
+ "",
+ str(boost::format("Feedback for Correlated Data, subband %s") % i),
+ str(format("%s") % itsParset.settings.momID),
+ str(format("%s") % itsParset.settings.observationID),
+ feedback.correlatedFeedback(i));
+
+ bus.send(msg);
+ }
+ }
+
+ if (itsParset.settings.beamFormer.enabled)
+ for (size_t i = 0; i < itsParset.settings.beamFormer.files.size(); ++i)
+ if (itsParset.settings.beamFormer.files[i].location.host == itsHostname) {
+ Protocols::TaskFeedbackDataproducts msg(
+ myName,
+ "",
+ str(boost::format("Feedback for Beamformed Data, file nr %s") % i),
+ str(format("%s") % itsParset.settings.momID),
+ str(format("%s") % itsParset.settings.observationID),
+ feedback.beamFormedFeedback(i));
+
+ bus.send(msg);
+ }
+ }
}
@@ -82,6 +127,12 @@ namespace LOFAR
}
+ bool StorageProcess::isSuccesful() const
+ {
+ return itsSuccessful;
+ }
+
+
bool StorageProcess::isDone() const
{
return itsThread->isDone();
@@ -95,15 +146,6 @@ namespace LOFAR
}
- ParameterSet StorageProcess::feedbackLTA() const
- {
- // Prevent read/write conflicts
- ASSERT(isDone());
-
- return itsFeedbackLTA;
- }
-
-
void StorageProcess::controlThread()
{
// Connect control stream
@@ -123,12 +165,11 @@ namespace LOFAR
itsFinalMetaData.write(stream);
LOG_DEBUG_STR(itsLogPrefix << "[ControlThread] sent final meta data");
- // Wait for LTA feedback
- LOG_DEBUG_STR(itsLogPrefix << "[ControlThread] reading LTA feedback");
- ParameterSet feedbackLTA;
- readParameterSet(stream, feedbackLTA);
- itsFeedbackLTA.adoptCollection(feedbackLTA);
- LOG_DEBUG_STR(itsLogPrefix << "[ControlThread] read LTA feedback");
+ // Wait for OutputProc to finish
+ LOG_DEBUG_STR(itsLogPrefix << "[ControlThread] waiting to finish");
+ stream.read(&itsSentFeedback, sizeof itsSentFeedback);
+ stream.read(&itsSuccessful, sizeof itsSuccessful);
+ LOG_DEBUG_STR(itsLogPrefix << "[ControlThread] finished");
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h
index f982196a70cd7b318984aac26cc08f2ebe6c3b89..a6e49c8429a9e9d48739861f26eab0b678568c0f 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h
@@ -67,11 +67,9 @@ namespace LOFAR
void start();
void stop( struct timespec deadline );
- bool isDone() const;
- // Returns feedback for the LTA -- only access this once the
- // StorageProcess has finished!
- ParameterSet feedbackLTA() const;
+ bool isSuccesful() const; // return whether communication with OutputProc went perfect
+ bool isDone() const;
void setFinalMetaData( const FinalMetaData &finalMetaData );
@@ -84,11 +82,12 @@ namespace LOFAR
const int itsRank;
const std::string itsHostname;
+ bool itsSentFeedback;
+ bool itsSuccessful;
+
FinalMetaData itsFinalMetaData;
Semaphore itsFinalMetaDataAvailable;
- ParameterSet itsFeedbackLTA;
-
SmartPtr<Thread> itsThread;
};
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc
index 402e43a5b95babbe6cb4323121fbbff80d8357b0..148c18dc8cf6bac77f56f03d69811122da6fed1b 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc
@@ -58,12 +58,6 @@ namespace LOFAR
}
- ParameterSet StorageProcesses::feedbackLTA() const
- {
- return itsFeedbackLTA;
- }
-
-
void StorageProcesses::start()
{
const vector<string> &hostnames = itsParset.settings.outputProcHosts;
@@ -91,9 +85,6 @@ namespace LOFAR
// stop storage process
itsStorageProcesses[rank]->stop(deadline_ts);
- // obtain feedback for LTA
- itsFeedbackLTA.adoptCollection(itsStorageProcesses[rank]->feedbackLTA());
-
// free the StorageProcess object
itsStorageProcesses[rank] = 0;
}
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h
index 04151848e90d6697d6297fb44e06c1136cf15f63..2e59471ed5cbec36b60917a4f8ef265dc33a7fde 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h
@@ -80,17 +80,12 @@ namespace LOFAR
// stop the processes and control threads, given an absolute time out.
void stop( time_t deadline );
- ParameterSet feedbackLTA() const;
-
private:
const Parset &itsParset;
const std::string itsLogPrefix;
std::vector<SmartPtr<StorageProcess> > itsStorageProcesses;
- // All feedback for the LTA obtained by the storage processes
- ParameterSet itsFeedbackLTA;
-
// start the processes and control threads
void start();
diff --git a/RTCP/Cobalt/GPUProc/src/rtcp.cc b/RTCP/Cobalt/GPUProc/src/rtcp.cc
index ee93600810e45156acec705f623fa00114cc9f95..544e829e0d20c256ba92abe6e8a6a3960c429641 100644
--- a/RTCP/Cobalt/GPUProc/src/rtcp.cc
+++ b/RTCP/Cobalt/GPUProc/src/rtcp.cc
@@ -52,6 +52,8 @@
#include <Common/SystemUtil.h>
#include <Common/StringUtil.h>
#include <Common/Thread/Trigger.h>
+#include <MessageBus/MsgBus.h>
+#include <MessageBus/Protocols/TaskFeedbackProcessing.h>
#include <ApplCommon/PVSSDatapointDefs.h>
#include <ApplCommon/StationInfo.h>
#include <MACIO/RTmetadata.h>
@@ -168,6 +170,10 @@ int main(int argc, char **argv)
LOG_INFO("===== INIT =====");
+ // Initialise message bus
+ LOG_INFO("----- Initialising MessageBus");
+ MessageBus::init();
+
// Create a parameters set object based on the inputs
LOG_INFO("----- Reading Parset");
Parset ps(argv[optind]);
@@ -593,31 +599,20 @@ int main(int argc, char **argv)
// graceful exit
storageProcesses->stop(time(0) + outputProcTimeout);
- LOG_INFO("Writing LTA feedback to disk");
+ // send processing feedback
+ ToBus bus("lofar.task.feedback.processing");
- // obtain LTA feedback
LTAFeedback fb(ps.settings);
- Parset feedbackLTA;
-
- // augment LTA feedback with global information
- feedbackLTA.adoptCollection(fb.allFeedback());
-
- // process updates from outputProc
- feedbackLTA.adoptCollection(storageProcesses->feedbackLTA());
- // write LTA feedback to disk
- const char *LOFARROOT = getenv("LOFARROOT");
- if (LOFARROOT != NULL) {
- string feedbackFilename = str(format("%s/var/run/Observation%s_feedback") % LOFARROOT % ps.settings.observationID);
+ Protocols::TaskFeedbackProcessing msg(
+ "Cobalt/GPUProc/rtcp",
+ "",
+ "Processing feedback",
+ str(format("%s") % ps.settings.momID),
+ str(format("%s") % ps.settings.observationID),
+ fb.processingFeedback());
- try {
- feedbackLTA.writeFile(feedbackFilename, false);
- } catch (APSException &ex) {
- LOG_ERROR_STR("Could not write feedback file " << feedbackFilename << ": " << ex);
- }
- } else {
- LOG_WARN("Could not write feedback file: $LOFARROOT not set.");
- }
+ bus.send(msg);
// final cleanup
storageProcesses = 0;
diff --git a/RTCP/Cobalt/GPUProc/src/scripts/CobaltVersions.sh b/RTCP/Cobalt/GPUProc/src/scripts/CobaltVersions.sh
index 6465f07bc5ffd29dc8571e24c726a202b04f2be3..55b901be83712424a910de98a09e3353711698dc 100755
--- a/RTCP/Cobalt/GPUProc/src/scripts/CobaltVersions.sh
+++ b/RTCP/Cobalt/GPUProc/src/scripts/CobaltVersions.sh
@@ -13,8 +13,8 @@ function error() {
if [ -z "$HOSTS" ]; then
case `hostname` in
cbt001|cbt002|cbt003|cbt004|cbt005|cbt006|cbt007|cbt008)
- # Production system
- HOSTS="cbm001 cbm002 cbm003 cbm004 cbm005 cbm006 cbm007 cbm008"
+ # Production system -- switch test system as well, in case we use cbm009/cbm010!
+ HOSTS="cbm001 cbm002 cbm003 cbm004 cbm005 cbm006 cbm007 cbm008 cbm009 cbm010"
;;
cbt009|cbt010)
diff --git a/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh b/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh
index 6b7dbad8514539717f6fa452c3947531636f7d44..0e594f82c00c88b753c045fda6688d15d3b56443 100755
--- a/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh
+++ b/RTCP/Cobalt/GPUProc/src/scripts/Cobalt_install.sh
@@ -19,7 +19,7 @@ if [ "${RELEASE_NAME}" = "" ]; then
exit 1
fi
-for HOST in ${HOSTS:-cbm001 cbm002 cbm003 cbm004 cbm005 cbm006 cbm007 cbm008 cbm009}; do
+for HOST in ${HOSTS:-cbm001 cbm002 cbm003 cbm004 cbm005 cbm006 cbm007 cbm008 cbm009 cbm010}; do
echo "ssh-ing to node $HOST"
# Escape double quotes below the following line!
diff --git a/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh b/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh
index d903cebebd7e4095958fa25e1d7f6baff2536ab1..fde6a641e02af789f2dfbee3b1189cfcd92089a1 100755
--- a/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh
+++ b/RTCP/Cobalt/GPUProc/src/scripts/runObservation.sh
@@ -12,7 +12,7 @@
function error {
echo -e "$@" >&2
- sendback_status 1
+ sendback_state 1
exit 1
}
@@ -51,7 +51,7 @@ function usage {
"\n -B: do NOT add broken antenna information"\
"\n -C: run with check tool specified in environment variable"\
"LOFAR_CHECKTOOL"\
- "\n -F: do NOT send feedback to OnlineControl and do NOT send data points to a PVSS gateway"\
+ "\n -F: do NOT send data points to a PVSS gateway"\
"\n -P: create PID file"\
"\n -l: run solely on localhost using 'nprocs' MPI processes (isolated test)"\
"\n -p: enable profiling" \
@@ -85,66 +85,26 @@ function command_retry {
done
}
-# Send the result status back to OnlineControl.
+# Send the result state back to LOFAR (MAC, MoM)
#
# to report success:
# sendback_status 0
# to report failure:
# sendback_status 1
-function sendback_status {
+function sendback_state {
OBSRESULT="$1"
- if [ -z "$PARSET" ]
+ if [ $OBSRESULT -eq 0 ]
then
- echo "Not communicating back to OnlineControl (no parset)"
- return 0
- fi
-
- if [ "$ONLINECONTROL_FEEDBACK" -eq "0" ]
- then
- echo "Not communicating back to OnlineControl (disabled on command line)"
- return 0
- fi
-
- if [ "$ONLINECONTROL_FEEDBACK" -eq "1" ]
- then
- ONLINECONTROL_USER=`getkey Cobalt.Feedback.userName $USER`
- ONLINECONTROL_HOST=`getkey Cobalt.Feedback.host`
-
- if [ $OBSRESULT -eq 0 ]
- then
- # ***** Observation ran successfully
-
- # Copy LTA feedback file to ccu001
- FEEDBACK_DEST="$ONLINECONTROL_USER@$ONLINECONTROL_HOST:`getkey Cobalt.Feedback.remotePath`"
-
- echo "Copying feedback to $FEEDBACK_DEST"
- timeout $KILLOPT 30s scp "$FEEDBACK_FILE" "$FEEDBACK_DEST"
- FEEDBACK_RESULT=$?
- if [ $FEEDBACK_RESULT -ne 0 ]
- then
- echo "Failed to copy file $FEEDBACK_FILE to $FEEDBACK_DEST (status: $FEEDBACK_RESULT)"
- OBSRESULT=$FEEDBACK_RESULT
- fi
- fi
-
- # Communicate result back to OnlineControl
- ONLINECONTROL_RESULT_PORT=$((21000 + $OBSID % 1000))
-
- if [ $OBSRESULT -eq 0 ]
- then
- # Signal success to OnlineControl
- echo "Signalling success to $ONLINECONTROL_HOST"
- echo -n "FINISHED" > /dev/tcp/$ONLINECONTROL_HOST/$ONLINECONTROL_RESULT_PORT
- else
- # ***** Observation or sending feedback failed for some reason
- # Signal failure to OnlineControl
- echo "Signalling failure to $ONLINECONTROL_HOST"
- echo -n "ABORT" > /dev/tcp/$ONLINECONTROL_HOST/$ONLINECONTROL_RESULT_PORT
- fi
+ echo "Signalling success"
+ SUCCESS=1
+ else
+ # ***** Observation or sending feedback failed for some reason
+ echo "Signalling failure"
+ SUCCESS=0
fi
- return 1
+ send_state "$PARSET" $SUCCESS
}
#############################
@@ -155,8 +115,8 @@ echo "Called as: $0 $@"
# Set default options
# ******************************
-# Provide feedback to OnlineControl and data points to PVSS?
-ONLINECONTROL_FEEDBACK=1
+# Provide data points to PVSS?
+STATUS_FEEDBACK=1
# Augment the parset with etc/parset-additions.d/* ?
AUGMENT_PARSET=1
@@ -192,7 +152,7 @@ while getopts ":ABCFP:l:o:p" opt; do
;;
C) CHECK_TOOL="$LOFAR_CHECKTOOL"
;;
- F) ONLINECONTROL_FEEDBACK=0
+ F) STATUS_FEEDBACK=0
;;
P) PIDFILE="$OPTARG"
;;
@@ -295,8 +255,6 @@ then
setkey Cobalt.OutputProc.executable "$LOFARROOT/bin/outputProc"
setkey Cobalt.OutputProc.StaticMetaDataDirectory "$LOFARROOT/etc"
setkey Cobalt.FinalMetaDataGatherer.database.host localhost
- setkey Cobalt.Feedback.host localhost
- setkey Cobalt.Feedback.remotePath "$LOFARROOT/var/run"
setkey Cobalt.PVSSGateway.host ""
# Redirect UDP/TCP input streams to any interface on the local machine
@@ -309,7 +267,7 @@ then
setkey Cobalt.FinalMetaDataGatherer.enabled false
fi
- if [ "$ONLINECONTROL_FEEDBACK" -eq "0" ]
+ if [ "$STATUS_FEEDBACK" -eq "0" ]
then
setkey Cobalt.PVSSGateway.host ""
fi
@@ -418,11 +376,13 @@ touch $PID_LIST_FILE
LIST_OF_HOSTS=$(getOutputProcHosts $PARSET)
RANK=0
+VARS="QUEUE_PREFIX=$QUEUE_PREFIX" # Variables to forward to outputProc
for HOST in $LIST_OF_HOSTS
do
- COMMAND="ssh -tt -l $SSH_USER_NAME $KEY_STRING $SSH_USER_NAME@$HOST $OUTPUT_PROC_EXECUTABLE $OBSERVATIONID $RANK"
+ COMMAND="ssh -tt -l $SSH_USER_NAME $KEY_STRING $SSH_USER_NAME@$HOST $VARS $OUTPUT_PROC_EXECUTABLE $OBSERVATIONID $RANK"
+ echo "Starting $COMMAND"
# keep a counter to allow determination of the rank (needed for binding to rtcp)
- RANK=$(($RANK + 1))
+ RANK=$(($RANK + 1))
command_retry "$COMMAND" & # Start retrying function in the background
PID=$! # get the pid
@@ -437,8 +397,10 @@ done
# Run in the background to allow signals to propagate
#
# -x LOFARROOT Propagate $LOFARROOT for rtcp to find GPU kernels, config files, etc.
+# -x QUEUE_PREFIX Propagate $QUEUE_PREFIX for test-specific interaction over the message bus
# -H The host list to run on, derived earlier.
mpirun.sh -x LOFARROOT="$LOFARROOT" \
+ -x QUEUE_PREFIX="$QUEUE_PREFIX" \
-H "$HOSTS" \
$MPIRUN_PARAMS \
$CHECK_TOOL \
@@ -480,7 +442,7 @@ fi
# Post-process the observation
# ******************************
-sendback_status "$OBSRESULT"
+sendback_state "$OBSRESULT"
# clean up outputProc children
echo "Allowing 120 second for normal end of outputProc"
diff --git a/RTCP/Cobalt/GPUProc/src/send_state.cc b/RTCP/Cobalt/GPUProc/src/send_state.cc
new file mode 100644
index 0000000000000000000000000000000000000000..777838919f6234c90f29b68e54e3021c4d87e439
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/send_state.cc
@@ -0,0 +1,91 @@
+//# send_state.cc: Send lofar.task.feedback.status information
+//# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include <Common/LofarLogger.h>
+#include <CoInterface/Parset.h>
+#include <MessageBus/MsgBus.h>
+#include <MessageBus/Protocols/TaskFeedbackState.h>
+
+#include <boost/format.hpp>
+
+#include <unistd.h>
+
+using namespace LOFAR;
+using namespace LOFAR::Cobalt;
+using namespace std;
+using boost::format;
+
+static void usage(const char *argv0)
+{
+ cerr << "Usage: " << argv0 << " parset success" << endl;
+ cerr << endl;
+ cerr << " -h: print this message" << endl;
+}
+
+int main(int argc, char **argv)
+{
+ /*
+ * Parse command-line options
+ */
+
+ int opt;
+ while ((opt = getopt(argc, argv, "h")) != -1) {
+ switch (opt) {
+
+ case 'h':
+ usage(argv[0]);
+ return EXIT_SUCCESS;
+
+ default: /* '?' */
+ usage(argv[0]);
+ return EXIT_FAILURE;
+ }
+ }
+
+ // we expect a parset filename and a success indicator as an additional parameter
+ if (optind + 1 >= argc) {
+ usage(argv[0]);
+ return EXIT_FAILURE;
+ }
+
+ INIT_LOGGER("send_status");
+ MessageBus::init();
+
+ Parset parset(argv[optind]);
+ int success = atoi(argv[optind+1]);
+
+ // send status feedback
+ ToBus bus("lofar.task.feedback.state");
+
+ Protocols::TaskFeedbackState msg(
+ "Cobalt/GPUProc/send_state",
+ "",
+ "State feedback",
+ str(format("%s") % parset.settings.momID),
+ str(format("%s") % parset.settings.observationID),
+ success);
+
+ bus.send(msg);
+
+ return 0;
+}
+
diff --git a/RTCP/Cobalt/GPUProc/test/Storage/CMakeLists.txt b/RTCP/Cobalt/GPUProc/test/Storage/CMakeLists.txt
index d27b53f7a22f1cf7738edb2441c6762c2de6dd1a..a01cc37bebb1f2974e6ef4e4341bd022715b6708 100644
--- a/RTCP/Cobalt/GPUProc/test/Storage/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/test/Storage/CMakeLists.txt
@@ -2,6 +2,10 @@
include(LofarCTest)
+configure_file(
+ ${CMAKE_CURRENT_SOURCE_DIR}/tStorageProcesses.run2.in
+ ${CMAKE_CURRENT_BINARY_DIR}/tStorageProcesses.run2 @ONLY)
+
# DummyStorage is started by tStorageProcesses to emulate an OutputProc process
lofar_add_executable(DummyStorage DummyStorage.cc)
lofar_add_test(tStorageProcesses tStorageProcesses.cc DEPENDS DummyStorage)
diff --git a/RTCP/Cobalt/GPUProc/test/Storage/DummyStorage.cc b/RTCP/Cobalt/GPUProc/test/Storage/DummyStorage.cc
index 7a8a1307b47f9050ce5b7fd59cf93dc17f3d8072..b246f19a15b8d111155473f90823f2637b97119d 100644
--- a/RTCP/Cobalt/GPUProc/test/Storage/DummyStorage.cc
+++ b/RTCP/Cobalt/GPUProc/test/Storage/DummyStorage.cc
@@ -43,10 +43,6 @@ unsigned rank;
Exception::TerminateHandler th(Exception::terminate);
-FinalMetaData origFinalMetaData;
-
-Mutex logMutex;
-
void emulateStorage()
{
// establish control connection
@@ -55,52 +51,22 @@ void emulateStorage()
// read and print parset
Parset parset(&stream);
- {
- ScopedLock sl(logMutex);
- cout << "Storage: Parset received." << endl;
- }
+ cout << "Storage: Parset received." << endl;
// read and print meta data
FinalMetaData finalMetaData;
finalMetaData.read(stream);
- {
- ScopedLock sl(logMutex);
-
- ASSERT(finalMetaData.brokenRCUsAtBegin == origFinalMetaData.brokenRCUsAtBegin);
- ASSERT(finalMetaData.brokenRCUsDuring == origFinalMetaData.brokenRCUsDuring);
-
- cout << "Storage: FinalMetaData received and matches." << endl;
- }
-
- // write LTA feedback
- Parset feedbackLTA;
- feedbackLTA.add("foo", "bar");
- feedbackLTA.write(&stream);
-}
-void emulateFinalMetaDataGatherer()
-{
- // establish control connection
- string resource = getStorageControlDescription(observationID, -1);
- PortBroker::ServerStream stream(resource);
-
- // read and print parset
- Parset parset(&stream);
- {
- ScopedLock sl(logMutex);
- cout << "FinalMetaDataGatherer: Parset received." << endl;
- }
+ //ASSERT(finalMetaData.brokenRCUsAtBegin == origFinalMetaData.brokenRCUsAtBegin);
+ //ASSERT(finalMetaData.brokenRCUsDuring == origFinalMetaData.brokenRCUsDuring);
- // set and write meta data
- origFinalMetaData.brokenRCUsAtBegin.push_back( FinalMetaData::BrokenRCU("CS001", "LBA", 2, "2012-01-01 12:34") );
- origFinalMetaData.brokenRCUsAtBegin.push_back( FinalMetaData::BrokenRCU("RS205", "HBA", 1, "2012-01-01 12:34") );
- origFinalMetaData.brokenRCUsDuring.push_back( FinalMetaData::BrokenRCU("DE601", "RCU", 3, "2012-01-01 12:34") );
- origFinalMetaData.write(stream);
+ cout << "Storage: FinalMetaData received and matches." << endl;
- {
- ScopedLock sl(logMutex);
- cout << "FinalMetaDataGatherer: FinalMetaData sent." << endl;
- }
+ // write completion signal
+ bool sentFeedback = false;
+ stream.write(&sentFeedback, sizeof sentFeedback);
+ bool success = true;
+ stream.write(&success, sizeof success);
}
int main(int argc, char **argv)
@@ -118,7 +84,7 @@ int main(int argc, char **argv)
// Make sure DummyStorage always dies, even if the test
// malfunctions.
- alarm(10);
+ alarm(20);
observationID = boost::lexical_cast<int>(argv[1]);
rank = boost::lexical_cast<unsigned>(argv[2]);
@@ -126,21 +92,10 @@ int main(int argc, char **argv)
// set up broker server
PortBroker::createInstance(storageBrokerPort(observationID));
-#pragma omp parallel sections
- {
-# pragma omp section
- try {
- emulateStorage();
- } catch (Exception &ex) {
- cout << "Storage caught exception: " << ex << endl;
- }
-
-# pragma omp section
- try {
- emulateFinalMetaDataGatherer();
- } catch (Exception &ex) {
- cout << "FinalMetaDataGatherer caught exception: " << ex << endl;
- }
+ try {
+ emulateStorage();
+ } catch (Exception &ex) {
+ cout << "Storage caught exception: " << ex << endl;
}
}
diff --git a/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.cc b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.cc
index 306ba933a95bcfead49498c0f89b84f422931259..0c1d9870b511cfd5362238f80f8969a07d620f3c 100644
--- a/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.cc
+++ b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.cc
@@ -56,6 +56,8 @@ void test_protocol()
THROW_SYSCALL("getcwd");
p.add("Observation.ObsID", "12345");
+ p.add("Observation.startTime", "2015-01-01 00:00:00");
+ p.add("Observation.stopTime", "2015-01-01 00:01:00");
p.add("Cobalt.OutputProc.userName", USER);
p.add("Cobalt.OutputProc.sshPublicKey", pubkey);
p.add("Cobalt.OutputProc.sshPrivateKey", privkey);
@@ -90,11 +92,6 @@ void test_protocol()
// Give 10 seconds to wrap up
sp.stop(time(0) + 10);
-
- // Obtain LTA feedback
- ParameterSet feedbackLTA(sp.feedbackLTA());
-
- ASSERT(feedbackLTA.getString("foo","") == "bar");
}
}
diff --git a/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.queue b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.queue
new file mode 100644
index 0000000000000000000000000000000000000000..4ec0ed39e87645060dab7300cda59f0907848b20
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.queue
@@ -0,0 +1,38 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>task.feedback.dataproducts</name>
+ <version>1.0.0</version>
+ </protocol>
+ <source>
+ <name>Cobalt/GPUProc/Storage/StorageProcess</name>
+ <user></user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>Feedback for Correlated Data, subband 0</summary>
+ </source>
+ <ids>
+ <momid>0</momid>
+ <sasid>12345</sasid>
+ </ids>
+ </header>
+ <payload>
+Observation.DataProducts.Output_Correlated_[0].SAP=0
+Observation.DataProducts.Output_Correlated_[0].centralFrequency=0.000000
+Observation.DataProducts.Output_Correlated_[0].channelWidth=3051.757812
+Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=64
+Observation.DataProducts.Output_Correlated_[0].duration=0
+Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+Observation.DataProducts.Output_Correlated_[0].filename=SB0.MS
+Observation.DataProducts.Output_Correlated_[0].integrationInterval=1.006633
+Observation.DataProducts.Output_Correlated_[0].location=localhost:.
+Observation.DataProducts.Output_Correlated_[0].percentageWritten=0
+Observation.DataProducts.Output_Correlated_[0].size=0
+Observation.DataProducts.Output_Correlated_[0].startTime=2015-01-01 00:00:00
+Observation.DataProducts.Output_Correlated_[0].stationSubband=0
+Observation.DataProducts.Output_Correlated_[0].subband=0
+
+ </payload>
+</message>
diff --git a/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run
index 474574333959f95d9e89aaee548627a21e85faea..855a8c12020b14cd2cc2c5e717edbf8c240ffe1b 100755
--- a/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run
+++ b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run
@@ -1,8 +1,4 @@
#!/bin/bash
-#
-DummyStorage 12345 0& # Run the command
-PID_ID=$! # get the PID
-tStorageProcesses || exit 1
-wait $PID_ID
-exit 0 # do not exit with DymmyStorage exit. If we are waiting the test should succeed
+# Defer to the .run2, which is constructed by CMake but won't be cleaned up by assay
+bash ./tStorageProcesses.run2
diff --git a/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run2.in b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run2.in
new file mode 100755
index 0000000000000000000000000000000000000000..4491adeaa30cc3022f0e8a033cfc3fa41f621c29
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/Storage/tStorageProcesses.run2.in
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+
+# Run a program to talk to
+DummyStorage 12345 0& # Run the command
+PID_ID=$! # get the PID
+
+source MessageFuncs.sh
+
+# Empty queues (if any)
+create_queue lofar.task.feedback.dataproducts
+
+# Start the test
+tStorageProcesses || exit 1
+wait $PID_ID
+
+# Validate queues
+if have_qpid; then
+ recv_msg lofar.task.feedback.dataproducts > tStorageProcesses.queue
+
+ compare_msg $srcdir/tStorageProcesses.queue tStorageProcesses.queue || exit 1
+fi
+
+# Signal success to parent
+exit 0
+
diff --git a/RTCP/Cobalt/OutputProc/CMakeLists.txt b/RTCP/Cobalt/OutputProc/CMakeLists.txt
index 2aed652557937b472bcd4b9640f6ccb4b1a73650..d12b4addfe84de38b51621ed4e1b1c37c73bf75e 100644
--- a/RTCP/Cobalt/OutputProc/CMakeLists.txt
+++ b/RTCP/Cobalt/OutputProc/CMakeLists.txt
@@ -1,6 +1,6 @@
# $Id$
-lofar_package(OutputProc 1.0 DEPENDS Common Stream CoInterface MSLofar LofarStMan MACIO)
+lofar_package(OutputProc 1.0 DEPENDS Common Stream CoInterface MSLofar LofarStMan MACIO MessageBus)
include(LofarFindPackage)
lofar_find_package(Casacore COMPONENTS casa ms tables REQUIRED)
diff --git a/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc b/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
index d3c6720f26bb3c7d86b60c91a81f411f88608ffc..fffaff1bdfbb114bd984ce757e4f7f4b5e3a9f0e 100644
--- a/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
+++ b/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
@@ -32,6 +32,8 @@
#include <Common/LofarLogger.h>
#include <Common/StringUtil.h>
#include <Common/Exceptions.h>
+#include <MessageBus/MsgBus.h>
+#include <MessageBus/Protocols/TaskFeedbackDataproducts.h>
#include <Stream/PortBroker.h>
#include <ApplCommon/PVSSDatapointDefs.h>
#include <ApplCommon/StationInfo.h>
@@ -249,22 +251,45 @@ bool process(Stream &controlStream, unsigned myRank)
* LTA FEEDBACK
*/
- LOG_DEBUG_STR("Retrieving LTA feedback");
- Parset feedbackLTA;
+ LOG_DEBUG_STR("Forwarding LTA feedback");
- for (size_t i = 0; i < subbandWriters.size(); ++i)
- feedbackLTA.adoptCollection(subbandWriters[i]->feedbackLTA());
- for (size_t i = 0; i < tabWriters.size(); ++i)
- feedbackLTA.adoptCollection(tabWriters[i]->feedbackLTA());
+ ToBus bus("lofar.task.feedback.dataproducts");
- LOG_DEBUG_STR("Forwarding LTA feedback");
- try {
- feedbackLTA.write(&controlStream);
- } catch (LOFAR::Exception &err) {
- success = false;
- LOG_ERROR_STR("Failed to forward LTA feedback information: " << err);
+ const std::string myName = str(boost::format("Cobalt/OutputProc on %s") % myHostName);
+
+ for (size_t i = 0; i < subbandWriters.size(); ++i) {
+ Protocols::TaskFeedbackDataproducts msg(
+ myName,
+ "",
+ str(boost::format("Feedback for Correlated Data, subband %s") % subbandWriters[i]->streamNr()),
+ str(format("%s") % parset.settings.momID),
+ str(format("%s") % parset.settings.observationID),
+ subbandWriters[i]->feedbackLTA());
+
+ bus.send(msg);
+ }
+
+ for (size_t i = 0; i < tabWriters.size(); ++i) {
+ Protocols::TaskFeedbackDataproducts msg(
+ myName,
+ "",
+ str(boost::format("Feedback for Beamformed Data, file nr %s") % tabWriters[i]->streamNr()),
+ str(format("%s") % parset.settings.momID),
+ str(format("%s") % parset.settings.observationID),
+ tabWriters[i]->feedbackLTA());
+
+ bus.send(msg);
}
+ /*
+ * SIGN OFF
+ */
+
+ bool sentFeedback = true;
+
+ controlStream.write(&sentFeedback, sizeof sentFeedback);
+ controlStream.write(&success, sizeof success);
+
return success;
}
}
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc b/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc
index 3914faeab3bc726991a6dc53df982b559a2c746f..711bc2f0a9c1f4c888367bfd19047641400a066b 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc
@@ -30,6 +30,7 @@
#include <Common/SystemUtil.h>
#include <MSLofar/FailedTileInfo.h>
#include <CoInterface/CorrelatedData.h>
+#include <CoInterface/LTAFeedback.h>
#include <tables/Tables/Table.h>
#include <casa/Quanta/MVTime.h>
@@ -54,6 +55,10 @@ namespace LOFAR
itsMSname(msName),
itsParset(parset)
{
+ // Add file-specific processing feedback
+ LTAFeedback fb(itsParset.settings);
+ itsConfiguration.adoptCollection(fb.correlatedFeedback(subbandIndex));
+
if (LofarStManVersion > 1) {
string seqfilename = str(format("%s/table.f0seqnr") % msName);
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc
index 51129879950f6e5bb798047e64e0ab22b33cab8d..260fab955922c19078ce90a3d0ab2eceb3eb0151 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc
@@ -40,6 +40,7 @@
#include <Common/StreamUtil.h>
#include <Common/Thread/Mutex.h>
#include <CoInterface/StreamableData.h>
+#include <CoInterface/LTAFeedback.h>
#include <OutputProc/Package__Version.h>
#include <dal/lofar/BF_File.h>
@@ -110,6 +111,10 @@ namespace LOFAR
itsNextSeqNr(0),
itsFileNr(fileno)
{
+ // Add file-specific processing feedback
+ LTAFeedback fb(itsParset.settings);
+ itsConfiguration.adoptCollection(fb.beamFormedFeedback(itsFileNr));
+
itsNrExpectedBlocks = itsParset.settings.nrBlocks();
string h5filename = forceextension(string(filename),".h5");
diff --git a/RTCP/Cobalt/OutputProc/src/outputProc.cc b/RTCP/Cobalt/OutputProc/src/outputProc.cc
index 39407435760cceb38492b31110eba99d723617b0..4e469d7fdae0533efa95bb419898ee1e30ae63da 100644
--- a/RTCP/Cobalt/OutputProc/src/outputProc.cc
+++ b/RTCP/Cobalt/OutputProc/src/outputProc.cc
@@ -38,6 +38,7 @@
#include <CoInterface/Stream.h>
#include <CoInterface/OMPThread.h>
#include <OutputProc/Package__Version.h>
+#include <MessageBus/MsgBus.h>
#include "GPUProcIO.h"
#include "IOPriority.h"
@@ -97,6 +98,8 @@ int main(int argc, char *argv[])
INIT_LOGGER("outputProc"); // also attaches to CasaLogSink
+ MessageBus::init();
+
LOG_DEBUG_STR("Started: " << argv[0] << ' ' << argv[1] << ' ' << argv[2]);
LOG_INFO_STR("OutputProc version " << OutputProcVersion::getVersion() << " r" << OutputProcVersion::getRevision());
diff --git a/SAS/CMakeLists.txt b/SAS/CMakeLists.txt
index 4cfb43c0523cb2092f2408ab783d27ab306d61da..6b44144c0fa6bba0d13ab3189af2fd86edb2c8f4 100644
--- a/SAS/CMakeLists.txt
+++ b/SAS/CMakeLists.txt
@@ -4,3 +4,4 @@ lofar_add_package(OTDB)
lofar_add_package(OTB)
lofar_add_package(OTDB_SQL OTDB/sql)
lofar_add_package(Scheduler)
+lofar_add_package(SAS_Feedback Feedback_Service)
diff --git a/SAS/Feedback_Service/CMakeLists.txt b/SAS/Feedback_Service/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..908452ac68c61484e7003af9453c98db30ba3c38
--- /dev/null
+++ b/SAS/Feedback_Service/CMakeLists.txt
@@ -0,0 +1,11 @@
+# $Id: CMakeLists.txt 23213 2012-12-07 13:00:09Z loose $
+
+lofar_package(SAS_Feedback 1.0 DEPENDS Common MACIO GCFTM OTDB MessageBus)
+
+include(LofarFindPackage)
+lofar_find_package(Boost REQUIRED COMPONENTS date_time)
+lofar_find_package(PQXX REQUIRED)
+
+add_subdirectory(src)
+#add_subdirectory(test)
+
diff --git a/SAS/Feedback_Service/package.dox b/SAS/Feedback_Service/package.dox
new file mode 100644
index 0000000000000000000000000000000000000000..23d33bb5e72cf84e9c9e18c6cb4fa5297406cb69
--- /dev/null
+++ b/SAS/Feedback_Service/package.dox
@@ -0,0 +1,6 @@
+//
+// \ingroup FeedbackService
+// \defgroup Feedback Service that receives feedback.* messages and updates
+// the database accordingly.
+//
+
diff --git a/SAS/Feedback_Service/src/CMakeLists.txt b/SAS/Feedback_Service/src/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..df3ad9e63ae1866a2bcfcdc4ebdcb42e8707e3c8
--- /dev/null
+++ b/SAS/Feedback_Service/src/CMakeLists.txt
@@ -0,0 +1,10 @@
+# $Id: CMakeLists.txt 30676 2015-01-07 14:09:34Z mol $
+
+include(LofarPackageVersion)
+
+lofar_add_library(sas_feedback Package__Version.cc)
+
+lofar_add_bin_program(versionsas_feedback versionsas_feedback.cc)
+lofar_add_bin_program(FeedbackService Feedback.cc FeedbackMain.cc)
+
+install(FILES FeedbackService.conf DESTINATION etc)
diff --git a/SAS/Feedback_Service/src/Feedback.cc b/SAS/Feedback_Service/src/Feedback.cc
new file mode 100644
index 0000000000000000000000000000000000000000..0526a7e1e176801ccf2056e9f1bff3bd87c9faec
--- /dev/null
+++ b/SAS/Feedback_Service/src/Feedback.cc
@@ -0,0 +1,263 @@
+//# Feedback.cc: one_line_description
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id: $
+
+//# Always #include <lofar_config.h> first!
+#include <lofar_config.h>
+#include <signal.h>
+
+//# Includes
+#include <Common/LofarLogger.h>
+#include <Common/LofarLocators.h>
+#include <Common/ParameterSet.h>
+#include <Common/ParameterRecord.h>
+#include <Common/Version.h>
+#include <GCF/TM/GCF_Control.h>
+#include <OTDB/TreeValue.h>
+#include <SAS_Feedback/Package__Version.h>
+#include "Feedback.h"
+
+#include <boost/date_time/posix_time/posix_time.hpp>
+using namespace boost::posix_time;
+
+namespace LOFAR {
+ using namespace OTDB;
+ using namespace GCF::TM;
+ namespace SAS {
+
+static Feedback* thisFeedback = 0;
+
+//
+// Feedback()
+//
+Feedback::Feedback() :
+ GCFTask ((State)&Feedback::connect2OTDB_state, "FeedbackService"),
+ itsTimer (0),
+ itsOTDBconn (0),
+ itsMsgQueue (0)
+{
+ // Log who we are.
+ LOG_INFO(Version::getInfo<SAS_FeedbackVersion>("FeedbackService"));
+
+ // attach timer
+ itsTimer = new GCFTimerPort(*this, "TimerPort");
+ ASSERTSTR(itsTimer, "Cannot allocate timer");
+
+ // get the name of the queues to monitor
+ vector<string> queuenames = globalParameterSet()->getStringVector("FeedbackQueuenames");
+ ASSERTSTR(!queuenames.empty(), "Queuenames not specified in label 'FeedbackQueuenames'");
+
+ // connect to bussystem
+ itsMsgQueue = new FromBus(queuenames[0]);
+ for (size_t i = 1; i < queuenames.size(); ++i) {
+ itsMsgQueue->addQueue(queuenames[i]);
+ }
+
+ // redirect signal handlers
+ thisFeedback = this;
+ signal (SIGINT, Feedback::sigintHandler); // ctrl-c
+ signal (SIGTERM, Feedback::sigintHandler); // kill
+}
+
+//
+// ~Feedback()
+//
+Feedback::~Feedback()
+{
+ delete itsTimer;
+ delete itsMsgQueue;
+ delete itsOTDBconn;
+}
+
+//
+// sigintHandler(signum)
+//
+void Feedback::sigintHandler(int signum)
+{
+ LOG_INFO (formatString("SIGINT signal detected (%d)",signum));
+
+ if (thisFeedback) {
+ thisFeedback->finish();
+ }
+}
+
+//
+// finish
+//
+void Feedback::finish()
+{
+ GCFScheduler::instance()->stop();
+}
+
+
+
+//
+// connect2OTDB_state(event, port);
+//
+GCFEvent::TResult Feedback::connect2OTDB_state(GCFEvent& event, GCFPortInterface& port)
+{
+ LOG_DEBUG_STR ("connect2OTDB:" << eventName(event) << "@" << port.getName());
+
+ switch (event.signal) {
+ case F_INIT: break;
+
+ case F_ENTRY:
+ case F_TIMER: {
+ // Try to setup the connection with the database
+ string confFile = globalParameterSet()->getString("OTDBconfFile", "SASGateway.conf");
+ ConfigLocator CL;
+ string filename = CL.locate(confFile);
+ LOG_DEBUG_STR("Trying to read database information from file " << filename);
+ ParameterSet otdbconf;
+ otdbconf.adoptFile(filename);
+ string database = otdbconf.getString("SASGateway.OTDBdatabase");
+ string dbhost = otdbconf.getString("SASGateway.OTDBhostname");
+ itsOTDBconn = new OTDBconnection("paulus", "boskabouter", database, dbhost);
+ if (!itsOTDBconn->connect()) {
+ LOG_FATAL_STR("Cannot connect to database " << database << " on machine " << dbhost << ", retry in 10 seconds");
+ itsTimer->setTimer(10.0);
+ return (GCFEvent::HANDLED);
+ }
+ LOG_INFO_STR("Connected to database " << database << " on machine " << dbhost);
+ TRAN(Feedback::operational_state);
+ } break;
+
+ default:
+ LOG_DEBUG_STR("connect2OTDB: not handled:" << eventName(event));
+ break;
+ } // switch
+
+ return (GCFEvent::HANDLED);
+}
+
+//
+// operational_state(event, port);
+//
+GCFEvent::TResult Feedback::operational_state(GCFEvent& event, GCFPortInterface& port)
+{
+ LOG_DEBUG_STR ("operational:" << eventName(event) << "@" << port.getName());
+
+ switch (event.signal) {
+ case F_ENTRY:
+ case F_TIMER: {
+ // wait (blocking) for next message
+ Message msg;
+ if (!itsMsgQueue->getMessage(msg)) {
+ LOG_FATAL_STR("Wow, lost connection with message bus");
+ GCFScheduler::instance()->stop(); // TODO better solution
+ }
+
+ if (!itsOTDBconn->connect()) {
+ LOG_ERROR("Lost connection with OTDB, starting reconnect cycle");
+ delete itsOTDBconn;
+ itsOTDBconn = 0;
+ itsMsgQueue->nack(msg);
+ TRAN (Feedback::connect2OTDB_state);
+ return (GCFEvent::HANDLED);
+ }
+
+ // Yeah, still connected.
+ if (passKVpairsToOTDB(atoi(msg.sasid().c_str()), msg.payload())) {
+ itsMsgQueue->ack(msg);
+ LOG_DEBUG("Message processed successful");
+ }
+ else {
+ itsMsgQueue->reject(msg);
+ LOG_DEBUG("Message rejected");
+ }
+ itsTimer->setTimer(0.0);
+ } break;
+
+ default:
+ LOG_DEBUG_STR("operational: not handled:" << eventName(event));
+ break;
+ } // switch
+
+ return (GCFEvent::HANDLED);
+}
+
+
+//
+// passKVpairsToOTDB()
+//
+// Handler for procssing the messages that are received on the bus.
+//
+bool Feedback::passKVpairsToOTDB(int obsID, const string& content)
+{
+ try {
+ TreeValue tv(itsOTDBconn, obsID);
+
+ // read parameterset
+ ParameterSet metadata;
+ metadata.adoptBuffer(content);
+
+ // Loop over the parameterset and send the information to the SAS database
+ // During the transition phase from parameter-based to record-based storage in OTDB the
+ // nodenames ending in '_' are implemented both as parameter and as record.
+ ParameterSet::iterator iter = metadata.begin();
+ ParameterSet::iterator end = metadata.end();
+ while (iter != end) {
+ string key(iter->first); // make destoyable copy
+ rtrim(key, "[]0123456789");
+ // bool doubleStorage(key[key.size()-1] == '_');
+ bool isRecord(iter->second.isRecord());
+ // isRecord doubleStorage
+ // --------------------------------------------------------------
+ // Y Y store as record and as parameters
+ // Y N store as parameters
+ // N * store parameter
+ if (!isRecord) {
+ LOG_DEBUG_STR("BASIC: " << iter->first << " = " << iter->second);
+ tv.addKVT(iter->first, iter->second, ptime(microsec_clock::local_time()));
+ }
+ else {
+ // if (doubleStorage) {
+ // LOG_DEBUG_STR("RECORD: " << iter->first << " = " << iter->second);
+ // tv.addKVT(iter->first, iter->second, ptime(microsec_clock::local_time()));
+ // }
+ // to store is a node/param values the last _ should be stipped of
+ key = iter->first; // destroyable copy
+ // string::size_type pos = key.find_last_of('_');
+ // key.erase(pos,1);
+ ParameterRecord pr(iter->second.getRecord());
+ ParameterRecord::const_iterator prIter = pr.begin();
+ ParameterRecord::const_iterator prEnd = pr.end();
+ while (prIter != prEnd) {
+ LOG_DEBUG_STR("ELEMENT: " << key+"."+prIter->first << " = " << prIter->second);
+ tv.addKVT(key+"."+prIter->first, prIter->second, ptime(microsec_clock::local_time()));
+ prIter++;
+ }
+ }
+ iter++;
+ }
+ LOG_INFO_STR(metadata.size() << " metadata values send to SAS");
+ return (true);
+ }
+ catch (Exception &ex) {
+ LOG_FATAL_STR(ex.what());
+ }
+ return (false);
+}
+
+
+
+ } // namespace SAS
+} // namespace LOFAR
diff --git a/SAS/Feedback_Service/src/Feedback.h b/SAS/Feedback_Service/src/Feedback.h
new file mode 100644
index 0000000000000000000000000000000000000000..cc85c8c0ad11c46d22f4fec63ca8132787dde70a
--- /dev/null
+++ b/SAS/Feedback_Service/src/Feedback.h
@@ -0,0 +1,82 @@
+//# Feedback.h: Pass key-value pairs to the database.
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id: $
+
+#ifndef FEEDBACK_SERVICE_H_
+#define FEEDBACK_SERVICE_H_
+
+// \file Feedback.h
+// Pass key-value pairs to the database.
+
+//# Never #include <config.h> or #include <lofar_config.h> in a header file!
+//# Includes
+#include <MACIO/GCF_Event.h>
+#include <GCF/TM/GCF_Control.h>
+#include <OTDB/OTDBconnection.h>
+#include <MessageBus/MsgBus.h>
+
+// Avoid 'using namespace' in headerfiles
+
+namespace LOFAR {
+ namespace SAS {
+
+using MACIO::GCFEvent;
+using GCF::TM::GCFTimerPort;
+using GCF::TM::GCFPort;
+using GCF::TM::GCFPortInterface;
+using GCF::TM::GCFTask;
+using OTDB::OTDBconnection;
+
+// \addtogroup package
+// @{
+
+
+class Feedback : public GCFTask
+{
+public:
+ Feedback();
+ ~Feedback();
+
+ // Interrupthandler for switching to finishingstate when exiting the program.
+ static void sigintHandler (int signum);
+ void finish();
+
+private:
+ GCFEvent::TResult connect2OTDB_state(GCFEvent& e, GCFPortInterface& p);
+ GCFEvent::TResult operational_state (GCFEvent& e, GCFPortInterface& p);
+ bool passKVpairsToOTDB(int obsID, const string& content);
+
+ // Copying is not allowed
+ Feedback(const Feedback& that);
+ Feedback& operator=(const Feedback& that);
+
+ //# --- Datamembers ---
+ GCFTimerPort* itsTimer; // generic timer
+ OTDBconnection* itsOTDBconn; // connection with OTDB
+ FromBus* itsMsgQueue; // session with Qpid.
+
+};
+
+// @}
+ } // namespace SAS
+} // namespace LOFAR
+
+#endif
diff --git a/SAS/Feedback_Service/src/FeedbackMain.cc b/SAS/Feedback_Service/src/FeedbackMain.cc
new file mode 100644
index 0000000000000000000000000000000000000000..1dffbed678f5f352aa07a2a41e833ba76227c62f
--- /dev/null
+++ b/SAS/Feedback_Service/src/FeedbackMain.cc
@@ -0,0 +1,53 @@
+//# FeedbackMain.cc: Main entry for the Feedback controller.
+//#
+//# Copyright (C) 2015
+//# ASTRON (Netherlands Foundation for Research in Astronomy)
+//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands, seg@astron.nl
+//#
+//# This program is free software; you can redistribute it and/or modify
+//# it under the terms of the GNU General Public License as published by
+//# the Free Software Foundation; either version 2 of the License, or
+//# (at your option) any later version.
+//#
+//# This program is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License
+//# along with this program; if not, write to the Free Software
+//# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//#
+//# $Id: FeedbackMain.cc 23491 2013-01-11 10:07:59Z overeem $
+//#
+#include <lofar_config.h>
+#include <Common/LofarLogger.h>
+#include <Common/Exception.h>
+#include <GCF/TM/GCF_Control.h>
+
+#include "Feedback.h"
+
+using namespace LOFAR::GCF::TM;
+using namespace LOFAR;
+using namespace LOFAR::SAS;
+
+// Use a terminate handler that can produce a backtrace.
+Exception::TerminateHandler t(Exception::terminate);
+
+int main(int argc, char* argv[])
+{
+ try {
+ GCFScheduler::instance()->init(argc, argv, "FeedbackService");
+
+ Feedback fbTask;
+ fbTask.start(); // make initial transition
+
+ GCFScheduler::instance()->run(); // until stop was called.
+ } catch( Exception &ex ) {
+ LOG_FATAL_STR("Caught exception: " << ex);
+ return 1;
+ }
+
+ return (0);
+}
+
diff --git a/SAS/Feedback_Service/src/FeedbackService.conf b/SAS/Feedback_Service/src/FeedbackService.conf
new file mode 100644
index 0000000000000000000000000000000000000000..64c745844ba358847dc43cc822834cf54423b1b2
--- /dev/null
+++ b/SAS/Feedback_Service/src/FeedbackService.conf
@@ -0,0 +1,7 @@
+#
+# FeedbackService.conf
+#
+# Parameters for FeedbackService to connect to Qpid.
+#
+FeedbackQueuenames = [ "otdb.task.feedback.dataproducts" , "otdb.task.feedback.processing" ]
+
diff --git a/SAS/Feedback_Service/src/fb_data_44883.txt b/SAS/Feedback_Service/src/fb_data_44883.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e6a7627720e1a4fcd441bdf2695137b23c75118f
--- /dev/null
+++ b/SAS/Feedback_Service/src/fb_data_44883.txt
@@ -0,0 +1,40 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>lofar.observation.start</name>
+ <version>1.0</version>
+ </protocol>
+ <source>
+ <name>mySubSystem</name>
+ <user>user</user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>some test message</summary>
+ </source>
+ <ids>
+ <momid>12345</momid>
+ <sasid>44883</sasid>
+ </ids>
+ </header>
+ <payload>
+Observation.Correlator.channelWidth=3051.7578125
+Observation.Correlator.channelsPerSubband=64
+Observation.Correlator.integrationInterval=1.00139008
+Observation.DataProducts.Output_Correlated_[0].SAP=0
+Observation.DataProducts.Output_Correlated_[0].centralFrequency=115039062.500000
+Observation.DataProducts.Output_Correlated_[0].channelWidth=3051.757812
+Observation.DataProducts.Output_Correlated_[0].channelsPerSubband=64
+Observation.DataProducts.Output_Correlated_[0].duration=119.165420
+Observation.DataProducts.Output_Correlated_[0].fileFormat=AIPS++/CASA
+Observation.DataProducts.Output_Correlated_[0].filename=L257915_SAP000_SB000_uv.MS
+Observation.DataProducts.Output_Correlated_[0].integrationInterval=1.001390
+Observation.DataProducts.Output_Correlated_[0].location=locus001:/data/L257915/
+Observation.DataProducts.Output_Correlated_[0].percentageWritten=100
+Observation.DataProducts.Output_Correlated_[0].size=268083200
+Observation.DataProducts.Output_Correlated_[0].startTime=2015-01-15 09:32:09
+Observation.DataProducts.Output_Correlated_[0].stationSubband=77
+Observation.DataProducts.Output_Correlated_[0].subband=0
+ </payload>
+</message>
diff --git a/SAS/OTDB/sql/exportTree_func.sql b/SAS/OTDB/sql/exportTree_func.sql
index 71a729345d6842c9a20f799ecabec303d1bb75c8..0d525c743bdda86d78378d7f0fb6c8f6cc7aab4d 100644
--- a/SAS/OTDB/sql/exportTree_func.sql
+++ b/SAS/OTDB/sql/exportTree_func.sql
@@ -22,6 +22,49 @@
-- $Id$
--
+-- exportIDs(treeID, prefixLen)
+--
+-- Return all the IDs that are attached to this tree (sas/mom/origin)
+--
+-- Authorisation: none
+--
+-- Tables: otdbtree read
+--
+-- Types: none
+--
+CREATE OR REPLACE FUNCTION exportIDs(INT4, INT4)
+ RETURNS TEXT AS $$
+ -- $Id$
+ DECLARE
+ vResult TEXT := '';
+ vPrefix TEXT;
+ vTreeID OTDBtree.treeID%TYPE;
+ vMoMID OTDBtree.momID%TYPE;
+ vOriginID OTDBtree.originID%TYPE;
+ aTreeID ALIAS FOR $1;
+ aPrefixLen ALIAS FOR $2;
+
+ BEGIN
+ -- get processInfo
+ SELECT treeid, momid, originid
+ INTO vTreeID, vMoMID, vOriginID
+ FROM OTDBtree
+ WHERE treeID = aTreeID;
+ IF NOT FOUND THEN
+ RAISE EXCEPTION 'Tree % does not exist', aTreeID;
+ END IF;
+
+ SELECT substr(name, aPrefixLen)
+ INTO vPrefix
+ FROM getVHitemList(aTreeID, '%.Observation');
+ vResult := vResult || vPrefix || '.otdbID=' || vTreeID || chr(10);
+ vResult := vResult || vPrefix || '.momID=' || vMoMID || chr(10);
+ vResult := vResult || vPrefix || '.originID=' || vOriginID || chr(10);
+
+ RETURN vResult;
+ END;
+$$ LANGUAGE plpgsql;
+
--
-- recursive helper function
-- fullTemplateNodeName (treeID, nodeID, lastPart)
@@ -288,7 +331,7 @@ CREATE OR REPLACE FUNCTION exportTree(INT4, INT4, INT4)
vResult := vResult || exportPICSubTree($2, $3, vPrefixLen+2);
ELSE
IF vIsVicTree THEN
- vResult := vResult || exportVICSubTree($2, $3, vPrefixLen+2) || exportProcessType($2, vPrefixLen+2) || exportCampaign($2, vPrefixLen+2);
+ vResult := vResult || exportVICSubTree($2, $3, vPrefixLen+2) || exportProcessType($2, vPrefixLen+2) || exportCampaign($2, vPrefixLen+2) || exportIDs($2, vPrefixLen+2);
ELSE
vResult := vResult || exportTemplateSubTree($2, $3, '');
END IF;
diff --git a/SubSystems/CCU_MAC/CMakeLists.txt b/SubSystems/CCU_MAC/CMakeLists.txt
index 3386fbd5067dc0bbe023eeb8a66dd46d6328386f..e8ddd38992d67474da2bcd49cf83d327eed2217e 100644
--- a/SubSystems/CCU_MAC/CMakeLists.txt
+++ b/SubSystems/CCU_MAC/CMakeLists.txt
@@ -1,5 +1,5 @@
# $Id$
lofar_package(CCU_MAC
- DEPENDS CEPCU CUDaemons CURTDBDaemons StaticMetaData)
+ DEPENDS CEPCU CUDaemons CURTDBDaemons StaticMetaData MessageDaemons)
diff --git a/SubSystems/MCU_MAC/CMakeLists.txt b/SubSystems/MCU_MAC/CMakeLists.txt
index 4cf6b00c6e7ffb24f136d5eb41ffffa887c9c5b6..b92b378f0fd3d9e00bf620a44cad1038020df66a 100644
--- a/SubSystems/MCU_MAC/CMakeLists.txt
+++ b/SubSystems/MCU_MAC/CMakeLists.txt
@@ -1,4 +1,4 @@
# $Id$
lofar_package(MCU_MAC
- DEPENDS MainCU CUDaemons CURTDBDaemons StaticMetaData)
+ DEPENDS MainCU CUDaemons CURTDBDaemons StaticMetaData SAS_Feedback)
diff --git a/SubSystems/Offline/CMakeLists.txt b/SubSystems/Offline/CMakeLists.txt
index f11dad6519c8ead9b279515f1a33952fb0577b33..f11418570ff5891292fe2deb87bfe63fa7004afe 100644
--- a/SubSystems/Offline/CMakeLists.txt
+++ b/SubSystems/Offline/CMakeLists.txt
@@ -1,4 +1,4 @@
# $Id$
-lofar_package(Offline DEPENDS CEP MSLofar StaticMetaData)
+lofar_package(Offline DEPENDS CEP MSLofar StaticMetaData MessageBus)
diff --git a/SubSystems/Online_Cobalt/install/install_qpid.sh b/SubSystems/Online_Cobalt/install/install_qpid.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f77144ac0c639abd23fcaaf38c4c71e35fbf9084
--- /dev/null
+++ b/SubSystems/Online_Cobalt/install/install_qpid.sh
@@ -0,0 +1,90 @@
+#!/bin/bash -eu
+
+# Note: this script is derived from the QPID build procedure
+# described on http://www.lofar.org/wiki/doku.php?id=qpid:build
+
+# We need to be lofarbuild to have the proper writing rights
+[ "`whoami`" == "lofarbuild" ]
+
+# Download location for the latest QPID source
+PROTON_SOURCE="http://svn.apache.org/repos/asf/qpid/proton/branches/0.8"
+QPID_SOURCE="http://svn.apache.org/repos/asf/qpid/branches/0.30/qpid/"
+
+QPID_INSTALLDIR=/localhome/lofar/qpid
+
+# ********************************************
+# Install latest PROTON & QPID
+#
+# into $QPID_INSTALLDIR
+# ********************************************
+echo "Configuring PROTON and QPID..."
+mkdir -p $QPID_INSTALLDIR
+
+QPID_BUILDDIR=`mktemp -d`
+pushd $QPID_BUILDDIR >/dev/null
+
+echo " Downloading PROTON..."
+svn co $PROTON_SOURCE proton >/dev/null
+
+echo " Configuring PROTON..."
+pushd proton >/dev/null
+mkdir build
+cd build
+cmake -DCMAKE_INSTALL_PREFIX=$QPID_INSTALLDIR -DBUILD_PERL=OFF .. > cmake.log
+
+echo " Building PROTON..."
+make -j 8 > make.log
+
+echo " Installing PROTON..."
+make -j 8 install > make_install.log
+
+# back to QPID_BUILDDIR
+popd >/dev/null
+
+echo " Downloading QPID..."
+svn co $QPID_SOURCE qpid >/dev/null
+
+echo " Configuring QPID C bindings..."
+pushd qpid/cpp >/dev/null
+mkdir build
+cd build
+cmake -DCMAKE_INSTALL_PREFIX=$QPID_INSTALLDIR -DProton_DIR=/localhome/lofar/qpid/lib/cmake/Proton -DBUILD_XML=OFF -DBUILD_SSL=OFF -DBUILD_BINDING_RUBY=OFF .. > cmake.log
+
+echo " Building QPID C bindings..."
+make -j 8 > make.log
+
+echo " Installing QPID C bindings..."
+make -j 8 install > make_install.log
+
+# back to QPID_BUILDDIR
+popd >/dev/null
+
+echo " Building and installing QPID Python bindings..."
+pushd qpid/python >/dev/null
+./setup.py build > setup_build.log
+./setup.py install --home=$QPID_INSTALLDIR > setup_install.log
+
+echo " Building and installing QPID Python QMF..."
+popd >/dev/null
+pushd qpid/extras/qmf >/dev/null
+./setup.py build > setup_build.log
+./setup.py install --home=$QPID_INSTALLDIR > setup_install.log
+
+echo " Building and installing QPID Python tools..."
+popd >/dev/null
+pushd qpid/tools >/dev/null
+./setup.py build > setup_build.log
+./setup.py install --home=$QPID_INSTALLDIR > setup_install.log
+
+echo " Creating .profile..."
+PYTHONVERSION=`python -c 'import platform; print "%s.%s" % platform.python_version_tuple()[0:2]'`
+cat > $QPID_INSTALLDIR/.profile << EOF
+export PATH=\$PATH:$QPID_INSTALLDIR/sbin/:$QPID_INSTALLDIR/bin/:$QPID_INSTALLDIR/local/bin/
+export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$QPID_INSTALLDIR/lib:$QPID_INSTALLDIR/local/lib/
+export PYTHONPATH=\$PYTHONPATH:$QPID_INSTALLDIR/lib/python/:$QPID_INSTALLDIR/lib/python$PYTHONVERSION/site-packages/
+EOF
+
+echo " Cleaning up..."
+popd >/dev/null
+rm -rf "$QPID_BUILDDIR"
+
diff --git a/SubSystems/Online_Cobalt/install/postinstall.sh b/SubSystems/Online_Cobalt/install/postinstall.sh
index a894b5fd5c6366e16c89506afa22d737d93db90c..f84f819ca187c5192751a64c6c35ce8c7999b052 100755
--- a/SubSystems/Online_Cobalt/install/postinstall.sh
+++ b/SubSystems/Online_Cobalt/install/postinstall.sh
@@ -35,6 +35,7 @@ function postinstall_lofarbuild {
./install_IERS.sh
./install_DAL.sh
./install_casacore.sh
+ ./install_qpid.sh
}
case "`whoami`" in
diff --git a/SubSystems/Online_Cobalt/test/CMakeLists.txt b/SubSystems/Online_Cobalt/test/CMakeLists.txt
index 67349fe148aef60bc99695d5d8e3cb4144023d35..9682fe742845f10afcbcea8ac4c0b8c8964a7754 100644
--- a/SubSystems/Online_Cobalt/test/CMakeLists.txt
+++ b/SubSystems/Online_Cobalt/test/CMakeLists.txt
@@ -2,15 +2,15 @@
include(LofarCTest)
-foreach(_file testFuncs runtest)
+foreach(_file testFuncs.sh runtest.sh tStatusFeedback.run2)
configure_file(
- ${CMAKE_CURRENT_SOURCE_DIR}/${_file}.sh.in
- ${CMAKE_CURRENT_BINARY_DIR}/${_file}.sh @ONLY)
+ ${CMAKE_CURRENT_SOURCE_DIR}/${_file}.in
+ ${CMAKE_CURRENT_BINARY_DIR}/${_file} @ONLY)
endforeach()
set(_tests
tgenerateStationStreams
- tMACfeedback
+ tStatusFeedback
tProductionParsets
tstartBGL
)
diff --git a/SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.processing b/SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.processing
new file mode 100644
index 0000000000000000000000000000000000000000..21dafb3099c1f3dcb20bc810853ade3d8f99cbec
--- /dev/null
+++ b/SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.processing
@@ -0,0 +1,30 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>task.feedback.processing</name>
+ <version>1.0.0</version>
+ </protocol>
+ <source>
+ <name>Cobalt/GPUProc/rtcp</name>
+ <user></user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>Processing feedback</summary>
+ </source>
+ <ids>
+ <momid>0</momid>
+ <sasid>76966</sasid>
+ </ids>
+ </header>
+ <payload>
+Observation.Correlator.channelWidth=12207.03125
+Observation.Correlator.channelsPerSubband=16
+Observation.Correlator.integrationInterval=0.25165824
+Observation.DataProducts.nrOfOutput_Beamformed_=0
+Observation.DataProducts.nrOfOutput_Correlated_=5
+_isCobalt=T
+
+ </payload>
+</message>
diff --git a/SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.state b/SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.state
new file mode 100644
index 0000000000000000000000000000000000000000..be60b26610df342f9d804c1ee123b3517b503d13
--- /dev/null
+++ b/SubSystems/Online_Cobalt/test/Correlator/tCorrelate_1sec_1st_5sb_noflagging.output/queues/lofar.task.feedback.state
@@ -0,0 +1,27 @@
+<message>
+ <header>
+ <system>LOFAR</system>
+ <version>1.0.0</version>
+ <protocol>
+ <name>task.feedback.state</name>
+ <version>1.0.0</version>
+ </protocol>
+ <source>
+ <name>Cobalt/GPUProc/send_state</name>
+ <user></user>
+ <uuid></uuid>
+ <timestamp></timestamp>
+ <summary>State feedback</summary>
+ </source>
+ <ids>
+ <momid>0</momid>
+ <sasid>76966</sasid>
+ </ids>
+ </header>
+ <payload>
+<task>
+ <type>observation</type>
+ <state>finished</state>
+</task>
+ </payload>
+</message>
diff --git a/SubSystems/Online_Cobalt/test/MockOnlineControl.sh b/SubSystems/Online_Cobalt/test/MockOnlineControl.sh
deleted file mode 100755
index d22f2c3ea63e2c27b6535f33de7d863a9a77aa00..0000000000000000000000000000000000000000
--- a/SubSystems/Online_Cobalt/test/MockOnlineControl.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-
-# A mock OnlineControl to receive the observation status
-# and the LTA feedback.
-
-# 1. Read parset
-PARSET="$1"
-
-function error {
- echo "$@"
- exit 1
-}
-
-function getkey {
- KEY=$1
- <$PARSET perl -ne '/^'$KEY'\s*=\s*"?(.*?)"?\s*$/ || next; print "$1";' | tail -n 1
-}
-
-[ -n "$PARSET" ] || error "No parset specified"
-[ -r "$PARSET" ] || error "Cannot read parset: $PARSET"
-
-# 2. Open port @ 21000 + 1000 % obsid
-OBSID=`getkey Observation.ObsID`
-RESULT_PORT=$((21000 + $OBSID % 1000))
-
-STATUSSTR=`timeout 120s nc -l $RESULT_PORT`
-
-# 3. Read string: ABORT or FINISHED
-if [ "$STATUSSTR" == "FINISHED" ]
-then
- echo "Observation reported success"
-
- # 4. If finished, check for existence of feedback file
- FEEDBACK_FILE=`getkey Cobalt.Feedback.remotePath`/Observation${OBSID}_feedback
-
- # Check existence and access rights
- [ -e $FEEDBACK_FILE ] || error "Feedback file not found: $FEEDBACK_FILE"
- [ -r $FEEDBACK_FILE ] || error "Feedback file not readable: $FEEDBACK_FILE"
- [ -f $FEEDBACK_FILE ] || error "Feedback file not a regular file: $FEEDBACK_FILE"
-
- # Check file size
- FILESIZE=`stat -c %s $FEEDBACK_FILE`
- [ $FILESIZE -ne 0 ] || error "Feedback file empty: $FEEDBACK_FILE"
-elif [ "$STATUSSTR" == "ABORT" ]
-then
- echo "Observation reported failure"
-else
- error "Invalid status string: '$STATUSSTR'"
-fi
-
-exit 0
-
diff --git a/SubSystems/Online_Cobalt/test/runtest.sh.in b/SubSystems/Online_Cobalt/test/runtest.sh.in
index d3fc83b64248ebdd4151599c9ec71a1fb1a23acb..04e5845e1ae4447d6dc566597ae5dedfda074c17 100755
--- a/SubSystems/Online_Cobalt/test/runtest.sh.in
+++ b/SubSystems/Online_Cobalt/test/runtest.sh.in
@@ -36,7 +36,7 @@ echo " in directory $(pwd)"
[ -n "$(echo *.raw)" ] || error "runObservation.sh produced no output files"
# create script to accept output (ie. copy it to the source dir for check in)
- echo "#!/bin/sh
+ echo "#!/bin/bash
cp ${PWD}/*.raw ${REFDIR}" > accept_output
chmod a+x accept_output
@@ -60,5 +60,21 @@ echo " in directory $(pwd)"
done
done
+ if have_qpid; then
+ # validate the contents of the queues
+ mkdir "queues" || error "Failed to create temporary directory ${OUTDIR}/queues"
+ echo "[ -d ${PWD}/queues ] && cp -r ${PWD}/queues ${REFDIR}" >> accept_output
+ QUEUES=`cd ${REFDIR}/queues 2>/dev/null && ls`
+ INCORRECT_QUEUES=""
+ for Q in $QUEUES; do
+ echo "Comparing output of queue $Q"
+ recv_all_msgs "$Q" > queues/$Q
+ compare_msg ${REFDIR}/queues/$Q queues/$Q || INCORRECT_QUEUES+="$Q "
+ done
+
+ if [ "$INCORRECT_QUEUES" != "" ]; then
+ error "Output of queue(s) $INCORRECT_QUEUES do(es) not match"
+ fi
+ fi
) || exit 1
diff --git a/SubSystems/Online_Cobalt/test/tMACfeedback.run b/SubSystems/Online_Cobalt/test/tMACfeedback.run
deleted file mode 100755
index 620ee8e3575357353286b735a09be3c3984c0dd8..0000000000000000000000000000000000000000
--- a/SubSystems/Online_Cobalt/test/tMACfeedback.run
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-. ./testFuncs.sh
-
-function test_parset {
- PARSET="$1"
- EXPECTED_OBSRESULT="$2"
- EXTRA_PARAMS="$3"
-
- echo "Testing $PARSET, expecting result $EXPECTED_OBSRESULT"
-
- # Add the connection information for this test
- echo "Cobalt.Feedback.host=localhost" >> $PARSET
- echo "Cobalt.Feedback.remotePath=$LOFARROOT/var/run" >> $PARSET
- echo "Cobalt.FinalMetaDataGatherer.host=localhost" >> $PARSET
-
- # Start a mock OnlineControl to verify the communications of runObservation.sh
- $srcdir/MockOnlineControl.sh $PARSET &
- ONLINECONTROL_PID=$!
-
- # Run the observation
- runObservation.sh -B -C -l 1 $EXTRA_PARAMS $PARSET
- OBSRESULT=$?
-
- # Wait for OnlineControl to finish
- wait $ONLINECONTROL_PID
- ONLINECONTROLRESULT=$?
-
- if [ $OBSRESULT -ne $EXPECTED_OBSRESULT ]; then
- echo "runObservation.sh failed (status: $OBSRESULT)"
- exit 1
- fi
-
- if [ $ONLINECONTROLRESULT -gt 0 ]; then
- echo "MockOnlineControl.sh failed (status: $ONLINECONTROLRESULT)"
- exit 1
- fi
-}
-
-test_parset $PWD/tMACfeedback.in_parset_success_1 0 ""
-test_parset $PWD/tMACfeedback.in_parset_failure_1 1 "-o Cobalt.Nodes=[unreachable]"
-
-# Everything went ok
-exit 0
-
diff --git a/SubSystems/Online_Cobalt/test/tMACfeedback.sh b/SubSystems/Online_Cobalt/test/tMACfeedback.sh
deleted file mode 100755
index 5ea49e61cbf0971b5b46ce0445e5c77bcb98576e..0000000000000000000000000000000000000000
--- a/SubSystems/Online_Cobalt/test/tMACfeedback.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tMACfeedback
diff --git a/SubSystems/Online_Cobalt/test/tMACfeedback.in_parset_failure_1 b/SubSystems/Online_Cobalt/test/tStatusFeedback.in_parset_failure_1
similarity index 100%
rename from SubSystems/Online_Cobalt/test/tMACfeedback.in_parset_failure_1
rename to SubSystems/Online_Cobalt/test/tStatusFeedback.in_parset_failure_1
diff --git a/SubSystems/Online_Cobalt/test/tMACfeedback.in_parset_success_1 b/SubSystems/Online_Cobalt/test/tStatusFeedback.in_parset_success_1
similarity index 100%
rename from SubSystems/Online_Cobalt/test/tMACfeedback.in_parset_success_1
rename to SubSystems/Online_Cobalt/test/tStatusFeedback.in_parset_success_1
diff --git a/SubSystems/Online_Cobalt/test/tStatusFeedback.run b/SubSystems/Online_Cobalt/test/tStatusFeedback.run
new file mode 100755
index 0000000000000000000000000000000000000000..a2dbabeefb09e4705a2861fb66940405d0f2fdee
--- /dev/null
+++ b/SubSystems/Online_Cobalt/test/tStatusFeedback.run
@@ -0,0 +1,2 @@
+#!/bin/sh
+./tStatusFeedback.run2
diff --git a/SubSystems/Online_Cobalt/test/tStatusFeedback.run2.in b/SubSystems/Online_Cobalt/test/tStatusFeedback.run2.in
new file mode 100755
index 0000000000000000000000000000000000000000..bca4e8a3a97275c552407909b5c9c5ea9dce8ea5
--- /dev/null
+++ b/SubSystems/Online_Cobalt/test/tStatusFeedback.run2.in
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. ./testFuncs.sh
+
+function test_parset {
+ PARSET="$1"
+ EXPECTED_OBSRESULT="$2"
+ EXTRA_PARAMS="$3"
+
+ echo "Testing $PARSET, expecting result $EXPECTED_OBSRESULT"
+
+ # Add the connection information for this test
+ echo "Cobalt.FinalMetaDataGatherer.host=localhost" >> $PARSET
+
+ # Run the observation
+ runObservation.sh -B -C -l 1 $EXTRA_PARAMS $PARSET
+ OBSRESULT=$?
+
+ if [ $OBSRESULT -ne $EXPECTED_OBSRESULT ]; then
+ echo "runObservation.sh failed (status: $OBSRESULT)"
+ exit 1
+ fi
+
+ # Pull the message from the status queue
+ @QPID_RECEIVE_EXECUTABLE@ -b 127.0.0.1 -q lofar.task.feedback.status || error 'Could not pull status message from bus'
+
+ # Check the message against the expected result
+ # <TODO>
+}
+
+test_parset $PWD/tStatusFeedback.in_parset_success_1 0 ""
+test_parset $PWD/tStatusFeedback.in_parset_failure_1 1 "-o Cobalt.Nodes=[unreachable]"
+
+# Everything went ok
+exit 0
diff --git a/SubSystems/Online_Cobalt/test/tStatusFeedback.sh b/SubSystems/Online_Cobalt/test/tStatusFeedback.sh
new file mode 100755
index 0000000000000000000000000000000000000000..6c8d582909b8210b477eaf031e19a21b31797b9e
--- /dev/null
+++ b/SubSystems/Online_Cobalt/test/tStatusFeedback.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+./runctest.sh tStatusFeedback
diff --git a/SubSystems/Online_Cobalt/test/testFuncs.sh.in b/SubSystems/Online_Cobalt/test/testFuncs.sh.in
index 7744152fef5983ed55bba2846b79005e6f27b404..51e146d427b31edcbfa49d1042334e3c5b09872f 100755
--- a/SubSystems/Online_Cobalt/test/testFuncs.sh.in
+++ b/SubSystems/Online_Cobalt/test/testFuncs.sh.in
@@ -13,9 +13,24 @@ error()
# Set LOFARROOT and other LOFAR env vars into install directory (var is always set).
. "@CMAKE_INSTALL_PREFIX@/lofarinit.sh" || error "Could not load our lofarinit.sh -- did you run 'make install'?"
+# MessageFuncs.sh provides QPID functionality, and is present in the build directory
+. "MessageFuncs.sh" || error "Could not load MessageFuncs.sh"
+
# Create runtime output directories if not exists.
# Not done at build, because it is a post-install setting. Different in production.
mkdir -p "$LOFARROOT/var/log" "$LOFARROOT/var/run" || error "Failed to create runtime output directories"
# Set all locales to "C" to avoid problems with, e.g., perl.
export LC_ALL="C"
+
+# Clear all used QPID queues to prevent leakage from previous tests
+# (if they exist, so ignore any errors regarding that)
+
+QUEUES="
+ lofar.task.feedback.state
+ lofar.task.feedback.dataproducts
+ lofar.task.feedback.processing"
+for Q in $QUEUES; do
+ create_queue "$Q"
+done
+
diff --git a/SubSystems/Online_Cobalt/test/tstartBGL.run b/SubSystems/Online_Cobalt/test/tstartBGL.run
index 5718de6be95091cf5c0745a5cde2d88036d9d61b..75e8e3ca24ae45148a8de2e20f8c6305334ab0e0 100755
--- a/SubSystems/Online_Cobalt/test/tstartBGL.run
+++ b/SubSystems/Online_Cobalt/test/tstartBGL.run
@@ -59,8 +59,6 @@ echo "Test 5: normal run"
echo " (expects success)"
echo "***************************"
# Add the connection information for this test
-echo "Cobalt.Feedback.host=localhost" >> tstartBGL.in_parset
-echo "Cobalt.Feedback.remotePath=$LOFARROOT/var/run" >> tstartBGL.in_parset
echo "Cobalt.FinalMetaDataGatherer.host=localhost" >> tstartBGL.in_parset
startBGL.sh 1 2 3 tstartBGL.in_parset 1000 || error "startBGL.sh failed"
@@ -88,8 +86,6 @@ echo "Test 6: stop a run"
echo " (expects success)"
echo "***************************"
# Add the connection information for this test
-echo "Cobalt.Feedback.host=localhost" >> tstartBGL.in_parset
-echo "Cobalt.Feedback.remotePath=$LOFARROOT/var/run" >> tstartBGL.in_parset
echo "Cobalt.FinalMetaDataGatherer.host=localhost" >> tstartBGL.in_parset
# Run forever
diff --git a/lofar_config.h.cmake b/lofar_config.h.cmake
index 4b033b2950ae56dafd2a7cd02d9a00bfe8df0953..0f04b602f1f5b5a1547ab863e58846e1f1b1cbd5 100644
--- a/lofar_config.h.cmake
+++ b/lofar_config.h.cmake
@@ -156,6 +156,9 @@
/* Define if using Rational Purify */
#cmakedefine HAVE_PURIFY 1
+/* Define if QPID is installed */
+#cmakedefine HAVE_QPID 1
+
/* Define if readline is installed */
#cmakedefine HAVE_READLINE 1
@@ -165,6 +168,9 @@
/* Defined if shared memory is used */
#cmakedefine HAVE_SHMEM 1
+/* Defined if uuid is installed */
+#cmakedefine HAVE_UUID 1
+
/* Define if WCSLIB is installed */
#cmakedefine HAVE_WCSLIB 1