From 93d98ba7e01b68d6ee463515a41274ae7523a12f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rn=20K=C3=BCnsem=C3=B6ller?=
<jkuensem@physik.uni-bielefeld.de>
Date: Thu, 24 Jan 2019 10:39:03 +0000
Subject: [PATCH] Task SW-560: Moved trigger recipe to tbbservice.
---
.gitattributes | 1 +
CMake/LofarPackageList.cmake | 2 +-
.../TBB/TBBClient/lib/tbbservice_rpc.py | 6 ++
MAC/Services/TBB/TBBServer/lib/CMakeLists.txt | 1 +
MAC/Services/TBB/TBBServer/lib/tbbservice.py | 87 +++++++++++++++++--
.../TBB/TBBServer/lib/tbbservice_config.py | 3 +
SAS/TriggerServices/lib/config.py | 3 -
SAS/TriggerServices/lib/trigger_service.py | 6 +-
8 files changed, 96 insertions(+), 13 deletions(-)
create mode 100644 MAC/Services/TBB/TBBServer/lib/tbbservice_config.py
diff --git a/.gitattributes b/.gitattributes
index e23eedc2712..2635a7e02ee 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3694,6 +3694,7 @@ MAC/Services/TBB/TBBServer/bin/tbbservice -text
MAC/Services/TBB/TBBServer/bin/tbbservice.ini -text
MAC/Services/TBB/TBBServer/lib/CMakeLists.txt -text
MAC/Services/TBB/TBBServer/lib/tbbservice.py -text
+MAC/Services/TBB/TBBServer/lib/tbbservice_config.py -text
MAC/Services/TBB/TBBServer/test/CMakeLists.txt -text
MAC/Services/TBB/TBBServer/test/t_tbbserver.py -text
MAC/Services/TBB/TBBServer/test/t_tbbserver.run -text
diff --git a/CMake/LofarPackageList.cmake b/CMake/LofarPackageList.cmake
index 689e7251422..d1c6d7c1b8c 100644
--- a/CMake/LofarPackageList.cmake
+++ b/CMake/LofarPackageList.cmake
@@ -1,7 +1,7 @@
# - Create for each LOFAR package a variable containing the absolute path to
# its source directory.
#
-# Generated by gen_LofarPackageList_cmake.sh at vr 18 jan 2019 15:16:58 CET
+# Generated by gen_LofarPackageList_cmake.sh at Do 24. Jan 11:12:48 CET 2019
#
# ---- DO NOT EDIT ----
#
diff --git a/MAC/Services/TBB/TBBClient/lib/tbbservice_rpc.py b/MAC/Services/TBB/TBBClient/lib/tbbservice_rpc.py
index 9c6f606e271..eb1c699c0a9 100644
--- a/MAC/Services/TBB/TBBClient/lib/tbbservice_rpc.py
+++ b/MAC/Services/TBB/TBBClient/lib/tbbservice_rpc.py
@@ -68,6 +68,12 @@ class TBBRPC(RPCWrapper):
logger.info("Received set storage request result %s" % result)
return result
+ def do_tbb_subband_dump(self, starttime, duration, dm, project, triggerid, stations, subbands, boards, nodes, voevent_xml, stoptime=None, rcus=None):
+ logger.info("Requesting full tbb dump to CEP for trigger %s and project %s" % (triggerid, project))
+ result = self.rpc('do_tbb_subband_dump', starttime, duration, dm, project, triggerid, stations, subbands, boards, nodes, voevent_xml, stoptime=stoptime, rcus=rcus)
+ logger.info("Received full dump to CEP result for trigger %s and project %s: %s" % (triggerid, project, result))
+ return result
+
if __name__ == '__main__':
'''little example usage'''
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
diff --git a/MAC/Services/TBB/TBBServer/lib/CMakeLists.txt b/MAC/Services/TBB/TBBServer/lib/CMakeLists.txt
index ab2ef4c8363..104760acdf5 100644
--- a/MAC/Services/TBB/TBBServer/lib/CMakeLists.txt
+++ b/MAC/Services/TBB/TBBServer/lib/CMakeLists.txt
@@ -2,4 +2,5 @@
include(PythonInstall)
python_install(tbbservice.py
+ tbbservice_config.py
DESTINATION lofar/mac/tbbservice/server)
diff --git a/MAC/Services/TBB/TBBServer/lib/tbbservice.py b/MAC/Services/TBB/TBBServer/lib/tbbservice.py
index 556e16d50b1..0c5113b586b 100644
--- a/MAC/Services/TBB/TBBServer/lib/tbbservice.py
+++ b/MAC/Services/TBB/TBBServer/lib/tbbservice.py
@@ -43,11 +43,13 @@ from lofar.mac.tbb.tbb_load_firmware import load_tbb_firmware
from lofar.mac.tbb.tbb_freeze import freeze_tbb
from lofar.mac.tbb.tbb_release_recording import release_tbb
from lofar.mac.tbb.tbb_restart_recording import restart_tbb_recording
-from lofar.mac.tbb.tbb_set_storage import set_tbb_storage
+from lofar.mac.tbb.tbb_set_storage import set_tbb_storage, create_mapping
from lofar.mac.tbb.tbb_start_recording import start_tbb
from lofar.mac.tbb.tbb_upload_to_cep import upload_tbb_data
from lofar.parameterset import parameterset
from lofar.mac.tbb.tbb_util import parse_parset_from_voevent
+from lofar.common.lcu_utils import stationname2hostname, hostname2stationname
+from . import tbbservice_config
class TBBServiceMessageHandler(MessageHandlerInterface):
'''
@@ -74,7 +76,9 @@ class TBBServiceMessageHandler(MessageHandlerInterface):
'restart_recording': self._tbb_control_service.restart_recording,
'upload_data': self._tbb_control_service.upload_data,
'freeze_data': self._tbb_control_service.freeze_data,
- 'set_storage': self._tbb_control_service.set_storage}
+ 'set_storage': self._tbb_control_service.set_storage,
+ 'do_tbb_subband_dump': self._tbb_control_service.do_tbb_subband_dump
+ }
class TBBControlService:
@@ -336,7 +340,7 @@ class TBBControlService:
log_message = "Switching TBB firmware on stations %s to \"%s\"" % (stations, mode)
logger.info(log_message + "...")
load_tbb_firmware(stations, mode)
- logger.info(log_message + "done.")
+ logger.info(log_message + " done.")
def start_recording(self, stations, mode, subbands):
"""
@@ -348,7 +352,7 @@ class TBBControlService:
log_message = "Starting the TBB data recording for mode %s on stations %s in sub-bands %s" % (mode, stations, subbands)
logger.info(log_message + "...")
start_tbb(stations, mode, subbands)
- logger.info(log_message + "done.")
+ logger.info(log_message + " done.")
def freeze_data(self, stations, dm, timesec, timensec):
"""
@@ -384,7 +388,7 @@ class TBBControlService:
"wait time = %f" % (stations, boards, sub_bands, dm, start_time, duration, wait_time)
logger.info(log_message + "...")
upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, boards)
- logger.info(log_message + "done.")
+ logger.info(log_message + " done.")
def release_data(self, stations):
"""
@@ -394,7 +398,7 @@ class TBBControlService:
log_message = "Releasing the TBB data recording on stations %s" % (stations)
logger.info(log_message + "...")
release_tbb(stations)
- logger.info(log_message + "done.")
+ logger.info(log_message + " done.")
def restart_recording(self, stations):
"""
@@ -404,7 +408,7 @@ class TBBControlService:
log_message = "Restarting the TBB data recording on stations %s" % (stations)
logger.info(log_message + "...")
restart_tbb_recording(stations)
- logger.info(log_message + "done.")
+ logger.info(log_message + " done.")
def set_storage(self, map):
"""
@@ -414,7 +418,74 @@ class TBBControlService:
log_message = "Setting the storage nodes where TBB data is received to: %s" % (map)
logger.info(log_message + "...")
set_tbb_storage(map)
- logger.info(log_message + "done.")
+ logger.info(log_message + " done.")
+
+ def do_tbb_subband_dump(self, starttime, duration, dm, project, triggerid,
+ stations, subbands, boards, nodes, voevent_xml,
+ stoptime=None, rcus=None):
+ """
+ This 'recipe' call freezes data on the boards and performs the tbbdump.
+ The first data of the dump is recorded at starttime in the subband of highest frequency. Subbands of lower frequency
+ are then delayed by an offset calculated from the provided dm, and each subband holds data of the provided duration.
+ Hence, the starttime should predate the expected time of arrival of the event at the frequency of the highest
+ subband by half the recording duration to center the event in the recorded data for all subbands.
+
+ :param starttime: The starttime of the recording
+ :param duration: The duration of the recording in seconds as float
+ :param dm: The dispersion measure (in pc/cm^3) of the event to capture as float
+ :param project: The project identifier as string
+ :param triggerid: The trigger identifier as string
+ :param stoptime: The stoptime in seconds since Epoch as float
+ :param stations: The stations to use as list of strings
+ :param subbands: The subbands to use as list of integers
+ :param rcus: The receivers to use as list of integers
+ :param boards: The tbb boards to use as list of integers
+ :param nodes: The CEP nodes to send data to
+ :param voevent_xml: the entire voevent in xml as string
+ :return:
+ """
+
+ log_message = "Performing TBB data dump to CEP for trigger %s and project %s " % (project, triggerid)
+ logger.info(log_message + "...")
+
+ # todo
+ if rcus is not None:
+ logger.warning("rcus option is not implemented. Ignoring that one.")
+ if stoptime is not None:
+ logger.warning(
+ "stoptime option is not implemented. Ignoring that one.") # todo: remove kwarg or make duration optional?
+
+ # convert station list to list of lcus, as the tbb service requires it
+ lcus = [stationname2hostname(station) for station in stations]
+ lcus_str = ",".join(lcus)
+
+ # convert float starttime to second and nanosecond component
+ # todo: Do we need higher precision up to here? Investigate!
+ # ...We agreed to try this out first, but it could be problematic fr use cases with extremely short recordings.
+ sec, nsec = ("%.9f" % starttime).split(".")
+ sec = int(sec)
+ nsec = int(nsec)
+
+ # determine time to wait between dump of individual subbands
+ waittime = tbbservice_config.DEFAULT_CONSTANT_TIME_BETWEEN_SUBBAND_DUMPS + \
+ tbbservice_config.DEFAULT_DURATION_FACTOR_BETWEEN_SUBBAND_DUMPS * float(duration)
+
+ datawriter_nodes = self.start_datawriters(voevent=voevent_xml)
+ if nodes is None:
+ logger.info('Nodes to use are not configured, using all with datawriter')
+ nodes = datawriter_nodes
+ else:
+ logger.info('Filtering node list for those who actually have a running datawriter')
+ nodes = [node for node in nodes if node in datawriter_nodes]
+
+ # create mapping for storage nodes
+ storage_map = create_mapping(lcus, nodes)
+ self.set_storage(storage_map)
+ self.upload_data(lcus_str, dm, starttime, duration, subbands, waittime, boards)
+
+ # restart recording
+ self.restart_recording(lcus_str)
+ logger.info(log_message + " done.")
def main():
diff --git a/MAC/Services/TBB/TBBServer/lib/tbbservice_config.py b/MAC/Services/TBB/TBBServer/lib/tbbservice_config.py
new file mode 100644
index 00000000000..1022c73c5a6
--- /dev/null
+++ b/MAC/Services/TBB/TBBServer/lib/tbbservice_config.py
@@ -0,0 +1,3 @@
+
+DEFAULT_CONSTANT_TIME_BETWEEN_SUBBAND_DUMPS = 0 # constant time in seconds to wait between subband dumps
+DEFAULT_DURATION_FACTOR_BETWEEN_SUBBAND_DUMPS = 0.00012 # linear scaling factor to increase time between dumps with duration
\ No newline at end of file
diff --git a/SAS/TriggerServices/lib/config.py b/SAS/TriggerServices/lib/config.py
index 53af0a20e53..c7bde61b6cd 100644
--- a/SAS/TriggerServices/lib/config.py
+++ b/SAS/TriggerServices/lib/config.py
@@ -42,6 +42,3 @@ DEFAULT_TBB_STATIONS = ['rs409'] # ['cs001','cs002','cs003','cs004','cs005','cs
DEFAULT_TBB_PROJECT = "COM_ALERT"
DEFAULT_TBB_ALERT_MODE = "subband"
DEFAULT_TBB_BOARDS = expand_list("0-5")
-
-DEFAULT_CONSTANT_TIME_BETWEEN_SUBBAND_DUMPS = 0 # constant time in seconds to wait between subband dumps
-DEFAULT_DURATION_FACTOR_BETWEEN_SUBBAND_DUMPS = 0.00012 # linear scaling factor to increase time between dumps with duration
\ No newline at end of file
diff --git a/SAS/TriggerServices/lib/trigger_service.py b/SAS/TriggerServices/lib/trigger_service.py
index 21918dbf6bc..1de8a24153b 100644
--- a/SAS/TriggerServices/lib/trigger_service.py
+++ b/SAS/TriggerServices/lib/trigger_service.py
@@ -238,6 +238,7 @@ def do_tbb_subband_dump(starttime, duration, dm, project, triggerid,
:param voevent_xml: the entire voevent in xml as string
:return:
"""
+ logger.warning("THIS IS DEPRECATED! User tbbservice instead!") # todo: remove this function when not needed any mre!
# We hardcode the subband mode here because we need a different procedure/parameters for regular dumps
tbbtype = "subband"
@@ -344,7 +345,10 @@ class ALERTHandler(VOEventListenerInterface):
# _send_notification('ALERT Broker', ALERT_BROKER_HOST, self.project, triggerid, voevent_xml) # todo: do we want that? do we want it on same bus?
logger.info('ALERT event %s is accepted. Initiating TBB dump: starttime %s, duration %ssec, dm %s' % (triggerid, starttime, duration, dm))
available_stations = self._determine_station_sets()['available']
- do_tbb_subband_dump(starttime, duration, dm, DEFAULT_TBB_PROJECT, triggerid, stoptime=stoptime, stations=available_stations)
+ # todo: remove the old call if the rpc service call is what we want
+ # do_tbb_subband_dump(starttime, duration, dm, DEFAULT_TBB_PROJECT, triggerid, stoptime=stoptime, stations=available_stations)
+ with TBBRPC() as rpc:
+ rpc.do_tbb_subband_dump(starttime, duration, dm, DEFAULT_TBB_PROJECT, triggerid, available_stations, DEFAULT_TBB_SUBBANDS, DEFAULT_TBB_BOARDS, DEFAULT_TBB_CEP_NODES, voevent_xml, stoptime=stoptime)
else:
raise Exception('ALERT event %s rejected by science pre-flight checks!' % triggerid)
else:
--
GitLab