Task SW-560: Moved trigger recipe to tbbservice.

.gitattributes

@@ -3694,6 +3694,7 @@ MAC/Services/TBB/TBBServer/bin/tbbservice -text
 MAC/Services/TBB/TBBServer/bin/tbbservice.ini -text
 MAC/Services/TBB/TBBServer/lib/CMakeLists.txt -text
 MAC/Services/TBB/TBBServer/lib/tbbservice.py -text
+MAC/Services/TBB/TBBServer/lib/tbbservice_config.py -text
 MAC/Services/TBB/TBBServer/test/CMakeLists.txt -text
 MAC/Services/TBB/TBBServer/test/t_tbbserver.py -text
 MAC/Services/TBB/TBBServer/test/t_tbbserver.run -text

CMake/LofarPackageList.cmake

 # - Create for each LOFAR package a variable containing the absolute path to
 # its source directory. 
 #
-# Generated by gen_LofarPackageList_cmake.sh at vr 18 jan 2019 15:16:58 CET
+# Generated by gen_LofarPackageList_cmake.sh at Do 24. Jan 11:12:48 CET 2019
 #
 #                      ---- DO NOT EDIT ----
 #

MAC/Services/TBB/TBBClient/lib/tbbservice_rpc.py

@@ -68,6 +68,12 @@ class TBBRPC(RPCWrapper):
         logger.info("Received set storage request result %s" % result)
         return result
 
+    def do_tbb_subband_dump(self, starttime, duration, dm, project, triggerid, stations, subbands, boards, nodes, voevent_xml, stoptime=None, rcus=None):
+        logger.info("Requesting full tbb dump to CEP for trigger %s and project %s" % (triggerid, project))
+        result = self.rpc('do_tbb_subband_dump', starttime, duration, dm, project, triggerid, stations, subbands, boards, nodes, voevent_xml, stoptime=stoptime, rcus=rcus)
+        logger.info("Received full dump to CEP result for trigger %s and project %s: %s" % (triggerid, project, result))
+        return result
+
 if __name__ == '__main__':
     '''little example usage'''
     logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',

MAC/Services/TBB/TBBServer/lib/CMakeLists.txt

@@ -2,4 +2,5 @@
 include(PythonInstall)
 
 python_install(tbbservice.py
+               tbbservice_config.py
                DESTINATION lofar/mac/tbbservice/server)

MAC/Services/TBB/TBBServer/lib/tbbservice.py

@@ -43,11 +43,13 @@ from lofar.mac.tbb.tbb_load_firmware import load_tbb_firmware
 from lofar.mac.tbb.tbb_freeze import freeze_tbb
 from lofar.mac.tbb.tbb_release_recording import release_tbb
 from lofar.mac.tbb.tbb_restart_recording import restart_tbb_recording
-from lofar.mac.tbb.tbb_set_storage import set_tbb_storage
+from lofar.mac.tbb.tbb_set_storage import set_tbb_storage, create_mapping
 from lofar.mac.tbb.tbb_start_recording import start_tbb
 from lofar.mac.tbb.tbb_upload_to_cep import upload_tbb_data
 from lofar.parameterset import parameterset
 from lofar.mac.tbb.tbb_util import parse_parset_from_voevent
+from lofar.common.lcu_utils import stationname2hostname, hostname2stationname
+from . import tbbservice_config
 
 class TBBServiceMessageHandler(MessageHandlerInterface):
     '''
@@ -74,7 +76,9 @@ class TBBServiceMessageHandler(MessageHandlerInterface):
                                   'restart_recording': self._tbb_control_service.restart_recording,
                                   'upload_data': self._tbb_control_service.upload_data,
                                   'freeze_data': self._tbb_control_service.freeze_data,
-                                  'set_storage': self._tbb_control_service.set_storage}
+                                  'set_storage': self._tbb_control_service.set_storage,
+                                  'do_tbb_subband_dump': self._tbb_control_service.do_tbb_subband_dump
+                                  }
 
 
 class TBBControlService:
@@ -336,7 +340,7 @@ class TBBControlService:
         log_message = "Switching TBB firmware on stations %s to \"%s\"" % (stations, mode)
         logger.info(log_message + "...")
         load_tbb_firmware(stations, mode)
-        logger.info(log_message + "done.")
+        logger.info(log_message + " done.")
 
     def start_recording(self, stations, mode, subbands):
         """
@@ -348,7 +352,7 @@ class TBBControlService:
         log_message = "Starting the TBB data recording for mode %s on stations %s in sub-bands %s" % (mode, stations, subbands)
         logger.info(log_message + "...")
         start_tbb(stations, mode, subbands)
-        logger.info(log_message + "done.")
+        logger.info(log_message + " done.")
 
     def freeze_data(self, stations, dm, timesec, timensec):
         """
@@ -384,7 +388,7 @@ class TBBControlService:
                       "wait time = %f" % (stations, boards, sub_bands, dm, start_time, duration, wait_time)
         logger.info(log_message + "...")
         upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, boards)
-        logger.info(log_message + "done.")
+        logger.info(log_message + " done.")
 
     def release_data(self, stations):
         """
@@ -394,7 +398,7 @@ class TBBControlService:
         log_message = "Releasing the TBB data recording on stations %s" % (stations)
         logger.info(log_message + "...")
         release_tbb(stations)
-        logger.info(log_message + "done.")
+        logger.info(log_message + " done.")
 
     def restart_recording(self, stations):
         """
@@ -404,7 +408,7 @@ class TBBControlService:
         log_message = "Restarting the TBB data recording on stations %s" % (stations)
         logger.info(log_message + "...")
         restart_tbb_recording(stations)
-        logger.info(log_message + "done.")
+        logger.info(log_message + " done.")
 
     def set_storage(self, map):
         """
@@ -414,7 +418,74 @@ class TBBControlService:
         log_message = "Setting the storage nodes where TBB data is received to: %s" % (map)
         logger.info(log_message + "...")
         set_tbb_storage(map)
-        logger.info(log_message + "done.")
+        logger.info(log_message + " done.")
+
+    def do_tbb_subband_dump(self, starttime, duration, dm, project, triggerid,
+                            stations, subbands, boards, nodes, voevent_xml,
+                            stoptime=None, rcus=None):
+        """
+        This 'recipe' call freezes data on the boards and performs the tbbdump.
+        The first data of the dump is recorded at starttime in the subband of highest frequency. Subbands of lower frequency
+        are then delayed by an offset calculated from the provided dm, and each subband holds data of the provided duration.
+        Hence, the starttime should predate the expected time of arrival of the event at the frequency of the highest
+        subband by half the recording duration to center the event in the recorded data for all subbands.
+
+        :param starttime: The starttime of the recording
+        :param duration: The duration of the recording in seconds as float
+        :param dm: The dispersion measure (in pc/cm^3) of the event to capture as float
+        :param project: The project identifier as string
+        :param triggerid: The trigger identifier as string
+        :param stoptime: The stoptime in seconds since Epoch as float
+        :param stations: The stations to use as list of strings
+        :param subbands: The subbands to use as list of integers
+        :param rcus: The receivers to use as list of integers
+        :param boards: The tbb boards to use as list of integers
+        :param nodes: The CEP nodes to send data to
+        :param voevent_xml: the entire voevent in xml as string
+        :return:
+        """
+
+        log_message = "Performing TBB data dump to CEP for trigger %s and project %s " % (project, triggerid)
+        logger.info(log_message + "...")
+
+        # todo
+        if rcus is not None:
+            logger.warning("rcus option is not implemented. Ignoring that one.")
+        if stoptime is not None:
+            logger.warning(
+                "stoptime option is not implemented. Ignoring that one.")  # todo: remove kwarg or make duration optional?
+
+        # convert station list to list of lcus, as the tbb service requires it
+        lcus = [stationname2hostname(station) for station in stations]
+        lcus_str = ",".join(lcus)
+
+        # convert float starttime to second and nanosecond component
+        # todo: Do we need higher precision up to here? Investigate!
+        #    ...We agreed to try this out first, but it could be problematic fr use cases with extremely short recordings.
+        sec, nsec = ("%.9f" % starttime).split(".")
+        sec = int(sec)
+        nsec = int(nsec)
+
+        # determine time to wait between dump of individual subbands
+        waittime = tbbservice_config.DEFAULT_CONSTANT_TIME_BETWEEN_SUBBAND_DUMPS + \
+                   tbbservice_config.DEFAULT_DURATION_FACTOR_BETWEEN_SUBBAND_DUMPS * float(duration)
+
+        datawriter_nodes = self.start_datawriters(voevent=voevent_xml)
+        if nodes is None:
+            logger.info('Nodes to use are not configured, using all with datawriter')
+            nodes = datawriter_nodes
+        else:
+            logger.info('Filtering node list for those who actually have a running datawriter')
+            nodes = [node for node in nodes if node in datawriter_nodes]
+
+        # create mapping for storage nodes
+        storage_map = create_mapping(lcus, nodes)
+        self.set_storage(storage_map)
+        self.upload_data(lcus_str, dm, starttime, duration, subbands, waittime, boards)
+
+        # restart recording
+        self.restart_recording(lcus_str)
+        logger.info(log_message + " done.")
 
 
 def main():

MAC/Services/TBB/TBBServer/lib/tbbservice_config.py

+
+DEFAULT_CONSTANT_TIME_BETWEEN_SUBBAND_DUMPS = 0          # constant time in seconds to wait between subband dumps
+DEFAULT_DURATION_FACTOR_BETWEEN_SUBBAND_DUMPS = 0.00012  # linear scaling factor to increase time between dumps with duration
\ No newline at end of file

SAS/TriggerServices/lib/config.py

@@ -42,6 +42,3 @@ DEFAULT_TBB_STATIONS = ['rs409']  # ['cs001','cs002','cs003','cs004','cs005','cs
 DEFAULT_TBB_PROJECT = "COM_ALERT"
 DEFAULT_TBB_ALERT_MODE = "subband"
 DEFAULT_TBB_BOARDS = expand_list("0-5")
-
-DEFAULT_CONSTANT_TIME_BETWEEN_SUBBAND_DUMPS = 0          # constant time in seconds to wait between subband dumps
-DEFAULT_DURATION_FACTOR_BETWEEN_SUBBAND_DUMPS = 0.00012  # linear scaling factor to increase time between dumps with duration
\ No newline at end of file

SAS/TriggerServices/lib/trigger_service.py

@@ -238,6 +238,7 @@ def do_tbb_subband_dump(starttime, duration, dm, project, triggerid,
     :param voevent_xml: the entire voevent in xml as string
     :return:
     """
+    logger.warning("THIS IS DEPRECATED! User tbbservice instead!")  # todo: remove this function when not needed any mre!
 
     # We hardcode the subband mode here because we need a different procedure/parameters for regular dumps
     tbbtype = "subband"
@@ -344,7 +345,10 @@ class ALERTHandler(VOEventListenerInterface):
                     # _send_notification('ALERT Broker', ALERT_BROKER_HOST, self.project, triggerid, voevent_xml)  # todo: do we want that? do we want it on same bus?
                     logger.info('ALERT event %s is accepted. Initiating TBB dump: starttime %s, duration %ssec, dm %s' % (triggerid, starttime, duration, dm))
                     available_stations = self._determine_station_sets()['available']
-                    do_tbb_subband_dump(starttime, duration, dm, DEFAULT_TBB_PROJECT, triggerid, stoptime=stoptime, stations=available_stations)
+                    # todo: remove the old call if the rpc service call is what we want
+                    # do_tbb_subband_dump(starttime, duration, dm, DEFAULT_TBB_PROJECT, triggerid, stoptime=stoptime, stations=available_stations)
+                    with TBBRPC() as rpc:
+                        rpc.do_tbb_subband_dump(starttime, duration, dm, DEFAULT_TBB_PROJECT, triggerid, available_stations, DEFAULT_TBB_SUBBANDS, DEFAULT_TBB_BOARDS, DEFAULT_TBB_CEP_NODES, voevent_xml, stoptime=stoptime)
                 else:
                     raise Exception('ALERT event %s rejected by science pre-flight checks!' % triggerid)
             else: