Task SW-534: Put TBB util funcs in tbb_util and config in tbb_config

.gitattributes

@@ -3719,12 +3719,14 @@ MAC/TBB/bin/tbb_set_storage -text
 MAC/TBB/bin/tbb_start_recording -text
 MAC/TBB/bin/tbb_upload_to_cep -text
 MAC/TBB/lib/CMakeLists.txt -text
+MAC/TBB/lib/tbb_config.py -text
 MAC/TBB/lib/tbb_freeze.py -text
 MAC/TBB/lib/tbb_load_firmware.py -text
 MAC/TBB/lib/tbb_release_recording.py -text
 MAC/TBB/lib/tbb_set_storage.py -text
 MAC/TBB/lib/tbb_start_recording.py -text
 MAC/TBB/lib/tbb_upload_to_cep.py -text
+MAC/TBB/lib/tbb_util.py -text
 MAC/Test/APL/PVSSproject/colorDB/Lofar[!!-~]colors -text svneol=native#application/octet-stream
 MAC/Test/APL/PVSSproject/colorDB/colorDB_de -text svneol=native#application/octet-stream
 MAC/Test/APL/PVSSproject/config/config -text svneol=native#application/octet-stream

MAC/TBB/lib/CMakeLists.txt

@@ -3,6 +3,8 @@
 include(PythonInstall)
 
 python_install(
+  tbb_config.py
+  tbb_util.py
   tbb_freeze.py
   tbb_load_firmware.py
   tbb_release_recording.py

MAC/TBB/lib/tbb_config.py

+# todo: The following list is not complete, make tbb operator aware of that!
+groups_with_6boards = ['superterp',
+                       'core',
+                       'remote',
+                       'nl',
+                       'even',
+                       'odd',
+                       'today_core',
+                       'today_remote',
+                       'today_nl', ]
+
+lowest_frequency = 100.0
+highest_frequency = 200.0
+sub_band_width = 0.1953125
+number_of_samples = 1024.0
+k_DM = 4148.808
+slice_size = number_of_samples / highest_frequency
+
+tbb_command = "/opt/lofar/bin/tbbctl "
+rsp_command = "/opt/lofar/bin/rspctl "
+lcurun_command = "lcurun"
+
+supported_firmwares = ['timeseries', 'subbands']

MAC/TBB/lib/tbb_freeze.py

@@ -10,40 +10,10 @@ import argparse
 import time
 import subprocess
 import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
+from lofar.mac.tbb.tbb_util import split_stations_by_boardnumber
 
 
-def split_stations_by_boardnumber(stations):
-    """
-    Stations are assumed to have 12 boards unless their name starts with 'cs' / 'rs'
-    Some alias groups are known to have only 6 boards, but others will be assumed to have 12 boards.
-
-    :param stations: comma-separated list of stations
-    :return: dict of lists with tbb board count as key and list of stations as value
-    """
-    logging.info('Determining number of TBB boards of provided stations')
-
-    # todo: The following list is not complete, make tbb operator aware of that!
-    groups_with_6boards = ['superterp',
-                           'core',
-                           'remote',
-                           'nl',
-                           'even',
-                           'odd',
-                           'today_core',
-                           'today_remote',
-                           'today_nl', ]
-
-    def has6boards(station):
-        return station in groups_with_6boards or station[:2] in ['cs', 'rs']
-
-    stationlist = stations.split(',')
-    stationslists = {6: [x for x in stationlist if has6boards(x)],
-                     12: [x for x in stationlist if not has6boards(x)]}
-
-    stationslists = {k: v for k, v in stationslists.items() if len(v) > 0} # remove empty
-    logging.debug("Board counts: %s" % stationslists)
-    return stationslists
-
 def freeze_tbb(stations, dm, timesec, timensec):
     """
     :param stations: comma-separated list of stations
@@ -64,14 +34,14 @@ def freeze_tbb(stations, dm, timesec, timensec):
         # batch handle all stations with same number of boards through lcurun
         for num_boards in stationlists.keys():
             logging.info('Handling stations with %s boards' % num_boards)
-            relay = ['lcurun', ','.join(stationlists[num_boards])]
+            relay = [lcurun_command, ','.join(stationlists[num_boards])]
 
             slicenr = timensec / 5
 
             # remotely iterate over tbb boards because issuing all separately takes too much time
             cmds = [
-                [";".join(['/opt/lofar/bin/tbbctl --dispmeas=%s,%s' % (board, dm) for board in range(num_boards)])],
-                [";".join(['/opt/lofar/bin/tbbctl --stoptimed=%s,%s,%s' % (board, timesec, slicenr) for board in range(num_boards)])]
+                [";".join([tbb_command + '--dispmeas=%s,%s' % (board, dm) for board in range(num_boards)])],
+                [";".join([tbb_command + '--stoptimed=%s,%s,%s' % (board, timesec, slicenr) for board in range(num_boards)])]
             ]
 
             for cmd in cmds:
@@ -96,8 +66,8 @@ def freeze_tbb(stations, dm, timesec, timensec):
         logging.info('Waiting %s seconds before stopping TBB boards' % sleeptime)
         time.sleep(sleeptime)
 
-    relay = ['lcurun', stations]
-    cmd = relay + ['/opt/lofar/bin/tbbctl', '--stop']
+    relay = [lcurun_command, stations]
+    cmd = relay + [tbb_command, '--stop']
     logging.info('Executing %s' % cmd)
     subprocess.check_call(cmd)
 

MAC/TBB/lib/tbb_load_firmware.py

@@ -10,6 +10,7 @@ import argparse
 import time
 import subprocess
 import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
 
 supported_firmwares = ['timeseries', 'subbands']
 
@@ -29,8 +30,8 @@ def load_tbb_firmware(stations, firmware):
 
     logging.info("It is assumed that the %s firmware is in slot %d!" % (firmware, slot))
 
-    relay = ['lcurun', stations]
-    cmd = ['/opt/lofar/bin/tbbctl', '--config=%s' % slot]
+    relay = [lcurun_command, stations]
+    cmd = [tbb_command, '--config=%s' % slot]
     cmd = relay + cmd
     logging.info('Executing %s' % cmd)
     subprocess.check_call(cmd)
@@ -45,7 +46,7 @@ def load_tbb_firmware(stations, firmware):
     logging.info("TBBs should now have the %s firmware loaded.  Check the output of the following command!" % (firmware))
 
     for board in range(6):
-        cmd = ['/opt/lofar/bin/tbbctl', '--imageinfo=%s' % str(board)]
+        cmd = [tbb_command, '--imageinfo=%s' % str(board)]
         cmd = relay + cmd
         logging.info('Executing %s' % cmd)
         logging.info(subprocess.check_output(cmd))

MAC/TBB/lib/tbb_release_recording.py

@@ -10,13 +10,14 @@ import argparse
 import time
 import subprocess
 import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
 
 def release_tbb(stations):
     logging.info('Releasing TBB recording')
 
-    relay = ['lcurun', stations]
+    relay = [lcurun_command, stations]
 
-    cmd = relay + ['/opt/lofar/bin/tbbctl', '--free']
+    cmd = relay + [tbb_command, '--free']
     logging.info('Executing %s' % cmd)
     subprocess.check_call(cmd)
 

MAC/TBB/lib/tbb_set_storage.py

@@ -10,17 +10,17 @@ import argparse
 import time
 import subprocess
 import logging
-
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
 
 def set_tbb_storage(map):
     logging.info('Setting TBB storage nodes')
 
     for stations, node in map.iteritems():
-        relay = ['lcurun', stations]
+        relay = [lcurun_command, stations]
 
         cmds = [
-            ['/opt/lofar/bin/tbbctl', '--storage=%s' % node],
-            ['/opt/lofar/bin/tbbctl', '--udp']
+            [tbb_command, '--storage=%s' % node],
+            [tbb_command, '--udp']
         ]
 
         for cmd in cmds:

MAC/TBB/lib/tbb_start_recording.py

@@ -10,6 +10,7 @@ import argparse
 import time
 import subprocess
 import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command, rsp_command
 
 supported_modes = ['transient', 'subbands']
 
@@ -18,19 +19,19 @@ def start_tbb(stations, mode, subbands):
     logging.info('Starting TBB recording')
 
     if mode == 'subbands':
-        rspctl_mode_cmd = ['/opt/lofar/bin/rspctl', '--tbbmode=%s,%s' % (mode, subbands)]
+        rspctl_mode_cmd = [rsp_command, '--tbbmode=%s,%s' % (mode, subbands)]
     else:
-        rspctl_mode_cmd = ['/opt/lofar/bin/rspctl', '--tbbmode=%s' % (mode)]
+        rspctl_mode_cmd = [rsp_command, '--tbbmode=%s' % (mode)]
 
     cmds = [
         rspctl_mode_cmd,
-        ['/opt/lofar/bin/tbbctl', '--mode=%s' % mode],
-        ['/opt/lofar/bin/tbbctl', '--free'],
-        ['/opt/lofar/bin/tbbctl', '--alloc'],
-        ['/opt/lofar/bin/tbbctl', '--record']
+        [tbb_command, '--mode=%s' % mode],
+        [tbb_command, '--free'],
+        [tbb_command, '--alloc'],
+        [tbb_command, '--record']
     ]
 
-    relay = ['lcurun', stations]
+    relay = [lcurun_command, stations]
 
     for cmd in cmds:
         cmd = relay + cmd

MAC/TBB/lib/tbb_upload_to_cep.py

@@ -10,140 +10,10 @@ import argparse
 import time
 import subprocess
 import logging
+from lofar.mac.tbb.tbb_config import *
+from lofar.mac.tbb.tbb_util import split_stations_by_boardnumber, expand_list, calculate_adjusted_start_time
 
 
-def calculate_adjusted_start_time(dm, start_time, sub_band):
-    """
-    Calculate an adjusted start time for data in a sub-band in TBB memory.
-    Interstellar and intergalactic dispersion smear out electro-magnetic pulses
-    so that higher frequencies of a pulse will arrive earlier at a receiver
-    than lower frequencies of the same pulse.  Mostly free electrons in the path
-    of the pulse "delay" the different frequencies.
-    The three parameters, dispersion measure, recording start time of the
-    highest frequency and the target frequency as sub-band allow us to
-    calculate the recording start time for the target frequency.  The calculated
-    start time will always be >= the start time of the highest frequency.
-    :param dm:  Dispersion measure
-    :param start_time:  Start time of the entire TBB 
-    :param sub_band:  The sub-band for which the time delay shall be calculated.
-    :return:  A tuple of two values.  The first value represents the adjusted
-    start time in seconds, the second value represents the remainder of the
-    adjusted start time in units of 5 microseconds.
-    """
-    lowest_frequency = 100.0
-    highest_frequency = 200.0
-    sub_band_width = 0.1953125
-    number_of_samples = 1024.0
-    slice_size = number_of_samples / highest_frequency
-    sub_band_frequency = lowest_frequency + sub_band_width * sub_band
-
-    k_DM = 4148.808
-
-    time_delay_sub_band = k_DM * dm * (sub_band_frequency**(-2) - highest_frequency**(-2))
-    adjusted_start_time = start_time + time_delay_sub_band
-
-    # The TBB needs the adjusted start time as two parameters:  full seconds and
-    # the fractional remainder in units of 5 microseconds.
-    full_seconds = int(adjusted_start_time)
-    slice_nr = int((adjusted_start_time - full_seconds) / slice_size)
-    
-    logging.info("Calculated the sub-band start time:  "
-        "lowest frequency = %fMHz, "
-        "highest frequency = %fMHz, "
-        "sub-band width = %fMHz, "
-        "slice size = %fs, "
-        "sub-band = %d, "
-        "sub-band frequency = %fMHz, "
-        "start time of the highest frequency = %fs, "
-        "dispersion measure = %f, "
-        "calculated start time delay = %fs, "
-        "sub-band start time = %fs, "
-        "sub-band start time, seconds only = %ds, "
-        "sub-band start time, slice # = %d."
-        % (lowest_frequency, highest_frequency, sub_band_width, slice_size,
-           sub_band, sub_band_frequency, start_time, dm, time_delay_sub_band,
-           adjusted_start_time, full_seconds, slice_nr))
-    return (full_seconds, slice_nr)
-
-def split_stations_by_boardnumber(stations):
-    """
-    Stations are assumed to have 12 boards unless their name starts with 'cs' / 'rs'
-    Some alias groups are known to have only 6 boards, but others will be assumed to have 12 boards.
-
-    :param stations: comma-separated list of stations
-    :return: dict of lists with tbb board count as key and list of stations as value
-    """
-    logging.info('Determining number of TBB boards of provided stations')
-
-    # todo: The following list is not complete, make tbb operator aware of that!
-    groups_with_6boards = ['superterp',
-                           'core',
-                           'remote',
-                           'nl',
-                           'even',
-                           'odd',
-                           'today_core',
-                           'today_remote',
-                           'today_nl', ]
-
-    def has6boards(station):
-        return station in groups_with_6boards or station[:2] in ['cs', 'rs']
-
-    stationlist = stations.split(',')
-    stationslists = {6: [x for x in stationlist if has6boards(x)],
-                     12: [x for x in stationlist if not has6boards(x)]}
-    stationslists = {k: v for k, v in stationslists.items() if len(v) > 0} # remove empty
-    logging.debug("Board counts: %s" % stationslists)
-    return stationslists
-
-def expand_range(input_range):
-    """
-    Take an input string that describes an integer range "7-29" and return the individual numbers in that range as strings in a list ["7", "8", ..., "29"]. 
-    :param input_range:  An input string of the format "7-29" or "16"
-    :return:  A list of the individual integer numbers in the range as strings. 
-    """
-    items = []
-    index = input_range.find("-")
-    if index == 0 or index == (len(input_range) - 1):
-        raise ValueError("Cannot convert %s into a number!" % (input_range))
-    elif index == -1:
-        # Could be a single value.  Check if it is a digit.
-        if input_range.isdigit() is True:
-            items.append(input_range)
-        else:
-            raise ValueError("Cannot convert %s into a number!" % (input_range))
-    else:
-        range_begin = input_range[:index]
-        if range_begin.isdigit() is True:
-            range_begin = int(range_begin)
-        else:
-            raise ValueError("Cannot convert %s into a number!" % (range_begin))
-        range_end = input_range[(index + 1):]
-        if range_end.isdigit() is True:
-            range_end = int(range_end)
-        else:
-            raise ValueError("Cannot convert %s into a number!" % (range_end))
-        for element in range(range_begin, range_end + 1):
-            items.append(str(element))
-    return items
-
-def expand_list(input_list):
-    """
-    Convert an input string of the form "1,17-33,64,1299-1344" to a list of strings that consists of each individual integer that is given and of all integers in every range.
-    :param input_list:  An input string containing integer ranges "7-29" or individual integers "33" all separated by ",".
-    :return:  A list of all the individual integer numbers and of all the integers in the ranges.  The list elements are strings.
-    """
-    items = []
-    split_by_comma = input_list.split(",")
-    for item in split_by_comma:
-        if item.isdigit() is True:
-            items.append(item)
-        else:
-            items += expand_range(item)
-
-    logging.debug("Input list: \"%s\"\nFound these elements in the list: \"%s\"" % (input_list, items))
-    return items
-
 def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, boards):
     """
     Set-up and execute tbbctl READBAND commands that will command TBBs in ALERT mode to upload part of their memory to CEP.
@@ -160,7 +30,7 @@ def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, bo
     # determine number of tbb boards per station:
     stationlists = split_stations_by_boardnumber(stations)
 
-    tbb_cmd = "/opt/lofar/bin/tbbctl --readband=${board},${channel},%s,%s,%s,%s"
+    tbb_cmd = tbb_command + " --readband=${board},${channel},%s,%s,%s,%s"
     # I have to wait until a sub-band has been uploaded.  But I can
     # upload multiple boards/channels in parallel.  Hence this can
     # parallelised down to the sub-bands.
@@ -170,7 +40,7 @@ def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, bo
         # batch handle all stations with same number of boards through lcurun
         for num_boards in stationlists.keys():
             logging.debug("Creating TBB commands for stations with %s boards..." % num_boards)
-            relay = ["lcurun", ",".join(stationlists[num_boards])]
+            relay = [lcurun_command, ",".join(stationlists[num_boards])]
 
             # iterate over tbb boards
             board_list = ""

MAC/TBB/lib/tbb_util.py

+import logging
+from lofar.mac.tbb.tbb_config import *
+
+
+def split_stations_by_boardnumber(stations):
+    """
+    Stations are assumed to have 12 boards unless their name starts with 'cs' / 'rs'
+    Some alias groups are known to have only 6 boards, but others will be assumed to have 12 boards.
+
+    :param stations: comma-separated list of stations
+    :return: dict of lists with tbb board count as key and list of stations as value
+    """
+    logging.info('Determining number of TBB boards of provided stations')
+
+    def has6boards(station):
+        return station in groups_with_6boards or station[:2] in ['cs', 'rs']
+
+    stationlist = stations.split(',')
+    stationslists = {6: [x for x in stationlist if has6boards(x)],
+                     12: [x for x in stationlist if not has6boards(x)]}
+
+    stationslists = {k: v for k, v in stationslists.items() if len(v) > 0} # remove empty
+    logging.debug("Board counts: %s" % stationslists)
+    return stationslists
+
+def expand_range(input_range):
+    """
+    Take an input string that describes an integer range "7-29" and return the individual numbers in that range as strings in a list ["7", "8", ..., "29"]. 
+    :param input_range:  An input string of the format "7-29" or "16"
+    :return:  A list of the individual integer numbers in the range as strings. 
+    """
+    items = []
+    index = input_range.find("-")
+    if index == 0 or index == (len(input_range) - 1):
+        raise ValueError("Cannot convert %s into a number!" % (input_range))
+    elif index == -1:
+        # Could be a single value.  Check if it is a digit.
+        if input_range.isdigit() is True:
+            items.append(input_range)
+        else:
+            raise ValueError("Cannot convert %s into a number!" % (input_range))
+    else:
+        range_begin = input_range[:index]
+        if range_begin.isdigit() is True:
+            range_begin = int(range_begin)
+        else:
+            raise ValueError("Cannot convert %s into a number!" % (range_begin))
+        range_end = input_range[(index + 1):]
+        if range_end.isdigit() is True:
+            range_end = int(range_end)
+        else:
+            raise ValueError("Cannot convert %s into a number!" % (range_end))
+        for element in range(range_begin, range_end + 1):
+            items.append(str(element))
+    return items
+
+def expand_list(input_list):
+    """
+    Convert an input string of the form "1,17-33,64,1299-1344" to a list of strings that consists of each individual integer that is given and of all integers in every range.
+    :param input_list:  An input string containing integer ranges "7-29" or individual integers "33" all separated by ",".
+    :return:  A list of all the individual integer numbers and of all the integers in the ranges.  The list elements are strings.
+    """
+    items = []
+    split_by_comma = input_list.split(",")
+    for item in split_by_comma:
+        if item.isdigit() is True:
+            items.append(item)
+        else:
+            items += expand_range(item)
+
+    logging.debug("Input list: \"%s\"\nFound these elements in the list: \"%s\"" % (input_list, items))
+    return items
+
+def calculate_adjusted_start_time(dm, start_time, sub_band):
+    """
+    Calculate an adjusted start time for data in a sub-band in TBB memory.
+    Interstellar and intergalactic dispersion smear out electro-magnetic pulses
+    so that higher frequencies of a pulse will arrive earlier at a receiver
+    than lower frequencies of the same pulse.  Mostly free electrons in the path
+    of the pulse "delay" the different frequencies.
+    The three parameters, dispersion measure, recording start time of the
+    highest frequency and the target frequency as sub-band allow us to
+    calculate the recording start time for the target frequency.  The calculated
+    start time will always be >= the start time of the highest frequency.
+    :param dm:  Dispersion measure
+    :param start_time:  Start time of the entire TBB 
+    :param sub_band:  The sub-band for which the time delay shall be calculated.
+    :return:  A tuple of two values.  The first value represents the adjusted
+    start time in seconds, the second value represents the remainder of the
+    adjusted start time in units of 5 microseconds.
+    """
+    sub_band_frequency = lowest_frequency + sub_band_width * sub_band
+    time_delay_sub_band = k_DM * dm * (sub_band_frequency**(-2) - highest_frequency**(-2))
+    adjusted_start_time = start_time + time_delay_sub_band
+
+    # The TBB needs the adjusted start time as two parameters:  full seconds and
+    # the fractional remainder in units of 5 microseconds.
+    full_seconds = int(adjusted_start_time)
+    slice_nr = int((adjusted_start_time - full_seconds) / slice_size)
+    
+    logging.info("Calculated the sub-band start time:  "
+        "lowest frequency = %fMHz, "
+        "highest frequency = %fMHz, "
+        "sub-band width = %fMHz, "
+        "slice size = %fs, "
+        "sub-band = %d, "
+        "sub-band frequency = %fMHz, "
+        "start time of the highest frequency = %fs, "
+        "dispersion measure = %f, "
+        "calculated start time delay = %fs, "
+        "sub-band start time = %fs, "
+        "sub-band start time, seconds only = %ds, "
+        "sub-band start time, slice # = %d."
+        % (lowest_frequency, highest_frequency, sub_band_width, slice_size,
+           sub_band, sub_band_frequency, start_time, dm, time_delay_sub_band,
+           adjusted_start_time, full_seconds, slice_nr))
+    return (full_seconds, slice_nr)