From ccceed64ee5a90471f9e33eb1745dacc48e488a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20J=C3=BCrges?= <jurges@astron.nl>
Date: Wed, 21 Nov 2018 15:13:47 +0000
Subject: [PATCH] Task SW-534: Put TBB util funcs in tbb_util and config in
tbb_config
---
.gitattributes | 2 +
MAC/TBB/lib/CMakeLists.txt | 2 +
MAC/TBB/lib/tbb_config.py | 23 +++++
MAC/TBB/lib/tbb_freeze.py | 44 ++-------
MAC/TBB/lib/tbb_load_firmware.py | 7 +-
MAC/TBB/lib/tbb_release_recording.py | 5 +-
MAC/TBB/lib/tbb_set_storage.py | 8 +-
MAC/TBB/lib/tbb_start_recording.py | 15 +--
MAC/TBB/lib/tbb_upload_to_cep.py | 138 +--------------------------
MAC/TBB/lib/tbb_util.py | 117 +++++++++++++++++++++++
10 files changed, 174 insertions(+), 187 deletions(-)
create mode 100644 MAC/TBB/lib/tbb_config.py
create mode 100644 MAC/TBB/lib/tbb_util.py
diff --git a/.gitattributes b/.gitattributes
index f4564edf956..10e472ee3c5 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3719,12 +3719,14 @@ MAC/TBB/bin/tbb_set_storage -text
MAC/TBB/bin/tbb_start_recording -text
MAC/TBB/bin/tbb_upload_to_cep -text
MAC/TBB/lib/CMakeLists.txt -text
+MAC/TBB/lib/tbb_config.py -text
MAC/TBB/lib/tbb_freeze.py -text
MAC/TBB/lib/tbb_load_firmware.py -text
MAC/TBB/lib/tbb_release_recording.py -text
MAC/TBB/lib/tbb_set_storage.py -text
MAC/TBB/lib/tbb_start_recording.py -text
MAC/TBB/lib/tbb_upload_to_cep.py -text
+MAC/TBB/lib/tbb_util.py -text
MAC/Test/APL/PVSSproject/colorDB/Lofar[!!-~]colors -text svneol=native#application/octet-stream
MAC/Test/APL/PVSSproject/colorDB/colorDB_de -text svneol=native#application/octet-stream
MAC/Test/APL/PVSSproject/config/config -text svneol=native#application/octet-stream
diff --git a/MAC/TBB/lib/CMakeLists.txt b/MAC/TBB/lib/CMakeLists.txt
index e24249baf3d..03e7806f788 100644
--- a/MAC/TBB/lib/CMakeLists.txt
+++ b/MAC/TBB/lib/CMakeLists.txt
@@ -3,6 +3,8 @@
include(PythonInstall)
python_install(
+ tbb_config.py
+ tbb_util.py
tbb_freeze.py
tbb_load_firmware.py
tbb_release_recording.py
diff --git a/MAC/TBB/lib/tbb_config.py b/MAC/TBB/lib/tbb_config.py
new file mode 100644
index 00000000000..22b07106c50
--- /dev/null
+++ b/MAC/TBB/lib/tbb_config.py
@@ -0,0 +1,23 @@
+# todo: The following list is not complete, make tbb operator aware of that!
+groups_with_6boards = ['superterp',
+ 'core',
+ 'remote',
+ 'nl',
+ 'even',
+ 'odd',
+ 'today_core',
+ 'today_remote',
+ 'today_nl', ]
+
+lowest_frequency = 100.0
+highest_frequency = 200.0
+sub_band_width = 0.1953125
+number_of_samples = 1024.0
+k_DM = 4148.808
+slice_size = number_of_samples / highest_frequency
+
+tbb_command = "/opt/lofar/bin/tbbctl "
+rsp_command = "/opt/lofar/bin/rspctl "
+lcurun_command = "lcurun"
+
+supported_firmwares = ['timeseries', 'subbands']
diff --git a/MAC/TBB/lib/tbb_freeze.py b/MAC/TBB/lib/tbb_freeze.py
index da5e7697c3b..e3b26f22b69 100755
--- a/MAC/TBB/lib/tbb_freeze.py
+++ b/MAC/TBB/lib/tbb_freeze.py
@@ -10,40 +10,10 @@ import argparse
import time
import subprocess
import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
+from lofar.mac.tbb.tbb_util import split_stations_by_boardnumber
-def split_stations_by_boardnumber(stations):
- """
- Stations are assumed to have 12 boards unless their name starts with 'cs' / 'rs'
- Some alias groups are known to have only 6 boards, but others will be assumed to have 12 boards.
-
- :param stations: comma-separated list of stations
- :return: dict of lists with tbb board count as key and list of stations as value
- """
- logging.info('Determining number of TBB boards of provided stations')
-
- # todo: The following list is not complete, make tbb operator aware of that!
- groups_with_6boards = ['superterp',
- 'core',
- 'remote',
- 'nl',
- 'even',
- 'odd',
- 'today_core',
- 'today_remote',
- 'today_nl', ]
-
- def has6boards(station):
- return station in groups_with_6boards or station[:2] in ['cs', 'rs']
-
- stationlist = stations.split(',')
- stationslists = {6: [x for x in stationlist if has6boards(x)],
- 12: [x for x in stationlist if not has6boards(x)]}
-
- stationslists = {k: v for k, v in stationslists.items() if len(v) > 0} # remove empty
- logging.debug("Board counts: %s" % stationslists)
- return stationslists
-
def freeze_tbb(stations, dm, timesec, timensec):
"""
:param stations: comma-separated list of stations
@@ -64,14 +34,14 @@ def freeze_tbb(stations, dm, timesec, timensec):
# batch handle all stations with same number of boards through lcurun
for num_boards in stationlists.keys():
logging.info('Handling stations with %s boards' % num_boards)
- relay = ['lcurun', ','.join(stationlists[num_boards])]
+ relay = [lcurun_command, ','.join(stationlists[num_boards])]
slicenr = timensec / 5
# remotely iterate over tbb boards because issuing all separately takes too much time
cmds = [
- [";".join(['/opt/lofar/bin/tbbctl --dispmeas=%s,%s' % (board, dm) for board in range(num_boards)])],
- [";".join(['/opt/lofar/bin/tbbctl --stoptimed=%s,%s,%s' % (board, timesec, slicenr) for board in range(num_boards)])]
+ [";".join([tbb_command + '--dispmeas=%s,%s' % (board, dm) for board in range(num_boards)])],
+ [";".join([tbb_command + '--stoptimed=%s,%s,%s' % (board, timesec, slicenr) for board in range(num_boards)])]
]
for cmd in cmds:
@@ -96,8 +66,8 @@ def freeze_tbb(stations, dm, timesec, timensec):
logging.info('Waiting %s seconds before stopping TBB boards' % sleeptime)
time.sleep(sleeptime)
- relay = ['lcurun', stations]
- cmd = relay + ['/opt/lofar/bin/tbbctl', '--stop']
+ relay = [lcurun_command, stations]
+ cmd = relay + [tbb_command, '--stop']
logging.info('Executing %s' % cmd)
subprocess.check_call(cmd)
diff --git a/MAC/TBB/lib/tbb_load_firmware.py b/MAC/TBB/lib/tbb_load_firmware.py
index 9149245bfa6..4ffcafd46d6 100755
--- a/MAC/TBB/lib/tbb_load_firmware.py
+++ b/MAC/TBB/lib/tbb_load_firmware.py
@@ -10,6 +10,7 @@ import argparse
import time
import subprocess
import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
supported_firmwares = ['timeseries', 'subbands']
@@ -29,8 +30,8 @@ def load_tbb_firmware(stations, firmware):
logging.info("It is assumed that the %s firmware is in slot %d!" % (firmware, slot))
- relay = ['lcurun', stations]
- cmd = ['/opt/lofar/bin/tbbctl', '--config=%s' % slot]
+ relay = [lcurun_command, stations]
+ cmd = [tbb_command, '--config=%s' % slot]
cmd = relay + cmd
logging.info('Executing %s' % cmd)
subprocess.check_call(cmd)
@@ -45,7 +46,7 @@ def load_tbb_firmware(stations, firmware):
logging.info("TBBs should now have the %s firmware loaded. Check the output of the following command!" % (firmware))
for board in range(6):
- cmd = ['/opt/lofar/bin/tbbctl', '--imageinfo=%s' % str(board)]
+ cmd = [tbb_command, '--imageinfo=%s' % str(board)]
cmd = relay + cmd
logging.info('Executing %s' % cmd)
logging.info(subprocess.check_output(cmd))
diff --git a/MAC/TBB/lib/tbb_release_recording.py b/MAC/TBB/lib/tbb_release_recording.py
index 82b6171cb15..bcac09f7baa 100755
--- a/MAC/TBB/lib/tbb_release_recording.py
+++ b/MAC/TBB/lib/tbb_release_recording.py
@@ -10,13 +10,14 @@ import argparse
import time
import subprocess
import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
def release_tbb(stations):
logging.info('Releasing TBB recording')
- relay = ['lcurun', stations]
+ relay = [lcurun_command, stations]
- cmd = relay + ['/opt/lofar/bin/tbbctl', '--free']
+ cmd = relay + [tbb_command, '--free']
logging.info('Executing %s' % cmd)
subprocess.check_call(cmd)
diff --git a/MAC/TBB/lib/tbb_set_storage.py b/MAC/TBB/lib/tbb_set_storage.py
index a300a600901..a3367e7d601 100755
--- a/MAC/TBB/lib/tbb_set_storage.py
+++ b/MAC/TBB/lib/tbb_set_storage.py
@@ -10,17 +10,17 @@ import argparse
import time
import subprocess
import logging
-
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command
def set_tbb_storage(map):
logging.info('Setting TBB storage nodes')
for stations, node in map.iteritems():
- relay = ['lcurun', stations]
+ relay = [lcurun_command, stations]
cmds = [
- ['/opt/lofar/bin/tbbctl', '--storage=%s' % node],
- ['/opt/lofar/bin/tbbctl', '--udp']
+ [tbb_command, '--storage=%s' % node],
+ [tbb_command, '--udp']
]
for cmd in cmds:
diff --git a/MAC/TBB/lib/tbb_start_recording.py b/MAC/TBB/lib/tbb_start_recording.py
index 8e3cd1b29e1..23651347b34 100755
--- a/MAC/TBB/lib/tbb_start_recording.py
+++ b/MAC/TBB/lib/tbb_start_recording.py
@@ -10,6 +10,7 @@ import argparse
import time
import subprocess
import logging
+from lofar.mac.tbb.tbb_config import lcurun_command, tbb_command, rsp_command
supported_modes = ['transient', 'subbands']
@@ -18,19 +19,19 @@ def start_tbb(stations, mode, subbands):
logging.info('Starting TBB recording')
if mode == 'subbands':
- rspctl_mode_cmd = ['/opt/lofar/bin/rspctl', '--tbbmode=%s,%s' % (mode, subbands)]
+ rspctl_mode_cmd = [rsp_command, '--tbbmode=%s,%s' % (mode, subbands)]
else:
- rspctl_mode_cmd = ['/opt/lofar/bin/rspctl', '--tbbmode=%s' % (mode)]
+ rspctl_mode_cmd = [rsp_command, '--tbbmode=%s' % (mode)]
cmds = [
rspctl_mode_cmd,
- ['/opt/lofar/bin/tbbctl', '--mode=%s' % mode],
- ['/opt/lofar/bin/tbbctl', '--free'],
- ['/opt/lofar/bin/tbbctl', '--alloc'],
- ['/opt/lofar/bin/tbbctl', '--record']
+ [tbb_command, '--mode=%s' % mode],
+ [tbb_command, '--free'],
+ [tbb_command, '--alloc'],
+ [tbb_command, '--record']
]
- relay = ['lcurun', stations]
+ relay = [lcurun_command, stations]
for cmd in cmds:
cmd = relay + cmd
diff --git a/MAC/TBB/lib/tbb_upload_to_cep.py b/MAC/TBB/lib/tbb_upload_to_cep.py
index 014ccddcb21..a4ea171611f 100755
--- a/MAC/TBB/lib/tbb_upload_to_cep.py
+++ b/MAC/TBB/lib/tbb_upload_to_cep.py
@@ -10,140 +10,10 @@ import argparse
import time
import subprocess
import logging
+from lofar.mac.tbb.tbb_config import *
+from lofar.mac.tbb.tbb_util import split_stations_by_boardnumber, expand_list, calculate_adjusted_start_time
-def calculate_adjusted_start_time(dm, start_time, sub_band):
- """
- Calculate an adjusted start time for data in a sub-band in TBB memory.
- Interstellar and intergalactic dispersion smear out electro-magnetic pulses
- so that higher frequencies of a pulse will arrive earlier at a receiver
- than lower frequencies of the same pulse. Mostly free electrons in the path
- of the pulse "delay" the different frequencies.
- The three parameters, dispersion measure, recording start time of the
- highest frequency and the target frequency as sub-band allow us to
- calculate the recording start time for the target frequency. The calculated
- start time will always be >= the start time of the highest frequency.
- :param dm: Dispersion measure
- :param start_time: Start time of the entire TBB
- :param sub_band: The sub-band for which the time delay shall be calculated.
- :return: A tuple of two values. The first value represents the adjusted
- start time in seconds, the second value represents the remainder of the
- adjusted start time in units of 5 microseconds.
- """
- lowest_frequency = 100.0
- highest_frequency = 200.0
- sub_band_width = 0.1953125
- number_of_samples = 1024.0
- slice_size = number_of_samples / highest_frequency
- sub_band_frequency = lowest_frequency + sub_band_width * sub_band
-
- k_DM = 4148.808
-
- time_delay_sub_band = k_DM * dm * (sub_band_frequency**(-2) - highest_frequency**(-2))
- adjusted_start_time = start_time + time_delay_sub_band
-
- # The TBB needs the adjusted start time as two parameters: full seconds and
- # the fractional remainder in units of 5 microseconds.
- full_seconds = int(adjusted_start_time)
- slice_nr = int((adjusted_start_time - full_seconds) / slice_size)
-
- logging.info("Calculated the sub-band start time: "
- "lowest frequency = %fMHz, "
- "highest frequency = %fMHz, "
- "sub-band width = %fMHz, "
- "slice size = %fs, "
- "sub-band = %d, "
- "sub-band frequency = %fMHz, "
- "start time of the highest frequency = %fs, "
- "dispersion measure = %f, "
- "calculated start time delay = %fs, "
- "sub-band start time = %fs, "
- "sub-band start time, seconds only = %ds, "
- "sub-band start time, slice # = %d."
- % (lowest_frequency, highest_frequency, sub_band_width, slice_size,
- sub_band, sub_band_frequency, start_time, dm, time_delay_sub_band,
- adjusted_start_time, full_seconds, slice_nr))
- return (full_seconds, slice_nr)
-
-def split_stations_by_boardnumber(stations):
- """
- Stations are assumed to have 12 boards unless their name starts with 'cs' / 'rs'
- Some alias groups are known to have only 6 boards, but others will be assumed to have 12 boards.
-
- :param stations: comma-separated list of stations
- :return: dict of lists with tbb board count as key and list of stations as value
- """
- logging.info('Determining number of TBB boards of provided stations')
-
- # todo: The following list is not complete, make tbb operator aware of that!
- groups_with_6boards = ['superterp',
- 'core',
- 'remote',
- 'nl',
- 'even',
- 'odd',
- 'today_core',
- 'today_remote',
- 'today_nl', ]
-
- def has6boards(station):
- return station in groups_with_6boards or station[:2] in ['cs', 'rs']
-
- stationlist = stations.split(',')
- stationslists = {6: [x for x in stationlist if has6boards(x)],
- 12: [x for x in stationlist if not has6boards(x)]}
- stationslists = {k: v for k, v in stationslists.items() if len(v) > 0} # remove empty
- logging.debug("Board counts: %s" % stationslists)
- return stationslists
-
-def expand_range(input_range):
- """
- Take an input string that describes an integer range "7-29" and return the individual numbers in that range as strings in a list ["7", "8", ..., "29"].
- :param input_range: An input string of the format "7-29" or "16"
- :return: A list of the individual integer numbers in the range as strings.
- """
- items = []
- index = input_range.find("-")
- if index == 0 or index == (len(input_range) - 1):
- raise ValueError("Cannot convert %s into a number!" % (input_range))
- elif index == -1:
- # Could be a single value. Check if it is a digit.
- if input_range.isdigit() is True:
- items.append(input_range)
- else:
- raise ValueError("Cannot convert %s into a number!" % (input_range))
- else:
- range_begin = input_range[:index]
- if range_begin.isdigit() is True:
- range_begin = int(range_begin)
- else:
- raise ValueError("Cannot convert %s into a number!" % (range_begin))
- range_end = input_range[(index + 1):]
- if range_end.isdigit() is True:
- range_end = int(range_end)
- else:
- raise ValueError("Cannot convert %s into a number!" % (range_end))
- for element in range(range_begin, range_end + 1):
- items.append(str(element))
- return items
-
-def expand_list(input_list):
- """
- Convert an input string of the form "1,17-33,64,1299-1344" to a list of strings that consists of each individual integer that is given and of all integers in every range.
- :param input_list: An input string containing integer ranges "7-29" or individual integers "33" all separated by ",".
- :return: A list of all the individual integer numbers and of all the integers in the ranges. The list elements are strings.
- """
- items = []
- split_by_comma = input_list.split(",")
- for item in split_by_comma:
- if item.isdigit() is True:
- items.append(item)
- else:
- items += expand_range(item)
-
- logging.debug("Input list: \"%s\"\nFound these elements in the list: \"%s\"" % (input_list, items))
- return items
-
def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, boards):
"""
Set-up and execute tbbctl READBAND commands that will command TBBs in ALERT mode to upload part of their memory to CEP.
@@ -160,7 +30,7 @@ def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, bo
# determine number of tbb boards per station:
stationlists = split_stations_by_boardnumber(stations)
- tbb_cmd = "/opt/lofar/bin/tbbctl --readband=${board},${channel},%s,%s,%s,%s"
+ tbb_cmd = tbb_command + " --readband=${board},${channel},%s,%s,%s,%s"
# I have to wait until a sub-band has been uploaded. But I can
# upload multiple boards/channels in parallel. Hence this can
# parallelised down to the sub-bands.
@@ -170,7 +40,7 @@ def upload_tbb_data(stations, dm, start_time, duration, sub_bands, wait_time, bo
# batch handle all stations with same number of boards through lcurun
for num_boards in stationlists.keys():
logging.debug("Creating TBB commands for stations with %s boards..." % num_boards)
- relay = ["lcurun", ",".join(stationlists[num_boards])]
+ relay = [lcurun_command, ",".join(stationlists[num_boards])]
# iterate over tbb boards
board_list = ""
diff --git a/MAC/TBB/lib/tbb_util.py b/MAC/TBB/lib/tbb_util.py
new file mode 100644
index 00000000000..391bc8d56c2
--- /dev/null
+++ b/MAC/TBB/lib/tbb_util.py
@@ -0,0 +1,117 @@
+import logging
+from lofar.mac.tbb.tbb_config import *
+
+
+def split_stations_by_boardnumber(stations):
+ """
+ Stations are assumed to have 12 boards unless their name starts with 'cs' / 'rs'
+ Some alias groups are known to have only 6 boards, but others will be assumed to have 12 boards.
+
+ :param stations: comma-separated list of stations
+ :return: dict of lists with tbb board count as key and list of stations as value
+ """
+ logging.info('Determining number of TBB boards of provided stations')
+
+ def has6boards(station):
+ return station in groups_with_6boards or station[:2] in ['cs', 'rs']
+
+ stationlist = stations.split(',')
+ stationslists = {6: [x for x in stationlist if has6boards(x)],
+ 12: [x for x in stationlist if not has6boards(x)]}
+
+ stationslists = {k: v for k, v in stationslists.items() if len(v) > 0} # remove empty
+ logging.debug("Board counts: %s" % stationslists)
+ return stationslists
+
+def expand_range(input_range):
+ """
+ Take an input string that describes an integer range "7-29" and return the individual numbers in that range as strings in a list ["7", "8", ..., "29"].
+ :param input_range: An input string of the format "7-29" or "16"
+ :return: A list of the individual integer numbers in the range as strings.
+ """
+ items = []
+ index = input_range.find("-")
+ if index == 0 or index == (len(input_range) - 1):
+ raise ValueError("Cannot convert %s into a number!" % (input_range))
+ elif index == -1:
+ # Could be a single value. Check if it is a digit.
+ if input_range.isdigit() is True:
+ items.append(input_range)
+ else:
+ raise ValueError("Cannot convert %s into a number!" % (input_range))
+ else:
+ range_begin = input_range[:index]
+ if range_begin.isdigit() is True:
+ range_begin = int(range_begin)
+ else:
+ raise ValueError("Cannot convert %s into a number!" % (range_begin))
+ range_end = input_range[(index + 1):]
+ if range_end.isdigit() is True:
+ range_end = int(range_end)
+ else:
+ raise ValueError("Cannot convert %s into a number!" % (range_end))
+ for element in range(range_begin, range_end + 1):
+ items.append(str(element))
+ return items
+
+def expand_list(input_list):
+ """
+ Convert an input string of the form "1,17-33,64,1299-1344" to a list of strings that consists of each individual integer that is given and of all integers in every range.
+ :param input_list: An input string containing integer ranges "7-29" or individual integers "33" all separated by ",".
+ :return: A list of all the individual integer numbers and of all the integers in the ranges. The list elements are strings.
+ """
+ items = []
+ split_by_comma = input_list.split(",")
+ for item in split_by_comma:
+ if item.isdigit() is True:
+ items.append(item)
+ else:
+ items += expand_range(item)
+
+ logging.debug("Input list: \"%s\"\nFound these elements in the list: \"%s\"" % (input_list, items))
+ return items
+
+def calculate_adjusted_start_time(dm, start_time, sub_band):
+ """
+ Calculate an adjusted start time for data in a sub-band in TBB memory.
+ Interstellar and intergalactic dispersion smear out electro-magnetic pulses
+ so that higher frequencies of a pulse will arrive earlier at a receiver
+ than lower frequencies of the same pulse. Mostly free electrons in the path
+ of the pulse "delay" the different frequencies.
+ The three parameters, dispersion measure, recording start time of the
+ highest frequency and the target frequency as sub-band allow us to
+ calculate the recording start time for the target frequency. The calculated
+ start time will always be >= the start time of the highest frequency.
+ :param dm: Dispersion measure
+ :param start_time: Start time of the entire TBB
+ :param sub_band: The sub-band for which the time delay shall be calculated.
+ :return: A tuple of two values. The first value represents the adjusted
+ start time in seconds, the second value represents the remainder of the
+ adjusted start time in units of 5 microseconds.
+ """
+ sub_band_frequency = lowest_frequency + sub_band_width * sub_band
+ time_delay_sub_band = k_DM * dm * (sub_band_frequency**(-2) - highest_frequency**(-2))
+ adjusted_start_time = start_time + time_delay_sub_band
+
+ # The TBB needs the adjusted start time as two parameters: full seconds and
+ # the fractional remainder in units of 5 microseconds.
+ full_seconds = int(adjusted_start_time)
+ slice_nr = int((adjusted_start_time - full_seconds) / slice_size)
+
+ logging.info("Calculated the sub-band start time: "
+ "lowest frequency = %fMHz, "
+ "highest frequency = %fMHz, "
+ "sub-band width = %fMHz, "
+ "slice size = %fs, "
+ "sub-band = %d, "
+ "sub-band frequency = %fMHz, "
+ "start time of the highest frequency = %fs, "
+ "dispersion measure = %f, "
+ "calculated start time delay = %fs, "
+ "sub-band start time = %fs, "
+ "sub-band start time, seconds only = %ds, "
+ "sub-band start time, slice # = %d."
+ % (lowest_frequency, highest_frequency, sub_band_width, slice_size,
+ sub_band, sub_band_frequency, start_time, dm, time_delay_sub_band,
+ adjusted_start_time, full_seconds, slice_nr))
+ return (full_seconds, slice_nr)
--
GitLab