diff --git a/devices/util/attribute_wrapper.py b/devices/clients/attribute_wrapper.py similarity index 100% rename from devices/util/attribute_wrapper.py rename to devices/clients/attribute_wrapper.py diff --git a/devices/util/comms_client.py b/devices/clients/comms_client.py similarity index 100% rename from devices/util/comms_client.py rename to devices/clients/comms_client.py diff --git a/devices/clients/opcua_connection.py b/devices/clients/opcua_client.py similarity index 100% rename from devices/clients/opcua_connection.py rename to devices/clients/opcua_client.py diff --git a/devices/common/__init__.py b/devices/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/util/lofar_git.py b/devices/common/lofar_git.py similarity index 97% rename from devices/util/lofar_git.py rename to devices/common/lofar_git.py index e95f6bdf369e9f21bfb89f0d3359a1328157d0a3..353748d985ccf72483792456cbcd8a0a5aa1056b 100644 --- a/devices/util/lofar_git.py +++ b/devices/common/lofar_git.py @@ -52,7 +52,7 @@ def get_version(repo: git.Repo = None) -> str: branch = repo.active_branch commit = repo.commit() - tags = { tag.commit: tag for tag in repo.tags } + tags = {tag.commit: tag for tag in repo.tags} if commit in tags: commit_str = "{}".format(tags[commit]) diff --git a/devices/util/lofar_logging.py b/devices/common/lofar_logging.py similarity index 100% rename from devices/util/lofar_logging.py rename to devices/common/lofar_logging.py diff --git a/devices/devices/__init__.py b/devices/devices/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/APSCTL.py b/devices/devices/apsctl.py similarity index 96% rename from devices/APSCTL.py rename to devices/devices/apsctl.py index 068ec2ab411c25395e06577121d78ffc575cdb55..5187aa8eb9f2a5231ec404636ccc5aae3ace15ce 100644 --- a/devices/APSCTL.py +++ b/devices/devices/apsctl.py @@ -17,12 +17,12 @@ from tango.server import device_property, attribute from tango import AttrWriteType # Additional import -from clients.opcua_connection import OPCUAConnection -from util.attribute_wrapper import attribute_wrapper -from util.hardware_device import hardware_device +from clients.opcua_client import OPCUAConnection +from devices.clients.attribute_wrapper import attribute_wrapper +from devices.devices.hardware_device import hardware_device -from util.lofar_logging import device_logging_to_python, log_exceptions -from util.lofar_git import get_version +from devices.common.lofar_logging import device_logging_to_python, log_exceptions +from devices.common.lofar_git import get_version import numpy @@ -188,7 +188,7 @@ class APSCTL(hardware_device): def main(args=None, **kwargs): """Main function of the SDP module.""" - from util.lofar_logging import configure_logger + from devices.common.lofar_logging import configure_logger import logging configure_logger(logging.getLogger()) diff --git a/devices/util/wrappers.py b/devices/devices/device_decorators.py similarity index 100% rename from devices/util/wrappers.py rename to devices/devices/device_decorators.py diff --git a/devices/util/hardware_device.py b/devices/devices/hardware_device.py similarity index 100% rename from devices/util/hardware_device.py rename to devices/devices/hardware_device.py diff --git a/devices/PCC.py b/devices/devices/pcc.py similarity index 95% rename from devices/PCC.py rename to devices/devices/pcc.py index e8eb9bc94b689bc616ce49dba9601b0ed99de7a7..fe2a1841d33fec9d6157a581c75dda3a01f2971f 100644 --- a/devices/PCC.py +++ b/devices/devices/pcc.py @@ -19,13 +19,13 @@ from tango import AttrWriteType import numpy # Additional import -from util.wrappers import * +from devices.devices.device_decorators import * -from clients.opcua_connection import OPCUAConnection -from util.attribute_wrapper import attribute_wrapper -from util.hardware_device import hardware_device -from util.lofar_logging import device_logging_to_python, log_exceptions -from util.lofar_git import get_version +from clients.opcua_client import OPCUAConnection +from devices.clients.attribute_wrapper import attribute_wrapper +from devices.devices.hardware_device import hardware_device +from devices.common.lofar_logging import device_logging_to_python, log_exceptions +from devices.common.lofar_git import get_version __all__ = ["PCC", "main"] @@ -244,7 +244,7 @@ class PCC(hardware_device): def main(args=None, **kwargs): """Main function of the PCC module.""" - from util.lofar_logging import configure_logger + from devices.common.lofar_logging import configure_logger import logging configure_logger(logging.getLogger()) diff --git a/devices/SDP.py b/devices/devices/sdp.py similarity index 96% rename from devices/SDP.py rename to devices/devices/sdp.py index 22d0a73fad15b7fdf81b74d1e56095d09d2a3852..c2aad2ddd5490b6da1e23d1a75c1b63f6776475f 100644 --- a/devices/SDP.py +++ b/devices/devices/sdp.py @@ -17,12 +17,12 @@ from tango.server import device_property, attribute from tango import AttrWriteType # Additional import -from clients.opcua_connection import OPCUAConnection -from util.attribute_wrapper import attribute_wrapper -from util.hardware_device import hardware_device +from clients.opcua_client import OPCUAConnection +from devices.clients.attribute_wrapper import attribute_wrapper +from devices.devices.hardware_device import hardware_device -from util.lofar_logging import device_logging_to_python, log_exceptions -from util.lofar_git import get_version +from devices.common.lofar_logging import device_logging_to_python, log_exceptions +from devices.common.lofar_git import get_version import numpy @@ -176,7 +176,7 @@ class SDP(hardware_device): def main(args=None, **kwargs): """Main function of the SDP module.""" - from util.lofar_logging import configure_logger + from devices.common.lofar_logging import configure_logger import logging configure_logger(logging.getLogger()) diff --git a/devices/devices/sdp/__init__.py b/devices/devices/sdp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/devices/sdp_statistics/__init__.py b/devices/devices/sdp_statistics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/devices/sdp_statistics/sst.py b/devices/devices/sdp_statistics/sst.py new file mode 100644 index 0000000000000000000000000000000000000000..28874a3d0556539f38d323d3a5b5dac1d0dd30fc --- /dev/null +++ b/devices/devices/sdp_statistics/sst.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# +# This file is part of the Statistics project +# +# +# +# Distributed under the terms of the APACHE license. +# See LICENSE.txt for more info. + +""" Statistics Device Server for LOFAR2.0 + +""" + +# PyTango imports +from tango.server import run +from tango.server import device_property +# Additional import + +from clients.sst_client import sst_client + +from devices.clients.attribute_wrapper import attribute_wrapper +from devices.devices.hardware_device import hardware_device + +from devices.common.lofar_logging import device_logging_to_python, log_exceptions + +import numpy + +__all__ = ["SST", "main"] + +@device_logging_to_python({"device": "SST"}) +class SST(hardware_device): + + # ----------------- + # Device Properties + # ----------------- + + SST_Port = device_property( + dtype='DevUShort', + mandatory=True + ) + + # ---------- + # Attributes + # ---------- + # -------- + + # SST client annotation consists of a dict that contains the parameter name that needs to be read. + # Example: comms_annotation={"parameter": "this_value_R"} + packet_count_R = attribute_wrapper(comms_annotation={"parameter": "packet_count_R"}, datatype=numpy.int64) + last_packet_timestamp_R = attribute_wrapper(comms_annotation={"parameter": "last_packet_timestamp_R"}, datatype=numpy.int64) + queue_percentage_used_R = attribute_wrapper(comms_annotation={"parameter": "queue_percentage_used_R"}, datatype=numpy.double) + + # -------- + # overloaded functions + def configure_for_off(self): + """ user code here. is called when the state is set to OFF """ + + # Stop keep-alive + try: + self.sst_client.stop() + except Exception as e: + self.warn_stream("Exception while stopping sst_client in configure_for_off function: {}. Exception ignored".format(e)) + + @log_exceptions() + def configure_for_initialise(self): + """ user code here. is called when the sate is set to INIT """ + """Initialises the attributes and properties of the statistics device.""" + + self.sst_client = sst_client("0.0.0.0", self.SST_Port, self.Fault, self) + + # map an access helper class + for i in self.attr_list(): + try: + i.set_comm_client(self.sst_client) + except Exception as e: + # use the pass function instead of setting read/write fails + i.set_pass_func() + self.warn_stream("error while setting the sst attribute {} read/write function. {}. using pass function instead".format(i, e)) + pass + + self.sst_client.start() + + # -------- + # Commands + # -------- + +# ---------- +# Run server +# ---------- +def main(args=None, **kwargs): + """Main function of the Statistics Device module.""" + + from devices.common.lofar_logging import configure_logger + import logging + configure_logger(logging.getLogger()) + + return run((Statistics,), args=args, **kwargs) + + +if __name__ == '__main__': + main() diff --git a/devices/devices/sdp_statistics/statistics_packet.py b/devices/devices/sdp_statistics/statistics_packet.py new file mode 100644 index 0000000000000000000000000000000000000000..37565935da2d6f10cd5898b6a3cc46f7dafcc259 --- /dev/null +++ b/devices/devices/sdp_statistics/statistics_packet.py @@ -0,0 +1,218 @@ +from struct import unpack, calcsize +from datetime import datetime, timezone +import numpy + +__all__ = ["StatisticsPacket"] + +def get_bit_value(value: bytes, first_bit: int, last_bit:int=None) -> int: + """ Return bits [first_bit:last_bit] from value, and return their integer value. Bit 0 = LSB. + + For example, extracting bits 2-3 from b'01100' returns 11 binary = 3 decimal: + get_bit_value(b'01100', 2, 3) == 3 + + If 'last_bit' is not given, just the value of bit 'first_bit' is returned. """ + + # default last_bit to first_bit + if last_bit is None: + last_bit = first_bit + + return value >> first_bit & ((1 << (last_bit - first_bit + 1)) - 1) + +class StatisticsPacket(object): + """ + Models a statistics UDP packet from SDP. + + Packets are expected to be UDP payload only (so no Ethernet/IP/UDP headers). + """ + + def __init__(self, packet: bytes): + self.packet = packet + + # Only parse valid packets + if self.marker not in 'SBX': + raise ValueError("Invalid SDP statistics packet: packet marker (first byte) is '{}', not one of 'SBX'.".format(self.marker)) + + @property + def marker(self) -> str: + """ Return the type of statistic: + + 'S' = SST + 'B' = BST + 'X' = XST + """ + + raw_marker = unpack("c",self.packet[0:1])[0] + + try: + return raw_marker.decode('ascii') + except UnicodeDecodeError: + # non-ascii (>127) character, return as binary + # + # this is typically not visible to the user, as these packets are not SDP statistics packets, + # which the constructor will refuse to accept. + return raw_marker + + @property + def version_id(self) -> int: + """ Return the version of this packet. """ + + return unpack("B",self.packet[1:2])[0] + + @property + def observation_id(self) -> int: + """ Return the ID of the observation running when this packet was generated. """ + + return unpack("<I",self.packet[2:6])[0] + + @property + def station_id(self) -> int: + """ Return the number of the station this packet was generated on. """ + + return unpack("<H",self.packet[6:8])[0] + + @property + def source_info(self) -> int: + """ Return a dict with the source_info flags. The dict contains the following fields: + + _raw: raw value of the source_info field in the packet, as an integer. + antenna_band_index: antenna type. 0 = low band, 1 = high band. + nyquist_zone_index: nyquist zone of filter: + 0 = 0 -- 1/2 * t_adc Hz (low band), + 1 = 1/2 * t_adc -- t_adc Hz (high band), + 2 = t_adc -- 3/2 * t_adc Hz (high band). + t_adc: sampling clock. 0 = 160 MHz, 1 = 200 MHz. + fsub_type: sampling method. 0 = critically sampled, 1 = oversampled. + payload_error: 0 = data is ok, 1 = data is corrupted (a fault was encountered). + beam_repositioning_flag: 0 = data is ok, 1 = beam got repositioned during packet construction (BST only). + subband_calibrated_flag: 1 = subband data had subband calibration values applied, 0 = not + reserved: reserved bits + gn_index: global index of FPGA that emitted this packet. """ + + bits = unpack("<H",self.packet[8:10])[0] + + return { + "_raw": bits, + "antenna_band_index": get_bit_value(bits, 15), + "nyquist_zone_index": get_bit_value(bits, 13, 14), + "t_adc": get_bit_value(bits, 12), + "fsub_type": get_bit_value(bits, 11), + "payload_error": get_bit_value(bits, 10), + "beam_repositioning_flag": get_bit_value(bits, 9), + "subband_calibrated_flag": get_bit_value(bits, 8), + "reserved": get_bit_value(bits, 5, 7), + "gn_index": get_bit_value(bits, 0, 4), + } + + @property + def reserved(self) -> bytes: + """ Reserved bytes. """ + + return self.packet[10:11] + + @property + def integration_interval_raw(self) -> int: + """ Returns the integration interval, in blocks. """ + + # This field is 3 bytes, little endian, so we need to append a 0 to parse it as a 32-bit integer. + return unpack("<I", self.packet[11:14] + b'0')[0] + + def integration_interval(self) -> float: + """ Returns the integration interval, in seconds. """ + + # Translate to seconds using the block period + return self.integration_interval_raw * self.block_period() + + @property + def data_id(self) -> int: + """ Returns the generic data identifier. """ + + return unpack("<I",self.packet[14:18])[0] + + @property + def nof_signal_inputs(self) -> int: + """ Number of inputs that were used for constructing the payload. """ + return unpack("<B",self.packet[18:19])[0] + + @property + def nof_bytes_per_statistic(self) -> int: + """ Word size for the payload. """ + + return unpack("<B",self.packet[19:20])[0] + + @property + def nof_statistics_per_packet(self) -> int: + """ Number of data points in the payload. """ + + return unpack("<H",self.packet[20:22])[0] + + @property + def block_period_raw(self) -> int: + """ Return the block period, in nanoseconds. """ + + return unpack("<H",self.packet[22:24])[0] + + def block_period(self) -> float: + """ Return the block period, in seconds. """ + + return self.block_period_raw / 1e9 + + @property + def block_serial_number(self) -> int: + """ Block index since epoch (1970). """ + + return unpack("<Q",self.packet[24:32])[0] + + def timestamp(self) -> datetime: + """ Returns the timestamp of the data in this packet. """ + + return datetime.fromtimestamp(self.block_serial_number * self.block_period(), timezone.utc) + + def header(self) -> dict: + """ Return all the header fields as a dict. """ + + return { + "marker": self.marker, + "version_id": self.version_id, + "observation_id": self.observation_id, + "station_id": self.station_id, + "source_info": self.source_info, + "reserved": self.reserved, + "integration_interval_raw": self.integration_interval_raw, + "integration_interval": self.integration_interval(), + "data_id": self.data_id, + "nof_signal_inputs": self.nof_signal_inputs, + "nof_bytes_per_statistic": self.nof_bytes_per_statistic, + "nof_statistics_per_packet": self.nof_statistics_per_packet, + "block_period_raw": self.block_period_raw, + "block_period": self.block_period(), + "block_serial_number": self.block_serial_number, + "timestamp": self.timestamp(), + } + + @property + def payload_sst(self) -> numpy.array: + """ The payload of this packet, interpreted as SST data. """ + + if self.marker != 'S': + raise Exception("Payload of SST requested of a non-SST packet. Actual packet marker is '{}', but must be 'S'.".format(self.marker)) + + # derive which and how many elements to read from the packet header + bytecount_to_unsigned_struct_type = { 1: 'B', 2: 'H', 4: 'I', 8: 'Q' } + format_str = "<{}{}".format(self.nof_statistics_per_packet, bytecount_to_unsigned_struct_type[self.nof_bytes_per_statistic]) + + return numpy.array(unpack(format_str, self.packet[32:32+calcsize(format_str)])) + + +if __name__ == "__main__": + # parse one packet from stdin + import sys + import pprint + + # read all of stdin, even though we only parse the first packet. we're too lazy to intelligently decide when + # the packet is complete and can stop reading. + data = sys.stdin.buffer.read() + packet = StatisticsPacket(data) + + # print header & payload + pprint.pprint(packet.header()) + pprint.pprint(packet.payload_sst) diff --git a/devices/examples/__init__.py b/devices/examples/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/examples/load_from_disk/__init__.py b/devices/examples/load_from_disk/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/clients/ini_client.py b/devices/examples/load_from_disk/ini_client.py similarity index 100% rename from devices/clients/ini_client.py rename to devices/examples/load_from_disk/ini_client.py diff --git a/devices/ini_device.py b/devices/examples/load_from_disk/ini_device.py similarity index 97% rename from devices/ini_device.py rename to devices/examples/load_from_disk/ini_device.py index dbc6e6159409449cfa7f5577e06eaa84e0620a06..c1c273171d268eb1ca6da587f7c616081e30dd0e 100644 --- a/devices/ini_device.py +++ b/devices/examples/load_from_disk/ini_device.py @@ -11,12 +11,10 @@ # PyTango imports from tango.server import run -from tango.server import device_property from tango import AttrWriteType -from tango import DevState # Additional import -from util.attribute_wrapper import attribute_wrapper -from util.hardware_device import hardware_device +from devices.clients.attribute_wrapper import attribute_wrapper +from devices.devices.hardware_device import hardware_device import configparser diff --git a/devices/examples/snmp/__init__.py b/devices/examples/snmp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/SNMP.py b/devices/examples/snmp/snmp.py similarity index 93% rename from devices/SNMP.py rename to devices/examples/snmp/snmp.py index eb90b276896f5e07f16eceb43c1dbf810da1b2ae..7962d2fd738898fa084c6530336ab61f65f604cf 100644 --- a/devices/SNMP.py +++ b/devices/examples/snmp/snmp.py @@ -17,9 +17,9 @@ from tango.server import device_property from tango import AttrWriteType # Additional import -from clients.SNMP_client import SNMP_client -from util.attribute_wrapper import attribute_wrapper -from util.hardware_device import hardware_device +from devices.examples.snmp.snmp_client import SNMP_client +from devices.clients.attribute_wrapper import attribute_wrapper +from devices.devices.hardware_device import hardware_device import numpy @@ -111,7 +111,7 @@ class SNMP(hardware_device): def main(args=None, **kwargs): """Main function of the PCC module.""" - from util.lofar_logging import configure_logger + from devices.common.lofar_logging import configure_logger import logging configure_logger(logging.getLogger()) diff --git a/devices/clients/SNMP_client.py b/devices/examples/snmp/snmp_client.py similarity index 100% rename from devices/clients/SNMP_client.py rename to devices/examples/snmp/snmp_client.py diff --git a/devices/test/clients/__init__.py b/devices/test/clients/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/clients/test_client.py b/devices/test/clients/test_client.py similarity index 100% rename from devices/clients/test_client.py rename to devices/test/clients/test_client.py diff --git a/devices/test/devices/__init__.py b/devices/test/devices/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/RandomData.py b/devices/test/devices/random_data.py similarity index 100% rename from devices/RandomData.py rename to devices/test/devices/random_data.py diff --git a/devices/test_device.py b/devices/test/devices/test_device.py similarity index 97% rename from devices/test_device.py rename to devices/test/devices/test_device.py index 6a62907112ea1cf081436285aa0d21532ba24d0a..22fe3c2bb7e6e52e887e029e8d189f41e3bad2c4 100644 --- a/devices/test_device.py +++ b/devices/test/devices/test_device.py @@ -17,8 +17,8 @@ from tango import DevState # Additional import from clients.test_client import test_client -from util.attribute_wrapper import * -from util.hardware_device import * +from devices.clients.attribute_wrapper import * +from devices.devices.hardware_device import * __all__ = ["test_device", "main"] diff --git a/devices/toolkit/__init__.py b/devices/toolkit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/devices/util/archiver.py b/devices/toolkit/archiver.py similarity index 94% rename from devices/util/archiver.py rename to devices/toolkit/archiver.py index 70f43d88a02b3f40e7324aceb966913a74fca51a..3764a7b161b9f01bd0cdd8cf5c9e9ee4a171d90d 100644 --- a/devices/util/archiver.py +++ b/devices/toolkit/archiver.py @@ -1,6 +1,5 @@ #! /usr/bin/env python3 -from .lofar2_config import configure_logging from tango import DeviceProxy diff --git a/devices/util/get_internal_attribute_history.py b/devices/toolkit/get_internal_attribute_history.py similarity index 100% rename from devices/util/get_internal_attribute_history.py rename to devices/toolkit/get_internal_attribute_history.py diff --git a/devices/util/lofar2_config.py b/devices/toolkit/lofar2_config.py similarity index 100% rename from devices/util/lofar2_config.py rename to devices/toolkit/lofar2_config.py diff --git a/devices/util/lts_cold_start.py b/devices/toolkit/lts_cold_start.py similarity index 98% rename from devices/util/lts_cold_start.py rename to devices/toolkit/lts_cold_start.py index 18b2bbb01fdff1508a65beec5333a4572369000f..baaadade255a55a3e730336786cca8ca03e73eef 100644 --- a/devices/util/lts_cold_start.py +++ b/devices/toolkit/lts_cold_start.py @@ -1,8 +1,8 @@ #! /usr/bin/env python3 import logging from time import sleep -from .startup import startup -from .lofar2_config import configure_logging +from devices.toolkit.startup import startup +from devices.toolkit.lofar2_config import configure_logging def start_device(device: str): diff --git a/devices/util/startup.py b/devices/toolkit/startup.py similarity index 100% rename from devices/util/startup.py rename to devices/toolkit/startup.py diff --git a/devices/udp_simulator.py b/devices/toolkit/udp_simulator.py similarity index 100% rename from devices/udp_simulator.py rename to devices/toolkit/udp_simulator.py