Merge branch 'L2SS-1576-add-statistics-to-observations' into 'master'

Resolve L2SS-1576 "Add statistics to observations"

Closes L2SS-1576

See merge request !862

README.md

@@ -166,6 +166,8 @@ Next change the version in the following places:
 
 # Release Notes
 
+* 0.30.2 Add XST/SST settings to Observation specifications
+         Fixed dimensionality of xst.FPGA_subband_select_R(W)
 * 0.30.1 Remove deprecated FPGA_beamlet_output_nof_beamlets
          Fix prometheus metric for counting calibrations
          Removed ALARM and DISABLED states

tangostationcontrol/VERSION

-0.30.1
+0.30.2

tangostationcontrol/integration_test/default/devices/test_device_observation_field.py

@@ -111,6 +111,8 @@ class TestDeviceObservationField(AbstractTestBases.TestDeviceBase):
         self.recv_proxy = self.setup_proxy("STAT/RECVH/H0", defaults=True)
         self.sdpfirmware_proxy = self.setup_proxy("STAT/SDPFirmware/HBA0")
         self.sdp_proxy = self.setup_proxy("STAT/SDP/HBA0")
+        self.sst_proxy = self.setup_proxy("STAT/SST/HBA0")
+        self.xst_proxy = self.setup_proxy("STAT/XST/HBA0")
         self.antennafield_proxy = self.setup_proxy(
             self.antennafield_name, cb=self.antennafield_configure
         )

tangostationcontrol/integration_test/default/devices/test_observation_client.py

@@ -36,6 +36,8 @@ class TestObservation(base.IntegrationTestCase):
             "STAT/DigitalBeam/HBA0",
             "STAT/TileBeam/HBA0",
             "STAT/AFH/HBA0",
+            "STAT/SST/HBA0",
+            "STAT/XST/HBA0",
         ]:
             proxy = TestDeviceProxy(device)
             proxy.off()

tangostationcontrol/tangostationcontrol/common/constants.py

@@ -66,7 +66,7 @@ CLK_200_MHZ = 200_000_000
 CLK_160_MHZ = 160_000_000
 
 # Maximum number of subbands for which we collect XSTs simultaneously
-MAX_PARALLEL_SUBBANDS = 8
+MAX_PARALLEL_SUBBANDS = 7
 # Expected block for XST's
 BLOCK_LENGTH = 12
 # Complex values are (real, imag).

tangostationcontrol/tangostationcontrol/configuration/__init__.py

@@ -8,6 +8,8 @@ from .observation_settings import ObservationSettings
 from .observation_field_settings import ObservationFieldSettings
 from .pointing import Pointing
 from .sap import Sap
+from .sst import SST
+from .xst import XST
 
 __all__ = [
     "ObservationSettings",
@@ -17,4 +19,6 @@ __all__ = [
     "HBA",
     "Dithering",
     "REGISTRY",
+    "SST",
+    "XST",
 ]

tangostationcontrol/tangostationcontrol/configuration/_json_parser.py

@@ -14,6 +14,8 @@ def _from_json_hook_t(primary: Type):
         ObservationFieldSettings,
         HBA,
         Dithering,
+        SST,
+        XST,
     )
 
     def actual_hook(json_dct):
@@ -21,6 +23,8 @@ def _from_json_hook_t(primary: Type):
         primary_ex = None
         # Order is critical, must match inheritance, deepest layers first
         for t in [
+            SST,
+            XST,
             Pointing,
             Sap,
             HBA,

tangostationcontrol/tangostationcontrol/configuration/_schemas.py

@@ -14,8 +14,12 @@ from referencing.jsonschema import SchemaRegistry as _SchemaRegistry
 
 
 def _schemas():
-    for schema in files(__package__).joinpath("schemas").iterdir():
+    for schema in files(__package__).joinpath("schemas").glob("*.json"):
+        try:
             contents = json.loads(schema.read_text(encoding="utf-8"))
+        except json.decoder.JSONDecodeError as ex:
+            raise ValueError(f"Error decoding JSON schema {schema}") from ex
+
         yield Resource.from_contents(contents)
 
 

tangostationcontrol/tangostationcontrol/configuration/configuration_base.py

@@ -33,6 +33,8 @@ class _ConfigurationBase(ABC):
         """Class name to json schema file conversion name"""
         cls_name = cls_name.replace("HBA", "hba")
         cls_name = cls_name.replace("LBA", "lba")
+        cls_name = cls_name.replace("SST", "sst")
+        cls_name = cls_name.replace("XST", "xst")
         return re.sub(r"(?<!^)(?=[A-Z])", "-", cls_name).lower()
 
     @classmethod
@@ -53,7 +55,11 @@ class _ConfigurationBase(ABC):
         pass
 
     def __str__(self):
+        try:
             return json.dumps(dict(self), ensure_ascii=False)
+        except TypeError:
+            # This happens if we want to dump a type that cannot be represented in JSON
+            return str(dict(self))
 
     def __repr__(self):
         return self.__str__()
@@ -76,10 +82,14 @@ class _ConfigurationBase(ABC):
 
     @classmethod
     def from_json(cls: Type[T], data: str) -> T:
+        try:
             s = json.loads(data, object_hook=_from_json_hook_t(cls))
+        except json.decoder.JSONDecodeError as ex:
+            raise ValueError(f"Error decoding JSON {data}") from ex
+
         if not isinstance(s, cls):
             raise ValidationError(
                 f"Unexpected type: expected <{cls.__class__.__name__}>, got "
-                f"<{type(s).__name__}>"
+                f"<{type(s).__name__}>: {s}"
             )
         return s

tangostationcontrol/tangostationcontrol/configuration/hba.py

@@ -9,7 +9,7 @@ class HBA(_ConfigurationBase):
     def __init__(
         self,
         tile_beam: Pointing,
-        DAB_filter: bool | None,
+        DAB_filter: bool | None = None,
         element_selection: str | None = "ALL",
     ):
         self.tile_beam = tile_beam

tangostationcontrol/tangostationcontrol/configuration/observation_field_settings.py

@@ -8,6 +8,8 @@ from tangostationcontrol.configuration.configuration_base import _ConfigurationB
 from tangostationcontrol.configuration.dithering import Dithering
 from tangostationcontrol.configuration.hba import HBA
 from tangostationcontrol.configuration.sap import Sap
+from tangostationcontrol.configuration.sst import SST
+from tangostationcontrol.configuration.xst import XST
 
 
 class ObservationFieldSettings(_ConfigurationBase):
@@ -24,6 +26,8 @@ class ObservationFieldSettings(_ConfigurationBase):
         first_beamlet: int = 0,
         lead_time: float | None = None,
         dithering: Dithering | None = None,
+        SST: SST | None = None,
+        XST: XST | None = None,
     ):
         self.observation_id = observation_id
         self.start_time = self._parse_and_convert_datetime(start_time)
@@ -36,6 +40,8 @@ class ObservationFieldSettings(_ConfigurationBase):
         self.first_beamlet = first_beamlet
         self.lead_time = lead_time
         self.dithering = dithering
+        self.SST = SST
+        self.XST = XST
 
     @staticmethod
     def _parse_and_convert_datetime(time: str | datetime | None):
@@ -68,6 +74,10 @@ class ObservationFieldSettings(_ConfigurationBase):
             yield "lead_time", self.lead_time
         if self.dithering is not None:
             yield "dithering", dict(self.dithering)
+        if self.SST is not None:
+            yield "SST", dict(self.SST)
+        if self.XST is not None:
+            yield "SST", dict(self.XST)
 
     @staticmethod
     def to_object(json_dct) -> "ObservationFieldSettings":
@@ -83,4 +93,6 @@ class ObservationFieldSettings(_ConfigurationBase):
             json_dct.get("first_beamlet", 0),
             json_dct.get("lead_time"),
             json_dct.get("dithering"),
+            json_dct.get("SST"),
+            json_dct.get("XST"),
         )

tangostationcontrol/tangostationcontrol/configuration/schemas/observation-field-settings.json

@@ -82,6 +82,12 @@
     },
     "HBA": {
       "$ref": "hba"
+    },
+    "sst": {
+      "$ref": "sst"
+    },
+    "xst": {
+      "$ref": "xst"
     }
   }
 }

tangostationcontrol/tangostationcontrol/configuration/schemas/sst.json

+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "$id": "sst",
+  "type": "object",
+  "required": [
+    "subbands_calibrated"
+  ],
+  "properties": {
+    "subbands_calibrated": {
+      "type": "boolean",
+      "default": true
+    }
+  }
+}

tangostationcontrol/tangostationcontrol/configuration/schemas/xst.json

+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "$id": "xst",
+  "type": "object",
+  "required": [
+    "subbands",
+    "integration_interval"
+  ],
+  "properties": {
+    "subbands": {
+      "type": "array",
+      "minItems": 0,
+      "maxItems": 7,
+      "items": {
+        "type": "number",
+        "minimum": 0,
+        "maximum": 488
+      }
+    },
+    "subbands_step": {
+      "type": "number",
+      "default": 0,
+      "minimum": 0,
+      "maximum": 488
+    },
+    "integration_interval": {
+      "type": "number",
+      "default": 1.0,
+      "minimum": 0.1,
+      "maximum": 10.7
+    }
+  }
+}

tangostationcontrol/tangostationcontrol/configuration/sst.py

+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+
+from tangostationcontrol.configuration.configuration_base import _ConfigurationBase
+
+
+class SST(_ConfigurationBase):
+    def __init__(
+        self,
+        subbands_calibrated: bool,
+    ):
+        self.subbands_calibrated = subbands_calibrated
+
+    def __iter__(self):
+        yield "subbands_calibrated", self.subbands_calibrated
+
+    @staticmethod
+    def to_object(json_dct) -> "SST":
+        return SST(
+            json_dct["subbands_calibrated"],
+        )

tangostationcontrol/tangostationcontrol/configuration/xst.py

+# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import List
+
+from tangostationcontrol.configuration.configuration_base import _ConfigurationBase
+
+
+class XST(_ConfigurationBase):
+    def __init__(
+        self,
+        integration_interval: float,
+        subbands: List[int],
+        subbands_step: int | None = None,
+    ):
+        self.subbands = subbands
+        self.subbands_step = subbands_step
+        self.integration_interval = integration_interval
+
+    def __iter__(self):
+        yield "integration_interval", self.integration_interval
+        yield "subbands", self.subbands
+
+        if self.subbands_step is not None:
+            yield "subbands_step", self.subbands_step
+
+    @staticmethod
+    def to_object(json_dct) -> "XST":
+        return XST(
+            json_dct["integration_interval"],
+            json_dct["subbands"],
+            json_dct.get("subbands_step"),
+        )

tangostationcontrol/tangostationcontrol/devices/observation_field.py

@@ -6,7 +6,7 @@ from datetime import datetime
 import logging
 from itertools import chain
 from time import time
-from typing import Optional
+from typing import Optional, List
 
 import numpy
 from jsonschema.exceptions import ValidationError
@@ -18,8 +18,10 @@ from tangostationcontrol.common.constants import (
     DEFAULT_METRICS_POLLING_PERIOD,
     DEFAULT_POLLING_PERIOD,
     MAX_ANTENNA,
+    MAX_PARALLEL_SUBBANDS,
     N_beamlets_ctrl,
     N_elements,
+    N_pn,
     N_point_prop,
     N_pol,
 )
@@ -254,6 +256,52 @@ class ObservationField(LOFARDevice):
         except AttributeError:
             return "ALL"
 
+    @attribute(
+        doc="",
+        dtype=bool,
+        fisallowed="is_attribute_access_allowed",
+    )
+    def SST_subbands_calibrated_R(self):
+        try:
+            return self._observation_field_settings.SST.subbands_calibrated
+        except AttributeError:
+            return True
+
+    @attribute(
+        doc="The indices of the subbands to emit every integration interval.",
+        dtype=(numpy.uint32,),
+        max_dim_x=MAX_PARALLEL_SUBBANDS,
+    )
+    def XST_subbands_R(self):
+        try:
+            return numpy.array(
+                self._observation_field_settings.XST.subbands,
+                dtype=numpy.uint32,
+            )
+        except AttributeError:
+            return numpy.array([], dtype=numpy.uint32)
+
+    @attribute(
+        doc="The subband indices are increased with this index every interval, causing different subbands to be emitted.",
+        dtype=numpy.uint32,
+    )
+    def XST_subbands_step_R(self):
+        try:
+            return self._observation_field_settings.XST.subbands_step
+        except AttributeError:
+            return 0
+
+    @attribute(
+        doc="",
+        dtype=numpy.float64,
+        fisallowed="is_attribute_access_allowed",
+    )
+    def XST_integration_interval_R(self):
+        try:
+            return self._observation_field_settings.XST.integration_interval
+        except AttributeError:
+            return 1.0
+
     observation_field_settings_RW = attribute(
         dtype=str, access=AttrWriteType.READ_WRITE
     )
@@ -372,6 +420,14 @@ class ObservationField(LOFARDevice):
                 f"{util.get_ds_inst_name()}/TileBeam/{antennafield}"
             )
 
+        self.sst_proxy = create_device_proxy(
+            f"{util.get_ds_inst_name()}/SST/{antennafield}"
+        )
+
+        self.xst_proxy = create_device_proxy(
+            f"{util.get_ds_inst_name()}/XST/{antennafield}"
+        )
+
     @log_exceptions()
     def _start_observation(self):
         """Configure the station for this observation and antenna field."""
@@ -438,6 +494,31 @@ class ObservationField(LOFARDevice):
                 element_selection
             )
 
+        # Configure SST
+        subbands_calibrated = self.read_attribute("SST_subbands_calibrated_R")
+        self.sst_proxy.FPGA_sst_offload_weighted_subbands_RW = [
+            subbands_calibrated
+        ] * N_pn
+
+        # Toggle statistics to activate new settings
+        self.sst_proxy.power_hardware_off()
+        self.sst_proxy.power_hardware_on()
+
+        # Configure XST
+        subbands = self.read_attribute("XST_subbands_R")
+        subbands_step = self.read_attribute("XST_subbands_step_R")
+        integration_interval = self.read_attribute("XST_integration_interval_R")
+
+        self.xst_proxy.FPGA_xst_subband_select_RW = self._apply_xst_subband_select(
+            subbands.tolist(),
+            subbands_step,
+        )
+        self.xst_proxy.FPGA_xst_integration_interval_RW = [integration_interval] * N_pn
+
+        # Toggle statistics to activate new settings
+        self.xst_proxy.power_hardware_off()
+        self.xst_proxy.power_hardware_on()
+
     @log_exceptions()
     def _stop_observation(self):
         """Tear down station resources we used."""
@@ -470,6 +551,21 @@ class ObservationField(LOFARDevice):
         """Return whether this observation should control a TileBeam device."""
         return self._observation_field_settings.antenna_field.startswith("HBA")
 
+    def _apply_xst_subband_select(
+        self, subbands: List[int], subbands_step: int
+    ) -> numpy.ndarray:
+
+        if len(subbands) > MAX_PARALLEL_SUBBANDS:
+            raise ValueError(
+                f"Requested more than {MAX_PARALLEL_SUBBANDS} subbands for the XSTs: {subbands}"
+            )
+
+        return numpy.array(
+            [[subbands_step] + subbands + [0] * (MAX_PARALLEL_SUBBANDS - len(subbands))]
+            * N_pn,
+            dtype=numpy.uint32,
+        )
+
     def _apply_antennafield_settings(self, filter_name: str):
         """Retrieve the RCU band from filter name, returning the correct format for
         AntennaField device

tangostationcontrol/tangostationcontrol/devices/sdp/bst.py

@@ -195,6 +195,14 @@ class BST(Statistics):
     # Overloaded functions
     # --------
 
+    def _power_hardware_on(self):
+        self.proxy.write_attribute(
+            "FPGA_bst_offload_enable_RW", self.FPGA_bst_offload_enable_RW_default
+        )
+
+    def _power_hardware_off(self):
+        self.proxy.write_attribute("FPGA_bst_offload_enable_RW", [False] * N_pn)
+
     # --------
     # Commands
     # --------

tangostationcontrol/tangostationcontrol/devices/sdp/sst.py

@@ -222,6 +222,14 @@ class SST(Statistics):
     # Overloaded functions
     # --------
 
+    def _power_hardware_on(self):
+        self.proxy.write_attribute(
+            "FPGA_sst_offload_enable_RW", self.FPGA_sst_offload_enable_RW_default
+        )
+
+    def _power_hardware_off(self):
+        self.proxy.write_attribute("FPGA_sst_offload_enable_RW", [False] * N_pn)
+
     # --------
     # Commands
     # --------

tangostationcontrol/tangostationcontrol/devices/sdp/xst.py

@@ -58,7 +58,7 @@ class XST(Statistics):
         first_signal_index = self.control.read_parent_attribute(
             "first_signal_input_index_R"
         )
-        return XSTCollector(nr_signal_inputs, first_signal_index)
+        return XSTCollector(nr_signal_inputs, first_signal_index, MAX_PARALLEL_SUBBANDS)
 
     # -----------------
     # Device Properties
@@ -210,17 +210,19 @@ class XST(Statistics):
         dims=(N_pn,),
     )
     FPGA_xst_subband_select_RW = AttributeWrapper(
+        doc="Which subbands to emit XSTs for. The first element is the step size, allowing different subbands to be emitted every interval.",
         comms_id=OPCUAConnection,
         comms_annotation=["FPGA_xst_subband_select_RW"],
         datatype=numpy.uint32,
-        dims=(MAX_PARALLEL_SUBBANDS, N_pn),
+        dims=(N_pn, MAX_PARALLEL_SUBBANDS + 1),
         access=AttrWriteType.READ_WRITE,
     )
     FPGA_xst_subband_select_R = AttributeWrapper(
+        doc="Which subbands to emit XSTs for. The first element is the step size, allowing different subbands to be emitted every interval.",
         comms_id=OPCUAConnection,
         comms_annotation=["FPGA_xst_subband_select_R"],
         datatype=numpy.uint32,
-        dims=(MAX_PARALLEL_SUBBANDS, N_pn),
+        dims=(N_pn, MAX_PARALLEL_SUBBANDS + 1),
     )
     FPGA_xst_start_time_RW = AttributeWrapper(
         doc="Start generating XSTs at this timestamp, or as soon as possible. Use this to synchronise the XSTs from different FPGAs.",
@@ -689,31 +691,6 @@ class XST(Statistics):
         fget=lambda self: self.read_xst_N_phase_R(6),
     )
 
-    xst_7_real_R = attribute(
-        max_dim_x=MAX_INPUTS,
-        max_dim_y=MAX_INPUTS,
-        dtype=((numpy.float32,),),
-        fget=lambda self: self.read_xst_N_real_R(7),
-    )
-    xst_7_imag_R = attribute(
-        max_dim_x=MAX_INPUTS,
-        max_dim_y=MAX_INPUTS,
-        dtype=((numpy.float32,),),
-        fget=lambda self: self.read_xst_N_imag_R(7),
-    )
-    xst_7_power_R = attribute(
-        max_dim_x=MAX_INPUTS,
-        max_dim_y=MAX_INPUTS,
-        dtype=((numpy.float32,),),
-        fget=lambda self: self.read_xst_N_power_R(7),
-    )
-    xst_7_phase_R = attribute(
-        max_dim_x=MAX_INPUTS,
-        max_dim_y=MAX_INPUTS,
-        dtype=((numpy.float32,),),
-        fget=lambda self: self.read_xst_N_phase_R(7),
-    )
-
     def read_xst_N_real_R(self, subband_idx):
         return numpy.real(self.statistics_client.collector.xst_values([subband_idx])[0])
 
@@ -745,6 +722,18 @@ class XST(Statistics):
     # Overloaded functions
     # --------
 
+    def _power_hardware_on(self):
+        self.proxy.write_attribute(
+            "FPGA_xst_processing_enable_RW", self.FPGA_xst_processing_enable_RW_default
+        )
+        self.proxy.write_attribute(
+            "FPGA_xst_offload_enable_RW", self.FPGA_xst_offload_enable_RW_default
+        )
+
+    def _power_hardware_off(self):
+        self.proxy.write_attribute("FPGA_xst_offload_enable_RW", [False] * N_pn)
+        self.proxy.write_attribute("FPGA_xst_processing_enable_RW", [False] * N_pn)
+
     # --------
     # Commands
     # --------