Merge branch 'support-higher-gn-indices' into 'main'

L2SS-1353: Record global node indices reported by each packet

See merge request !45

README.md

@@ -105,6 +105,7 @@ tox -e debug tests.requests.test_prometheus
 ```
 
 ## Releasenotes
+- 0.14.5 - Added `gn_indices` and support for global node indices > 16.
 - 0.14.4 - Fixed bug on `writer_version` retrieval
 - 0.14.3 - Added `rcu_pcb_id` and `rcu_pcb_version` to Hdf5 file header
 - 0.14.2 - Added `station_name` attribute to Hdf5 file header

VERSION

-0.14.4
+0.14.5

lofar_station_client/statistics/collector.py

@@ -99,6 +99,8 @@ class SSTCollector(StatisticsCollector):
                 "nof_payload_errors": numpy.zeros(
                     (self.MAX_FPGAS,), dtype=numpy.uint64
                 ),
+                # gn_index of each FPGA, for verification purposes
+                "gn_indices": numpy.full((self.MAX_FPGAS,), -1, dtype=numpy.uint64),
                 # Last value array we've constructed out of the packets
                 "sst_values": numpy.zeros(
                     (self.MAX_INPUTS, self.MAX_SUBBANDS), dtype=numpy.uint64
@@ -120,6 +122,10 @@ class SSTCollector(StatisticsCollector):
     def _parse_packet(self, packet):
         fields = SSTPacket(packet)
 
+        self.parameters["gn_indices"][
+            fields.gn_index % self.MAX_FPGAS
+        ] = fields.gn_index
+
         # determine which input this packet contains data for
         if fields.signal_input_index >= self.MAX_INPUTS:
             # packet describes an input that is out of bounds for us
@@ -132,13 +138,17 @@ class SSTCollector(StatisticsCollector):
 
         if fields.payload_error:
             # cannot trust the data if a payload error is reported
-            self.parameters["nof_payload_errors"][fields.gn_index] += numpy.uint64(1)
+            self.parameters["nof_payload_errors"][
+                fields.gn_index % self.MAX_FPGAS
+            ] += numpy.uint64(1)
 
             # don't raise, as packet is valid
             return
 
         # process the packet
-        self.parameters["nof_valid_payloads"][fields.gn_index] += numpy.uint64(1)
+        self.parameters["nof_valid_payloads"][
+            fields.gn_index % self.MAX_FPGAS
+        ] += numpy.uint64(1)
         self.parameters["sst_values"][input_index][
             : fields.nof_statistics_per_packet
         ] = fields.payload()
@@ -206,6 +216,8 @@ class XSTCollector(StatisticsCollector):
                 "nof_payload_errors": numpy.zeros(
                     (self.MAX_FPGAS,), dtype=numpy.uint64
                 ),
+                # gn_index of each FPGA, for verification purposes
+                "gn_indices": numpy.full((self.MAX_FPGAS,), -1, dtype=numpy.uint64),
                 # Last value array we've constructed out of the packets
                 "xst_blocks": numpy.zeros(
                     (
@@ -261,9 +273,15 @@ class XSTCollector(StatisticsCollector):
     def _parse_packet(self, packet):
         fields = XSTPacket(packet)
 
+        self.parameters["gn_indices"][
+            fields.gn_index % self.MAX_FPGAS
+        ] = fields.gn_index
+
         if fields.payload_error:
             # cannot trust the data if a payload error is reported
-            self.parameters["nof_payload_errors"][fields.gn_index] += numpy.uint64(1)
+            self.parameters["nof_payload_errors"][
+                fields.gn_index % self.MAX_FPGAS
+            ] += numpy.uint64(1)
 
             # don't raise, as packet is valid
             return
@@ -337,7 +355,9 @@ class XSTCollector(StatisticsCollector):
         )
 
         # process the packet
-        self.parameters["nof_valid_payloads"][fields.gn_index] += numpy.uint64(1)
+        self.parameters["nof_valid_payloads"][
+            fields.gn_index % self.MAX_FPGAS
+        ] += numpy.uint64(1)
 
         self.parameters["xst_blocks"][
             subband_slot, block_index, : fields.nof_statistics_per_packet
@@ -427,6 +447,8 @@ class BSTCollector(StatisticsCollector):
                     (self.MAX_FPGAS,),
                     dtype=numpy.uint64,
                 ),
+                # gn_index of each FPGA, for verification purposes
+                "gn_indices": numpy.full((self.MAX_FPGAS,), -1, dtype=numpy.uint64),
                 # Last value array we've constructed out of the packets
                 "bst_values": numpy.zeros(
                     (self.MAX_BLOCKS, self.MAX_BEAMLETS),
@@ -445,6 +467,10 @@ class BSTCollector(StatisticsCollector):
     def _parse_packet(self, packet):
         fields = BSTPacket(packet)
 
+        self.parameters["gn_indices"][
+            fields.gn_index % self.MAX_FPGAS
+        ] = fields.gn_index
+
         # To get the block_index we floor divide this beamlet_index by the max amount
         # of beamlets per block
         block_index = fields.beamlet_index // self.MAX_BEAMLETS
@@ -459,13 +485,17 @@ class BSTCollector(StatisticsCollector):
 
         if fields.payload_error:
             # cannot trust the data if a payload error is reported
-            self.parameters["nof_payload_errors"][fields.gn_index] += numpy.uint64(1)
+            self.parameters["nof_payload_errors"][
+                fields.gn_index % self.MAX_FPGAS
+            ] += numpy.uint64(1)
 
             # don't raise, as packet is valid
             return
 
         # process the packet
-        self.parameters["nof_valid_payloads"][fields.gn_index] += numpy.uint64(1)
+        self.parameters["nof_valid_payloads"][
+            fields.gn_index % self.MAX_FPGAS
+        ] += numpy.uint64(1)
         self.parameters["bst_values"][block_index][
             : self.MAX_BEAMLETS
         ] = fields.payload()

lofar_station_client/statistics/statistics_data.py

@@ -121,12 +121,16 @@ class StatisticsData(ndarray):
     BST the number of the beamlet for which this packet holds statistics.
     """
 
-    nof_valid_payloads: int = attribute()
+    nof_valid_payloads: ndarray = attribute()
     """ Number of packets received so far that we could parse correctly and do not have
-        a payload error """
+        a payload error. One value per FPGA. """
 
-    nof_payload_errors: int = attribute()
-    """ Number of packets that reported a payload error """
+    nof_payload_errors: ndarray = attribute()
+    """ Number of packets that reported a payload error. One value per FPGA. """
+
+    gn_indices: ndarray = attribute()
+    """ Global node index (gn) as reported by each FPGA, for each element in the payload
+        counters. """
 
     tile_beam_pointing_direction: str = attribute(optional=True)
     """ Direction of the tile beam """

lofar_station_client/statistics/writer/hdf5.py

@@ -254,6 +254,7 @@ class HDF5Writer(ABC):
         fields = {
             "nof_payload_errors": self.current_matrix.parameters["nof_payload_errors"],
             "nof_valid_payloads": self.current_matrix.parameters["nof_valid_payloads"],
+            "gn_indices": self.current_matrix.parameters["gn_indices"],
         }
 
         # add the packet headers
@@ -369,11 +370,6 @@ class HDF5Writer(ABC):
         tstamp = self.current_timestamp.isoformat(timespec="milliseconds")
         matrix = self.get_values_matrix()
 
-        # Stores the header of the packet received for this matrix as a list of
-        # attributes
-        matrix.nof_payload_errors = self.current_matrix.parameters["nof_payload_errors"]
-        matrix.nof_valid_payloads = self.current_matrix.parameters["nof_valid_payloads"]
-
         # Stores the header of the packet received for this matrix as a list of
         # attributes
         header_attributes = self.hdf5_matrix_header(self.statistics_packet_header)