Merge branch 'L2SS-624-implement-boot-sequence' into 'master'

L2SS-624: Boot sequence & error reporting updates

Closes L2SS-624

See merge request !247

tangostationcontrol/docs/source/devices/recv.rst

@@ -13,3 +13,28 @@ The ``recv == DeviceProxy("STAT/RECV/1")`` device controls the RCUs, the LBA ant
 
 Typically, ``N_RCUs == 32``, and ``N_antennas_per_RCU == 3``.
 
+Error information
+---------------------
+
+These attributes summarise the basic state of the device. Any elements which are not present in ``FPGA_mask_RW`` will be ignored and thus not report errors:
+
+:RCU_error_R: Whether the RCUs appear usable.
+
+  :type: ``bool[N_RCUs]``
+
+:ANT_error_R: Whether the antennas appear usable.
+
+  :type: ``bool[N_RCUs][N_antennas_per_RCU]``
+
+:RCU_IOUT_error_R: Whether there are alarms on any of the amplitudes in the measured currents.
+
+  :type: ``bool[N_RCUs]``
+
+:RCU_VOUT_error_R: Whether there are alarms on any of the voltages in the measured currents.
+
+  :type: ``bool[N_RCUs]``
+
+:RCU_TEMP_error_R: Whether there are alarms on any of the temperatures. NB: These values are also exposed for unused RCUs (the ``RCU_mask_RW`` is ignored).
+
+  :type: ``bool[N_RCUs]``
+

tangostationcontrol/docs/source/devices/sdp.rst

@@ -30,16 +30,29 @@ The following points are significant for the operations of this device:
 Data-quality information
 ---------------------------
 
-The following fields describe the data quality:
+The following fields describe the data quality (see also :doc:`../signal_chain`):
 
 :FPGA_signal_input_mean_R: Mean value of the last second of input (in ADC quantisation units). Should be close to 0.
 
   :type: ``double[N_fpgas][N_ants_per_fpga]``
 
-:FPGA_signal_input_rms_R: Root means square value of the last second of input (in ADC quantisation units). ``rms^2 = mean^2 + std^2``. Values above 2048 indicate strong RFI.
+:FPGA_signal_input_rms_R: Root means square value of the last second of input (in ADC quantisation units). ``rms^2 = mean^2 + std^2``. Values above 2048 indicate strong RFI. Values of 0 indicate a lack of signal input.
 
   :type: ``double[N_fpgas][N_ants_per_fpga]``
 
+Error information
+---------------------
+
+These attributes summarise the basic state of the device. Any elements which are not present in ``FPGA_mask_RW`` will be ignored and thus not report errors:
+
+:FPGA_error_R: Whether the FPGAs appear usable.
+
+  :type: ``bool[N_fpgas]``
+
+:FPGA_procesing_error_R: Whether the FPGAs are processing their input from the RCUs. NB: This will also raise an error if the Waveform Generator is enabled.
+
+  :type: ``bool[N_fpgas]``
+
 Version Information
 ---------------------
 

tangostationcontrol/docs/source/devices/using.rst

@@ -59,13 +59,12 @@ typically involves the following sequence of commands::
   # turn the device off completely first.
   device.off()
 
-  # setup any connections and threads
-  device.initialise()
-
-  # turn on the device
-  device.on()
+  # turn on the device and fully reinitialise it
+  # alternatively, device.warm_boot() can be used,
+  # in which case no hardware is reinitialised.
+  device.boot()
 
-Of course, the device could go into ``FAULT`` again, even during the ``initialise()`` command, for example because the hardware it manages is unreachable. To debug the fault condition, check the :doc:`../interfaces/logs` of the device in question.
+Of course, the device could go into ``FAULT`` again, even during the ``boot()`` command, for example because the hardware it manages is unreachable. To debug the fault condition, check the :doc:`../interfaces/logs` of the device in question.
 
 Initialise hardware
 ````````````````````
@@ -74,6 +73,10 @@ Most devices provide the following commands, in order to configure the hardware
 
 :initialise(): Initialise the device (connect to the hardware). Moves from ``OFF`` to ``STANDBY``.
 
+:set_translator_defaults(): Select the hardware to configure and monitor.
+
+:prepare_hardware(): For devices that control hardware, this command prepares the hardware to accept commands (f.e. load firmware).
+
 :set_defaults(): Upload default attribute settings from the TangoDB to the hardware.
 
 :initialise_hardware(): For devices that control hardware, this command runs the hardware initialisation procedure.
@@ -145,4 +148,3 @@ For example, the ``RCU_mask_RW`` array is the RCU mask in the ``recv`` device. I
   # <--- only LED0 on RCU3 is now on
   # recv.RCU_LED0_R should show this,
   # if you have the RCU hardware installed.
-

tangostationcontrol/tangostationcontrol/devices/lofar_device.py

@@ -62,7 +62,10 @@ class lofar_device(Device, metaclass=DeviceMeta):
 
     version_R = attribute(dtype=str, access=AttrWriteType.READ, fget=lambda self: get_version())
 
-    # list of property names too be set first by set_defaults
+    # list of translator property names to be set by set_translator_defaults
+    translator_default_settings = []
+
+    # list of hardware property names to be set first by set_defaults
     first_default_settings = []
 
     @classmethod
@@ -224,6 +227,27 @@ class lofar_device(Device, metaclass=DeviceMeta):
         """Method always executed before any TANGO command is executed."""
         pass
 
+    def _set_defaults(self, attribute_names: list):
+        """ Set hardware points to their default value.
+
+            attribute_names: The names of the attributes to set to their default value.
+
+            A hardware point XXX is set to the value of the object member named XXX_default, if it exists.
+            XXX_default can be f.e. a constant, or a device_property.
+        """
+
+        # set them all
+        for name in attribute_names:
+            try:
+                default_value = getattr(self, f"{name}_default")
+
+                # set the attribute to the configured default
+                logger.debug(f"Setting attribute {name} to {default_value}")
+                self.proxy.write_attribute(name, default_value)
+            except Exception as e:
+                # log which attribute we're addressing
+                raise Exception(f"Cannot assign default to attribute {name}") from e
+
     def properties_changed(self):
         pass
 
@@ -240,7 +264,7 @@ class lofar_device(Device, metaclass=DeviceMeta):
             The points are set in the following order:
                 1) The python class property 'first_default_settings' is read, as an array of strings denoting property names. Each property
                    is set in that order.
-                2) Any remaining default properties are set.
+                2) Any remaining default properties are set, except the translators (those in 'translator_default_settings').
         """
 
         # collect all attributes for which defaults are provided
@@ -248,22 +272,33 @@ class lofar_device(Device, metaclass=DeviceMeta):
                                     # collect all attribute members
                                     if isinstance(getattr(self, name), Attribute)
                                     # with a default set
-                                    and hasattr(self, f"{name}_default")]
+                                    and hasattr(self, f"{name}_default")
+                                    and name not in self.translator_default_settings]
 
         # determine the order: first do the ones mentioned in default_settings_order
         attributes_to_set = self.first_default_settings + [name for name in attributes_with_defaults if name not in self.first_default_settings]
 
-        # set them all
-        for name in attributes_to_set:
-            try:
-                default_value = getattr(self, f"{name}_default")
+        # set them
+        self._set_defaults(attributes_to_set)
 
-                # set the attribute to the configured default
-                logger.debug(f"Setting attribute {name} to {default_value}")
-                self.proxy.write_attribute(name, default_value)
-            except Exception as e:
-                # log which attribute we're addressing
-                raise Exception(f"Cannot assign default to attribute {name}") from e
+
+    @only_in_states([DevState.STANDBY, DevState.INIT, DevState.ON])
+    @fault_on_error()
+    @command()
+    def set_translator_defaults(self):
+        """ Initialise the translator translators to their configured settings. """
+
+        # This is just the command version of _set_translator_defaults().
+        self._set_translator_defaults()
+
+    @only_in_states([DevState.STANDBY, DevState.INIT, DevState.ON])
+    @fault_on_error()
+    @command()
+    def prepare_hardware(self):
+        """ Load firmware required before configuring anything. """
+
+        # This is just the command version of _prepare_hardware().
+        self._prepare_hardware()
 
     @only_in_states([DevState.STANDBY, DevState.INIT, DevState.ON])
     @fault_on_error()
@@ -278,7 +313,12 @@ class lofar_device(Device, metaclass=DeviceMeta):
         # setup connections
         self.Initialise()
 
+        self.set_translator_defaults()
+
         if initialise_hardware:
+            # prepare hardware to accept settings
+            self.prepare_hardware()
+
             # initialise settings
             self.set_defaults()
 
@@ -288,7 +328,6 @@ class lofar_device(Device, metaclass=DeviceMeta):
         # make device available
         self.On()
 
-
     @only_in_states([DevState.OFF])
     @command()
     def boot(self):
@@ -299,6 +338,15 @@ class lofar_device(Device, metaclass=DeviceMeta):
     def warm_boot(self):
         self._boot(initialise_hardware=False)
 
+    def _set_translator_defaults(self):
+        """ Initialise any translators to their default settings. """
+
+        self._set_defaults(self.translator_default_settings)
+
+    def _prepare_hardware(self):
+        """ Override this method to load any firmware before configuring the hardware. """
+        pass
+
     def _initialise_hardware(self):
         """ Override this method to initialise any hardware after configuring it. """
         pass
@@ -309,13 +357,21 @@ class lofar_device(Device, metaclass=DeviceMeta):
 
             Raises an Exception if it has not after the timeout.
 
+            value:       The value that needs to be matched, or a function
+                         that needs to evaluate to True given the attribute.
             timeout:     time until an Exception is raised, in seconds.
             pollperiod:  how often to check the attribute, in seconds.
         """
+        if type(value) == type(lambda x: True):
+            # evaluate function
+            is_correct = value
+        else:
+            # compare to value
+            is_correct = lambda x: x == value
 
         # Poll every half a second
         for _ in range(math.ceil(timeout/pollperiod)):
-            if getattr(self.proxy, attr_name) != value:
+            if is_correct(getattr(self.proxy, attr_name)):
                 return
 
             time.sleep(pollperiod)
@@ -335,7 +391,7 @@ class lofar_device(Device, metaclass=DeviceMeta):
         # fetch attribute value as an array
         value = self.proxy.read_attribute(attr_name).value
         if is_scalar:
-            value = numpy.array(value)
+            value = numpy.array(value) # this stays a scalar in numpy
 
         # construct alarm state, in the same shape as the attribute
         alarm_state = numpy.zeros(value.shape, dtype=bool)
@@ -346,9 +402,6 @@ class lofar_device(Device, metaclass=DeviceMeta):
         if alarms.min_alarm != 'Not specified':
             alarm_state |= value <= value.dtype.type(alarms.min_alarm)
 
-        # return alarm state, as the same type as the attribute
-        if is_scalar:
-            return alarm_state[0]
-        else:
+        # return alarm state, with the same dimensions as the attribute
         return alarm_state
 

tangostationcontrol/tangostationcontrol/devices/recv.py

@@ -109,8 +109,7 @@ class RECV(opcua_device):
         default_value = 2015.5
     )
 
-    first_default_settings = [
-        # set the masks first, as those filter any subsequent settings
+    translator_default_settings = [
         'ANT_mask_RW',
         'RCU_mask_RW'
     ]

tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py

@@ -89,8 +89,7 @@ class SDP(opcua_device):
         default_value=[[8192] * 12 * 512] * 16
     )
 
-    first_default_settings = [
-        # set the masks first, as those filter any subsequent settings
+    translator_default_settings = [
         'TR_fpga_mask_RW'
     ]
 
@@ -109,8 +108,8 @@ class SDP(opcua_device):
     FPGA_beamlet_output_scale_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_scale_R"], datatype=numpy.uint32, dims=(16,))
     FPGA_beamlet_output_scale_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_scale_RW"], datatype=numpy.uint32, dims=(16,), access=AttrWriteType.READ_WRITE)
     FPGA_firmware_version_R = attribute_wrapper(comms_annotation=["FPGA_firmware_version_R"], datatype=numpy.str, dims=(16,))
-    FPGA_reboot_R = attribute_wrapper(comms_annotation=["FPGA_reboot_R"], datatype=numpy.uint32, dims=(16,), doc="Active FPGA image (0=factory, 1=user)")
-    FPGA_reboot_RW = attribute_wrapper(comms_annotation=["FPGA_reboot_R"], datatype=numpy.uint32, dims=(16,), access=AttrWriteType.READ_WRITE)
+    FPGA_boot_image_R = attribute_wrapper(comms_annotation=["FPGA_reboot_R"], datatype=numpy.uint32, dims=(16,), doc="Active FPGA image (0=factory, 1=user)")
+    FPGA_boot_image_RW = attribute_wrapper(comms_annotation=["FPGA_reboot_R"], datatype=numpy.uint32, dims=(16,), access=AttrWriteType.READ_WRITE)
     FPGA_global_node_index_R = attribute_wrapper(comms_annotation=["FPGA_global_node_index_R"], datatype=numpy.uint32, dims=(16,))
     FPGA_hardware_version_R = attribute_wrapper(comms_annotation=["FPGA_hardware_version_R"], datatype=numpy.str, dims=(16,))
     FPGA_processing_enable_R = attribute_wrapper(comms_annotation=["FPGA_processing_enable_R"], datatype=numpy.bool_, dims=(16,))
@@ -223,14 +222,23 @@ class SDP(opcua_device):
     def read_FPGA_processing_error_R(self):
         return self.proxy.TR_fpga_mask_RW & (
                  ~self.proxy.FPGA_processing_enable_R
-               | (self.proxy.FPGA_reboot_R == 0)
-               | ~self.proxy.FPGA_wg_enable_R.any(axis=1)
+               | (self.proxy.FPGA_boot_image_R == 0)
+               | self.proxy.FPGA_wg_enable_R.any(axis=1)
+               | (self.proxy.FPGA_signal_input_rms_R == 0)
                )
 
     # --------
     # overloaded functions
     # --------
 
+    def _prepare_hardware(self):
+        # FPGAs need the correct firmware loaded 
+        self.FPGA_boot_image_RW = [1] * N_pn
+
+        # wait for the firmware to be loaded (ignoring masked out elements)
+        mask = self.proxy.TR_fpga_mask_RW
+        self.wait_attribute("FPGA_boot_image_R", lambda attr: (attr == 1) | ~mask, 10)
+
     # --------
     # Commands
     # --------

tangostationcontrol/tangostationcontrol/devices/unb2.py

@@ -59,8 +59,7 @@ class UNB2(opcua_device):
         default_value=[True] * 2
     )
 
-    first_default_settings = [
-        # set the masks first, as those filter any subsequent settings
+    translator_default_settings = [
         'UNB2_mask_RW'
     ]