diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 010b73887fcb75a74f0f3782762639c6a720dac1..fc6db953c0aae40d87013784890a73848acadfa3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,7 +115,6 @@ docker_build_image: parallel: matrix: - IMAGE: - - lofar-device-base - ec-sim - http-json-schemas - prometheus @@ -137,6 +136,15 @@ docker_build_image: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh $IMAGE $tag +docker_build_image_device_base: + extends: .base_docker_images + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG + script: + # Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag + newline_at_eof: stage: linting before_script: diff --git a/docker-compose/jupyter-lab/requirements.txt b/docker-compose/jupyter-lab/requirements.txt index eba971cf6e501feeb8aff045dd7ea49ceaf528db..c3e82e317c170d8c79d5e14efc8aff8894a87aef 100644 --- a/docker-compose/jupyter-lab/requirements.txt +++ b/docker-compose/jupyter-lab/requirements.txt @@ -1,3 +1,4 @@ +# Jupyter & enhancements ipython >=7.27.0,!=7.28.0 # BSD jupyter jupyterlab >=3,<4 # until https://github.com/jupyterlab/jupyterlab-git/issues/1245 @@ -5,26 +6,32 @@ jupyterlab_h5web[full] # MIT jupyterlab-git jupyterlab-skip-traceback ipykernel -jupyter_bokeh -matplotlib -jupyterplot nbconvert notebook-as-pdf PyPDF2==2.12.1 # until https://github.com/betatim/notebook-as-pdf/issues/40 hits a notebook-as-pdf release -python-logstash-async -PyMySQL[rsa] -psycopg2-binary >= 2.9.2 #LGPL + +# low-level access to station components +opcua +asyncua >= 0.9.90 # LGPLv3 pyvisa pyvisa-py -opcua + +# antenna locations lofarantpos >= 0.5.0 # Apache 2 python-geohash >= 0.8.5 # Apache 2 / MIT -asyncua >= 0.9.90 # LGPLv3 +etrs-itrs@git+https://github.com/brentjens/etrs-itrs # Apache 2 -numpy -scipy +# plotting +matplotlib +jupyter_bokeh +jupyterplot -pabeam@git+https://git.astron.nl/mevius/grate # Apache2 +# useful LOFAR software +pabeam@git+https://git.astron.nl/mevius/pabeam # Apache2 lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client # Apache2 attributewrapper@git+https://git.astron.nl/lofar2.0/attributewrapper # Apache2 -etrs-itrs@git+https://github.com/brentjens/etrs-itrs # Apache 2 + +# user packages +numpy +scipy +astropy diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index e39c081533b1edaaaf45f3cbb4f712f243877452..600a5b219fdda78938ba93908537d5aeeca4eecf 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -45,7 +45,7 @@ function integration_test { } # list of arguments expected in the input -optstring_long="help,no-build,preserve,save-logs" +optstring_long="help,no-build,skip-tests,preserve,save-logs" optstring="hnb" options=$(getopt -l ${optstring_long} -o ${optstring} -- "$@") @@ -63,6 +63,10 @@ while true; do export no_build=1 export NO_BASE=${no_build} ;; + --skip-tests) + echo "Only setup and configure environment don't run any tests" + export no_tests=1 + ;; --) shift break;; @@ -209,6 +213,10 @@ make start "${DEVICES[@]}" # Wait for devices to restart make await "${DEVICES[@]}" +if [ -n "${no_tests}" ]; then + exit 0 +fi + # Start the integration test cd "$LOFAR20_DIR/docker-compose" || exit 1 make up integration-test diff --git a/tangostationcontrol/integration_test/default/devices/test_device_calibration.py b/tangostationcontrol/integration_test/default/devices/test_device_calibration.py index 3d9329525be4078999fd0d03f89f0b539b09d890..fd0fdad70fdb93fc8927a6275e43e3dac164a184 100644 --- a/tangostationcontrol/integration_test/default/devices/test_device_calibration.py +++ b/tangostationcontrol/integration_test/default/devices/test_device_calibration.py @@ -153,16 +153,17 @@ class TestCalibrationDevice(AbstractTestBases.TestDeviceBase): [[1, x + DEFAULT_N_HBA_TILES] for x in range(0, DEFAULT_N_HBA_TILES)] ).flatten(), # [1, 48, 1, 49, x ... 1, 95] + "Frequency_Band_RW_default": ["HBA_110_190"] * (DEFAULT_N_HBA_TILES * 2), } - self.antennafield_proxy = self.setup_proxy("STAT/AntennaField/LBA") + self.antennafield_proxy = self.setup_proxy("STAT/AntennaField/HBA0") self.antennafield_proxy.put_property(calibration_properties) self.antennafield_proxy.boot() self.proxy.boot() # calibrate - self.proxy.calibrate_recv("STAT/AntennaField/LBA") + self.proxy.calibrate_recv("STAT/AntennaField/HBA0") # check the results rcu_attenuator_db_pwr = self.antennafield_proxy.RCU_attenuator_dB_RW[:, 0] @@ -213,6 +214,7 @@ class TestCalibrationDevice(AbstractTestBases.TestDeviceBase): [[1, x + DEFAULT_N_HBA_TILES] for x in range(0, DEFAULT_N_HBA_TILES)] ).flatten(), # [1, 48, 1, 49, x ... 1, 95] + "Frequency_Band_RW_default": ["HBA_110_190"] * (DEFAULT_N_HBA_TILES * 2), } self.antennafield_proxy = self.setup_proxy("STAT/AntennaField/HBA0") diff --git a/tangostationcontrol/tangostationcontrol/devices/base_device_classes/mapper.py b/tangostationcontrol/tangostationcontrol/devices/base_device_classes/mapper.py index 9e3bd89d0431a118416866d2ca014c4284008469..56d91572f08b150d66e46f9d5d5582af402885ac 100644 --- a/tangostationcontrol/tangostationcontrol/devices/base_device_classes/mapper.py +++ b/tangostationcontrol/tangostationcontrol/devices/base_device_classes/mapper.py @@ -260,7 +260,10 @@ class AntennaMapper: ) in enumerate(zip(power_mapping, control_mapping)): if antenna_type == "LBA" and pwr_attribute: if dev_power > 0 and dev_control > 0: - mapped_values[dev_power] = set_values[idx] + mapped_values[dev_power - 1, dev_input_power] = set_values[idx] + mapped_values[dev_control - 1, dev_input_control] = set_values[ + idx + ] else: if dev_power > 0: mapped_values[dev_power - 1, dev_input_power] = set_values[idx][ diff --git a/tangostationcontrol/tangostationcontrol/devices/base_device_classes/power_hierarchy.py b/tangostationcontrol/tangostationcontrol/devices/base_device_classes/power_hierarchy.py index bc1810c30cae64824962ecb41485d36ac5ae57aa..2fb996242bde7eb0c97dd50ff2d0e3cf41733a40 100644 --- a/tangostationcontrol/tangostationcontrol/devices/base_device_classes/power_hierarchy.py +++ b/tangostationcontrol/tangostationcontrol/devices/base_device_classes/power_hierarchy.py @@ -41,9 +41,6 @@ class PowerHierarchyDevice(AbstractHierarchyDevice): def _boot_device(self, device: DeviceProxy): """Default sequence of device booting operations""" - if device.state() == DevState.ON: - logger.info(f"Booting {device}: Succesful: It's already ON?") - return logger.info(f"Booting {device}: off()") device.off() diff --git a/tangostationcontrol/tangostationcontrol/devices/base_device_classes/recv_device.py b/tangostationcontrol/tangostationcontrol/devices/base_device_classes/recv_device.py index b8e6ad462ad19ce2db7af798bf0af2fa682b317e..1860aea67773fbd446cd11dc4ec2b44d6085521f 100644 --- a/tangostationcontrol/tangostationcontrol/devices/base_device_classes/recv_device.py +++ b/tangostationcontrol/tangostationcontrol/devices/base_device_classes/recv_device.py @@ -100,7 +100,7 @@ class RECVDevice(OPCUADevice): doc="Maximum amount of time to wait after turning RCU(s) on or off", dtype="DevFloat", mandatory=False, - default_value=30.0, + default_value=60.0, ) RCU_DTH_On_Off_timeout = device_property( @@ -388,10 +388,10 @@ class RECVDevice(OPCUADevice): self.RCU_on() self.wait_attribute("RECVTR_translator_busy_R", False, self.RCU_On_Off_timeout) - self.RCU_DTH_off() - self.wait_attribute( - "RECVTR_translator_busy_R", False, self.RCU_DTH_On_Off_timeout - ) + + # NB: Powering on RCUs causes DTH to be turned off, which is what we want + # to create a steady baseline after powerup. This is done by RECVTR + # even if the RCUs are already powered on. def _power_hardware_off(self): """Turns off the RCUs.""" diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/bst.py b/tangostationcontrol/tangostationcontrol/devices/sdp/bst.py index 0614ceabf03f768f2dd5d46fcdf987c9869337d4..621c3ef6e4a74128b4112f863e54bbe0d67e5ad5 100644 --- a/tangostationcontrol/tangostationcontrol/devices/sdp/bst.py +++ b/tangostationcontrol/tangostationcontrol/devices/sdp/bst.py @@ -162,7 +162,7 @@ class BST(Statistics): comms_id=StatisticsClient, comms_annotation={"type": "statistics", "parameter": "bst_timestamps"}, dims=(N_pn,), - datatype=numpy.float32, + datatype=numpy.uint64, ) # reported integration interval # from each FPGA in the latest BSTs diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/firmware.py b/tangostationcontrol/tangostationcontrol/devices/sdp/firmware.py index 866529f4851d940ef1578d60d5eaceede4015ecd..37c87ccf3abda72d329ca956b974b46f93c090e5 100644 --- a/tangostationcontrol/tangostationcontrol/devices/sdp/firmware.py +++ b/tangostationcontrol/tangostationcontrol/devices/sdp/firmware.py @@ -296,12 +296,12 @@ class SDPFirmware(OPCUADevice): def _power_hardware_on(self): """Boot the SDP Firmware user image""" - self._boot_to_image(0) + self._boot_to_image(1) def _power_hardware_off(self): """Use the SDP Firmware factory image""" - self._boot_to_image(1) + self._boot_to_image(0) # -------- # Commands diff --git a/tangostationcontrol/tangostationcontrol/devices/station_manager.py b/tangostationcontrol/tangostationcontrol/devices/station_manager.py index 987d7ea247255f05f6c5ef73153b88478bb3590e..dc5d6c3f2cb3aa2432903e04bcdf7c80cd787da7 100644 --- a/tangostationcontrol/tangostationcontrol/devices/station_manager.py +++ b/tangostationcontrol/tangostationcontrol/devices/station_manager.py @@ -195,6 +195,9 @@ class StationManager(LOFARDevice): Switch the station into OFF state. It can only be executed from state HIBERNATE. """ + if self.station_state == StationState.OFF: + return + if not self._is_transition_allowed(StationState.OFF): raise Exception(f"Station did not transition to {StationState.OFF.name}") @@ -214,6 +217,9 @@ class StationManager(LOFARDevice): Switch the station into HIBERNATE state. It can only be executed from either state OFF or STANDBY. """ + if self.station_state == StationState.HIBERNATE: + return + if not self._is_transition_allowed(StationState.HIBERNATE): raise Exception( f"Station did not transition to {StationState.HIBERNATE.name}" @@ -247,6 +253,9 @@ class StationManager(LOFARDevice): Switch the station into STANDBY state. It can only be executed from either state HIBERNATE or ON. """ + if self.station_state == StationState.STANDBY: + return + if not self._is_transition_allowed(StationState.STANDBY): raise Exception( f"Station did not transition to {StationState.STANDBY.name}" @@ -278,6 +287,9 @@ class StationManager(LOFARDevice): Switch the station into ON state. It can only be executed from state STANDBY. """ + if self.station_state == StationState.ON: + return + if not self._is_transition_allowed(StationState.ON): raise Exception(f"Station did not transition to {StationState.ON.name}")