diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8b1a90b70cc2a34955ca4d2c0588ec323da8a9af..82cbc3f552989949dbe2d5ac79cdd47b83152971 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -97,6 +97,8 @@ docker_build_image_all:
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki latest
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh ccd-sim latest
@@ -187,6 +189,28 @@ docker_build_image_grafana:
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
+docker_build_image_loki:
+ extends: .base_docker_images_except
+ only:
+ refs:
+ - merge_requests
+ changes:
+ - docker-compose/loki.yml
+ - docker-compose/loki/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki $tag
+docker_build_image_logstash:
+ extends: .base_docker_images_except
+ only:
+ refs:
+ - merge_requests
+ changes:
+ - docker-compose/logstash.yml
+ - docker-compose/logstash/*
+ script:
+# Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash $tag
docker_build_image_jupyter:
extends: .base_docker_images_except
only:
diff --git a/CDB/LOFAR_ConfigDb.json b/CDB/LOFAR_ConfigDb.json
index dcf8ce83abb4ddba759be6340bbe84a93c783505..b2d2f517e83667a1659c39d091f73bc2580df01f 100644
--- a/CDB/LOFAR_ConfigDb.json
+++ b/CDB/LOFAR_ConfigDb.json
@@ -113,6 +113,60 @@
"Beamlet": {
"STAT/Beamlet/1": {
"properties": {
+ "FPGA_beamlet_output_hdr_eth_source_mac_RW_default": [
+ "00:22:86:08:00:00",
+ "00:22:86:08:00:01",
+ "00:22:86:08:00:02",
+ "00:22:86:08:00:03",
+ "00:22:86:08:01:00",
+ "00:22:86:08:01:01",
+ "00:22:86:08:01:02",
+ "00:22:86:08:01:03",
+ "00:22:86:08:02:00",
+ "00:22:86:08:02:01",
+ "00:22:86:08:02:02",
+ "00:22:86:08:02:03",
+ "00:22:86:08:03:00",
+ "00:22:86:08:03:01",
+ "00:22:86:08:03:02",
+ "00:22:86:08:03:03"
+ ],
+ "FPGA_beamlet_output_hdr_ip_source_address_RW_default": [
+ "192.168.0.1",
+ "192.168.0.2",
+ "192.168.0.3",
+ "192.168.0.4",
+ "192.168.1.1",
+ "192.168.1.2",
+ "192.168.1.3",
+ "192.168.1.4",
+ "192.168.2.1",
+ "192.168.2.2",
+ "192.168.2.3",
+ "192.168.2.4",
+ "192.168.3.1",
+ "192.168.3.2",
+ "192.168.3.3",
+ "192.168.3.4"
+ ],
+ "FPGA_beamlet_output_hdr_udp_source_port_RW_default": [
+ "53248",
+ "53249",
+ "53250",
+ "53251",
+ "53252",
+ "53253",
+ "53254",
+ "53255",
+ "53256",
+ "53257",
+ "53258",
+ "53259",
+ "53260",
+ "53261",
+ "53262",
+ "53263"
+ ],
"FPGA_beamlet_output_hdr_udp_destination_port_RW_default": [
"10001",
"10001",
diff --git a/CDB/stations/CS001_ConfigDb.json b/CDB/stations/CS001_ConfigDb.json
index 12411a45994cd0cd9697190204cf1a13d37f3e6d..e078b8c7a9daf205a064a665a934bfb15671225b 100644
--- a/CDB/stations/CS001_ConfigDb.json
+++ b/CDB/stations/CS001_ConfigDb.json
@@ -126,6 +126,60 @@
"OPC_Time_Out": [
"5.0"
],
+ "FPGA_beamlet_output_hdr_eth_source_mac_RW_default": [
+ "00:22:86:00:01:00",
+ "00:22:86:00:01:01",
+ "00:22:86:00:01:02",
+ "00:22:86:00:01:03",
+ "00:22:86:00:01:04",
+ "00:22:86:00:01:05",
+ "00:22:86:00:01:06",
+ "00:22:86:00:01:07",
+ "00:22:86:00:01:08",
+ "00:22:86:00:01:09",
+ "00:22:86:00:01:10",
+ "00:22:86:00:01:11",
+ "00:22:86:00:01:12",
+ "00:22:86:00:01:13",
+ "00:22:86:00:01:14",
+ "00:22:86:00:01:15"
+ ],
+ "FPGA_beamlet_output_hdr_ip_source_address_RW_default": [
+ "10.175.1.0",
+ "10.175.1.1",
+ "10.175.1.2",
+ "10.175.1.3",
+ "10.175.1.4",
+ "10.175.1.5",
+ "10.175.1.6",
+ "10.175.1.7",
+ "10.175.1.8",
+ "10.175.1.9",
+ "10.175.1.10",
+ "10.175.1.11",
+ "10.175.1.12",
+ "10.175.1.13",
+ "10.175.1.14",
+ "10.175.1.15"
+ ],
+ "FPGA_beamlet_output_hdr_udp_source_port_RW_default": [
+ "4346",
+ "4347",
+ "4348",
+ "4349",
+ "4350",
+ "4351",
+ "4352",
+ "4353",
+ "4354",
+ "4355",
+ "4356",
+ "4357",
+ "4358",
+ "4359",
+ "4360",
+ "4361"
+ ],
"FPGA_beamlet_output_hdr_eth_destination_mac_RW_default": [
"3c:ec:ef:86:2f:b7",
"3c:ec:ef:86:2f:b7",
@@ -220,8 +274,8 @@
],
"Antenna_to_SDP_Mapping": [
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
- "0", "-1", "0", "-1", "3", "0", "3", "1", "3", "2", "0", "-1",
- "0", "-1", "0", "-1", "3", "3", "3", "4", "3", "5", "0", "-1",
+ "0", "-1", "0", "-1", "2", "0", "2", "1", "2", "2", "0", "-1",
+ "0", "-1", "0", "-1", "2", "3", "2", "4", "2", "5", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
@@ -392,9 +446,9 @@
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "4", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "5",
- "0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "6", "1", "7",
+ "0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "0", "1", "1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
- "0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "8", "0", "-1"
+ "0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "2", "0", "-1"
],
"Antenna_Field_Reference_ETRS": [
"3826923.942", "460915.117", "5064643.229"
diff --git a/CDB/stations/DTS_ConfigDb.json b/CDB/stations/DTS_ConfigDb.json
index 21941beaa1056f2a28360b9f055e91504bedd3ef..1263040256ab94d74f87bdbbd9d593aa3cbda701 100644
--- a/CDB/stations/DTS_ConfigDb.json
+++ b/CDB/stations/DTS_ConfigDb.json
@@ -102,40 +102,40 @@
"5.0"
],
"FPGA_beamlet_output_hdr_eth_destination_mac_RW_default": [
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1",
- "0c:c4:7a:c0:30:f1"
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc",
+ "ec:0d:9a:bf:f2:dc"
],
"FPGA_beamlet_output_hdr_ip_destination_address_RW_default": [
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250",
- "10.99.250.250"
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250",
+ "192.168.1.250"
]
}
}
@@ -147,6 +147,9 @@
"AntennaField": {
"STAT/AntennaField/1": {
"properties": {
+ "Antenna_Type": [
+ "HBA"
+ ],
"RECV_devices": [
"STAT/RECV/1"
],
@@ -164,6 +167,13 @@
"1", "28",
"1", "29"
],
+ "Antenna_to_SDP_Mapping": [
+ "0", "0",
+ "0", "1",
+ "0", "2",
+ "0", "3",
+ "0", "4"
+ ],
"Antenna_Field_Reference_ETRS": [
"3839371.416", "430339.901", "5057958.886"
],
@@ -196,24 +206,6 @@
"DigitalBeam": {
"STAT/DigitalBeam/1": {
"properties": {
- "Input_to_Antenna_Mapping": [
- "0", "1", "2", "3", "4", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1"
- ]
}
}
}
diff --git a/CDB/stations/DTS_Outside_ConfigDb.json b/CDB/stations/DTS_Outside_ConfigDb.json
index c2ed04867ab540dc1ca913aac6680f8bd2019dbf..ce52a70b9094cfa87bca8d3fd03d3cc595145898 100644
--- a/CDB/stations/DTS_Outside_ConfigDb.json
+++ b/CDB/stations/DTS_Outside_ConfigDb.json
@@ -172,6 +172,9 @@
"AntennaField": {
"STAT/AntennaField/2": {
"properties": {
+ "Antenna_Type": [
+ "HBA"
+ ],
"RECV_devices": [
"STAT/RECV/1"
],
@@ -196,6 +199,13 @@
"1","25",
"1","26"
],
+ "Antenna_to_SDP_Mapping": [
+ "3", "0",
+ "0", "-1",
+ "0", "-1",
+ "3", "1",
+ "3", "2"
+ ],
"Antenna_Field_Reference_ETRS": [
"3839371.416","430339.901","5057958.886"
],
@@ -222,6 +232,9 @@
},
"STAT/AntennaField/1": {
"properties": {
+ "Antenna_Type": [
+ "LBA"
+ ],
"RECV_devices": [
"STAT/RECV/1"
],
@@ -258,6 +271,17 @@
"1","15",
"1","17"
],
+ "Antenna_to_SDP_Mapping": [
+ "0", "0",
+ "0", "1",
+ "0", "2",
+ "0", "3",
+ "0", "4",
+ "0", "5",
+ "1", "0",
+ "1", "1",
+ "1", "2"
+ ],
"Antenna_Field_Reference_ETRS": [
"3839358.189",
"430354.482",
@@ -286,24 +310,6 @@
"properties": {
"AntennaField_Device": [
"STAT/AntennaField/2"
- ],
- "Input_to_Antenna_Mapping": [
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "0", "3", "4", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1"
]
}
},
@@ -312,24 +318,6 @@
"AntennaField_Device": [
"STAT/AntennaField/1"
],
- "Input_to_Antenna_Mapping": [
- "0", "1", "2", "3", "4", "5",
- "6", "7", "8", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1"
- ],
"Tracking_enabled_RW_default": [
"False"
]
@@ -362,9 +350,6 @@
"SDP": {
"STAT/SDP/1": {
"properties": {
- "AntennaType": [
- "LBA"
- ],
"OPC_Server_Name": [
"10.99.0.250"
],
diff --git a/CDB/stations/dummy_positions_ConfigDb.json b/CDB/stations/dummy_positions_ConfigDb.json
index 7403bc8bb43ae1c4b9495b12bba38f8bf6e99a9d..6a293343b622d365a4cde0eb84b5aa2c18a55360 100644
--- a/CDB/stations/dummy_positions_ConfigDb.json
+++ b/CDB/stations/dummy_positions_ConfigDb.json
@@ -105,6 +105,16 @@
"1", "94",
"1", "95"
],
+ "Antenna_to_SDP_Mapping": [
+ "0", "0", "0", "1", "0", "2", "0", "3", "0", "4", "0", "5",
+ "1", "0", "1", "1", "1", "2", "1", "3", "1", "4", "1", "5",
+ "2", "0", "2", "1", "2", "2", "2", "3", "2", "4", "2", "5",
+ "3", "0", "3", "1", "3", "2", "3", "3", "3", "4", "3", "5",
+ "4", "0", "4", "1", "4", "2", "4", "3", "4", "4", "4", "5",
+ "5", "0", "5", "1", "5", "2", "5", "3", "5", "4", "5", "5",
+ "6", "0", "6", "1", "6", "2", "6", "3", "6", "4", "6", "5",
+ "7", "0", "7", "1", "7", "2", "7", "3", "7", "4", "7", "5"
+ ],
"Antenna_Field_Reference_ETRS": [
"3826896.631", "460979.131", "5064657.943"
],
@@ -223,24 +233,6 @@
"DigitalBeam": {
"STAT/DigitalBeam/1": {
"properties": {
- "Input_to_Antenna_Mapping": [
- "0", "1", "2", "3", "4", "5",
- "6", "7", "8", "9", "10", "11",
- "12", "13", "14", "15", "16", "17",
- "18", "19", "20", "21", "22", "23",
- "24", "25", "26", "27", "28", "29",
- "30", "31", "32", "33", "34", "35",
- "36", "37", "38", "39", "40", "41",
- "42", "43", "44", "45", "46", "47",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1"
- ]
}
}
}
diff --git a/README.md b/README.md
index c72d120f54c46d5bf9eba273863b4572de9595ff..57ed52816e57541249ed224d5a5d2260c469c2e4 100644
--- a/README.md
+++ b/README.md
@@ -8,11 +8,16 @@ Station Control software related to Tango devices.
# Index
+* [Installation](#installation)
+ * [Prerequisites](#prerequisites)
+ * [Bootstrap](#bootstrap)
+* [User documentation (ReadTheDocs (Sphinx / ReStructuredText))](tangostationcontrol/docs/README.md)
* [Docker compose documentation](docker-compose/README.md)
* [Timescaledb](docker-compose/timescaledb/README.md)
* [Jupyter startup files](docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md)
* [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter)
-* [ReadTheDocs (Sphinx / ReStructuredText) documentation](tangostationcontrol/docs/README.md)
+* [Developer Documentation](#development)
+ * [Versioning](#versioning)
* Source code documentation
* [Attribute wrapper documentation](tangostationcontrol/tangostationcontrol/clients/README.md)
* [Archiver documentation](tangostationcontrol/tangostationcontrol/toolkit/README.md)
@@ -20,6 +25,7 @@ Station Control software related to Tango devices.
* [HDF5 statistics](tangostationcontrol/tangostationcontrol/statistics/README.md)
* [Unit tests](tangostationcontrol/tangostationcontrol/test/README.md)
* [Integration tests](tangostationcontrol/tangostationcontrol/integration_test/README.md)
+* [Release Notes](#release-notes)
# Installation
@@ -41,20 +47,16 @@ You will also need:
## Bootstrap
-The bootstrap procedure is needed only once. First we build all docker containers, and load the initial configuration. This may take a while:
+The bootstrap procedure is needed only once. First we build all docker
+containers, and load the initial configuration. This may take a while:
```
cd docker-compose
make bootstrap
```
-If you lack access to LOFAR station hardware, configure the devices to use their simulators instead:
-
-```
-for sim in ../CDB/*-sim-config.json; do
- ../sbin/update_ConfigDb.sh ../CDB${sim}-config.json
-done
-```
+By default bootstrap will configure the station to use simulators. You can
+lookup alternative configurations in the CDB directory.
Now we can start all containers, and make sure everything is up:
@@ -63,6 +65,44 @@ make start
make status
```
-If not, you can inspect why with `docker logs <container>`. The containers will automatically be restarted on reboot or failure. Stop them explicitly to bring them down (`make stop <container>`).
+If not, you can inspect why with `docker logs <container>`. The containers will
+automatically be restarted on reboot or failure. Stop them explicitly to bring
+them down (`make stop <container>`).
+
+Most notably, you will have web interfaces available at:
+ - http://localhost:8888 (Jupyter Notebook)
+ - http://localhost:3000 (Grafana).
+
+# Development
+
+For development you will need several dependencies including:
+
+```
+git g++ gcc make docker docker-compose shellcheck graphviz python3-dev \
+python3-pip python3-tox libboost-python-dev libtango-cpp pkg-config
+```
+
+Of these docker-compose must be at least 2.0 and Python 3.7 or higher.
+Alternatively, tox can be installed through pip using `pip install tox`.
+
+Finally, running unit tests relies on availability of casacore data see:
+[lofar-device-base Dockerfile](docker-compose/lofar-device-base/Dockerfile)
+for details.
+
+## Versioning
+
+When changing behavior a new version for Lofar Station Control should be
+reserved. To do this please follow [semantic versioning](https://semver.org/).
+
+Next change the version in the following places:
+
+1. The [VERSION](VERSION) file.
+2. In [test_writer_sst.py](tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py)
+ for the `test_header_info` test.
+3. Add a [Release note](#release-notes) for the given version.
+3. Once the merge requests is merged to master, add a tag with the version (just x.x.x not Vx.x.x)
+
+# Release Notes
-Most notably, you will have web interfaces available at http://localhost:8888 (Jupyter Notebook), and http://localhost:3000 (Grafana).
+* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter
+* 0.2.0 Extend `Beamlet` device with FPGA source address attributes
diff --git a/docker-compose/elk.yml b/docker-compose/elk.yml
index 786e843ce85c16e7604341a7138c5030f1356fed..d671ba0c7708d3ae9cb37956f9bc7884462f7389 100644
--- a/docker-compose/elk.yml
+++ b/docker-compose/elk.yml
@@ -44,10 +44,10 @@ services:
ports:
- "5601:5601" # kibana
- "9200:9200" # elasticsearch
- - "5044:5044" # logstash beats input
- - "1514:1514/tcp" # logstash syslog input
- - "1514:1514/udp" # logstash syslog input
- - "5959:5959" # logstash tcp json input
+ # - "5044:5044" # logstash beats input
+ # - "1514:1514/tcp" # logstash syslog input
+ # - "1514:1514/udp" # logstash syslog input
+ # - "5959:5959" # logstash tcp json input
depends_on:
- elk-configure-host
restart: unless-stopped
diff --git a/docker-compose/grafana/datasources/loki.yaml b/docker-compose/grafana/datasources/loki.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f9108f15f3791de72fb8c80cc24ae156e0bfea73
--- /dev/null
+++ b/docker-compose/grafana/datasources/loki.yaml
@@ -0,0 +1,45 @@
+apiVersion: 1
+
+datasources:
+ # <string, required> name of the datasource. Required
+ - name: Loki
+ # <string, required> datasource type. Required
+ type: loki
+ # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
+ access: proxy
+ # <int> org id. will default to orgId 1 if not specified
+ orgId: 1
+ # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
+ uid: loki
+ # <string> url
+ url: http://loki:3100
+ # <string> Deprecated, use secureJsonData.password
+ password:
+ # <string> database user, if used
+ user:
+ # <string> database name, if used
+ database:
+ # <bool> enable/disable basic auth
+ basicAuth: false
+ # <string> basic auth username
+ basicAuthUser:
+ # <string> Deprecated, use secureJsonData.basicAuthPassword
+ basicAuthPassword:
+ # <bool> enable/disable with credentials headers
+ withCredentials:
+ # <bool> mark as default datasource. Max one per org
+ isDefault: false
+ # <map> fields that will be converted to json and stored in jsonData
+ jsonData:
+ esVersion: 7.10.0
+ includeFrozen: false
+ logLevelField:
+ logMessageField:
+ maxConcurrentShardRequests: 5
+ timeField: "@timestamp"
+ # <string> json object of data that will be encrypted.
+ secureJsonData:
+ version: 1
+ # <bool> allow users to edit datasources from the UI.
+ editable: false
+
diff --git a/docker-compose/jupyter-lab.yml b/docker-compose/jupyter-lab.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2c5984dc6eac7327f0af29a9251e4834a7f9c2ff
--- /dev/null
+++ b/docker-compose/jupyter-lab.yml
@@ -0,0 +1,42 @@
+#
+# Docker compose file that launches Jupyter Lab for interactive iTango sessions over HTTP.
+#
+# Connect by surfing to http://localhost:8889/
+# View logs through 'docker logs -f -t jupyter-lab'
+#
+# Defines:
+# - jupyter-lab: Jupyter Lab with iTango support
+#
+
+version: '2.1'
+
+services:
+ jupyter-lab:
+ build:
+ context: jupyterlab
+ args:
+ CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
+ SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
+ container_name: ${CONTAINER_NAME_PREFIX}jupyter-lab
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ networks:
+ - control
+ volumes:
+ - ..:/opt/lofar/tango:rw
+ - ../jupyter-notebooks:/jupyter-notebooks:rw
+ - ${HOME}:/hosthome
+ - ${SCRATCH}:/scratch:rw
+ environment:
+ - TANGO_HOST=${TANGO_HOST}
+ ports:
+ - "8889:8889"
+ user: ${CONTAINER_EXECUTION_UID}
+ working_dir: /jupyter-notebooks
+ entrypoint:
+ - /opt/lofar/tango/bin/start-ds.sh
+ - jupyter lab --port=8889 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
+ restart: unless-stopped
diff --git a/docker-compose/jupyter/Dockerfile b/docker-compose/jupyter/Dockerfile
index ec2292a885abb5eaa8f6bba978fd0f4a7b32815d..abb6f8872b202f952f587165a17fb08856cc653c 100644
--- a/docker-compose/jupyter/Dockerfile
+++ b/docker-compose/jupyter/Dockerfile
@@ -13,6 +13,9 @@ RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
RUN sudo apt-get update -y
RUN sudo apt-get install -y g++ gcc python3-dev
+# start-ds file synchronization requirements
+RUN sudo apt-get install -y rsync
+
# Install git to install pip requirements from git
RUN sudo apt-get install -y git
diff --git a/docker-compose/jupyterlab/Dockerfile b/docker-compose/jupyterlab/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..448d9a7928ed879b285baf75e02a2c1604a433da
--- /dev/null
+++ b/docker-compose/jupyterlab/Dockerfile
@@ -0,0 +1,61 @@
+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# UID if the user that this container will run under. This is needed to give directories
+# that are needed for temporary storage the proper owner and access rights.
+ARG CONTAINER_EXECUTION_UID=1000
+
+# Create new user with uid but only if uid not used
+RUN sudo adduser --disabled-password --system --uid ${CONTAINER_EXECUTION_UID} --no-create-home --home ${HOME} user || exit 0
+RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
+
+# Add compiler to install python packages which come with C++ code
+RUN sudo apt-get update -y
+RUN sudo apt-get install -y g++ gcc python3-dev
+
+# Install git to install pip requirements from git
+RUN sudo apt-get install -y git
+
+# Install dependencies of our scripts (bin/start-ds.sh)
+RUN sudo apt-get install -y rsync
+
+COPY requirements.txt ./
+RUN sudo pip3 install -r requirements.txt
+
+# Install some version of the casacore measures tables, to allow basic delay computation analysis in the notebooks
+RUN sudo apt-get install -y casacore-data
+
+# see https://github.com/jupyter/nbconvert/issues/1434
+RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py"
+RUN sudo apt-get update -y
+RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
+
+# Allow Download as -> PDF via LaTeX
+RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended cm-super
+
+# Configure jupyter_bokeh
+RUN sudo mkdir -p /usr/share/jupyter /usr/etc
+RUN sudo chmod a+rwx /usr/share/jupyter /usr/etc
+RUN sudo jupyter nbextension install --sys-prefix --symlink --py jupyter_bokeh
+RUN sudo jupyter nbextension enable jupyter_bokeh --py --sys-prefix
+
+# Install profiles for ipython & jupyter
+COPY ipython-profiles /opt/ipython-profiles/
+RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles
+COPY jupyter-kernels /usr/local/share/jupyter/kernels/
+
+# Install patched jupyter executable
+COPY jupyter-notebook /usr/local/bin/jupyter-notebook
+
+# Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes.
+ENV TINI_VERSION v0.6.0
+ENV JUPYTER_RUNTIME_DIR=/tmp
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
+RUN sudo chmod +x /usr/bin/tini
+
+USER ${CONTAINER_EXECUTION_UID}
+# pyppeteer-install installs in the homedir, so run it as the user that will execute the notebook
+RUN pyppeteer-install
+
+# Enable Jupyter lab
+ENV JUPYTER_ENABLE_LAB=yes
diff --git a/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/ipython_config.py b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/ipython_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..91b04aaa3a20232b60e5ced00a99648891955ce5
--- /dev/null
+++ b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/ipython_config.py
@@ -0,0 +1,578 @@
+# Configuration file for ipython.
+
+#------------------------------------------------------------------------------
+# InteractiveShellApp(Configurable) configuration
+#------------------------------------------------------------------------------
+
+## A Mixin for applications that start InteractiveShell instances.
+#
+# Provides configurables for loading extensions and executing files as part of
+# configuring a Shell environment.
+#
+# The following methods should be called by the :meth:`initialize` method of the
+# subclass:
+#
+# - :meth:`init_path`
+# - :meth:`init_shell` (to be implemented by the subclass)
+# - :meth:`init_gui_pylab`
+# - :meth:`init_extensions`
+# - :meth:`init_code`
+
+## Execute the given command string.
+#c.InteractiveShellApp.code_to_run = ''
+
+## Run the file referenced by the PYTHONSTARTUP environment variable at IPython
+# startup.
+#c.InteractiveShellApp.exec_PYTHONSTARTUP = True
+
+## List of files to run at IPython startup.
+#c.InteractiveShellApp.exec_files = []
+
+## lines of code to run at IPython startup.
+#c.InteractiveShellApp.exec_lines = []
+
+## A list of dotted module names of IPython extensions to load.
+#c.InteractiveShellApp.extensions = []
+
+## dotted module name of an IPython extension to load.
+#c.InteractiveShellApp.extra_extension = ''
+
+## A file to be run
+#c.InteractiveShellApp.file_to_run = ''
+
+## Enable GUI event loop integration with any of ('glut', 'gtk', 'gtk2', 'gtk3',
+# 'osx', 'pyglet', 'qt', 'qt4', 'qt5', 'tk', 'wx', 'gtk2', 'qt4').
+#c.InteractiveShellApp.gui = None
+
+## Should variables loaded at startup (by startup files, exec_lines, etc.) be
+# hidden from tools like %who?
+#c.InteractiveShellApp.hide_initial_ns = True
+
+## Configure matplotlib for interactive use with the default matplotlib backend.
+#c.InteractiveShellApp.matplotlib = None
+
+## Run the module as a script.
+#c.InteractiveShellApp.module_to_run = ''
+
+## Pre-load matplotlib and numpy for interactive use, selecting a particular
+# matplotlib backend and loop integration.
+#c.InteractiveShellApp.pylab = None
+
+## If true, IPython will populate the user namespace with numpy, pylab, etc. and
+# an ``import *`` is done from numpy and pylab, when using pylab mode.
+#
+# When False, pylab mode should not import any names into the user namespace.
+#c.InteractiveShellApp.pylab_import_all = True
+
+## Reraise exceptions encountered loading IPython extensions?
+#c.InteractiveShellApp.reraise_ipython_extension_failures = False
+
+#------------------------------------------------------------------------------
+# Application(SingletonConfigurable) configuration
+#------------------------------------------------------------------------------
+
+## This is an application.
+
+## The date format used by logging formatters for %(asctime)s
+#c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S'
+
+## The Logging format template
+#c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s'
+
+## Set the log level by value or name.
+#c.Application.log_level = 30
+
+#------------------------------------------------------------------------------
+# BaseIPythonApplication(Application) configuration
+#------------------------------------------------------------------------------
+
+## IPython: an enhanced interactive Python shell.
+
+## Whether to create profile dir if it doesn't exist
+#c.BaseIPythonApplication.auto_create = False
+
+## Whether to install the default config files into the profile dir. If a new
+# profile is being created, and IPython contains config files for that profile,
+# then they will be staged into the new directory. Otherwise, default config
+# files will be automatically generated.
+#c.BaseIPythonApplication.copy_config_files = False
+
+## Path to an extra config file to load.
+#
+# If specified, load this config file in addition to any other IPython config.
+#c.BaseIPythonApplication.extra_config_file = ''
+
+## The name of the IPython directory. This directory is used for logging
+# configuration (through profiles), history storage, etc. The default is usually
+# $HOME/.ipython. This option can also be specified through the environment
+# variable IPYTHONDIR.
+#c.BaseIPythonApplication.ipython_dir = ''
+
+## Whether to overwrite existing config files when copying
+#c.BaseIPythonApplication.overwrite = False
+
+## The IPython profile to use.
+#c.BaseIPythonApplication.profile = 'default'
+
+## Create a massive crash report when IPython encounters what may be an internal
+# error. The default is to append a short message to the usual traceback
+#c.BaseIPythonApplication.verbose_crash = False
+
+#------------------------------------------------------------------------------
+# TerminalIPythonApp(BaseIPythonApplication,InteractiveShellApp) configuration
+#------------------------------------------------------------------------------
+
+## Whether to display a banner upon starting IPython.
+#c.TerminalIPythonApp.display_banner = True
+
+## If a command or file is given via the command-line, e.g. 'ipython foo.py',
+# start an interactive shell after executing the file or command.
+#c.TerminalIPythonApp.force_interact = False
+
+## Class to use to instantiate the TerminalInteractiveShell object. Useful for
+# custom Frontends
+#c.TerminalIPythonApp.interactive_shell_class = 'IPython.terminal.interactiveshell.TerminalInteractiveShell'
+
+## Start IPython quickly by skipping the loading of config files.
+#c.TerminalIPythonApp.quick = False
+
+#------------------------------------------------------------------------------
+# InteractiveShell(SingletonConfigurable) configuration
+#------------------------------------------------------------------------------
+
+## An enhanced, interactive shell for Python.
+
+## 'all', 'last', 'last_expr' or 'none', specifying which nodes should be run
+# interactively (displaying output from expressions).
+#c.InteractiveShell.ast_node_interactivity = 'last_expr'
+
+## A list of ast.NodeTransformer subclass instances, which will be applied to
+# user input before code is run.
+#c.InteractiveShell.ast_transformers = []
+
+## Make IPython automatically call any callable object even if you didn't type
+# explicit parentheses. For example, 'str 43' becomes 'str(43)' automatically.
+# The value can be '0' to disable the feature, '1' for 'smart' autocall, where
+# it is not applied if there are no more arguments on the line, and '2' for
+# 'full' autocall, where all callable objects are automatically called (even if
+# no arguments are present).
+#c.InteractiveShell.autocall = 0
+
+## Autoindent IPython code entered interactively.
+#c.InteractiveShell.autoindent = True
+
+## Enable magic commands to be called without the leading %.
+#c.InteractiveShell.automagic = True
+
+## The part of the banner to be printed before the profile
+#c.InteractiveShell.banner1 = 'Python 3.7.3 (default, Jul 25 2020, 13:03:44) \nType "copyright", "credits" or "license" for more information.\n\nIPython 5.8.0 -- An enhanced Interactive Python.\n? -> Introduction and overview of IPython\'s features.\n%quickref -> Quick reference.\nhelp -> Python\'s own help system.\nobject? -> Details about \'object\', use \'object??\' for extra details.\n'
+
+## The part of the banner to be printed after the profile
+#c.InteractiveShell.banner2 = ''
+
+## Set the size of the output cache. The default is 1000, you can change it
+# permanently in your config file. Setting it to 0 completely disables the
+# caching system, and the minimum value accepted is 20 (if you provide a value
+# less than 20, it is reset to 0 and a warning is issued). This limit is
+# defined because otherwise you'll spend more time re-flushing a too small cache
+# than working
+#c.InteractiveShell.cache_size = 1000
+
+## Use colors for displaying information about objects. Because this information
+# is passed through a pager (like 'less'), and some pagers get confused with
+# color codes, this capability can be turned off.
+#c.InteractiveShell.color_info = True
+
+## Set the color scheme (NoColor, Neutral, Linux, or LightBG).
+#c.InteractiveShell.colors = 'Neutral'
+
+##
+#c.InteractiveShell.debug = False
+
+## **Deprecated**
+#
+# Will be removed in IPython 6.0
+#
+# Enable deep (recursive) reloading by default. IPython can use the deep_reload
+# module which reloads changes in modules recursively (it replaces the reload()
+# function, so you don't need to change anything to use it). `deep_reload`
+# forces a full reload of modules whose code may have changed, which the default
+# reload() function does not. When deep_reload is off, IPython will use the
+# normal reload(), but deep_reload will still be available as dreload().
+#c.InteractiveShell.deep_reload = False
+
+## Don't call post-execute functions that have failed in the past.
+#c.InteractiveShell.disable_failing_post_execute = False
+
+## If True, anything that would be passed to the pager will be displayed as
+# regular output instead.
+#c.InteractiveShell.display_page = False
+
+## (Provisional API) enables html representation in mime bundles sent to pagers.
+#c.InteractiveShell.enable_html_pager = False
+
+## Total length of command history
+#c.InteractiveShell.history_length = 10000
+
+## The number of saved history entries to be loaded into the history buffer at
+# startup.
+#c.InteractiveShell.history_load_length = 1000
+
+##
+#c.InteractiveShell.ipython_dir = ''
+
+## Start logging to the given file in append mode. Use `logfile` to specify a log
+# file to **overwrite** logs to.
+#c.InteractiveShell.logappend = ''
+
+## The name of the logfile to use.
+#c.InteractiveShell.logfile = ''
+
+## Start logging to the default log file in overwrite mode. Use `logappend` to
+# specify a log file to **append** logs to.
+#c.InteractiveShell.logstart = False
+
+##
+#c.InteractiveShell.object_info_string_level = 0
+
+## Automatically call the pdb debugger after every exception.
+#c.InteractiveShell.pdb = False
+
+## Deprecated since IPython 4.0 and ignored since 5.0, set
+# TerminalInteractiveShell.prompts object directly.
+#c.InteractiveShell.prompt_in1 = 'In [\\#]: '
+
+## Deprecated since IPython 4.0 and ignored since 5.0, set
+# TerminalInteractiveShell.prompts object directly.
+#c.InteractiveShell.prompt_in2 = ' .\\D.: '
+
+## Deprecated since IPython 4.0 and ignored since 5.0, set
+# TerminalInteractiveShell.prompts object directly.
+#c.InteractiveShell.prompt_out = 'Out[\\#]: '
+
+## Deprecated since IPython 4.0 and ignored since 5.0, set
+# TerminalInteractiveShell.prompts object directly.
+#c.InteractiveShell.prompts_pad_left = True
+
+##
+#c.InteractiveShell.quiet = False
+
+##
+#c.InteractiveShell.separate_in = '\n'
+
+##
+#c.InteractiveShell.separate_out = ''
+
+##
+#c.InteractiveShell.separate_out2 = ''
+
+## Show rewritten input, e.g. for autocall.
+#c.InteractiveShell.show_rewritten_input = True
+
+## Enables rich html representation of docstrings. (This requires the docrepr
+# module).
+#c.InteractiveShell.sphinxify_docstring = False
+
+##
+#c.InteractiveShell.wildcards_case_sensitive = True
+
+##
+#c.InteractiveShell.xmode = 'Context'
+
+#------------------------------------------------------------------------------
+# TerminalInteractiveShell(InteractiveShell) configuration
+#------------------------------------------------------------------------------
+
+## Set to confirm when you try to exit IPython with an EOF (Control-D in Unix,
+# Control-Z/Enter in Windows). By typing 'exit' or 'quit', you can force a
+# direct exit without any confirmation.
+#c.TerminalInteractiveShell.confirm_exit = True
+
+## Options for displaying tab completions, 'column', 'multicolumn', and
+# 'readlinelike'. These options are for `prompt_toolkit`, see `prompt_toolkit`
+# documentation for more information.
+#c.TerminalInteractiveShell.display_completions = 'multicolumn'
+
+## Shortcut style to use at the prompt. 'vi' or 'emacs'.
+#c.TerminalInteractiveShell.editing_mode = 'emacs'
+
+## Set the editor used by IPython (default to $EDITOR/vi/notepad).
+#c.TerminalInteractiveShell.editor = 'vi'
+
+## Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. This is
+# in addition to the F2 binding, which is always enabled.
+#c.TerminalInteractiveShell.extra_open_editor_shortcuts = False
+
+## Highlight matching brackets.
+#c.TerminalInteractiveShell.highlight_matching_brackets = True
+
+## The name or class of a Pygments style to use for syntax highlighting. To see
+# available styles, run `pygmentize -L styles`.
+#c.TerminalInteractiveShell.highlighting_style = traitlets.Undefined
+
+## Override highlighting format for specific tokens
+#c.TerminalInteractiveShell.highlighting_style_overrides = {}
+
+## Enable mouse support in the prompt
+#c.TerminalInteractiveShell.mouse_support = False
+
+## Class used to generate Prompt token for prompt_toolkit
+#c.TerminalInteractiveShell.prompts_class = 'IPython.terminal.prompts.Prompts'
+
+## Use `raw_input` for the REPL, without completion and prompt colors.
+#
+# Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR.
+# Known usage are: IPython own testing machinery, and emacs inferior-shell
+# integration through elpy.
+#
+# This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT` environment
+# variable is set, or the current terminal is not a tty.
+#c.TerminalInteractiveShell.simple_prompt = False
+
+## Number of line at the bottom of the screen to reserve for the completion menu
+#c.TerminalInteractiveShell.space_for_menu = 6
+
+## Automatically set the terminal title
+#c.TerminalInteractiveShell.term_title = True
+
+## Use 24bit colors instead of 256 colors in prompt highlighting. If your
+# terminal supports true color, the following command should print 'TRUECOLOR'
+# in orange: printf "\x1b[38;2;255;100;0mTRUECOLOR\x1b[0m\n"
+#c.TerminalInteractiveShell.true_color = False
+
+#------------------------------------------------------------------------------
+# HistoryAccessor(HistoryAccessorBase) configuration
+#------------------------------------------------------------------------------
+
+## Access the history database without adding to it.
+#
+# This is intended for use by standalone history tools. IPython shells use
+# HistoryManager, below, which is a subclass of this.
+
+## Options for configuring the SQLite connection
+#
+# These options are passed as keyword args to sqlite3.connect when establishing
+# database conenctions.
+#c.HistoryAccessor.connection_options = {}
+
+## enable the SQLite history
+#
+# set enabled=False to disable the SQLite history, in which case there will be
+# no stored history, no SQLite connection, and no background saving thread.
+# This may be necessary in some threaded environments where IPython is embedded.
+#c.HistoryAccessor.enabled = True
+
+## Path to file to use for SQLite history database.
+#
+# By default, IPython will put the history database in the IPython profile
+# directory. If you would rather share one history among profiles, you can set
+# this value in each, so that they are consistent.
+#
+# Due to an issue with fcntl, SQLite is known to misbehave on some NFS mounts.
+# If you see IPython hanging, try setting this to something on a local disk,
+# e.g::
+#
+# ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
+#
+# you can also use the specific value `:memory:` (including the colon at both
+# end but not the back ticks), to avoid creating an history file.
+#c.HistoryAccessor.hist_file = ''
+
+#------------------------------------------------------------------------------
+# HistoryManager(HistoryAccessor) configuration
+#------------------------------------------------------------------------------
+
+## A class to organize all history-related functionality in one place.
+
+## Write to database every x commands (higher values save disk access & power).
+# Values of 1 or less effectively disable caching.
+#c.HistoryManager.db_cache_size = 0
+
+## Should the history database include output? (default: no)
+#c.HistoryManager.db_log_output = False
+
+#------------------------------------------------------------------------------
+# ProfileDir(LoggingConfigurable) configuration
+#------------------------------------------------------------------------------
+
+## An object to manage the profile directory and its resources.
+#
+# The profile directory is used by all IPython applications, to manage
+# configuration, logging and security.
+#
+# This object knows how to find, create and manage these directories. This
+# should be used by any code that wants to handle profiles.
+
+## Set the profile location directly. This overrides the logic used by the
+# `profile` option.
+#c.ProfileDir.location = ''
+
+#------------------------------------------------------------------------------
+# BaseFormatter(Configurable) configuration
+#------------------------------------------------------------------------------
+
+## A base formatter class that is configurable.
+#
+# This formatter should usually be used as the base class of all formatters. It
+# is a traited :class:`Configurable` class and includes an extensible API for
+# users to determine how their objects are formatted. The following logic is
+# used to find a function to format an given object.
+#
+# 1. The object is introspected to see if it has a method with the name
+# :attr:`print_method`. If is does, that object is passed to that method
+# for formatting.
+# 2. If no print method is found, three internal dictionaries are consulted
+# to find print method: :attr:`singleton_printers`, :attr:`type_printers`
+# and :attr:`deferred_printers`.
+#
+# Users should use these dictionaries to register functions that will be used to
+# compute the format data for their objects (if those objects don't have the
+# special print methods). The easiest way of using these dictionaries is through
+# the :meth:`for_type` and :meth:`for_type_by_name` methods.
+#
+# If no function/callable is found to compute the format data, ``None`` is
+# returned and this format type is not used.
+
+##
+#c.BaseFormatter.deferred_printers = {}
+
+##
+#c.BaseFormatter.enabled = True
+
+##
+#c.BaseFormatter.singleton_printers = {}
+
+##
+#c.BaseFormatter.type_printers = {}
+
+#------------------------------------------------------------------------------
+# PlainTextFormatter(BaseFormatter) configuration
+#------------------------------------------------------------------------------
+
+## The default pretty-printer.
+#
+# This uses :mod:`IPython.lib.pretty` to compute the format data of the object.
+# If the object cannot be pretty printed, :func:`repr` is used. See the
+# documentation of :mod:`IPython.lib.pretty` for details on how to write pretty
+# printers. Here is a simple example::
+#
+# def dtype_pprinter(obj, p, cycle):
+# if cycle:
+# return p.text('dtype(...)')
+# if hasattr(obj, 'fields'):
+# if obj.fields is None:
+# p.text(repr(obj))
+# else:
+# p.begin_group(7, 'dtype([')
+# for i, field in enumerate(obj.descr):
+# if i > 0:
+# p.text(',')
+# p.breakable()
+# p.pretty(field)
+# p.end_group(7, '])')
+
+##
+#c.PlainTextFormatter.float_precision = ''
+
+## Truncate large collections (lists, dicts, tuples, sets) to this size.
+#
+# Set to 0 to disable truncation.
+#c.PlainTextFormatter.max_seq_length = 1000
+
+##
+#c.PlainTextFormatter.max_width = 79
+
+##
+#c.PlainTextFormatter.newline = '\n'
+
+##
+#c.PlainTextFormatter.pprint = True
+
+##
+#c.PlainTextFormatter.verbose = False
+
+#------------------------------------------------------------------------------
+# Completer(Configurable) configuration
+#------------------------------------------------------------------------------
+
+## Enable unicode completions, e.g. \alpha<tab> . Includes completion of latex
+# commands, unicode names, and expanding unicode characters back to latex
+# commands.
+#c.Completer.backslash_combining_completions = True
+
+## Activate greedy completion PENDING DEPRECTION. this is now mostly taken care
+# of with Jedi.
+#
+# This will enable completion on elements of lists, results of function calls,
+# etc., but can be unsafe because the code is actually evaluated on TAB.
+#c.Completer.greedy = False
+
+#------------------------------------------------------------------------------
+# IPCompleter(Completer) configuration
+#------------------------------------------------------------------------------
+
+## Extension of the completer class with IPython-specific features
+
+## DEPRECATED as of version 5.0.
+#
+# Instruct the completer to use __all__ for the completion
+#
+# Specifically, when completing on ``object.<tab>``.
+#
+# When True: only those names in obj.__all__ will be included.
+#
+# When False [default]: the __all__ attribute is ignored
+#c.IPCompleter.limit_to__all__ = False
+
+## Whether to merge completion results into a single list
+#
+# If False, only the completion results from the first non-empty completer will
+# be returned.
+#c.IPCompleter.merge_completions = True
+
+## Instruct the completer to omit private method names
+#
+# Specifically, when completing on ``object.<tab>``.
+#
+# When 2 [default]: all names that start with '_' will be excluded.
+#
+# When 1: all 'magic' names (``__foo__``) will be excluded.
+#
+# When 0: nothing will be excluded.
+#c.IPCompleter.omit__names = 2
+
+#------------------------------------------------------------------------------
+# ScriptMagics(Magics) configuration
+#------------------------------------------------------------------------------
+
+## Magics for talking to scripts
+#
+# This defines a base `%%script` cell magic for running a cell with a program in
+# a subprocess, and registers a few top-level magics that call %%script with
+# common interpreters.
+
+## Extra script cell magics to define
+#
+# This generates simple wrappers of `%%script foo` as `%%foo`.
+#
+# If you want to add script magics that aren't on your path, specify them in
+# script_paths
+#c.ScriptMagics.script_magics = []
+
+## Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
+#
+# Only necessary for items in script_magics where the default path will not find
+# the right interpreter.
+#c.ScriptMagics.script_paths = {}
+
+#------------------------------------------------------------------------------
+# StoreMagics(Magics) configuration
+#------------------------------------------------------------------------------
+
+## Lightweight persistence for python variables.
+#
+# Provides the %store magic.
+
+## If True, any %store-d variables will be automatically restored when IPython
+# starts.
+#c.StoreMagics.autorestore = False
diff --git a/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py
new file mode 100644
index 0000000000000000000000000000000000000000..38fcb84c3417c6b19d89527be6f8122bd0249765
--- /dev/null
+++ b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py
@@ -0,0 +1 @@
+from tango import *
diff --git a/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py
new file mode 100644
index 0000000000000000000000000000000000000000..350ecb1e87f4829ddd60698831bbf75d941782a9
--- /dev/null
+++ b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py
@@ -0,0 +1,21 @@
+# Create shortcuts for our devices
+apsct = DeviceProxy("STAT/APSCT/1")
+ccd = DeviceProxy("STAT/CCD/1")
+apspu = DeviceProxy("STAT/APSPU/1")
+recv = DeviceProxy("STAT/RECV/1")
+sdp = DeviceProxy("STAT/SDP/1")
+bst = DeviceProxy("STAT/BST/1")
+sst = DeviceProxy("STAT/SST/1")
+xst = DeviceProxy("STAT/XST/1")
+unb2 = DeviceProxy("STAT/UNB2/1")
+boot = DeviceProxy("STAT/Boot/1")
+tilebeam = DeviceProxy("STAT/TileBeam/1")
+psoc = DeviceProxy("STAT/PSOC/1")
+beamlet = DeviceProxy("STAT/Beamlet/1")
+digitalbeam = DeviceProxy("STAT/DigitalBeam/1")
+antennafield = DeviceProxy("STAT/AntennaField/1")
+docker = DeviceProxy("STAT/Docker/1")
+temperaturemanager = DeviceProxy("STAT/TemperatureManager/1")
+
+# Put them in a list in case one wants to iterate
+devices = [apsct, ccd, apspu, recv, sdp, bst, sst, xst, unb2, boot, tilebeam, beamlet, digitalbeam, antennafield, temperaturemanager, docker]
diff --git a/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py
new file mode 100644
index 0000000000000000000000000000000000000000..d21ed1cf013d73b700cbc72e3d89ef9541efcacc
--- /dev/null
+++ b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py
@@ -0,0 +1 @@
+import tangostationcontrol
diff --git a/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..61d470004218ae459ce7bfdc974f7c86e0790486
--- /dev/null
+++ b/docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md
@@ -0,0 +1,11 @@
+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+ 00-first.py
+ 50-middle.py
+ 99-last.ipy
diff --git a/docker-compose/jupyterlab/jupyter-kernels/stationcontrol/kernel.json b/docker-compose/jupyterlab/jupyter-kernels/stationcontrol/kernel.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff6d4a1a01d0f7bd6eda3a40886eae74b451a5a4
--- /dev/null
+++ b/docker-compose/jupyterlab/jupyter-kernels/stationcontrol/kernel.json
@@ -0,0 +1,13 @@
+ {
+ "argv": [
+ "python",
+ "-m",
+ "ipykernel",
+ "-f",
+ "{connection_file}",
+ "--profile-dir",
+ "/opt/ipython-profiles/stationcontrol-jupyter/"
+ ],
+ "language": "python",
+ "display_name": "StationControl"
+}
diff --git a/docker-compose/jupyterlab/jupyter-notebook b/docker-compose/jupyterlab/jupyter-notebook
new file mode 100755
index 0000000000000000000000000000000000000000..59613a137cc1bb5c86b4cd7c82f3a2cb1f9abde3
--- /dev/null
+++ b/docker-compose/jupyterlab/jupyter-notebook
@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# An adjustment of the `jupyter-notebook' executable patched to:
+# - log to the ELK stack
+#
+# We go straight for the notebook executable here, as the "jupyter" command
+# execvp's into the requested notebook subcommand, erasing all configuration
+# we set here.
+import re
+import sys
+
+from notebook.notebookapp import main
+
+from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
+import logging
+
+if __name__ == '__main__':
+ # log to the tcp_input of logstash in our ELK stack
+ handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/pending_log_messages.db')
+
+ # add to logger of Jupyter traitlets Application. As that logger is configured not to propagate
+ # messages upward, we need to configure it directly.
+ logger = logging.getLogger("NotebookApp")
+ logger.addHandler(handler)
+ logger.setLevel(logging.DEBUG)
+
+ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+ sys.exit(main())
diff --git a/docker-compose/jupyterlab/requirements.txt b/docker-compose/jupyterlab/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..70494373dff1a7d5010f4a939362c7bfc87998c8
--- /dev/null
+++ b/docker-compose/jupyterlab/requirements.txt
@@ -0,0 +1,26 @@
+ipython >=7.27.0,!=7.28.0 # BSD
+jupyter
+jupyterlab
+jupyterlab_h5web[full] # MIT
+ipykernel
+jupyter_bokeh
+matplotlib
+jupyterplot
+nbconvert
+notebook-as-pdf
+python-logstash-async
+PyMySQL[rsa]
+psycopg2-binary >= 2.9.2 #LGPL
+sqlalchemy
+pyvisa
+pyvisa-py
+opcua
+lofarantpos >= 0.5.0 # Apache 2
+python-geohash >= 0.8.5 # Apache 2 / MIT
+
+numpy
+scipy
+
+pabeam@git+https://git.astron.nl/mevius/grate # Apache2
+lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client # Apache2
+etrs-itrs@git+https://github.com/brentjens/etrs-itrs # Apache 2
diff --git a/docker-compose/logstash.yml b/docker-compose/logstash.yml
new file mode 100644
index 0000000000000000000000000000000000000000..73a13e346433a3a337bf66383132d8c1e24e0352
--- /dev/null
+++ b/docker-compose/logstash.yml
@@ -0,0 +1,29 @@
+#
+# Docker compose file that launches Logstash-output-loki
+#
+#
+
+version: '2.1'
+
+services:
+ logstash:
+ image: logstash
+ build:
+ context: logstash
+ args:
+ SOURCE_IMAGE: grafana/logstash-output-loki:main
+ container_name: ${CONTAINER_NAME_PREFIX}logstash
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ networks:
+ - control
+ ports:
+ - "5044:5044" # logstash beats input
+ - "1514:1514/tcp" # logstash syslog input
+ - "1514:1514/udp" # logstash syslog input
+ - "5959:5959" # logstash tcp json input
+ - "9600:9600"
+ restart: unless-stopped
diff --git a/docker-compose/logstash/Dockerfile b/docker-compose/logstash/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..94fa5ab4bfe9a6d5946da6dda9a018c8385ef592
--- /dev/null
+++ b/docker-compose/logstash/Dockerfile
@@ -0,0 +1,10 @@
+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# Disable Elastic Search connection
+ENV ELASTIC_CONTAINER=false
+
+# Provide our logstash config
+COPY loki.conf /home/logstash/
+COPY logstash.yml /usr/share/logstash/config/logstash.yml
+COPY loki.conf /usr/share/logstash/pipeline/logstash.conf
diff --git a/docker-compose/logstash/README.md b/docker-compose/logstash/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..156aaae85a210ee963a83baadc39ccdadbbef756
--- /dev/null
+++ b/docker-compose/logstash/README.md
@@ -0,0 +1,41 @@
+# Logstash
+
+Grafana Loki has a Logstash output plugin called logstash-output-loki that enables shipping logs to a Loki instance
+
+## Usage and configuration
+
+To configure Logstash to forward logs to Loki, simply add the loki output to your Logstash configuration file as documented below:
+
+ output {
+ loki {
+ [url => "" | default = none | required=true]
+
+ [tenant_id => string | default = nil | required=false]
+
+ [message_field => string | default = "message" | required=false]
+
+ [include_fields => array | default = [] | required=false]
+
+ [batch_wait => number | default = 1(s) | required=false]
+
+ [batch_size => number | default = 102400(bytes) | required=false]
+
+ [min_delay => number | default = 1(s) | required=false]
+
+ [max_delay => number | default = 300(s) | required=false]
+
+ [retries => number | default = 10 | required=false]
+
+ [username => string | default = nil | required=false]
+
+ [password => secret | default = nil | required=false]
+
+ [cert => path | default = nil | required=false]
+
+ [key => path | default = nil| required=false]
+
+ [ca_cert => path | default = nil | required=false]
+
+ [insecure_skip_verify => boolean | default = false | required=false]
+ }
+ }
diff --git a/docker-compose/logstash/logstash.yml b/docker-compose/logstash/logstash.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5f80650fe6fc635570fd9f7e4888da17eddf4e70
--- /dev/null
+++ b/docker-compose/logstash/logstash.yml
@@ -0,0 +1,2 @@
+http.host: "0.0.0.0"
+#xpack.monitoring.elasticsearch.hosts: [ "http://loki:3100" ]
diff --git a/docker-compose/logstash/loki.conf b/docker-compose/logstash/loki.conf
new file mode 100644
index 0000000000000000000000000000000000000000..e22f53dbee2bac168d7e4d3bc572c854f6e81663
--- /dev/null
+++ b/docker-compose/logstash/loki.conf
@@ -0,0 +1,152 @@
+input {
+ beats {
+ port => 5044
+ # ssl => true
+ # ssl_certificate => "/etc/pki/tls/certs/logstash-beats.crt"
+ # ssl_key => "/etc/pki/tls/private/logstash-beats.key"
+ }
+}
+
+input {
+ syslog {
+ port => 1514
+ }
+}
+
+input {
+ tcp {
+ port => 5959
+ codec => json
+ }
+}
+
+filter {
+ if [type] == "syslog" {
+ grok {
+ match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
+ add_field => [ "received_at", "%{@timestamp}" ]
+ add_field => [ "received_from", "%{host}" ]
+ }
+ syslog_pri { }
+ date {
+ match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
+ }
+ }
+}
+
+# filter {
+# if [type] == "nginx-access" {
+# grok {
+# match => { "message" => "%{NGINXACCESS}" }
+# }
+# }
+# }
+
+filter {
+ if [program] == "grafana" {
+ kv { }
+ mutate {
+ rename => {
+ "t" => "timestamp"
+ "lvl" => "level"
+ "msg" => "message"
+ }
+ uppercase => [ "level" ]
+ }
+ date {
+ match => [ "timestamp", "ISO8601" ]
+ }
+ }
+}
+
+filter {
+ if [program] == "prometheus" {
+ kv { }
+ mutate {
+ rename => {
+ "ts" => "timestamp"
+ "msg" => "message"
+ }
+ uppercase => [ "level" ]
+ }
+ date {
+ match => [ "timestamp", "ISO8601" ]
+ }
+ }
+}
+
+filter {
+ if [program] == "prometheus" {
+ kv { }
+ mutate {
+ rename => {
+ "ts" => "timestamp"
+ "msg" => "message"
+ }
+ uppercase => [ "level" ]
+ }
+ date {
+ match => [ "timestamp", "ISO8601" ]
+ }
+ }
+}
+
+filter {
+ if [program] == "tango-rest" {
+ grok {
+ match => {
+ "message" => "%{TIMESTAMP_ISO8601:timestamp} %{WORD:level} %{GREEDYDATA:message}"
+ }
+ "overwrite" => [ "timestamp", "level", "message" ]
+ }
+ date {
+ match => [ "timestamp", "YYYY-MM-dd HH:mm:ss,SSS" ]
+ timezone => "UTC"
+ }
+ }
+}
+
+filter {
+ # mark all our mariadb instances
+ grok {
+ match => {
+ "program" => [ "archiver-maria-db", "tangodb" ]
+ }
+ add_tag => [ "mariadb" ]
+ }
+
+ # parse mariadb output
+ if "mariadb" in [tags] {
+ grok {
+ match => {
+ "message" => [
+ "%{TIMESTAMP_ISO8601:timestamp} .%{WORD:level}. %{GREEDYDATA:message}",
+ "%{TIMESTAMP_ISO8601:timestamp} 0 .%{WORD:level}. %{GREEDYDATA:message}"
+ ]
+ }
+ "overwrite" => [ "timestamp", "level", "message" ]
+ }
+ mutate {
+ gsub => [
+ "level", "Note", "Info"
+ ]
+ uppercase => [ "level" ]
+ }
+ date {
+ match => [ "timestamp", "YYYY-MM-dd HH:mm:ssZZ", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd H:mm:ss" ]
+ timezone => "UTC"
+ }
+ }
+}
+
+output {
+ # elasticsearch {
+ # hosts => ["localhost"]
+ # manage_template => false
+ # index => "logstash-%{+YYYY.MM.dd}"
+ # }
+ loki {
+ url => "http://loki:3100/loki/api/v1/push"
+ }
+}
+
diff --git a/docker-compose/loki.yml b/docker-compose/loki.yml
new file mode 100644
index 0000000000000000000000000000000000000000..2007e16aa7e06726cc5b8be75553e4ef2df56475
--- /dev/null
+++ b/docker-compose/loki.yml
@@ -0,0 +1,40 @@
+#
+# Docker compose file that launches a LOKI instance.
+# See https://grafana.com/docs/loki/latest/installation/docker/
+#
+#
+
+version: "3"
+
+services:
+ loki:
+ image: grafana/loki:2.6.0
+ container_name: ${CONTAINER_NAME_PREFIX}loki
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ networks:
+ - control
+ ports:
+ - "3100:3100"
+ command: -config.file=/etc/loki/local-config.yaml
+ restart: unless-stopped
+
+ promtail:
+ image: grafana/promtail:2.6.0
+ container_name: ${CONTAINER_NAME_PREFIX}promtail
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "100m"
+ max-file: "10"
+ volumes:
+ - /var/log:/var/log
+ command: -config.file=/etc/promtail/config.yml
+ networks:
+ - control
+ ports:
+ - "9080:9080"
+ restart: unless-stopped
diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh
index 57e4fff1458ead7733250ad130da5ff1602a1bdb..563b2c7fbf793bcbd9db84a45635433c0b742c26 100755
--- a/sbin/run_integration_test.sh
+++ b/sbin/run_integration_test.sh
@@ -86,14 +86,16 @@ SIMULATORS="sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim ccd-sim"
# shellcheck disable=SC2086
make build $DEVICES $SIMULATORS
-make build elk integration-test
+# make build elk integration-test # L2SS-970: elk temporarily disabled
+make build logstash integration-test
make build archiver-timescale hdbppts-cm hdbppts-es
# Start and stop sequence
# shellcheck disable=SC2086
make stop $DEVICES $SIMULATORS hdbppts-es hdbppts-cm archiver-timescale
make stop device-docker # this one does not test well in docker-in-docker
-make stop elk
+# make stop elk # L2SS-970: elk temporarily disabled
+make stop logstash
# Run dummy integration test to install pytango in tox virtualenv without
# the memory pressure of the ELK stack.
@@ -103,7 +105,8 @@ make stop elk
# TODO(L2SS-992): Remove me and above documentation
integration_test dummy
-make start elk
+# make start elk # L2SS-970: elk temporarily disabled
+make start logstash
# Give elk time to start
# TODO(L2SS-988): Use a nicer more reliable mechanism
diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh
index 3e10110ed1ac4dad5e0f6fb0521a314c0ae06fbe..d3b5ea894e0b13eb3bcb7755656381235d664a5a 100755
--- a/sbin/tag_and_push_docker_image.sh
+++ b/sbin/tag_and_push_docker_image.sh
@@ -61,6 +61,7 @@ REMOTE_IMAGES=(
# TODO(Corne): Have this list generated from the .yml files
LOCAL_IMAGES=(
"elk elk y" "elk-configure-host elk-configure-host y"
+ "logstash logstash y"
"lofar-device-base lofar-device-base y"
"apsct-sim docker-compose_apsct-sim y" "apspu-sim docker-compose_apspu-sim y"
diff --git a/tangostationcontrol/VERSION b/tangostationcontrol/VERSION
index 6da28dde76d6550e3d398a70a9a8231256774669..0ea3a944b399d25f7e1b8fe684d754eb8da9fe7f 100644
--- a/tangostationcontrol/VERSION
+++ b/tangostationcontrol/VERSION
@@ -1 +1 @@
-0.1.1
\ No newline at end of file
+0.2.0
diff --git a/tangostationcontrol/requirements.txt b/tangostationcontrol/requirements.txt
index 1101bd06624daa6fb01aaaa1a6aab0e5c8a82ddc..a6662e4b263ca26aa93ad216c9b104ffd0e8e735 100644
--- a/tangostationcontrol/requirements.txt
+++ b/tangostationcontrol/requirements.txt
@@ -3,7 +3,7 @@
# integration process, which may cause wedges in the gate later.
importlib-metadata<2.0.0,>=0.12;python_version<"3.8"
-lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.6.0
+lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client@0.9.1
numpy
mock
asyncua >= 0.9.90 # LGPLv3
diff --git a/tangostationcontrol/setup.cfg b/tangostationcontrol/setup.cfg
index 510034a343b89b2bc2a24bd212bafe77a4565a67..f42357ef73a2f50734d03f26444b32b905ac0042 100644
--- a/tangostationcontrol/setup.cfg
+++ b/tangostationcontrol/setup.cfg
@@ -26,7 +26,7 @@ package_dir=
packages=find:
python_requires => 3.7
install_requires =
- importlib-metadata>=0.12;python_version<"3.8"
+ importlib-metadata>=0.12, <5.0;python_version<"3.8"
pip>=1.5
[options.packages.find]
diff --git a/tangostationcontrol/tangostationcontrol/clients/statistics/consumer.py b/tangostationcontrol/tangostationcontrol/clients/statistics/consumer.py
index 8930cdb371cd4ac23b299d479e19dcd28dd08d38..497c7eb763c362ee3ba3c8c7854be8d03664efda 100644
--- a/tangostationcontrol/tangostationcontrol/clients/statistics/consumer.py
+++ b/tangostationcontrol/tangostationcontrol/clients/statistics/consumer.py
@@ -10,6 +10,7 @@
import logging
from threading import Thread
from queue import Queue
+import time
from lofar_station_client.statistics.collector import StatisticsCollector
@@ -24,6 +25,9 @@ class StatisticsConsumer(Thread, StatisticsClientThread):
# Maximum time to wait for the Thread to get unstuck, if we want to stop
DISCONNECT_TIMEOUT = 10.0
+ # Minimum time between packet exception logging
+ LOGGING_TIME = 30
+
# No default options required, for now?
_DEFAULT_OPTIONS = {}
@@ -35,10 +39,30 @@ class StatisticsConsumer(Thread, StatisticsClientThread):
super().__init__()
self.start()
+ self.last_exception_time = time.time()
+ self.exception_counter = 0
+
@property
def _options(self) -> dict:
return StatisticsConsumer._DEFAULT_OPTIONS
+ def _exception_logging(self, err):
+ # get the time since we last logged a message
+ time_since_log = time.time() - self.last_exception_time
+ self.exception_counter += 1
+
+ # if the time since we last logged an exeption is greater than LOGGING TIME
+ if time_since_log < self.LOGGING_TIME:
+ return
+
+ if self.exception_counter == 1:
+ logger.exception(f"Could not parse statistics packet")
+ else:
+ logger.exception(f"Could not parse {self.exception_counter} statistics packets in the last {int(time_since_log)} seconds")
+
+ self.last_exception_time = time.time()
+ self.exception_counter = 0
+
def run(self):
logger.info("Starting statistics thread")
@@ -53,8 +77,7 @@ class StatisticsConsumer(Thread, StatisticsClientThread):
try:
self.collector.process_packet(self.last_packet)
except ValueError as e:
- logger.exception("Could not parse statistics packet")
-
+ self._exception_logging()
# continue processing
logger.info("Stopped statistics thread")
diff --git a/tangostationcontrol/tangostationcontrol/common/lofar_logging.py b/tangostationcontrol/tangostationcontrol/common/lofar_logging.py
index 89ab11c0db3b31af805b5cda78ce21a77ee9318a..13d20551b0def7a72cd75ff750232b861ebd9b0e 100644
--- a/tangostationcontrol/tangostationcontrol/common/lofar_logging.py
+++ b/tangostationcontrol/tangostationcontrol/common/lofar_logging.py
@@ -56,7 +56,7 @@ class LogSuppressErrorSpam(logging.Formatter):
self.error_suppress_interval = error_suppress_interval
def is_error_to_suppress(self, record):
- # Errors occuring by not being able to connect to the ELK stack, f.e. because it is down.
+ # Errors occuring by not being able to connect to the log processing container, f.e. because it is down.
return record.name == "LogProcessingWorker" and record.msg == "An error occurred while sending events: %s"
def filter(self, record):
@@ -105,7 +105,7 @@ class LogAnnotator(logging.Formatter):
def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
"""
Configure the given logger (or root if None) to:
- - send logs to the ELK stack
+ - send logs to Loki through Logstash
- send logs to Tango
- send logs to stdout
"""
@@ -128,7 +128,7 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
# don't spam debug messages when fetching URLs
logging.getLogger("urllib3").setLevel(logging.INFO)
- # don't spam error messages when having trouble connecting to ELK
+ # don't spam error messages when having connection troubles
logging.getLogger("LogProcessingWorker").setLevel(logging.CRITICAL)
# for now, also log to stderr
@@ -151,12 +151,12 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
if debug:
return logger
- # Log to ELK stack
+ # Log to Logstash-Loki
try:
from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
- # log to the tcp_input of logstash in our ELK stack
- handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/lofar_pending_log_messages.db')
+ # log to the tcp_input of logstash in our logstash-loki container
+ handler = AsynchronousLogstashHandler("logstash", 5959, database_path='/tmp/lofar_pending_log_messages.db')
# configure log messages
formatter = LogstashFormatter(extra=log_extra, tags=["python", "lofar"])
@@ -167,9 +167,9 @@ def configure_logger(logger: logging.Logger=None, log_extra=None, debug=False):
# install the handler
logger.addHandler(handler)
except ImportError:
- logger.exception("Cannot forward logs to ELK: logstash_async module not found.")
+ logger.exception("Cannot forward logs to Logstash-Loki: logstash_async module not found.")
except Exception:
- logger.exception("Cannot forward logs to ELK.")
+ logger.exception("Cannot forward logs to Logstash-Loki.")
# Don't log to Tango to reduce log spam
"""
diff --git a/tangostationcontrol/tangostationcontrol/devices/README.md b/tangostationcontrol/tangostationcontrol/devices/README.md
index 4b7923faf4804b6fc0e33ab78d4765f16860e253..64390631beb1222e68c536757766e50fba64bcbc 100644
--- a/tangostationcontrol/tangostationcontrol/devices/README.md
+++ b/tangostationcontrol/tangostationcontrol/devices/README.md
@@ -7,7 +7,7 @@ This directory contains the sources for our custom Tango devices.
If a new device is added, it will (likely) need to be referenced in several places. Adjust or add the following files (referenced from the repository root), following the pattern shown by the devices already there:
- Adjust `CDB/LOFAR_ConfigDb.json` to create the device in the Tango device database,
-- Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter,
+- Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` and `docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter and Jupyter-Lab,
- Adjust `tangostationcontrol/tangostationcontrol/devices/boot.py` to add the device to the station initialisation sequence,
- Add to `docker-compose/` to create a YaML file to start the device in a docker container. NOTE: it needs a unique 57xx port assigned (current _unused_ port value: 5722), a unique 58xx port for ZMQ events, and a unique 59xx port for ZMQ heartbeat
- Adjust `tangostationcontrol/setup.cfg` to add an entry point for the device in the package installation,
diff --git a/tangostationcontrol/tangostationcontrol/devices/antennafield.py b/tangostationcontrol/tangostationcontrol/devices/antennafield.py
index fbd4506ebec411d37ccda3ac855f2756239fc784..52a4ccd359867f329e3ae18ad889ce52c920c7e7 100644
--- a/tangostationcontrol/tangostationcontrol/devices/antennafield.py
+++ b/tangostationcontrol/tangostationcontrol/devices/antennafield.py
@@ -26,14 +26,16 @@ from tangostationcontrol.beam.geo import ETRS_to_ITRF, ITRF_to_GEO, GEO_to_GEOHA
from tangostationcontrol.beam.hba_tile import HBATAntennaOffsets
import logging
+
logger = logging.getLogger()
__all__ = ["AntennaField", "AntennaToRecvMapper", "main"]
class AntennaUse(IntEnum):
- AUTO = 0 # use antenna only if its OK or SUSPICIOUS
- ON = 1 # force antenna to be on, regardless of quality
- OFF = 2 # force antenna to be off, regardless of quality
+ AUTO = 0 # use antenna only if its OK or SUSPICIOUS
+ ON = 1 # force antenna to be on, regardless of quality
+ OFF = 2 # force antenna to be off, regardless of quality
+
class AntennaQuality(IntEnum):
OK = 0
@@ -41,6 +43,7 @@ class AntennaQuality(IntEnum):
BROKEN = 2
BEYOND_REPAIR = 3
+
class mapped_attribute(attribute):
def __init__(self, mapping_attribute, dtype, max_dim_x, max_dim_y=0, access=AttrWriteType.READ, **kwargs):
@@ -60,12 +63,12 @@ class mapped_attribute(attribute):
self.fget = read_func_wrapper
- super().__init__(dtype=dtype, max_dim_y=max_dim_y, max_dim_x=max_dim_x, access=access, fisallowed="is_attribute_access_allowed", **kwargs)
+ super().__init__(dtype=dtype, max_dim_y=max_dim_y, max_dim_x=max_dim_x, access=access,
+ fisallowed="is_attribute_access_allowed", **kwargs)
@device_logging_to_python()
class AntennaField(lofar_device):
-
""" Manages the antennas in a single antenna field, by acting as a
a mapping onto one or more RECV devices.
@@ -88,8 +91,8 @@ class AntennaField(lofar_device):
calculated, as well as the geohash.
"""
- MAX_NUMBER_OF_HBAT = constants.MAX_NUMBER_OF_HBAT
- NUMBER_OF_ELEMENTS_PER_TILE = constants.N_te
+ MAX_NUMBER_OF_HBAT = constants.MAX_NUMBER_OF_HBAT
+ N_te = constants.N_te
# ----- Antenna names
@@ -97,7 +100,7 @@ class AntennaField(lofar_device):
doc="Name of each antenna",
dtype='DevVarStringArray',
mandatory=False,
- default_value = [f'Antenna{n+1}' for n in range(MAX_NUMBER_OF_HBAT)]
+ default_value=[f'Antenna{n + 1}' for n in range(MAX_NUMBER_OF_HBAT)]
)
# ----- Antenna states
@@ -106,23 +109,30 @@ class AntennaField(lofar_device):
doc="Operational quality state of each antenna",
dtype='DevVarUShortArray',
mandatory=False,
- default_value = numpy.array([AntennaQuality.OK] * MAX_NUMBER_OF_HBAT)
+ default_value=numpy.array([AntennaQuality.OK] * MAX_NUMBER_OF_HBAT)
)
Antenna_Use = device_property(
doc="Operational State of each antenna",
dtype='DevVarUShortArray',
mandatory=False,
- default_value = numpy.array([AntennaUse.AUTO] * MAX_NUMBER_OF_HBAT)
+ default_value=numpy.array([AntennaUse.AUTO] * MAX_NUMBER_OF_HBAT)
)
# ----- Antenna properties
+ Antenna_Type = device_property(
+ doc="Type of antenna in this field (LBA or HBA)",
+ dtype='DevString',
+ mandatory=False,
+ default_value = "LBA"
+ )
+
Antenna_Needs_Power = device_property(
doc="Whether to provide power to each antenna (False for noise sources)",
dtype='DevVarBooleanArray',
mandatory=False,
- default_value = numpy.array([False] * MAX_NUMBER_OF_HBAT)
+ default_value=numpy.array([False] * MAX_NUMBER_OF_HBAT)
)
# ----- Position information
@@ -155,37 +165,52 @@ class AntennaField(lofar_device):
doc="Reference frame in which the ITRF coordinates are provided, or are to be computed from ETRS89",
dtype='DevString',
mandatory=False,
- default_value = "ITRF2005"
+ default_value="ITRF2005"
)
ITRF_Reference_Epoch = device_property(
doc="Reference epoch in which the ITRF coordinates are provided, or are to be extrapolated from ETRS89",
dtype='DevFloat',
mandatory=False,
- default_value = 2015.5
+ default_value=2015.5
)
HBAT_PQR_rotation_angles_deg = device_property(
doc='Rotation of each tile in the PQ plane ("horizontal") in degrees.',
dtype='DevVarFloatArray',
mandatory=False,
- default_value = [0.0] * MAX_NUMBER_OF_HBAT
+ default_value=[0.0] * MAX_NUMBER_OF_HBAT
)
PQR_to_ETRS_rotation_matrix = device_property(
doc="Field-specific rotation matrix to convert PQR offsets to ETRS/ITRF offsets.",
dtype='DevVarFloatArray',
mandatory=False,
- default_value = numpy.array([ # PQR->ETRS rotation matrix for the core stations
- [-0.1195951054, -0.7919544517, 0.5987530018],
- [ 0.9928227484, -0.0954186800, 0.0720990002],
- [ 0.0000330969, 0.6030782884, 0.7976820024]]).flatten()
+ default_value=numpy.array([ # PQR->ETRS rotation matrix for the core stations
+ [-0.1195951054, -0.7919544517, 0.5987530018],
+ [0.9928227484, -0.0954186800, 0.0720990002],
+ [0.0000330969, 0.6030782884, 0.7976820024]]).flatten()
)
HBAT_base_antenna_offsets = device_property(
doc="Offsets of the antennas in a HBAT, with respect to its reference center (16x3).",
dtype='DevVarFloatArray',
mandatory=False,
- default_value = HBATAntennaOffsets.HBAT1_BASE_ANTENNA_OFFSETS.flatten()
+ default_value=HBATAntennaOffsets.HBAT1_BASE_ANTENNA_OFFSETS.flatten()
+ )
+
+ # ----- SDP mapping
+
+ Antenna_to_SDP_Mapping = device_property(
+ dtype=(numpy.int32,),
+ doc='The mapping of Antennas to FPGA input pairs. Each FPGA can handle 6 inputs, and SDP has 16 FPGAs. Each antenna is represented with a (fpga, input) value pair. The array is flattened, so must be reshaped upon use. An input=-1 means the antenna is unconnected.',
+ default_value=numpy.array([-1] * MAX_NUMBER_OF_HBAT * 2, dtype=numpy.int32)
+ )
+
+ SDP_device = device_property(
+ dtype=str,
+ doc='Which SDP device is processing this AntennaField.',
+ mandatory=False,
+ default_value="STAT/SDP/1"
)
# ----- RECV mapping
@@ -194,56 +219,70 @@ class AntennaField(lofar_device):
dtype=(numpy.int32,),
doc='The mapping of Antenna power lines to RECV mapping. Each RECV can handle 96 inputs. The Antenna number is the index and the value shows to which receiver device it is connected and on which input. The first integer is the input. The second integer is the RECV id. Example: [0, 3] = first receiver of property RECV_devices with input 3. -1 means that the Antenna is not connected. The property is stored in a one dimensional structure. It needs to be reshaped to a list of lists of two items.',
mandatory=False,
- default_value = [-1] * MAX_NUMBER_OF_HBAT * 2
+ default_value=[-1] * MAX_NUMBER_OF_HBAT * 2
)
Control_to_RECV_mapping = device_property(
dtype=(numpy.int32,),
doc='The mapping of Antenna control lines to RECV mapping. Each RECV can handle 96 inputs. The Antenna number is the index and the value shows to which receiver device it is connected and on which input. The first integer is the input. The second interger is the RECV id. Example: [1, 3] = STAT/RECV/1 with input 3. -1 means that the Antenna is not connected. The property is stored in a one dimensional structure. It needs to be reshaped to a list of lists of two items.',
mandatory=False,
- default_value = [-1] * MAX_NUMBER_OF_HBAT * 2
- )
-
- Antenna_to_FPGA_mapping = device_property(
- dtype=(numpy.int32,),
- doc='The mapping of Antenna control lines to FPGA inputs. Each array element describes which antenna of the AntennaField is connected there (-1 = no antenna connected)',
- mandatory=False,
- default_value = [-1] * MAX_NUMBER_OF_HBAT
+ default_value=[-1] * MAX_NUMBER_OF_HBAT * 2
)
RECV_devices = device_property(
dtype=(str,),
doc='Which Recv devices are in use by the AntennaField. The order is important and it should match up with the order of the mapping.',
mandatory=False,
- default_value = []
+ default_value=[]
)
+ Antenna_Type_R = attribute(doc='The type of antenna in this field (LBA or HBA).',
+ dtype=str)
+
Antenna_Names_R = attribute(access=AttrWriteType.READ,
- dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT)
+ dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT)
Antenna_Quality_R = attribute(doc='The quality of each antenna. 0=OK, 1=SUSPICIOUS, 2=BROKEN, 3=BEYOND_REPAIR.',
- dtype=(numpy.uint32,), max_dim_x=MAX_NUMBER_OF_HBAT)
- Antenna_Use_R = attribute(doc='Whether each antenna should be used. 0=AUTO, 1=ON, 2=OFF. In AUTO mode, the antenna is used if it is not BROKEN or BEYOND_REPAIR.',
+ dtype=(numpy.uint32,), max_dim_x=MAX_NUMBER_OF_HBAT)
+ Antenna_Use_R = attribute(
+ doc='Whether each antenna should be used. 0=AUTO, 1=ON, 2=OFF. In AUTO mode, the antenna is used if it is not BROKEN or BEYOND_REPAIR.',
dtype=(numpy.uint32,), max_dim_x=MAX_NUMBER_OF_HBAT)
Antenna_Quality_str_R = attribute(doc='The quality of each antenna, as a string.',
- dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT)
+ dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT)
Antenna_Use_str_R = attribute(doc='Whether each antenna should be used, as a string.',
- dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT)
+ dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT)
Antenna_Usage_Mask_R = attribute(doc='Whether each antenna will be used.',
- dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT)
-
- ANT_mask_RW = mapped_attribute("ANT_mask_RW", dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
- RCU_PWR_ANT_on_R = mapped_attribute("RCU_PWR_ANT_on_R", dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT)
- RCU_PWR_ANT_on_RW = mapped_attribute("RCU_PWR_ANT_on_RW", dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
- HBAT_BF_delay_steps_R = mapped_attribute("HBAT_BF_delay_steps_R", dtype=((numpy.int64,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT)
- HBAT_BF_delay_steps_RW = mapped_attribute("HBAT_BF_delay_steps_RW", dtype=((numpy.int64,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
- HBAT_LED_on_R = mapped_attribute("HBAT_LED_on_R", dtype=((bool,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT)
- HBAT_LED_on_RW = mapped_attribute("HBAT_LED_on_RW", dtype=((bool,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
- HBAT_PWR_LNA_on_R = mapped_attribute("HBAT_PWR_LNA_on_R", dtype=((bool,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT)
- HBAT_PWR_LNA_on_RW = mapped_attribute("HBAT_PWR_LNA_on_RW", dtype=((bool,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
- HBAT_PWR_on_R = mapped_attribute("HBAT_PWR_on_R", dtype=((bool,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT)
- HBAT_PWR_on_RW = mapped_attribute("HBAT_PWR_on_RW", dtype=((bool,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
- RCU_band_select_RW = mapped_attribute("RCU_band_select_RW", dtype=(numpy.int64,), max_dim_x=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
+
+ dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT)
+
+ Antenna_to_SDP_Mapping_R = attribute(doc='To which (fpga, input) pair each antenna is connected. -1=unconnected.',
+ dtype=((numpy.int32,),), max_dim_x=constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT)
+
+ ANT_mask_RW = mapped_attribute("ANT_mask_RW", dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT,
+ access=AttrWriteType.READ_WRITE)
+ RCU_PWR_ANT_on_R = mapped_attribute("RCU_PWR_ANT_on_R", dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT)
+ RCU_PWR_ANT_on_RW = mapped_attribute("RCU_PWR_ANT_on_RW", dtype=(bool,), max_dim_x=MAX_NUMBER_OF_HBAT,
+ access=AttrWriteType.READ_WRITE)
+ HBAT_BF_delay_steps_R = mapped_attribute("HBAT_BF_delay_steps_R", dtype=((numpy.int64,),),
+ max_dim_x=N_te * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT)
+ HBAT_BF_delay_steps_RW = mapped_attribute("HBAT_BF_delay_steps_RW", dtype=((numpy.int64,),),
+ max_dim_x=N_te * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT,
+ access=AttrWriteType.READ_WRITE)
+ HBAT_LED_on_R = mapped_attribute("HBAT_LED_on_R", dtype=((bool,),), max_dim_x=N_te * constants.N_pol,
+ max_dim_y=MAX_NUMBER_OF_HBAT)
+ HBAT_LED_on_RW = mapped_attribute("HBAT_LED_on_RW", dtype=((bool,),), max_dim_x=N_te * constants.N_pol,
+ max_dim_y=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
+ HBAT_PWR_LNA_on_R = mapped_attribute("HBAT_PWR_LNA_on_R", dtype=((bool,),),
+ max_dim_x=N_te * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT)
+ HBAT_PWR_LNA_on_RW = mapped_attribute("HBAT_PWR_LNA_on_RW", dtype=((bool,),),
+ max_dim_x=N_te * constants.N_pol, max_dim_y=MAX_NUMBER_OF_HBAT,
+ access=AttrWriteType.READ_WRITE)
+ HBAT_PWR_on_R = mapped_attribute("HBAT_PWR_on_R", dtype=((bool,),), max_dim_x=N_te * constants.N_pol,
+ max_dim_y=MAX_NUMBER_OF_HBAT)
+ HBAT_PWR_on_RW = mapped_attribute("HBAT_PWR_on_RW", dtype=((bool,),), max_dim_x=N_te * constants.N_pol,
+ max_dim_y=MAX_NUMBER_OF_HBAT, access=AttrWriteType.READ_WRITE)
+ RCU_band_select_RW = mapped_attribute("RCU_band_select_RW", dtype=(numpy.int64,), max_dim_x=MAX_NUMBER_OF_HBAT,
+ access=AttrWriteType.READ_WRITE)
# ----- Position information
@@ -252,33 +291,37 @@ class AntennaField(lofar_device):
dtype=(numpy.float64,), max_dim_x=constants.N_xyz)
Antenna_Field_Reference_GEO_R = attribute(access=AttrWriteType.READ,
- doc='Absolute reference position of antenna field, in latitude/longitude (degrees)',
- dtype=(numpy.float64,), max_dim_x=constants.N_coord)
+ doc='Absolute reference position of antenna field, in latitude/longitude (degrees)',
+ dtype=(numpy.float64,), max_dim_x=constants.N_coord)
Antenna_Field_Reference_GEOHASH_R = attribute(access=AttrWriteType.READ,
- doc='Absolute reference position of antenna field, as a geohash string',
- dtype=str)
+ doc='Absolute reference position of antenna field, as a geohash string',
+ dtype=str)
HBAT_antenna_ITRF_offsets_R = attribute(access=AttrWriteType.READ,
doc='For each tile, the offsets of the antennas within that, in ITRF ("iHBADeltas"). True shape: nrtiles x 16 x 3.',
- dtype=((numpy.float64,),), max_dim_x=NUMBER_OF_ELEMENTS_PER_TILE * constants.N_xyz, max_dim_y=MAX_NUMBER_OF_HBAT)
+ dtype=((numpy.float64,),), max_dim_x=MAX_NUMBER_OF_HBAT * constants.N_xyz,
+ max_dim_y=MAX_NUMBER_OF_HBAT)
Antenna_Reference_ITRF_R = attribute(access=AttrWriteType.READ,
doc='Absolute reference position of each tile, in ITRF (XYZ)',
dtype=((numpy.float64,),), max_dim_x=constants.N_xyz, max_dim_y=MAX_NUMBER_OF_HBAT)
Antenna_Reference_GEO_R = attribute(access=AttrWriteType.READ,
- doc='Absolute reference position of each tile, in latitude/longitude (degrees)',
- dtype=((numpy.float64,),), max_dim_x=constants.N_coord, max_dim_y=MAX_NUMBER_OF_HBAT)
+ doc='Absolute reference position of each tile, in latitude/longitude (degrees)',
+ dtype=((numpy.float64,),), max_dim_x=constants.N_coord, max_dim_y=MAX_NUMBER_OF_HBAT)
Antenna_Reference_GEOHASH_R = attribute(access=AttrWriteType.READ,
- doc='Absolute reference position of each tile, as geohash strings',
- dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT,)
+ doc='Absolute reference position of each tile, as geohash strings',
+ dtype=(str,), max_dim_x=MAX_NUMBER_OF_HBAT, )
nr_antennas_R = attribute(
doc='Number of Antennas in this field',
dtype=numpy.int32)
+ def read_Antenna_Type_R(self):
+ return self.Antenna_Type
+
def read_Antenna_Names_R(self):
return self.Antenna_Names
@@ -295,14 +338,17 @@ class AntennaField(lofar_device):
return [AntennaQuality(x).name for x in self.Antenna_Quality]
def read_Antenna_Usage_Mask_R(self):
- use = numpy.array(self.Antenna_Use)
+ use = numpy.array(self.Antenna_Use)
quality = numpy.array(self.Antenna_Quality)
antennas_forced_on = use == AntennaUse.ON
- antennas_auto_on = numpy.logical_and(use == AntennaUse.AUTO, quality <= AntennaQuality.SUSPICIOUS)
+ antennas_auto_on = numpy.logical_and(use == AntennaUse.AUTO, quality <= AntennaQuality.SUSPICIOUS)
return numpy.logical_or(antennas_forced_on, antennas_auto_on)
+ def read_Antenna_to_SDP_Mapping_R(self):
+ return numpy.array(self.Antenna_to_SDP_Mapping).reshape(-1, 2)
+
def read_nr_antennas_R(self):
# The number of antennas should be equal to:
# * the number of elements in the Control_to_RECV_mapping (after reshaping),
@@ -310,6 +356,7 @@ class AntennaField(lofar_device):
# * the number of antennas exposed through Antenna_Reference_ITRF_R.
# * the number of elements in Antenna_Use
# * the number of elements in Antenna_Quality
+ # * the number of elements in Antenna_to_SDP_Mapping
#
# Parsing a property here is quickest, so we chose that.
return len(self.Control_to_RECV_mapping) // 2
@@ -345,17 +392,17 @@ class AntennaField(lofar_device):
tiles lie on the same plane in ITRF. """
# the relative offsets between the elements is fixed in HBAT_base_antenna_offsets
- base_antenna_offsets = numpy.array(self.HBAT_base_antenna_offsets).reshape(constants.N_te, constants.N_xyz)
+ base_antenna_offsets = numpy.array(self.HBAT_base_antenna_offsets).reshape(constants.N_te, constants.N_xyz)
PQR_to_ETRS_rotation_matrix = numpy.array(self.PQR_to_ETRS_rotation_matrix).reshape(constants.N_xyz, constants.N_xyz)
# each tile has its own rotation angle, resulting in different offsets per tile
all_offsets = numpy.array(
- [HBATAntennaOffsets.ITRF_offsets(
- base_antenna_offsets,
- angle_deg * pi / 180,
- PQR_to_ETRS_rotation_matrix)
- for angle_deg in self.HBAT_PQR_rotation_angles_deg])
+ [HBATAntennaOffsets.ITRF_offsets(
+ base_antenna_offsets,
+ angle_deg * pi / 180,
+ PQR_to_ETRS_rotation_matrix)
+ for angle_deg in self.HBAT_PQR_rotation_angles_deg])
return all_offsets.reshape(-1, constants.N_te * constants.N_xyz)
@@ -366,6 +413,7 @@ class AntennaField(lofar_device):
# calculate them from ETRS coordinates if not, using the configured ITRF reference
ETRS_coordinates = numpy.array(self.Antenna_Reference_ETRS).reshape(-1, constants.N_xyz)
+
return ETRS_to_ITRF(ETRS_coordinates, self.ITRF_Reference_Frame, self.ITRF_Reference_Epoch)
def read_Antenna_Reference_GEO_R(self):
@@ -374,7 +422,7 @@ class AntennaField(lofar_device):
def read_Antenna_Reference_GEOHASH_R(self):
return GEO_to_GEOHASH(self.read_Antenna_Reference_GEO_R())
- def __setup_all_receiver_proxies(self):
+ def __setup_all_proxies(self):
self.recv_proxies = []
for recv in self.RECV_devices:
@@ -383,6 +431,9 @@ class AntennaField(lofar_device):
recv_proxy.set_source(DevSource.DEV)
self.recv_proxies.append(recv_proxy)
+ self.sdp_proxy = DeviceProxy(self.SDP_device)
+ self.sdp_proxy.set_source(DevSource.DEV)
+
def __setup_mapper(self):
number_of_receivers = len(self.RECV_devices)
# Reshape of mapping is needed because properties are stored in 1d arrays
@@ -427,12 +478,21 @@ class AntennaField(lofar_device):
@log_exceptions()
def configure_for_initialise(self):
super().configure_for_initialise()
- self.__setup_all_receiver_proxies()
+ self.__setup_all_proxies()
self.__setup_mapper()
@log_exceptions()
def _prepare_hardware(self):
- usage_mask = self.read_attribute('Antenna_Usage_Mask_R')
+ # Configure the devices that process our antennas
+ self.configure_recv()
+ self.configure_sdp()
+
+ # --------
+ # Commands
+ # --------
+ @command()
+ def configure_recv(self):
+ """ Configure RECV to process our antennas. """
# Disable controlling the tiles that fall outside the mask
# WARN: Needed in configure_for_initialise but Tango does not allow to write attributes in INIT state
@@ -441,11 +501,24 @@ class AntennaField(lofar_device):
# Turn on power to antennas that need it (and due to the ANT_mask, that we're using)
self.proxy.write_attribute('RCU_PWR_ANT_on_RW', self.Antenna_Needs_Power)
- # --------
- # Commands
- # --------
+ @command()
+ def configure_sdp(self):
+ """ Configure SDP to process our antennas. """
+
+ # Upload which antenna type we're using
+
+ # read-modify-write on [fpga][(input, polarisation)]
+ sdp_antenna_type = numpy.array(self.sdp_proxy.antenna_type_RW, dtype=object)
+ for fpga_nr, input_nr in self.read_attribute("Antenna_to_SDP_Mapping_R"):
+ # set for x polarisation
+ sdp_antenna_type[fpga_nr, input_nr * 2 + 0] = self.Antenna_Type
+ # set for y polarisation
+ sdp_antenna_type[fpga_nr, input_nr * 2 + 1] = self.Antenna_Type
+
+ self.sdp_proxy.antenna_type_RW = tuple(sdp_antenna_type)
+
@command(dtype_in=DevVarFloatArray, dtype_out=DevVarLongArray)
- def calculate_HBAT_bf_delay_steps(self, delays: numpy.ndarray):
+ def calculate_HBAT_bf_delay_steps(self, delays: numpy.ndarray):
num_tiles = self.read_nr_antennas_R()
delays = delays.reshape(num_tiles, constants.N_te)
@@ -455,7 +528,7 @@ class AntennaField(lofar_device):
for recv_idx, recv_proxy in enumerate(self.recv_proxies):
# collect all delays for this recv_proxy
- recv_result_indices = numpy.where(control_mapping[:,0] == (recv_idx + 1))
+ recv_result_indices = numpy.where(control_mapping[:, 0] == (recv_idx + 1))
recv_delays = delays[recv_result_indices]
if not recv_result_indices:
@@ -463,7 +536,8 @@ class AntennaField(lofar_device):
continue
# convert them into delay steps
- flatten_delay_steps = numpy.array(recv_proxy.calculate_HBAT_bf_delay_steps(recv_delays.flatten()), dtype=numpy.int64)
+ flatten_delay_steps = numpy.array(recv_proxy.calculate_HBAT_bf_delay_steps(recv_delays.flatten()),
+ dtype=numpy.int64)
delay_steps = numpy.reshape(flatten_delay_steps, (-1, constants.N_te * constants.N_pol))
# write back into same positions we collected them from
@@ -490,33 +564,39 @@ class AntennaToRecvMapper(object):
self._power_mapping = power_to_recv_mapping
self._number_of_receivers = number_of_receivers
self._default_value_mapping_read = {
- "ANT_mask_RW": value_map_ant_bool,
- "RCU_PWR_ANT_on_R": value_map_ant_bool,
- "RCU_PWR_ANT_on_RW": value_map_ant_bool,
- "HBAT_BF_delay_steps_R": value_map_ant_32_int,
- "HBAT_BF_delay_steps_RW": value_map_ant_32_int,
- "HBAT_LED_on_R": value_map_ant_32_bool,
- "HBAT_LED_on_RW": value_map_ant_32_bool,
- "HBAT_PWR_LNA_on_R": value_map_ant_32_bool,
- "HBAT_PWR_LNA_on_RW": value_map_ant_32_bool,
- "HBAT_PWR_on_R": value_map_ant_32_bool,
- "HBAT_PWR_on_RW": value_map_ant_32_bool,
- "RCU_band_select_RW": numpy.zeros(number_of_antennas, dtype=numpy.int64)
+ "ANT_mask_RW": value_map_ant_bool,
+ "HBAT_BF_delay_steps_R": value_map_ant_32_int,
+ "HBAT_BF_delay_steps_RW": value_map_ant_32_int,
+ "HBAT_LED_on_R": value_map_ant_32_bool,
+ "HBAT_LED_on_RW": value_map_ant_32_bool,
+ "HBAT_PWR_LNA_on_R": value_map_ant_32_bool,
+ "HBAT_PWR_LNA_on_RW": value_map_ant_32_bool,
+ "HBAT_PWR_on_R": value_map_ant_32_bool,
+ "HBAT_PWR_on_RW": value_map_ant_32_bool,
+ "RCU_PWR_ANT_on_R": value_map_ant_bool,
+ "RCU_PWR_ANT_on_RW": value_map_ant_bool,
+ "RCU_band_select_RW": numpy.zeros(number_of_antennas, dtype=numpy.int64)
}
self._masked_value_mapping_write = {
- "ANT_mask_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96,
- "RCU_PWR_ANT_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96,
- "HBAT_BF_delay_steps_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
- "HBAT_LED_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
- "HBAT_PWR_LNA_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
- "HBAT_PWR_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
- "RCU_band_select_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96,
+ "ANT_mask_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96,
+ "HBAT_BF_delay_steps_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
+ "HBAT_LED_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
+ "HBAT_PWR_LNA_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
+ "HBAT_PWR_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96_32,
+ "RCU_PWR_ANT_on_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96,
+ "RCU_band_select_RW": AntennaToRecvMapper._VALUE_MAP_NONE_96,
}
self._reshape_attributes_in = {
"HBAT_BF_delay_steps_RW": (constants.MAX_NUMBER_OF_HBAT, constants.N_rcu),
+ "RCU_PWR_ANT_on_R": (constants.MAX_NUMBER_OF_HBAT,),
+ "RCU_PWR_ANT_on_RW": (constants.MAX_NUMBER_OF_HBAT,),
+ "RCU_band_select_RW": (constants.MAX_NUMBER_OF_HBAT,),
}
self._reshape_attributes_out = {
"HBAT_BF_delay_steps_RW": (constants.MAX_NUMBER_OF_HBAT, constants.N_rcu),
+ "RCU_PWR_ANT_on_R": (constants.N_rcu, constants.N_rcu_inp),
+ "RCU_PWR_ANT_on_RW": (constants.N_rcu, constants.N_rcu_inp),
+ "RCU_band_select_RW": (constants.N_rcu, constants.N_rcu_inp),
}
def map_read(self, mapped_attribute: str, recv_results: List[any]) -> List[any]:
@@ -533,15 +613,14 @@ class AntennaToRecvMapper(object):
if mapped_attribute in self._reshape_attributes_in:
recv_results = numpy.reshape(recv_results,
- (self._number_of_receivers,) + self._reshape_attributes_in[mapped_attribute])
+ (self._number_of_receivers,) + self._reshape_attributes_in[mapped_attribute])
return self._mapped_r_values(recv_results, default_values)
def map_write(self, mapped_attribute: str, set_values: List[any]) -> List[any]:
"""Perform a mapped write for the attribute using the set_values
- :param mapped_attribute: attribute identifier as present in
- py:attribute:`~_default_value_mapping_write`
+ :param mapped_attribute: attribute identifier as present in py:attribute:`~_default_value_mapping_write`
:param set_values: The values to be set for the specified attribute
:return: set_values as mapped given attribute dimensions and control mapping
"""
@@ -586,6 +665,7 @@ class AntennaToRecvMapper(object):
return mapped_values
+
# ----------
# Run server
# ----------
diff --git a/tangostationcontrol/tangostationcontrol/devices/docker_device.py b/tangostationcontrol/tangostationcontrol/devices/docker_device.py
index 8ee1301c6eeb89d91fa166bc4691a9d84b0bb77a..71e13119de523f829092d040ab62bc11330ac4ad 100644
--- a/tangostationcontrol/tangostationcontrol/devices/docker_device.py
+++ b/tangostationcontrol/tangostationcontrol/devices/docker_device.py
@@ -103,6 +103,10 @@ class Docker(lofar_device):
elk_RW = attribute_wrapper(comms_annotation={"container": "elk"}, datatype=bool, access=AttrWriteType.READ_WRITE)
grafana_R = attribute_wrapper(comms_annotation={"container": "grafana"}, datatype=bool)
grafana_RW = attribute_wrapper(comms_annotation={"container": "grafana"}, datatype=bool, access=AttrWriteType.READ_WRITE)
+ logstash_R = attribute_wrapper(comms_annotation={"container": "logstash"}, datatype=bool)
+ logstash_RW = attribute_wrapper(comms_annotation={"container": "logstash"}, datatype=bool, access=AttrWriteType.READ_WRITE)
+ loki_R = attribute_wrapper(comms_annotation={"container": "loki"}, datatype=bool)
+ loki_RW = attribute_wrapper(comms_annotation={"container": "loki"}, datatype=bool, access=AttrWriteType.READ_WRITE)
hdbppts_cm_R = attribute_wrapper(comms_annotation={"container": "hdbppts-cm"}, datatype=bool)
hdbppts_cm_RW = attribute_wrapper(comms_annotation={"container": "hdbppts-cm"}, datatype=bool, access=AttrWriteType.READ_WRITE)
hdbppts_es_R = attribute_wrapper(comms_annotation={"container": "hdbppts-es"}, datatype=bool)
diff --git a/tangostationcontrol/tangostationcontrol/devices/recv.py b/tangostationcontrol/tangostationcontrol/devices/recv.py
index 7d7cba44c2f8da666a6c8f8c5db780cf47b0cfeb..a08abce8656750e05e4fe7f67ba2217ca8a466ce 100644
--- a/tangostationcontrol/tangostationcontrol/devices/recv.py
+++ b/tangostationcontrol/tangostationcontrol/devices/recv.py
@@ -29,17 +29,20 @@ from tangostationcontrol.devices.device_decorators import only_in_states
from tangostationcontrol.devices.opcua_device import opcua_device
import logging
+
logger = logging.getLogger()
__all__ = ["RECV", "main"]
+
@device_logging_to_python()
class RECV(opcua_device):
S_ant = constants.S_ant
N_rcu = constants.N_rcu
- NUMBER_OF_ELEMENTS_PER_TILE = constants.N_te
+ N_te = constants.N_te
N_pol = constants.N_pol
+ N_rcu_inp = constants.N_rcu_inp
FILTER_RCU_DICT = {
"LBA_10_90": 1,
@@ -84,7 +87,8 @@ class RECV(opcua_device):
RCU_PWR_ANT_on_RW_default = device_property(
dtype='DevVarBooleanArray',
mandatory=False,
- default_value=[False] * S_ant # turn power off by default in test setups, f.e. to prevent blowing up the noise sources
+ default_value=[False] * S_ant
+ # turn power off by default in test setups, f.e. to prevent blowing up the noise sources
)
RECVTR_monitor_rate_RW_default = device_property(
@@ -128,62 +132,80 @@ class RECV(opcua_device):
10.0463E-9, 10.5774E-9, 11.0509E-9, 11.5289E-9, 11.9374E-9,
12.4524E-9, 13.0842E-9, 13.5936E-9, 13.9198E-9, 14.4087E-9,
14.9781E-9, 15.5063E-9
- ],dtype=numpy.float64)
+ ], dtype=numpy.float64)
)
HBAT_signal_input_delays = device_property(
doc='Signal input delay calibration values for the elements within a tile.',
dtype='DevVarFloatArray',
mandatory=False,
- default_value = numpy.zeros((N_rcu,), dtype=numpy.float64)
+ default_value=numpy.zeros((N_rcu,), dtype=numpy.float64)
)
# ----------
# Attributes
# ----------
- ANT_mask_RW = attribute_wrapper(comms_annotation=["ANT_mask_RW" ],datatype=bool , dims=(S_ant,), access=AttrWriteType.READ_WRITE)
+
+ ANT_mask_RW = attribute_wrapper(comms_annotation=["ANT_mask_RW"], datatype=bool, dims=(S_ant,),
+ access=AttrWriteType.READ_WRITE)
# The HBAT beamformer delays represent 32 delays for each of the 96 inputs.
# The 32 delays deconstruct as delays[polarisation][dipole], and each delay is the number of 'delay steps' to apply (0.5ns for HBAT1).
- HBAT_BF_delay_steps_R = attribute_wrapper(comms_annotation=["HBAT_BF_delay_steps_R" ],datatype=numpy.int64 , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE))
- HBAT_BF_delay_steps_RW = attribute_wrapper(comms_annotation=["HBAT_BF_delay_steps_RW" ],datatype=numpy.int64 , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE), access=AttrWriteType.READ_WRITE)
- HBAT_LED_on_R = attribute_wrapper(comms_annotation=["HBAT_LED_on_R" ],datatype=bool , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE))
- HBAT_LED_on_RW = attribute_wrapper(comms_annotation=["HBAT_LED_on_RW" ],datatype=bool , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE), access=AttrWriteType.READ_WRITE)
- HBAT_PWR_LNA_on_R = attribute_wrapper(comms_annotation=["HBAT_PWR_LNA_on_R" ],datatype=bool , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE))
- HBAT_PWR_LNA_on_RW = attribute_wrapper(comms_annotation=["HBAT_PWR_LNA_on_RW" ],datatype=bool , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE), access=AttrWriteType.READ_WRITE)
- HBAT_PWR_on_R = attribute_wrapper(comms_annotation=["HBAT_PWR_on_R" ],datatype=bool , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE))
- HBAT_PWR_on_RW = attribute_wrapper(comms_annotation=["HBAT_PWR_on_RW" ],datatype=bool , dims=(S_ant,N_pol,NUMBER_OF_ELEMENTS_PER_TILE), access=AttrWriteType.READ_WRITE)
- RCU_ADC_locked_R = attribute_wrapper(comms_annotation=["RCU_ADC_locked_R" ],datatype=bool , dims=(S_ant,))
- RCU_attenuator_dB_R = attribute_wrapper(comms_annotation=["RCU_attenuator_dB_R" ],datatype=numpy.int64 , dims=(S_ant,))
- RCU_attenuator_dB_RW = attribute_wrapper(comms_annotation=["RCU_attenuator_dB_RW" ],datatype=numpy.int64 , dims=(S_ant,), access=AttrWriteType.READ_WRITE)
- RCU_band_select_R = attribute_wrapper(comms_annotation=["RCU_band_select_R" ],datatype=numpy.int64 , dims=(S_ant,))
- RCU_band_select_RW = attribute_wrapper(comms_annotation=["RCU_band_select_RW" ],datatype=numpy.int64 , dims=(S_ant,), access=AttrWriteType.READ_WRITE)
- RCU_DTH_freq_R = attribute_wrapper(comms_annotation=["RCU_DTH_freq_R" ],datatype=numpy.int64 , dims=(S_ant,))
- RCU_DTH_freq_RW = attribute_wrapper(comms_annotation=["RCU_DTH_freq_RW" ],datatype=numpy.int64 , dims=(S_ant,), access=AttrWriteType.READ_WRITE)
- RCU_DTH_on_R = attribute_wrapper(comms_annotation=["RCU_DTH_on_R" ],datatype=bool , dims=(S_ant,))
- RCU_LED_green_on_R = attribute_wrapper(comms_annotation=["RCU_LED_green_on_R" ],datatype=bool , dims=(N_rcu,))
- RCU_LED_green_on_RW = attribute_wrapper(comms_annotation=["RCU_LED_green_on_RW" ],datatype=bool , dims=(N_rcu,), access=AttrWriteType.READ_WRITE)
- RCU_LED_red_on_R = attribute_wrapper(comms_annotation=["RCU_LED_red_on_R" ],datatype=bool , dims=(N_rcu,))
- RCU_LED_red_on_RW = attribute_wrapper(comms_annotation=["RCU_LED_red_on_RW" ],datatype=bool , dims=(N_rcu,), access=AttrWriteType.READ_WRITE)
- RCU_mask_RW = attribute_wrapper(comms_annotation=["RCU_mask_RW" ],datatype=bool , dims=(N_rcu,), access=AttrWriteType.READ_WRITE)
- RCU_PCB_ID_R = attribute_wrapper(comms_annotation=["RCU_PCB_ID_R" ],datatype=numpy.int64 , dims=(N_rcu,))
- RCU_PCB_number_R = attribute_wrapper(comms_annotation=["RCU_PCB_number_R" ],datatype=str , dims=(N_rcu,))
- RCU_PCB_version_R = attribute_wrapper(comms_annotation=["RCU_PCB_version_R" ],datatype=str , dims=(N_rcu,))
- RCU_PWR_1V8_R = attribute_wrapper(comms_annotation=["RCU_PWR_1V8_R" ],datatype=numpy.float64, dims=(N_rcu,))
- RCU_PWR_2V5_R = attribute_wrapper(comms_annotation=["RCU_PWR_2V5_R" ],datatype=numpy.float64, dims=(N_rcu,))
- RCU_PWR_3V3_R = attribute_wrapper(comms_annotation=["RCU_PWR_3V3_R" ],datatype=numpy.float64, dims=(N_rcu,))
- RCU_PWR_ANALOG_on_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANALOG_on_R" ],datatype=bool , dims=(N_rcu,))
- RCU_PWR_ANT_IOUT_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_IOUT_R" ],datatype=numpy.float64, dims=(S_ant,))
- RCU_PWR_ANT_on_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_on_R" ],datatype=bool , dims=(S_ant,))
- RCU_PWR_ANT_on_RW = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_on_RW" ],datatype=bool , dims=(S_ant,), access=AttrWriteType.READ_WRITE)
- RCU_PWR_ANT_VIN_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_VIN_R" ],datatype=numpy.float64, dims=(S_ant,))
- RCU_PWR_ANT_VOUT_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_VOUT_R" ],datatype=numpy.float64, dims=(S_ant,))
- RCU_PWR_DIGITAL_on_R = attribute_wrapper(comms_annotation=["RCU_PWR_DIGITAL_on_R" ],datatype=bool , dims=(N_rcu,))
- RCU_PWR_good_R = attribute_wrapper(comms_annotation=["RCU_PWR_good_R" ],datatype=bool , dims=(N_rcu,))
- RCU_TEMP_R = attribute_wrapper(comms_annotation=["RCU_TEMP_R" ],datatype=numpy.float64, dims=(N_rcu,))
- RECVTR_I2C_error_R = attribute_wrapper(comms_annotation=["RECVTR_I2C_error_R" ],datatype=numpy.int64 , dims=(N_rcu,))
- RECVTR_monitor_rate_RW = attribute_wrapper(comms_annotation=["RECVTR_monitor_rate_RW" ],datatype=numpy.int64 , access=AttrWriteType.READ_WRITE)
- RECVTR_translator_busy_R = attribute_wrapper(comms_annotation=["RECVTR_translator_busy_R" ],datatype=bool)
+ HBAT_BF_delay_steps_R = attribute_wrapper(comms_annotation=["HBAT_BF_delay_steps_R"], datatype=numpy.int64,
+ dims=(S_ant, N_te, N_pol))
+ HBAT_BF_delay_steps_RW = attribute_wrapper(comms_annotation=["HBAT_BF_delay_steps_RW"], datatype=numpy.int64,
+ dims=(S_ant, N_te, N_pol), access=AttrWriteType.READ_WRITE)
+ HBAT_LED_on_R = attribute_wrapper(comms_annotation=["HBAT_LED_on_R"], datatype=bool, dims=(S_ant, N_te, N_pol))
+ HBAT_LED_on_RW = attribute_wrapper(comms_annotation=["HBAT_LED_on_RW"], datatype=bool, dims=(S_ant, N_te, N_pol),
+ access=AttrWriteType.READ_WRITE)
+ HBAT_PWR_LNA_on_R = attribute_wrapper(comms_annotation=["HBAT_PWR_LNA_on_R"], datatype=bool, dims=(S_ant, N_te, N_pol))
+ HBAT_PWR_LNA_on_RW = attribute_wrapper(comms_annotation=["HBAT_PWR_LNA_on_RW"], datatype=bool, dims=(S_ant, N_te, N_pol),
+ access=AttrWriteType.READ_WRITE)
+ HBAT_PWR_on_R = attribute_wrapper(comms_annotation=["HBAT_PWR_on_R"], datatype=bool, dims=(S_ant, N_te, N_pol))
+ HBAT_PWR_on_RW = attribute_wrapper(comms_annotation=["HBAT_PWR_on_RW"], datatype=bool, dims=(S_ant, N_te, N_pol),
+ access=AttrWriteType.READ_WRITE)
+ RCU_ADC_locked_R = attribute_wrapper(comms_annotation=["RCU_ADC_locked_R"], datatype=bool, dims=(N_rcu, N_rcu_inp))
+ RCU_attenuator_dB_R = attribute_wrapper(comms_annotation=["RCU_attenuator_dB_R"], datatype=numpy.int64,
+ dims=(N_rcu, N_rcu_inp))
+ RCU_attenuator_dB_RW = attribute_wrapper(comms_annotation=["RCU_attenuator_dB_RW"], datatype=numpy.int64,
+ dims=(N_rcu, N_rcu_inp), access=AttrWriteType.READ_WRITE)
+ RCU_band_select_R = attribute_wrapper(comms_annotation=["RCU_band_select_R"], datatype=numpy.int64, dims=(N_rcu, N_rcu_inp))
+ RCU_band_select_RW = attribute_wrapper(comms_annotation=["RCU_band_select_RW"], datatype=numpy.int64, dims=(N_rcu, N_rcu_inp),
+ access=AttrWriteType.READ_WRITE)
+ RCU_DTH_freq_R = attribute_wrapper(comms_annotation=["RCU_DTH_freq_R"], datatype=numpy.int64, dims=(N_rcu, N_rcu_inp))
+ RCU_DTH_freq_RW = attribute_wrapper(comms_annotation=["RCU_DTH_freq_RW"], datatype=numpy.int64, dims=(N_rcu, N_rcu_inp),
+ access=AttrWriteType.READ_WRITE)
+ RCU_DTH_on_R = attribute_wrapper(comms_annotation=["RCU_DTH_on_R"], datatype=bool, dims=(N_rcu, N_rcu_inp))
+ RCU_LED_green_on_R = attribute_wrapper(comms_annotation=["RCU_LED_green_on_R"], datatype=bool, dims=(N_rcu,))
+ RCU_LED_green_on_RW = attribute_wrapper(comms_annotation=["RCU_LED_green_on_RW"], datatype=bool, dims=(N_rcu,),
+ access=AttrWriteType.READ_WRITE)
+ RCU_LED_red_on_R = attribute_wrapper(comms_annotation=["RCU_LED_red_on_R"], datatype=bool, dims=(N_rcu,))
+ RCU_LED_red_on_RW = attribute_wrapper(comms_annotation=["RCU_LED_red_on_RW"], datatype=bool, dims=(N_rcu,),
+ access=AttrWriteType.READ_WRITE)
+ RCU_mask_RW = attribute_wrapper(comms_annotation=["RCU_mask_RW"], datatype=bool, dims=(N_rcu,),
+ access=AttrWriteType.READ_WRITE)
+ RCU_PCB_ID_R = attribute_wrapper(comms_annotation=["RCU_PCB_ID_R"], datatype=numpy.int64, dims=(N_rcu,))
+ RCU_PCB_number_R = attribute_wrapper(comms_annotation=["RCU_PCB_number_R"], datatype=str, dims=(N_rcu,))
+ RCU_PCB_version_R = attribute_wrapper(comms_annotation=["RCU_PCB_version_R"], datatype=str, dims=(N_rcu,))
+ RCU_PWR_1V8_R = attribute_wrapper(comms_annotation=["RCU_PWR_1V8_R"], datatype=numpy.float64, dims=(N_rcu,))
+ RCU_PWR_2V5_R = attribute_wrapper(comms_annotation=["RCU_PWR_2V5_R"], datatype=numpy.float64, dims=(N_rcu,))
+ RCU_PWR_3V3_R = attribute_wrapper(comms_annotation=["RCU_PWR_3V3_R"], datatype=numpy.float64, dims=(N_rcu,))
+ RCU_PWR_ANALOG_on_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANALOG_on_R"], datatype=bool, dims=(N_rcu,))
+ RCU_PWR_ANT_IOUT_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_IOUT_R"], datatype=numpy.float64,
+ dims=(N_rcu, N_rcu_inp))
+ RCU_PWR_ANT_on_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_on_R"], datatype=bool, dims=(N_rcu, N_rcu_inp))
+ RCU_PWR_ANT_on_RW = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_on_RW"], datatype=bool, dims=(N_rcu, N_rcu_inp),
+ access=AttrWriteType.READ_WRITE)
+ RCU_PWR_ANT_VIN_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_VIN_R"], datatype=numpy.float64, dims=(N_rcu, N_rcu_inp))
+ RCU_PWR_ANT_VOUT_R = attribute_wrapper(comms_annotation=["RCU_PWR_ANT_VOUT_R"], datatype=numpy.float64,
+ dims=(N_rcu, N_rcu_inp))
+ RCU_PWR_DIGITAL_on_R = attribute_wrapper(comms_annotation=["RCU_PWR_DIGITAL_on_R"], datatype=bool, dims=(N_rcu,))
+ RCU_PWR_good_R = attribute_wrapper(comms_annotation=["RCU_PWR_good_R"], datatype=bool, dims=(N_rcu,))
+ RCU_TEMP_R = attribute_wrapper(comms_annotation=["RCU_TEMP_R"], datatype=numpy.float64, dims=(N_rcu,))
+ RECVTR_I2C_error_R = attribute_wrapper(comms_annotation=["RECVTR_I2C_error_R"], datatype=numpy.int64, dims=(N_rcu,))
+ RECVTR_monitor_rate_RW = attribute_wrapper(comms_annotation=["RECVTR_monitor_rate_RW"], datatype=numpy.int64,
+ access=AttrWriteType.READ_WRITE)
+ RECVTR_translator_busy_R = attribute_wrapper(comms_annotation=["RECVTR_translator_busy_R"], datatype=bool)
# ----------
# Summarising Attributes
@@ -191,48 +213,50 @@ class RECV(opcua_device):
RCU_LED_colour_R = attribute(dtype=(numpy.uint32,), max_dim_x=N_rcu, fisallowed="is_attribute_access_allowed")
def read_RCU_LED_colour_R(self):
- return (2 * self.read_attribute("RCU_LED_green_on_R") + 4 * self.read_attribute("RCU_LED_red_on_R")).astype(numpy.uint32)
+ return (2 * self.read_attribute("RCU_LED_green_on_R") + 4 * self.read_attribute("RCU_LED_red_on_R")).astype(
+ numpy.uint32)
- RCU_error_R = attribute(dtype=(bool,), max_dim_x=N_rcu, fisallowed="is_attribute_access_allowed")
- ANT_error_R = attribute(dtype=(bool,), max_dim_x=S_ant, fisallowed="is_attribute_access_allowed")
+ RCU_error_R = attribute(dtype=(bool,), max_dim_x=N_rcu, fisallowed="is_attribute_access_allowed")
+ ANT_error_R = attribute(dtype=(bool,), max_dim_x=S_ant, fisallowed="is_attribute_access_allowed")
def read_RCU_error_R(self):
return self.read_attribute("RCU_mask_RW") & (
- (self.read_attribute("RECVTR_I2C_error_R") > 0)
- | self.alarm_val("RCU_PCB_ID_R")
- )
+ (self.read_attribute("RECVTR_I2C_error_R") > 0)
+ | self.alarm_val("RCU_PCB_ID_R")
+ )
def read_ANT_error_R(self):
return self.read_attribute("ANT_mask_RW") & (
- ~self.read_attribute("RCU_ADC_locked_R")
- )
+ ~self.read_attribute("RCU_ADC_locked_R").flatten()
+ )
- RECV_IOUT_error_R = attribute(dtype=(bool,), max_dim_x=S_ant, fisallowed="is_attribute_access_allowed")
- RECV_TEMP_error_R = attribute(dtype=(bool,), max_dim_x=N_rcu, fisallowed="is_attribute_access_allowed", polling_period=constants.DEFAULT_POLLING_PERIOD)
- RECV_VOUT_error_R = attribute(dtype=(bool,), max_dim_x=N_rcu, fisallowed="is_attribute_access_allowed")
+ RECV_IOUT_error_R = attribute(dtype=(bool,), max_dim_x=S_ant, fisallowed="is_attribute_access_allowed")
+ RECV_TEMP_error_R = attribute(dtype=(bool,), max_dim_x=N_rcu, fisallowed="is_attribute_access_allowed",
+ polling_period=constants.DEFAULT_POLLING_PERIOD)
+ RECV_VOUT_error_R = attribute(dtype=(bool,), max_dim_x=N_rcu, fisallowed="is_attribute_access_allowed")
def read_RECV_IOUT_error_R(self):
return self.read_attribute("ANT_mask_RW") & (
- self.alarm_val("RCU_PWR_ANT_IOUT_R")
- )
+ self.alarm_val("RCU_PWR_ANT_IOUT_R").flatten()
+ )
def read_RECV_TEMP_error_R(self):
# Don't apply the mask here --- we always want to know if things get too hot!
return (
- self.alarm_val("RCU_TEMP_R")
- )
+ self.alarm_val("RCU_TEMP_R")
+ )
def read_RECV_VOUT_error_R(self):
return (self.read_attribute("ANT_mask_RW") & (
- self.alarm_val("RCU_PWR_ANT_VIN_R")
- | self.alarm_val("RCU_PWR_ANT_VOUT_R")
- )).reshape(constants.N_rcu,constants.N_rcu_inp).any(axis=1) | (self.read_attribute("RCU_mask_RW") & (
- self.alarm_val("RCU_PWR_1V8_R")
- | self.alarm_val("RCU_PWR_2V5_R")
- | self.alarm_val("RCU_PWR_3V3_R")
- | ~self.read_attribute("RCU_PWR_DIGITAL_on_R")
- | ~self.read_attribute("RCU_PWR_good_R")
- ))
+ self.alarm_val("RCU_PWR_ANT_VIN_R").flatten()
+ | self.alarm_val("RCU_PWR_ANT_VOUT_R").flatten()
+ )).reshape(constants.N_rcu,constants.N_rcu_inp).any(axis=1) | (self.read_attribute("RCU_mask_RW") & (
+ self.alarm_val("RCU_PWR_1V8_R")
+ | self.alarm_val("RCU_PWR_2V5_R")
+ | self.alarm_val("RCU_PWR_3V3_R")
+ | ~self.read_attribute("RCU_PWR_DIGITAL_on_R")
+ | ~self.read_attribute("RCU_PWR_good_R")
+ ))
# --------
# overloaded functions
@@ -280,7 +304,7 @@ class RECV(opcua_device):
which is a value per tile per dipole per polarisation.
"""
# Duplicate delay values per polarisation
- polarised_delays = numpy.tile(delays, constants.N_pol) # output dims -> 96x32
+ polarised_delays = numpy.repeat(delays, constants.N_pol, axis=1) # output dims -> 96x32
# Add signal input delay
calibrated_delays = numpy.add(polarised_delays, self.HBAT_signal_input_delays)
@@ -301,7 +325,7 @@ class RECV(opcua_device):
def calculate_HBAT_bf_delay_steps(self, delays: numpy.ndarray):
""" converts a signal path delay (in seconds) to an analog beam weight """
- # Reshape the flatten input array, into whatever how many tiles we get
+ # Reshape the flattened input array, into whatever how many tiles we get
delays = numpy.array(delays).reshape(-1, constants.N_te)
# Calculate the beam weight array
@@ -354,6 +378,7 @@ class RECV(opcua_device):
"""
self.opcua_connection.call_method(["RCU_DTH_on"])
+
# ----------
# Run server
# ----------
diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py b/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py
index 63486ef8edb947292b929929baf06265371b4b8a..ad21851b4478f4a864fb7de755c01f8ae783437f 100644
--- a/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py
+++ b/tangostationcontrol/tangostationcontrol/devices/sdp/beamlet.py
@@ -42,6 +42,21 @@ class Beamlet(opcua_device):
# Device Properties
# -----------------
+ FPGA_beamlet_output_hdr_eth_source_mac_RW_default = device_property(
+ dtype='DevVarStringArray',
+ mandatory=True
+ )
+
+ FPGA_beamlet_output_hdr_ip_source_address_RW_default = device_property(
+ dtype='DevVarStringArray',
+ mandatory=True
+ )
+
+ FPGA_beamlet_output_hdr_udp_source_port_RW_default = device_property(
+ dtype='DevVarUShortArray',
+ mandatory=True
+ )
+
FPGA_beamlet_output_hdr_eth_destination_mac_RW_default = device_property(
dtype='DevVarStringArray',
mandatory=True
@@ -82,6 +97,9 @@ class Beamlet(opcua_device):
)
FIRST_DEFAULT_SETTINGS = [
+ 'FPGA_beamlet_output_hdr_eth_source_mac_RW',
+ 'FPGA_beamlet_output_hdr_ip_source_address_RW',
+ 'FPGA_beamlet_output_hdr_udp_source_port_RW',
'FPGA_beamlet_output_hdr_eth_destination_mac_RW',
'FPGA_beamlet_output_hdr_ip_destination_address_RW',
'FPGA_beamlet_output_hdr_udp_destination_port_RW',
@@ -94,13 +112,22 @@ class Beamlet(opcua_device):
# ----------
FPGA_beamlet_output_enable_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_enable_R"], datatype=bool, dims=(N_pn,))
- FPGA_beamlet_output_enable_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_enable_RW"], datatype=bool, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
+ FPGA_beamlet_output_enable_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_enable_RW"], datatype=bool, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
+
+ FPGA_beamlet_output_hdr_eth_source_mac_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_eth_source_mac_R"], datatype=str, dims=(N_pn,))
+ FPGA_beamlet_output_hdr_eth_source_mac_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_eth_source_mac_RW"], datatype=str, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
+ FPGA_beamlet_output_hdr_ip_source_address_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_ip_source_address_R"], datatype=str, dims=(N_pn,))
+ FPGA_beamlet_output_hdr_ip_source_address_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_ip_source_address_RW"], datatype=str, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
+ FPGA_beamlet_output_hdr_udp_source_port_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_udp_source_port_R"], datatype=numpy.uint16, dims=(N_pn,))
+ FPGA_beamlet_output_hdr_udp_source_port_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_udp_source_port_RW"], datatype=numpy.uint16, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
+
FPGA_beamlet_output_hdr_eth_destination_mac_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_eth_destination_mac_R"], datatype=str, dims=(N_pn,))
FPGA_beamlet_output_hdr_eth_destination_mac_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_eth_destination_mac_RW"], datatype=str, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
FPGA_beamlet_output_hdr_ip_destination_address_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_ip_destination_address_R"], datatype=str, dims=(N_pn,))
FPGA_beamlet_output_hdr_ip_destination_address_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_ip_destination_address_RW"], datatype=str, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
FPGA_beamlet_output_hdr_udp_destination_port_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_udp_destination_port_R"], datatype=numpy.uint16, dims=(N_pn,))
FPGA_beamlet_output_hdr_udp_destination_port_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_hdr_udp_destination_port_RW"], datatype=numpy.uint16, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
+
FPGA_beamlet_output_scale_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_scale_R"], datatype=numpy.double, dims=(N_pn,))
FPGA_beamlet_output_scale_RW = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_scale_RW"], datatype=numpy.double, dims=(N_pn,), access=AttrWriteType.READ_WRITE)
FPGA_beamlet_output_bsn_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_bsn_R"], datatype=numpy.int64, dims=(N_pn, N_beamsets_ctrl))
@@ -113,7 +140,7 @@ class Beamlet(opcua_device):
# boolean[N_pn][N_beamsets_ctrl]
FPGA_beamlet_output_xon_R = attribute_wrapper(comms_annotation=["FPGA_beamlet_output_xon_R"], datatype=bool, dims=(N_pn, N_beamsets_ctrl))
- # uint16[N_pn][A_pn][N_pol][N_beamlets_ctrl]
+ # uint16[N_pn][A_PN][N_POL][N_beamsets_ctrl]
# Select subband per dual-polarisation beamlet.
# 0 for antenna polarization X in beamlet polarization X,
# 1 for antenna polarization Y in beamlet polarization Y.
@@ -290,14 +317,14 @@ class Beamlet(opcua_device):
return bf_weights.reshape(orig_shape)
@staticmethod
- def _subband_frequencies(subbands: numpy.ndarray, clock: int, nyquist_zone: int) -> numpy.ndarray:
+ def _subband_frequencies(subbands: numpy.ndarray, clock: int, nyquist_zones: numpy.ndarray) -> numpy.ndarray:
""" Obtain the frequencies of each subband, given a clock and an antenna type. """
subband_width = clock / constants.N_sub_pts
- base_subband = nyquist_zone * constants.N_sub
+ base_subbands = nyquist_zones * constants.N_sub
# broadcast clock across frequencies
- frequencies = (subbands + base_subband) * subband_width
+ frequencies = (subbands + base_subbands) * subband_width
return frequencies
@@ -310,7 +337,13 @@ class Beamlet(opcua_device):
# obtain which subband is selected for each input and beamlet
beamlet_subbands = self.read_attribute("FPGA_beamlet_subband_select_RW") # (fpga_nr, [input_nr][pol][beamlet_nr])
- return self._subband_frequencies(beamlet_subbands, self.sdp_proxy.clock_RW, self.sdp_proxy.nyquist_zone_R)
+ nyquist_zones = self.sdp_proxy.nyquist_zone_R # (fpga_nr, [input_nr][pol])
+
+ # repeat nyquist zone for all beamlets, to match the shape of beamlet_subbands
+ nyquist_zones = numpy.repeat(nyquist_zones, self.N_BEAMLETS_CTRL, axis=1)
+
+ # compute the frequency of each beamlet for each input
+ return self._subband_frequencies(beamlet_subbands, self.sdp_proxy.clock_RW, nyquist_zones)
@staticmethod
def _calculate_bf_weights(delays: numpy.ndarray, beamlet_frequencies: numpy.ndarray):
diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/digitalbeam.py b/tangostationcontrol/tangostationcontrol/devices/sdp/digitalbeam.py
index 633a8d56988d254674d5f46ea540e1935773c52b..92c5c578f589e5aeb16407a75b1ec4b266fd93ae 100644
--- a/tangostationcontrol/tangostationcontrol/devices/sdp/digitalbeam.py
+++ b/tangostationcontrol/tangostationcontrol/devices/sdp/digitalbeam.py
@@ -44,6 +44,9 @@ class DigitalBeam(beam_device):
"""
+ # number of antennas connected to each FPGA in SDP
+ A_pn = 6
+
# -----------------
# Device Properties
# -----------------
@@ -62,13 +65,6 @@ class DigitalBeam(beam_device):
default_value = "STAT/Beamlet/1"
)
- Input_to_Antenna_Mapping = device_property(
- dtype=(numpy.int32,),
- doc='Which antenna of the antennafield is connected to each input. -1 if no antenna is present.',
- mandatory=False,
- default_value = [-1] * constants.S_ant
- )
-
# ----------
# Attributes
# ----------
@@ -88,7 +84,7 @@ class DigitalBeam(beam_device):
def nr_inputs(self):
""" Return the number of configured inputs. """
- return max(self.Input_to_Antenna_Mapping) + 1
+ return len(self.antennafield_proxy.Antenna_to_SDP_Mapping_R)
def read_input_select_RW(self):
return self._input_select
@@ -100,9 +96,9 @@ class DigitalBeam(beam_device):
# select only the rows from self.__input_select for which a mapping onto antennas is defined.
antenna_select = [[False] * constants.N_beamlets_ctrl] * self.nr_inputs()
- for input_nr, antenna_nr in enumerate(self.Input_to_Antenna_Mapping):
- if antenna_nr >= 0:
- antenna_select[antenna_nr] = self._input_select[input_nr]
+ for antenna_nr, (fpga_nr, input_nr) in enumerate(self.antennafield_proxy.Antenna_to_SDP_Mapping_R):
+ if input_nr >= 0:
+ antenna_select[antenna_nr] = self._input_select[fpga_nr * self.A_pn + input_nr]
return antenna_select
@@ -111,14 +107,14 @@ class DigitalBeam(beam_device):
# to select the antennas they would like to use.
antenna_usage_mask = self.antennafield_proxy.Antenna_Usage_Mask_R
- for input_nr, antenna_nr in enumerate(self.Input_to_Antenna_Mapping):
- if antenna_nr >= 0:
+ for antenna_nr, (fpga_nr, input_nr) in enumerate(self.antennafield_proxy.Antenna_to_SDP_Mapping_R):
+ if input_nr >= 0:
if antenna_usage_mask[antenna_nr]:
# use antenna for the beamlets as supplied by the client
- self._input_select[input_nr] = antennas[antenna_nr]
+ self._input_select[fpga_nr * self.A_pn + input_nr] = antennas[antenna_nr]
else:
# do not use antenna for any beamlet
- self._input_select[input_nr] = False
+ self._input_select[fpga_nr * self.A_pn + input_nr] = False
# ----------
# Summarising Attributes
@@ -148,10 +144,11 @@ class DigitalBeam(beam_device):
# Generate positions for all FPGA inputs.
# Use reference position for any missing antennas so they always get a delay of 0
- input_itrf = numpy.array([reference_itrf] * constants.S_ant)
- for input_nr, antenna_nr in enumerate(self.Input_to_Antenna_Mapping):
- if antenna_nr >= 0:
- input_itrf[input_nr] = antenna_itrf[antenna_nr]
+ input_itrf = numpy.array([reference_itrf] * constants.MAX_INPUTS)
+ for antenna_nr, (fpga_nr, input_nr) in enumerate(self.antennafield_proxy.Antenna_to_SDP_Mapping_R):
+ if input_nr >= 0:
+ input_itrf[fpga_nr * constants.A_pn + input_nr] = antenna_itrf[antenna_nr]
+
# a delay calculator
self.delay_calculator = Delays(reference_itrf)
diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py b/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py
index e1c742d2e47c27d8f605b806d5061e3ac24d815c..794d5ebaa349596cb227cb6eec068b9a3d70f582 100644
--- a/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py
+++ b/tangostationcontrol/tangostationcontrol/devices/sdp/sdp.py
@@ -39,13 +39,6 @@ class SDP(opcua_device):
# Device Properties
# -----------------
- AntennaType = device_property(
- doc='Antenna type (LBA or HBA) we control',
- dtype='DevString',
- mandatory=False,
- default_value = "HBA"
- )
-
TR_fpga_mask_RW_default = device_property(
dtype='DevVarBooleanArray',
mandatory=False,
@@ -188,15 +181,36 @@ class SDP(opcua_device):
FPGA_bst_offload_bsn_R = attribute_wrapper(comms_annotation=["FPGA_bst_offload_bsn_R"], datatype=numpy.int64, dims=(N_pn, constants.N_beamsets_ctrl))
- antenna_type_R = attribute(doc='Type of antenna (LBA or HBA) attached to the FPGAs',
- dtype=str, fget=lambda self: self.AntennaType)
- nyquist_zone_R = attribute(doc='Nyquist zone of the input frequencies',
- dtype=numpy.uint32, fisallowed="is_attribute_access_allowed",
+ antenna_type_RW = attribute(doc='Type of antenna (LBA or HBA) attached to each input of the FPGAs',
+ dtype=((str,),), max_dim_y=N_pn, max_dim_x=S_pn,
+ access=AttrWriteType.READ_WRITE, fisallowed="is_attribute_access_allowed")
+ nyquist_zone_R = attribute(doc='Nyquist zone of each input.',
+ dtype=((numpy.uint32,),), max_dim_y=N_pn, max_dim_x=S_pn,
+ fisallowed="is_attribute_access_allowed",
polling_period=constants.DEFAULT_POLLING_PERIOD, abs_change=1)
clock_RW = attribute(doc='Configured sampling clock (Hz)',
dtype=numpy.uint32, access=AttrWriteType.READ_WRITE, fisallowed="is_attribute_access_allowed",
polling_period=constants.DEFAULT_POLLING_PERIOD, abs_change=1)
+ def read_antenna_type_RW(self):
+ return self._antenna_type
+
+ def write_antenna_type_RW(self, value):
+ # use numpy for easy processing
+ value = numpy.array(value)
+
+ # validate shape
+ if value.shape != (self.N_pn, self.S_pn):
+ raise ValueError(f"Dimension mismatch. Expected ({self.N_pn}, {self.S_pn}), got {value.shape}.")
+
+ # validate content
+ for val in value.flatten():
+ if val not in ["LBA", "HBA"]:
+ raise ValueError(f"Unsupported antenna type: {val}. Must be one of [LBA, HBA].")
+
+ # adopt new value
+ self._antenna_type = value
+
def _nyquist_zone(self, clock):
""" Return the Nyquist zone for the given clock (in Hz).
@@ -205,7 +219,7 @@ class SDP(opcua_device):
NOTE: Only 160 and 200 MHz clocks are supported. """
- # (AntennaType, clockMHz) -> Nyquist zone
+ # (antenna type, clockMHz) -> Nyquist zone
nyquist_zones = {
("LBA", 160): 0,
("LBA", 200): 0,
@@ -213,17 +227,17 @@ class SDP(opcua_device):
("HBA", 200): 2,
}
- try:
- return nyquist_zones[(self.AntennaType), clock // 1000000]
- except KeyError:
- raise ValueError(f"Could not determine Nyquist zone for antenna type {self.AntennaType} with clock {clock} Hz")
+ def antenna_type_to_nyquist_zone(antenna_type):
+ try:
+ return nyquist_zones[(antenna_type, clock // 1_000_000)]
+ except KeyError:
+ # sane default
+ return 0
+
+ return numpy.vectorize(antenna_type_to_nyquist_zone)(self._antenna_type)
def read_nyquist_zone_R(self):
- try:
- return self._nyquist_zone(self.read_attribute("clock_RW"))
- except ValueError:
- # Supply a sane default for computations in tests until L2SDP-725 allows us to read back the set clock
- return 0
+ return self._nyquist_zone(self.read_attribute("clock_RW"))
def read_clock_RW(self):
# We can only return a single value, so we assume the FPGA is configured coherently. Which is something
@@ -242,8 +256,8 @@ class SDP(opcua_device):
# Tell all FPGAs to use this clock
self.proxy.FPGA_pps_expected_cnt_RW = [clock] * self.N_pn
- # Also update the packet headers
- self.proxy.FPGA_sdp_info_nyquist_sampling_zone_index_RW = [self._nyquist_zone(clock)] * self.N_pn
+ # Also update the packet headers. We assume the first Nyquist zone of each FPGA is representative
+ self.proxy.FPGA_sdp_info_nyquist_sampling_zone_index_RW = self._nyquist_zone(clock)[:,0]
# ----------
# Summarising Attributes
@@ -276,6 +290,14 @@ class SDP(opcua_device):
# overloaded functions
# --------
+ def configure_for_initialise(self):
+ super().configure_for_initialise()
+
+ # Store which type of antenna is connected to each input.
+ #
+ # We need to be told this by AntennaField, through configure_for_antennafield.
+ self._antenna_type = numpy.array([["LBA"] * self.S_pn] * self.N_pn, dtype=str)
+
def _prepare_hardware(self):
# FPGAs that are actually reachable and we care about
wait_for = ~(self.read_attribute("TR_fpga_communication_error_R")) & self.read_attribute("TR_fpga_mask_R")
diff --git a/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py b/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py
index 89da1c4f4362a605185f9b47558696e3f7d74a24..117742d5fe4ac5acf6846664b556f6903f8b9c0c 100644
--- a/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py
+++ b/tangostationcontrol/tangostationcontrol/devices/sdp/statistics.py
@@ -68,6 +68,7 @@ class Statistics(opcua_device):
# when last packet was received
last_packet_timestamp_R = attribute_wrapper(comms_id=StatisticsClient, comms_annotation={"type": "udp", "parameter": "last_packet_timestamp"}, datatype=numpy.uint64)
+
# queue fill percentage, as reported by the consumer
queue_collector_fill_percentage_R = attribute_wrapper(comms_id=StatisticsClient, comms_annotation={"type": "queue", "parameter": "collector_fill_percentage"}, datatype=numpy.uint64)
queue_replicator_fill_percentage_R = attribute_wrapper(comms_id=StatisticsClient, comms_annotation={"type": "queue", "parameter": "replicator_fill_percentage"}, datatype=numpy.uint64)
@@ -84,6 +85,8 @@ class Statistics(opcua_device):
nof_invalid_packets_R = attribute_wrapper(comms_id=StatisticsClient, comms_annotation={"type": "statistics", "parameter": "nof_invalid_packets"}, datatype=numpy.uint64)
# last packet that could not be parsed
last_invalid_packet_R = attribute_wrapper(comms_id=StatisticsClient, comms_annotation={"type": "statistics", "parameter": "last_invalid_packet"}, dims=(constants.MAX_PACKET_SIZE,), datatype=numpy.uint8)
+ # what the last exception was
+ last_invalid_packet_exception_R = attribute_wrapper(comms_id=StatisticsClient, comms_annotation={"type": "statistics", "parameter": "last_invalid_packet_exception"}, datatype=str)
# --------
# Overloaded functions
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py
index cfec09cb457b2039be8feec2fd20d0f45ef3ed04..24e2c1191425cf80389490eeb34fcc2ec45414e5 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_antennafield.py
@@ -24,9 +24,11 @@ class TestAntennaFieldDevice(AbstractTestBases.TestDeviceBase):
"Power_to_RECV_mapping": [1, 1, 1, 0] + [-1] * 92
})
self.recv_proxy = self.setup_recv_proxy()
+ self.sdp_proxy = self.setup_sdp_proxy()
self.addCleanup(self.restore_antennafield)
self.addCleanup(self.shutdown_recv)
+ self.addCleanup(self.shutdown_sdp)
def restore_antennafield(self):
self.proxy.put_property({
@@ -40,6 +42,11 @@ class TestAntennaFieldDevice(AbstractTestBases.TestDeviceBase):
recv_proxy = TestDeviceProxy("STAT/RECV/1")
recv_proxy.off()
+ @staticmethod
+ def shutdown_sdp():
+ sdp_proxy = TestDeviceProxy("STAT/SDP/1")
+ sdp_proxy.off()
+
def setup_recv_proxy(self):
# setup RECV
recv_proxy = TestDeviceProxy("STAT/RECV/1")
@@ -48,6 +55,13 @@ class TestAntennaFieldDevice(AbstractTestBases.TestDeviceBase):
recv_proxy.set_defaults()
return recv_proxy
+ def setup_sdp_proxy(self):
+ # setup SDP
+ sdp_proxy = TestDeviceProxy("STAT/SDP/1")
+ sdp_proxy.off()
+ sdp_proxy.warm_boot()
+ return sdp_proxy
+
def test_property_recv_devices_has_one_receiver(self):
result = self.proxy.get_property("RECV_devices")
self.assertSequenceEqual(result["RECV_devices"], ["STAT/RECV/1"])
@@ -255,20 +269,20 @@ class TestAntennaFieldDevice(AbstractTestBases.TestDeviceBase):
antennafield_proxy.put_property(mapping_properties)
antennafield_proxy.boot()
- self.recv_proxy.write_attribute("RCU_band_select_RW", [False] * 96)
+ self.recv_proxy.write_attribute("RCU_band_select_RW", [[False] * 3] * 32)
try:
antennafield_proxy.write_attribute(
"RCU_band_select_RW", [True] * 96
)
numpy.testing.assert_equal(
- numpy.array([True] * 96),
+ numpy.array([[True] * 3] * 32),
self.recv_proxy.read_attribute("RCU_band_select_RW").value
)
finally:
# Always restore recv again
self.recv_proxy.write_attribute(
- "RCU_band_select_RW", [False] * 96
+ "RCU_band_select_RW", [[False] * 3] * 32
)
# Verify device did not enter FAULT state
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py
index 108491bca005157b42c8200467b4ada841d3ee9f..6aa67918d4b76867109234478c281ad68789da08 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_beamlet.py
@@ -30,12 +30,17 @@ class TestDeviceBeamlet(AbstractTestBases.TestDeviceBase):
super().test_device_read_all_attributes()
- def setup_sdp(self):
+ def setup_sdp(self, antenna_type="HBA", clock=200_000_000):
# setup SDP, on which this device depends
sdp_proxy = TestDeviceProxy("STAT/SDP/1")
sdp_proxy.off()
sdp_proxy.warm_boot()
sdp_proxy.set_defaults()
+
+ # setup the frequencies as expected in the test
+ sdp_proxy.antenna_type_RW = [[antenna_type] * 12] * 16
+ sdp_proxy.clock_RW = clock
+
return sdp_proxy
def test_pointing_to_zenith(self):
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py
index 3c48ad0adbc0704995f0c1b4dbdae6e1dc058532..9a7d22306e6aa842e8976a37c7a1d2b8e1923c1e 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py
@@ -87,8 +87,8 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
TestDeviceProxy.test_device_turn_off, self.antennafield_iden
)
- self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.sdp_proxy = self.setup_sdp_proxy()
+ self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.beamlet_proxy = self.initialise_beamlet_proxy()
self.beamlet_proxy.on()
@@ -126,8 +126,8 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
TestDeviceProxy.test_device_turn_off, self.antennafield_iden
)
- self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.sdp_proxy = self.setup_sdp_proxy()
+ self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.beamlet_proxy = self.initialise_beamlet_proxy()
self.beamlet_proxy.subband_select_RW = numpy.array(list(range(317)) + [316] + list(range(318,488)), dtype=numpy.uint32)
@@ -160,8 +160,8 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
TestDeviceProxy.test_device_turn_off, self.antennafield_iden
)
- self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.setup_sdp_proxy()
+ self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.proxy.initialise()
self.proxy.Tracking_enabled_RW = False
@@ -190,8 +190,8 @@ class TestDeviceDigitalBeam(AbstractTestBases.TestDeviceBase):
TestDeviceProxy.test_device_turn_off, self.antennafield_iden
)
- self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.setup_sdp_proxy()
+ self.setup_antennafield_proxy(self.antenna_qualities_ok, self.antenna_use_ok)
self.proxy.initialise()
self.proxy.Tracking_enabled_RW = False
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py
index 8a4a2b1c65323a83f216c386130de66eb903f07e..46c77246eb5fd876079998c341bc7b4693d48ec3 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation.py
@@ -22,32 +22,24 @@ class TestDeviceObservation(AbstractTestBases.TestDeviceBase):
NUM_TILES = 48
NUM_BEAMLETS_CTRL = 488
NUM_INPUTS = 96
- INPUT_TO_ANTENNA_MAPPING = [
- "0", "1", "2", "3", "4", "5",
- "6", "7", "8", "9", "10", "11",
- "12", "13", "14", "15", "16", "17",
- "18", "19", "20", "21", "22", "23",
- "24", "25", "26", "27", "28", "29",
- "30", "31", "32", "33", "34", "35",
- "36", "37", "38", "39", "40", "41",
- "42", "43", "44", "45", "46", "47",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1"
- ]
+ ANTENNA_TO_SDP_MAPPING = [
+ "0", "0", "0", "1", "0", "2", "0", "3", "0", "4", "0", "5",
+ "1", "0", "1", "1", "1", "2", "1", "3", "1", "4", "1", "5",
+ "2", "0", "2", "1", "2", "2", "2", "3", "2", "4", "2", "5",
+ "3", "0", "3", "1", "3", "2", "3", "3", "3", "4", "3", "5",
+ "4", "0", "4", "1", "4", "2", "4", "3", "4", "4", "4", "5",
+ "5", "0", "5", "1", "5", "2", "5", "3", "5", "4", "5", "5",
+ "6", "0", "6", "1", "6", "2", "6", "3", "6", "4", "6", "5",
+ "7", "0", "7", "1", "7", "2", "7", "3", "7", "4", "7", "5",
+ ]
def setUp(self):
super().setUp("STAT/Observation/1")
self.VALID_JSON = TestObservationBase.VALID_JSON
self.recv_proxy = self.setup_recv_proxy()
+ self.sdp_proxy = self.setup_sdp_proxy()
self.antennafield_proxy = self.setup_antennafield_proxy()
self.beamlet_proxy = self.setup_beamlet_proxy()
- self.sdp_proxy = self.setup_sdp_proxy()
self.digitalbeam_proxy = self.setup_digitalbeam_proxy()
self.tilebeam_proxy = self.setup_tilebeam_proxy()
@@ -74,6 +66,7 @@ class TestDeviceObservation(AbstractTestBases.TestDeviceBase):
antenna_use = numpy.array([AntennaUse.AUTO] * 96)
antennafield_proxy.put_property({"RECV_devices": ["STAT/RECV/1"],
"Control_to_RECV_mapping": numpy.array(control_mapping).flatten(),
+ "Antenna_to_SDP_Mapping": self.ANTENNA_TO_SDP_MAPPING,
'Antenna_Quality': antenna_qualities, 'Antenna_Use': antenna_use})
antennafield_proxy.off()
antennafield_proxy.boot()
@@ -90,7 +83,6 @@ class TestDeviceObservation(AbstractTestBases.TestDeviceBase):
def setup_digitalbeam_proxy(self):
# setup Digitalbeam
digitalbeam_proxy = TestDeviceProxy("STAT/DigitalBeam/1")
- digitalbeam_proxy.put_property({"Input_to_Antenna_Mapping": numpy.array(self.INPUT_TO_ANTENNA_MAPPING).flatten()})
digitalbeam_proxy.off()
digitalbeam_proxy.warm_boot()
digitalbeam_proxy.set_defaults()
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation_control.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation_control.py
index 1e6aff8eff8870e2bc224c9ca3e0c7908363c69f..ac325e0cbcc9f5dd663ce21c104461516819a9cf 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation_control.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_observation_control.py
@@ -24,24 +24,16 @@ class TestObservationControlDevice(AbstractTestBases.TestDeviceBase):
NUM_TILES = 48
NUM_BEAMLETS_CTRL = 488
NUM_INPUTS = 96
- INPUT_TO_ANTENNA_MAPPING = [
- "0", "1", "2", "3", "4", "5",
- "6", "7", "8", "9", "10", "11",
- "12", "13", "14", "15", "16", "17",
- "18", "19", "20", "21", "22", "23",
- "24", "25", "26", "27", "28", "29",
- "30", "31", "32", "33", "34", "35",
- "36", "37", "38", "39", "40", "41",
- "42", "43", "44", "45", "46", "47",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1",
- "-1", "-1", "-1", "-1", "-1", "-1"
- ]
+ ANTENNA_TO_SDP_MAPPING = [
+ "0", "0", "0", "1", "0", "2", "0", "3", "0", "4", "0", "5",
+ "1", "0", "1", "1", "1", "2", "1", "3", "1", "4", "1", "5",
+ "2", "0", "2", "1", "2", "2", "2", "3", "2", "4", "2", "5",
+ "3", "0", "3", "1", "3", "2", "3", "3", "3", "4", "3", "5",
+ "4", "0", "4", "1", "4", "2", "4", "3", "4", "4", "4", "5",
+ "5", "0", "5", "1", "5", "2", "5", "3", "5", "4", "5", "5",
+ "6", "0", "6", "1", "6", "2", "6", "3", "6", "4", "6", "5",
+ "7", "0", "7", "1", "7", "2", "7", "3", "7", "4", "7", "5",
+ ]
def setUp(self):
super().setUp("STAT/ObservationControl/1")
@@ -73,7 +65,8 @@ class TestObservationControlDevice(AbstractTestBases.TestDeviceBase):
antennafield_proxy = TestDeviceProxy("STAT/AntennaField/1")
control_mapping = [[1,i] for i in range(self.NUM_TILES)]
antennafield_proxy.put_property({"RECV_devices": ["STAT/RECV/1"],
- "Power_to_RECV_mapping": numpy.array(control_mapping).flatten()})
+ "Power_to_RECV_mapping": numpy.array(control_mapping).flatten(),
+ "Antenna_to_SDP_Mapping": self.ANTENNA_TO_SDP_MAPPING})
antennafield_proxy.off()
antennafield_proxy.warm_boot()
antennafield_proxy.set_defaults()
@@ -90,7 +83,6 @@ class TestObservationControlDevice(AbstractTestBases.TestDeviceBase):
def setup_digitalbeam_proxy(self):
# setup Digitalbeam
digitalbeam_proxy = TestDeviceProxy("STAT/DigitalBeam/1")
- digitalbeam_proxy.put_property({"Input_to_Antenna_Mapping": numpy.array(self.INPUT_TO_ANTENNA_MAPPING).flatten()})
digitalbeam_proxy.off()
digitalbeam_proxy.warm_boot()
digitalbeam_proxy.set_defaults()
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_tilebeam.py b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_tilebeam.py
index 541e04bc4e878043dfa80569b4342712be22cf0e..c474e6628d695d4fd29eba7e0fed04e296477a8a 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_tilebeam.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/devices/test_device_tilebeam.py
@@ -122,7 +122,7 @@ class TestDeviceTileBeam(AbstractTestBases.TestDeviceBase):
self.proxy.set_pointing(["AZELGEO","0deg","0deg"] * self.NR_TILES)
# obtain delays of the X polarisation of all the elements of the first tile
- north_beam_delay_steps = antennafield_proxy.HBAT_BF_delay_steps_RW[0].reshape(2,4,4)[0]
+ north_beam_delay_steps = antennafield_proxy.HBAT_BF_delay_steps_RW[0].reshape(4,4,2)[:,:,0]
# delays must differ under rotation, or our test will give a false positive
self.assertNotEqual(north_beam_delay_steps.tolist(), numpy.rot90(north_beam_delay_steps).tolist())
@@ -132,7 +132,7 @@ class TestDeviceTileBeam(AbstractTestBases.TestDeviceBase):
self.proxy.set_pointing(["AZELGEO",f"{angle}deg","0deg"] * self.NR_TILES)
# obtain delays of the X polarisation of all the elements of the first tile
- angled_beam_delay_steps = antennafield_proxy.HBAT_BF_delay_steps_RW[0].reshape(2,4,4)[0]
+ angled_beam_delay_steps = antennafield_proxy.HBAT_BF_delay_steps_RW[0].reshape(4,4,2)[:,:,0]
expected_delay_steps = numpy.rot90(north_beam_delay_steps, k=-(angle/90))
@@ -166,7 +166,7 @@ class TestDeviceTileBeam(AbstractTestBases.TestDeviceBase):
# The [3] = 28 diff is explained that we match the closest delay step and LOFAR 1 wants the one with
# in 0.2ns but if it can't it will do a int(delay / 0.5ns) so we get slightly different results but
# they can be explained.
- expected_HBAT_delay_steps = numpy.array([24, 25, 27, 29, 17, 18, 20, 21, 10, 11, 13, 14, 3, 4, 5, 7] * 2, dtype=numpy.int64)
+ expected_HBAT_delay_steps = numpy.repeat(numpy.array([24, 25, 27, 29, 17, 18, 20, 21, 10, 11, 13, 14, 3, 4, 5, 7], dtype=numpy.int64), 2)
numpy.testing.assert_equal(calculated_HBAT_delay_steps[0], expected_HBAT_delay_steps)
numpy.testing.assert_equal(calculated_HBAT_delay_steps[self.NR_TILES - 1], expected_HBAT_delay_steps)
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
index cf730e6461dfc4e438fc77ea8d8150cb59dfa90f..89eb31d8ef6db7f74345aab8b0ba7bf35de741e1 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
@@ -82,25 +82,25 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
'2021-09-20T12:17:40.000+00:00'
)
self.assertIsNotNone(stat)
- self.assertEqual("0.1.1", stat.station_version_id)
+ self.assertEqual("0.2.0", stat.station_version_id)
self.assertEqual("0.1", stat.writer_version_id)
def test_insert_tango_SST_statistics(self):
self.assertEqual(DevState.ON, self.recv_proxy.state())
collector = StationSSTCollector(device=self.recv_proxy)
- # Test attribute values retrieval
+ # Test attribute values retrieval
collector.parse_device_attributes()
numpy.testing.assert_equal(
- collector.parameters["rcu_attenuator_dB"].flatten(),
+ collector.parameters["rcu_attenuator_dB"],
self.recv_proxy.rcu_attenuator_dB_r
)
numpy.testing.assert_equal(
- collector.parameters["rcu_band_select"].flatten(),
+ collector.parameters["rcu_band_select"],
self.recv_proxy.rcu_band_select_r.tolist()
)
numpy.testing.assert_equal(
- collector.parameters["rcu_dth_on"].flatten(),
+ collector.parameters["rcu_dth_on"],
self.recv_proxy.rcu_dth_on_r.tolist()
)
@@ -186,7 +186,7 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
self.assertEqual(stat.rcu_band_select, None)
self.assertEqual(stat.rcu_dth_on, None)
- def test_SST_statistics_with_device_in_off(self):
+ def test_SST_statistics_with_device_in_off(self):
self.setup_recv_proxy()
self.recv_proxy.Off()
self.assertEqual(DevState.OFF, self.recv_proxy.state())
@@ -221,7 +221,7 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
stat = stat_parser.get_statistic('2021-09-20T12:17:40.000+00:00') # same as stat_parser.statistics[0]
self.assertIsNotNone(stat)
self.assertEqual(121, stat.data_id_signal_input_index)
- # Test RECV attributes
+ # Test RECV attributes
self.assertEqual(stat.rcu_attenuator_dB, None)
self.assertEqual(stat.rcu_band_select, None)
self.assertEqual(stat.rcu_dth_on, None)
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/recv_cluster/test_recv_cluster.py b/tangostationcontrol/tangostationcontrol/integration_test/recv_cluster/test_recv_cluster.py
index 6c7aad92b3df90414633109bcd049162a6aec05e..92db0ffbf8607f375c1b5a9993889f0089e44087 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/recv_cluster/test_recv_cluster.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/recv_cluster/test_recv_cluster.py
@@ -35,6 +35,14 @@ class TestRecvCluster(base.IntegrationTestCase):
antenna_field_proxies = []
recv_proxies = []
+ # SDP must be ready before AntennaField
+ sdp_proxy = TestDeviceProxy("STAT/SDP/1")
+ sdp_proxy.off()
+ self.assertTrue(sdp_proxy.state() is DevState.OFF)
+ sdp_proxy.warm_boot()
+ sdp_proxy.set_defaults()
+ self.assertTrue(sdp_proxy.state() is DevState.ON)
+
# Beam / Recv 1,2,3,4
for i in range(1, 5):
recv_proxies.append(TestDeviceProxy(f"STAT/RECV/{i}"))
diff --git a/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py b/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py
index c614ea34a4a4c2fef8c75e3f719c88b8c59bc352..5e73c6ae50986b76d52c376926d623f93b07b0cb 100644
--- a/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py
+++ b/tangostationcontrol/tangostationcontrol/test/devices/test_antennafield_device.py
@@ -224,11 +224,35 @@ class TestAntennaToRecvMapper(base.TestCase):
actual = mapper.map_write("ANT_mask_RW", set_values)
numpy.testing.assert_equal(expected, actual)
+ def test_map_write_rcu_pwr_ant_on_no_mapping_and_one_receiver(self):
+ mapper = AntennaToRecvMapper(self.CONTROL_NOT_CONNECTED, self.POWER_NOT_CONNECTED, 1)
+
+ set_values = [None] * 48
+ expected = [[[None, None, None]] * 32]
+ actual = mapper.map_write("RCU_PWR_ANT_on_RW", set_values)
+ numpy.testing.assert_equal(expected, actual)
+
+ def test_map_write_rcu_pwr_ant_on_no_mapping_and_two_receivers(self):
+ mapper = AntennaToRecvMapper(self.CONTROL_NOT_CONNECTED, self.POWER_NOT_CONNECTED, 2)
+
+ set_values = [None] * 48
+ expected = [[[None, None, None]] * 32] * 2
+ actual = mapper.map_write("RCU_PWR_ANT_on_RW", set_values)
+ numpy.testing.assert_equal(expected, actual)
+
+ def test_map_write_rcu_pwr_ant_on_hba_0_and_1_on_rcu_1_and_0_of_recv_1(self):
+ mapper = AntennaToRecvMapper(self.CONTROL_HBA_0_AND_1_ON_RCU_1_AND_0_OF_RECV_1, self.POWER_NOT_CONNECTED, 1)
+
+ set_values = [1, 0] + [None] * 46
+ expected = [[[0, 1, None]] + [[None, None, None]] * 31]
+ actual = mapper.map_write("RCU_PWR_ANT_on_RW", set_values)
+ numpy.testing.assert_equal(expected, actual)
+
def test_map_write_rcu_band_select_no_mapping_and_one_receiver(self):
mapper = AntennaToRecvMapper(self.CONTROL_NOT_CONNECTED, self.POWER_NOT_CONNECTED, 1)
set_values = [None] * 48
- expected = [[None] * 96]
+ expected = [[[None, None, None]] * 32]
actual = mapper.map_write("RCU_band_select_RW", set_values)
numpy.testing.assert_equal(expected, actual)
@@ -236,7 +260,7 @@ class TestAntennaToRecvMapper(base.TestCase):
mapper = AntennaToRecvMapper(self.CONTROL_NOT_CONNECTED, self.POWER_NOT_CONNECTED, 2)
set_values = [None] * 48
- expected = [[None] * 96] * 2
+ expected = [[[None, None, None]] * 32] * 2
actual = mapper.map_write("RCU_band_select_RW", set_values)
numpy.testing.assert_equal(expected, actual)
@@ -244,7 +268,7 @@ class TestAntennaToRecvMapper(base.TestCase):
mapper = AntennaToRecvMapper(self.CONTROL_HBA_0_AND_1_ON_RCU_1_AND_0_OF_RECV_1, self.POWER_NOT_CONNECTED, 1)
set_values = [1, 0] + [None] * 46
- expected = [[0, 1] + [None] * 94]
+ expected = [[[0, 1, None]] + [[None, None, None]] * 31]
actual = mapper.map_write("RCU_band_select_RW", set_values)
numpy.testing.assert_equal(expected, actual)
diff --git a/tangostationcontrol/tangostationcontrol/test/devices/test_beamlet_device.py b/tangostationcontrol/tangostationcontrol/test/devices/test_beamlet_device.py
index e7b82ec29ccd1e83abb481ed63db897b1a98f2d2..ff43c4ada4f8f73257f32b13c3e1261afd13aa47 100644
--- a/tangostationcontrol/tangostationcontrol/test/devices/test_beamlet_device.py
+++ b/tangostationcontrol/tangostationcontrol/test/devices/test_beamlet_device.py
@@ -97,38 +97,35 @@ class TestBeamletDevice(base.TestCase):
[0, 1, 102],
])
- clocks = numpy.array([
- 200 * 1000000,
- 160 * 1000000
- ])
-
- subband_width = 200e6 / 1024
+ nyquist_zones_0 = numpy.zeros(subbands.shape)
+ nyquist_zones_1 = numpy.ones(subbands.shape)
+ nyquist_zones_2 = numpy.ones(subbands.shape) * 2
# for reference values, see https://proxy.lofar.eu/rtsm/tests/
- lba_frequencies = Beamlet._subband_frequencies(subbands, 160 * 1000000, 0)
+ lba_frequencies = Beamlet._subband_frequencies(subbands, 160 * 1000000, nyquist_zones_0)
self.assertAlmostEqual(lba_frequencies[0][0], 0.0000000e6)
self.assertAlmostEqual(lba_frequencies[0][1], 0.1562500e6)
self.assertAlmostEqual(lba_frequencies[0][2], 15.9375000e6)
- lba_frequencies = Beamlet._subband_frequencies(subbands, 200 * 1000000, 0)
+ lba_frequencies = Beamlet._subband_frequencies(subbands, 200 * 1000000, nyquist_zones_0)
self.assertAlmostEqual(lba_frequencies[0][0], 0.0000000e6)
self.assertAlmostEqual(lba_frequencies[0][1], 0.1953125e6)
self.assertAlmostEqual(lba_frequencies[0][2], 19.9218750e6)
# Nyquist zone 1 is not used in 160 MHz
- hba_low_frequencies = Beamlet._subband_frequencies(subbands, 200 * 1000000, 1)
+ hba_low_frequencies = Beamlet._subband_frequencies(subbands, 200 * 1000000, nyquist_zones_1)
self.assertAlmostEqual(hba_low_frequencies[0][0], 100.0000000e6)
self.assertAlmostEqual(hba_low_frequencies[0][1], 100.1953125e6)
self.assertAlmostEqual(hba_low_frequencies[0][2], 119.9218750e6)
- hba_high_frequencies = Beamlet._subband_frequencies(subbands, 160 * 1000000, 2)
+ hba_high_frequencies = Beamlet._subband_frequencies(subbands, 160 * 1000000, nyquist_zones_2)
self.assertAlmostEqual(hba_high_frequencies[0][0], 160.0000000e6)
self.assertAlmostEqual(hba_high_frequencies[0][1], 160.1562500e6)
self.assertAlmostEqual(hba_high_frequencies[0][2], 175.9375000e6)
- hba_high_frequencies = Beamlet._subband_frequencies(subbands, 200 * 1000000, 2)
+ hba_high_frequencies = Beamlet._subband_frequencies(subbands, 200 * 1000000, nyquist_zones_2)
self.assertAlmostEqual(hba_high_frequencies[0][0], 200.0000000e6)
self.assertAlmostEqual(hba_high_frequencies[0][1], 200.1953125e6)
self.assertAlmostEqual(hba_high_frequencies[0][2], 219.9218750e6)