Skip to content
Snippets Groups Projects
Commit 8ac86cbb authored by Taya Snijder's avatar Taya Snijder
Browse files

Merge branch 'master' of https://git.astron.nl/lofar2.0/tango into... 

Merge branch 'master' of https://git.astron.nl/lofar2.0/tango into L2SS-954_create_single_source_of_truth
parents 8b5caada 9bda21ff
No related branches found
No related tags found
1 merge request!465added constants.py and replaced most magic numbers
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 1097 additions and 126 deletions
.gitlab-ci.yml
+ 24
− 0
View file @ 8ac86cbb
......@@ -97,6 +97,8 @@ docker_build_image_all:
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh ccd-sim latest
......@@ -187,6 +189,28 @@ docker_build_image_grafana:
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
docker_build_image_loki:
extends: .base_docker_images_except
only:
refs:
- merge_requests
changes:
- docker-compose/loki.yml
- docker-compose/loki/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki $tag
docker_build_image_logstash:
extends: .base_docker_images_except
only:
refs:
- merge_requests
changes:
- docker-compose/logstash.yml
- docker-compose/logstash/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash $tag
docker_build_image_jupyter:
extends: .base_docker_images_except
only:
......
CDB/LOFAR_ConfigDb.json
+ 54
− 0
View file @ 8ac86cbb
......@@ -113,6 +113,60 @@
"Beamlet": {
"STAT/Beamlet/1": {
"properties": {
"FPGA_beamlet_output_hdr_eth_source_mac_RW_default": [
"00:22:86:08:00:00",
"00:22:86:08:00:01",
"00:22:86:08:00:02",
"00:22:86:08:00:03",
"00:22:86:08:01:00",
"00:22:86:08:01:01",
"00:22:86:08:01:02",
"00:22:86:08:01:03",
"00:22:86:08:02:00",
"00:22:86:08:02:01",
"00:22:86:08:02:02",
"00:22:86:08:02:03",
"00:22:86:08:03:00",
"00:22:86:08:03:01",
"00:22:86:08:03:02",
"00:22:86:08:03:03"
],
"FPGA_beamlet_output_hdr_ip_source_address_RW_default": [
"192.168.0.1",
"192.168.0.2",
"192.168.0.3",
"192.168.0.4",
"192.168.1.1",
"192.168.1.2",
"192.168.1.3",
"192.168.1.4",
"192.168.2.1",
"192.168.2.2",
"192.168.2.3",
"192.168.2.4",
"192.168.3.1",
"192.168.3.2",
"192.168.3.3",
"192.168.3.4"
],
"FPGA_beamlet_output_hdr_udp_source_port_RW_default": [
"53248",
"53249",
"53250",
"53251",
"53252",
"53253",
"53254",
"53255",
"53256",
"53257",
"53258",
"53259",
"53260",
"53261",
"53262",
"53263"
],
"FPGA_beamlet_output_hdr_udp_destination_port_RW_default": [
"10001",
"10001",
......
CDB/stations/CS001_ConfigDb.json
+ 58
− 4
View file @ 8ac86cbb
......@@ -126,6 +126,60 @@
"OPC_Time_Out": [
"5.0"
],
"FPGA_beamlet_output_hdr_eth_source_mac_RW_default": [
"00:22:86:00:01:00",
"00:22:86:00:01:01",
"00:22:86:00:01:02",
"00:22:86:00:01:03",
"00:22:86:00:01:04",
"00:22:86:00:01:05",
"00:22:86:00:01:06",
"00:22:86:00:01:07",
"00:22:86:00:01:08",
"00:22:86:00:01:09",
"00:22:86:00:01:10",
"00:22:86:00:01:11",
"00:22:86:00:01:12",
"00:22:86:00:01:13",
"00:22:86:00:01:14",
"00:22:86:00:01:15"
],
"FPGA_beamlet_output_hdr_ip_source_address_RW_default": [
"10.175.1.0",
"10.175.1.1",
"10.175.1.2",
"10.175.1.3",
"10.175.1.4",
"10.175.1.5",
"10.175.1.6",
"10.175.1.7",
"10.175.1.8",
"10.175.1.9",
"10.175.1.10",
"10.175.1.11",
"10.175.1.12",
"10.175.1.13",
"10.175.1.14",
"10.175.1.15"
],
"FPGA_beamlet_output_hdr_udp_source_port_RW_default": [
"4346",
"4347",
"4348",
"4349",
"4350",
"4351",
"4352",
"4353",
"4354",
"4355",
"4356",
"4357",
"4358",
"4359",
"4360",
"4361"
],
"FPGA_beamlet_output_hdr_eth_destination_mac_RW_default": [
"3c:ec:ef:86:2f:b7",
"3c:ec:ef:86:2f:b7",
......@@ -220,8 +274,8 @@
],
"Antenna_to_SDP_Mapping": [
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "3", "0", "3", "1", "3", "2", "0", "-1",
"0", "-1", "0", "-1", "3", "3", "3", "4", "3", "5", "0", "-1",
"0", "-1", "0", "-1", "2", "0", "2", "1", "2", "2", "0", "-1",
"0", "-1", "0", "-1", "2", "3", "2", "4", "2", "5", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
......@@ -392,9 +446,9 @@
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "4", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "5",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "6", "1", "7",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "0", "1", "1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1", "0", "-1",
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "8", "0", "-1"
"0", "-1", "0", "-1", "0", "-1", "0", "-1", "1", "2", "0", "-1"
],
"Antenna_Field_Reference_ETRS": [
"3826923.942", "460915.117", "5064643.229"
......
CDB/stations/DTS_ConfigDb.json
+ 42
− 50
View file @ 8ac86cbb
......@@ -102,40 +102,40 @@
"5.0"
],
"FPGA_beamlet_output_hdr_eth_destination_mac_RW_default": [
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1",
"0c:c4:7a:c0:30:f1"
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc",
"ec:0d:9a:bf:f2:dc"
],
"FPGA_beamlet_output_hdr_ip_destination_address_RW_default": [
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250",
"10.99.250.250"
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250",
"192.168.1.250"
]
}
}
......@@ -147,6 +147,9 @@
"AntennaField": {
"STAT/AntennaField/1": {
"properties": {
"Antenna_Type": [
"HBA"
],
"RECV_devices": [
"STAT/RECV/1"
],
......@@ -164,6 +167,13 @@
"1", "28",
"1", "29"
],
"Antenna_to_SDP_Mapping": [
"0", "0",
"0", "1",
"0", "2",
"0", "3",
"0", "4"
],
"Antenna_Field_Reference_ETRS": [
"3839371.416", "430339.901", "5057958.886"
],
......@@ -196,24 +206,6 @@
"DigitalBeam": {
"STAT/DigitalBeam/1": {
"properties": {
"Input_to_Antenna_Mapping": [
"0", "1", "2", "3", "4", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1"
]
}
}
}
......
CDB/stations/DTS_Outside_ConfigDb.json
+ 24
− 39
View file @ 8ac86cbb
......@@ -172,6 +172,9 @@
"AntennaField": {
"STAT/AntennaField/2": {
"properties": {
"Antenna_Type": [
"HBA"
],
"RECV_devices": [
"STAT/RECV/1"
],
......@@ -196,6 +199,13 @@
"1","25",
"1","26"
],
"Antenna_to_SDP_Mapping": [
"3", "0",
"0", "-1",
"0", "-1",
"3", "1",
"3", "2"
],
"Antenna_Field_Reference_ETRS": [
"3839371.416","430339.901","5057958.886"
],
......@@ -222,6 +232,9 @@
},
"STAT/AntennaField/1": {
"properties": {
"Antenna_Type": [
"LBA"
],
"RECV_devices": [
"STAT/RECV/1"
],
......@@ -258,6 +271,17 @@
"1","15",
"1","17"
],
"Antenna_to_SDP_Mapping": [
"0", "0",
"0", "1",
"0", "2",
"0", "3",
"0", "4",
"0", "5",
"1", "0",
"1", "1",
"1", "2"
],
"Antenna_Field_Reference_ETRS": [
"3839358.189",
"430354.482",
......@@ -286,24 +310,6 @@
"properties": {
"AntennaField_Device": [
"STAT/AntennaField/2"
],
"Input_to_Antenna_Mapping": [
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"0", "3", "4", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1"
]
}
},
......@@ -312,24 +318,6 @@
"AntennaField_Device": [
"STAT/AntennaField/1"
],
"Input_to_Antenna_Mapping": [
"0", "1", "2", "3", "4", "5",
"6", "7", "8", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1"
],
"Tracking_enabled_RW_default": [
"False"
]
......@@ -362,9 +350,6 @@
"SDP": {
"STAT/SDP/1": {
"properties": {
"AntennaType": [
"LBA"
],
"OPC_Server_Name": [
"10.99.0.250"
],
......
CDB/stations/dummy_positions_ConfigDb.json
+ 10
− 18
View file @ 8ac86cbb
......@@ -105,6 +105,16 @@
"1", "94",
"1", "95"
],
"Antenna_to_SDP_Mapping": [
"0", "0", "0", "1", "0", "2", "0", "3", "0", "4", "0", "5",
"1", "0", "1", "1", "1", "2", "1", "3", "1", "4", "1", "5",
"2", "0", "2", "1", "2", "2", "2", "3", "2", "4", "2", "5",
"3", "0", "3", "1", "3", "2", "3", "3", "3", "4", "3", "5",
"4", "0", "4", "1", "4", "2", "4", "3", "4", "4", "4", "5",
"5", "0", "5", "1", "5", "2", "5", "3", "5", "4", "5", "5",
"6", "0", "6", "1", "6", "2", "6", "3", "6", "4", "6", "5",
"7", "0", "7", "1", "7", "2", "7", "3", "7", "4", "7", "5"
],
"Antenna_Field_Reference_ETRS": [
"3826896.631", "460979.131", "5064657.943"
],
......@@ -223,24 +233,6 @@
"DigitalBeam": {
"STAT/DigitalBeam/1": {
"properties": {
"Input_to_Antenna_Mapping": [
"0", "1", "2", "3", "4", "5",
"6", "7", "8", "9", "10", "11",
"12", "13", "14", "15", "16", "17",
"18", "19", "20", "21", "22", "23",
"24", "25", "26", "27", "28", "29",
"30", "31", "32", "33", "34", "35",
"36", "37", "38", "39", "40", "41",
"42", "43", "44", "45", "46", "47",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1",
"-1", "-1", "-1", "-1", "-1", "-1"
]
}
}
}
......
README.md
+ 51
− 11
View file @ 8ac86cbb
......@@ -8,11 +8,16 @@ Station Control software related to Tango devices.
# Index
* [Installation](#installation)
* [Prerequisites](#prerequisites)
* [Bootstrap](#bootstrap)
* [User documentation (ReadTheDocs (Sphinx / ReStructuredText))](tangostationcontrol/docs/README.md)
* [Docker compose documentation](docker-compose/README.md)
* [Timescaledb](docker-compose/timescaledb/README.md)
* [Jupyter startup files](docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md)
* [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter)
* [ReadTheDocs (Sphinx / ReStructuredText) documentation](tangostationcontrol/docs/README.md)
* [Developer Documentation](#development)
* [Versioning](#versioning)
* Source code documentation
* [Attribute wrapper documentation](tangostationcontrol/tangostationcontrol/clients/README.md)
* [Archiver documentation](tangostationcontrol/tangostationcontrol/toolkit/README.md)
......@@ -20,6 +25,7 @@ Station Control software related to Tango devices.
* [HDF5 statistics](tangostationcontrol/tangostationcontrol/statistics/README.md)
* [Unit tests](tangostationcontrol/tangostationcontrol/test/README.md)
* [Integration tests](tangostationcontrol/tangostationcontrol/integration_test/README.md)
* [Release Notes](#release-notes)
# Installation
......@@ -41,20 +47,16 @@ You will also need:
## Bootstrap
The bootstrap procedure is needed only once. First we build all docker containers, and load the initial configuration. This may take a while:
The bootstrap procedure is needed only once. First we build all docker
containers, and load the initial configuration. This may take a while:
```
cd docker-compose
make bootstrap
```
If you lack access to LOFAR station hardware, configure the devices to use their simulators instead:
```
for sim in ../CDB/*-sim-config.json; do
../sbin/update_ConfigDb.sh ../CDB${sim}-config.json
done
```
By default bootstrap will configure the station to use simulators. You can
lookup alternative configurations in the CDB directory.
Now we can start all containers, and make sure everything is up:
......@@ -63,6 +65,44 @@ make start
make status
```
If not, you can inspect why with `docker logs <container>`. The containers will automatically be restarted on reboot or failure. Stop them explicitly to bring them down (`make stop <container>`).
If not, you can inspect why with `docker logs <container>`. The containers will
automatically be restarted on reboot or failure. Stop them explicitly to bring
them down (`make stop <container>`).
Most notably, you will have web interfaces available at:
- http://localhost:8888 (Jupyter Notebook)
- http://localhost:3000 (Grafana).
# Development
For development you will need several dependencies including:
```
git g++ gcc make docker docker-compose shellcheck graphviz python3-dev \
python3-pip python3-tox libboost-python-dev libtango-cpp pkg-config
```
Of these docker-compose must be at least 2.0 and Python 3.7 or higher.
Alternatively, tox can be installed through pip using `pip install tox`.
Finally, running unit tests relies on availability of casacore data see:
[lofar-device-base Dockerfile](docker-compose/lofar-device-base/Dockerfile)
for details.
## Versioning
When changing behavior a new version for Lofar Station Control should be
reserved. To do this please follow [semantic versioning](https://semver.org/).
Next change the version in the following places:
1. The [VERSION](VERSION) file.
2. In [test_writer_sst.py](tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py)
for the `test_header_info` test.
3. Add a [Release note](#release-notes) for the given version.
3. Once the merge requests is merged to master, add a tag with the version (just x.x.x not Vx.x.x)
# Release Notes
Most notably, you will have web interfaces available at http://localhost:8888 (Jupyter Notebook), and http://localhost:3000 (Grafana).
* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter
* 0.2.0 Extend `Beamlet` device with FPGA source address attributes
docker-compose/elk.yml
+ 4
− 4
View file @ 8ac86cbb
......@@ -44,10 +44,10 @@ services:
ports:
- "5601:5601" # kibana
- "9200:9200" # elasticsearch
- "5044:5044" # logstash beats input
- "1514:1514/tcp" # logstash syslog input
- "1514:1514/udp" # logstash syslog input
- "5959:5959" # logstash tcp json input
# - "5044:5044" # logstash beats input
# - "1514:1514/tcp" # logstash syslog input
# - "1514:1514/udp" # logstash syslog input
# - "5959:5959" # logstash tcp json input
depends_on:
- elk-configure-host
restart: unless-stopped
docker-compose/grafana/datasources/loki.yaml 0 → 100644
+ 45
− 0
View file @ 8ac86cbb
apiVersion: 1
datasources:
# <string, required> name of the datasource. Required
- name: Loki
# <string, required> datasource type. Required
type: loki
# <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
access: proxy
# <int> org id. will default to orgId 1 if not specified
orgId: 1
# <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
uid: loki
# <string> url
url: http://loki:3100
# <string> Deprecated, use secureJsonData.password
password:
# <string> database user, if used
user:
# <string> database name, if used
database:
# <bool> enable/disable basic auth
basicAuth: false
# <string> basic auth username
basicAuthUser:
# <string> Deprecated, use secureJsonData.basicAuthPassword
basicAuthPassword:
# <bool> enable/disable with credentials headers
withCredentials:
# <bool> mark as default datasource. Max one per org
isDefault: false
# <map> fields that will be converted to json and stored in jsonData
jsonData:
esVersion: 7.10.0
includeFrozen: false
logLevelField:
logMessageField:
maxConcurrentShardRequests: 5
timeField: "@timestamp"
# <string> json object of data that will be encrypted.
secureJsonData:
version: 1
# <bool> allow users to edit datasources from the UI.
editable: false
docker-compose/jupyter-lab.yml 0 → 100644
+ 42
− 0
View file @ 8ac86cbb
#
# Docker compose file that launches Jupyter Lab for interactive iTango sessions over HTTP.
#
# Connect by surfing to http://localhost:8889/
# View logs through 'docker logs -f -t jupyter-lab'
#
# Defines:
# - jupyter-lab: Jupyter Lab with iTango support
#
version: '2.1'
services:
jupyter-lab:
build:
context: jupyterlab
args:
CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
container_name: ${CONTAINER_NAME_PREFIX}jupyter-lab
logging:
driver: "json-file"
options:
max-size: "100m"
max-file: "10"
networks:
- control
volumes:
- ..:/opt/lofar/tango:rw
- ../jupyter-notebooks:/jupyter-notebooks:rw
- ${HOME}:/hosthome
- ${SCRATCH}:/scratch:rw
environment:
- TANGO_HOST=${TANGO_HOST}
ports:
- "8889:8889"
user: ${CONTAINER_EXECUTION_UID}
working_dir: /jupyter-notebooks
entrypoint:
- /opt/lofar/tango/bin/start-ds.sh
- jupyter lab --port=8889 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
restart: unless-stopped
docker-compose/jupyter/Dockerfile
+ 3
− 0
View file @ 8ac86cbb
......@@ -13,6 +13,9 @@ RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
RUN sudo apt-get update -y
RUN sudo apt-get install -y g++ gcc python3-dev
# start-ds file synchronization requirements
RUN sudo apt-get install -y rsync
# Install git to install pip requirements from git
RUN sudo apt-get install -y git
......
docker-compose/jupyterlab/Dockerfile 0 → 100644
+ 61
− 0
View file @ 8ac86cbb
ARG SOURCE_IMAGE
FROM ${SOURCE_IMAGE}
# UID if the user that this container will run under. This is needed to give directories
# that are needed for temporary storage the proper owner and access rights.
ARG CONTAINER_EXECUTION_UID=1000
# Create new user with uid but only if uid not used
RUN sudo adduser --disabled-password --system --uid ${CONTAINER_EXECUTION_UID} --no-create-home --home ${HOME} user || exit 0
RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
# Add compiler to install python packages which come with C++ code
RUN sudo apt-get update -y
RUN sudo apt-get install -y g++ gcc python3-dev
# Install git to install pip requirements from git
RUN sudo apt-get install -y git
# Install dependencies of our scripts (bin/start-ds.sh)
RUN sudo apt-get install -y rsync
COPY requirements.txt ./
RUN sudo pip3 install -r requirements.txt
# Install some version of the casacore measures tables, to allow basic delay computation analysis in the notebooks
RUN sudo apt-get install -y casacore-data
# see https://github.com/jupyter/nbconvert/issues/1434
RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py"
RUN sudo apt-get update -y
RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
# Allow Download as -> PDF via LaTeX
RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended cm-super
# Configure jupyter_bokeh
RUN sudo mkdir -p /usr/share/jupyter /usr/etc
RUN sudo chmod a+rwx /usr/share/jupyter /usr/etc
RUN sudo jupyter nbextension install --sys-prefix --symlink --py jupyter_bokeh
RUN sudo jupyter nbextension enable jupyter_bokeh --py --sys-prefix
# Install profiles for ipython & jupyter
COPY ipython-profiles /opt/ipython-profiles/
RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles
COPY jupyter-kernels /usr/local/share/jupyter/kernels/
# Install patched jupyter executable
COPY jupyter-notebook /usr/local/bin/jupyter-notebook
# Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes.
ENV TINI_VERSION v0.6.0
ENV JUPYTER_RUNTIME_DIR=/tmp
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
RUN sudo chmod +x /usr/bin/tini
USER ${CONTAINER_EXECUTION_UID}
# pyppeteer-install installs in the homedir, so run it as the user that will execute the notebook
RUN pyppeteer-install
# Enable Jupyter lab
ENV JUPYTER_ENABLE_LAB=yes
docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py 0 → 100644
+ 1
− 0
View file @ 8ac86cbb
from tango import *
docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py 0 → 100644
+ 21
− 0
View file @ 8ac86cbb
# Create shortcuts for our devices
apsct = DeviceProxy("STAT/APSCT/1")
ccd = DeviceProxy("STAT/CCD/1")
apspu = DeviceProxy("STAT/APSPU/1")
recv = DeviceProxy("STAT/RECV/1")
sdp = DeviceProxy("STAT/SDP/1")
bst = DeviceProxy("STAT/BST/1")
sst = DeviceProxy("STAT/SST/1")
xst = DeviceProxy("STAT/XST/1")
unb2 = DeviceProxy("STAT/UNB2/1")
boot = DeviceProxy("STAT/Boot/1")
tilebeam = DeviceProxy("STAT/TileBeam/1")
psoc = DeviceProxy("STAT/PSOC/1")
beamlet = DeviceProxy("STAT/Beamlet/1")
digitalbeam = DeviceProxy("STAT/DigitalBeam/1")
antennafield = DeviceProxy("STAT/AntennaField/1")
docker = DeviceProxy("STAT/Docker/1")
temperaturemanager = DeviceProxy("STAT/TemperatureManager/1")
# Put them in a list in case one wants to iterate
devices = [apsct, ccd, apspu, recv, sdp, bst, sst, xst, unb2, boot, tilebeam, beamlet, digitalbeam, antennafield, temperaturemanager, docker]
docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py 0 → 100644
+ 1
− 0
View file @ 8ac86cbb
import tangostationcontrol
docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md 0 → 100644
+ 11
− 0
View file @ 8ac86cbb
This is the IPython startup directory
.py and .ipy files in this directory will be run *prior* to any code or files specified
via the exec_lines or exec_files configurables whenever you load this profile.
Files will be run in lexicographical order, so you can control the execution order of files
with a prefix, e.g.::
00-first.py
50-middle.py
99-last.ipy
docker-compose/jupyterlab/jupyter-kernels/stationcontrol/kernel.json 0 → 100644
+ 13
− 0
View file @ 8ac86cbb
{
"argv": [
"python",
"-m",
"ipykernel",
"-f",
"{connection_file}",
"--profile-dir",
"/opt/ipython-profiles/stationcontrol-jupyter/"
],
"language": "python",
"display_name": "StationControl"
}
docker-compose/jupyterlab/jupyter-notebook 0 → 100755
+ 28
− 0
View file @ 8ac86cbb
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# An adjustment of the `jupyter-notebook' executable patched to:
# - log to the ELK stack
#
# We go straight for the notebook executable here, as the "jupyter" command
# execvp's into the requested notebook subcommand, erasing all configuration
# we set here.
import re
import sys
from notebook.notebookapp import main
from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
import logging
if __name__ == '__main__':
# log to the tcp_input of logstash in our ELK stack
handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/pending_log_messages.db')
# add to logger of Jupyter traitlets Application. As that logger is configured not to propagate
# messages upward, we need to configure it directly.
logger = logging.getLogger("NotebookApp")
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())
docker-compose/jupyterlab/requirements.txt 0 → 100644
+ 26
− 0
View file @ 8ac86cbb
ipython >=7.27.0,!=7.28.0 # BSD
jupyter
jupyterlab
jupyterlab_h5web[full] # MIT
ipykernel
jupyter_bokeh
matplotlib
jupyterplot
nbconvert
notebook-as-pdf
python-logstash-async
PyMySQL[rsa]
psycopg2-binary >= 2.9.2 #LGPL
sqlalchemy
pyvisa
pyvisa-py
opcua
lofarantpos >= 0.5.0 # Apache 2
python-geohash >= 0.8.5 # Apache 2 / MIT
numpy
scipy
pabeam@git+https://git.astron.nl/mevius/grate # Apache2
lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client # Apache2
etrs-itrs@git+https://github.com/brentjens/etrs-itrs # Apache 2
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment