Merge branch 'master' of https://git.astron.nl/lofar2.0/tango into...

Merge branch 'master' of https://git.astron.nl/lofar2.0/tango into L2SS-363_01-12-2021-get_RW_values_from_hardware

docker-compose/device-sdp.yml

@@ -20,7 +20,7 @@ services:
     build:
         context: lofar-device-base
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-sdp
     networks:
       - control

docker-compose/device-sst.yml

@@ -20,7 +20,7 @@ services:
     build:
         context: lofar-device-base
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-sst
     networks:
         - control

docker-compose/device-unb2.yml

@@ -20,7 +20,7 @@ services:
     build:
         context: lofar-device-base
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-unb2
     networks:
       - control

docker-compose/device-xst.yml

@@ -20,7 +20,7 @@ services:
     build:
         context: lofar-device-base
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}device-xst
     networks:
         - control

docker-compose/elk.yml

@@ -6,6 +6,7 @@
 #   - elk-configure-host: Configures the hosts's kernel to be able to use the ELK stack
 #   - elk: ELK stack
 #
+
 version: '2'
 
 volumes:

docker-compose/grafana.yml

@@ -4,6 +4,7 @@
 # Defines:
 #   - grafana: Grafana
 #
+
 version: '2'
 
 #volumes:

docker-compose/grafana/dashboards/archiver_timescale.json

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": "TimescaleDB",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "showHeader": true
+      },
+      "pluginVersion": "8.2.5",
+      "targets": [
+        {
+          "format": "table",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "select name as \"Attribute\",\r\ntable_name as \"Type\"\r\nfrom att_conf ac \r\nwhere \"domain\" ='stat' and \"family\" = 'sdp' and \"member\" ='1'\r\norder by name",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "value_r"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "att_scalar_devdouble",
+          "timeColumn": "data_time",
+          "timeColumnType": "timestamp",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "STAT/SDP/1 archived attributes",
+      "type": "table"
+    },
+    {
+      "datasource": "TimescaleDB",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 8,
+      "options": {
+        "showHeader": true
+      },
+      "pluginVersion": "8.2.5",
+      "targets": [
+        {
+          "format": "table",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "select name as \"Attribute\",\r\ntable_name as \"Type\" \r\nfrom att_conf ac \r\nwhere \"domain\" ='stat' and \"family\" = 'recv' and \"member\" ='1'\r\norder by name",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "value_r"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "att_scalar_devdouble",
+          "timeColumn": "data_time",
+          "timeColumnType": "timestamp",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "LTS/RECV/1 archived attributes",
+      "type": "table"
+    },
+    {
+      "datasource": "TimescaleDB",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-BlYlRd"
+          },
+          "custom": {
+            "fillOpacity": 70,
+            "lineWidth": 1
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 9,
+      "interval": "10s",
+      "maxDataPoints": 1000,
+      "options": {
+        "colWidth": 0.3,
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "rowHeight": 0.78,
+        "showValue": "auto",
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "pluginVersion": "8.2.4",
+      "targets": [
+        {
+          "format": "time_series",
+          "group": [],
+          "hide": false,
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "SELECT time_bucket('00:00:01'::interval, q1.\"time\") AS time,\r\n    mask(array[q1.value[1]], array[q2.\"value\"[1]]) AS \"temperature[1]\",\r\n    mask(array[q1.value[2]], array[q2.\"value\"[2]]) AS \"temperature[2]\",\r\n    mask(array[q1.value[3]], array[q2.\"value\"[3]]) AS \"temperature[3]\",\r\n    mask(array[q1.value[4]], array[q2.\"value\"[4]]) AS \"temperature[4]\",\r\n    mask(array[q1.value[5]], array[q2.\"value\"[5]]) AS \"temperature[5]\",\r\n    mask(array[q1.value[6]], array[q2.\"value\"[6]]) AS \"temperature[6]\",\r\n    mask(array[q1.value[7]], array[q2.\"value\"[7]]) AS \"temperature[7]\",\r\n    mask(array[q1.value[8]], array[q2.\"value\"[8]]) AS \"temperature[8]\",\r\n    mask(array[q1.value[9]], array[q2.\"value\"[9]]) AS \"temperature[9]\",\r\n    mask(array[q1.value[10]], array[q2.\"value\"[10]]) AS \"temperature[10]\",\r\n    mask(array[q1.value[11]], array[q2.\"value\"[11]]) AS \"temperature[11]\",\r\n    mask(array[q1.value[12]], array[q2.\"value\"[12]]) AS \"temperature[12]\",\r\n    mask(array[q1.value[13]], array[q2.\"value\"[13]]) AS \"temperature[13]\",\r\n    mask(array[q1.value[14]], array[q2.\"value\"[14]]) AS \"temperature[14]\",\r\n    mask(array[q1.value[15]], array[q2.\"value\"[15]]) AS \"temperature[15]\"\r\n   FROM ( SELECT aad.data_time AS \"time\",\r\n            aad.value_r AS value\r\n           FROM att_array_devdouble aad\r\n             JOIN att_conf ac ON aad.att_conf_id = ac.att_conf_id\r\n          WHERE aad.value_r IS NOT NULL AND ac.domain = 'stat'::text AND ac.family = 'sdp'::text AND ac.member = '1'::text AND ac.name = 'fpga_temp_r'::text\r\n          ORDER BY aad.data_time) q1\r\n     JOIN ( SELECT aab.data_time AS \"time\",\r\n            aab.value_r AS value\r\n           FROM att_array_devboolean aab\r\n             JOIN att_conf ac ON aab.att_conf_id = ac.att_conf_id\r\n          WHERE aab.value_r IS NOT NULL AND ac.domain = 'stat'::text AND ac.family = 'sdp'::text AND ac.member = '1'::text AND ac.name = 'tr_fpga_mask_r'::text\r\n          ORDER BY aab.data_time) q2 ON time_bucket('00:00:01'::interval, q1.\"time\") = time_bucket('00:00:01'::interval, q2.\"time\")\r\n  WHERE $__timeFilter(q1.\"time\")\r\n  ORDER BY q1.\"time\";",
+          "refId": "B",
+          "select": [
+            [
+              {
+                "params": [
+                  "temperature[1]"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "table": "sdp_masked_temp_values",
+          "timeColumn": "temp_time",
+          "timeColumnType": "timestamptz",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "SDP - FPGA Temperature",
+      "type": "status-history"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 32,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Archiver-Timescale",
+  "uid": "M7zA7Hc7k",
+  "version": 1
+}

docker-compose/grafana/datasources/archiver-timescale.yaml

+apiVersion: 1
+
+datasources:
+  # <string, required> name of the datasource. Required
+  - name: TimescaleDB
+    # <string, required> datasource type. Required
+    type: postgres
+    # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required
+    access: proxy
+    # <int> org id. will default to orgId 1 if not specified
+    orgId: 1
+    # <string> custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically
+    uid: ZqArtG97z
+    # <string> url
+    url: archiver-timescale
+    # <string> Deprecated, use secureJsonData.password
+    password:
+    # <string> database user, if used
+    user: postgres
+    # <string> database name, if used
+    database: hdb
+    # <bool> enable/disable basic auth
+    basicAuth: false
+    # <string> basic auth username
+    basicAuthUser:
+    # <string> Deprecated, use secureJsonData.basicAuthPassword
+    basicAuthPassword:
+    # <bool> enable/disable with credentials headers
+    withCredentials:
+    # <bool> mark as default datasource. Max one per org
+    isDefault: false
+    # <map> fields that will be converted to json and stored in jsonData
+    jsonData:
+      # <string> determines whether or with what priority a secure TLS/SSL TCP/IP connection will be negotiated with the server.
+      sslmode: "disable"
+      # <bool> enable TimescaleDB
+      timescaledb: true
+    # <string> json object of data that will be encrypted.
+    secureJsonData:
+      # <string> database password, if used
+      password: password
+    version: 1
+    # <bool> allow users to edit datasources from the UI.
+    editable: false
+    

docker-compose/integration-test.yml

@@ -11,7 +11,7 @@ services:
     build:
         context: itango
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}integration-test
     networks:
       - control

docker-compose/itango.yml

@@ -17,7 +17,7 @@ services:
     build:
         context: itango
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}itango
     networks:
       - control

docker-compose/jive.yml

@@ -18,7 +18,7 @@ version: '2'
 
 services:
   jive:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}jive
     network_mode: host
     volumes:

docker-compose/jupyter.yml

@@ -7,6 +7,7 @@
 # Defines:
 #   - jupyter: Jupyter Notebook with iTango support
 #
+
 version: '2'
 
 services:
@@ -15,7 +16,7 @@ services:
         context: jupyter
         args:
             CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}jupyter
     networks:
       - control
@@ -30,10 +31,6 @@ services:
     user: ${CONTAINER_EXECUTION_UID}
     working_dir: /jupyter-notebooks
     entrypoint:
-      - /usr/local/bin/wait-for-it.sh
-      - ${TANGO_HOST}
-      - --timeout=30
-      - --strict
-      - --
+      - /opt/lofar/tango/bin/start-ds.sh
       - /usr/bin/tini -- /usr/local/bin/jupyter-notebook --port=8888 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
     restart: unless-stopped

docker-compose/jupyter/Dockerfile

@@ -10,23 +10,13 @@ ENV HOME=/home/user
 RUN sudo mkdir -p ${HOME}
 RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
 
-# ipython 7.28 is broken in combination with Jupyter, it causes connection errors with notebooks
-RUN sudo pip3 install ipython==7.27.0
-
-RUN sudo pip3 install jupyter
-RUN sudo pip3 install ipykernel
-RUN sudo pip3 install jupyter_bokeh
-# Install matplotlib, jupyterplot
-RUN sudo pip3 install matplotlib jupyterplot
-
-# Allow Download as -> PDF via html
-RUN sudo pip3 install nbconvert
-RUN sudo pip3 install notebook-as-pdf
+COPY requirements.txt ./
+RUN sudo pip3 install -r requirements.txt
 
 # see https://github.com/jupyter/nbconvert/issues/1434
 RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py"
 RUN sudo apt-get update -y
-RUN sudo apt-get install -y gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
+RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
 
 # Allow Download as -> PDF via LaTeX
 RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended
@@ -43,15 +33,8 @@ RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles
 COPY jupyter-kernels /usr/local/share/jupyter/kernels/
 
 # Install patched jupyter executable
-RUN sudo pip3 install python-logstash-async
 COPY jupyter-notebook /usr/local/bin/jupyter-notebook
 
-#Install further python modules
-RUN sudo pip3 install PyMySQL[rsa] sqlalchemy
-
-# Packages to interface with testing hardware directly
-RUN sudo pip3 install pyvisa pyvisa-py opcua
-
 # Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes.
 ENV TINI_VERSION v0.6.0
 ENV JUPYTER_RUNTIME_DIR=/tmp

docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py

@@ -7,7 +7,8 @@ sst = DeviceProxy("STAT/SST/1")
 xst = DeviceProxy("STAT/XST/1")
 unb2 = DeviceProxy("STAT/UNB2/1")
 boot = DeviceProxy("STAT/Boot/1")
+beam = DeviceProxy("STAT/Beam/1")
 docker = DeviceProxy("STAT/Docker/1")
 
 # Put them in a list in case one wants to iterate
-devices = [apsct, apspu, recv, sdp, sst, xst, unb2, boot, docker]
+devices = [apsct, apspu, recv, sdp, sst, xst, unb2, boot, beam, docker]

docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py

+import tangostationcontrol

docker-compose/jupyter/requirements.txt

+GitPython >= 3.1.24 # BSD
+ipython >=7.27.0,!=7.28.0 # BSD
+jupyter
+ipykernel
+jupyter_bokeh
+matplotlib
+jupyterplot
+nbconvert
+notebook-as-pdf
+python-logstash-async
+PyMySQL[rsa]
+psycopg2-binary >= 2.9.2 #LGPL
+sqlalchemy
+pyvisa
+pyvisa-py
+opcua

docker-compose/lofar-device-base.yml

@@ -10,6 +10,7 @@
 # Requires:
 #   - tango.yml
 #
+
 version: '2'
 
 services:
@@ -18,7 +19,7 @@ services:
     build:
         context: lofar-device-base
         args:
-            SOURCE_IMAGE: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-itango:${TANGO_ITANGO_VERSION}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}lofar-device-base
     # These parameters are just visual queues, you have to define them again
     # in derived docker-compose files!

docker-compose/lofar-device-base/lofar-requirements.txt

-# Do not put tangostationcontrol dependencies here
-astropy
-
-# requirements to build tangocontrol 
+# Do not put tangostationcontrol dependencies here, only setup.py / __init__.py
 GitPython >= 3.1.24 # BSD

docker-compose/logviewer.yml

@@ -12,7 +12,7 @@ version: '2'
 
 services:
   logviewer:
-    image: ${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-tango-java:${TANGO_JAVA_VERSION}
+    image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-java:${TANGO_JAVA_VERSION}
     container_name: ${CONTAINER_NAME_PREFIX}logviewer
     networks:
       - control