L2SS-949: add jupyter lab container

docker-compose/jupyter-lab.yml

+#
+# Docker compose file that launches Jupyter Lab for interactive iTango sessions over HTTP.
+#
+# Connect by surfing to http://localhost:8889/
+# View logs through 'docker logs -f -t jupyter-lab'
+#
+# Defines:
+#   - jupyter-lab: Jupyter Lab with iTango support
+#
+
+version: '2.1'
+
+services:
+  jupyter-lab:
+    build:
+        context: jupyterlab
+        args:
+            CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
+    container_name: ${CONTAINER_NAME_PREFIX}jupyter-lab
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "10"
+    networks:
+      - control
+    volumes:
+      - ..:/opt/lofar/tango:rw
+      - ../jupyter-notebooks:/jupyter-notebooks:rw
+      - ${HOME}:/hosthome
+      - ${SCRATCH}:/scratch:rw
+    environment:
+      - TANGO_HOST=${TANGO_HOST}
+    ports:
+      - "8889:8889"
+    user: ${CONTAINER_EXECUTION_UID}
+    working_dir: /jupyter-notebooks
+    entrypoint:
+      - /opt/lofar/tango/bin/start-ds.sh
+      - jupyter lab --port=8889 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
+    restart: unless-stopped

docker-compose/jupyterlab/Dockerfile

+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# UID if the user that this container will run under. This is needed to give directories
+# that are needed for temporary storage the proper owner and access rights.
+ARG CONTAINER_EXECUTION_UID=1000
+
+# Create new user with uid but only if uid not used
+RUN sudo adduser --disabled-password --system --uid ${CONTAINER_EXECUTION_UID} --no-create-home --home ${HOME} user || exit 0
+RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
+
+# Add compiler to install python packages which come with C++ code
+RUN sudo apt-get update -y
+RUN sudo apt-get install -y g++ gcc python3-dev
+
+# Install git to install pip requirements from git
+RUN sudo apt-get install -y git
+
+# Install dependencies of our scripts (bin/start-ds.sh)
+RUN sudo apt-get install -y rsync
+
+COPY requirements.txt ./
+RUN sudo pip3 install -r requirements.txt
+
+# Install some version of the casacore measures tables, to allow basic delay computation analysis in the notebooks
+RUN sudo apt-get install -y casacore-data
+
+# see https://github.com/jupyter/nbconvert/issues/1434
+RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py"
+RUN sudo apt-get update -y
+RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
+
+# Allow Download as -> PDF via LaTeX
+RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended cm-super
+
+# Configure jupyter_bokeh
+RUN sudo mkdir -p /usr/share/jupyter /usr/etc
+RUN sudo chmod a+rwx /usr/share/jupyter /usr/etc
+RUN sudo jupyter nbextension install --sys-prefix --symlink --py jupyter_bokeh
+RUN sudo jupyter nbextension enable jupyter_bokeh --py --sys-prefix
+
+# Install profiles for ipython & jupyter
+COPY ipython-profiles /opt/ipython-profiles/
+RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles
+COPY jupyter-kernels /usr/local/share/jupyter/kernels/
+
+# Install patched jupyter executable
+COPY jupyter-notebook /usr/local/bin/jupyter-notebook
+
+# Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes.
+ENV TINI_VERSION v0.6.0
+ENV JUPYTER_RUNTIME_DIR=/tmp
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
+RUN sudo chmod +x /usr/bin/tini
+
+USER ${CONTAINER_EXECUTION_UID}
+# pyppeteer-install installs in the homedir, so run it as the user that will execute the notebook
+RUN pyppeteer-install
+
+# Enable Jupyter lab
+ENV JUPYTER_ENABLE_LAB=yes

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py

+from tango import *

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py

+# Create shortcuts for our devices
+apsct = DeviceProxy("STAT/APSCT/1")
+ccd = DeviceProxy("STAT/CCD/1")
+apspu = DeviceProxy("STAT/APSPU/1")
+recv = DeviceProxy("STAT/RECV/1")
+sdp = DeviceProxy("STAT/SDP/1")
+bst = DeviceProxy("STAT/BST/1")
+sst = DeviceProxy("STAT/SST/1")
+xst = DeviceProxy("STAT/XST/1")
+unb2 = DeviceProxy("STAT/UNB2/1")
+boot = DeviceProxy("STAT/Boot/1")
+tilebeam = DeviceProxy("STAT/TileBeam/1")
+psoc = DeviceProxy("STAT/PSOC/1")
+beamlet = DeviceProxy("STAT/Beamlet/1")
+digitalbeam = DeviceProxy("STAT/DigitalBeam/1")
+antennafield = DeviceProxy("STAT/AntennaField/1")
+docker = DeviceProxy("STAT/Docker/1")
+temperaturemanager = DeviceProxy("STAT/TemperatureManager/1")
+
+# Put them in a list in case one wants to iterate
+devices = [apsct, ccd, apspu, recv, sdp, bst, sst, xst, unb2, boot, tilebeam, beamlet, digitalbeam, antennafield, temperaturemanager, docker]

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py

+import tangostationcontrol

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md

+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+    00-first.py
+    50-middle.py
+    99-last.ipy

docker-compose/jupyterlab/jupyter-kernels/stationcontrol/kernel.json

+ {
+     "argv": [
+	  "python",
+	  "-m",
+	  "ipykernel",
+	  "-f",
+	  "{connection_file}",
+	  "--profile-dir",
+	  "/opt/ipython-profiles/stationcontrol-jupyter/"
+     ],
+     "language": "python",
+     "display_name": "StationControl"
+}

docker-compose/jupyterlab/jupyter-notebook

+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# An adjustment of the `jupyter-notebook' executable patched to:
+#  - log to the ELK stack
+#
+# We go straight for the notebook executable here, as the "jupyter" command
+# execvp's into the requested notebook subcommand, erasing all configuration
+# we set here.
+import re
+import sys
+
+from notebook.notebookapp import main 
+
+from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
+import logging
+
+if __name__ == '__main__':
+    # log to the tcp_input of logstash in our ELK stack
+    handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/pending_log_messages.db')
+
+    # add to logger of Jupyter traitlets Application. As that logger is configured not to propagate
+    # messages upward, we need to configure it directly.
+    logger = logging.getLogger("NotebookApp")
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG)
+
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

docker-compose/jupyterlab/requirements.txt

+ipython >=7.27.0,!=7.28.0 # BSD
+jupyter
+jupyterlab
+ipykernel
+jupyter_bokeh
+matplotlib
+jupyterplot
+nbconvert
+notebook-as-pdf
+python-logstash-async
+PyMySQL[rsa]
+psycopg2-binary >= 2.9.2 #LGPL
+sqlalchemy
+pyvisa
+pyvisa-py
+opcua
+lofarantpos >= 0.5.0 # Apache 2
+python-geohash >= 0.8.5 # Apache 2 / MIT
+
+numpy
+scipy
+
+pabeam@git+https://git.astron.nl/mevius/grate # Apache2
+lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client # Apache2
+etrs-itrs@git+https://github.com/brentjens/etrs-itrs # Apache 2