Merge branch 'L2SS-949-add-jupyter-lab' into 'master'

Resolve L2SS-949: add jupyter lab

Closes L2SS-949

See merge request !454

docker-compose/jupyter-lab.yml

+#
+# Docker compose file that launches Jupyter Lab for interactive iTango sessions over HTTP.
+#
+# Connect by surfing to http://localhost:8889/
+# View logs through 'docker logs -f -t jupyter-lab'
+#
+# Defines:
+#   - jupyter-lab: Jupyter Lab with iTango support
+#
+
+version: '2.1'
+
+services:
+  jupyter-lab:
+    build:
+        context: jupyterlab
+        args:
+            CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
+            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
+    container_name: ${CONTAINER_NAME_PREFIX}jupyter-lab
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "10"
+    networks:
+      - control
+    volumes:
+      - ..:/opt/lofar/tango:rw
+      - ../jupyter-notebooks:/jupyter-notebooks:rw
+      - ${HOME}:/hosthome
+      - ${SCRATCH}:/scratch:rw
+    environment:
+      - TANGO_HOST=${TANGO_HOST}
+    ports:
+      - "8889:8889"
+    user: ${CONTAINER_EXECUTION_UID}
+    working_dir: /jupyter-notebooks
+    entrypoint:
+      - /opt/lofar/tango/bin/start-ds.sh
+      - jupyter lab --port=8889 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
+    restart: unless-stopped

docker-compose/jupyterlab/Dockerfile

+ARG SOURCE_IMAGE
+FROM ${SOURCE_IMAGE}
+
+# UID if the user that this container will run under. This is needed to give directories
+# that are needed for temporary storage the proper owner and access rights.
+ARG CONTAINER_EXECUTION_UID=1000
+
+# Create new user with uid but only if uid not used
+RUN sudo adduser --disabled-password --system --uid ${CONTAINER_EXECUTION_UID} --no-create-home --home ${HOME} user || exit 0
+RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
+
+# Add compiler to install python packages which come with C++ code
+RUN sudo apt-get update -y
+RUN sudo apt-get install -y g++ gcc python3-dev
+
+# Install git to install pip requirements from git
+RUN sudo apt-get install -y git
+
+# Install dependencies of our scripts (bin/start-ds.sh)
+RUN sudo apt-get install -y rsync
+
+COPY requirements.txt ./
+RUN sudo pip3 install -r requirements.txt
+
+# Install some version of the casacore measures tables, to allow basic delay computation analysis in the notebooks
+RUN sudo apt-get install -y casacore-data
+
+# see https://github.com/jupyter/nbconvert/issues/1434
+RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py"
+RUN sudo apt-get update -y
+RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
+
+# Allow Download as -> PDF via LaTeX
+RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended cm-super
+
+# Configure jupyter_bokeh
+RUN sudo mkdir -p /usr/share/jupyter /usr/etc
+RUN sudo chmod a+rwx /usr/share/jupyter /usr/etc
+RUN sudo jupyter nbextension install --sys-prefix --symlink --py jupyter_bokeh
+RUN sudo jupyter nbextension enable jupyter_bokeh --py --sys-prefix
+
+# Install profiles for ipython & jupyter
+COPY ipython-profiles /opt/ipython-profiles/
+RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles
+COPY jupyter-kernels /usr/local/share/jupyter/kernels/
+
+# Install patched jupyter executable
+COPY jupyter-notebook /usr/local/bin/jupyter-notebook
+
+# Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes.
+ENV TINI_VERSION v0.6.0
+ENV JUPYTER_RUNTIME_DIR=/tmp
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
+RUN sudo chmod +x /usr/bin/tini
+
+USER ${CONTAINER_EXECUTION_UID}
+# pyppeteer-install installs in the homedir, so run it as the user that will execute the notebook
+RUN pyppeteer-install
+
+# Enable Jupyter lab
+ENV JUPYTER_ENABLE_LAB=yes

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py

+from tango import *

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py

+# Create shortcuts for our devices
+apsct = DeviceProxy("STAT/APSCT/1")
+ccd = DeviceProxy("STAT/CCD/1")
+apspu = DeviceProxy("STAT/APSPU/1")
+recv = DeviceProxy("STAT/RECV/1")
+sdp = DeviceProxy("STAT/SDP/1")
+bst = DeviceProxy("STAT/BST/1")
+sst = DeviceProxy("STAT/SST/1")
+xst = DeviceProxy("STAT/XST/1")
+unb2 = DeviceProxy("STAT/UNB2/1")
+boot = DeviceProxy("STAT/Boot/1")
+tilebeam = DeviceProxy("STAT/TileBeam/1")
+psoc = DeviceProxy("STAT/PSOC/1")
+beamlet = DeviceProxy("STAT/Beamlet/1")
+digitalbeam = DeviceProxy("STAT/DigitalBeam/1")
+antennafield = DeviceProxy("STAT/AntennaField/1")
+docker = DeviceProxy("STAT/Docker/1")
+temperaturemanager = DeviceProxy("STAT/TemperatureManager/1")
+
+# Put them in a list in case one wants to iterate
+devices = [apsct, ccd, apspu, recv, sdp, bst, sst, xst, unb2, boot, tilebeam, beamlet, digitalbeam, antennafield, temperaturemanager, docker]

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py

+import tangostationcontrol

docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md

+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+    00-first.py
+    50-middle.py
+    99-last.ipy

docker-compose/jupyterlab/jupyter-kernels/stationcontrol/kernel.json

+ {
+     "argv": [
+	  "python",
+	  "-m",
+	  "ipykernel",
+	  "-f",
+	  "{connection_file}",
+	  "--profile-dir",
+	  "/opt/ipython-profiles/stationcontrol-jupyter/"
+     ],
+     "language": "python",
+     "display_name": "StationControl"
+}

docker-compose/jupyterlab/jupyter-notebook

+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+# An adjustment of the `jupyter-notebook' executable patched to:
+#  - log to the ELK stack
+#
+# We go straight for the notebook executable here, as the "jupyter" command
+# execvp's into the requested notebook subcommand, erasing all configuration
+# we set here.
+import re
+import sys
+
+from notebook.notebookapp import main 
+
+from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
+import logging
+
+if __name__ == '__main__':
+    # log to the tcp_input of logstash in our ELK stack
+    handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/pending_log_messages.db')
+
+    # add to logger of Jupyter traitlets Application. As that logger is configured not to propagate
+    # messages upward, we need to configure it directly.
+    logger = logging.getLogger("NotebookApp")
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG)
+
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())

docker-compose/jupyterlab/requirements.txt

+ipython >=7.27.0,!=7.28.0 # BSD
+jupyter
+jupyterlab
+ipykernel
+jupyter_bokeh
+matplotlib
+jupyterplot
+nbconvert
+notebook-as-pdf
+python-logstash-async
+PyMySQL[rsa]
+psycopg2-binary >= 2.9.2 #LGPL
+sqlalchemy
+pyvisa
+pyvisa-py
+opcua
+lofarantpos >= 0.5.0 # Apache 2
+python-geohash >= 0.8.5 # Apache 2 / MIT
+
+numpy
+scipy
+
+pabeam@git+https://git.astron.nl/mevius/grate # Apache2
+lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client # Apache2
+etrs-itrs@git+https://github.com/brentjens/etrs-itrs # Apache 2

tangostationcontrol/tangostationcontrol/devices/README.md

@@ -7,7 +7,7 @@ This directory contains the sources for our custom Tango devices.
 If a new device is added, it will (likely) need to be referenced in several places. Adjust or add the following files (referenced from the repository root), following the pattern shown by the devices already there:
 
 - Adjust `CDB/LOFAR_ConfigDb.json` to create the device in the Tango device database,
-- Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter,
+- Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` and `docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter and Jupyter-Lab,
 - Adjust `tangostationcontrol/tangostationcontrol/devices/boot.py` to add the device to the station initialisation sequence,
 - Add to `docker-compose/` to create a YaML file to start the device in a docker container. NOTE: it needs a unique 57xx port assigned (current _unused_ port value: 5722), a unique 58xx port for ZMQ events, and a unique 59xx port for ZMQ heartbeat
 - Adjust `tangostationcontrol/setup.cfg` to add an entry point for the device in the package installation,