Resolve L2SS-1049 "Remove jupyter notebooks"

.gitlab-ci.yml

@@ -96,7 +96,7 @@ docker_build_image_all:
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter latest
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh ccd-sim latest
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim latest
@@ -215,11 +215,11 @@ docker_build_image_jupyter:
     refs:
       - merge_requests
     changes:
-      - docker-compose/jupyter.yml
-      - docker-compose/jupyter/*
+      - docker-compose/jupyter-lab.yml
+      - docker-compose/jupyterlab/*
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab $tag
 docker_build_image_apsct_sim:
   extends: .base_docker_images_except
   only:

README.md

@@ -14,7 +14,7 @@ Station Control software related to Tango devices.
 * [User documentation (ReadTheDocs (Sphinx / ReStructuredText))](tangostationcontrol/docs/README.md)
 * [Docker compose documentation](docker-compose/README.md)
   * [Timescaledb](docker-compose/timescaledb/README.md)
-  * [Jupyter startup files](docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md)
+  * [Jupyter startup files](docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md)
   * [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter)
 * [Developer Documentation](#development)
   * [Deployments](deploy/README.md)
@@ -71,7 +71,7 @@ automatically be restarted on reboot or failure. Stop them explicitly to bring
 them down (`make stop <container>`).
 
 Most notably, you will have web interfaces available at:
- - http://localhost:8888 (Jupyter Notebook)
+ - http://localhost:8888 (Jupyter Lab)
  - http://localhost:5601 (ElasticSearch)
  - http://localhost:3000 (Grafana).
 

bootstrap/sbin/rebuild_system_from_scratch.sh

@@ -106,7 +106,7 @@ function configure_elk()
 function start_support_images()
 {
     (cd ${HOME_DIR}/docker-compose && make start elk
-    make start jupyter)
+    make start jupyter-lab)
 }
 
 function start_lofar_images()

docker-compose/jupyter-lab.yml

 #
 # Docker compose file that launches Jupyter Lab for interactive iTango sessions over HTTP.
 #
-# Connect by surfing to http://localhost:8889/
+# Connect by surfing to http://localhost:8888/
 # View logs through 'docker logs -f -t jupyter-lab'
 #
 # Defines:
@@ -33,10 +33,10 @@ services:
     environment:
       - TANGO_HOST=${TANGO_HOST}
     ports:
-      - "8889:8889"
+      - "8888:80"
     user: ${CONTAINER_EXECUTION_UID}
     working_dir: /jupyter-notebooks
     entrypoint:
       - /opt/lofar/tango/bin/start-ds.sh
-      - jupyter lab --port=8889 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
+      - jupyter lab --port=80 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
     restart: unless-stopped

docker-compose/jupyter.yml

-#
-# Docker compose file that launches Jupyter Notebook for interactive iTango sessions over HTTP.
-#
-# Connect by surfing to http://localhost:8888/
-# View logs through 'docker logs -f -t jupyter'
-#
-# Defines:
-#   - jupyter: Jupyter Notebook with iTango support
-#
-
-version: '2.1'
-
-services:
-  jupyter:
-    build:
-        context: jupyter
-        args:
-            CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID}
-            SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION}
-    container_name: ${CONTAINER_NAME_PREFIX}jupyter
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "100m"
-        max-file: "10"
-    networks:
-      - control
-    volumes:
-        - ..:/opt/lofar/tango:rw
-        - ../jupyter-notebooks:/jupyter-notebooks:rw
-        - ${HOME}:/hosthome
-        - ${SCRATCH}:/scratch:rw
-    environment:
-      - TANGO_HOST=${TANGO_HOST}
-    ports:
-      - "8888:8888"
-    user: ${CONTAINER_EXECUTION_UID}
-    working_dir: /jupyter-notebooks
-    entrypoint:
-      - /opt/lofar/tango/bin/start-ds.sh
-      - /usr/bin/tini -- /usr/local/bin/jupyter-notebook --port=8888 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password=
-    restart: unless-stopped

docker-compose/jupyter/Dockerfile

-ARG SOURCE_IMAGE
-FROM ${SOURCE_IMAGE}
-
-# UID if the user that this container will run under. This is needed to give directories
-# that are needed for temporary storage the proper owner and access rights.
-ARG CONTAINER_EXECUTION_UID=1000
-
-# Create new user with uid but only if uid not used
-RUN sudo adduser --disabled-password --system --uid ${CONTAINER_EXECUTION_UID} --no-create-home --home ${HOME} user || exit 0
-RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME}
-
-# Add compiler to install python packages which come with C++ code
-RUN sudo apt-get update -y
-RUN sudo apt-get install -y g++ gcc python3-dev
-
-# start-ds file synchronization requirements
-RUN sudo apt-get install -y rsync
-
-# Install git to install pip requirements from git
-RUN sudo apt-get install -y git
-
-# Install dependencies of our scripts (bin/start-ds.sh)
-RUN sudo apt-get install -y rsync
-
-COPY requirements.txt ./
-RUN sudo pip3 install -r requirements.txt
-
-# Install some version of the casacore measures tables, to allow basic delay computation analysis in the notebooks
-RUN sudo apt-get install -y casacore-data
-
-# see https://github.com/jupyter/nbconvert/issues/1434
-RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py"
-RUN sudo apt-get update -y
-RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1
-
-# Allow Download as -> PDF via LaTeX
-RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended cm-super
-
-# Configure jupyter_bokeh
-RUN sudo mkdir -p /usr/share/jupyter /usr/etc
-RUN sudo chmod a+rwx /usr/share/jupyter /usr/etc
-RUN sudo jupyter nbextension install --sys-prefix --symlink --py jupyter_bokeh
-RUN sudo jupyter nbextension enable jupyter_bokeh --py --sys-prefix
-
-# Install profiles for ipython & jupyter
-COPY ipython-profiles /opt/ipython-profiles/
-RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles
-COPY jupyter-kernels /usr/local/share/jupyter/kernels/
-
-# Install patched jupyter executable
-COPY jupyter-notebook /usr/local/bin/jupyter-notebook
-
-# Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes.
-ENV TINI_VERSION v0.6.0
-ENV JUPYTER_RUNTIME_DIR=/tmp
-ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
-RUN sudo chmod +x /usr/bin/tini
-
-USER ${CONTAINER_EXECUTION_UID}
-# pyppeteer-install installs in the homedir, so run it as the user that will execute the notebook
-RUN pyppeteer-install

docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py

-from tango import *

docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py

-# Create shortcuts for our devices
-apsct = DeviceProxy("STAT/APSCT/1")
-ccd = DeviceProxy("STAT/CCD/1")
-apspu = DeviceProxy("STAT/APSPU/1")
-recv = DeviceProxy("STAT/RECV/1")
-sdp = DeviceProxy("STAT/SDP/1")
-bst = DeviceProxy("STAT/BST/1")
-sst = DeviceProxy("STAT/SST/1")
-xst = DeviceProxy("STAT/XST/1")
-unb2 = DeviceProxy("STAT/UNB2/1")
-boot = DeviceProxy("STAT/Boot/1")
-tilebeam = DeviceProxy("STAT/TileBeam/1")
-pcon = DeviceProxy("STAT/PCON/1")
-psoc = DeviceProxy("STAT/PSOC/1")
-beamlet = DeviceProxy("STAT/Beamlet/1")
-digitalbeam = DeviceProxy("STAT/DigitalBeam/1")
-antennafield = DeviceProxy("STAT/AntennaField/1")
-docker = DeviceProxy("STAT/Docker/1")
-temperaturemanager = DeviceProxy("STAT/TemperatureManager/1")
-configuration = DeviceProxy("STAT/Configuration/1")
-
-# Put them in a list in case one wants to iterate
-devices = [apsct, ccd, apspu, recv, sdp, bst, sst, xst, unb2, boot, tilebeam, beamlet, digitalbeam, antennafield, temperaturemanager, docker, pcon, psoc, configuration]

docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py

-import tangostationcontrol

docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md

-This is the IPython startup directory
-
-.py and .ipy files in this directory will be run *prior* to any code or files specified
-via the exec_lines or exec_files configurables whenever you load this profile.
-
-Files will be run in lexicographical order, so you can control the execution order of files
-with a prefix, e.g.::
-
-    00-first.py
-    50-middle.py
-    99-last.ipy

docker-compose/jupyter/jupyter-kernels/stationcontrol/kernel.json

- {
-     "argv": [
-	  "python",
-	  "-m",
-	  "ipykernel",
-	  "-f",
-	  "{connection_file}",
-	  "--profile-dir",
-	  "/opt/ipython-profiles/stationcontrol-jupyter/"
-     ],
-     "language": "python",
-     "display_name": "StationControl"
-}

docker-compose/jupyter/jupyter-notebook

-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-# An adjustment of the `jupyter-notebook' executable patched to:
-#  - log to the ELK stack
-#
-# We go straight for the notebook executable here, as the "jupyter" command
-# execvp's into the requested notebook subcommand, erasing all configuration
-# we set here.
-import re
-import sys
-
-from notebook.notebookapp import main 
-
-from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter
-import logging
-
-if __name__ == '__main__':
-    # log to the tcp_input of logstash in our ELK stack
-    handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/pending_log_messages.db')
-
-    # add to logger of Jupyter traitlets Application. As that logger is configured not to propagate
-    # messages upward, we need to configure it directly.
-    logger = logging.getLogger("NotebookApp")
-    logger.addHandler(handler)
-    logger.setLevel(logging.DEBUG)
-
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
-    sys.exit(main())

docker-compose/jupyter/requirements.txt

-ipython >=7.27.0,!=7.28.0 # BSD
-jupyter
-ipykernel
-jupyter_bokeh
-matplotlib
-jupyterplot
-nbconvert
-notebook-as-pdf
-python-logstash-async
-PyMySQL[rsa]
-psycopg2-binary >= 2.9.2 #LGPL
-sqlalchemy
-pyvisa
-pyvisa-py
-opcua
-
-numpy
-scipy
-
-pabeam@git+https://git.astron.nl/mevius/grate # Apache2
-lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client # Apache2

sbin/tag_and_push_docker_image.sh

@@ -88,7 +88,7 @@ LOCAL_IMAGES=(
   "hdbppts-es hdbppts-es y"
 
   "grafana grafana n" "prometheus prometheus n"
-  "jupyter docker-compose_jupyter n"
+  "jupyter-lab docker-compose_jupyter-lab n"
   "integration-test docker-compose_integration-test n"
   "tango-prometheus-exporter docker-compose_tango-prometheus-exporter n"
 )

tangostationcontrol/docs/source/interfaces/control.rst

@@ -12,7 +12,7 @@ The station offers Juypyter notebooks On http://localhost:8888, which allow one
 
 The notebooks provide some predefined variables, so you don't have to look them up:
 
-.. literalinclude:: ../../../../docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py
+.. literalinclude:: ../../../../docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py
 
 Note: the Jupyter notebooks use enhancements from the ``itango`` suite, which provide tab completions, but also the ``Device`` alias for ``DeviceProxy`` as was used in the Python examples in the next section.
 

tangostationcontrol/tangostationcontrol/devices/README.md

@@ -7,7 +7,7 @@ This directory contains the sources for our custom Tango devices.
 If a new device is added, it will (likely) need to be referenced in several places. Adjust or add the following files (referenced from the repository root), following the pattern shown by the devices already there:
 
 - Adjust `CDB/LOFAR_ConfigDb.json` to create the device in the Tango device database,
-- Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` and `docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter and Jupyter-Lab,
+- Adjust `docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter-Lab,
 - Adjust `tangostationcontrol/tangostationcontrol/devices/boot.py` to add the device to the station initialisation sequence,
 - Add to `docker-compose/` to create a YaML file to start the device in a docker container. NOTE: it needs a unique 57xx port assigned (current _unused_ port value: 5723), a unique 58xx port for ZMQ events, and a unique 59xx port for ZMQ heartbeat
 - Adjust `tangostationcontrol/setup.cfg` to add an entry point for the device in the package installation,

tangostationcontrol/tangostationcontrol/devices/docker_device.py

@@ -115,8 +115,8 @@ class Docker(lofar_device):
     hdbppts_es_RW = attribute_wrapper(comms_annotation={"container": "hdbppts-es"}, datatype=bool, access=AttrWriteType.READ_WRITE)
     itango_R = attribute_wrapper(comms_annotation={"container": "itango"}, datatype=bool)
     itango_RW = attribute_wrapper(comms_annotation={"container": "itango"}, datatype=bool, access=AttrWriteType.READ_WRITE)
-    jupyter_R = attribute_wrapper(comms_annotation={"container": "jupyter"}, datatype=bool)
-    jupyter_RW = attribute_wrapper(comms_annotation={"container": "jupyter"}, datatype=bool, access=AttrWriteType.READ_WRITE)
+    jupyter_lab_R = attribute_wrapper(comms_annotation={"container": "jupyter-lab"}, datatype=bool)
+    jupyter_lab_RW = attribute_wrapper(comms_annotation={"container": "jupyter-lab"}, datatype=bool, access=AttrWriteType.READ_WRITE)
     tangodb_R = attribute_wrapper(comms_annotation={"container": "tangodb"}, datatype=bool)
     tangodb_RW = attribute_wrapper(comms_annotation={"container": "tangodb"}, datatype=bool, access=AttrWriteType.READ_WRITE)
     prometheus_R = attribute_wrapper(comms_annotation={"container": "prometheus"}, datatype=bool)