diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e99cc9037001fe0c058c4ba36c72ea878821ff74..f485339fb83baac3381ae6002fdc4db469302e1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,7 +96,7 @@ docker_build_image_all: - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh loki latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh logstash latest - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter latest + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh ccd-sim latest - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim latest @@ -215,11 +215,11 @@ docker_build_image_jupyter: refs: - merge_requests changes: - - docker-compose/jupyter.yml - - docker-compose/jupyter/* + - docker-compose/jupyter-lab.yml + - docker-compose/jupyterlab/* script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter-lab $tag docker_build_image_apsct_sim: extends: .base_docker_images_except only: diff --git a/README.md b/README.md index b05bf6bfc86a5b55ea1043fa977fe0ac0c2a0f89..f97dc2a507de58910c863766bab831b17b71e05d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Station Control software related to Tango devices. * [User documentation (ReadTheDocs (Sphinx / ReStructuredText))](tangostationcontrol/docs/README.md) * [Docker compose documentation](docker-compose/README.md) * [Timescaledb](docker-compose/timescaledb/README.md) - * [Jupyter startup files](docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md) + * [Jupyter startup files](docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/README.md) * [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter) * [Developer Documentation](#development) * [Deployments](deploy/README.md) @@ -71,7 +71,7 @@ automatically be restarted on reboot or failure. Stop them explicitly to bring them down (`make stop <container>`). Most notably, you will have web interfaces available at: - - http://localhost:8888 (Jupyter Notebook) + - http://localhost:8888 (Jupyter Lab) - http://localhost:5601 (ElasticSearch) - http://localhost:3000 (Grafana). diff --git a/bootstrap/sbin/rebuild_system_from_scratch.sh b/bootstrap/sbin/rebuild_system_from_scratch.sh index 0af4d0b19d6fd85f48040265055235399c107a9e..387cb94faaa89d3b1e563da0b9dc7c10c5ee74a0 100755 --- a/bootstrap/sbin/rebuild_system_from_scratch.sh +++ b/bootstrap/sbin/rebuild_system_from_scratch.sh @@ -106,7 +106,7 @@ function configure_elk() function start_support_images() { (cd ${HOME_DIR}/docker-compose && make start elk - make start jupyter) + make start jupyter-lab) } function start_lofar_images() diff --git a/docker-compose/jupyter-lab.yml b/docker-compose/jupyter-lab.yml index 2c5984dc6eac7327f0af29a9251e4834a7f9c2ff..6c4b6e75047a84bdc5268d37f2ca3cef15a76977 100644 --- a/docker-compose/jupyter-lab.yml +++ b/docker-compose/jupyter-lab.yml @@ -1,7 +1,7 @@ # # Docker compose file that launches Jupyter Lab for interactive iTango sessions over HTTP. # -# Connect by surfing to http://localhost:8889/ +# Connect by surfing to http://localhost:8888/ # View logs through 'docker logs -f -t jupyter-lab' # # Defines: @@ -33,10 +33,10 @@ services: environment: - TANGO_HOST=${TANGO_HOST} ports: - - "8889:8889" + - "8888:80" user: ${CONTAINER_EXECUTION_UID} working_dir: /jupyter-notebooks entrypoint: - /opt/lofar/tango/bin/start-ds.sh - - jupyter lab --port=8889 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password= + - jupyter lab --port=80 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password= restart: unless-stopped diff --git a/docker-compose/jupyter.yml b/docker-compose/jupyter.yml deleted file mode 100644 index 49824c9f0ce40632553e699cf407e5a195b38386..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Docker compose file that launches Jupyter Notebook for interactive iTango sessions over HTTP. -# -# Connect by surfing to http://localhost:8888/ -# View logs through 'docker logs -f -t jupyter' -# -# Defines: -# - jupyter: Jupyter Notebook with iTango support -# - -version: '2.1' - -services: - jupyter: - build: - context: jupyter - args: - CONTAINER_EXECUTION_UID: ${CONTAINER_EXECUTION_UID} - SOURCE_IMAGE: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/tango-itango:${TANGO_ITANGO_VERSION} - container_name: ${CONTAINER_NAME_PREFIX}jupyter - logging: - driver: "json-file" - options: - max-size: "100m" - max-file: "10" - networks: - - control - volumes: - - ..:/opt/lofar/tango:rw - - ../jupyter-notebooks:/jupyter-notebooks:rw - - ${HOME}:/hosthome - - ${SCRATCH}:/scratch:rw - environment: - - TANGO_HOST=${TANGO_HOST} - ports: - - "8888:8888" - user: ${CONTAINER_EXECUTION_UID} - working_dir: /jupyter-notebooks - entrypoint: - - /opt/lofar/tango/bin/start-ds.sh - - /usr/bin/tini -- /usr/local/bin/jupyter-notebook --port=8888 --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --NotebookApp.password= - restart: unless-stopped diff --git a/docker-compose/jupyter/Dockerfile b/docker-compose/jupyter/Dockerfile deleted file mode 100644 index abb6f8872b202f952f587165a17fb08856cc653c..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -ARG SOURCE_IMAGE -FROM ${SOURCE_IMAGE} - -# UID if the user that this container will run under. This is needed to give directories -# that are needed for temporary storage the proper owner and access rights. -ARG CONTAINER_EXECUTION_UID=1000 - -# Create new user with uid but only if uid not used -RUN sudo adduser --disabled-password --system --uid ${CONTAINER_EXECUTION_UID} --no-create-home --home ${HOME} user || exit 0 -RUN sudo chown ${CONTAINER_EXECUTION_UID} -R ${HOME} - -# Add compiler to install python packages which come with C++ code -RUN sudo apt-get update -y -RUN sudo apt-get install -y g++ gcc python3-dev - -# start-ds file synchronization requirements -RUN sudo apt-get install -y rsync - -# Install git to install pip requirements from git -RUN sudo apt-get install -y git - -# Install dependencies of our scripts (bin/start-ds.sh) -RUN sudo apt-get install -y rsync - -COPY requirements.txt ./ -RUN sudo pip3 install -r requirements.txt - -# Install some version of the casacore measures tables, to allow basic delay computation analysis in the notebooks -RUN sudo apt-get install -y casacore-data - -# see https://github.com/jupyter/nbconvert/issues/1434 -RUN sudo bash -c "echo DEFAULT_ARGS += [\\\"--no-sandbox\\\"] >> /usr/local/lib/python3.7/dist-packages/pyppeteer/launcher.py" -RUN sudo apt-get update -y -RUN sudo apt-get install -y git gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libcairo-gobject2 libxinerama1 libgtk2.0-0 libpangoft2-1.0-0 libthai0 libpixman-1-0 libxcb-render0 libharfbuzz0b libdatrie1 libgraphite2-3 libgbm1 - -# Allow Download as -> PDF via LaTeX -RUN sudo apt-get install -y texlive-xetex texlive-fonts-recommended texlive-latex-recommended cm-super - -# Configure jupyter_bokeh -RUN sudo mkdir -p /usr/share/jupyter /usr/etc -RUN sudo chmod a+rwx /usr/share/jupyter /usr/etc -RUN sudo jupyter nbextension install --sys-prefix --symlink --py jupyter_bokeh -RUN sudo jupyter nbextension enable jupyter_bokeh --py --sys-prefix - -# Install profiles for ipython & jupyter -COPY ipython-profiles /opt/ipython-profiles/ -RUN sudo chown ${CONTAINER_EXECUTION_UID} -R /opt/ipython-profiles -COPY jupyter-kernels /usr/local/share/jupyter/kernels/ - -# Install patched jupyter executable -COPY jupyter-notebook /usr/local/bin/jupyter-notebook - -# Add Tini. Tini operates as a process subreaper for jupyter. This prevents kernel crashes. -ENV TINI_VERSION v0.6.0 -ENV JUPYTER_RUNTIME_DIR=/tmp -ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini -RUN sudo chmod +x /usr/bin/tini - -USER ${CONTAINER_EXECUTION_UID} -# pyppeteer-install installs in the homedir, so run it as the user that will execute the notebook -RUN pyppeteer-install diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/ipython_config.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/ipython_config.py deleted file mode 100644 index 91b04aaa3a20232b60e5ced00a99648891955ce5..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/ipython_config.py +++ /dev/null @@ -1,578 +0,0 @@ -# Configuration file for ipython. - -#------------------------------------------------------------------------------ -# InteractiveShellApp(Configurable) configuration -#------------------------------------------------------------------------------ - -## A Mixin for applications that start InteractiveShell instances. -# -# Provides configurables for loading extensions and executing files as part of -# configuring a Shell environment. -# -# The following methods should be called by the :meth:`initialize` method of the -# subclass: -# -# - :meth:`init_path` -# - :meth:`init_shell` (to be implemented by the subclass) -# - :meth:`init_gui_pylab` -# - :meth:`init_extensions` -# - :meth:`init_code` - -## Execute the given command string. -#c.InteractiveShellApp.code_to_run = '' - -## Run the file referenced by the PYTHONSTARTUP environment variable at IPython -# startup. -#c.InteractiveShellApp.exec_PYTHONSTARTUP = True - -## List of files to run at IPython startup. -#c.InteractiveShellApp.exec_files = [] - -## lines of code to run at IPython startup. -#c.InteractiveShellApp.exec_lines = [] - -## A list of dotted module names of IPython extensions to load. -#c.InteractiveShellApp.extensions = [] - -## dotted module name of an IPython extension to load. -#c.InteractiveShellApp.extra_extension = '' - -## A file to be run -#c.InteractiveShellApp.file_to_run = '' - -## Enable GUI event loop integration with any of ('glut', 'gtk', 'gtk2', 'gtk3', -# 'osx', 'pyglet', 'qt', 'qt4', 'qt5', 'tk', 'wx', 'gtk2', 'qt4'). -#c.InteractiveShellApp.gui = None - -## Should variables loaded at startup (by startup files, exec_lines, etc.) be -# hidden from tools like %who? -#c.InteractiveShellApp.hide_initial_ns = True - -## Configure matplotlib for interactive use with the default matplotlib backend. -#c.InteractiveShellApp.matplotlib = None - -## Run the module as a script. -#c.InteractiveShellApp.module_to_run = '' - -## Pre-load matplotlib and numpy for interactive use, selecting a particular -# matplotlib backend and loop integration. -#c.InteractiveShellApp.pylab = None - -## If true, IPython will populate the user namespace with numpy, pylab, etc. and -# an ``import *`` is done from numpy and pylab, when using pylab mode. -# -# When False, pylab mode should not import any names into the user namespace. -#c.InteractiveShellApp.pylab_import_all = True - -## Reraise exceptions encountered loading IPython extensions? -#c.InteractiveShellApp.reraise_ipython_extension_failures = False - -#------------------------------------------------------------------------------ -# Application(SingletonConfigurable) configuration -#------------------------------------------------------------------------------ - -## This is an application. - -## The date format used by logging formatters for %(asctime)s -#c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S' - -## The Logging format template -#c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s' - -## Set the log level by value or name. -#c.Application.log_level = 30 - -#------------------------------------------------------------------------------ -# BaseIPythonApplication(Application) configuration -#------------------------------------------------------------------------------ - -## IPython: an enhanced interactive Python shell. - -## Whether to create profile dir if it doesn't exist -#c.BaseIPythonApplication.auto_create = False - -## Whether to install the default config files into the profile dir. If a new -# profile is being created, and IPython contains config files for that profile, -# then they will be staged into the new directory. Otherwise, default config -# files will be automatically generated. -#c.BaseIPythonApplication.copy_config_files = False - -## Path to an extra config file to load. -# -# If specified, load this config file in addition to any other IPython config. -#c.BaseIPythonApplication.extra_config_file = '' - -## The name of the IPython directory. This directory is used for logging -# configuration (through profiles), history storage, etc. The default is usually -# $HOME/.ipython. This option can also be specified through the environment -# variable IPYTHONDIR. -#c.BaseIPythonApplication.ipython_dir = '' - -## Whether to overwrite existing config files when copying -#c.BaseIPythonApplication.overwrite = False - -## The IPython profile to use. -#c.BaseIPythonApplication.profile = 'default' - -## Create a massive crash report when IPython encounters what may be an internal -# error. The default is to append a short message to the usual traceback -#c.BaseIPythonApplication.verbose_crash = False - -#------------------------------------------------------------------------------ -# TerminalIPythonApp(BaseIPythonApplication,InteractiveShellApp) configuration -#------------------------------------------------------------------------------ - -## Whether to display a banner upon starting IPython. -#c.TerminalIPythonApp.display_banner = True - -## If a command or file is given via the command-line, e.g. 'ipython foo.py', -# start an interactive shell after executing the file or command. -#c.TerminalIPythonApp.force_interact = False - -## Class to use to instantiate the TerminalInteractiveShell object. Useful for -# custom Frontends -#c.TerminalIPythonApp.interactive_shell_class = 'IPython.terminal.interactiveshell.TerminalInteractiveShell' - -## Start IPython quickly by skipping the loading of config files. -#c.TerminalIPythonApp.quick = False - -#------------------------------------------------------------------------------ -# InteractiveShell(SingletonConfigurable) configuration -#------------------------------------------------------------------------------ - -## An enhanced, interactive shell for Python. - -## 'all', 'last', 'last_expr' or 'none', specifying which nodes should be run -# interactively (displaying output from expressions). -#c.InteractiveShell.ast_node_interactivity = 'last_expr' - -## A list of ast.NodeTransformer subclass instances, which will be applied to -# user input before code is run. -#c.InteractiveShell.ast_transformers = [] - -## Make IPython automatically call any callable object even if you didn't type -# explicit parentheses. For example, 'str 43' becomes 'str(43)' automatically. -# The value can be '0' to disable the feature, '1' for 'smart' autocall, where -# it is not applied if there are no more arguments on the line, and '2' for -# 'full' autocall, where all callable objects are automatically called (even if -# no arguments are present). -#c.InteractiveShell.autocall = 0 - -## Autoindent IPython code entered interactively. -#c.InteractiveShell.autoindent = True - -## Enable magic commands to be called without the leading %. -#c.InteractiveShell.automagic = True - -## The part of the banner to be printed before the profile -#c.InteractiveShell.banner1 = 'Python 3.7.3 (default, Jul 25 2020, 13:03:44) \nType "copyright", "credits" or "license" for more information.\n\nIPython 5.8.0 -- An enhanced Interactive Python.\n? -> Introduction and overview of IPython\'s features.\n%quickref -> Quick reference.\nhelp -> Python\'s own help system.\nobject? -> Details about \'object\', use \'object??\' for extra details.\n' - -## The part of the banner to be printed after the profile -#c.InteractiveShell.banner2 = '' - -## Set the size of the output cache. The default is 1000, you can change it -# permanently in your config file. Setting it to 0 completely disables the -# caching system, and the minimum value accepted is 20 (if you provide a value -# less than 20, it is reset to 0 and a warning is issued). This limit is -# defined because otherwise you'll spend more time re-flushing a too small cache -# than working -#c.InteractiveShell.cache_size = 1000 - -## Use colors for displaying information about objects. Because this information -# is passed through a pager (like 'less'), and some pagers get confused with -# color codes, this capability can be turned off. -#c.InteractiveShell.color_info = True - -## Set the color scheme (NoColor, Neutral, Linux, or LightBG). -#c.InteractiveShell.colors = 'Neutral' - -## -#c.InteractiveShell.debug = False - -## **Deprecated** -# -# Will be removed in IPython 6.0 -# -# Enable deep (recursive) reloading by default. IPython can use the deep_reload -# module which reloads changes in modules recursively (it replaces the reload() -# function, so you don't need to change anything to use it). `deep_reload` -# forces a full reload of modules whose code may have changed, which the default -# reload() function does not. When deep_reload is off, IPython will use the -# normal reload(), but deep_reload will still be available as dreload(). -#c.InteractiveShell.deep_reload = False - -## Don't call post-execute functions that have failed in the past. -#c.InteractiveShell.disable_failing_post_execute = False - -## If True, anything that would be passed to the pager will be displayed as -# regular output instead. -#c.InteractiveShell.display_page = False - -## (Provisional API) enables html representation in mime bundles sent to pagers. -#c.InteractiveShell.enable_html_pager = False - -## Total length of command history -#c.InteractiveShell.history_length = 10000 - -## The number of saved history entries to be loaded into the history buffer at -# startup. -#c.InteractiveShell.history_load_length = 1000 - -## -#c.InteractiveShell.ipython_dir = '' - -## Start logging to the given file in append mode. Use `logfile` to specify a log -# file to **overwrite** logs to. -#c.InteractiveShell.logappend = '' - -## The name of the logfile to use. -#c.InteractiveShell.logfile = '' - -## Start logging to the default log file in overwrite mode. Use `logappend` to -# specify a log file to **append** logs to. -#c.InteractiveShell.logstart = False - -## -#c.InteractiveShell.object_info_string_level = 0 - -## Automatically call the pdb debugger after every exception. -#c.InteractiveShell.pdb = False - -## Deprecated since IPython 4.0 and ignored since 5.0, set -# TerminalInteractiveShell.prompts object directly. -#c.InteractiveShell.prompt_in1 = 'In [\\#]: ' - -## Deprecated since IPython 4.0 and ignored since 5.0, set -# TerminalInteractiveShell.prompts object directly. -#c.InteractiveShell.prompt_in2 = ' .\\D.: ' - -## Deprecated since IPython 4.0 and ignored since 5.0, set -# TerminalInteractiveShell.prompts object directly. -#c.InteractiveShell.prompt_out = 'Out[\\#]: ' - -## Deprecated since IPython 4.0 and ignored since 5.0, set -# TerminalInteractiveShell.prompts object directly. -#c.InteractiveShell.prompts_pad_left = True - -## -#c.InteractiveShell.quiet = False - -## -#c.InteractiveShell.separate_in = '\n' - -## -#c.InteractiveShell.separate_out = '' - -## -#c.InteractiveShell.separate_out2 = '' - -## Show rewritten input, e.g. for autocall. -#c.InteractiveShell.show_rewritten_input = True - -## Enables rich html representation of docstrings. (This requires the docrepr -# module). -#c.InteractiveShell.sphinxify_docstring = False - -## -#c.InteractiveShell.wildcards_case_sensitive = True - -## -#c.InteractiveShell.xmode = 'Context' - -#------------------------------------------------------------------------------ -# TerminalInteractiveShell(InteractiveShell) configuration -#------------------------------------------------------------------------------ - -## Set to confirm when you try to exit IPython with an EOF (Control-D in Unix, -# Control-Z/Enter in Windows). By typing 'exit' or 'quit', you can force a -# direct exit without any confirmation. -#c.TerminalInteractiveShell.confirm_exit = True - -## Options for displaying tab completions, 'column', 'multicolumn', and -# 'readlinelike'. These options are for `prompt_toolkit`, see `prompt_toolkit` -# documentation for more information. -#c.TerminalInteractiveShell.display_completions = 'multicolumn' - -## Shortcut style to use at the prompt. 'vi' or 'emacs'. -#c.TerminalInteractiveShell.editing_mode = 'emacs' - -## Set the editor used by IPython (default to $EDITOR/vi/notepad). -#c.TerminalInteractiveShell.editor = 'vi' - -## Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. This is -# in addition to the F2 binding, which is always enabled. -#c.TerminalInteractiveShell.extra_open_editor_shortcuts = False - -## Highlight matching brackets. -#c.TerminalInteractiveShell.highlight_matching_brackets = True - -## The name or class of a Pygments style to use for syntax highlighting. To see -# available styles, run `pygmentize -L styles`. -#c.TerminalInteractiveShell.highlighting_style = traitlets.Undefined - -## Override highlighting format for specific tokens -#c.TerminalInteractiveShell.highlighting_style_overrides = {} - -## Enable mouse support in the prompt -#c.TerminalInteractiveShell.mouse_support = False - -## Class used to generate Prompt token for prompt_toolkit -#c.TerminalInteractiveShell.prompts_class = 'IPython.terminal.prompts.Prompts' - -## Use `raw_input` for the REPL, without completion and prompt colors. -# -# Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. -# Known usage are: IPython own testing machinery, and emacs inferior-shell -# integration through elpy. -# -# This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT` environment -# variable is set, or the current terminal is not a tty. -#c.TerminalInteractiveShell.simple_prompt = False - -## Number of line at the bottom of the screen to reserve for the completion menu -#c.TerminalInteractiveShell.space_for_menu = 6 - -## Automatically set the terminal title -#c.TerminalInteractiveShell.term_title = True - -## Use 24bit colors instead of 256 colors in prompt highlighting. If your -# terminal supports true color, the following command should print 'TRUECOLOR' -# in orange: printf "\x1b[38;2;255;100;0mTRUECOLOR\x1b[0m\n" -#c.TerminalInteractiveShell.true_color = False - -#------------------------------------------------------------------------------ -# HistoryAccessor(HistoryAccessorBase) configuration -#------------------------------------------------------------------------------ - -## Access the history database without adding to it. -# -# This is intended for use by standalone history tools. IPython shells use -# HistoryManager, below, which is a subclass of this. - -## Options for configuring the SQLite connection -# -# These options are passed as keyword args to sqlite3.connect when establishing -# database conenctions. -#c.HistoryAccessor.connection_options = {} - -## enable the SQLite history -# -# set enabled=False to disable the SQLite history, in which case there will be -# no stored history, no SQLite connection, and no background saving thread. -# This may be necessary in some threaded environments where IPython is embedded. -#c.HistoryAccessor.enabled = True - -## Path to file to use for SQLite history database. -# -# By default, IPython will put the history database in the IPython profile -# directory. If you would rather share one history among profiles, you can set -# this value in each, so that they are consistent. -# -# Due to an issue with fcntl, SQLite is known to misbehave on some NFS mounts. -# If you see IPython hanging, try setting this to something on a local disk, -# e.g:: -# -# ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite -# -# you can also use the specific value `:memory:` (including the colon at both -# end but not the back ticks), to avoid creating an history file. -#c.HistoryAccessor.hist_file = '' - -#------------------------------------------------------------------------------ -# HistoryManager(HistoryAccessor) configuration -#------------------------------------------------------------------------------ - -## A class to organize all history-related functionality in one place. - -## Write to database every x commands (higher values save disk access & power). -# Values of 1 or less effectively disable caching. -#c.HistoryManager.db_cache_size = 0 - -## Should the history database include output? (default: no) -#c.HistoryManager.db_log_output = False - -#------------------------------------------------------------------------------ -# ProfileDir(LoggingConfigurable) configuration -#------------------------------------------------------------------------------ - -## An object to manage the profile directory and its resources. -# -# The profile directory is used by all IPython applications, to manage -# configuration, logging and security. -# -# This object knows how to find, create and manage these directories. This -# should be used by any code that wants to handle profiles. - -## Set the profile location directly. This overrides the logic used by the -# `profile` option. -#c.ProfileDir.location = '' - -#------------------------------------------------------------------------------ -# BaseFormatter(Configurable) configuration -#------------------------------------------------------------------------------ - -## A base formatter class that is configurable. -# -# This formatter should usually be used as the base class of all formatters. It -# is a traited :class:`Configurable` class and includes an extensible API for -# users to determine how their objects are formatted. The following logic is -# used to find a function to format an given object. -# -# 1. The object is introspected to see if it has a method with the name -# :attr:`print_method`. If is does, that object is passed to that method -# for formatting. -# 2. If no print method is found, three internal dictionaries are consulted -# to find print method: :attr:`singleton_printers`, :attr:`type_printers` -# and :attr:`deferred_printers`. -# -# Users should use these dictionaries to register functions that will be used to -# compute the format data for their objects (if those objects don't have the -# special print methods). The easiest way of using these dictionaries is through -# the :meth:`for_type` and :meth:`for_type_by_name` methods. -# -# If no function/callable is found to compute the format data, ``None`` is -# returned and this format type is not used. - -## -#c.BaseFormatter.deferred_printers = {} - -## -#c.BaseFormatter.enabled = True - -## -#c.BaseFormatter.singleton_printers = {} - -## -#c.BaseFormatter.type_printers = {} - -#------------------------------------------------------------------------------ -# PlainTextFormatter(BaseFormatter) configuration -#------------------------------------------------------------------------------ - -## The default pretty-printer. -# -# This uses :mod:`IPython.lib.pretty` to compute the format data of the object. -# If the object cannot be pretty printed, :func:`repr` is used. See the -# documentation of :mod:`IPython.lib.pretty` for details on how to write pretty -# printers. Here is a simple example:: -# -# def dtype_pprinter(obj, p, cycle): -# if cycle: -# return p.text('dtype(...)') -# if hasattr(obj, 'fields'): -# if obj.fields is None: -# p.text(repr(obj)) -# else: -# p.begin_group(7, 'dtype([') -# for i, field in enumerate(obj.descr): -# if i > 0: -# p.text(',') -# p.breakable() -# p.pretty(field) -# p.end_group(7, '])') - -## -#c.PlainTextFormatter.float_precision = '' - -## Truncate large collections (lists, dicts, tuples, sets) to this size. -# -# Set to 0 to disable truncation. -#c.PlainTextFormatter.max_seq_length = 1000 - -## -#c.PlainTextFormatter.max_width = 79 - -## -#c.PlainTextFormatter.newline = '\n' - -## -#c.PlainTextFormatter.pprint = True - -## -#c.PlainTextFormatter.verbose = False - -#------------------------------------------------------------------------------ -# Completer(Configurable) configuration -#------------------------------------------------------------------------------ - -## Enable unicode completions, e.g. \alpha<tab> . Includes completion of latex -# commands, unicode names, and expanding unicode characters back to latex -# commands. -#c.Completer.backslash_combining_completions = True - -## Activate greedy completion PENDING DEPRECTION. this is now mostly taken care -# of with Jedi. -# -# This will enable completion on elements of lists, results of function calls, -# etc., but can be unsafe because the code is actually evaluated on TAB. -#c.Completer.greedy = False - -#------------------------------------------------------------------------------ -# IPCompleter(Completer) configuration -#------------------------------------------------------------------------------ - -## Extension of the completer class with IPython-specific features - -## DEPRECATED as of version 5.0. -# -# Instruct the completer to use __all__ for the completion -# -# Specifically, when completing on ``object.<tab>``. -# -# When True: only those names in obj.__all__ will be included. -# -# When False [default]: the __all__ attribute is ignored -#c.IPCompleter.limit_to__all__ = False - -## Whether to merge completion results into a single list -# -# If False, only the completion results from the first non-empty completer will -# be returned. -#c.IPCompleter.merge_completions = True - -## Instruct the completer to omit private method names -# -# Specifically, when completing on ``object.<tab>``. -# -# When 2 [default]: all names that start with '_' will be excluded. -# -# When 1: all 'magic' names (``__foo__``) will be excluded. -# -# When 0: nothing will be excluded. -#c.IPCompleter.omit__names = 2 - -#------------------------------------------------------------------------------ -# ScriptMagics(Magics) configuration -#------------------------------------------------------------------------------ - -## Magics for talking to scripts -# -# This defines a base `%%script` cell magic for running a cell with a program in -# a subprocess, and registers a few top-level magics that call %%script with -# common interpreters. - -## Extra script cell magics to define -# -# This generates simple wrappers of `%%script foo` as `%%foo`. -# -# If you want to add script magics that aren't on your path, specify them in -# script_paths -#c.ScriptMagics.script_magics = [] - -## Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby' -# -# Only necessary for items in script_magics where the default path will not find -# the right interpreter. -#c.ScriptMagics.script_paths = {} - -#------------------------------------------------------------------------------ -# StoreMagics(Magics) configuration -#------------------------------------------------------------------------------ - -## Lightweight persistence for python variables. -# -# Provides the %store magic. - -## If True, any %store-d variables will be automatically restored when IPython -# starts. -#c.StoreMagics.autorestore = False diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py deleted file mode 100644 index 38fcb84c3417c6b19d89527be6f8122bd0249765..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/00-tango.py +++ /dev/null @@ -1 +0,0 @@ -from tango import * diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py deleted file mode 100644 index 1123fde3afca3a742f2fb239d12bb361d91e5e12..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py +++ /dev/null @@ -1,23 +0,0 @@ -# Create shortcuts for our devices -apsct = DeviceProxy("STAT/APSCT/1") -ccd = DeviceProxy("STAT/CCD/1") -apspu = DeviceProxy("STAT/APSPU/1") -recv = DeviceProxy("STAT/RECV/1") -sdp = DeviceProxy("STAT/SDP/1") -bst = DeviceProxy("STAT/BST/1") -sst = DeviceProxy("STAT/SST/1") -xst = DeviceProxy("STAT/XST/1") -unb2 = DeviceProxy("STAT/UNB2/1") -boot = DeviceProxy("STAT/Boot/1") -tilebeam = DeviceProxy("STAT/TileBeam/1") -pcon = DeviceProxy("STAT/PCON/1") -psoc = DeviceProxy("STAT/PSOC/1") -beamlet = DeviceProxy("STAT/Beamlet/1") -digitalbeam = DeviceProxy("STAT/DigitalBeam/1") -antennafield = DeviceProxy("STAT/AntennaField/1") -docker = DeviceProxy("STAT/Docker/1") -temperaturemanager = DeviceProxy("STAT/TemperatureManager/1") -configuration = DeviceProxy("STAT/Configuration/1") - -# Put them in a list in case one wants to iterate -devices = [apsct, ccd, apspu, recv, sdp, bst, sst, xst, unb2, boot, tilebeam, beamlet, digitalbeam, antennafield, temperaturemanager, docker, pcon, psoc, configuration] diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py deleted file mode 100644 index d21ed1cf013d73b700cbc72e3d89ef9541efcacc..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/02-stationcontrol.py +++ /dev/null @@ -1 +0,0 @@ -import tangostationcontrol diff --git a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md b/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md deleted file mode 100644 index 61d470004218ae459ce7bfdc974f7c86e0790486..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md +++ /dev/null @@ -1,11 +0,0 @@ -This is the IPython startup directory - -.py and .ipy files in this directory will be run *prior* to any code or files specified -via the exec_lines or exec_files configurables whenever you load this profile. - -Files will be run in lexicographical order, so you can control the execution order of files -with a prefix, e.g.:: - - 00-first.py - 50-middle.py - 99-last.ipy diff --git a/docker-compose/jupyter/jupyter-kernels/stationcontrol/kernel.json b/docker-compose/jupyter/jupyter-kernels/stationcontrol/kernel.json deleted file mode 100644 index ff6d4a1a01d0f7bd6eda3a40886eae74b451a5a4..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/jupyter-kernels/stationcontrol/kernel.json +++ /dev/null @@ -1,13 +0,0 @@ - { - "argv": [ - "python", - "-m", - "ipykernel", - "-f", - "{connection_file}", - "--profile-dir", - "/opt/ipython-profiles/stationcontrol-jupyter/" - ], - "language": "python", - "display_name": "StationControl" -} diff --git a/docker-compose/jupyter/jupyter-notebook b/docker-compose/jupyter/jupyter-notebook deleted file mode 100755 index 59613a137cc1bb5c86b4cd7c82f3a2cb1f9abde3..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/jupyter-notebook +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- -# An adjustment of the `jupyter-notebook' executable patched to: -# - log to the ELK stack -# -# We go straight for the notebook executable here, as the "jupyter" command -# execvp's into the requested notebook subcommand, erasing all configuration -# we set here. -import re -import sys - -from notebook.notebookapp import main - -from logstash_async.handler import AsynchronousLogstashHandler, LogstashFormatter -import logging - -if __name__ == '__main__': - # log to the tcp_input of logstash in our ELK stack - handler = AsynchronousLogstashHandler("elk", 5959, database_path='/tmp/pending_log_messages.db') - - # add to logger of Jupyter traitlets Application. As that logger is configured not to propagate - # messages upward, we need to configure it directly. - logger = logging.getLogger("NotebookApp") - logger.addHandler(handler) - logger.setLevel(logging.DEBUG) - - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/docker-compose/jupyter/requirements.txt b/docker-compose/jupyter/requirements.txt deleted file mode 100644 index 27a6d1ed38ea1edc00323ae7b7e20a69242746e6..0000000000000000000000000000000000000000 --- a/docker-compose/jupyter/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -ipython >=7.27.0,!=7.28.0 # BSD -jupyter -ipykernel -jupyter_bokeh -matplotlib -jupyterplot -nbconvert -notebook-as-pdf -python-logstash-async -PyMySQL[rsa] -psycopg2-binary >= 2.9.2 #LGPL -sqlalchemy -pyvisa -pyvisa-py -opcua - -numpy -scipy - -pabeam@git+https://git.astron.nl/mevius/grate # Apache2 -lofar-station-client@git+https://git.astron.nl/lofar2.0/lofar-station-client # Apache2 diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index c785e4c8f27981f393dd353f814ce6a0b127e7d3..0464c8b33c341929936b6c1b810e816869bc8937 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -88,7 +88,7 @@ LOCAL_IMAGES=( "hdbppts-es hdbppts-es y" "grafana grafana n" "prometheus prometheus n" - "jupyter docker-compose_jupyter n" + "jupyter-lab docker-compose_jupyter-lab n" "integration-test docker-compose_integration-test n" "tango-prometheus-exporter docker-compose_tango-prometheus-exporter n" ) diff --git a/tangostationcontrol/docs/source/interfaces/control.rst b/tangostationcontrol/docs/source/interfaces/control.rst index 4dac94eb9112e959b664eaa671e2317dc3569ca3..e2f436f378acd97280845886f32cf0bf4f48eb22 100644 --- a/tangostationcontrol/docs/source/interfaces/control.rst +++ b/tangostationcontrol/docs/source/interfaces/control.rst @@ -12,7 +12,7 @@ The station offers Juypyter notebooks On http://localhost:8888, which allow one The notebooks provide some predefined variables, so you don't have to look them up: -.. literalinclude:: ../../../../docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py +.. literalinclude:: ../../../../docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py Note: the Jupyter notebooks use enhancements from the ``itango`` suite, which provide tab completions, but also the ``Device`` alias for ``DeviceProxy`` as was used in the Python examples in the next section. diff --git a/tangostationcontrol/tangostationcontrol/devices/README.md b/tangostationcontrol/tangostationcontrol/devices/README.md index 0a1081baac732665cb9693fd9cb43f40c0b7dcfa..d1ced11fcd5a0352f85da4aab3ecd18601eb7916 100644 --- a/tangostationcontrol/tangostationcontrol/devices/README.md +++ b/tangostationcontrol/tangostationcontrol/devices/README.md @@ -7,7 +7,7 @@ This directory contains the sources for our custom Tango devices. If a new device is added, it will (likely) need to be referenced in several places. Adjust or add the following files (referenced from the repository root), following the pattern shown by the devices already there: - Adjust `CDB/LOFAR_ConfigDb.json` to create the device in the Tango device database, -- Adjust `docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` and `docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter and Jupyter-Lab, +- Adjust `docker-compose/jupyterlab/ipython-profiles/stationcontrol-jupyter/startup/01-devices.py` to make an alias for it available in Jupyter-Lab, - Adjust `tangostationcontrol/tangostationcontrol/devices/boot.py` to add the device to the station initialisation sequence, - Add to `docker-compose/` to create a YaML file to start the device in a docker container. NOTE: it needs a unique 57xx port assigned (current _unused_ port value: 5723), a unique 58xx port for ZMQ events, and a unique 59xx port for ZMQ heartbeat - Adjust `tangostationcontrol/setup.cfg` to add an entry point for the device in the package installation, diff --git a/tangostationcontrol/tangostationcontrol/devices/docker_device.py b/tangostationcontrol/tangostationcontrol/devices/docker_device.py index a746ad1cd949ed488a90802d0ac7ccd903133ec3..35009237c2018f760e3c00c2bb82d4c6ce064907 100644 --- a/tangostationcontrol/tangostationcontrol/devices/docker_device.py +++ b/tangostationcontrol/tangostationcontrol/devices/docker_device.py @@ -115,8 +115,8 @@ class Docker(lofar_device): hdbppts_es_RW = attribute_wrapper(comms_annotation={"container": "hdbppts-es"}, datatype=bool, access=AttrWriteType.READ_WRITE) itango_R = attribute_wrapper(comms_annotation={"container": "itango"}, datatype=bool) itango_RW = attribute_wrapper(comms_annotation={"container": "itango"}, datatype=bool, access=AttrWriteType.READ_WRITE) - jupyter_R = attribute_wrapper(comms_annotation={"container": "jupyter"}, datatype=bool) - jupyter_RW = attribute_wrapper(comms_annotation={"container": "jupyter"}, datatype=bool, access=AttrWriteType.READ_WRITE) + jupyter_lab_R = attribute_wrapper(comms_annotation={"container": "jupyter-lab"}, datatype=bool) + jupyter_lab_RW = attribute_wrapper(comms_annotation={"container": "jupyter-lab"}, datatype=bool, access=AttrWriteType.READ_WRITE) tangodb_R = attribute_wrapper(comms_annotation={"container": "tangodb"}, datatype=bool) tangodb_RW = attribute_wrapper(comms_annotation={"container": "tangodb"}, datatype=bool, access=AttrWriteType.READ_WRITE) prometheus_R = attribute_wrapper(comms_annotation={"container": "prometheus"}, datatype=bool)