Merge branch 'master' of https://git.astron.nl/lofar2.0/tango into...

Merge branch 'master' of https://git.astron.nl/lofar2.0/tango into L2SS-216_2021-05-17-Branched_from_master-SDP_hotfix

bootstrap/bin/updatePythonEnv.sh

+#! /usr/bin/env bash
+
+venv=${VIRTUAL_ENV:?This is currently not a Python3 venv!  Will not upgrade venv packages.}
+echo -e "\nFound a Python3 venv in \"${VIRTUAL_ENV}\".\nWill now proceed with package upgrades.\n"
+
+pip="python3 -m pip"
+upgrade_command="${pip} install --upgrade"
+
+if [ -z ${OS} ]; then
+    OS=$(uname)
+fi
+if [ ${OS} = Darwin ]; then
+    find=gfind
+else
+    find=find
+fi
+
+function upgrade()
+{
+    if [ ${#} -ge 1 ]; then
+        echo "Upgrading ${@}..."
+        ${upgrade_command} ${@}
+        if [ ${?} -eq 0 ]; then
+            echo -e "\nUpgrading ${@} done.\n\nNow \"deactivate\" and \"source ${venv}/bin/activate\".\n"
+        else
+            echo -e "\nSomething went wrong during the upgrade.\nCheck the output!\n"
+        fi
+    else
+        echo -e "\nNo package for upgrading given.\n"
+    fi
+}
+
+upgrade pip wheel
+
+installed_packages=$(for i in $(python3 -m pip freeze | cut -d'=' -f1); do printf "%s " ${i}; done)
+upgrade ${installed_packages}

bootstrap/etc/lofar20rc.sh

+# Set up the LOFAR2.0 environment.
+# For the time being it is assumend that the LOFAR2.0 environment has to
+# co-exist with a LOFAR1 environment.
+
+# Set these for the host where you run SKA's Tango Docker images.
+# And export those directories for LOFAR in Tango Docker images.
+
+# Pass a directory as first parameter to this script.  This will
+# then be used as LOFAR20_DIR.  Otherwise the current directory will
+# be used.
+export LOFAR20_DIR=${1:-${PWD}}
+
+# This needs to be modified for a development environment.
+# Example:  ~/lofar2.0/tango
+# The current setting is for a production environment.
+export TANGO_LOFAR_LOCAL_DIR=${LOFAR20_DIR}/
+
+export TANGO_LOFAR_CONTAINER_DIR=${LOFAR20_DIR}/
+export TANGO_LOFAR_CONTAINER_MOUNT=${TANGO_LOFAR_LOCAL_DIR}:${TANGO_LOFAR_CONTAINER_DIR}:rw
+
+# This needs to be modified for a development environment.
+# In case you run multiple Docker networks on the same host in parallel, you need to specify a unique
+# network name for each of them.
+export NETWORK_MODE=lofar
+
+# It is assumed that the Tango host, the computer that runs the TangoDB, is this host.
+# If this is not true, then modify to the Tango host's FQDN and port.
+# Example:  export TANGO_HOST=station-xk25.astron.nl:10000
+export TANGO_HOST=$(hostname):10000
+
+
+#
+# NO MODIFICATION BEYOND THIS POINT!
+#
+
+# Provide the -v parameters for Docker and the -e ENV variables.
+export TANGO_CONTAINER_ENV="-e TANGO_LOFAR_CONTAINER_DIR=${TANGO_LOFAR_CONTAINER_DIR}"
+
+# Remove all LOFAR1 related environment modifications
+function remove_lofar()
+{
+    tmp=${1//:/ }
+    echo "$(for new in $(for i in ${tmp}; do printf "%s\n" ${i}; done | egrep -v '/opt/lofar/|/opt/WinCC|/opt/stationtest|/opt/operations'); do printf "%s:" ${new}; done)"
+}
+
+unset LOFARROOT
+export PATH=$(remove_lofar ${PATH})
+export LD_LIBRARY_PATH=$(remove_lofar ${LD_LIBRARY_PATH})
+export PYTHON_PATH=$(remove_lofar ${PYTHON_PATH})
+
+
+# Allow read access for everybody to allow Docker the forwarding of X11.
+chmod a+r ~/.Xauthority
+
+# Source the LOFAR2.0 Python3 venv if it exists.
+[ -z ${VIRTUAL_ENV} ] && [ -d ${LOFAR20_DIR}/lofar2.0_venv ] && source ${LOFAR20_DIR}/lofar2.0_venv/bin/activate

bootstrap/etc/requirements.txt

+astropy
+jupyter
+matplotlib
+numpy
+opcua-client
+pyqtgraph
+PyQt5
+asyncua
+dataclasses

bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh

+#! /usr/bin/env bash
+
+# Clean out local dirs and then clone a shallow copy
+# of LOFAR2.0 repos.
+
+# 2020-10-13, thomas
+# Currently correct.
+astron_gitlab=https://git.astron.nl/
+repo_dir=lofar2.0/
+branch=master
+
+for repo in lmc-base-classes ska-logging ska-docker opcua tango pypcc; do
+    cd ${repo}
+    # This will return 2 if the repo is clean.
+    clean="$(git status -u | wc -l)"
+    cd -
+    if [ "${clean}" = "2" ]; then
+        # The repo is clean, no new files, no modifications.
+        rm -rf ${repo}
+        git clone --depth 1 --branch ${branch} ${astron_gitlab}${repo_dir}${repo}.git
+    else
+        echo -e "*********\n\tThe repository ${repo} contains modifications.\n\tRestore the original content first before continuing.\nAborting now.\n"
+        exit -1
+    fi
+done
+
+# 2020-10-13, thomas
+# I need to move this repo to lofar2.0.
+repo=crossecho
+repo_dir=jurges
+rm -rf crossecho && git clone --depth 1 --branch add_simulation_mode_plus_patches ${astron_gitlab}${repo_dir}/${repo}.git
+
+if [ ! -s ska_logging -o $(readlink ska_logging) != ska-logging ]; then
+    echo "Creating symlink ska-logging -> ska_logging"
+    rm -f ska_logging
+    ln -s ska-logging ska_logging
+fi
+
+if [ ! -s skabase -o $(readlink skabase) != lmc-base-classes ]; then
+    echo "Creating symlink lmc-base-classes -> skabase"
+    rm -f skabase
+    ln -s lmc-base-classes skabase
+fi

bootstrap/sbin/delete_all_docker_images.sh

+#! /usr/bin/env bash
+
+function help()
+{
+    echo "You need to add a parameter \"YES_DELETE_ALL\" in order to really remove all Docker images."
+    exit 0
+}
+
+if [ ${#} -ne 1 ]; then
+    help
+elif [ "${1}" != "YES_DELETE_ALL" ]; then
+    help
+fi
+
+read -p "If you certain that you want to delete all Docker images, then enter now \"YES_DO_IT\" " reply
+if [ "${reply}" != "YES_DO_IT" ]; then
+    echo "You really need to enter \"YES_DO_IT\" in order to confirm."
+    exit 0
+else
+    images="$(for i in $(docker images | egrep -v "REPOSITORY" | awk '{printf "%s:%s\n", $1, $2}'); do printf "%s " ${i}; done)"
+    if [ ${#images} -eq 0 ]; then
+        echo "There are no images to delete."
+    else
+        echo -e "Will now delete the following Docker images:"
+        for image in ${images}; do
+            printf "\t%s\n" "${image}"
+        done
+        docker rmi ${images}
+    fi
+fi
+

bootstrap/sbin/rebuild_system_from_scratch.sh

+#! /usr/bin/env bash -e
+
+HOME_DIR=${LOFAR20_DIR:-${PWD}}
+if [ ! -d ${HOME_DIR}/bootstrap ]; then
+    # HOME_DIR/bootstrap needs to exist for this script to work.
+    echo -e "$(basename ${0}):\n\nERROR\n\tCannot run this script because the \"bootstrap\" cannot be found!\n\n"
+    exit 1
+fi
+
+# Save for the moment when tango will be cloned.
+OLD_HOME_DIR=${HOME_DIR}.old
+
+trap ' exit ${?} ' ABRT EXIT HUP INT TERM QUIT ERR
+
+
+function pull_images()
+{
+    (cd ${HOME_DIR}/docker-compose && make pull)
+}
+
+function build_lofar_images()
+{
+    (cd ${HOME_DIR}/docker-compose && make build)
+}
+
+function move_tango_dir_out_of_the_way()
+{
+    mv ${HOME_DIR} ${OLD_HOME_DIR}
+}
+
+function remove_images()
+{
+    ${HOME_DIR}/bootstrap/sbin/delete_all_docker_images.sh YES_DELETE_ALL
+}
+
+function pull_tango()
+{
+    git clone https://git.astron.nl/lofar2.0/tango.git ${HOME_DIR}
+    # Now remove the old tango dir.
+    rm -rf ${OLD_HOME_DIR}
+}
+
+function clean_images()
+{
+    # This can return a non-0 return code if the system
+    # has already been shut down.
+    # Therefore disable Bash's exit on error flag
+    set +e
+    (cd ${HOME_DIR}/docker-compose && make clean)
+    # And enable it again.
+    set -e
+}
+
+function start_minimal_tango()
+{
+    (cd ${HOME_DIR}/docker-compose
+    make minimal
+    # There is an issue with the dsconfig container that it every
+    # other time just exits without telling why.
+    # Therefore start dsconfig, then wait for 10s to allow it
+    # to die and just start it again.
+    # Don't they say that the second time is always a charm?
+    make start dsconfig
+    echo -e "\tWaiting for dsconfig to settle down..."
+    sleep 10
+    echo -e "\tDone.\n\tStarting dsconfig again."
+    make start dsconfig)
+}
+
+function configure_tango_db()
+{
+    # This somehow returns with a non-0 return code, make Bash happy.
+    ${HOME_DIR}/sbin/update_ConfigDb.sh ${HOME_DIR}/CDB/LOFAR_ConfigDb.json || true
+}
+
+function configure_elk()
+{
+    (cd ${HOME_DIR}/docker-compose && make start elk-configure-host)
+}
+
+function start_support_images()
+{
+    (cd ${HOME_DIR}/docker-compose && make start elk
+    make start jupyter)
+}
+
+function start_lofar_images()
+{
+    (cd ${HOME_DIR}/docker-compose
+    make start device-pcc
+    make start device-sdp)
+}
+
+
+# Clean out the Docker volumes.
+echo "-> Stopping Docker images and cleaning cleaning up their volumes..."
+clean_images
+echo -e "\tDone.\n"
+
+# Move the tango repo dir out of the way.
+echo "-> Moving the tango directory ${HOME_DIR} out of the way..."
+move_tango_dir_out_of_the_way
+echo -e "\tDone.\n"
+
+# Now pull the repo.
+echo "-> Cloning tango's master branch to ${HOME_DIR}..."
+pull_tango
+echo -e "\tDone.\n"
+
+# Now clean out the docker images.
+echo "-> Deleting all Docker images from this host..."
+remove_images
+echo -e "\tDone.\n"
+
+# Pull SKA's Docker images.
+echo "-> Installing latest SKA Docker images on this host..."
+pull_images
+echo -e "\tDone.\n"
+
+# Build all of the LOFAR Docker images.
+echo "-> Building LOFAR2.0 Docker images..."
+build_lofar_images
+echo -e "\tDone.\n"
+
+# Now start the basic Tango system, including dsconfig.
+echo "-> Staring up a minimal Tango session..."
+start_minimal_tango
+echo -e "\tDone.\n"
+
+# Load LOFAR's TangoDB.
+echo "-> Configuring the Tango DB for LOFAR2.0..."
+configure_tango_db
+echo -e "\tDone.\n"
+
+# Now configure the ELK container.
+echo "-> Configuring the ELK container for this host..."
+configure_elk
+echo -e "\tDone.\n"
+
+# Here I start ELK & Jupyter.
+echo "-> Start LOFAR2.0 support containers (ELK, Jupyter, etc.)..."
+start_support_images
+echo -e "\tDone.\n"
+
+# And finally start all the LOFAR images.
+echo "-> Start LOFAR2.0 containers..."
+start_lofar_images
+echo -e "\tDone.\n"
+
+# Now the system should be ready to use.
+# For instance the cold start script could now be executed.
+echo -e "-> LOFAR2.0 system rebuild was successful.\n\n"

docker-compose/Makefile

@@ -101,7 +101,6 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
     TANGO_HOST=$(TANGO_HOST) \
     NETWORK_MODE=$(NETWORK_MODE) \
     XAUTHORITY_MOUNT=$(XAUTHORITY_MOUNT) \
-    TANGO_SKA_CONTAINER_MOUNT=$(TANGO_SKA_CONTAINER_MOUNT) \
     TANGO_LOFAR_CONTAINER_MOUNT=$(TANGO_LOFAR_CONTAINER_MOUNT) \
     TANGO_LOFAR_CONTAINER_DIR=${TANGO_LOFAR_CONTAINER_DIR} MYSQL_HOST=$(MYSQL_HOST) \
     CONTAINER_NAME_PREFIX=$(CONTAINER_NAME_PREFIX) \

docker-compose/archiver.yml

@@ -63,7 +63,6 @@ services:
              json2tango -w -a -u /tango-archiver/data/archiver-devices.json &&
              sleep infinity"
     volumes:
-      - ${TANGO_SKA_CONTAINER_MOUNT}
       - ${TANGO_LOFAR_CONTAINER_MOUNT}
       - ${HOME}:/hosthome
       - ../docker/tango/tango-archiver:/tango-archiver

docker-compose/itango.yml

@@ -21,7 +21,6 @@ services:
     container_name: ${CONTAINER_NAME_PREFIX}itango
     network_mode: ${NETWORK_MODE}
     volumes:
-        - ${TANGO_SKA_CONTAINER_MOUNT}
         - ${TANGO_LOFAR_CONTAINER_MOUNT}
         - ${HOME}:/hosthome
     environment:

docker-compose/jive.yml

@@ -3,7 +3,7 @@
 # display.
 #
 # This container will always run on the same network as the host,
-# to make sure the DISPLAY variable can be used verbatim. For the 
+# to make sure the DISPLAY variable can be used verbatim. For the
 # same reason, TANGO_HOST is hardcoded to be at localhost:10000:
 # the docker network offering our tangodb also exposes it on port 10000
 # on the host.
@@ -23,7 +23,6 @@ services:
     network_mode: host
     volumes:
       - ${XAUTHORITY_MOUNT}
-      - ${TANGO_SKA_CONTAINER_MOUNT}
       - ${TANGO_LOFAR_CONTAINER_MOUNT}
       - ${HOME}:/hosthome
     environment:

docker-compose/jupyter.yml

@@ -18,7 +18,6 @@ services:
     container_name: ${CONTAINER_NAME_PREFIX}jupyter
     network_mode: ${NETWORK_MODE}
     volumes:
-        - ${TANGO_SKA_CONTAINER_MOUNT}
         - ${TANGO_LOFAR_CONTAINER_MOUNT}
         - ${TANGO_LOFAR_LOCAL_DIR}/jupyter-notebooks:/jupyter-notebooks:rw
         - ${HOME}:/hosthome

docker-compose/pogo.yml

@@ -26,7 +26,6 @@ services:
     volumes:
       - pogo:/home/tango
       - ${XAUTHORITY_MOUNT}
-      - ${TANGO_SKA_CONTAINER_MOUNT}
       - ${TANGO_LOFAR_CONTAINER_MOUNT}
       - ${HOME}:/hosthome:rw
     environment: