From d8e8aff8c799b94629fd0ee9914e207d52623e0d Mon Sep 17 00:00:00 2001
From: Thomas Juerges <4-jurges@users.noreply.git.astron.nl>
Date: Mon, 17 May 2021 21:08:06 +0200
Subject: [PATCH] L2SS-218 Move helper scripts to bootstrap dir
---
bootstrap/bin/updatePythonEnv.sh | 36 +++++++
bootstrap/etc/lofar20rc.sh | 66 +++++++++++++
bootstrap/etc/requirements.txt | 9 ++
.../sbin/checkout_shallow_copy_lofar_repos.sh | 43 +++++++++
bootstrap/sbin/delete_all_docker_images.sh | 31 +++++++
bootstrap/sbin/rebuild_system_from_scratch.sh | 93 +++++++++++++++++++
6 files changed, 278 insertions(+)
create mode 100755 bootstrap/bin/updatePythonEnv.sh
create mode 100755 bootstrap/etc/lofar20rc.sh
create mode 100644 bootstrap/etc/requirements.txt
create mode 100755 bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh
create mode 100755 bootstrap/sbin/delete_all_docker_images.sh
create mode 100755 bootstrap/sbin/rebuild_system_from_scratch.sh
diff --git a/bootstrap/bin/updatePythonEnv.sh b/bootstrap/bin/updatePythonEnv.sh
new file mode 100755
index 000000000..0f0d1becd
--- /dev/null
+++ b/bootstrap/bin/updatePythonEnv.sh
@@ -0,0 +1,36 @@
+#! /usr/bin/env bash
+
+venv=${VIRTUAL_ENV:?This is currently not a Python3 venv! Will not upgrade venv packages.}
+echo -e "\nFound a Python3 venv in \"${VIRTUAL_ENV}\".\nWill now proceed with package upgrades.\n"
+
+pip="python3 -m pip"
+upgrade_command="${pip} install --upgrade"
+
+if [ -z ${OS} ]; then
+ OS=$(uname)
+fi
+if [ ${OS} = Darwin ]; then
+ find=gfind
+else
+ find=find
+fi
+
+function upgrade()
+{
+ if [ ${#} -ge 1 ]; then
+ echo "Upgrading ${@}..."
+ ${upgrade_command} ${@}
+ if [ ${?} -eq 0 ]; then
+ echo -e "\nUpgrading ${@} done.\n\nNow \"deactivate\" and \"source ${venv}/bin/activate\".\n"
+ else
+ echo -e "\nSomething went wrong during the upgrade.\nCheck the output!\n"
+ fi
+ else
+ echo -e "\nNo package for upgrading given.\n"
+ fi
+}
+
+upgrade pip wheel
+
+installed_packages=$(for i in $(python3 -m pip freeze | cut -d'=' -f1); do printf "%s " ${i}; done)
+upgrade ${installed_packages}
diff --git a/bootstrap/etc/lofar20rc.sh b/bootstrap/etc/lofar20rc.sh
new file mode 100755
index 000000000..9cccdceb4
--- /dev/null
+++ b/bootstrap/etc/lofar20rc.sh
@@ -0,0 +1,66 @@
+# Set up the LOFAR2.0 environment.
+# For the time being it is assumend that the LOFAR2.0 environment has to
+# co-exist with a LOFAR1 environment.
+
+# Set these for the host where you run SKA's Tango Docker images.
+# And export those directories for LOFAR in Tango Docker images.
+
+# Pass a directory as first parameter to this script. This will
+# then be used as LOFAR20_DIR. Otherwise the current directory will
+# be used.
+export LOFAR20_DIR=${1:-${PWD}}
+echo "Setting LOFAR20_DIR=${LOFAR20_DIR}"
+
+# This needs to be modified for a development environment.
+# Example: ~/lofar2.0/
+# The current setting is for a production environment.
+export TANGO_SKA_LOCAL_DIR=${LOFAR20_DIR}
+
+export TANGO_SKA_CONTAINER_DIR=${LOFAR20_DIR}/ska-docker/
+export TANGO_SKA_CONTAINER_MOUNT=${TANGO_SKA_LOCAL_DIR}:${TANGO_SKA_CONTAINER_DIR}:ro
+
+
+# This needs to be modified for a development environment.
+# Example: ~/lofar2.0/tango
+# The current setting is for a production environment.
+export TANGO_LOFAR_LOCAL_DIR=${LOFAR20_DIR}/
+
+export TANGO_LOFAR_CONTAINER_DIR=${LOFAR20_DIR}/
+export TANGO_LOFAR_CONTAINER_MOUNT=${TANGO_LOFAR_LOCAL_DIR}:${TANGO_LOFAR_CONTAINER_DIR}:rw
+
+# This needs to be modified for a development environment.
+# In case you run multiple Docker networks on the same host in parallel, you need to specify a unique
+# network name for each of them.
+export NETWORK_MODE=lofar
+
+# It is assumed that the Tango host, the computer that runs the TangoDB, is this host.
+# If this is not true, then modify to the Tango host's FQDN and port.
+# Example: export TANGO_HOST=station-xk25.astron.nl:10000
+export TANGO_HOST=$(hostname):10000
+
+
+#
+# NO MODIFICATION BEYOND THIS POINT!
+#
+
+# Provide the -v parameters for Docker and the -e ENV variables.
+export TANGO_CONTAINER_ENV="-e TANGO_SKA_CONTAINER_DIR=${TANGO_SKA_CONTAINER_DIR} -e TANGO_LOFAR_CONTAINER_DIR=${TANGO_LOFAR_CONTAINER_DIR}"
+
+# Remove all LOFAR1 related environment modifications
+function remove_lofar()
+{
+ tmp=${1//:/ }
+ echo "$(for new in $(for i in ${tmp}; do printf "%s\n" ${i}; done | egrep -v '/opt/lofar/|/opt/WinCC|/opt/stationtest|/opt/operations'); do printf "%s:" ${new}; done)"
+}
+
+unset LOFARROOT
+export PATH=$(remove_lofar ${PATH})
+export LD_LIBRARY_PATH=$(remove_lofar ${LD_LIBRARY_PATH})
+export PYTHON_PATH=$(remove_lofar ${PYTHON_PATH})
+
+
+# Allow read access for everybody to allow Docker the forwarding of X11.
+chmod a+r ~/.Xauthority
+
+# Source the LOFAR2.0 Python3 venv if it exists.
+[ -z ${VIRTUAL_ENV} ] && [ -d ${LOFAR20_DIR}/lofar2.0_venv ] && source ${LOFAR20_DIR}/lofar2.0_venv/bin/activate
diff --git a/bootstrap/etc/requirements.txt b/bootstrap/etc/requirements.txt
new file mode 100644
index 000000000..5502737a6
--- /dev/null
+++ b/bootstrap/etc/requirements.txt
@@ -0,0 +1,9 @@
+astropy
+jupyter
+matplotlib
+numpy
+opcua-client
+pyqtgraph
+PyQt5
+asyncua
+dataclasses
diff --git a/bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh b/bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh
new file mode 100755
index 000000000..eec4919f4
--- /dev/null
+++ b/bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh
@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+
+# Clean out local dirs and then clone a shallow copy
+# of LOFAR2.0 repos.
+
+# 2020-10-13, thomas
+# Currently correct.
+astron_gitlab=https://git.astron.nl/
+repo_dir=lofar2.0/
+branch=master
+
+for repo in lmc-base-classes ska-logging ska-docker opcua tango pypcc; do
+ cd ${repo}
+ # This will return 2 if the repo is clean.
+ clean="$(git status -u | wc -l)"
+ cd -
+ if [ "${clean}" = "2" ]; then
+ # The repo is clean, no new files, no modifications.
+ rm -rf ${repo}
+ git clone --depth 1 --branch ${branch} ${astron_gitlab}${repo_dir}${repo}.git
+ else
+ echo -e "*********\n\tThe repository ${repo} contains modifications.\n\tRestore the original content first before continuing.\nAborting now.\n"
+ exit -1
+ fi
+done
+
+# 2020-10-13, thomas
+# I need to move this repo to lofar2.0.
+repo=crossecho
+repo_dir=jurges
+rm -rf crossecho && git clone --depth 1 --branch add_simulation_mode_plus_patches ${astron_gitlab}${repo_dir}/${repo}.git
+
+if [ ! -s ska_logging -o $(readlink ska_logging) != ska-logging ]; then
+ echo "Creating symlink ska-logging -> ska_logging"
+ rm -f ska_logging
+ ln -s ska-logging ska_logging
+fi
+
+if [ ! -s skabase -o $(readlink skabase) != lmc-base-classes ]; then
+ echo "Creating symlink lmc-base-classes -> skabase"
+ rm -f skabase
+ ln -s lmc-base-classes skabase
+fi
diff --git a/bootstrap/sbin/delete_all_docker_images.sh b/bootstrap/sbin/delete_all_docker_images.sh
new file mode 100755
index 000000000..7cccb90c3
--- /dev/null
+++ b/bootstrap/sbin/delete_all_docker_images.sh
@@ -0,0 +1,31 @@
+#! /usr/bin/env bash
+
+function help()
+{
+ echo "You need to add a parameter \"YES_DELETE_ALL\" in order to really remove all Docker images."
+ exit 0
+}
+
+if [ ${#} -ne 1 ]; then
+ help
+elif [ "${1}" != "YES_DELETE_ALL" ]; then
+ help
+fi
+
+read -p "If you certain that you want to delete all Docker images, then enter now \"YES_DO_IT\" " reply
+if [ "${reply}" != "YES_DO_IT" ]; then
+ echo "You really need to enter \"YES_DO_IT\" in order to confirm."
+ exit 0
+else
+ images="$(for i in $(docker images | egrep -v "REPOSITORY" | awk '{printf "%s:%s\n", $1, $2}'); do printf "%s " ${i}; done)"
+ if [ ${#images} -eq 0 ]; then
+ echo "There are no images to delete."
+ else
+ echo -e "Will now delete the following Docker images:"
+ for image in ${images}; do
+ printf "\t%s\n" "${image}"
+ done
+ docker rmi ${images}
+ fi
+fi
+
diff --git a/bootstrap/sbin/rebuild_system_from_scratch.sh b/bootstrap/sbin/rebuild_system_from_scratch.sh
new file mode 100755
index 000000000..c315a26f9
--- /dev/null
+++ b/bootstrap/sbin/rebuild_system_from_scratch.sh
@@ -0,0 +1,93 @@
+#! /usr/bin/env bash -e
+
+HOME_DIR=/opt/lofar2.0
+
+trap ' popd; exit ${?} ' ABRT EXIT HUP INT TERM QUIT ERR
+
+function pull_images()
+{
+ pushd ${HOME_DIR}/tango/docker-compose && make pull
+ popd
+}
+
+function build_lofar_images()
+{
+ pushd ${HOME_DIR}/tango/docker-compose && make build
+ popd
+}
+
+function remove_tango_dir()
+{
+ pushd ${HOME_DIR} && rm -rf tango
+ popd
+}
+
+function remove_images()
+{
+ ${HOME_DIR}/delete_all_docker_images.sh YES_DO_IT
+}
+
+function pull_tango()
+{
+ git clone https://git.astron.nl/lofar2.0/tango.git ${HOME_DIR}/tango
+}
+
+function stop_images()
+{
+ pushd ${HOME_DIR}/tango/docker-compose && make stop
+ popd
+}
+
+function clean_images()
+{
+ pushd ${HOME_DIR}/tango/docker-compose && make clean
+ popd
+}
+
+function start_tango()
+{
+ pushd ${HOME_DIR}/tango/docker-compose
+ make start itango
+ make start dsconfig
+ popd
+}
+
+function configure_tango_db()
+{
+ ${HOME_DIR}/tango/sbin/update_ConfigDb.sh ${HOME_DIR}/tango/CDB/LOFAR_ConfigDb.json
+}
+
+function configure_elk()
+{
+ pushd ${HOME_DIR}/tango/docker-compose && make start elk-configure-host
+ popd
+}
+
+function start_support_images()
+{
+ pushd ${HOME_DIR}/tango/docker-compose && make start elk
+ make start jupyter
+ popd
+}
+
+function start_lofar_images()
+{
+ pushd ${HOME_DIR}/tango/docker-compose
+ make start device-pcc
+ make start device-sdp
+ popd
+}
+
+stop_images
+clean_images
+remove_tango_dir
+remove_images
+pull_tango
+pull_images
+build_lofar_images
+start_tango
+configure_tango_db
+configure_elk
+start_support_images
+start_lofar_images
+
--
GitLab