diff --git a/bootstrap/bin/updatePythonEnv.sh b/bootstrap/bin/updatePythonEnv.sh new file mode 100755 index 0000000000000000000000000000000000000000..0f0d1becdcc2db08a6a7c3f93682df63124a9c67 --- /dev/null +++ b/bootstrap/bin/updatePythonEnv.sh @@ -0,0 +1,36 @@ +#! /usr/bin/env bash + +venv=${VIRTUAL_ENV:?This is currently not a Python3 venv! Will not upgrade venv packages.} +echo -e "\nFound a Python3 venv in \"${VIRTUAL_ENV}\".\nWill now proceed with package upgrades.\n" + +pip="python3 -m pip" +upgrade_command="${pip} install --upgrade" + +if [ -z ${OS} ]; then + OS=$(uname) +fi +if [ ${OS} = Darwin ]; then + find=gfind +else + find=find +fi + +function upgrade() +{ + if [ ${#} -ge 1 ]; then + echo "Upgrading ${@}..." + ${upgrade_command} ${@} + if [ ${?} -eq 0 ]; then + echo -e "\nUpgrading ${@} done.\n\nNow \"deactivate\" and \"source ${venv}/bin/activate\".\n" + else + echo -e "\nSomething went wrong during the upgrade.\nCheck the output!\n" + fi + else + echo -e "\nNo package for upgrading given.\n" + fi +} + +upgrade pip wheel + +installed_packages=$(for i in $(python3 -m pip freeze | cut -d'=' -f1); do printf "%s " ${i}; done) +upgrade ${installed_packages} diff --git a/bootstrap/etc/lofar20rc.sh b/bootstrap/etc/lofar20rc.sh new file mode 100755 index 0000000000000000000000000000000000000000..e3c75df48457342a6ba9d8e061a322e324ee17c4 --- /dev/null +++ b/bootstrap/etc/lofar20rc.sh @@ -0,0 +1,56 @@ +# Set up the LOFAR2.0 environment. +# For the time being it is assumend that the LOFAR2.0 environment has to +# co-exist with a LOFAR1 environment. + +# Set these for the host where you run SKA's Tango Docker images. +# And export those directories for LOFAR in Tango Docker images. + +# Pass a directory as first parameter to this script. This will +# then be used as LOFAR20_DIR. Otherwise the current directory will +# be used. +export LOFAR20_DIR=${1:-${PWD}} + +# This needs to be modified for a development environment. +# Example: ~/lofar2.0/tango +# The current setting is for a production environment. +export TANGO_LOFAR_LOCAL_DIR=${LOFAR20_DIR}/ + +export TANGO_LOFAR_CONTAINER_DIR=${LOFAR20_DIR}/ +export TANGO_LOFAR_CONTAINER_MOUNT=${TANGO_LOFAR_LOCAL_DIR}:${TANGO_LOFAR_CONTAINER_DIR}:rw + +# This needs to be modified for a development environment. +# In case you run multiple Docker networks on the same host in parallel, you need to specify a unique +# network name for each of them. +export NETWORK_MODE=lofar + +# It is assumed that the Tango host, the computer that runs the TangoDB, is this host. +# If this is not true, then modify to the Tango host's FQDN and port. +# Example: export TANGO_HOST=station-xk25.astron.nl:10000 +export TANGO_HOST=$(hostname):10000 + + +# +# NO MODIFICATION BEYOND THIS POINT! +# + +# Provide the -v parameters for Docker and the -e ENV variables. +export TANGO_CONTAINER_ENV="-e TANGO_LOFAR_CONTAINER_DIR=${TANGO_LOFAR_CONTAINER_DIR}" + +# Remove all LOFAR1 related environment modifications +function remove_lofar() +{ + tmp=${1//:/ } + echo "$(for new in $(for i in ${tmp}; do printf "%s\n" ${i}; done | egrep -v '/opt/lofar/|/opt/WinCC|/opt/stationtest|/opt/operations'); do printf "%s:" ${new}; done)" +} + +unset LOFARROOT +export PATH=$(remove_lofar ${PATH}) +export LD_LIBRARY_PATH=$(remove_lofar ${LD_LIBRARY_PATH}) +export PYTHON_PATH=$(remove_lofar ${PYTHON_PATH}) + + +# Allow read access for everybody to allow Docker the forwarding of X11. +chmod a+r ~/.Xauthority + +# Source the LOFAR2.0 Python3 venv if it exists. +[ -z ${VIRTUAL_ENV} ] && [ -d ${LOFAR20_DIR}/lofar2.0_venv ] && source ${LOFAR20_DIR}/lofar2.0_venv/bin/activate diff --git a/bootstrap/etc/requirements.txt b/bootstrap/etc/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5502737a6308c9939be7a2fa4981707f965918ac --- /dev/null +++ b/bootstrap/etc/requirements.txt @@ -0,0 +1,9 @@ +astropy +jupyter +matplotlib +numpy +opcua-client +pyqtgraph +PyQt5 +asyncua +dataclasses diff --git a/bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh b/bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh new file mode 100755 index 0000000000000000000000000000000000000000..eec4919f44e0b2c36faae7ebc796b4ce41c12315 --- /dev/null +++ b/bootstrap/sbin/checkout_shallow_copy_lofar_repos.sh @@ -0,0 +1,43 @@ +#! /usr/bin/env bash + +# Clean out local dirs and then clone a shallow copy +# of LOFAR2.0 repos. + +# 2020-10-13, thomas +# Currently correct. +astron_gitlab=https://git.astron.nl/ +repo_dir=lofar2.0/ +branch=master + +for repo in lmc-base-classes ska-logging ska-docker opcua tango pypcc; do + cd ${repo} + # This will return 2 if the repo is clean. + clean="$(git status -u | wc -l)" + cd - + if [ "${clean}" = "2" ]; then + # The repo is clean, no new files, no modifications. + rm -rf ${repo} + git clone --depth 1 --branch ${branch} ${astron_gitlab}${repo_dir}${repo}.git + else + echo -e "*********\n\tThe repository ${repo} contains modifications.\n\tRestore the original content first before continuing.\nAborting now.\n" + exit -1 + fi +done + +# 2020-10-13, thomas +# I need to move this repo to lofar2.0. +repo=crossecho +repo_dir=jurges +rm -rf crossecho && git clone --depth 1 --branch add_simulation_mode_plus_patches ${astron_gitlab}${repo_dir}/${repo}.git + +if [ ! -s ska_logging -o $(readlink ska_logging) != ska-logging ]; then + echo "Creating symlink ska-logging -> ska_logging" + rm -f ska_logging + ln -s ska-logging ska_logging +fi + +if [ ! -s skabase -o $(readlink skabase) != lmc-base-classes ]; then + echo "Creating symlink lmc-base-classes -> skabase" + rm -f skabase + ln -s lmc-base-classes skabase +fi diff --git a/bootstrap/sbin/delete_all_docker_images.sh b/bootstrap/sbin/delete_all_docker_images.sh new file mode 100755 index 0000000000000000000000000000000000000000..7cccb90c3ede668f9e97eeb9956eac82176ac9a9 --- /dev/null +++ b/bootstrap/sbin/delete_all_docker_images.sh @@ -0,0 +1,31 @@ +#! /usr/bin/env bash + +function help() +{ + echo "You need to add a parameter \"YES_DELETE_ALL\" in order to really remove all Docker images." + exit 0 +} + +if [ ${#} -ne 1 ]; then + help +elif [ "${1}" != "YES_DELETE_ALL" ]; then + help +fi + +read -p "If you certain that you want to delete all Docker images, then enter now \"YES_DO_IT\" " reply +if [ "${reply}" != "YES_DO_IT" ]; then + echo "You really need to enter \"YES_DO_IT\" in order to confirm." + exit 0 +else + images="$(for i in $(docker images | egrep -v "REPOSITORY" | awk '{printf "%s:%s\n", $1, $2}'); do printf "%s " ${i}; done)" + if [ ${#images} -eq 0 ]; then + echo "There are no images to delete." + else + echo -e "Will now delete the following Docker images:" + for image in ${images}; do + printf "\t%s\n" "${image}" + done + docker rmi ${images} + fi +fi + diff --git a/bootstrap/sbin/rebuild_system_from_scratch.sh b/bootstrap/sbin/rebuild_system_from_scratch.sh new file mode 100755 index 0000000000000000000000000000000000000000..8840c5a11d1863abe2e94cdda6268b9801d14d10 --- /dev/null +++ b/bootstrap/sbin/rebuild_system_from_scratch.sh @@ -0,0 +1,152 @@ +#! /usr/bin/env bash -e + +HOME_DIR=${LOFAR20_DIR:-${PWD}} +if [ ! -d ${HOME_DIR}/bootstrap ]; then + # HOME_DIR/bootstrap needs to exist for this script to work. + echo -e "$(basename ${0}):\n\nERROR\n\tCannot run this script because the \"bootstrap\" cannot be found!\n\n" + exit 1 +fi + +# Save for the moment when tango will be cloned. +OLD_HOME_DIR=${HOME_DIR}.old + +trap ' exit ${?} ' ABRT EXIT HUP INT TERM QUIT ERR + + +function pull_images() +{ + (cd ${HOME_DIR}/docker-compose && make pull) +} + +function build_lofar_images() +{ + (cd ${HOME_DIR}/docker-compose && make build) +} + +function move_tango_dir_out_of_the_way() +{ + mv ${HOME_DIR} ${OLD_HOME_DIR} +} + +function remove_images() +{ + ${HOME_DIR}/bootstrap/sbin/delete_all_docker_images.sh YES_DELETE_ALL +} + +function pull_tango() +{ + git clone https://git.astron.nl/lofar2.0/tango.git ${HOME_DIR} + # Now remove the old tango dir. + rm -rf ${OLD_HOME_DIR} +} + +function clean_images() +{ + # This can return a non-0 return code if the system + # has already been shut down. + # Therefore disable Bash's exit on error flag + set +e + (cd ${HOME_DIR}/docker-compose && make clean) + # And enable it again. + set -e +} + +function start_minimal_tango() +{ + (cd ${HOME_DIR}/docker-compose + make minimal + # There is an issue with the dsconfig container that it every + # other time just exits without telling why. + # Therefore start dsconfig, then wait for 10s to allow it + # to die and just start it again. + # Don't they say that the second time is always a charm? + make start dsconfig + echo -e "\tWaiting for dsconfig to settle down..." + sleep 10 + echo -e "\tDone.\n\tStarting dsconfig again." + make start dsconfig) +} + +function configure_tango_db() +{ + # This somehow returns with a non-0 return code, make Bash happy. + ${HOME_DIR}/sbin/update_ConfigDb.sh ${HOME_DIR}/CDB/LOFAR_ConfigDb.json || true +} + +function configure_elk() +{ + (cd ${HOME_DIR}/docker-compose && make start elk-configure-host) +} + +function start_support_images() +{ + (cd ${HOME_DIR}/docker-compose && make start elk + make start jupyter) +} + +function start_lofar_images() +{ + (cd ${HOME_DIR}/docker-compose + make start device-pcc + make start device-sdp) +} + + +# Clean out the Docker volumes. +echo "-> Stopping Docker images and cleaning cleaning up their volumes..." +clean_images +echo -e "\tDone.\n" + +# Move the tango repo dir out of the way. +echo "-> Moving the tango directory ${HOME_DIR} out of the way..." +move_tango_dir_out_of_the_way +echo -e "\tDone.\n" + +# Now pull the repo. +echo "-> Cloning tango's master branch to ${HOME_DIR}..." +pull_tango +echo -e "\tDone.\n" + +# Now clean out the docker images. +echo "-> Deleting all Docker images from this host..." +remove_images +echo -e "\tDone.\n" + +# Pull SKA's Docker images. +echo "-> Installing latest SKA Docker images on this host..." +pull_images +echo -e "\tDone.\n" + +# Build all of the LOFAR Docker images. +echo "-> Building LOFAR2.0 Docker images..." +build_lofar_images +echo -e "\tDone.\n" + +# Now start the basic Tango system, including dsconfig. +echo "-> Staring up a minimal Tango session..." +start_minimal_tango +echo -e "\tDone.\n" + +# Load LOFAR's TangoDB. +echo "-> Configuring the Tango DB for LOFAR2.0..." +configure_tango_db +echo -e "\tDone.\n" + +# Now configure the ELK container. +echo "-> Configuring the ELK container for this host..." +configure_elk +echo -e "\tDone.\n" + +# Here I start ELK & Jupyter. +echo "-> Start LOFAR2.0 support containers (ELK, Jupyter, etc.)..." +start_support_images +echo -e "\tDone.\n" + +# And finally start all the LOFAR images. +echo "-> Start LOFAR2.0 containers..." +start_lofar_images +echo -e "\tDone.\n" + +# Now the system should be ready to use. +# For instance the cold start script could now be executed. +echo -e "-> LOFAR2.0 system rebuild was successful.\n\n" diff --git a/docker-compose/Makefile b/docker-compose/Makefile index f1b2cc5788c9f75cead9c8fdfaa1793e86f2cb81..1b7983ed19bd5ada525d91d30f45fb1d45fea012 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -101,7 +101,6 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \ TANGO_HOST=$(TANGO_HOST) \ NETWORK_MODE=$(NETWORK_MODE) \ XAUTHORITY_MOUNT=$(XAUTHORITY_MOUNT) \ - TANGO_SKA_CONTAINER_MOUNT=$(TANGO_SKA_CONTAINER_MOUNT) \ TANGO_LOFAR_CONTAINER_MOUNT=$(TANGO_LOFAR_CONTAINER_MOUNT) \ TANGO_LOFAR_CONTAINER_DIR=${TANGO_LOFAR_CONTAINER_DIR} MYSQL_HOST=$(MYSQL_HOST) \ CONTAINER_NAME_PREFIX=$(CONTAINER_NAME_PREFIX) \ diff --git a/docker-compose/archiver.yml b/docker-compose/archiver.yml index f471de6285a23e9d4969b4d840e9b83accaac22e..98200f610c887a6a40cdc27c8e87dfbdac8b22a7 100644 --- a/docker-compose/archiver.yml +++ b/docker-compose/archiver.yml @@ -63,7 +63,6 @@ services: json2tango -w -a -u /tango-archiver/data/archiver-devices.json && sleep infinity" volumes: - - ${TANGO_SKA_CONTAINER_MOUNT} - ${TANGO_LOFAR_CONTAINER_MOUNT} - ${HOME}:/hosthome - ../docker/tango/tango-archiver:/tango-archiver diff --git a/docker-compose/itango.yml b/docker-compose/itango.yml index d131d405371bc6ff7e59892587b2bd2aba9d1fd6..4c12fe3a00fe8ee5fda3a26668bc24adcbf25a72 100644 --- a/docker-compose/itango.yml +++ b/docker-compose/itango.yml @@ -21,7 +21,6 @@ services: container_name: ${CONTAINER_NAME_PREFIX}itango network_mode: ${NETWORK_MODE} volumes: - - ${TANGO_SKA_CONTAINER_MOUNT} - ${TANGO_LOFAR_CONTAINER_MOUNT} - ${HOME}:/hosthome environment: diff --git a/docker-compose/jive.yml b/docker-compose/jive.yml index 5148d1d15c5ef74501e21f9777839e3c4cd724bf..d3b8b06f9778e3e5f53953a5f2eb26725d657e60 100644 --- a/docker-compose/jive.yml +++ b/docker-compose/jive.yml @@ -3,7 +3,7 @@ # display. # # This container will always run on the same network as the host, -# to make sure the DISPLAY variable can be used verbatim. For the +# to make sure the DISPLAY variable can be used verbatim. For the # same reason, TANGO_HOST is hardcoded to be at localhost:10000: # the docker network offering our tangodb also exposes it on port 10000 # on the host. @@ -23,7 +23,6 @@ services: network_mode: host volumes: - ${XAUTHORITY_MOUNT} - - ${TANGO_SKA_CONTAINER_MOUNT} - ${TANGO_LOFAR_CONTAINER_MOUNT} - ${HOME}:/hosthome environment: diff --git a/docker-compose/jupyter.yml b/docker-compose/jupyter.yml index dbabfadba86064c724d9073f3077132716da5438..0ac641b22f78a6a1d332e572dc3d9b6db470a5bf 100644 --- a/docker-compose/jupyter.yml +++ b/docker-compose/jupyter.yml @@ -18,7 +18,6 @@ services: container_name: ${CONTAINER_NAME_PREFIX}jupyter network_mode: ${NETWORK_MODE} volumes: - - ${TANGO_SKA_CONTAINER_MOUNT} - ${TANGO_LOFAR_CONTAINER_MOUNT} - ${TANGO_LOFAR_LOCAL_DIR}/jupyter-notebooks:/jupyter-notebooks:rw - ${HOME}:/hosthome diff --git a/docker-compose/pogo.yml b/docker-compose/pogo.yml index c9770ca7b355ea57a96e06b7d76a2d995c3e41a6..9e2d377c1c40f3276f979af621d986a04a04a6b7 100644 --- a/docker-compose/pogo.yml +++ b/docker-compose/pogo.yml @@ -26,7 +26,6 @@ services: volumes: - pogo:/home/tango - ${XAUTHORITY_MOUNT} - - ${TANGO_SKA_CONTAINER_MOUNT} - ${TANGO_LOFAR_CONTAINER_MOUNT} - ${HOME}:/hosthome:rw environment: