fixes

sbin/dsconfig.sh

@@ -38,6 +38,10 @@ while true; do
   shift
 done
 
+if [ -z "$TAG" ]; then
+  TAG="latest"
+fi
+
 docker_args=(run --rm  -e "TANGO_HOST=$TANGO_HOST" --network=station -i)
 docker_image="git.astron.nl:5000/lofar2.0/tango/dsconfig:$TAG"
 

sbin/run_integration_test.sh

@@ -79,7 +79,7 @@ fi
 export TANGO_SKIP_BUILD=1
 
 if [ -z "$TAG" ]; then
-  TAG="latest"
+  export TAG="latest"
 fi
 
 # Build dsconfig first, `-z ${y+x}` is the inverse of `-n ${y}`

sbin/run_integration_test.sh.orig

+#!/bin/bash -e
+#
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# Usage function explains how parameters are parsed
+function usage {
+    echo "./$(basename "$0")
+      no arguments, builds and configures all docker containers and starts each
+      stage of the integration test one after the other. Between each stage the
+      dsconfig is updated accordingly."
+    echo ""
+    echo "./$(basename "$0") -h --help
+      displays this help message"
+    echo ""
+    echo "./$(basename "$0") --no-build
+      disables building of docker images"
+}
+
+# Configure the config database, restart containers and run a specific
+# integration module or even specific tests
+# integration_test module restarted_containers config_files specific_test
+function integration_test {
+  IFS=" " read -r -a restarts <<< "${2}"
+  IFS=" " read -r -a configs <<< "${3}"
+  for config in "${configs[@]}"; do
+    echo "Updating config ${config} ..."
+    bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${config}"
+  done
+  if [ -n "${2+x}" ]; then
+    # shellcheck disable=SC2145
+    echo "make restart ${restarts[@]} ..."
+    make restart "${restarts[@]}"
+    # shellcheck disable=SC2145
+    echo "make await ${restarts[@]} ..."
+    make await "${restarts[@]}"
+  fi
+
+  echo "make integration ${1} ..."
+  make integration "${1}"
+}
+
+# list of arguments expected in the input
+optstring_long="help,no-build,preserve,save-logs"
+optstring="hnb"
+
+options=$(getopt -l ${optstring_long} -o ${optstring} -- "$@")
+
+eval set -- "$options"
+
+while true; do
+  case ${1} in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    --no-build)
+      echo "Disable docker compose build step"
+      export no_build=1
+      export NO_BASE=${no_build}
+      ;;
+    --preserve)
+      echo "Preserve test environment"
+      export preserve=1
+      ;;
+    --save-logs)
+      echo "Save logs after execution"
+      export save_logs=1
+      ;;
+    --)
+    shift
+    break;;
+  esac
+  shift
+done
+
+if [ -z "$LOFAR20_DIR" ]; then
+    # We assume we aren't in the PATH, so we can derive our path.
+    # We need our parent directory.
+    LOFAR20_DIR_RELATIVE=$(dirname "$0")/..
+
+    # As an absolute path
+    LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}")
+fi
+
+<<<<<<< HEAD
+if [ -z "$TAG" ]; then
+  TAG="latest"
+fi
+
+[ -n "${no_build}" ] || make build dsconfig
+
+docker network rm station || true
+
+if ! [ -x "$(command -v jumppad)" ]; then
+  echo 'Install jumppad'
+  mkdir -p ./.bin
+  wget https://git.astron.nl/lofar2.0/tango/-/package_files/37347/download -O ./.bin/jumppad
+  chmod +x ./.bin/jumppad
+  bin_dir=$(realpath ./.bin)
+  PATH="$PATH:$bin_dir"
+fi
+
+make -C infra/jobs/station DIR_OUT="$( realpath "infra/dev/jobs/station")" render
+
+# prepare a docker volume for nomad
+tmp_volume="test_$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)"
+
+function cleanup {
+  cd "$LOFAR20_DIR"
+  if [ -n "${save_logs}" ]; then
+    mkdir -p log
+    for container in $(docker ps -a --format "{{.Names}}")
+    do
+      echo "Saving log for container $container"
+      docker logs "${container}" >& "log/${container}.log"
+    done
+    bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --dump >& log/dump_ConfigDb.log
+  fi
+  if [ -z "${preserve}" ]; then
+    make stop > /dev/null 2>&1
+    HOME="$JUMPPAD_HOME" jumppad down infra/dev
+    docker volume rm "$tmp_volume" || true
+  fi
+}
+
+trap cleanup EXIT
+
+docker volume create "$tmp_volume"
+
+docker pull -q bash
+docker run -i --rm -v "$tmp_volume":/mnt bash bash -c 'mkdir -p /mnt/volumes/tango-database'
+
+rm -rf ~/.jumppad/
+
+if [ -z "$JUMPPAD_HOME" ]; then
+  JUMPPAD_HOME="$HOME"
+fi
+
+HOME="$JUMPPAD_HOME" jumppad up --var="host_volume=$tmp_volume" infra/dev || true
+
+echo -n "Waiting for tango service to become available .."
+until [[ $(dig @127.0.0.1 -p 8600 tango.service.consul +short) ]]; do
+  sleep 2
+  echo -n "."
+done
+echo ". [ok]"
+
+TANGO_PORT=$(dig @127.0.0.1 -p 8600 tango.service.consul SRV +short  | awk '{printf "%s",$3}')
+TANGO_HOST=$(dig @127.0.0.1 -p 8600 tango.service.consul +short)
+
+export TANGO_HOST="$TANGO_HOST:$TANGO_PORT"
+
+export HOSTNAME=192.168.123.1
+export DNS=192.168.123.100
+
+echo "Using tango host $TANGO_HOST"
+
+cd "$LOFAR20_DIR/docker-compose" || exit 1
+
+=======
+export TANGO_SKIP_BUILD=1
+
+cd "$LOFAR20_DIR/docker-compose" || exit 1
+
+# Start the database server first, `-z ${y+x}` is the inverse of `-n ${y}`
+if [ -z "${no_build+x}" ]; then
+    rm -rf "${LOFAR20_DIR}/tangostationcontrol/dist"
+    cd "${LOFAR20_DIR}/tangostationcontrol" || exit 1
+    tox -e build
+    cd "$LOFAR20_DIR/docker-compose" || exit 1
+    make build databaseds dsconfig
+fi
+
+make start databaseds dsconfig
+
+# Give dsconfig and databaseds time to start
+sleep 5 # dsconfig container must be up and running...
+# shellcheck disable=SC2016
+echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true' | make run dsconfig bash -
+
+>>>>>>> master
+# Devices list is used to explitly word split when supplied to commands, must
+# disable shellcheck SC2086 for each case.
+DEVICES=(device-station-manager device-boot device-aps device-apsct device-ccd device-ec device-apspu device-sdpfirmware device-sdp device-recvh device-recvl device-bst device-sst device-unb2 device-xst device-beamlet device-digitalbeam device-tilebeam device-psoc device-pcon device-antennafield device-temperature-manager device-observation-control device-configuration device-calibration)
+
+SIMULATORS=(sdptr-sim recvh-sim recvl-sim unb2-sim apsct-sim apspu-sim ccd-sim ec-sim)
+
+# Build only the required images, please do not build everything that makes CI
+# take really long to finish, especially grafana / jupyter / prometheus.
+# jupyter is physically large > 2.5gb and overlayfs is really slow.
+
+[ -n "${no_build}" ] || make build "${SIMULATORS[@]}"
+[ -n "${no_build}" ] || make build logstash integration-test http-json-schemas
+
+# Start and stop sequence
+make stop http-json-schemas
+make stop object-storage init-object-storage
+make stop "${DEVICES[@]}" "${SIMULATORS[@]}"
+make stop device-docker # this one does not test well in docker-in-docker
+make stop logstash
+
+make start logstash http-json-schemas object-storage init-object-storage
+
+# Update the dsconfig
+# Do not remove `bash`, otherwise statement ignored by gitlab ci shell!
+docker pull -q "git.astron.nl:5000/lofar2.0/tango/dsconfig:$TAG"
+
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --load "${LOFAR20_DIR}"/CDB/stations/common.json
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/l0.json
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/l1.json
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/lba.json
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/h0.json
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/hba_core.json
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/cs001.json
+bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/testenv_cs001.json
+
+cd "$LOFAR20_DIR/docker-compose" || exit 1
+make start "${SIMULATORS[@]}"
+
+# Give the simulators time to start
+sleep 5
+
+# shellcheck disable=SC2086
+make start "${DEVICES[@]}"
+
+# Wait for devices to restart
+make await "${DEVICES[@]}"
+
+# Start the integration test
+cd "$LOFAR20_DIR/docker-compose" || exit 1
+make up integration-test
+
+integration_test default
+
+integration_test tilebeam_performance "device-sdpfirmware device-sdp device-recvh device-recvl device-tilebeam device-antennafield" "${LOFAR20_DIR}/CDB/integrations/tilebeam_cluster_ConfigDb.json"
+
+integration_test digitalbeam_performance "device-sdpfirmware device-sdp device-recvh device-recvl device-digitalbeam device-beamlet device-antennafield" "${LOFAR20_DIR}/CDB/integrations/digitalbeam_cluster_ConfigDb.json"
+
+integration_test configuration "device-configuration"
+
+make restart "${DEVICES[@]}"