L2SS-723: Wait up to 5 minutes for databaseds

bin/wait-for-it.sh

+#!/usr/bin/env bash
+# shellcheck disable=SC2086,SC2064,SC2206,SC2124
+#   Use this script to test if a given TCP host/port are available
+#
+# Source: https://github.com/vishnubob/wait-for-it
+# License: MIT
+
+cmdname=$(basename $0)
+
+echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $TIMEOUT -gt 0 ]]; then
+        echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
+    else
+        echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
+    fi
+    start_ts=$(date +%s)
+    while :
+    do
+        (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
+        result=$?
+        if [[ $result -eq 0 ]]; then
+            end_ts=$(date +%s)
+            echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $QUIET -eq 1 ]]; then
+        timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    else
+        timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    fi
+    PID=$!
+    trap "kill -INT -$PID" INT
+    wait $PID
+    RESULT=$?
+    if [[ $RESULT -ne 0 ]]; then
+        echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
+    fi
+    return $RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        hostport=(${1//:/ })
+        HOST=${hostport[0]}
+        PORT=${hostport[1]}
+        shift 1
+        ;;
+        --child)
+        CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        STRICT=1
+        shift 1
+        ;;
+        -h)
+        HOST="$2"
+        if [[ $HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        PORT="$2"
+        if [[ $PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        TIMEOUT="$2"
+        if [[ $TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        CLI="$@"
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$HOST" == "" || "$PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+TIMEOUT=${TIMEOUT:-15}
+STRICT=${STRICT:-0}
+CHILD=${CHILD:-0}
+QUIET=${QUIET:-0}
+
+if [[ $CHILD -gt 0 ]]; then
+    wait_for
+    RESULT=$?
+    exit $RESULT
+else
+    if [[ $TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        RESULT=$?
+    else
+        wait_for
+        RESULT=$?
+    fi
+fi
+
+if [[ $CLI != "" ]]; then
+    if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
+        echoerr "$cmdname: strict mode, refusing to execute subprocess"
+        exit $RESULT
+    fi
+    exec $CLI
+else
+    exit $RESULT
+fi

sbin/run_integration_test.sh

@@ -11,20 +11,26 @@ fi
 
 cd "$LOFAR20_DIR/docker-compose" || exit 1
 
+# Start the database server first
+make build databaseds dsconfig
+make start databaseds dsconfig
+
+# Give dsconfig and databaseds time to start
+sleep 1 # dsconfig container must be up and running...
+# shellcheck disable=SC2016
+echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true' | make run dsconfig bash -
+
 # Build only the required images, please do not build everything that makes CI
 # take really long to finish, especially grafana / jupyter / prometheus.
 # jupyter is physically large > 2.5gb and overlayfs is really slow.
 make build device-sdp device-recv device-sst device-unb2 device-xst device-beamlet device-digitalbeam device-tilebeam 
 make build sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim
-make build databaseds dsconfig elk integration-test
+make build elk integration-test
 make build archiver-timescale hdbppts-cm hdbppts-es
 
 # Start and stop sequence
 make stop device-boot device-docker device-apsct device-apspu device-sdp device-recv device-sst device-unb2 device-xst device-beamlet device-digitalbeam device-tilebeam sdptr-sim recv-sim unb2-sim apsct-sim apspu-sim hdbppts-es hdbppts-cm archiver-timescale
-make start databaseds dsconfig elk
-
-# Give dsconfig and databaseds time to start
-sleep 60
+make start elk
 
 # Update the dsconfig
 # Do not remove `bash`, otherwise statement ignored by gitlab ci shell!