diff --git a/docker-compose/lofar-device-base/Dockerfile b/docker-compose/lofar-device-base/Dockerfile index d0cf589c3e2124079b969bbc173635d7c235ccc3..14dcd0804a33961a5048a8688b881b87e88309f9 100644 --- a/docker-compose/lofar-device-base/Dockerfile +++ b/docker-compose/lofar-device-base/Dockerfile @@ -20,15 +20,20 @@ RUN --mount=type=cache,target=/var/cache/apt \ sudo apt-get install -y netcat COPY tmp/requirements.txt /tangostationcontrol-requirements.txt -COPY tmp/*.whl / RUN echo "TANGO_STATION_CONTROL: ${TANGO_STATION_CONTROL}" +RUN if [ -z $TANGO_STATION_CONTROL ]; then \ + echo "Installing requirements only"; \ + sudo pip3 install -r /tangostationcontrol-requirements.txt; \ + fi + +# This COPY typically triggers a new build instead of pulling from cache. +# We thus perform it after installing the requirements.txt in case +# $TANGO_STATION_CONTROL is not defined. +COPY tmp/*.whl / RUN if [ $TANGO_STATION_CONTROL ]; then \ echo "Installing prebuild Station Control wheel"; \ sudo pip3 install /*.whl; \ - else \ - echo "Installing requirements only"; \ - sudo pip3 install -r /tangostationcontrol-requirements.txt; \ fi # install and use ephimerides and geodetic ("measures") tables for casacore. diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 1afd339d7be804b5763d9a96cffb6fb8245c827d..9ac9fe831a89caeab74c2718df8218b9ea23d3b4 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -121,6 +121,21 @@ function cleanup { docker logs "${container}" >& "log/${container}.log" done bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --dump >& log/dump_ConfigDb.log + + mkdir -p log/allocations + # obtain the list of tasks, but only those that have actually started. + docker exec server.station.nomad.nomad-cluster.jumppad.dev nomad alloc status -json | jq --raw-output '.[] | (.ID + " " + .Name + " " + (.TaskStates | map_values(select(.StartedAt)) | keys[]) + " " + (.CreateTime/1e9 | todate))' | while read -r line + do + read -r -a parts <<< "${line}" + ALLOC_ID="${parts[0]}" + ALLOC_NAME="${parts[1]}" + TASK_NAME="${parts[2]}" + CREATE_TIME="${parts[3]}" + + echo "Saving logs for job ${TASK_NAME} allocation ${ALLOC_ID}" + docker exec server.station.nomad.nomad-cluster.jumppad.dev nomad alloc logs -stderr "${ALLOC_ID}" "${TASK_NAME}" > "log/allocations/${ALLOC_NAME}-${TASK_NAME}-${CREATE_TIME}-${ALLOC_ID}.stderr.log" + docker exec server.station.nomad.nomad-cluster.jumppad.dev nomad alloc logs -stdout "${ALLOC_ID}" "${TASK_NAME}" > "log/allocations/${ALLOC_NAME}-${TASK_NAME}-${CREATE_TIME}-${ALLOC_ID}.stdout.log" + done fi if [ -z "${preserve}" ]; then make stop > /dev/null 2>&1 @@ -155,7 +170,7 @@ function integration_test { -v "$LOFAR20_DIR":/opt/lofar/tango:rw -w="/opt/lofar/tango/tangostationcontrol" ) - echo "run integration ${1} ..." + echo "run integration ${1} starting at $(date +'%F %T')..." docker "${docker_args[@]}" -e "TEST_MODULE=${1}" "git.astron.nl:5000/lofar2.0/tango/ci-build-runner:$TAG" \ tox -e integration } @@ -169,7 +184,7 @@ function await { echo -n '.'; sleep 2; current_time="$(date -u +%s)" - elapsed_seconds=$("$current_time"-"$start_time") + elapsed_seconds=$(("$current_time" - "$start_time")) if [ "${elapsed_seconds}" -gt "${timeout_sec}" ]; then echo ". [timeout]"; exit 1;