diff --git a/bin/fix-tango-exit-status.sh b/bin/fix-tango-exit-status.sh new file mode 100755 index 0000000000000000000000000000000000000000..9b775d5f3b4a10414d911ae87063aea288a3c67b --- /dev/null +++ b/bin/fix-tango-exit-status.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 +# +# This script waraps Tango Controls device servers such that they return +# exit code 0 when they should not be restarted. This allows them to +# be run effectively under the "restart: on-failure" strategy of running +# Docker containers. +# +# Exit code 0 will be returned under the following conditions: +# - Process completed succesfully (Tango returns exit code 0). +# - Process failed but an immediate restart would be useless +# (Tango returns exit code 255 ("exit(-1)" in its code base). +# +# Failures for which an immediate restart of the process would be +# useless include: +# - Device server is not found in the Tango Database +# - Mandatory properties of devices are missing in the Tango Database +# - Tango Database cannot be reached +# +# In all other cases, the non-zero exit code is propagated to Docker, +# resulting in a restart of the container. This includes for example +# Segmentation Faults. +# +# If the process is not wrapped with this script, the above conditions +# would result in a flurry of non-stop restarts of the container, +# thus stressing the system needlessly and spamming the logs. + +set -e + +# run command in background +"$@" & +PID=$! + +# propagate signals sent to us to the command +trap 'kill -INT $PID' INT +trap 'kill -TERM $PID' TERM + +# wait for the command to finish +wait $PID +RESULT=$? + +if [[ $RESULT -eq 255 ]]; then + # ignore exit status 255, which indicates a fundamental + # error that is useless to retry execution for. + RESULT=0 +fi + +# our exit code is the command's +exit $RESULT diff --git a/bin/start-ds.sh b/bin/start-ds.sh index a046c48789417fc2ddb0a171f9a374dbbab079f5..e05e25beebd09ba043a89f8ccb9421956eadc371 100755 --- a/bin/start-ds.sh +++ b/bin/start-ds.sh @@ -50,4 +50,4 @@ cd "$CWD" || exit 1 # Replace this script's process with the actual command, allowing any signals # send to the bash PID to be sent to the command directly. -exec /usr/local/bin/wait-for-it.sh "$TANGO_HOST" --timeout=30 --strict -- "$@" +exec /opt/lofar/tango/bin/fix-tango-exit-status.sh /usr/local/bin/wait-for-it.sh "$TANGO_HOST" --timeout=30 --strict -- "$@" diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml index 5272302432c4ef5cc69aa2c7236e567234198b46..b1f4dc6c09fd07954aa8d0c02d06a85a192167ad 100644 --- a/docker-compose/device-antennafield.yml +++ b/docker-compose/device-antennafield.yml @@ -51,6 +51,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-antennafield AntennaField STAT -v -ORBendPoint giop:tcp:0:5715 -ORBendPointPublish giop:tcp:${HOSTNAME}:5715 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-beamlet.yml b/docker-compose/device-beamlet.yml index 1cf7dc5a2f265d45be7700e2c368f7a83a67b53c..def12d9abc92affccd01b4668d64f7a2da974b14 100644 --- a/docker-compose/device-beamlet.yml +++ b/docker-compose/device-beamlet.yml @@ -50,6 +50,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-beamlet Beamlet STAT -v -ORBendPoint giop:tcp:0:5712 -ORBendPointPublish giop:tcp:${HOSTNAME}:5712 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-boot.yml b/docker-compose/device-boot.yml index 8472267963ef41407cbca5e10a297c304eba7933..5c4a58689fa0809c648f13f081ae5c314348490e 100644 --- a/docker-compose/device-boot.yml +++ b/docker-compose/device-boot.yml @@ -49,6 +49,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-boot Boot STAT -v -ORBendPoint giop:tcp:0:5708 -ORBendPointPublish giop:tcp:${HOSTNAME}:5708 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-bst.yml b/docker-compose/device-bst.yml index 69d71ff97f2006d943fa535196b760c4c86f0787..c6cd7814954d81c423d1fa13ac545ce7ee81b08d 100644 --- a/docker-compose/device-bst.yml +++ b/docker-compose/device-bst.yml @@ -55,6 +55,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-bst BST STAT -v -ORBendPoint giop:tcp:0:5717 -ORBendPointPublish giop:tcp:${HOSTNAME}:5717 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-calibration.yml b/docker-compose/device-calibration.yml index a024ab8a2a0e9207e3c9ea294cca6a667c705bf0..3ff6353c543a6afd30117edf9e28e833d5c55d62 100644 --- a/docker-compose/device-calibration.yml +++ b/docker-compose/device-calibration.yml @@ -52,7 +52,7 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-calibration Calibration STAT -v -ORBendPoint giop:tcp:0:5724 -ORBendPointPublish giop:tcp:${HOSTNAME}:5724 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s depends_on: diff --git a/docker-compose/device-digitalbeam.yml b/docker-compose/device-digitalbeam.yml index 2ad5c7d24050b3b8f5021df8f627c3fcd38178aa..766c4317df094bd10f9cbd050ecb2d46ca0ca677 100644 --- a/docker-compose/device-digitalbeam.yml +++ b/docker-compose/device-digitalbeam.yml @@ -50,6 +50,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-digitalbeam DigitalBeam STAT -v -ORBendPoint giop:tcp:0:5713 -ORBendPointPublish giop:tcp:${HOSTNAME}:5713 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml index c31130996c9e92291f9b9a521b5fb6ded181184d..bf51663ae58d8a897c4fab30b62be96da9069a63 100644 --- a/docker-compose/device-docker.yml +++ b/docker-compose/device-docker.yml @@ -52,6 +52,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-docker Docker STAT -v -ORBendPoint giop:tcp:0:5705 -ORBendPointPublish giop:tcp:${HOSTNAME}:5705 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml index c4a3e4bf0bb7bc5861c30335dbe7acf49e86266f..f5fcaa1a644ce34459f4187bebdc51d78f64fb50 100644 --- a/docker-compose/device-observation-control.yml +++ b/docker-compose/device-observation-control.yml @@ -49,7 +49,7 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-observationcontrol ObservationControl STAT -v -ORBendPoint giop:tcp:0:5703 -ORBendPointPublish giop:tcp:${HOSTNAME}:5703 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s depends_on: diff --git a/docker-compose/device-observation.yml b/docker-compose/device-observation.yml index 3d4b24168c96306c67bd4a0c7b91485b8bf77794..9fa593a967a1a73413de306f349c53f9aca4558a 100644 --- a/docker-compose/device-observation.yml +++ b/docker-compose/device-observation.yml @@ -48,6 +48,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-observation Observation STAT -v -ORBendPoint giop:tcp:0:5718 -ORBendPointPublish giop:tcp:${HOSTNAME}:5718 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml index 5b900769107dc097af35c0efdd49fbf814e417a2..093c22c10ee186f481bae57ae27af63b18e2eb78 100644 --- a/docker-compose/device-pcon.yml +++ b/docker-compose/device-pcon.yml @@ -45,6 +45,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-pcon pcon STAT -v -ORBendPoint giop:tcp:device-pcon:5720 -ORBendPointPublish giop:tcp:${HOSTNAME}:5720 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml index ea3ef27d9af6ae2475601ab2064c0e1752da8d1b..3610a7e34abadbead2da62903c8d1665d4aa6f0f 100644 --- a/docker-compose/device-psoc.yml +++ b/docker-compose/device-psoc.yml @@ -45,6 +45,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-psoc PSOC STAT -v -ORBendPoint giop:tcp:device-psoc:5719 -ORBendPointPublish giop:tcp:${HOSTNAME}:5719 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-rcu2h.yml b/docker-compose/device-rcu2h.yml index 48594945a5a297940f2347cb8e894bf695312362..8bfcdf2f99c79b9fa8fc214b5719284aec1c5c6f 100644 --- a/docker-compose/device-rcu2h.yml +++ b/docker-compose/device-rcu2h.yml @@ -50,6 +50,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-rcu2h RCU2H STAT -v -ORBendPoint giop:tcp:device-rcu2h:5725 -ORBendPointPublish giop:tcp:${HOSTNAME}:5725 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-rcu2l.yml b/docker-compose/device-rcu2l.yml index a499d8b7c81479487b3fa82042d6c3576aba9092..8cb03dd03822e49a0accd4abc24f84634a472bd1 100644 --- a/docker-compose/device-rcu2l.yml +++ b/docker-compose/device-rcu2l.yml @@ -50,6 +50,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-rcu2l RCU2L STAT -v -ORBendPoint giop:tcp:device-rcu2l:5726 -ORBendPointPublish giop:tcp:${HOSTNAME}:5726 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml index f9e45c368cb5998926d80f8fe27183a5091d7fa8..e471859258feb61c96f1cacb8b9262d190e7de8e 100644 --- a/docker-compose/device-sdp.yml +++ b/docker-compose/device-sdp.yml @@ -50,6 +50,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-sdp SDP STAT -v -ORBendPoint giop:tcp:device-sdp:5701 -ORBendPointPublish giop:tcp:${HOSTNAME}:5701 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml index 1e430f5bfe2d7ec15886b63f4f288120e5a98431..2139b3de6488956dff17a315b14d8b84b8c5fa59 100644 --- a/docker-compose/device-sst.yml +++ b/docker-compose/device-sst.yml @@ -55,6 +55,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-sst SST STAT -v -ORBendPoint giop:tcp:0:5702 -ORBendPointPublish giop:tcp:${HOSTNAME}:5702 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-station-manager.yml b/docker-compose/device-station-manager.yml index 24c124256b74238495b8288d305dcef44c931f9a..fb27bec2d6618380fc5e963615419a4b1abeea70 100644 --- a/docker-compose/device-station-manager.yml +++ b/docker-compose/device-station-manager.yml @@ -45,6 +45,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-station-manager StationManager STAT -v -ORBendPoint giop:tcp:device-station-manager:5723 -ORBendPointPublish giop:tcp:${HOSTNAME}:5723 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-temperature-manager.yml b/docker-compose/device-temperature-manager.yml index 9bb1a4589ff5effb6ebe2cfae19cbc10b117d273..a62613803092ea3b5b5de87402da6e26bec1b128 100644 --- a/docker-compose/device-temperature-manager.yml +++ b/docker-compose/device-temperature-manager.yml @@ -45,6 +45,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-temperaturemanager TemperatureManager STAT -v -ORBendPoint giop:tcp:0:5716 -ORBendPointPublish giop:tcp:${HOSTNAME}:5716 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-tilebeam.yml b/docker-compose/device-tilebeam.yml index 3bd65a9ca2a0eb0db3a0b2714199c0d5160fe233..5f36c7b3053cc43a2df095168a1b43129e8dfb96 100644 --- a/docker-compose/device-tilebeam.yml +++ b/docker-compose/device-tilebeam.yml @@ -46,6 +46,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-tilebeam TileBeam STAT -v -ORBendPoint giop:tcp:0:5711 -ORBendPointPublish giop:tcp:${HOSTNAME}:5711 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml index ed5a4309808db5edfbab6c063bae3c2abdc5a204..fdfa1938e4655e784bad21ae479f53d02411644c 100644 --- a/docker-compose/device-unb2.yml +++ b/docker-compose/device-unb2.yml @@ -50,6 +50,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-unb2 UNB2 STAT -v -ORBendPoint giop:tcp:device-unb2:5704 -ORBendPointPublish giop:tcp:${HOSTNAME}:5704 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml index e064d4eca68c008efd91fe504ea81236e557cecf..bf199067d9982a3fe3a50ba6b921216d8e27559b 100644 --- a/docker-compose/device-xst.yml +++ b/docker-compose/device-xst.yml @@ -55,6 +55,6 @@ services: # configure CORBA to _listen_ on 0:port, but tell others we're _reachable_ through ${HOSTNAME}:port, since CORBA # can't know about our Docker port forwarding - l2ss-xst XST STAT -v -ORBendPoint giop:tcp:0:5706 -ORBendPointPublish giop:tcp:${HOSTNAME}:5706 - restart: unless-stopped + restart: on-failure stop_signal: SIGINT # request a graceful shutdown of Tango stop_grace_period: 2s