Merge branch 'L2SS-988' into 'master'

L2SS-988: Docker health status with make await used by integration script

Closes L2SS-988

See merge request !467

README.md

@@ -115,6 +115,7 @@ Next change the version in the following places:
 
 # Release Notes
 
+* 0.4.0 Have most containers report health status and add `make await` command
 * 0.3.1 Fix for applying boot device dsconfig
 * 0.3.0 Initial version of deployment scripts and functionality
 * 0.2.0 Extend `Beamlet` device with FPGA source address attributes

docker-compose/Makefile

@@ -13,6 +13,9 @@ ATTACH_COMPOSE_FILE_ARGS := $(foreach yml,$(filter-out tango.yml,$(COMPOSE_FILES
 # But we allow to overwrite it.
 NETWORK_MODE ?= tangonet
 
+# Timeout used to await services to become healthy
+TIMEOUT ?= 300
+
 SCRATCH ?= /tmp
 
 # Host name through which others can reach our control interfaces.
@@ -50,6 +53,8 @@ else ifeq (attach,$(firstword $(MAKECMDGOALS)))
     ifndef TANGO_HOST
         $(error TANGO_HOST must specify the Tango database device, e.g., make TANGO_HOST=powersupply-databaseds:10000 ...)
     endif
+else ifeq (await,$(firstword $(MAKECMDGOALS)))
+    SERVICE_TARGET = true
 else ifeq (run,$(firstword $(MAKECMDGOALS)))
     RUN_TARGET = true
 else ifeq (integration,$(firstword $(MAKECMDGOALS)))
@@ -220,6 +225,28 @@ restart: ## restart a service (usage: make restart <servicename>)
 attach:  ## attach a service to an existing Tango network
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(ATTACH_COMPOSE_FILE_ARGS) up --no-recreate -d $(SERVICE)
 
+time := 0
+await:  ## Await every container with total max timeout of 300, do not reset timeout
+	for i in $(SERVICE); do \
+		current_service=$$($(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) ps -q $${i}); \
+		if [ -z "$${current_service}" ]; then \
+		    continue; \
+        fi; \
+        service_has_health=$$(docker inspect -f '{{.State.Health.Status}}' $${current_service}); \
+        if [ -z "$${service_has_health}" ]; then \
+            continue; \
+        fi; \
+		while [ "$$(docker inspect -f '{{.State.Health.Status}}' $${current_service})" != "healthy" ] ; do \
+			sleep 1; \
+			time=$$(expr $$time + 1); \
+			if [ $${time} -gt $(TIMEOUT) ]; then \
+				echo "Timeout reached waiting for $${i} to become healthy"; \
+				exit 1; \
+			fi; \
+		done; \
+		echo "Service $${i} is healthy"; \
+	done
+
 status:  ## show the container status
 	$(DOCKER_COMPOSE_ARGS) docker-compose $(COMPOSE_FILE_ARGS) ps
 

docker-compose/archiver-timescale.yml

@@ -25,6 +25,12 @@ services:
       - POSTGRES_PASSWORD=${PG_SUPERUSER_PASSWORD}
       - PG_HDB_PASSWORD=${PG_HDB_PASSWORD}
       - TANGO_HOST=${TANGO_HOST}
+    healthcheck:
+      test: nc -z -v localhost 5432
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     logging:
       driver: syslog
       options:

docker-compose/device-antennafield.yml

@@ -43,6 +43,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5815
       - TANGO_ZMQ_HEARTBEAT_PORT=5915
+    healthcheck:
+      test: l2ss-health STAT/AntennaField/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-apsct.yml

@@ -42,6 +42,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5809
       - TANGO_ZMQ_HEARTBEAT_PORT=5909
+    healthcheck:
+      test: l2ss-health STAT/Apsct/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-apspu.yml

@@ -42,6 +42,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5810
       - TANGO_ZMQ_HEARTBEAT_PORT=5910
+    healthcheck:
+      test: l2ss-health STAT/Apspu/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-beamlet.yml

@@ -42,6 +42,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5812
       - TANGO_ZMQ_HEARTBEAT_PORT=5912
+    healthcheck:
+      test: l2ss-health STAT/Beamlet/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-boot.yml

@@ -41,6 +41,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5808
       - TANGO_ZMQ_HEARTBEAT_PORT=5908
+    healthcheck:
+      test: l2ss-health STAT/BOOT/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-bst.yml

@@ -45,6 +45,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5817
       - TANGO_ZMQ_HEARTBEAT_PORT=5917
+    healthcheck:
+      test: l2ss-health STAT/BST/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-ccd.yml

@@ -42,6 +42,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5821
       - TANGO_ZMQ_HEARTBEAT_PORT=5921
+    healthcheck:
+      test: l2ss-health STAT/CCD/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-digitalbeam.yml

@@ -42,6 +42,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5813
       - TANGO_ZMQ_HEARTBEAT_PORT=5913
+    healthcheck:
+      test: l2ss-health STAT/DigitalBeam/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-docker.yml

@@ -44,6 +44,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5805
       - TANGO_ZMQ_HEARTBEAT_PORT=5905
+    healthcheck:
+      test: l2ss-health STAT/Docker/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-observation-control.yml

@@ -41,6 +41,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5803
       - TANGO_ZMQ_HEARTBEAT_PORT=5903
+    healthcheck:
+      test: l2ss-health STAT/ObservationControl/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-observation.yml

@@ -40,6 +40,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5818
       - TANGO_ZMQ_HEARTBEAT_PORT=5918
+    healthcheck:
+      test: nc -z -v device-observation 5718
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-pcon.yml

@@ -37,6 +37,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5820
       - TANGO_ZMQ_HEARTBEAT_PORT=5920
+    healthcheck:
+      test: l2ss-health STAT/pcon/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-psoc.yml

@@ -37,6 +37,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5819
       - TANGO_ZMQ_HEARTBEAT_PORT=5919
+    healthcheck:
+      test: l2ss-health STAT/PSOC/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-recv.yml

@@ -42,6 +42,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5807
       - TANGO_ZMQ_HEARTBEAT_PORT=5907
+    healthcheck:
+      test: l2ss-health STAT/RECV/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-sdp.yml

@@ -42,6 +42,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5801
       - TANGO_ZMQ_HEARTBEAT_PORT=5901
+    healthcheck:
+      test: l2ss-health STAT/SDP/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-sst.yml

@@ -45,6 +45,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5802
       - TANGO_ZMQ_HEARTBEAT_PORT=5902
+    healthcheck:
+      test: l2ss-health STAT/SST/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh

docker-compose/device-temperature-manager.yml

@@ -37,6 +37,12 @@ services:
       - TANGO_HOST=${TANGO_HOST}
       - TANGO_ZMQ_EVENT_PORT=5816
       - TANGO_ZMQ_HEARTBEAT_PORT=5916
+    healthcheck:
+      test: l2ss-health STAT/TemperatureManager/1
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     working_dir: /opt/lofar/tango
     entrypoint:
       - bin/start-ds.sh