L2SS-1544: Prevent Grafana permission issues and test for regression

.gitlab-ci.yml

@@ -268,12 +268,12 @@ unit_test:
       - tangostationcontrol/cover/*
       - tangostationcontrol/.coverage
 
-integration_test_docker:
+.test_docker:
   stage: integration-tests
   image: docker:23.0.5 # latest ships with docker compose v2.19, which has the following bug: https://github.com/docker/compose/issues/10668
-  needs:
-    - unit_test
-    - docker_build_image_device_base
+#  needs:
+#    - unit_test
+#    - docker_build_image_device_base
   tags:
     - privileged
   services:
@@ -294,7 +294,6 @@ integration_test_docker:
     - apk add --update bind-tools
     - apk add --update postgresql14-client gzip
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-  script:
     - touch /root/.Xauthority
     #    Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
     - export BASH_SOURCE=$(pwd)/setup.sh
@@ -306,9 +305,7 @@ integration_test_docker:
     - unset TANGO_HOST
     #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
-    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - export TAG="$tag"
-    - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build
   after_script:
     #    Collect output of all containers
     - |
@@ -325,10 +322,24 @@ integration_test_docker:
     paths:
       - log/
 
+integration_test_docker:
+  extends: .test_docker
+  script:
+    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build
+
+service_test_docker:
+  extends: .test_docker
+  script:
+    # Fix logstash healthcheck
+    - export HOSTNAME=localhost
+    #    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash -e $CI_PROJECT_DIR/sbin/run_service_test.sh
+
 multi_project_integration_test:
   stage: integration-tests
-  needs:
-    - docker_build_image_device_base
+#  needs:
+#    - docker_build_image_device_base
   variables:
     TANGO_DEFAULT_BRANCH: $CI_DEFAULT_BRANCH
     TANGO_CURRENT_BRANCH: $CI_COMMIT_BRANCH

docker-compose/grafana.yml

@@ -9,10 +9,6 @@
 
 version: '2.1'
 
-volumes:
-  grafana-database: {}
-#  grafana-configs: {}
-
 services:
   grafana:
     image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/grafana:${TAG}
@@ -23,12 +19,19 @@ services:
     networks:
       - control
     volumes:
-      - grafana-database:/var/lib/grafana/database
+      - grafana-database:/var/lib/grafana/database:rw
     #  - grafana-configs:/etc/grafana
     ports:
       - "3000:3000"
     environment:
       - GF_SERVER_DOMAIN=${HOSTNAME}
+    healthcheck:
+      test: >
+        bash -c "printf 'GET /nc.1 HTTP/1.1\r\nHost: astron.nl\r\n\r\n' | nc localhost 3000"
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     logging:
       driver: syslog
       options:
@@ -36,3 +39,7 @@ services:
         syslog-format: rfc3164
         tag: "{{.Name}}"
     restart: unless-stopped
+
+volumes:
+  grafana-database:
+# grafana-configs:

docker-compose/grafana/Dockerfile

 FROM git.astron.nl:5000/lofar2.0/grafana-station-dashboards:latest
 
 COPY stationcontrol-dashboards.yaml /etc/grafana/provisioning/dashboards/
+
+RUN mkdir -p /var/lib/grafana/database
+RUN chown grafana -R /var/lib/grafana/database

docker-compose/loki.yml

@@ -23,6 +23,13 @@ services:
       options:
         max-size: "100m"
         max-file: "10"
+    healthcheck:
+      test: >
+        sh -c "printf 'GET /nc.1 HTTP/1.1\r\nHost: astron.nl\r\n\r\n' | nc localhost 3100"
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     networks:
       - control
     volumes:

docker-compose/prometheus.yml

@@ -27,6 +27,13 @@ services:
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"
+    healthcheck:
+      test: >
+        sh -c "printf 'GET /nc.1 HTTP/1.1\r\nHost: astron.nl\r\n\r\n' | nc localhost 9090"
+      interval: 1m
+      timeout: 30s
+      retries: 3
+      start_period: 30s
     logging:
       driver: syslog
       options:

sbin/run_service_test.sh

+#!/bin/bash -e
+#
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+#
+
+if [ -z "$LOFAR20_DIR" ]; then
+    # We assume we aren't in the PATH, so we can derive our path.
+    # We need our parent directory.
+    LOFAR20_DIR_RELATIVE=$(dirname "$0")/..
+
+    # As an absolute path
+    LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}")
+fi
+
+export TANGO_SKIP_BUILD=1
+export NO_BASE=1
+
+cd "$LOFAR20_DIR/docker-compose" || exit 1
+
+make start grafana loki logstash prometheus
+make await grafana loki logstash prometheus