From 5e1630d11f9112e5a9c05a41c6a3197a12a73bdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Corn=C3=A9=20Lukken?= <lukken@astron.nl>
Date: Wed, 4 Oct 2023 12:29:44 +0000
Subject: [PATCH] L2SS-1544: Prevent Grafana permission issues and test for
regression
---
.gitlab-ci.yml | 29 ++++++++++++++++++++---------
docker-compose/grafana.yml | 17 ++++++++++++-----
docker-compose/grafana/Dockerfile | 3 +++
docker-compose/loki.yml | 7 +++++++
docker-compose/prometheus.yml | 7 +++++++
sbin/run_service_test.sh | 22 ++++++++++++++++++++++
6 files changed, 71 insertions(+), 14 deletions(-)
create mode 100644 sbin/run_service_test.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index db3d0c5cf..627270c72 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -268,12 +268,12 @@ unit_test:
- tangostationcontrol/cover/*
- tangostationcontrol/.coverage
-integration_test_docker:
+.test_docker:
stage: integration-tests
image: docker:23.0.5 # latest ships with docker compose v2.19, which has the following bug: https://github.com/docker/compose/issues/10668
- needs:
- - unit_test
- - docker_build_image_device_base
+# needs:
+# - unit_test
+# - docker_build_image_device_base
tags:
- privileged
services:
@@ -294,7 +294,6 @@ integration_test_docker:
- apk add --update bind-tools
- apk add --update postgresql14-client gzip
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- script:
- touch /root/.Xauthority
# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
- export BASH_SOURCE=$(pwd)/setup.sh
@@ -306,9 +305,7 @@ integration_test_docker:
- unset TANGO_HOST
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
- # Do not remove 'bash' or statement will be ignored by primitive docker shell
- export TAG="$tag"
- - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build
after_script:
# Collect output of all containers
- |
@@ -325,10 +322,24 @@ integration_test_docker:
paths:
- log/
+integration_test_docker:
+ extends: .test_docker
+ script:
+ # Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build
+
+service_test_docker:
+ extends: .test_docker
+ script:
+ # Fix logstash healthcheck
+ - export HOSTNAME=localhost
+ # Do not remove 'bash' or statement will be ignored by primitive docker shell
+ - bash -e $CI_PROJECT_DIR/sbin/run_service_test.sh
+
multi_project_integration_test:
stage: integration-tests
- needs:
- - docker_build_image_device_base
+# needs:
+# - docker_build_image_device_base
variables:
TANGO_DEFAULT_BRANCH: $CI_DEFAULT_BRANCH
TANGO_CURRENT_BRANCH: $CI_COMMIT_BRANCH
diff --git a/docker-compose/grafana.yml b/docker-compose/grafana.yml
index 7e09b616c..f6e37341a 100644
--- a/docker-compose/grafana.yml
+++ b/docker-compose/grafana.yml
@@ -9,10 +9,6 @@
version: '2.1'
-volumes:
- grafana-database: {}
-# grafana-configs: {}
-
services:
grafana:
image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/grafana:${TAG}
@@ -23,12 +19,19 @@ services:
networks:
- control
volumes:
- - grafana-database:/var/lib/grafana/database
+ - grafana-database:/var/lib/grafana/database:rw
# - grafana-configs:/etc/grafana
ports:
- "3000:3000"
environment:
- GF_SERVER_DOMAIN=${HOSTNAME}
+ healthcheck:
+ test: >
+ bash -c "printf 'GET /nc.1 HTTP/1.1\r\nHost: astron.nl\r\n\r\n' | nc localhost 3000"
+ interval: 1m
+ timeout: 30s
+ retries: 3
+ start_period: 30s
logging:
driver: syslog
options:
@@ -36,3 +39,7 @@ services:
syslog-format: rfc3164
tag: "{{.Name}}"
restart: unless-stopped
+
+volumes:
+ grafana-database:
+# grafana-configs:
diff --git a/docker-compose/grafana/Dockerfile b/docker-compose/grafana/Dockerfile
index 1d13c0b4c..b8d793c0b 100644
--- a/docker-compose/grafana/Dockerfile
+++ b/docker-compose/grafana/Dockerfile
@@ -1,3 +1,6 @@
FROM git.astron.nl:5000/lofar2.0/grafana-station-dashboards:latest
COPY stationcontrol-dashboards.yaml /etc/grafana/provisioning/dashboards/
+
+RUN mkdir -p /var/lib/grafana/database
+RUN chown grafana -R /var/lib/grafana/database
diff --git a/docker-compose/loki.yml b/docker-compose/loki.yml
index d2165ea14..ce364dab0 100644
--- a/docker-compose/loki.yml
+++ b/docker-compose/loki.yml
@@ -23,6 +23,13 @@ services:
options:
max-size: "100m"
max-file: "10"
+ healthcheck:
+ test: >
+ sh -c "printf 'GET /nc.1 HTTP/1.1\r\nHost: astron.nl\r\n\r\n' | nc localhost 3100"
+ interval: 1m
+ timeout: 30s
+ retries: 3
+ start_period: 30s
networks:
- control
volumes:
diff --git a/docker-compose/prometheus.yml b/docker-compose/prometheus.yml
index a1ce03c21..977197e1b 100644
--- a/docker-compose/prometheus.yml
+++ b/docker-compose/prometheus.yml
@@ -27,6 +27,13 @@ services:
- prometheus-data:/prometheus
ports:
- "9090:9090"
+ healthcheck:
+ test: >
+ sh -c "printf 'GET /nc.1 HTTP/1.1\r\nHost: astron.nl\r\n\r\n' | nc localhost 9090"
+ interval: 1m
+ timeout: 30s
+ retries: 3
+ start_period: 30s
logging:
driver: syslog
options:
diff --git a/sbin/run_service_test.sh b/sbin/run_service_test.sh
new file mode 100644
index 000000000..1a357205e
--- /dev/null
+++ b/sbin/run_service_test.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -e
+#
+# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+# SPDX-License-Identifier: Apache-2.0
+#
+
+if [ -z "$LOFAR20_DIR" ]; then
+ # We assume we aren't in the PATH, so we can derive our path.
+ # We need our parent directory.
+ LOFAR20_DIR_RELATIVE=$(dirname "$0")/..
+
+ # As an absolute path
+ LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}")
+fi
+
+export TANGO_SKIP_BUILD=1
+export NO_BASE=1
+
+cd "$LOFAR20_DIR/docker-compose" || exit 1
+
+make start grafana loki logstash prometheus
+make await grafana loki logstash prometheus
--
GitLab