Skip to content
Snippets Groups Projects
Commit eeaa88a0 authored by Hannes Feldt's avatar Hannes Feldt
Browse files

Merge branch 'L2SS-1341-migrate_tango2nomad' into L2SS-1340-deploy_monitoring_to_nomad

parents b85833d4 e600ed49
Branches
No related tags found
4 merge requests!752Resolve L2SS-1525 "Migrate minio",!751Resolve L2SS-1340 "Deploy monitoring to nomad",!749Resolve L2SS-1342 "Migrate jupyter",!685Resolve L2SS-1340 "Deploy monitoring to nomad"
Showing
with 119 additions and 88 deletions
...@@ -34,4 +34,11 @@ deploy/hosts ...@@ -34,4 +34,11 @@ deploy/hosts
docker-compose/alerta-web/alerta-secrets.json docker-compose/alerta-web/alerta-secrets.json
docker-compose/tmp docker-compose/tmp
infra/dev/jobs/*.nomad
infra/dev/tmp/*
!infra/dev/tmp/.keep
**/CDB/dump*.json **/CDB/dump*.json
bin/jumppad
...@@ -111,6 +111,7 @@ docker_build_image: ...@@ -111,6 +111,7 @@ docker_build_image:
- loki - loki
- logstash - logstash
- jupyter-lab - jupyter-lab
- dsconfig
rules: rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes: changes:
...@@ -225,15 +226,18 @@ unit_test: ...@@ -225,15 +226,18 @@ unit_test:
integration_test_docker: integration_test_docker:
stage: integration-tests stage: integration-tests
image: docker:23.0.5 # latest ships with docker compose v2.19, which has the following bug: https://github.com/docker/compose/issues/10668 image: docker:latest
needs: needs:
- unit_test - unit_test
tags: tags:
- privileged - privileged
- integration_tests
services: services:
- name: docker:dind - name: docker:dind
variables: variables:
DOCKER_TLS_CERTDIR: "/certs" DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
JUMPPAD_HOME: $CI_PROJECT_DIR
before_script: before_script:
- | - |
if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
...@@ -244,16 +248,15 @@ integration_test_docker: ...@@ -244,16 +248,15 @@ integration_test_docker:
echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
fi fi
- apk update - apk update
- apk add git
- apk add --update make bash - apk add --update make bash
- apk add --update bind-tools - apk add --update bind-tools
- apk add --update postgresql14-client gzip - apk add --update postgresql14-client gzip socat
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
script: script:
- touch /root/.Xauthority - touch /root/.Xauthority
# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this # Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
- export BASH_SOURCE=$(pwd)/setup.sh - export BASH_SOURCE=$(pwd)/setup.sh
# Hack HOSTNAME env variable into host.docker.internal, set in docker-compose
- export HOSTNAME=host.docker.internal
# source the lofarrc file and mask its non zero exit code # source the lofarrc file and mask its non zero exit code
- . setup.sh || true - . setup.sh || true
# TANGO_HOST must be unset our databaseds will be unreachable # TANGO_HOST must be unset our databaseds will be unreachable
...@@ -262,18 +265,9 @@ integration_test_docker: ...@@ -262,18 +265,9 @@ integration_test_docker:
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag
# Do not remove 'bash' or statement will be ignored by primitive docker shell # Do not remove 'bash' or statement will be ignored by primitive docker shell
- export TAG="$tag" - export TAG="$tag"
- bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build - socat tcp-listen:4646,reuseaddr,fork tcp:docker:4646 &
after_script: - socat udp-listen:8600,reuseaddr,fork udp:docker:8600 &
# Collect output of all containers - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs
- |
mkdir -p log
for container in $(docker ps -a --format "{{.Names}}")
do
echo "Saving log for container $container"
docker logs "${container}" >& "log/${container}.log"
done
# Collect content of the TangoDB
- $CI_PROJECT_DIR/bin/dump_ConfigDb.sh >& log/dump_ConfigDb.log
artifacts: artifacts:
when: always when: always
paths: paths:
......
...@@ -4,7 +4,7 @@ The ConfigDb.json files in this directory are used to populate the Tango configu ...@@ -4,7 +4,7 @@ The ConfigDb.json files in this directory are used to populate the Tango configu
To load a configuration file, use To load a configuration file, use
```bash ```bash
../sbin/update_ConfigDb.sh file.json ../sbin/dsconfig.sh --update file.json
``` ```
The tool ``tangostationcontrol.toolkit.analyse_dsconfig_hierarchies`` is provided to check the consistency of The tool ``tangostationcontrol.toolkit.analyse_dsconfig_hierarchies`` is provided to check the consistency of
...@@ -13,7 +13,7 @@ hierarchies defined between the devices in a set of configuration files. ...@@ -13,7 +13,7 @@ hierarchies defined between the devices in a set of configuration files.
The following files are provided: The following files are provided:
| File | Description | Usage | | File | Description | Usage |
|--------------------------------------------|-------------------------------------------------------------|---------------------| |------------------------------------------|-------------------------------------------------------------|---------------------|
| `LOFAR_ConfigDb.json` | Generic base configuration, registering all of the devices. | Always | | `LOFAR_ConfigDb.json` | Generic base configuration, registering all of the devices. | Always |
| `test_environment_ConfigDb.json` | Base delta for the unit- and integration test suites. | Tests & development | | `test_environment_ConfigDb.json` | Base delta for the unit- and integration test suites. | Tests & development |
| `stations/simulators_ConfigDb.json` | A "station" configuration that points to our simulators. | Tests & development | | `stations/simulators_ConfigDb.json` | A "station" configuration that points to our simulators. | Tests & development |
......
...@@ -47,6 +47,25 @@ You will also need: ...@@ -47,6 +47,25 @@ You will also need:
* make * make
* bash * bash
## Start dev environment
For local development a dev environment is needed. To setup this environment run
```
./sbin/prepare_dev_env.sh
```
This will install `jumppad`, if not present yet as well as creating the docker volume needed to simulate the station
nomad cluster.
Afterwards run
```
jumppad up infra/dev
```
to start the dev environment including tango.
## Bootstrap ## Bootstrap
The bootstrap procedure is needed only once. First we build all docker The bootstrap procedure is needed only once. First we build all docker
......
#!/bin/bash #!/bin/bash
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) #
# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# writes the JSON dump to stdout, Do not change -i into -it incompatible with gitlab ci! "${LOFAR20_DIR}/sbin/dsconfig.sh" --dump
docker exec -i dsconfig bash -c '
python -m dsconfig.dump > /tmp/dsconfig-configdb-dump.json
/manage_object_properties.py -r > /tmp/dsconfig-objectdb-dump.json
/merge_json.py /tmp/dsconfig-objectdb-dump.json /tmp/dsconfig-configdb-dump.json'
#!/bin/bash #!/bin/bash
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) #
# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
exec docker exec -it itango itango3 exec docker exec -e "TANGO_HOST=$TANGO_HOST:$TANGO_PORT" -it itango itango3
#!/bin/bash #!/bin/bash
# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) #
# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
exec docker exec -it itango /bin/bash exec docker exec -e "TANGO_HOST=$TANGO_HOST:$TANGO_PORT" -it itango /bin/bash
- name: Update Database Config - name: Update Database Config
changed_when: false changed_when: false
shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }}" shell: "./sbin/dsconfig.sh --load CDB/{{ database_config }}"
args: args:
chdir: "{{ base_path }}" chdir: "{{ base_path }}"
- name: Get Database Config - name: Get Database Config
changed_when: false changed_when: false
shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }} 2>&1" shell: "./sbin/dsconfig.sh --load CDB/{{ database_config }} 2>&1"
register: dsconfig_result register: dsconfig_result
args: args:
chdir: "{{ base_path }}" chdir: "{{ base_path }}"
......
...@@ -6,19 +6,20 @@ MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST))) ...@@ -6,19 +6,20 @@ MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST)))
BASEDIR := $(notdir $(patsubst %/,%,$(dir $(MAKEPATH)))) BASEDIR := $(notdir $(patsubst %/,%,$(dir $(MAKEPATH))))
TAG ?= latest TAG ?= latest
DNS ?= 127.0.0.11
DOCKER_COMPOSE ?= docker compose DOCKER_COMPOSE ?= docker compose
DOCKER_COMPOSE_ENV_FILE := $(abspath .env) DOCKER_COMPOSE_ENV_FILE := $(abspath .env)
COMPOSE_FILES := $(wildcard *.yml) COMPOSE_FILES := $(wildcard *.yml)
COMPOSE_FILE_ARGS := --env-file $(DOCKER_COMPOSE_ENV_FILE) $(foreach yml,$(COMPOSE_FILES),-f $(yml))
ATTACH_COMPOSE_FILE_ARGS := $(foreach yml,$(filter-out tango.yml,$(COMPOSE_FILES)),-f $(yml)) ATTACH_COMPOSE_FILE_ARGS := $(foreach yml,$(filter-out tango.yml,$(COMPOSE_FILES)),-f $(yml))
COMPOSE_FILE_ARGS := --env-file $(DOCKER_COMPOSE_ENV_FILE) $(ATTACH_COMPOSE_FILE_ARGS)
# The default Docker network mode is tangonet. The "host" network # The default Docker network mode is tangonet. The "host" network
# mode will make the tangodb and archiverdb ports clash, # mode will make the tangodb and archiverdb ports clash,
# But we allow to overwrite it. # But we allow to overwrite it.
NETWORK_MODE ?= tangonet NETWORK_MODE ?= station
# Timeout used to await services to become healthy # Timeout used to await services to become healthy
TIMEOUT ?= 300 TIMEOUT ?= 300
...@@ -97,11 +98,6 @@ endif ...@@ -97,11 +98,6 @@ endif
# prevent collisions with jobs from the same project running at the same # prevent collisions with jobs from the same project running at the same
# time. # time.
# #
ifneq ($(CI_JOB_ID),)
NETWORK_MODE := tangonet-$(CI_JOB_ID)
else
$(info Network mode cannot be host for the archiver! It won't work unless you set the env var CI_JOB_ID=local)
endif
ifeq ($(OS),Windows_NT) ifeq ($(OS),Windows_NT)
$(error Sorry, Windows is not supported yet) $(error Sorry, Windows is not supported yet)
...@@ -166,14 +162,15 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \ ...@@ -166,14 +162,15 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
CONTAINER_EXECUTION_UID=$(shell id -u) \ CONTAINER_EXECUTION_UID=$(shell id -u) \
DOCKER_GID=$(DOCKER_GID) \ DOCKER_GID=$(DOCKER_GID) \
TEST_MODULE=$(INTEGRATION_MODULE) \ TEST_MODULE=$(INTEGRATION_MODULE) \
TAG=$(TAG) TAG=$(TAG) \
DNS=$(DNS)
.PHONY: up base base-nocache down minimal context run integration start stop restart build build-nocache status clean pull help await .PHONY: up base base-nocache down minimal context run integration start stop restart build build-nocache status clean pull help await
.DEFAULT_GOAL := help .DEFAULT_GOAL := help
pull: ## pull the images from the Docker hub pull: ## pull the images from the Docker hub
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull --ignore-pull-failures $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull --quiet-pull --ignore-pull-failures
base: context ## Build base lofar device image base: context ## Build base lofar device image
ifneq ($(NO_BASE),1) ifneq ($(NO_BASE),1)
...@@ -185,11 +182,11 @@ base-nocache: context ## Rebuild base lofar device image ...@@ -185,11 +182,11 @@ base-nocache: context ## Rebuild base lofar device image
build: base ## build images build: base ## build images
# docker-compose does not support build dependencies, so manage those here # docker-compose does not support build dependencies, so manage those here
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --parallel --progress=plain $(SERVICE) $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) -f tango.yml build --parallel --progress=plain $(SERVICE)
build-nocache: base-nocache ## rebuild images from scratch build-nocache: base-nocache ## rebuild images from scratch
# docker-compose does not support build dependencies, so manage those here # docker-compose does not support build dependencies, so manage those here
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE) $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) -f tango.yml build --no-cache --progress=plain $(SERVICE)
up: base minimal ## start the base TANGO system and prepare requested services up: base minimal ## start the base TANGO system and prepare requested services
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE) $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE)
...@@ -219,7 +216,7 @@ ifneq ($(NETWORK_MODE),host) ...@@ -219,7 +216,7 @@ ifneq ($(NETWORK_MODE),host)
docker network inspect 9000-$(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create 9000-$(NETWORK_MODE) -o com.docker.network.driver.mtu=9000) docker network inspect 9000-$(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create 9000-$(NETWORK_MODE) -o com.docker.network.driver.mtu=9000)
endif endif
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) -f tango.yml -f networks.yml up --no-recreate -d
context: ## Move the necessary files to create minimal docker context context: ## Move the necessary files to create minimal docker context
@mkdir -p tmp @mkdir -p tmp
...@@ -228,8 +225,8 @@ context: ## Move the necessary files to create minimal docker context ...@@ -228,8 +225,8 @@ context: ## Move the necessary files to create minimal docker context
bootstrap: pull build # first start, initialise from scratch bootstrap: pull build # first start, initialise from scratch
$(MAKE) start dsconfig # boot up containers to load configurations $(MAKE) start dsconfig # boot up containers to load configurations
sleep 5 # wait for dsconfig container to come up sleep 5 # wait for dsconfig container to come up
../sbin/update_ConfigDb.sh ../CDB/LOFAR_ConfigDb.json # load default configuration ../sbin/dsconfig.sh --update ../CDB/LOFAR_ConfigDb.json # load default configuration
../sbin/update_ConfigDb.sh ../CDB/stations/simulators_ConfigDb.json # by default, use simulators ../sbin/dsconfig.sh --update ../CDB/stations/simulators_ConfigDb.json # by default, use simulators
start: up ## start a service (usage: make start <servicename>) start: up ## start a service (usage: make start <servicename>)
if [ $(UNAME_S) = Linux ]; then touch ~/.Xauthority; chmod a+r ~/.Xauthority; fi if [ $(UNAME_S) = Linux ]; then touch ~/.Xauthority; chmod a+r ~/.Xauthority; fi
...@@ -258,16 +255,18 @@ await: ## Await every container with total max timeout of 300, do not reset tim ...@@ -258,16 +255,18 @@ await: ## Await every container with total max timeout of 300, do not reset tim
if [ -z "$${service_has_health}" ]; then \ if [ -z "$${service_has_health}" ]; then \
continue; \ continue; \
fi; \ fi; \
echo -n "Whait for service $${i} to become healthy .."; \
while [ "$$(docker inspect -f '{{.State.Health.Status}}' $${current_service})" != "healthy" ] ; do \ while [ "$$(docker inspect -f '{{.State.Health.Status}}' $${current_service})" != "healthy" ] ; do \
sleep 1; \ echo -n '.'; \
sleep 2; \
time=$$(expr $$time + 1); \ time=$$(expr $$time + 1); \
if [ $${time} -gt $(TIMEOUT) ]; then \ if [ $${time} -gt $(TIMEOUT) ]; then \
echo "Timeout reached waiting for $${i} to become healthy"; \ echo "timeout"; \
docker logs $${i}; \ docker logs $${i}; \
exit 1; \ exit 1; \
fi; \ fi; \
done; \ done; \
echo "Service $${i} is healthy"; \ echo ". [ok]"; \
done done
status: ## show the container status status: ## show the container status
......
...@@ -27,6 +27,7 @@ services: ...@@ -27,6 +27,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5715:5715" # unique port for this DS - "5715:5715" # unique port for this DS
- "5815:5815" # ZeroMQ event port - "5815:5815" # ZeroMQ event port
......
...@@ -18,6 +18,7 @@ services: ...@@ -18,6 +18,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5728:5728" # unique port for this DS - "5728:5728" # unique port for this DS
- "5828:5828" # ZeroMQ event port - "5828:5828" # ZeroMQ event port
......
...@@ -26,6 +26,7 @@ services: ...@@ -26,6 +26,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5709:5709" # unique port for this DS - "5709:5709" # unique port for this DS
- "5809:5809" # ZeroMQ event port - "5809:5809" # ZeroMQ event port
......
...@@ -26,6 +26,7 @@ services: ...@@ -26,6 +26,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5710:5710" # unique port for this DS - "5710:5710" # unique port for this DS
- "5810:5810" # ZeroMQ event port - "5810:5810" # ZeroMQ event port
......
...@@ -26,6 +26,7 @@ services: ...@@ -26,6 +26,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5712:5712" # unique port for this DS - "5712:5712" # unique port for this DS
- "5812:5812" # ZeroMQ event port - "5812:5812" # ZeroMQ event port
......
...@@ -25,6 +25,7 @@ services: ...@@ -25,6 +25,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5708:5708" # unique port for this DS - "5708:5708" # unique port for this DS
- "5808:5808" # ZeroMQ event port - "5808:5808" # ZeroMQ event port
......
...@@ -27,6 +27,7 @@ services: ...@@ -27,6 +27,7 @@ services:
networks: networks:
- control - control
- data - data
dns: ${DNS}
ports: ports:
- "5003:5003/udp" # port to receive SST UDP packets on (first antennafield) - "5003:5003/udp" # port to receive SST UDP packets on (first antennafield)
- "5103:5103/tcp" # port to emit SST TCP packets on - "5103:5103/tcp" # port to emit SST TCP packets on
......
...@@ -26,6 +26,7 @@ services: ...@@ -26,6 +26,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5721:5721" # unique port for this DS - "5721:5721" # unique port for this DS
- "5821:5821" # ZeroMQ event port - "5821:5821" # ZeroMQ event port
......
...@@ -26,6 +26,7 @@ services: ...@@ -26,6 +26,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5722:5722" # unique port for this DS - "5722:5722" # unique port for this DS
- "5822:5822" # ZeroMQ event port - "5822:5822" # ZeroMQ event port
......
...@@ -26,6 +26,7 @@ services: ...@@ -26,6 +26,7 @@ services:
max-file: "10" max-file: "10"
networks: networks:
- control - control
dns: ${DNS}
ports: ports:
- "5713:5713" # unique port for this DS - "5713:5713" # unique port for this DS
- "5813:5813" # ZeroMQ event port - "5813:5813" # ZeroMQ event port
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment