diff --git a/.gitignore b/.gitignore index 95968218c813af7e5461d064a864d4b93d8cefd6..5a270622ef1b10d68ee0508933a29c0cb9db6431 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,11 @@ deploy/hosts docker-compose/alerta-web/alerta-secrets.json docker-compose/tmp +infra/dev/jobs/*.nomad +infra/dev/tmp/* +!infra/dev/tmp/.keep + **/CDB/dump*.json + +bin/jumppad + diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4f72625dbc60ab4bb34e43651208fa27e9ee901..b4a9452c092334f5dee9faff89cce4fadbea93a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,6 +124,7 @@ docker_build_image: - loki - logstash - jupyter-lab + - dsconfig rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" changes: @@ -247,18 +248,20 @@ unit_test: integration_test_docker: stage: integration-tests - image: docker:23.0.5 # latest ships with docker compose v2.19, which has the following bug: https://github.com/docker/compose/issues/10668 + image: docker:latest needs: - unit_test - wheel_packaging dependencies: - wheel_packaging tags: - - privileged + - integration_tests services: - name: docker:dind variables: - DOCKER_TLS_CERTDIR: "/certs" + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + JUMPPAD_HOME: $CI_PROJECT_DIR before_script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then @@ -269,16 +272,15 @@ integration_test_docker: echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - apk update + - apk add git - apk add --update make bash - apk add --update bind-tools - - apk add --update postgresql14-client gzip + - apk add --update postgresql14-client gzip socat - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: - touch /root/.Xauthority # Hack BASH_SOURCE into sourced files, docker its sh shell won't set this - export BASH_SOURCE=$(pwd)/setup.sh - # Hack HOSTNAME env variable into host.docker.internal, set in docker-compose - - export HOSTNAME=host.docker.internal # source the lofarrc file and mask its non zero exit code - . setup.sh || true # TANGO_HOST must be unset our databaseds will be unreachable @@ -287,18 +289,9 @@ integration_test_docker: - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag # Do not remove 'bash' or statement will be ignored by primitive docker shell - export TAG="$tag" - - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build - after_script: - # Collect output of all containers - - | - mkdir -p log - for container in $(docker ps -a --format "{{.Names}}") - do - echo "Saving log for container $container" - docker logs "${container}" >& "log/${container}.log" - done - # Collect content of the TangoDB - - $CI_PROJECT_DIR/bin/dump_ConfigDb.sh >& log/dump_ConfigDb.log + - socat tcp-listen:4646,reuseaddr,fork tcp:docker:4646 & + - socat udp-listen:8600,reuseaddr,fork udp:docker:8600 & + - bash -e $CI_PROJECT_DIR/sbin/run_integration_test.sh --no-build --save-logs artifacts: when: always paths: @@ -344,41 +337,26 @@ release_job: tag_name: '$CI_COMMIT_TAG' description: '$CI_COMMIT_TAG' -.base_deploy: +deploy_nomad: stage: deploy - image: ubuntu:bionic + image: + name: hashicorp/levant + entrypoint: [ "" ] when: manual rules: - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG - before_script: - - apt-get update - - apt-get install ansible openssh-client -y - # Use Gitlab protected variable to provide key - - echo "$DEPLOY_KEY" > id_rsa - - chmod 400 id_rsa - - ssh-keygen -y -f id_rsa > id_rsa.pub - # Add ssh key to agent - - eval $(ssh-agent) - - ssh-add id_rsa - - ansible --version - script: - - echo "Deploying version $CI_COMMIT_TAG" - - cd deploy - # Prevent error of ansible being run in world writeable directory - - chmod o-w . - # Generate hosts file for deployment - - echo "[all]" > hosts - - echo "stat ansible_host=$DEPLOY_HOST ansible_user=$DEPLOY_USER" >> hosts - # Run deployment with populated variables - - ansible-playbook -v deploy.yml --extra-vars station_version=$CI_COMMIT_TAG --extra-vars station_config=$DEPLOY_CONFIG -.deploy_l2ts_base: - extends: .base_deploy - variables: - DEPLOY_USER: $L2TS_USERNAME - DEPLOY_HOST: $L2TS_HOSTNAME - DEPLOY_CONFIG: $L2TS_CONFIG - DEPLOY_KEY: $L2TS_DEPLOY_KEY -deploy_l2ts_start: - extends: .deploy_l2ts_base + parallel: + matrix: + - STATION: + - cs001 + COMPONENT: + - tango environment: - name: l2ts + name: $STATION + script: + - | + levant deploy \ + -address="http://${STATION}c.control.lofar:4646" \ + -var image_tag="latest" \ + -var station="${STATION}" \ + infra/jobs/station/${COMPONENT}.levant.hcl diff --git a/CDB/README.md b/CDB/README.md index 794fd25f3ef5080d67eca27676839fdb125ca188..3ec4da94dea9bf4d5751c83ee600bc9f5a62aa93 100644 --- a/CDB/README.md +++ b/CDB/README.md @@ -4,7 +4,7 @@ The ConfigDb.json files in this directory are used to populate the Tango configu To load a configuration file, use ```bash -../sbin/update_ConfigDb.sh file.json +../sbin/dsconfig.sh --update file.json ``` The tool ``tangostationcontrol.toolkit.analyse_dsconfig_hierarchies`` is provided to check the consistency of @@ -12,13 +12,13 @@ hierarchies defined between the devices in a set of configuration files. The following files are provided: -| File | Description | Usage | -|--------------------------------------------|-------------------------------------------------------------|---------------------| -| `LOFAR_ConfigDb.json` | Generic base configuration, registering all of the devices. | Always | -| `test_environment_ConfigDb.json` | Base delta for the unit- and integration test suites. | Tests & development | -| `stations/simulators_ConfigDb.json` | A "station" configuration that points to our simulators. | Tests & development | -| `stations/dummy_positions_ConfigDb.json` | An antenna configuration, just to have one (it's CS001). | Tests & development | -| `stations/LTS_ConfigDb.json` | The configuration for the Lab Test Station. | Load on LTS | -| `stations/DTS_ConfigDb.json` | The configuration for the Dwingeloo Test Station. | Load on DTS-lab | -| `stations/DTS_Outside_ConfigDb.json` | The configuration for the Dwingeloo Test Station Outside. | Load on DTS-outside | -| `integrations/` | Configurations required by the integration test suites. | Integration tests | +| File | Description | Usage | +|------------------------------------------|-------------------------------------------------------------|---------------------| +| `LOFAR_ConfigDb.json` | Generic base configuration, registering all of the devices. | Always | +| `test_environment_ConfigDb.json` | Base delta for the unit- and integration test suites. | Tests & development | +| `stations/simulators_ConfigDb.json` | A "station" configuration that points to our simulators. | Tests & development | +| `stations/dummy_positions_ConfigDb.json` | An antenna configuration, just to have one (it's CS001). | Tests & development | +| `stations/LTS_ConfigDb.json` | The configuration for the Lab Test Station. | Load on LTS | +| `stations/DTS_ConfigDb.json` | The configuration for the Dwingeloo Test Station. | Load on DTS-lab | +| `stations/DTS_Outside_ConfigDb.json` | The configuration for the Dwingeloo Test Station Outside. | Load on DTS-outside | +| `integrations/` | Configurations required by the integration test suites. | Integration tests | diff --git a/README.md b/README.md index e454040c5e624063132c88639a6470f38c355c7a..afa1c4339e31944a78489beb85be46f6bceacab7 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,25 @@ You will also need: * make * bash +## Start dev environment + +For local development a dev environment is needed. To setup this environment run + +``` +./sbin/prepare_dev_env.sh +``` + +This will install `jumppad`, if not present yet as well as creating the docker volume needed to simulate the station +nomad cluster. + +Afterwards run + +``` +jumppad up infra/dev +``` + +to start the dev environment including tango. + ## Bootstrap The bootstrap procedure is needed only once. First we build all docker @@ -115,6 +134,7 @@ Next change the version in the following places: # Release Notes +* 0.22.0 Migrate execution environment to nomad * 0.21.3 Added DigitalBeam.Antenna_Usage_Mask_R to expose antennas used in beamforming * 0.21.2 Removed deprecated "Boot" device (use StationManager now) * 0.21.1 Implement multi project integration downstream pipeline diff --git a/bin/dump_ConfigDb.sh b/bin/dump_ConfigDb.sh index af76d68da825fbe583156f472cc796f7786b4d5c..b58beff1b37b7b8401bcb937e8167840706201c0 100755 --- a/bin/dump_ConfigDb.sh +++ b/bin/dump_ConfigDb.sh @@ -1,9 +1,7 @@ #!/bin/bash -# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +# -# writes the JSON dump to stdout, Do not change -i into -it incompatible with gitlab ci! -docker exec -i dsconfig bash -c ' - python -m dsconfig.dump > /tmp/dsconfig-configdb-dump.json - /manage_object_properties.py -r > /tmp/dsconfig-objectdb-dump.json - /merge_json.py /tmp/dsconfig-objectdb-dump.json /tmp/dsconfig-configdb-dump.json' +"${LOFAR20_DIR}/sbin/dsconfig.sh" --dump diff --git a/bin/itango_console.sh b/bin/itango_console.sh index 20f2c2923da0cb12a6feaca853104418b4ff7026..c577d4592190ed711bfbed73ac03403d4882addc 100755 --- a/bin/itango_console.sh +++ b/bin/itango_console.sh @@ -1,5 +1,7 @@ #!/bin/bash -# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +# -exec docker exec -it itango itango3 +exec docker exec -e "TANGO_HOST=$TANGO_HOST:$TANGO_PORT" -it itango itango3 diff --git a/bin/itango_shell.sh b/bin/itango_shell.sh index 8a8d5e38080bb71d884b43645ce5c892fce0a07e..92e92eda3c5cddce689676c0bcac0efe0d0d441a 100755 --- a/bin/itango_shell.sh +++ b/bin/itango_shell.sh @@ -1,5 +1,7 @@ #!/bin/bash -# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +# -exec docker exec -it itango /bin/bash +exec docker exec -e "TANGO_HOST=$TANGO_HOST:$TANGO_PORT" -it itango /bin/bash diff --git a/deploy/README.md b/deploy/README.md deleted file mode 100644 index 5f49ffc5c340dacf671e607f55c36f68f4db774a..0000000000000000000000000000000000000000 --- a/deploy/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Deployments - -Deploying is achieved through gitlab environments as well as ansible. -Concretely the following files are involved: - -1. [deploy.yml](deploy.yml) -2. [ansible.cfg](ansible.cfg) -3. [.gitlab-ci.yml](../.gitlab-ci.yml) - -Once a tagged commit hits master a job is created for this tag that needs to -be manually triggered. You should be able to find this pipeline on the -[tags overview](https://git.astron.nl/lofar2.0/tango/-/tags). - -From here, with prior consent, the deployment can be started for any -environment. - -The gitlab-ci.yml job defines each of the environments in parameterized way. -The templated `.base_deploy` needs to be extended and have several variables -defined. These variables need to be fed from -[protected variables](https://git.astron.nl/lofar2.0/tango/-/settings/ci_cd) -within gitlab. - -Below is a practical example of how to define an environment. - -``` -.deploy_example_base: - extends: .base_deploy - variables: - DEPLOY_USER: $EXAMPLE_USERNAME - DEPLOY_HOST: $EXAMPLE_HOSTNAME - DEPLOY_CONFIG: $EXAMPLE_CONFIG - DEPLOY_KEY: $EXAMPLE_KEY -deploy_l2ts_start: - extends: .deploy_example_base - environment: - name: example -``` - -Such an environment will only appear in the -[environments overview](https://git.astron.nl/lofar2.0/tango/-/environments) -after the first deployment. - -The process of this deployment is handled through ansible as found in -`deploy.yml`. diff --git a/deploy/ansible.cfg b/deploy/ansible.cfg deleted file mode 100644 index 8da5e9439f11320c5e90d51bbe6ad7d7f8d9798c..0000000000000000000000000000000000000000 --- a/deploy/ansible.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[defaults] -host_key_checking = False -inventory = hosts diff --git a/deploy/deploy.yml b/deploy/deploy.yml deleted file mode 100644 index 455c9da59ea78e724dc448e476df8fffa0ff6522..0000000000000000000000000000000000000000 --- a/deploy/deploy.yml +++ /dev/null @@ -1,106 +0,0 @@ ---- -- name: StationControl Early Deployment - hosts: all - vars: - base_station_config: "LOFAR_ConfigDb.json" - install_path: ~/git/tango - databaseds_port: 10000 - tasks: - - name: Check make installed - include_tasks: - file: tasks/check_binary_install.yml - vars: - program: make - - name: Check git installed - include_tasks: - file: tasks/check_binary_install.yml - vars: - program: git - - name: Register tango directory status - shell: cd {{ install_path }} - args: - chdir: ~ - changed_when: false - failed_when: tango_directory.rc not in [0,1] - register: tango_directory - - name: Register pending changes - shell: "! (git status | grep Changes)" - args: - chdir: "{{ install_path }}" - changed_when: false - failed_when: pending_changes.rc not in [0,1] - register: pending_changes -# Several issues with this variable being unset in ansible 2.2 -# where encountered, keep debug in case needed - - debug: var=pending_changes - - name: Check tango directory status - fail: - msg: "Tango directory appears to be missing!" - when: tango_directory.rc not in [0] - - name: Check pending changes - fail: - msg: "Deployment repository seems to have pending changes!" - when: pending_changes.rc not in [0] - - name: Stop Current Station - changed_when: false - shell: "make stop" - args: - chdir: "{{ install_path }}/docker-compose" - - name: Git Fetch All - changed_when: false - shell: "git fetch --all" - args: - chdir: "{{ install_path }}" - - name: Update Sources - changed_when: false - shell: "git checkout {{ station_version }}" - args: - chdir: "{{ install_path }}" - - name: Pull Images - changed_when: false - shell: "make pull" - args: - chdir: "{{ install_path }}/docker-compose" - - name: Build Images - changed_when: false - shell: "make build" - args: - chdir: "{{ install_path }}/docker-compose" - - name: Start Database - changed_when: false - shell: "make minimal" - args: - chdir: "{{ install_path }}/docker-compose" - - name: Wait for databaseds - ansible.builtin.wait_for: - port: "{{ databaseds_port }}" - delay: 10 - - name: Update Base Database Config - include_tasks: - file: tasks/update_database_config.yml - vars: - database_config: "{{ base_station_config }}" - base_path: "{{ install_path }}" - - name: Verify Base Database Config - include_tasks: - file: tasks/verify_database_config.yml - vars: - database_config: "{{ base_station_config }}" - base_path: "{{ install_path }}" - - name: Update Station Database Config - include_tasks: - file: tasks/update_database_config.yml - vars: - database_config: "stations/{{ station_config }}" - base_path: "{{ install_path }}" - - name: Verify Station Database Config - include_tasks: - file: tasks/verify_database_config.yml - vars: - database_config: "stations/{{ station_config }}" - base_path: "{{ install_path }}" - - name: Start Station - changed_when: false - shell: "make start" - args: - chdir: "{{ install_path }}/docker-compose" diff --git a/deploy/tasks/check_binary_install.yml b/deploy/tasks/check_binary_install.yml deleted file mode 100644 index f114fd87e7b7bd25d7f6b848b25c32aa9324efa2..0000000000000000000000000000000000000000 --- a/deploy/tasks/check_binary_install.yml +++ /dev/null @@ -1,9 +0,0 @@ - - name: Register installation status - command: which {{ program }} - changed_when: false - failed_when: installed.rc not in [0,1] - register: installed - - name: Check installation status - fail: - msg: "{{ program }} does not appear to be installed!" - when: installed.rc not in [0] \ No newline at end of file diff --git a/deploy/tasks/update_database_config.yml b/deploy/tasks/update_database_config.yml deleted file mode 100644 index be321c5769d48a6443de18340033c8cdaffd661f..0000000000000000000000000000000000000000 --- a/deploy/tasks/update_database_config.yml +++ /dev/null @@ -1,5 +0,0 @@ - - name: Update Database Config - changed_when: false - shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }}" - args: - chdir: "{{ base_path }}" \ No newline at end of file diff --git a/deploy/tasks/verify_database_config.yml b/deploy/tasks/verify_database_config.yml deleted file mode 100644 index 1df3681e4eae0fe7d04cede8f5cd3820aa6cbd67..0000000000000000000000000000000000000000 --- a/deploy/tasks/verify_database_config.yml +++ /dev/null @@ -1,9 +0,0 @@ - - name: Get Database Config - changed_when: false - shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }} 2>&1" - register: dsconfig_result - args: - chdir: "{{ base_path }}" - - name: Verify Database Config - when: '"No changes needed" in dsconfig_result.stdout' - debug: msg="Database changes stored" \ No newline at end of file diff --git a/docker-compose/Makefile b/docker-compose/Makefile index 4929af30299e3eea95c6c03c93f05690e6394258..5950a98ec596424a22a235f92a7dda6d22f0ffd7 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -6,19 +6,21 @@ MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST))) BASEDIR := $(notdir $(patsubst %/,%,$(dir $(MAKEPATH)))) TAG ?= latest +DNS ?= 127.0.0.11 DOCKER_COMPOSE ?= docker compose DOCKER_COMPOSE_ENV_FILE := $(abspath .env) COMPOSE_FILES := $(wildcard *.yml) -COMPOSE_FILE_ARGS := --env-file $(DOCKER_COMPOSE_ENV_FILE) $(foreach yml,$(COMPOSE_FILES),-f $(yml)) - ATTACH_COMPOSE_FILE_ARGS := $(foreach yml,$(filter-out tango.yml,$(COMPOSE_FILES)),-f $(yml)) +BUILD_ONLY_FILE_ARGS := -f tango.yml +COMPOSE_FILE_ARGS := --env-file $(DOCKER_COMPOSE_ENV_FILE) $(ATTACH_COMPOSE_FILE_ARGS) + -# The default Docker network mode is tangonet. The "host" network +# The default Docker network mode is station. The "host" network # mode will make the tangodb and archiverdb ports clash, # But we allow to overwrite it. -NETWORK_MODE ?= tangonet +NETWORK_MODE ?= station # Timeout used to await services to become healthy TIMEOUT ?= 300 @@ -97,11 +99,6 @@ endif # prevent collisions with jobs from the same project running at the same # time. # -ifneq ($(CI_JOB_ID),) - NETWORK_MODE := tangonet-$(CI_JOB_ID) -else - $(info Network mode cannot be host for the archiver! It won't work unless you set the env var CI_JOB_ID=local) -endif ifeq ($(OS),Windows_NT) $(error Sorry, Windows is not supported yet) @@ -166,14 +163,15 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \ CONTAINER_EXECUTION_UID=$(shell id -u) \ DOCKER_GID=$(DOCKER_GID) \ TEST_MODULE=$(INTEGRATION_MODULE) \ - TAG=$(TAG) + TAG=$(TAG) \ + DNS=$(DNS) .PHONY: up base base-nocache down minimal context run integration start stop restart build build-nocache status clean pull help await .DEFAULT_GOAL := help pull: ## pull the images from the Docker hub - $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull --ignore-pull-failures + $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) pull --quiet-pull --ignore-pull-failures base: context ## Build base lofar device image ifneq ($(NO_BASE),1) @@ -181,15 +179,15 @@ ifneq ($(NO_BASE),1) endif base-nocache: context ## Rebuild base lofar device image - $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain lofar-device-base + $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) $(BUILD_ONLY_FILE_ARGS) build --no-cache --progress=plain lofar-device-base build: base ## build images # docker-compose does not support build dependencies, so manage those here - $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --parallel --progress=plain $(SERVICE) + $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) $(BUILD_ONLY_FILE_ARGS) build --parallel --progress=plain $(SERVICE) build-nocache: base-nocache ## rebuild images from scratch # docker-compose does not support build dependencies, so manage those here - $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --no-cache --progress=plain $(SERVICE) + $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) $(BUILD_ONLY_FILE_ARGS) build --no-cache --progress=plain $(SERVICE) up: base minimal ## start the base TANGO system and prepare requested services $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) up --no-start --no-recreate $(SERVICE) @@ -219,7 +217,7 @@ ifneq ($(NETWORK_MODE),host) docker network inspect 9000-$(NETWORK_MODE) &> /dev/null || ([ $$? -ne 0 ] && docker network create 9000-$(NETWORK_MODE) -o com.docker.network.driver.mtu=9000) endif - $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) -f tango.yml -f networks.yml up --no-recreate -d + context: ## Move and build the necessary files to create minimal docker context rm -rf ./tmp; \ @@ -236,8 +234,8 @@ context: ## Move and build the necessary files to create minimal docker context bootstrap: pull build # first start, initialise from scratch $(MAKE) start dsconfig # boot up containers to load configurations sleep 5 # wait for dsconfig container to come up - ../sbin/update_ConfigDb.sh ../CDB/LOFAR_ConfigDb.json # load default configuration - ../sbin/update_ConfigDb.sh ../CDB/stations/simulators_ConfigDb.json # by default, use simulators + ../sbin/dsconfig.sh --update ../CDB/LOFAR_ConfigDb.json # load default configuration + ../sbin/dsconfig.sh --update ../CDB/stations/simulators_ConfigDb.json # by default, use simulators start: up ## start a service (usage: make start <servicename>) if [ $(UNAME_S) = Linux ]; then touch ~/.Xauthority; chmod a+r ~/.Xauthority; fi @@ -266,16 +264,18 @@ await: ## Await every container with total max timeout of 300, do not reset tim if [ -z "$${service_has_health}" ]; then \ continue; \ fi; \ + echo -n "Whait for service $${i} to become healthy .."; \ while [ "$$(docker inspect -f '{{.State.Health.Status}}' $${current_service})" != "healthy" ] ; do \ - sleep 1; \ + echo -n '.'; \ + sleep 2; \ time=$$(expr $$time + 1); \ if [ $${time} -gt $(TIMEOUT) ]; then \ - echo "Timeout reached waiting for $${i} to become healthy"; \ + echo "timeout"; \ docker logs $${i}; \ exit 1; \ fi; \ done; \ - echo "Service $${i} is healthy"; \ + echo ". [ok]"; \ done status: ## show the container status diff --git a/docker-compose/device-antennafield.yml b/docker-compose/device-antennafield.yml index 705e43b517718da08e682e72e2ebdaed3504310c..8470975a049b7fb828b61847fba4b874d6a3f152 100644 --- a/docker-compose/device-antennafield.yml +++ b/docker-compose/device-antennafield.yml @@ -27,6 +27,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5715:5715" # unique port for this DS - "5815:5815" # ZeroMQ event port diff --git a/docker-compose/device-aps.yml b/docker-compose/device-aps.yml index 23690af220e129e34730c6bcd62c95d98520e937..292dce5712baf1fa404701ee66be5f48f18bbb13 100644 --- a/docker-compose/device-aps.yml +++ b/docker-compose/device-aps.yml @@ -18,6 +18,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5728:5728" # unique port for this DS - "5828:5828" # ZeroMQ event port diff --git a/docker-compose/device-apsct.yml b/docker-compose/device-apsct.yml index 86c24b6a69b8462d61fe480fb8c39083c982c130..a8a0e946568950ef3d9aa83979c75673749852dd 100644 --- a/docker-compose/device-apsct.yml +++ b/docker-compose/device-apsct.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5709:5709" # unique port for this DS - "5809:5809" # ZeroMQ event port diff --git a/docker-compose/device-apspu.yml b/docker-compose/device-apspu.yml index ac585e6c480c957546d55b6c95ff7852e026ca2a..89f2ac5c3d20e218c70b4762d09537956cb03541 100644 --- a/docker-compose/device-apspu.yml +++ b/docker-compose/device-apspu.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5710:5710" # unique port for this DS - "5810:5810" # ZeroMQ event port diff --git a/docker-compose/device-beamlet.yml b/docker-compose/device-beamlet.yml index 7c734e0e3ed33c6ec2e25fdeb6df91b3dead4f96..f7a234ed125fc3741c892f6e7828d212cb864b29 100644 --- a/docker-compose/device-beamlet.yml +++ b/docker-compose/device-beamlet.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5712:5712" # unique port for this DS - "5812:5812" # ZeroMQ event port diff --git a/docker-compose/device-bst.yml b/docker-compose/device-bst.yml index e1fd186004c2d6016751f08778306d66a530eb3f..0090acf8b3e07420fdc8e9794a02501a8a1feb52 100644 --- a/docker-compose/device-bst.yml +++ b/docker-compose/device-bst.yml @@ -27,6 +27,7 @@ services: networks: - control - data + dns: ${DNS} ports: - "5003:5003/udp" # port to receive SST UDP packets on (first antennafield) - "5103:5103/tcp" # port to emit SST TCP packets on diff --git a/docker-compose/device-ccd.yml b/docker-compose/device-ccd.yml index 950800e33372a84d3d1f5204cb8aaea1bf3b8d99..a43f0df86da9c3ef8096cc229b380526a08873e7 100644 --- a/docker-compose/device-ccd.yml +++ b/docker-compose/device-ccd.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5721:5721" # unique port for this DS - "5821:5821" # ZeroMQ event port diff --git a/docker-compose/device-configuration.yml b/docker-compose/device-configuration.yml index 1a923f15b33585c8f2c397e7ac0afa5855ac464f..8bd1a17c23652d66ea75314f7f87cd38c1549a59 100644 --- a/docker-compose/device-configuration.yml +++ b/docker-compose/device-configuration.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5722:5722" # unique port for this DS - "5822:5822" # ZeroMQ event port diff --git a/docker-compose/device-digitalbeam.yml b/docker-compose/device-digitalbeam.yml index b093ee63e050f20205b6b2197fbe87f45daf92d6..f6b42d37fd53529e8394b5936e7e33612963891a 100644 --- a/docker-compose/device-digitalbeam.yml +++ b/docker-compose/device-digitalbeam.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5713:5713" # unique port for this DS - "5813:5813" # ZeroMQ event port diff --git a/docker-compose/device-docker.yml b/docker-compose/device-docker.yml index a9080eeb5263437fe42d22e91dc6c2509f213596..fd5102f7fb68b6d51fb80c2e68666c21a2242d90 100644 --- a/docker-compose/device-docker.yml +++ b/docker-compose/device-docker.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5705:5705" # unique port for this DS - "5805:5805" # ZeroMQ event port diff --git a/docker-compose/device-ec.yml b/docker-compose/device-ec.yml index f944c751acb3ae72cba986a63b4f7e5816d68e04..2d4020000844361945a674a810a8b1887e9d237c 100644 --- a/docker-compose/device-ec.yml +++ b/docker-compose/device-ec.yml @@ -21,6 +21,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5729:5729" # unique port for this DS - "5829:5829" # ZeroMQ event port diff --git a/docker-compose/device-observation-control.yml b/docker-compose/device-observation-control.yml index b5b7f9fe50defba4487766a576e6213e6fb4ac66..e47729037c63342367392dbe09da3859512c4d43 100644 --- a/docker-compose/device-observation-control.yml +++ b/docker-compose/device-observation-control.yml @@ -25,6 +25,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5703:5703" # unique port for this DS - "5803:5803" # ZeroMQ event port diff --git a/docker-compose/device-pcon.yml b/docker-compose/device-pcon.yml index e833bb55d4e35eb842cf91a42897dcf1ceb6e2aa..e479a5bf74bf9ca2067d13243c1d3ebcf95772d7 100644 --- a/docker-compose/device-pcon.yml +++ b/docker-compose/device-pcon.yml @@ -21,6 +21,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5720:5720" # unique port for this DS - "5820:5820" # ZeroMQ event port diff --git a/docker-compose/device-psoc.yml b/docker-compose/device-psoc.yml index 998a0ad2fda23aab5232ebd40632392fad353f08..e0efad7e58c089013feb4a76901271e302178f00 100644 --- a/docker-compose/device-psoc.yml +++ b/docker-compose/device-psoc.yml @@ -21,6 +21,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5719:5719" # unique port for this DS - "5819:5819" # ZeroMQ event port diff --git a/docker-compose/device-recvh.yml b/docker-compose/device-recvh.yml index feb0ce875303bc84aec7c6eec1465d45958e7685..28a17dab70481c4d49237edd6d8fc8104805deaf 100644 --- a/docker-compose/device-recvh.yml +++ b/docker-compose/device-recvh.yml @@ -18,6 +18,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5725:5725" # unique port for this DS - "5825:5825" # ZeroMQ event port diff --git a/docker-compose/device-recvl.yml b/docker-compose/device-recvl.yml index 810505463033f32f2afcc4a029a8ecb3f106c0c5..1c959a3cedb2e78e32d7cbbf33406f5c23fddd21 100644 --- a/docker-compose/device-recvl.yml +++ b/docker-compose/device-recvl.yml @@ -18,6 +18,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5726:5726" # unique port for this DS - "5826:5826" # ZeroMQ event port diff --git a/docker-compose/device-sdp.yml b/docker-compose/device-sdp.yml index 02ce3f99ebc945f93ec340e7da14a2e318f41608..50ad319a1b5d9c7564c49005c82689375b04aa17 100644 --- a/docker-compose/device-sdp.yml +++ b/docker-compose/device-sdp.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5701:5701" # unique port for this DS - "5801:5801" # ZeroMQ event port diff --git a/docker-compose/device-sdpfirmware.yml b/docker-compose/device-sdpfirmware.yml index 5fd34bf0f42bfc3df1ae14e7d50134d1f0faa62b..926e0b23552a766f15f1fadbec4699a0d8ebe123 100644 --- a/docker-compose/device-sdpfirmware.yml +++ b/docker-compose/device-sdpfirmware.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5727:5727" # unique port for this DS - "5827:5827" # ZeroMQ event port diff --git a/docker-compose/device-sst.yml b/docker-compose/device-sst.yml index a6ba4f2b2dc7b32f328cd137e1d8e50159dc58d9..a0b5d60c05f6ee920bc1204d7c006c97aef90d7f 100644 --- a/docker-compose/device-sst.yml +++ b/docker-compose/device-sst.yml @@ -27,6 +27,7 @@ services: networks: - control - data + dns: ${DNS} ports: - "5001:5001/udp" # port to receive SST UDP packets on (first antennafield) - "5101:5101/tcp" # port to emit SST TCP packets on diff --git a/docker-compose/device-station-manager.yml b/docker-compose/device-station-manager.yml index 044de2d17f7cf878a8e4449cb3029bae71d09197..50fa4c7aa04d878c441a1c0433ab08380b8a41a3 100644 --- a/docker-compose/device-station-manager.yml +++ b/docker-compose/device-station-manager.yml @@ -21,6 +21,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5723:5723" # unique port for this DS - "5823:5823" # ZeroMQ event port diff --git a/docker-compose/device-temperature-manager.yml b/docker-compose/device-temperature-manager.yml index 35e2defff2f45a67ddc42f632dc4565536617fe8..7b2624c13e0b0753c7386217152ab6daefcb63ed 100644 --- a/docker-compose/device-temperature-manager.yml +++ b/docker-compose/device-temperature-manager.yml @@ -21,6 +21,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5716:5716" # unique port for this DS - "5816:5816" # ZeroMQ event port diff --git a/docker-compose/device-tilebeam.yml b/docker-compose/device-tilebeam.yml index c67cb289cabfec8f73c0cb586e8633ca401b8489..65a39c153ef859be4e3258a5a5c5b3e2dbcac956 100644 --- a/docker-compose/device-tilebeam.yml +++ b/docker-compose/device-tilebeam.yml @@ -21,6 +21,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5711:5711" # unique port for this DS - "5811:5811" # ZeroMQ event port diff --git a/docker-compose/device-unb2.yml b/docker-compose/device-unb2.yml index 82ef3846066fd78efe7eae2c26f15ddbd2d62d31..968aad48ea1db99aa294e11e459036a73c4326ee 100644 --- a/docker-compose/device-unb2.yml +++ b/docker-compose/device-unb2.yml @@ -26,6 +26,7 @@ services: max-file: "10" networks: - control + dns: ${DNS} ports: - "5704:5704" # unique port for this DS - "5804:5804" # ZeroMQ event port diff --git a/docker-compose/device-xst.yml b/docker-compose/device-xst.yml index 0890e91d748ebf14754a51b5b927a38987716084..a8d9d83e2ec7476c943c1749a9f51775e3560cbc 100644 --- a/docker-compose/device-xst.yml +++ b/docker-compose/device-xst.yml @@ -27,6 +27,7 @@ services: networks: - control - data + dns: ${DNS} ports: - "5002:5002/udp" # port to receive XST UDP packets on (first antennafield) - "5102:5102/tcp" # port to emit XST TCP packets on diff --git a/docker-compose/integration-test.yml b/docker-compose/integration-test.yml index 8b2cc0a33f72865ee65c0cd66d5bc1af036fe8f2..d31670e157af42391c12f8e8b8393d608da18f75 100644 --- a/docker-compose/integration-test.yml +++ b/docker-compose/integration-test.yml @@ -20,6 +20,7 @@ services: container_name: integration-test networks: - control + dns: ${DNS} extra_hosts: - "host.docker.internal:host-gateway" volumes: diff --git a/docker-compose/tango-prometheus-exporter.yml b/docker-compose/tango-prometheus-exporter.yml index 14b66f988d36fd6377e894e02108c5ad37573b8d..69e0df046b7ed01625e28376c9516d1da840d146 100644 --- a/docker-compose/tango-prometheus-exporter.yml +++ b/docker-compose/tango-prometheus-exporter.yml @@ -28,8 +28,6 @@ services: - TANGO_HOST=${TANGO_HOST} ports: - "8000:8000" - depends_on: - - databaseds restart: unless-stopped tango-prometheus-fast-exporter: @@ -54,8 +52,6 @@ services: - TANGO_HOST=${TANGO_HOST} ports: - "8001:8000" - depends_on: - - databaseds restart: unless-stopped tango-prometheus-slow-exporter: @@ -80,6 +76,4 @@ services: - TANGO_HOST=${TANGO_HOST} ports: - "8002:8000" - depends_on: - - databaseds restart: unless-stopped diff --git a/docker-compose/tango.yml b/docker-compose/tango.yml index 9de7d9e1dc6d594b21ea967266c383886e82970e..c4a2827d97d6597df9f0a55b38c90c6dfe361cc4 100644 --- a/docker-compose/tango.yml +++ b/docker-compose/tango.yml @@ -75,6 +75,7 @@ services: restart: unless-stopped dsconfig: + image: ${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/dsconfig:${TAG} build: context: dsconfig args: @@ -83,8 +84,6 @@ services: container_name: dsconfig networks: - control - depends_on: - - databaseds environment: - TANGO_HOST=${TANGO_HOST} command: > diff --git a/infra/dev/config/nomad/client.hcl b/infra/dev/config/nomad/client.hcl new file mode 100644 index 0000000000000000000000000000000000000000..0989925d182a0e7e35129c65d8ff09899a521dff --- /dev/null +++ b/infra/dev/config/nomad/client.hcl @@ -0,0 +1,34 @@ +plugin "docker" { + config { + allow_privileged = true + volumes { + enabled = true + selinuxlabel = "z" + } + } +} + +client { + host_volume "tango-database" { + path = "/localdata/volumes/tango-database" + } + options = { + "driver.allowlist" = "docker,exec" + } +} +consul { + address = "localhost:8500" + server_service_name = "nomad" + client_service_name = "nomad-client" + auto_advertise = true + server_auto_join = true + client_auto_join = true + checks_use_advertise = true +} + +advertise { + + http = "{{ GetInterfaceIP \"eth0\" }}" + rpc = "{{ GetInterfaceIP \"eth0\" }}" + serf = "{{ GetInterfaceIP \"eth0\" }}" +} diff --git a/infra/dev/config/nomad/consul.hcl b/infra/dev/config/nomad/consul.hcl new file mode 100644 index 0000000000000000000000000000000000000000..512ae3b7188bf969d7e1541f2836da3fc3e7628d --- /dev/null +++ b/infra/dev/config/nomad/consul.hcl @@ -0,0 +1,21 @@ +data_dir = "/tmp/" +log_level = "trace" + +server = false +datacenter = "dev-stat" + +bind_addr = "0.0.0.0" +advertise_addr = "{{GetInterfaceIP \"eth0\"}}" + +ports { + grpc = 8502 + grpc_tls = 8503 +} + +connect { + enabled = true +} + +retry_join = ["192.168.123.100"] +skip_leave_on_interrupt = true +leave_on_terminate = false diff --git a/infra/dev/jobs/.keep b/infra/dev/jobs/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/infra/dev/jobs/station/.keep b/infra/dev/jobs/station/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/infra/dev/main.hcl b/infra/dev/main.hcl new file mode 100644 index 0000000000000000000000000000000000000000..42920543b5271a846b9f2c441d5e68fff1ac88d7 --- /dev/null +++ b/infra/dev/main.hcl @@ -0,0 +1,120 @@ +variable "host_volume" { + default = "dev_nomad_station" +} + +resource "network" "station" { + subnet = "192.168.123.0/24" +} + +resource "template" "consul_config" { + + source = <<-EOF + data_dir = "/consul/data/" + log_level = "trace" + + + datacenter = "dev-stat" + + server = true + + bootstrap_expect = 1 + ui = true + + bind_addr = "0.0.0.0" + client_addr = "127.0.0.1 {{GetInterfaceIP \"eth0\"}}" + advertise_addr = "{{GetInterfaceIP \"eth0\"}}" + + ports { + grpc = 8502 + grpc_tls = 8503 + dns = 53 + } + connect { + enabled = true + } + + auto_encrypt { + allow_tls = true + } + skip_leave_on_interrupt = true + leave_on_terminate = false + recursors = ["127.0.0.11"] + EOF + + destination = "./tmp/consul.hcl" +} + + +resource "container" "consul" { + depends_on = ["resource.template.consul_config"] + network { + id = resource.network.station.id + ip_address = "192.168.123.100" + } + + image { + name = "hashicorp/consul:latest" + } + + command = [ + "consul", + "agent", + "-config-file=/consul/config/config.hcl" + ] + + environment = { + CONSUL_HTTP_ADDR = "http://localhost:8500" + } + + volume { + source = resource.template.consul_config.destination + destination = "/consul/config/config.hcl" + type = "bind" + } + + port { + local = 8500 + remote = 8500 + host = 18500 + } + + port { + local = 53 + host = 8600 + remote = 53 + protocol = "udp" + } + privileged = false +} + +resource "nomad_cluster" "station" { + depends_on = ["resource.container.consul"] + client_nodes = 1 + client_config = "./config/nomad/client.hcl" + consul_config = "./config/nomad/consul.hcl" + datacenter = "stat" + + network { + id = resource.network.station.id + } + environment = { + NO_PROXY = "git.astron.nl:5000" + } + + volume { + source = "${variable.host_volume}" + destination = "/localdata" + type = "volume" + } +} + +resource "nomad_job" "tango" { + cluster = resource.nomad_cluster.station + + paths = ["./jobs/station/tango.nomad"] + + health_check { + timeout = "300s" + jobs = ["tango"] + } +} diff --git a/infra/dev/tmp/.keep b/infra/dev/tmp/.keep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/infra/env.yaml b/infra/env.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42911253929c26c5e4e0d42cb457f7ba6cb8e6f --- /dev/null +++ b/infra/env.yaml @@ -0,0 +1,5 @@ +tango: + db: + version: 11.0.2 + databaseds: + version: 5.22.0 diff --git a/infra/jobs/Makefile b/infra/jobs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9300046e1e52f20d73aaf809a99ad9b740c00b9a --- /dev/null +++ b/infra/jobs/Makefile @@ -0,0 +1,12 @@ + + +DIR_OUT ?= . + +SUBDIRS := $(wildcard */.) + +.PHONY: render $(SUBDIRS) + +render: $(SUBDIRS) + +$(SUBDIRS): + $(MAKE) -C $@ diff --git a/infra/jobs/station/Makefile b/infra/jobs/station/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a64b4a40045c56f78af0dc6e10e8bb856904dca3 --- /dev/null +++ b/infra/jobs/station/Makefile @@ -0,0 +1,16 @@ + +DIR_OUT ?= . +DIR_SRC += . +SRC_JOBS += $(wildcard $(addsuffix /*.levant.nomad, $(DIR_SRC))) +JOBS := $(patsubst %.levant.nomad, %.nomad, $(SRC_JOBS)) +ENV ?= ../../env.yaml + +.PHONY: render + +render: pull $(JOBS) + +pull: + docker pull -q hashicorp/levant + +%.nomad: %.levant.nomad + docker run --rm -v $(realpath $(ENV)):/env.yaml -v $(DIR_SRC):/in -v $(realpath $(DIR_OUT)):/out hashicorp/levant render -var-file=/env.yaml -out=/out/$@ /in/$< diff --git a/infra/jobs/station/tango.levant.nomad b/infra/jobs/station/tango.levant.nomad new file mode 100644 index 0000000000000000000000000000000000000000..36153022c01529368bbbdd50c284e6bfd7dac28c --- /dev/null +++ b/infra/jobs/station/tango.levant.nomad @@ -0,0 +1,164 @@ +job "tango" { + datacenters = ["stat"] + type = "service" + + group "database" { + count = 1 + + restart { + attempts = 10 + interval = "5m" + delay = "25s" + mode = "delay" + } + + network { + mode = "bridge" + port "mysql" { + # should be migrated to port 3000 when fully replaces docker-compose setup + static = 3306 + to = 3306 + } + } + + volume "database" { + type = "host" + read_only = false + source = "tango-database" + } + + service { + name = "tangodb" + port = "mysql" + + check { + type = "tcp" + interval = "10s" + timeout = "2s" + } + } + + task "database" { + driver = "docker" + + volume_mount { + volume = "database" + destination = "/var/lib/mysql" + read_only = false + } + + config { + image = "git.astron.nl:5000/lofar2.0/tango/tango-db:[[.tango.db.version]]" + ports = ["mysql"] + } + + env { + MYSQL_ROOT_PASSWORD = "secret" + MYSQL_DATABASE = "tango" + MYSQL_USER = "tango" + MYSQL_PASSWORD = "tango" + } + + resources { + cpu = 1024 + memory = 512 + } + } + } + + group "device-server" { + count = 1 + + restart { + attempts = 10 + interval = "10m" + delay = "60s" + mode = "delay" + } + + network { + mode = "bridge" + port "tango" { + static = 10000 + to = 10000 + } + } + + service { + name = "tango" + port = "tango" + + check { + type = "tcp" + interval = "10s" + timeout = "20s" + } + } + + task "wait-for-db" { + lifecycle { + hook = "prestart" + sidecar = false + } + driver = "docker" + + config { + image = "busybox" + command = "sh" + args = ["-c", "while ! nc -z $MYSQL_HOST $MYSQL_PORT; do sleep 1; done"] + } + + template { + data = <<EOH + {{ range service "tangodb" }} + MYSQL_HOST = "{{ .Address }}" + MYSQL_PORT = "{{ .Port }}" + {{ end }} + EOH + destination = "local/env.txt" + env = true + } + } + + task "database-ds" { + driver = "docker" + + + config { + image = "git.astron.nl:5000/lofar2.0/tango/tango-databaseds:[[.tango.databaseds.version]]" + ports = ["tango"] + entrypoint = [ + "/usr/local/bin/DataBaseds", + "2", + "-ORBendPoint", + "giop:tcp:0.0.0.0:10000", + "-ORBendPointPublish", + "giop:tcp:tango.service.consul:10000" + ] + } + + env { + MYSQL_DATABASE = "tango" + MYSQL_USER = "tango" + MYSQL_PASSWORD = "tango" + TANGO_HOST = "tango.service.consul:10000" + } + + template { + data = <<EOH + {{ range service "tangodb" }} + MYSQL_HOST = "{{ .Address }}:{{ .Port }}" + {{ end }} + EOH + destination = "local/env.txt" + env = true + } + + + resources { + cpu = 1024 + memory = 512 + } + } + } +} diff --git a/infra/station-config/nomad.hcl b/infra/station-config/nomad.hcl index 709db4bf0ce3117ea3b336784befc63f5f9f7407..b0dd76c492868b00d7e95f6546cb9a8efc732486 100644 --- a/infra/station-config/nomad.hcl +++ b/infra/station-config/nomad.hcl @@ -30,6 +30,10 @@ client { host_volume "monitoring-prometheus-data" { path = "/localdata/volumes/monitoring-prometheus-data" } + + host_volume "tango-database" { + path = "/localdata/volumes/tango-database" + } } telemetry { diff --git a/sbin/dsconfig.sh b/sbin/dsconfig.sh new file mode 100755 index 0000000000000000000000000000000000000000..a51b6c61351ea78b968a0c8e882ac44dd107955c --- /dev/null +++ b/sbin/dsconfig.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 +# + +# defaults +network="station" + +# list of arguments expected in the input +optstring_long="help,load,update,dump,file,network::" +optstring="hludf:n::" + +options=$(getopt -l ${optstring_long} -o ${optstring} -- "$@") + +eval set -- "$options" + + +while true; do + case ${1} in + -h|--help) + usage + exit 0 + ;; + -l|--load) + echo "Load config" + action="load" + ;; + -u|--update) + echo "Update config" + action="update" + ;; + -d|--dump) + echo "Dump config" + action="dump" + ;; + -n|--network) + shift + network="$1" + ;; + --) + shift + break;; + esac + shift +done + +if [ -z "$TAG" ]; then + TAG="latest" +fi + +echo "Use docker network '$network'" +docker_args=(run --rm -e "TANGO_HOST=$TANGO_HOST" --network="$network" -i) +docker_image="git.astron.nl:5000/lofar2.0/tango/dsconfig:$TAG" + +if [[ -n "$DNS" ]]; then + echo "Set docker dns to $DNS" + docker_args+=(--dns "$DNS") +fi + +if [[ "$action" != "dump" ]]; then + + if [ ${#} -eq 1 ]; then + file=$(realpath "${1}") + else + echo "A file name must be provided." + exit 1 + fi + + cmd_args=(--write) + docker_args+=(-v "${file}":/tmp/dsconfig-update-settings.json) + + if [[ "$action" == "update" ]]; then + "${LOFAR20_DIR}/sbin/dsconfig.sh" --dump > "${LOFAR20_DIR}"/CDB/dump_"$(date "+%Y.%m.%d_%H.%M.%S")".json + + # update settings, Do not change -i into -it this will break integration tests in gitlab ci! + docker "${docker_args[@]}" "$docker_image" \ + sh -c '/manage_object_properties.py --write < /tmp/dsconfig-update-settings.json' + cmd_args+=(--update) + fi + + + # update settings, Do not change -i into -it this will break integration tests in gitlab ci! + docker "${docker_args[@]}" "$docker_image" \ + json2tango "${cmd_args[@]}" /tmp/dsconfig-update-settings.json + + # somehow json2tango does not return 0 on success + exit 0 +fi + +# writes the JSON dump to stdout, Do not change -i into -it incompatible with gitlab ci! +docker "${docker_args[@]}" "$docker_image" \ + bash -c ' + python -m dsconfig.dump > /tmp/dsconfig-configdb-dump.json + /manage_object_properties.py -r > /tmp/dsconfig-objectdb-dump.json + /merge_json.py /tmp/dsconfig-objectdb-dump.json /tmp/dsconfig-configdb-dump.json' diff --git a/sbin/load_ConfigDb.sh b/sbin/load_ConfigDb.sh index 707eee508b7dd0019775037e95b6a31e0a41d00b..cc7c8b254f5cd4f56e9bf71fa2fa116b703c405c 100755 --- a/sbin/load_ConfigDb.sh +++ b/sbin/load_ConfigDb.sh @@ -1,20 +1,7 @@ #!/bin/bash -# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +# -if [ ${#} -eq 1 ]; then - file=${1} -else - echo "A file name must be provided." - exit 1 -fi - -# copy file into container to read it from container, as the file's location -# in the container won't be the same as on the host. -docker cp "${file}" dsconfig:/tmp/dsconfig-update-settings.json || exit 1 - -# update settings, Do not change -i into -it this will break integration tests in gitlab ci! -docker exec -i dsconfig json2tango --write /tmp/dsconfig-update-settings.json - -# somehow json2tango does not return 0 on success -exit 0 +"${LOFAR20_DIR}/sbin/dsconfig.sh" --load "${1}" diff --git a/sbin/prepare_dev_env.sh b/sbin/prepare_dev_env.sh new file mode 100755 index 0000000000000000000000000000000000000000..403e41aaa8db7f2d41cccfe89d6cc12b990a2f73 --- /dev/null +++ b/sbin/prepare_dev_env.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 +# + + +if [ -z "$LOFAR20_DIR" ]; then + # We assume we aren't in the PATH, so we can derive our path. + # We need our parent directory. + LOFAR20_DIR_RELATIVE=$(dirname "$0")/.. + + # As an absolute path + LOFAR20_DIR=$(readlink -f "${LOFAR20_DIR_RELATIVE}") +fi + +if ! [ -x "$(command -v jumppad)" ]; then + echo 'Install jumppad' + mkdir -p "$LOFAR20_DIR/bin" + wget https://git.astron.nl/lofar2.0/tango/-/package_files/37347/download -O "$LOFAR20_DIR/bin/jumppad" + chmod +x "$LOFAR20_DIR/bin/jumppad" + PATH="$PATH:$LOFAR20_DIR/bin" +fi + + +if [ "$(docker volume list | grep -c dev_nomad_station)" = "0" ]; then + docker volume create "dev_nomad_station" + docker pull -q bash + docker run -i --rm -v dev_nomad_station:/mnt bash bash -c 'mkdir -p /mnt/volumes/tango-database' +fi diff --git a/sbin/run_integration_test.sh b/sbin/run_integration_test.sh index 2c4af601e1e46f8aff974b455f7dc1de8a7d50d0..600a5b219fdda78938ba93908537d5aeeca4eecf 100755 --- a/sbin/run_integration_test.sh +++ b/sbin/run_integration_test.sh @@ -4,6 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 # +# Url to the jumppad download location +jumppad_download="https://git.astron.nl/lofar2.0/tango/-/package_files/37347/download" + # Usage function explains how parameters are parsed function usage { echo "./$(basename "$0") @@ -26,7 +29,7 @@ function integration_test { IFS=" " read -r -a configs <<< "${3}" for config in "${configs[@]}"; do echo "Updating config ${config} ..." - bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${config}" + bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${config}" done if [ -n "${2+x}" ]; then # shellcheck disable=SC2145 @@ -42,7 +45,7 @@ function integration_test { } # list of arguments expected in the input -optstring_long="help,no-build,skip-tests" +optstring_long="help,no-build,skip-tests,preserve,save-logs" optstring="hnb" options=$(getopt -l ${optstring_long} -o ${optstring} -- "$@") @@ -82,23 +85,86 @@ fi export TANGO_SKIP_BUILD=1 -cd "$LOFAR20_DIR/docker-compose" || exit 1 +if [ -z "$TAG" ]; then + export TAG="latest" +fi -# Start the database server first, `-z ${y+x}` is the inverse of `-n ${y}` +# Build dsconfig first, `-z ${y+x}` is the inverse of `-n ${y}` if [ -z "${no_build+x}" ]; then rm -rf "${LOFAR20_DIR}/tangostationcontrol/dist" cd "${LOFAR20_DIR}/tangostationcontrol" || exit 1 tox -e build cd "$LOFAR20_DIR/docker-compose" || exit 1 - make build databaseds dsconfig + make build dsconfig +fi + +docker network rm station || true + +if ! [ -x "$(command -v jumppad)" ]; then + echo 'Install jumppad' + mkdir -p ./.bin + wget "${jumppad_download}" -O ./.bin/jumppad + chmod +x ./.bin/jumppad + bin_dir=$(realpath ./.bin) + PATH="$PATH:$bin_dir" fi -make start databaseds dsconfig +make -C infra/jobs/station DIR_OUT="$( realpath "infra/dev/jobs/station")" render + +# prepare a docker volume for nomad +tmp_volume="test_$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)" + +function cleanup { + cd "$LOFAR20_DIR" + if [ -n "${save_logs}" ]; then + mkdir -p log + for container in $(docker ps -a --format "{{.Names}}") + do + echo "Saving log for container $container" + docker logs "${container}" >& "log/${container}.log" + done + bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --dump >& log/dump_ConfigDb.log + fi + if [ -z "${preserve}" ]; then + make stop > /dev/null 2>&1 + HOME="$JUMPPAD_HOME" jumppad down infra/dev + docker volume rm "$tmp_volume" || true + fi +} + +trap cleanup EXIT + +docker volume create "$tmp_volume" + +docker pull -q bash +docker run -i --rm -v "$tmp_volume":/mnt bash bash -c 'mkdir -p /mnt/volumes/tango-database' -# Give dsconfig and databaseds time to start -sleep 5 # dsconfig container must be up and running... -# shellcheck disable=SC2016 -echo '/usr/local/bin/wait-for-it.sh ${TANGO_HOST} --strict --timeout=300 -- true' | make run dsconfig bash - +rm -rf ~/.jumppad/ + +if [ -z "$JUMPPAD_HOME" ]; then + JUMPPAD_HOME="$HOME" +fi + +HOME="$JUMPPAD_HOME" jumppad up --var="host_volume=$tmp_volume" infra/dev || true + +echo -n "Waiting for tango service to become available .." +until [[ $(dig @127.0.0.1 -p 8600 tango.service.consul +short) ]]; do + sleep 2 + echo -n "." +done +echo ". [ok]" + +TANGO_PORT=$(dig @127.0.0.1 -p 8600 tango.service.consul SRV +short | awk '{printf "%s",$3}') +TANGO_HOST=$(dig @127.0.0.1 -p 8600 tango.service.consul +short) + +export TANGO_HOST="$TANGO_HOST:$TANGO_PORT" + +export HOSTNAME=192.168.123.1 +export DNS=192.168.123.100 + +echo "Using tango host $TANGO_HOST" + +cd "$LOFAR20_DIR/docker-compose" || exit 1 # Devices list is used to explitly word split when supplied to commands, must # disable shellcheck SC2086 for each case. @@ -124,14 +190,16 @@ make start logstash http-json-schemas object-storage init-object-storage # Update the dsconfig # Do not remove `bash`, otherwise statement ignored by gitlab ci shell! -bash "${LOFAR20_DIR}"/sbin/load_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/common.json -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/l0.json -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/l1.json -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/lba.json -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/h0.json -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/hba_core.json -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/cs001.json -bash "${LOFAR20_DIR}"/sbin/update_ConfigDb.sh "${LOFAR20_DIR}"/CDB/stations/testenv_cs001.json +docker pull -q "git.astron.nl:5000/lofar2.0/tango/dsconfig:$TAG" + +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --load "${LOFAR20_DIR}"/CDB/stations/common.json +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/l0.json +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/l1.json +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/lba.json +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/h0.json +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/hba_core.json +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/cs001.json +bash "${LOFAR20_DIR}"/sbin/dsconfig.sh --update "${LOFAR20_DIR}"/CDB/stations/testenv_cs001.json cd "$LOFAR20_DIR/docker-compose" || exit 1 make start "${SIMULATORS[@]}" diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index 63daae8ef2650ef9b0a4264744f438bb3be24e57..d77b085b8515f78327269b7fb9490ec5dccb7975 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -71,6 +71,8 @@ LOCAL_IMAGES=( "lofar-device-base lofar-device-base y" "http-json-schemas http-json-schemas y" + "dsconfig dsconfig n" + "ec-sim ec-sim y" "itango itango n" diff --git a/sbin/update_ConfigDb.sh b/sbin/update_ConfigDb.sh index 4e3fc4b210e1bcc2b299e51521cb9e92b94377b4..dd46a81ebee566bfeccad3603d28751cb5e4fdac 100755 --- a/sbin/update_ConfigDb.sh +++ b/sbin/update_ConfigDb.sh @@ -1,31 +1,7 @@ #!/bin/bash -# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +# -if [ ${#} -eq 1 ]; then - file=${1} -else - echo "A file name must be provided." - exit 1 -fi - -# dump a copy of the database before updating -# Do not change -i into -it this will break integration tests in gitlab ci! -docker exec -i dsconfig bash -c ' - python -m dsconfig.dump > /tmp/dsconfig-configdb-dump.json - /manage_object_properties.py -r > /tmp/dsconfig-objectdb-dump.json - /merge_json.py /tmp/dsconfig-objectdb-dump.json /tmp/dsconfig-configdb-dump.json' \ - > "${LOFAR20_DIR}"/CDB/dump_"$(date "+%Y.%m.%d_%H.%M.%S")".json - -# copy file into container to read it from container, as the file's location -# in the container won't be the same as on the host. -docker cp "${file}" dsconfig:/tmp/dsconfig-update-settings.json || exit 1 - -# update settings, Do not change -i into -it this will break integration tests in gitlab ci! -docker exec -i dsconfig /manage_object_properties.py --write < "${file}" - -# update settings, Do not change -i into -it this will break integration tests in gitlab ci! -docker exec -i dsconfig json2tango --write --update /tmp/dsconfig-update-settings.json - -# somehow json2tango does not return 0 on success -exit 0 +"${LOFAR20_DIR}/sbin/dsconfig.sh" --update "${1}" diff --git a/setup.sh b/setup.sh index ee556bb84e2e5badd8767145db70dd3ef368064d..75c179f124a74262547d7e76ac7fc0795d95613b 100755 --- a/setup.sh +++ b/setup.sh @@ -1,11 +1,17 @@ #! /usr/bin/env bash -# Copyright (C) 2022 ASTRON (Netherlands Institute for Radio Astronomy) +# +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) # SPDX-License-Identifier: Apache-2.0 +# # Set up station control development # This file's directory is used to determine the station control directory # location. +if [ -z ${BASH_SOURCE} ]; then + BASH_SOURCE=${(%):-%x} +fi + ABSOLUTE_PATH=$(realpath $(dirname ${BASH_SOURCE})) export LOFAR20_DIR=${1:-${ABSOLUTE_PATH}} @@ -18,12 +24,20 @@ fi # This needs to be modified for a development environment. # In case you run multiple Docker networks on the same host in parallel, # you need to specify a unique network name for each of them. -export NETWORK_MODE=tangonet +export NETWORK_MODE=station # It is assumed that the Tango host, the computer that runs the TangoDB, # is this host. If this is not true, then modify to the Tango host's FQDN and # port. Example: export TANGO_HOST=station-xk25.astron.nl:10000 -export TANGO_HOST=$(hostname):10000 +if dig @127.0.0.1 -p 8600 tango.service.consul +short > /dev/null; then + TANGO_PORT=$(dig @127.0.0.1 -p 8600 tango.service.consul SRV +short | awk '{printf "%s",$3}') + TANGO_HOST=$(dig @127.0.0.1 -p 8600 tango.service.consul +short) + export TANGO_HOST="$TANGO_HOST:$TANGO_PORT" +else + export TANGO_HOST=$(hostname):10000 +fi + +echo "Using tango host $TANGO_HOST" # Configure to install station control in lofar-device-base image # TODO(L2SS-520): Extend to support debug and expose as property in devices diff --git a/tangostationcontrol/VERSION b/tangostationcontrol/VERSION index 16eb94e711f823f8085064e79f74d9ce2382c240..2157409059873c80aa93884ecb847639add77b7a 100644 --- a/tangostationcontrol/VERSION +++ b/tangostationcontrol/VERSION @@ -1 +1 @@ -0.21.3 +0.22.0 diff --git a/tangostationcontrol/docs/source/configure_station.rst b/tangostationcontrol/docs/source/configure_station.rst index fa45b4c34549eae0cde573751916282f2a00c674..4c08ada9bbe9449eb580e823b1961e1628647bb1 100644 --- a/tangostationcontrol/docs/source/configure_station.rst +++ b/tangostationcontrol/docs/source/configure_station.rst @@ -5,7 +5,7 @@ The software will need to be told various aspects of your station configuration, Stock configurations are provided for several stations, as well as using simulators to simulate the station's interface (which is the default after bootstrapping a station). These are provided in the ``CDB/stations/`` directory, and can be loaded using for example:: - sbin/update_ConfigDb.sh CDB/stations/LTS_ConfigDb.json + sbin/dsconfig.sh --update CDB/stations/LTS_ConfigDb.json The following sections describe the settings that are station dependent, and thus must or can be set. diff --git a/tangostationcontrol/docs/source/devices/configure.rst b/tangostationcontrol/docs/source/devices/configure.rst index 0ed05a0fba41d10c9a73a91da355ddbf3f192b97..c8573f029082d1d88fd8601547c1c334311d6ef0 100644 --- a/tangostationcontrol/docs/source/devices/configure.rst +++ b/tangostationcontrol/docs/source/devices/configure.rst @@ -39,10 +39,10 @@ Command-line interaction The content of the TangoDB can be dumped from the command line using:: - bin/dump_ConfigDb.sh > tangodb-dump.json + sbin/dsconfig.sh --dump > tangodb-dump.json and changes can be applied using:: - bin/update_ConfigDb.sh changeset.json + sbin/dsconfig.sh --update changeset.json .. note:: The ``dsconfig`` docker container needs to be running for these commands to work. diff --git a/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py b/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py index 32c3b5666b87e407771f7ae86782d40c5ddfaf20..9c996a70636f1b1ef5fe189e2e659610056b3fdb 100644 --- a/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py +++ b/tangostationcontrol/integration_test/default/devices/test_device_digitalbeam.py @@ -6,7 +6,8 @@ import time import numpy import timeout_decorator - +from integration_test.default.devices.base import AbstractTestBases +from integration_test.device_proxy import TestDeviceProxy from tangostationcontrol.common.constants import ( MAX_ANTENNA, N_beamlets_ctrl, @@ -18,9 +19,6 @@ from tangostationcontrol.common.constants import ( ) from tangostationcontrol.devices.antennafield import AntennaQuality, AntennaUse -from integration_test.device_proxy import TestDeviceProxy -from integration_test.default.devices.base import AbstractTestBases - logger = logging.getLogger()