From dddffe3538750784d55d38c63c2c910766b72dd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Corn=C3=A9=20Lukken?= <lukken@astron.nl>
Date: Thu, 3 Nov 2022 12:08:49 +0000
Subject: [PATCH] L2SS-983: Deployment through environments and ansible
---
.gitignore | 3 +
.gitlab-ci.yml | 53 +++++++--
CDB/LOFAR_ConfigDb.json | 2 +-
CDB/stations/CS001_ConfigDb.json | 2 +-
README.md | 8 +-
deploy/README.md | 44 ++++++++
deploy/ansible.cfg | 3 +
deploy/deploy.yml | 106 ++++++++++++++++++
deploy/tasks/check_binary_install.yml | 9 ++
deploy/tasks/update_database_config.yml | 5 +
deploy/tasks/verify_database_config.yml | 9 ++
docker-compose/Makefile | 1 +
tangostationcontrol/VERSION | 2 +-
.../clients/comms_client.py | 2 +-
.../default/statistics/test_writer_sst.py | 2 +-
tangostationcontrol/tox.ini | 2 +-
16 files changed, 236 insertions(+), 17 deletions(-)
create mode 100644 deploy/README.md
create mode 100644 deploy/ansible.cfg
create mode 100644 deploy/deploy.yml
create mode 100644 deploy/tasks/check_binary_install.yml
create mode 100644 deploy/tasks/update_database_config.yml
create mode 100644 deploy/tasks/verify_database_config.yml
diff --git a/.gitignore b/.gitignore
index f77736405..247edda4a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,5 +27,8 @@ tangostationcontrol/docs/build
**/pending_log_messages.db
**/.eggs
+deploy/*.retry
+deploy/hosts
+
docker-compose/alerta-web/alerta-secrets.json
docker-compose/tmp
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 82cbc3f55..fb162af72 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,8 @@ stages:
- unit-tests
- integration-tests
- packaging
-# See docker-compose/README.md for docker image behavior and explanation
+ - deploy
+ # See docker-compose/README.md for docker image behavior and explanation
.base_docker_images:
stage: images
image: docker:latest
@@ -63,10 +64,8 @@ stages:
# master builds
docker_store_images_master_tag:
extends: .base_docker_store_images
- only:
- refs:
- - tags
- - master
+ rules:
+ - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG
# Download all remote images and store them on our image registry if .env changes
# on a merge request
@@ -85,10 +84,8 @@ docker_store_images_changes:
# Build and push all our custom images on tagged or master builds
docker_build_image_all:
extends: .base_docker_images
- only:
- refs:
- - tags
- - master
+ rules:
+ - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk latest
@@ -664,3 +661,41 @@ wheel_packaging:
script:
- cd tangostationcontrol
- tox -e build
+.base_deploy:
+ stage: deploy
+ image: ubuntu:bionic
+ when: manual
+ rules:
+ - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG
+ before_script:
+ - apt-get update
+ - apt-get install ansible -y
+ # Use Gitlab protected variable to provide key
+ - echo "$DEPLOY_KEY" > id_rsa
+ - chmod 400 id_rsa
+ - ssh-keygen -y -f id_rsa > id_rsa.pub
+ # Add ssh key to agent
+ - eval $(ssh-agent)
+ - ssh-add id_rsa
+ - ansible --version
+ script:
+ - echo "Deploying version $CI_COMMIT_TAG"
+ - cd deploy
+ # Prevent error of ansible being run in world writeable directory
+ - chmod o-w .
+ # Generate hosts file for deployment
+ - echo "[all]" > hosts
+ - echo "stat ansible_host=$DEPLOY_HOST ansible_user=$DEPLOY_USER" >> hosts
+ # Run deployment with populated variables
+ - ansible-playbook -v deploy.yml --extra-vars station_version=$CI_COMMIT_TAG --extra-vars station_config=$DEPLOY_CONFIG
+.deploy_l2ts_base:
+ extends: .base_deploy
+ variables:
+ DEPLOY_USER: $L2TS_USERNAME
+ DEPLOY_HOST: $L2TS_HOSTNAME
+ DEPLOY_CONFIG: $L2TS_CONFIG
+ DEPLOY_KEY: $L2TS_DEPLOY_KEY
+deploy_l2ts_start:
+ extends: .deploy_l2ts_base
+ environment:
+ name: l2ts
diff --git a/CDB/LOFAR_ConfigDb.json b/CDB/LOFAR_ConfigDb.json
index b2d2f517e..1df2e30fd 100644
--- a/CDB/LOFAR_ConfigDb.json
+++ b/CDB/LOFAR_ConfigDb.json
@@ -206,7 +206,7 @@
}
}
},
- "boot": {
+ "Boot": {
"STAT": {
"Boot": {
"STAT/Boot/1": {}
diff --git a/CDB/stations/CS001_ConfigDb.json b/CDB/stations/CS001_ConfigDb.json
index e078b8c7a..5ea7c3d0b 100644
--- a/CDB/stations/CS001_ConfigDb.json
+++ b/CDB/stations/CS001_ConfigDb.json
@@ -10,7 +10,7 @@
}
},
"servers": {
- "boot": {
+ "Boot": {
"STAT": {
"Boot": {
"STAT/Boot/1": {
diff --git a/README.md b/README.md
index f1f8eae13..795e8a0aa 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,7 @@ Station Control software related to Tango devices.
* [Jupyter startup files](docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md)
* [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter)
* [Developer Documentation](#development)
+ * [Deployments](deploy/README.md)
* [Versioning](#versioning)
* Source code documentation
* [Attribute wrapper documentation](tangostationcontrol/tangostationcontrol/clients/README.md)
@@ -109,9 +110,12 @@ Next change the version in the following places:
2. In [test_writer_sst.py](tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py)
for the `test_header_info` test.
3. Add a [Release note](#release-notes) for the given version.
-3. Once the merge requests is merged to master, add a tag with the version (just x.x.x not Vx.x.x)
+4. Once the merge requests is merged to master, add a tag with the version such as v0.3.2 or v0.3.3 etc
+5. The tag can be deployed to the environments, manually, through [https://git.astron.nl/lofar2.0/tango/-/tags](Deploy Tags)
# Release Notes
-* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter
+* 0.3.1 Fix for applying boot device dsconfig
+* 0.3.0 Initial version of deployment scripts and functionality
* 0.2.0 Extend `Beamlet` device with FPGA source address attributes
+* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter
diff --git a/deploy/README.md b/deploy/README.md
new file mode 100644
index 000000000..5f49ffc5c
--- /dev/null
+++ b/deploy/README.md
@@ -0,0 +1,44 @@
+# Deployments
+
+Deploying is achieved through gitlab environments as well as ansible.
+Concretely the following files are involved:
+
+1. [deploy.yml](deploy.yml)
+2. [ansible.cfg](ansible.cfg)
+3. [.gitlab-ci.yml](../.gitlab-ci.yml)
+
+Once a tagged commit hits master a job is created for this tag that needs to
+be manually triggered. You should be able to find this pipeline on the
+[tags overview](https://git.astron.nl/lofar2.0/tango/-/tags).
+
+From here, with prior consent, the deployment can be started for any
+environment.
+
+The gitlab-ci.yml job defines each of the environments in parameterized way.
+The templated `.base_deploy` needs to be extended and have several variables
+defined. These variables need to be fed from
+[protected variables](https://git.astron.nl/lofar2.0/tango/-/settings/ci_cd)
+within gitlab.
+
+Below is a practical example of how to define an environment.
+
+```
+.deploy_example_base:
+ extends: .base_deploy
+ variables:
+ DEPLOY_USER: $EXAMPLE_USERNAME
+ DEPLOY_HOST: $EXAMPLE_HOSTNAME
+ DEPLOY_CONFIG: $EXAMPLE_CONFIG
+ DEPLOY_KEY: $EXAMPLE_KEY
+deploy_l2ts_start:
+ extends: .deploy_example_base
+ environment:
+ name: example
+```
+
+Such an environment will only appear in the
+[environments overview](https://git.astron.nl/lofar2.0/tango/-/environments)
+after the first deployment.
+
+The process of this deployment is handled through ansible as found in
+`deploy.yml`.
diff --git a/deploy/ansible.cfg b/deploy/ansible.cfg
new file mode 100644
index 000000000..8da5e9439
--- /dev/null
+++ b/deploy/ansible.cfg
@@ -0,0 +1,3 @@
+[defaults]
+host_key_checking = False
+inventory = hosts
diff --git a/deploy/deploy.yml b/deploy/deploy.yml
new file mode 100644
index 000000000..455c9da59
--- /dev/null
+++ b/deploy/deploy.yml
@@ -0,0 +1,106 @@
+---
+- name: StationControl Early Deployment
+ hosts: all
+ vars:
+ base_station_config: "LOFAR_ConfigDb.json"
+ install_path: ~/git/tango
+ databaseds_port: 10000
+ tasks:
+ - name: Check make installed
+ include_tasks:
+ file: tasks/check_binary_install.yml
+ vars:
+ program: make
+ - name: Check git installed
+ include_tasks:
+ file: tasks/check_binary_install.yml
+ vars:
+ program: git
+ - name: Register tango directory status
+ shell: cd {{ install_path }}
+ args:
+ chdir: ~
+ changed_when: false
+ failed_when: tango_directory.rc not in [0,1]
+ register: tango_directory
+ - name: Register pending changes
+ shell: "! (git status | grep Changes)"
+ args:
+ chdir: "{{ install_path }}"
+ changed_when: false
+ failed_when: pending_changes.rc not in [0,1]
+ register: pending_changes
+# Several issues with this variable being unset in ansible 2.2
+# where encountered, keep debug in case needed
+ - debug: var=pending_changes
+ - name: Check tango directory status
+ fail:
+ msg: "Tango directory appears to be missing!"
+ when: tango_directory.rc not in [0]
+ - name: Check pending changes
+ fail:
+ msg: "Deployment repository seems to have pending changes!"
+ when: pending_changes.rc not in [0]
+ - name: Stop Current Station
+ changed_when: false
+ shell: "make stop"
+ args:
+ chdir: "{{ install_path }}/docker-compose"
+ - name: Git Fetch All
+ changed_when: false
+ shell: "git fetch --all"
+ args:
+ chdir: "{{ install_path }}"
+ - name: Update Sources
+ changed_when: false
+ shell: "git checkout {{ station_version }}"
+ args:
+ chdir: "{{ install_path }}"
+ - name: Pull Images
+ changed_when: false
+ shell: "make pull"
+ args:
+ chdir: "{{ install_path }}/docker-compose"
+ - name: Build Images
+ changed_when: false
+ shell: "make build"
+ args:
+ chdir: "{{ install_path }}/docker-compose"
+ - name: Start Database
+ changed_when: false
+ shell: "make minimal"
+ args:
+ chdir: "{{ install_path }}/docker-compose"
+ - name: Wait for databaseds
+ ansible.builtin.wait_for:
+ port: "{{ databaseds_port }}"
+ delay: 10
+ - name: Update Base Database Config
+ include_tasks:
+ file: tasks/update_database_config.yml
+ vars:
+ database_config: "{{ base_station_config }}"
+ base_path: "{{ install_path }}"
+ - name: Verify Base Database Config
+ include_tasks:
+ file: tasks/verify_database_config.yml
+ vars:
+ database_config: "{{ base_station_config }}"
+ base_path: "{{ install_path }}"
+ - name: Update Station Database Config
+ include_tasks:
+ file: tasks/update_database_config.yml
+ vars:
+ database_config: "stations/{{ station_config }}"
+ base_path: "{{ install_path }}"
+ - name: Verify Station Database Config
+ include_tasks:
+ file: tasks/verify_database_config.yml
+ vars:
+ database_config: "stations/{{ station_config }}"
+ base_path: "{{ install_path }}"
+ - name: Start Station
+ changed_when: false
+ shell: "make start"
+ args:
+ chdir: "{{ install_path }}/docker-compose"
diff --git a/deploy/tasks/check_binary_install.yml b/deploy/tasks/check_binary_install.yml
new file mode 100644
index 000000000..f114fd87e
--- /dev/null
+++ b/deploy/tasks/check_binary_install.yml
@@ -0,0 +1,9 @@
+ - name: Register installation status
+ command: which {{ program }}
+ changed_when: false
+ failed_when: installed.rc not in [0,1]
+ register: installed
+ - name: Check installation status
+ fail:
+ msg: "{{ program }} does not appear to be installed!"
+ when: installed.rc not in [0]
\ No newline at end of file
diff --git a/deploy/tasks/update_database_config.yml b/deploy/tasks/update_database_config.yml
new file mode 100644
index 000000000..be321c576
--- /dev/null
+++ b/deploy/tasks/update_database_config.yml
@@ -0,0 +1,5 @@
+ - name: Update Database Config
+ changed_when: false
+ shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }}"
+ args:
+ chdir: "{{ base_path }}"
\ No newline at end of file
diff --git a/deploy/tasks/verify_database_config.yml b/deploy/tasks/verify_database_config.yml
new file mode 100644
index 000000000..1df3681e4
--- /dev/null
+++ b/deploy/tasks/verify_database_config.yml
@@ -0,0 +1,9 @@
+ - name: Get Database Config
+ changed_when: false
+ shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }} 2>&1"
+ register: dsconfig_result
+ args:
+ chdir: "{{ base_path }}"
+ - name: Verify Database Config
+ when: '"No changes needed" in dsconfig_result.stdout'
+ debug: msg="Database changes stored"
\ No newline at end of file
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index 5cdb54531..0ca1c240b 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -149,6 +149,7 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
XAUTHORITY_MOUNT=$(XAUTHORITY_MOUNT) \
CONTAINER_NAME_PREFIX=$(CONTAINER_NAME_PREFIX) \
COMPOSE_IGNORE_ORPHANS=true \
+ COMPOSE_HTTP_TIMEOUT=180 \
CONTAINER_EXECUTION_UID=$(shell id -u) \
DOCKER_GID=$(DOCKER_GID) \
TEST_MODULE=$(INTEGRATION_MODULE)
diff --git a/tangostationcontrol/VERSION b/tangostationcontrol/VERSION
index 0ea3a944b..9e11b32fc 100644
--- a/tangostationcontrol/VERSION
+++ b/tangostationcontrol/VERSION
@@ -1 +1 @@
-0.2.0
+0.3.1
diff --git a/tangostationcontrol/tangostationcontrol/clients/comms_client.py b/tangostationcontrol/tangostationcontrol/clients/comms_client.py
index deb37f671..0188753a2 100644
--- a/tangostationcontrol/tangostationcontrol/clients/comms_client.py
+++ b/tangostationcontrol/tangostationcontrol/clients/comms_client.py
@@ -15,7 +15,7 @@ class AbstractCommClient(ABC):
def stop(self):
""" Stop communication with the client. """
- def ping(self): # noqa: B027
+ def ping(self): # noqa: B027
""" Check whether the connection is still alive.
Clients that override this method must raise an Exception if the
diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
index 89eb31d8e..90c1c9161 100644
--- a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
+++ b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py
@@ -82,7 +82,7 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
'2021-09-20T12:17:40.000+00:00'
)
self.assertIsNotNone(stat)
- self.assertEqual("0.2.0", stat.station_version_id)
+ self.assertEqual("0.3.1", stat.station_version_id)
self.assertEqual("0.1", stat.writer_version_id)
def test_insert_tango_SST_statistics(self):
diff --git a/tangostationcontrol/tox.ini b/tangostationcontrol/tox.ini
index 332d3777f..d528ccc75 100644
--- a/tangostationcontrol/tox.ini
+++ b/tangostationcontrol/tox.ini
@@ -118,4 +118,4 @@ commands =
[flake8]
filename = *.py,.stestr.conf,.txt
ignore = B014, B019, W291, W293, W391, E111, E114, E121, E122, E123, E124, E126, E127, E128, E131, E201, E201, E202, E203, E221, E222, E225, E226, E231, E241, E251, E252, E261, E262, E265, E271, E301, E302, E303, E305, E306, E401, E402, E501, E502, E701, E712, E721, E731, F403, F523, F541, F841, H301, H306, H401, H403, H404, H405, W503
-exclude=.tox,.egg-info,libhdbpp-python, SNMP_mib_loading
+exclude=.tox,build,.egg-info,libhdbpp-python, SNMP_mib_loading
--
GitLab