Merge branch 'L2SS-983' into 'master'

L2SS-983: Deployment through environments and ansible

Closes L2SS-983

See merge request !469

.gitignore

@@ -27,5 +27,8 @@ tangostationcontrol/docs/build
 **/pending_log_messages.db
 **/.eggs
 
+deploy/*.retry
+deploy/hosts
+
 docker-compose/alerta-web/alerta-secrets.json
 docker-compose/tmp

.gitlab-ci.yml

@@ -14,7 +14,8 @@ stages:
   - unit-tests
   - integration-tests
   - packaging
-# See docker-compose/README.md for docker image behavior and explanation
+  - deploy
+ # See docker-compose/README.md for docker image behavior and explanation
 .base_docker_images:
   stage: images
   image: docker:latest
@@ -63,10 +64,8 @@ stages:
 # master builds
 docker_store_images_master_tag:
   extends: .base_docker_store_images
-  only:
-    refs:
-      - tags
-      - master
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG
 
 # Download all remote images and store them on our image registry if .env changes
 # on a merge request
@@ -85,10 +84,8 @@ docker_store_images_changes:
 # Build and push all our custom images on tagged or master builds
 docker_build_image_all:
   extends: .base_docker_images
-  only:
-    refs:
-      - tags
-      - master
+  rules:
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk latest
@@ -664,3 +661,41 @@ wheel_packaging:
   script:
     - cd tangostationcontrol
     - tox -e build
+.base_deploy:
+  stage: deploy
+  image: ubuntu:bionic
+  when: manual
+  rules:
+  - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG
+  before_script:
+    - apt-get update
+    - apt-get install ansible -y
+    # Use Gitlab protected variable to provide key
+    - echo "$DEPLOY_KEY" > id_rsa
+    - chmod 400 id_rsa
+    - ssh-keygen -y -f id_rsa > id_rsa.pub
+    # Add ssh key to agent
+    - eval $(ssh-agent)
+    - ssh-add id_rsa
+    - ansible --version
+  script:
+    - echo "Deploying version $CI_COMMIT_TAG"
+    - cd deploy
+    # Prevent error of ansible being run in world writeable directory
+    - chmod o-w .
+    # Generate hosts file for deployment
+    - echo "[all]" > hosts
+    - echo "stat ansible_host=$DEPLOY_HOST ansible_user=$DEPLOY_USER" >> hosts
+    # Run deployment with populated variables
+    - ansible-playbook -v deploy.yml --extra-vars station_version=$CI_COMMIT_TAG --extra-vars station_config=$DEPLOY_CONFIG
+.deploy_l2ts_base:
+  extends: .base_deploy
+  variables:
+    DEPLOY_USER: $L2TS_USERNAME
+    DEPLOY_HOST: $L2TS_HOSTNAME
+    DEPLOY_CONFIG: $L2TS_CONFIG
+    DEPLOY_KEY: $L2TS_DEPLOY_KEY
+deploy_l2ts_start:
+  extends: .deploy_l2ts_base
+  environment:
+    name: l2ts

CDB/LOFAR_ConfigDb.json

@@ -206,7 +206,7 @@
                 }
             }
         },
-        "boot": {
+        "Boot": {
             "STAT": {
                 "Boot": {
                     "STAT/Boot/1": {}

CDB/stations/CS001_ConfigDb.json

@@ -10,7 +10,7 @@
         }
     },
     "servers": {
-        "boot": {
+        "Boot": {
             "STAT": {
                 "Boot": {
                     "STAT/Boot/1": {

README.md

@@ -17,6 +17,7 @@ Station Control software related to Tango devices.
   * [Jupyter startup files](docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md)
   * [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter)
 * [Developer Documentation](#development)
+  * [Deployments](deploy/README.md)
   * [Versioning](#versioning)
 * Source code documentation
   * [Attribute wrapper documentation](tangostationcontrol/tangostationcontrol/clients/README.md)
@@ -109,9 +110,12 @@ Next change the version in the following places:
 2. In [test_writer_sst.py](tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py)
    for the `test_header_info` test.
 3. Add a [Release note](#release-notes) for the given version.
-3. Once the merge requests is merged to master, add a tag with the version (just x.x.x not Vx.x.x)
+4. Once the merge requests is merged to master, add a tag with the version such as v0.3.2 or v0.3.3 etc
+5. The tag can be deployed to the environments, manually, through [https://git.astron.nl/lofar2.0/tango/-/tags](Deploy Tags)
 
 # Release Notes
 
-* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter
+* 0.3.1 Fix for applying boot device dsconfig
+* 0.3.0 Initial version of deployment scripts and functionality
 * 0.2.0 Extend `Beamlet` device with FPGA source address attributes
+* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter

deploy/README.md

+# Deployments
+
+Deploying is achieved through gitlab environments as well as ansible.
+Concretely the following files are involved:
+
+1. [deploy.yml](deploy.yml)
+2. [ansible.cfg](ansible.cfg)
+3. [.gitlab-ci.yml](../.gitlab-ci.yml)
+
+Once a tagged commit hits master a job is created for this tag that needs to
+be manually triggered. You should be able to find this pipeline on the
+[tags overview](https://git.astron.nl/lofar2.0/tango/-/tags).
+
+From here, with prior consent, the deployment can be started for any
+environment.
+
+The gitlab-ci.yml job defines each of the environments in parameterized way.
+The templated `.base_deploy` needs to be extended and have several variables
+defined. These variables need to be fed from
+[protected variables](https://git.astron.nl/lofar2.0/tango/-/settings/ci_cd)
+within gitlab.
+
+Below is a practical example of how to define an environment.
+
+```
+.deploy_example_base:
+  extends: .base_deploy
+  variables:
+    DEPLOY_USER: $EXAMPLE_USERNAME
+    DEPLOY_HOST: $EXAMPLE_HOSTNAME
+    DEPLOY_CONFIG: $EXAMPLE_CONFIG
+    DEPLOY_KEY: $EXAMPLE_KEY
+deploy_l2ts_start:
+  extends: .deploy_example_base
+  environment:
+    name: example
+```
+
+Such an environment will only appear in the
+[environments overview](https://git.astron.nl/lofar2.0/tango/-/environments)
+after the first deployment.
+
+The process of this deployment is handled through ansible as found in
+`deploy.yml`.

deploy/ansible.cfg

+[defaults]
+host_key_checking = False
+inventory = hosts

deploy/deploy.yml

+---
+- name: StationControl Early Deployment
+  hosts: all
+  vars:
+    base_station_config: "LOFAR_ConfigDb.json"
+    install_path: ~/git/tango
+    databaseds_port: 10000
+  tasks:
+     - name: Check make installed
+       include_tasks:
+         file: tasks/check_binary_install.yml
+       vars:
+         program: make
+     - name: Check git installed
+       include_tasks:
+         file: tasks/check_binary_install.yml
+       vars:
+         program: git
+     - name: Register tango directory status
+       shell: cd {{ install_path }}
+       args:
+         chdir: ~
+       changed_when: false
+       failed_when: tango_directory.rc not in [0,1]
+       register: tango_directory
+     - name: Register pending changes
+       shell: "! (git status | grep Changes)"
+       args:
+         chdir: "{{ install_path }}"
+       changed_when: false
+       failed_when: pending_changes.rc not in [0,1]
+       register: pending_changes
+# Several issues with this variable being unset in ansible 2.2
+# where encountered, keep debug in case needed
+     - debug: var=pending_changes
+     - name: Check tango directory status
+       fail:
+         msg: "Tango directory appears to be missing!"
+       when: tango_directory.rc not in [0]
+     - name: Check pending changes
+       fail:
+         msg: "Deployment repository seems to have pending changes!"
+       when: pending_changes.rc not in [0]
+     - name: Stop Current Station
+       changed_when: false
+       shell: "make stop"
+       args:
+         chdir: "{{ install_path }}/docker-compose"
+     - name: Git Fetch All
+       changed_when: false
+       shell: "git fetch --all"
+       args:
+         chdir: "{{ install_path }}"
+     - name: Update Sources
+       changed_when: false
+       shell: "git checkout {{ station_version }}"
+       args:
+         chdir: "{{ install_path }}"
+     - name: Pull Images
+       changed_when: false
+       shell: "make pull"
+       args:
+         chdir: "{{ install_path }}/docker-compose"
+     - name: Build Images
+       changed_when: false
+       shell: "make build"
+       args:
+         chdir: "{{ install_path }}/docker-compose"
+     - name: Start Database
+       changed_when: false
+       shell: "make minimal"
+       args:
+         chdir: "{{ install_path }}/docker-compose"
+     - name: Wait for databaseds
+       ansible.builtin.wait_for:
+         port: "{{ databaseds_port }}"
+         delay: 10
+     - name: Update Base Database Config
+       include_tasks:
+         file: tasks/update_database_config.yml
+       vars:
+         database_config: "{{ base_station_config }}"
+         base_path: "{{ install_path }}"
+     - name: Verify Base Database Config
+       include_tasks:
+         file: tasks/verify_database_config.yml
+       vars:
+         database_config: "{{ base_station_config }}"
+         base_path: "{{ install_path }}"
+     - name: Update Station Database Config
+       include_tasks:
+         file: tasks/update_database_config.yml
+       vars:
+         database_config: "stations/{{ station_config }}"
+         base_path: "{{ install_path }}"
+     - name: Verify Station Database Config
+       include_tasks:
+         file: tasks/verify_database_config.yml
+       vars:
+         database_config: "stations/{{ station_config }}"
+         base_path: "{{ install_path }}"
+     - name: Start Station
+       changed_when: false
+       shell: "make start"
+       args:
+         chdir: "{{ install_path }}/docker-compose"

deploy/tasks/check_binary_install.yml

+ - name: Register installation status
+   command: which {{ program }}
+   changed_when: false
+   failed_when: installed.rc not in [0,1]
+   register: installed
+ - name: Check installation status
+   fail:
+     msg: "{{ program }} does not appear to be installed!"
+   when: installed.rc not in [0]
\ No newline at end of file

deploy/tasks/update_database_config.yml

+ - name: Update Database Config
+   changed_when: false
+   shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }}"
+   args:
+     chdir: "{{ base_path }}"
\ No newline at end of file

deploy/tasks/verify_database_config.yml

+ - name: Get Database Config
+   changed_when: false
+   shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }} 2>&1"
+   register: dsconfig_result
+   args:
+     chdir: "{{ base_path }}"
+ - name: Verify Database Config
+   when: '"No changes needed" in dsconfig_result.stdout'
+   debug: msg="Database changes stored"
\ No newline at end of file

docker-compose/Makefile

@@ -149,6 +149,7 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \
     XAUTHORITY_MOUNT=$(XAUTHORITY_MOUNT) \
     CONTAINER_NAME_PREFIX=$(CONTAINER_NAME_PREFIX) \
     COMPOSE_IGNORE_ORPHANS=true \
+    COMPOSE_HTTP_TIMEOUT=180 \
     CONTAINER_EXECUTION_UID=$(shell id -u) \
     DOCKER_GID=$(DOCKER_GID) \
     TEST_MODULE=$(INTEGRATION_MODULE)

tangostationcontrol/VERSION

-0.2.0
+0.3.1

tangostationcontrol/tangostationcontrol/clients/comms_client.py

@@ -15,7 +15,7 @@ class AbstractCommClient(ABC):
     def stop(self):
         """ Stop communication with the client. """
 
-    def ping(self): # noqa: B027
+    def ping(self):  # noqa: B027
         """ Check whether the connection is still alive.
 
             Clients that override this method must raise an Exception if the

tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py

@@ -82,7 +82,7 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase):
                     '2021-09-20T12:17:40.000+00:00'
                 )
                 self.assertIsNotNone(stat)
-                self.assertEqual("0.2.0", stat.station_version_id)
+                self.assertEqual("0.3.1", stat.station_version_id)
                 self.assertEqual("0.1", stat.writer_version_id)
 
     def test_insert_tango_SST_statistics(self):

tangostationcontrol/tox.ini

@@ -118,4 +118,4 @@ commands =
 [flake8]
 filename = *.py,.stestr.conf,.txt
 ignore = B014, B019, W291, W293, W391, E111, E114, E121, E122, E123, E124, E126, E127, E128, E131, E201, E201, E202, E203, E221, E222, E225, E226, E231, E241, E251, E252, E261, E262, E265, E271, E301, E302, E303, E305, E306, E401, E402, E501, E502, E701, E712, E721, E731, F403, F523, F541, F841, H301, H306, H401, H403, H404, H405, W503
-exclude=.tox,.egg-info,libhdbpp-python, SNMP_mib_loading
+exclude=.tox,build,.egg-info,libhdbpp-python, SNMP_mib_loading