diff --git a/.gitignore b/.gitignore index f777364050f38eddf7f7867a0326b5dd3199074c..247edda4ab41b5110b3af96f7d2ecd4660970a0c 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,8 @@ tangostationcontrol/docs/build **/pending_log_messages.db **/.eggs +deploy/*.retry +deploy/hosts + docker-compose/alerta-web/alerta-secrets.json docker-compose/tmp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82cbc3f552989949dbe2d5ac79cdd47b83152971..fb162af725be87fdec3d1beb3649ed90b78aca42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,7 +14,8 @@ stages: - unit-tests - integration-tests - packaging -# See docker-compose/README.md for docker image behavior and explanation + - deploy + # See docker-compose/README.md for docker image behavior and explanation .base_docker_images: stage: images image: docker:latest @@ -63,10 +64,8 @@ stages: # master builds docker_store_images_master_tag: extends: .base_docker_store_images - only: - refs: - - tags - - master + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG # Download all remote images and store them on our image registry if .env changes # on a merge request @@ -85,10 +84,8 @@ docker_store_images_changes: # Build and push all our custom images on tagged or master builds docker_build_image_all: extends: .base_docker_images - only: - refs: - - tags - - master + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk latest @@ -664,3 +661,41 @@ wheel_packaging: script: - cd tangostationcontrol - tox -e build +.base_deploy: + stage: deploy + image: ubuntu:bionic + when: manual + rules: + - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) && $CI_COMMIT_TAG + before_script: + - apt-get update + - apt-get install ansible -y + # Use Gitlab protected variable to provide key + - echo "$DEPLOY_KEY" > id_rsa + - chmod 400 id_rsa + - ssh-keygen -y -f id_rsa > id_rsa.pub + # Add ssh key to agent + - eval $(ssh-agent) + - ssh-add id_rsa + - ansible --version + script: + - echo "Deploying version $CI_COMMIT_TAG" + - cd deploy + # Prevent error of ansible being run in world writeable directory + - chmod o-w . + # Generate hosts file for deployment + - echo "[all]" > hosts + - echo "stat ansible_host=$DEPLOY_HOST ansible_user=$DEPLOY_USER" >> hosts + # Run deployment with populated variables + - ansible-playbook -v deploy.yml --extra-vars station_version=$CI_COMMIT_TAG --extra-vars station_config=$DEPLOY_CONFIG +.deploy_l2ts_base: + extends: .base_deploy + variables: + DEPLOY_USER: $L2TS_USERNAME + DEPLOY_HOST: $L2TS_HOSTNAME + DEPLOY_CONFIG: $L2TS_CONFIG + DEPLOY_KEY: $L2TS_DEPLOY_KEY +deploy_l2ts_start: + extends: .deploy_l2ts_base + environment: + name: l2ts diff --git a/CDB/LOFAR_ConfigDb.json b/CDB/LOFAR_ConfigDb.json index b2d2f517e83667a1659c39d091f73bc2580df01f..1df2e30fdc1854edf45328cd3c61e64c0d1f4479 100644 --- a/CDB/LOFAR_ConfigDb.json +++ b/CDB/LOFAR_ConfigDb.json @@ -206,7 +206,7 @@ } } }, - "boot": { + "Boot": { "STAT": { "Boot": { "STAT/Boot/1": {} diff --git a/CDB/stations/CS001_ConfigDb.json b/CDB/stations/CS001_ConfigDb.json index e078b8c7a9daf205a064a665a934bfb15671225b..5ea7c3d0bc87365f3de6cdf4c3e06aedcc4d65f1 100644 --- a/CDB/stations/CS001_ConfigDb.json +++ b/CDB/stations/CS001_ConfigDb.json @@ -10,7 +10,7 @@ } }, "servers": { - "boot": { + "Boot": { "STAT": { "Boot": { "STAT/Boot/1": { diff --git a/README.md b/README.md index f1f8eae1392d18924664aab90db19473caf2e33f..795e8a0aa36fa3c50df45ade18ab9b75327f4c11 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Station Control software related to Tango devices. * [Jupyter startup files](docker-compose/jupyter/ipython-profiles/stationcontrol-jupyter/startup/README.md) * [Tango Prometheus exporter](https://git.astron.nl/lofar2.0/ska-tango-grafana-exporter) * [Developer Documentation](#development) + * [Deployments](deploy/README.md) * [Versioning](#versioning) * Source code documentation * [Attribute wrapper documentation](tangostationcontrol/tangostationcontrol/clients/README.md) @@ -109,9 +110,12 @@ Next change the version in the following places: 2. In [test_writer_sst.py](tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py) for the `test_header_info` test. 3. Add a [Release note](#release-notes) for the given version. -3. Once the merge requests is merged to master, add a tag with the version (just x.x.x not Vx.x.x) +4. Once the merge requests is merged to master, add a tag with the version such as v0.3.2 or v0.3.3 etc +5. The tag can be deployed to the environments, manually, through [https://git.astron.nl/lofar2.0/tango/-/tags](Deploy Tags) # Release Notes -* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter +* 0.3.1 Fix for applying boot device dsconfig +* 0.3.0 Initial version of deployment scripts and functionality * 0.2.0 Extend `Beamlet` device with FPGA source address attributes +* 0.1.2 Fix `StatisticsClient` accessing `last_invalid_packet_exception` parameter diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f49ffc5c340dacf671e607f55c36f68f4db774a --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,44 @@ +# Deployments + +Deploying is achieved through gitlab environments as well as ansible. +Concretely the following files are involved: + +1. [deploy.yml](deploy.yml) +2. [ansible.cfg](ansible.cfg) +3. [.gitlab-ci.yml](../.gitlab-ci.yml) + +Once a tagged commit hits master a job is created for this tag that needs to +be manually triggered. You should be able to find this pipeline on the +[tags overview](https://git.astron.nl/lofar2.0/tango/-/tags). + +From here, with prior consent, the deployment can be started for any +environment. + +The gitlab-ci.yml job defines each of the environments in parameterized way. +The templated `.base_deploy` needs to be extended and have several variables +defined. These variables need to be fed from +[protected variables](https://git.astron.nl/lofar2.0/tango/-/settings/ci_cd) +within gitlab. + +Below is a practical example of how to define an environment. + +``` +.deploy_example_base: + extends: .base_deploy + variables: + DEPLOY_USER: $EXAMPLE_USERNAME + DEPLOY_HOST: $EXAMPLE_HOSTNAME + DEPLOY_CONFIG: $EXAMPLE_CONFIG + DEPLOY_KEY: $EXAMPLE_KEY +deploy_l2ts_start: + extends: .deploy_example_base + environment: + name: example +``` + +Such an environment will only appear in the +[environments overview](https://git.astron.nl/lofar2.0/tango/-/environments) +after the first deployment. + +The process of this deployment is handled through ansible as found in +`deploy.yml`. diff --git a/deploy/ansible.cfg b/deploy/ansible.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8da5e9439f11320c5e90d51bbe6ad7d7f8d9798c --- /dev/null +++ b/deploy/ansible.cfg @@ -0,0 +1,3 @@ +[defaults] +host_key_checking = False +inventory = hosts diff --git a/deploy/deploy.yml b/deploy/deploy.yml new file mode 100644 index 0000000000000000000000000000000000000000..455c9da59ea78e724dc448e476df8fffa0ff6522 --- /dev/null +++ b/deploy/deploy.yml @@ -0,0 +1,106 @@ +--- +- name: StationControl Early Deployment + hosts: all + vars: + base_station_config: "LOFAR_ConfigDb.json" + install_path: ~/git/tango + databaseds_port: 10000 + tasks: + - name: Check make installed + include_tasks: + file: tasks/check_binary_install.yml + vars: + program: make + - name: Check git installed + include_tasks: + file: tasks/check_binary_install.yml + vars: + program: git + - name: Register tango directory status + shell: cd {{ install_path }} + args: + chdir: ~ + changed_when: false + failed_when: tango_directory.rc not in [0,1] + register: tango_directory + - name: Register pending changes + shell: "! (git status | grep Changes)" + args: + chdir: "{{ install_path }}" + changed_when: false + failed_when: pending_changes.rc not in [0,1] + register: pending_changes +# Several issues with this variable being unset in ansible 2.2 +# where encountered, keep debug in case needed + - debug: var=pending_changes + - name: Check tango directory status + fail: + msg: "Tango directory appears to be missing!" + when: tango_directory.rc not in [0] + - name: Check pending changes + fail: + msg: "Deployment repository seems to have pending changes!" + when: pending_changes.rc not in [0] + - name: Stop Current Station + changed_when: false + shell: "make stop" + args: + chdir: "{{ install_path }}/docker-compose" + - name: Git Fetch All + changed_when: false + shell: "git fetch --all" + args: + chdir: "{{ install_path }}" + - name: Update Sources + changed_when: false + shell: "git checkout {{ station_version }}" + args: + chdir: "{{ install_path }}" + - name: Pull Images + changed_when: false + shell: "make pull" + args: + chdir: "{{ install_path }}/docker-compose" + - name: Build Images + changed_when: false + shell: "make build" + args: + chdir: "{{ install_path }}/docker-compose" + - name: Start Database + changed_when: false + shell: "make minimal" + args: + chdir: "{{ install_path }}/docker-compose" + - name: Wait for databaseds + ansible.builtin.wait_for: + port: "{{ databaseds_port }}" + delay: 10 + - name: Update Base Database Config + include_tasks: + file: tasks/update_database_config.yml + vars: + database_config: "{{ base_station_config }}" + base_path: "{{ install_path }}" + - name: Verify Base Database Config + include_tasks: + file: tasks/verify_database_config.yml + vars: + database_config: "{{ base_station_config }}" + base_path: "{{ install_path }}" + - name: Update Station Database Config + include_tasks: + file: tasks/update_database_config.yml + vars: + database_config: "stations/{{ station_config }}" + base_path: "{{ install_path }}" + - name: Verify Station Database Config + include_tasks: + file: tasks/verify_database_config.yml + vars: + database_config: "stations/{{ station_config }}" + base_path: "{{ install_path }}" + - name: Start Station + changed_when: false + shell: "make start" + args: + chdir: "{{ install_path }}/docker-compose" diff --git a/deploy/tasks/check_binary_install.yml b/deploy/tasks/check_binary_install.yml new file mode 100644 index 0000000000000000000000000000000000000000..f114fd87e7b7bd25d7f6b848b25c32aa9324efa2 --- /dev/null +++ b/deploy/tasks/check_binary_install.yml @@ -0,0 +1,9 @@ + - name: Register installation status + command: which {{ program }} + changed_when: false + failed_when: installed.rc not in [0,1] + register: installed + - name: Check installation status + fail: + msg: "{{ program }} does not appear to be installed!" + when: installed.rc not in [0] \ No newline at end of file diff --git a/deploy/tasks/update_database_config.yml b/deploy/tasks/update_database_config.yml new file mode 100644 index 0000000000000000000000000000000000000000..be321c5769d48a6443de18340033c8cdaffd661f --- /dev/null +++ b/deploy/tasks/update_database_config.yml @@ -0,0 +1,5 @@ + - name: Update Database Config + changed_when: false + shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }}" + args: + chdir: "{{ base_path }}" \ No newline at end of file diff --git a/deploy/tasks/verify_database_config.yml b/deploy/tasks/verify_database_config.yml new file mode 100644 index 0000000000000000000000000000000000000000..1df3681e4eae0fe7d04cede8f5cd3820aa6cbd67 --- /dev/null +++ b/deploy/tasks/verify_database_config.yml @@ -0,0 +1,9 @@ + - name: Get Database Config + changed_when: false + shell: "./sbin/load_ConfigDb.sh CDB/{{ database_config }} 2>&1" + register: dsconfig_result + args: + chdir: "{{ base_path }}" + - name: Verify Database Config + when: '"No changes needed" in dsconfig_result.stdout' + debug: msg="Database changes stored" \ No newline at end of file diff --git a/docker-compose/Makefile b/docker-compose/Makefile index 5cdb54531b472901dbda22902937e62a0ab2902b..0ca1c240ba67498ebcef67c9c0516dc9d9e2b32d 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -149,6 +149,7 @@ DOCKER_COMPOSE_ARGS := DISPLAY=$(DISPLAY) \ XAUTHORITY_MOUNT=$(XAUTHORITY_MOUNT) \ CONTAINER_NAME_PREFIX=$(CONTAINER_NAME_PREFIX) \ COMPOSE_IGNORE_ORPHANS=true \ + COMPOSE_HTTP_TIMEOUT=180 \ CONTAINER_EXECUTION_UID=$(shell id -u) \ DOCKER_GID=$(DOCKER_GID) \ TEST_MODULE=$(INTEGRATION_MODULE) diff --git a/tangostationcontrol/VERSION b/tangostationcontrol/VERSION index 0ea3a944b399d25f7e1b8fe684d754eb8da9fe7f..9e11b32fcaa96816319e5d0dcff9fb2873f04061 100644 --- a/tangostationcontrol/VERSION +++ b/tangostationcontrol/VERSION @@ -1 +1 @@ -0.2.0 +0.3.1 diff --git a/tangostationcontrol/tangostationcontrol/clients/comms_client.py b/tangostationcontrol/tangostationcontrol/clients/comms_client.py index deb37f67153c429911a417a743a5a9d73d530fae..0188753a2f6ca1d439873903d9b8d021dcc7ee62 100644 --- a/tangostationcontrol/tangostationcontrol/clients/comms_client.py +++ b/tangostationcontrol/tangostationcontrol/clients/comms_client.py @@ -15,7 +15,7 @@ class AbstractCommClient(ABC): def stop(self): """ Stop communication with the client. """ - def ping(self): # noqa: B027 + def ping(self): # noqa: B027 """ Check whether the connection is still alive. Clients that override this method must raise an Exception if the diff --git a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py index 89eb31d8ef6db7f74345aab8b0ba7bf35de741e1..90c1c9161908ba8bd2724e364c0e0c500cce29df 100644 --- a/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py +++ b/tangostationcontrol/tangostationcontrol/integration_test/default/statistics/test_writer_sst.py @@ -82,7 +82,7 @@ class TestStatisticsWriterSST(BaseIntegrationTestCase): '2021-09-20T12:17:40.000+00:00' ) self.assertIsNotNone(stat) - self.assertEqual("0.2.0", stat.station_version_id) + self.assertEqual("0.3.1", stat.station_version_id) self.assertEqual("0.1", stat.writer_version_id) def test_insert_tango_SST_statistics(self): diff --git a/tangostationcontrol/tox.ini b/tangostationcontrol/tox.ini index 332d3777fd58491bfe96c270568a12e8d10f5f77..d528ccc75b0280f93740f5859987c225166f2c33 100644 --- a/tangostationcontrol/tox.ini +++ b/tangostationcontrol/tox.ini @@ -118,4 +118,4 @@ commands = [flake8] filename = *.py,.stestr.conf,.txt ignore = B014, B019, W291, W293, W391, E111, E114, E121, E122, E123, E124, E126, E127, E128, E131, E201, E201, E202, E203, E221, E222, E225, E226, E231, E241, E251, E252, E261, E262, E265, E271, E301, E302, E303, E305, E306, E401, E402, E501, E502, E701, E712, E721, E731, F403, F523, F541, F841, H301, H306, H401, H403, H404, H405, W503 -exclude=.tox,.egg-info,libhdbpp-python, SNMP_mib_loading +exclude=.tox,build,.egg-info,libhdbpp-python, SNMP_mib_loading