Merge branch 'fix-deployment-rs' into 'main'

Fix deployment for remote stations

See merge request !15

.deploy.gitlab-ci.yml

+variables:
+  STATION: "LOFAR station to deploy on"
+
+stages:
+  - deploy
+
+deploy_nomad:
+  stage: deploy
+  image:
+    name: hashicorp/levant
+    entrypoint: [ "" ]
+  parallel:
+    matrix:
+      - COMPONENT:
+          - stingray
+          - stingray-bucket-replication
+  environment:
+    name: $STATION
+  script:
+    - |
+      if [ "${STATION}" == "dts-lab" ]; then
+          # dts-lab test station
+          HOSTNAME="dts-lab.lofar.net"
+          EXTRA_VARFILES="-var-file=infra/env/cs.yaml"
+      elif [ "${STATION::2}" == "rs" ]; then
+          # remote station
+          HOSTNAME="${STATION}c.control.lofar"
+          EXTRA_VARFILES="-var-file=infra/env/rs.yaml"
+      else
+          # core station
+          HOSTNAME="${STATION}c.control.lofar"
+          EXTRA_VARFILES="-var-file=infra/env/cs.yaml"
+      fi
+
+      # To deploy manually, get the env.yaml and the .levant.nomad file,
+      # and run docker run --rm -i --net=host -v /path/to/env.yaml:/env.yaml:ro hashicorp/levant deploy --var-file=/env.yaml /dev/stdin < /path/to/file.levant.nomad
+      levant deploy \
+        -address="http://${HOSTNAME}:4646" \
+        -var-file=infra/env/common.yaml ${EXTRA_VARFILES} \
+        -var image_tag="${CI_COMMIT_REF_SLUG}" \
+        -var station="${STATION}" \
+        -ignore-no-changes \
+        infra/jobs/station/${COMPONENT}.levant.nomad
+
+workflow:
+  name: 'deploy on $STATION'
+  rules:
+    - when: always

.gitlab-ci.yml

@@ -113,7 +113,7 @@ package_docs:
   script:
     - tox -e docs
 
-docker_build:
+docker_build_image:
   stage: images
   image: docker:latest
   needs:
@@ -163,30 +163,24 @@ publish_to_readthedocs:
 
 deploy_nomad_station:
   stage: deploy
-  image:
-    name: hashicorp/levant
-    entrypoint: [ "" ]
-  when: manual
   rules:
-    - if: $CI_COMMIT_TAG
+    - if: ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH) || $CI_COMMIT_TAG
+  when: manual
+  needs:
+    - docker_build_image
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+  trigger:
+    include: .deploy.gitlab-ci.yml
+    forward:
+      pipeline_variables: true
   parallel:
     matrix:
       - STATION:
           - cs001
           - cs032
-        COMPONENT:
-          - stingray
-          - stingray-bucket-replication
-  environment:
-    name: $STATION
-  script:
-    - |
-      levant deploy \
-        -address="http://${STATION}c.control.lofar:4646" \
-        -var-file=infra/env.yaml \
-        -var station="${STATION}" \
-        -var image_tag="$CI_COMMIT_REF_SLUG" \
-        infra/jobs/station/${COMPONENT}.levant.nomad
+          - rs307
+          - dts-lab
 
 deploy_nomad_central:
   stage: deploy
@@ -194,6 +188,8 @@ deploy_nomad_central:
     name: hashicorp/levant
     entrypoint: [ "" ]
   when: manual
+  needs:
+    - docker_build_image
   rules:
     - if: $CI_COMMIT_TAG
   parallel:
@@ -205,7 +201,7 @@ deploy_nomad_central:
   script:
     - |
       levant deploy \
-        -address="http://codlet01.control.lofar:4646" \
+        -address="http://nomad.central.lofar.net:4646" \
         -var-file=infra/env.yaml \
         -var image_tag="$CI_COMMIT_REF_SLUG" \
         infra/jobs/central/${COMPONENT}.levant.nomad

infra/env/README.md

+This directory contains the station-specific deployment settings, and selected versions of external images. The YAML files
+are used by levant for rendering the nomad templates.
+
+ For production, they are loaded in `.gitlab-ci.yml` as
+part of deployment.
+
+NB:
+
+* For each section, only the one in the last applied file will be used. The sections will not be merged. So be careful when specifying a section in multiple files.

infra/env/common.yaml

+object_storage:
+  mc:
+    version: RELEASE.2024-08-26T10-49-58Z
+  user:
+    name: minioadmin
+    pass: minioadmin

infra/env.yaml → infra/env/cs.yaml

@@ -11,10 +11,3 @@ stingray:
     sst: 5121
     xst: 5122
     bst: 5123
-
-object_storage:
-  mc:
-    version: RELEASE.2024-03-09T06-43-06Z
-  user:
-    name: minioadmin
-    pass: minioadmin

infra/env/rs.yaml

+stingray:
+  lba:
+    sst: 5101
+    xst: 5102
+    bst: 5103
+  hba:
+    sst: 5111
+    xst: 5112
+    bst: 5113

lofar_stingray/aggregator/_sst_aggregator.py

@@ -31,7 +31,7 @@ class SstAggregator(BaseAggregator):
         packets: Iterable,
         nr_signal_inputs: int = MAX_INPUTS,
         first_signal_input_index: int = 0,
-    ):  # pylint: disable=too-many-arguments
+    ):  # pylint: disable=too-many-arguments,too-many-positional-arguments
         super().__init__(station, antennafield, metadata_packets)
         self.package_loader = packets
         self.nr_signal_inputs = nr_signal_inputs

lofar_stingray/aggregator/_xst_aggregator.py

@@ -49,7 +49,7 @@ class XstAggregator(BaseAggregator):
         packets: Iterable,
         nr_signal_inputs=MAX_INPUTS,
         first_signal_input_index=0,
-    ):  # pylint: disable=too-many-arguments
+    ):  # pylint: disable=too-many-arguments,too-many-positional-arguments
         super().__init__(station, antennafield, metadata_packets)
         self.packets = packets
         self.nr_signal_inputs = nr_signal_inputs