Merge branch 'L2SS-1618_fix_lifecycle_configuration' into 'main'

L2SS-1618: Fix lifecycle configuration

Closes L2SS-1618

See merge request !2

.gitignore

@@ -10,7 +10,8 @@ build
 dist
 
 # Documentation
-docs/source/source_documentation
+docs/source/source_documentation/*
+!docs/source/source_documentation/index.rst
 docs/build
 
 # Setuptools SCM

.gitlab-ci.yml

@@ -18,8 +18,6 @@ stages:
 # Caching of dependencies to speed up builds
 variables:
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  DOCKER_DRIVER: overlay2
-  DOCKER_TLS_CERTDIR: ""
 
 include:
   - template: Security/SAST.gitlab-ci.yml
@@ -124,7 +122,7 @@ docker_build:
     - name: docker:dind
   script:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker build . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG
+    - docker build -f docker/package/Dockerfile . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG
     - docker push $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG
 
 run_integration_tests:
@@ -186,5 +184,5 @@ deploy_nomad:
       levant deploy \
         -address="http://${STATION}c.control.lofar:4646" \
         -var-file=infra/env.yaml \
-        -var image_tag="$CI_COMMIT_TAG" \
+        -var image_tag="$CI_COMMIT_REF_SLUG" \
         infra/jobs/station/stingray.levant.nomad

.prepare.gitlab-ci.yml

@@ -6,9 +6,6 @@ build_ci_runner_image:
   image: docker:stable
   services:
     - docker:dind
-  variables:
-    DOCKER_DRIVER: overlay2
-    DOCKER_TLS_CERTDIR: ""
   script:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - |

Dockerfile → docker/package/Dockerfile

 ARG BUILD_ENV=no_copy
 
 FROM python:3.12 AS build_no_copy
-ADD requirements.txt .
-COPY . /work
+ADD ../../requirements.txt .
+COPY ../.. /work
 RUN rm -r /work/dist | true
 RUN python -m pip install --user tox
 WORKDIR /work

docs/cleanup.py

+#!/usr/bin/env python3
+
+#  Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+#  SPDX-License-Identifier: Apache-2.0
+
+import os
+
+file_dir = os.path.dirname(os.path.realpath(__file__))
+
+clean_dir = os.path.join(file_dir, "source", "source_documentation")
+print(f"Cleaning.. {clean_dir}/*")
+
+for file_name in os.listdir(clean_dir):
+    file = os.path.join(clean_dir, file_name)
+    match file_name:
+        case "index.rst": continue
+        case _:
+            print(f"Removing.. {file}")
+            os.remove(file)

docs/cleanup.sh

-#!/bin/bash
-
-#
-# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
-# SPDX-License-Identifier: Apache-2.0
-#
-
-FILE_DIR=$(dirname -- "$(readlink -f -- "${0}")")
-
-echo "Cleaning.. ${FILE_DIR}/source/source_documentation/*"
-
-for f in "${FILE_DIR}"/source/source_documentation/*
-do
-
-  case $f in
-    */index.rst) true;;
-    *) echo "Removing.. ${f}"; rm "${f}";;
-  esac
-done

docs/requirements.txt

@@ -3,3 +3,4 @@ sphinx-rtd-theme>=0.4.3 #MIT
 sphinxcontrib-apidoc>=0.3.0 #BSD
 myst-parser>=2.0 # MIT
 docutils>=0.17 # BSD
+diagrams # MIT

docs/source/deployment.rst

+Deployment / Infrastructure
+----------------------------
+
+Stingray is deployed as containers running on the station.
+For each statistic type (XST, SST, BST) and each antenna field, one container instance is running.
+
+Each container is receiving the statistics packets for a specific type and antenna field via TCP,
+recording them in chunks of up to 5 minutes and writing the result JSON encoded to a S3 bucket.
+
+The bucket is located on the stations MinIO instance and configured to replicate the individual objects
+to the central MinIO instance. To reduce the risk of data loss the objects are stored on the station for 24 hours.
+
+.. image:: infrastructure.png
+

docs/source/index.rst

@@ -14,3 +14,4 @@ Contents:
 
    readme
    source_documentation/index
+   deployment

docs/source/infra_diagrams.py

+#  Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
+#  SPDX-License-Identifier: Apache-2.0
+
+from diagrams import Diagram, Cluster, Edge
+from diagrams.aws.storage import SimpleStorageServiceS3BucketWithObjects
+from diagrams.digitalocean.compute import Docker
+from diagrams.onprem.compute import Server
+
+with Diagram("Infrastructure", show=False, direction="LR"):
+    with Cluster("Central"):
+        central_minio = SimpleStorageServiceS3BucketWithObjects("MinIO")
+
+    with Cluster("Station"):
+        minio = SimpleStorageServiceS3BucketWithObjects("MinIO")
+        sd = Server("statistics devices")
+        statistics_container = Docker("statistics writers")
+        sd >> Edge(label="statistics packets") >> statistics_container >> Edge(label="write blocks of 5m") >> minio
+        minio >> Edge(label="replicate") >> central_minio
+
+

docs/source/source_documentation/index.rst

+Source code documentation
+=========================
+
+.. toctree::
+   :maxdepth: 3
+
+   lofar_stingray

infra/env.yaml

 stingray:
-  sst:
-    lba: 5101
-    hba0: 5111
-    hba1: 5121
-  bst:
-    lba: 5103
-    hba0: 5113
-    hba1: 5123
-  xst:
-    lba: 5102
-    hba0: 5112
-    hba1: 5122
+  lba:
+    sst: 5101
+    xst: 5102
+    bst: 5103
+  hba0:
+    sst: 5111
+    xst: 5112
+    bst: 5113
+  hba1:
+    sst: 5121
+    xst: 5122
+    bst: 5123

infra/jobs/station/stingray.levant.nomad

@@ -8,12 +8,12 @@ job "statistics" {
       delay_function = "constant"
     }
 
-    [[ range $st, $fields := $.stingray ]]
-    [[ range $af, $port := $fields ]]
-    group "stingray-[[ $st ]]-[[ $af ]]" {
-        count = 1
+    [[ range $af, $fields := $.stingray ]]
+    group "stingray-[[ $af ]]" {
+      count = 1
 
-        task "stingray-[[ $st ]]-[[ $af ]]" {
+      [[ range $st, $port := $fields ]]
+        task "stingray-[[ $af ]]-[[ $st ]]" {
             driver = "docker"
 
             config {
@@ -37,7 +37,7 @@ job "statistics" {
                 memory = 512
             }
         }
+      [[ end ]]
     }
     [[ end ]]
-    [[ end ]]
 }

lofar_stingray/writer.py

@@ -71,7 +71,7 @@ class Storage:
                     Rule(
                         ENABLED,
                         expiration=Expiration(days=1),
-                        rule_filter=Filter(prefix="/"),
+                        rule_filter=Filter(prefix=""),
                     )
                 ]
             ),

requirements.txt

 importlib-metadata>=0.12, <5.0;python_version<"3.8"
 numpy
 minio~=7.1.17
-lofar-station-client==0.18.3.dev0  # Apache 2
+lofar-station-client>=0.18.3  # Apache 2
 setuptools~=68.2.0
 logfmter # MIT

tox.ini

@@ -9,8 +9,6 @@ package = wheel
 wheel_build_env = .pkg
 
 setenv =
-    LANGUAGE=en_US
-    LC_ALL=en_US.UTF-8
     PYTHONWARNINGS=default::DeprecationWarning
     PIP_EXTRA_INDEX_URL={env:PIP_EXTRA_INDEX_URL:https://git.astron.nl/api/v4/projects/395/packages/pypi/simple}
 deps =
@@ -42,14 +40,12 @@ commands =
     format: {envpython} -m black -v lofar_stingray tests
 
 [testenv:docs]
-allowlist_externals =
-    bash
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/docs/requirements.txt
 changedir = {toxinidir}
 commands =
-    bash docs/cleanup.sh
+    {envpython} docs/cleanup.py
     sphinx-build -b html docs/source docs/build/html
 
 [testenv:build]