diff --git a/.gitignore b/.gitignore index 02e835b9647310478d3b41ffb2f5315bac921dfe..ec464cde5866c8010c6d4bf89d929139fb606609 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,8 @@ build dist # Documentation -docs/source/source_documentation +docs/source/source_documentation/* +!docs/source/source_documentation/index.rst docs/build # Setuptools SCM diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fedcccbac1ef39d9ec89b2c55bf34d0ff68ec1ec..af8ea8dbd378dff8bdebf4540a9c5ffd26a8e63f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,8 +18,6 @@ stages: # Caching of dependencies to speed up builds variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" include: - template: Security/SAST.gitlab-ci.yml @@ -124,7 +122,7 @@ docker_build: - name: docker:dind script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker build . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG + - docker build -f docker/package/Dockerfile . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG - docker push $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG run_integration_tests: @@ -186,5 +184,5 @@ deploy_nomad: levant deploy \ -address="http://${STATION}c.control.lofar:4646" \ -var-file=infra/env.yaml \ - -var image_tag="$CI_COMMIT_TAG" \ + -var image_tag="$CI_COMMIT_REF_SLUG" \ infra/jobs/station/stingray.levant.nomad diff --git a/.prepare.gitlab-ci.yml b/.prepare.gitlab-ci.yml index 718ab4cfa4c02c2dcd28491a16e1c4018d6b2e31..e531b6dfa16369347e85f91c5587fb8ed1607074 100644 --- a/.prepare.gitlab-ci.yml +++ b/.prepare.gitlab-ci.yml @@ -6,9 +6,6 @@ build_ci_runner_image: image: docker:stable services: - docker:dind - variables: - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - | diff --git a/compose.yaml b/docker/compose.yaml similarity index 100% rename from compose.yaml rename to docker/compose.yaml diff --git a/Dockerfile b/docker/package/Dockerfile similarity index 90% rename from Dockerfile rename to docker/package/Dockerfile index bce53c97e8815219bcc85ae2a87781019b74f1fb..22d9ef5f5ba7c05aaf6a0f6fd87ba5b57ab87d79 100644 --- a/Dockerfile +++ b/docker/package/Dockerfile @@ -1,8 +1,8 @@ ARG BUILD_ENV=no_copy FROM python:3.12 AS build_no_copy -ADD requirements.txt . -COPY . /work +ADD ../../requirements.txt . +COPY ../.. /work RUN rm -r /work/dist | true RUN python -m pip install --user tox WORKDIR /work diff --git a/docs/cleanup.py b/docs/cleanup.py new file mode 100644 index 0000000000000000000000000000000000000000..f5890d9cc7cc976261dfb986e403ed4ce68bcf93 --- /dev/null +++ b/docs/cleanup.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 + +import os + +file_dir = os.path.dirname(os.path.realpath(__file__)) + +clean_dir = os.path.join(file_dir, "source", "source_documentation") +print(f"Cleaning.. {clean_dir}/*") + +for file_name in os.listdir(clean_dir): + file = os.path.join(clean_dir, file_name) + match file_name: + case "index.rst": continue + case _: + print(f"Removing.. {file}") + os.remove(file) diff --git a/docs/cleanup.sh b/docs/cleanup.sh deleted file mode 100644 index 282221a5a82ed473b350aba469dcb11a777bc4dd..0000000000000000000000000000000000000000 --- a/docs/cleanup.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# -# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) -# SPDX-License-Identifier: Apache-2.0 -# - -FILE_DIR=$(dirname -- "$(readlink -f -- "${0}")") - -echo "Cleaning.. ${FILE_DIR}/source/source_documentation/*" - -for f in "${FILE_DIR}"/source/source_documentation/* -do - - case $f in - */index.rst) true;; - *) echo "Removing.. ${f}"; rm "${f}";; - esac -done diff --git a/docs/requirements.txt b/docs/requirements.txt index 3c6e46c6db7ddaf65e47cfa22c9ec0b914f7fd38..8fb62dbaea4129f09c7e10c0b012dee15ad45d3f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,3 +3,4 @@ sphinx-rtd-theme>=0.4.3 #MIT sphinxcontrib-apidoc>=0.3.0 #BSD myst-parser>=2.0 # MIT docutils>=0.17 # BSD +diagrams # MIT diff --git a/docs/source/deployment.rst b/docs/source/deployment.rst new file mode 100644 index 0000000000000000000000000000000000000000..566775f80a4f164ee8edaf71b9ca1d9ef89d6e8e --- /dev/null +++ b/docs/source/deployment.rst @@ -0,0 +1,14 @@ +Deployment / Infrastructure +---------------------------- + +Stingray is deployed as containers running on the station. +For each statistic type (XST, SST, BST) and each antenna field, one container instance is running. + +Each container is receiving the statistics packets for a specific type and antenna field via TCP, +recording them in chunks of up to 5 minutes and writing the result JSON encoded to a S3 bucket. + +The bucket is located on the stations MinIO instance and configured to replicate the individual objects +to the central MinIO instance. To reduce the risk of data loss the objects are stored on the station for 24 hours. + +.. image:: infrastructure.png + diff --git a/docs/source/index.rst b/docs/source/index.rst index b9b73e302704037e9e93e3eca2e153143a37f173..b0505ce88b14ff54fb0c13709b1088cc45c564ee 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,3 +14,4 @@ Contents: readme source_documentation/index + deployment diff --git a/docs/source/infra_diagrams.py b/docs/source/infra_diagrams.py new file mode 100644 index 0000000000000000000000000000000000000000..f808fae780023f569c99d3ea4eb6d4c839a5acea --- /dev/null +++ b/docs/source/infra_diagrams.py @@ -0,0 +1,20 @@ +# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy) +# SPDX-License-Identifier: Apache-2.0 + +from diagrams import Diagram, Cluster, Edge +from diagrams.aws.storage import SimpleStorageServiceS3BucketWithObjects +from diagrams.digitalocean.compute import Docker +from diagrams.onprem.compute import Server + +with Diagram("Infrastructure", show=False, direction="LR"): + with Cluster("Central"): + central_minio = SimpleStorageServiceS3BucketWithObjects("MinIO") + + with Cluster("Station"): + minio = SimpleStorageServiceS3BucketWithObjects("MinIO") + sd = Server("statistics devices") + statistics_container = Docker("statistics writers") + sd >> Edge(label="statistics packets") >> statistics_container >> Edge(label="write blocks of 5m") >> minio + minio >> Edge(label="replicate") >> central_minio + + diff --git a/docs/source/infrastructure.png b/docs/source/infrastructure.png new file mode 100644 index 0000000000000000000000000000000000000000..76f70278716df14152c9ce0e59d745575d6512d8 Binary files /dev/null and b/docs/source/infrastructure.png differ diff --git a/docs/source/source_documentation/index.rst b/docs/source/source_documentation/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..ec205bedffc1b931277c2bae1da46f6b3f6d3381 --- /dev/null +++ b/docs/source/source_documentation/index.rst @@ -0,0 +1,7 @@ +Source code documentation +========================= + +.. toctree:: + :maxdepth: 3 + + lofar_stingray diff --git a/infra/env.yaml b/infra/env.yaml index a0617c54d4f8ea7e8bf6ecf2ef5293263de8f6bd..6597eeacca3c3f7382543bbf1ef207707f6e2a72 100644 --- a/infra/env.yaml +++ b/infra/env.yaml @@ -1,13 +1,13 @@ stingray: - sst: - lba: 5101 - hba0: 5111 - hba1: 5121 - bst: - lba: 5103 - hba0: 5113 - hba1: 5123 - xst: - lba: 5102 - hba0: 5112 - hba1: 5122 + lba: + sst: 5101 + xst: 5102 + bst: 5103 + hba0: + sst: 5111 + xst: 5112 + bst: 5113 + hba1: + sst: 5121 + xst: 5122 + bst: 5123 diff --git a/infra/jobs/station/stingray.levant.nomad b/infra/jobs/station/stingray.levant.nomad index 532c6e512662fa5d7165e6ad76d5ff4382a16e91..0575250dad8c8f7f160d833024c438f381c67842 100644 --- a/infra/jobs/station/stingray.levant.nomad +++ b/infra/jobs/station/stingray.levant.nomad @@ -8,12 +8,12 @@ job "statistics" { delay_function = "constant" } - [[ range $st, $fields := $.stingray ]] - [[ range $af, $port := $fields ]] - group "stingray-[[ $st ]]-[[ $af ]]" { - count = 1 + [[ range $af, $fields := $.stingray ]] + group "stingray-[[ $af ]]" { + count = 1 - task "stingray-[[ $st ]]-[[ $af ]]" { + [[ range $st, $port := $fields ]] + task "stingray-[[ $af ]]-[[ $st ]]" { driver = "docker" config { @@ -37,7 +37,7 @@ job "statistics" { memory = 512 } } + [[ end ]] } [[ end ]] - [[ end ]] } diff --git a/lofar_stingray/writer.py b/lofar_stingray/writer.py index 9109f2942bea03fc3b16d2b685e06ae4a9b7d581..a1cd6b7a32dc4c87d017b516152e2df4067391c5 100644 --- a/lofar_stingray/writer.py +++ b/lofar_stingray/writer.py @@ -71,7 +71,7 @@ class Storage: Rule( ENABLED, expiration=Expiration(days=1), - rule_filter=Filter(prefix="/"), + rule_filter=Filter(prefix=""), ) ] ), diff --git a/requirements.txt b/requirements.txt index c2b2c4d3bdfe02a4333ef0ba0ee50d6581d417f7..f9e7926ee28da48e2aa6ba5c00fc1ed7d3c8bd3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ importlib-metadata>=0.12, <5.0;python_version<"3.8" numpy minio~=7.1.17 -lofar-station-client==0.18.3.dev0 # Apache 2 +lofar-station-client>=0.18.3 # Apache 2 setuptools~=68.2.0 logfmter # MIT diff --git a/tox.ini b/tox.ini index ec3b6ac26767dfe151ef380522440d40c129188c..15f8c057a7d24783afd9c744a3450275304f5dc3 100644 --- a/tox.ini +++ b/tox.ini @@ -9,8 +9,6 @@ package = wheel wheel_build_env = .pkg setenv = - LANGUAGE=en_US - LC_ALL=en_US.UTF-8 PYTHONWARNINGS=default::DeprecationWarning PIP_EXTRA_INDEX_URL={env:PIP_EXTRA_INDEX_URL:https://git.astron.nl/api/v4/projects/395/packages/pypi/simple} deps = @@ -42,14 +40,12 @@ commands = format: {envpython} -m black -v lofar_stingray tests [testenv:docs] -allowlist_externals = - bash deps = -r{toxinidir}/requirements.txt -r{toxinidir}/docs/requirements.txt changedir = {toxinidir} commands = - bash docs/cleanup.sh + {envpython} docs/cleanup.py sphinx-build -b html docs/source docs/build/html [testenv:build]