Skip to content
Snippets Groups Projects
Commit 33b85f5e authored by Hannes Feldt's avatar Hannes Feldt
Browse files

L2SS-1618: Fix lifecycle configuration

parent 226b65c5
No related branches found
No related tags found
1 merge request!2L2SS-1618: Fix lifecycle configuration
......@@ -10,7 +10,8 @@ build
dist
# Documentation
docs/source/source_documentation
docs/source/source_documentation/*
!docs/source/source_documentation/index.rst
docs/build
# Setuptools SCM
......
......@@ -18,8 +18,6 @@ stages:
# Caching of dependencies to speed up builds
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
include:
- template: Security/SAST.gitlab-ci.yml
......@@ -124,7 +122,7 @@ docker_build:
- name: docker:dind
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker build . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG
- docker build -f docker/package/Dockerfile . --build-arg BUILD_ENV=copy --tag $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG
- docker push $CI_REGISTRY_IMAGE/stingray:$CI_COMMIT_REF_SLUG
run_integration_tests:
......@@ -186,5 +184,5 @@ deploy_nomad:
levant deploy \
-address="http://${STATION}c.control.lofar:4646" \
-var-file=infra/env.yaml \
-var image_tag="$CI_COMMIT_TAG" \
-var image_tag="$CI_COMMIT_REF_SLUG" \
infra/jobs/station/stingray.levant.nomad
......@@ -6,9 +6,6 @@ build_ci_runner_image:
image: docker:stable
services:
- docker:dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- |
......
File moved
ARG BUILD_ENV=no_copy
FROM python:3.12 AS build_no_copy
ADD requirements.txt .
COPY . /work
ADD ../../requirements.txt .
COPY ../.. /work
RUN rm -r /work/dist | true
RUN python -m pip install --user tox
WORKDIR /work
......
#!/usr/bin/env python3
# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0
import os
file_dir = os.path.dirname(os.path.realpath(__file__))
clean_dir = os.path.join(file_dir, "source", "source_documentation")
print(f"Cleaning.. {clean_dir}/*")
for file_name in os.listdir(clean_dir):
file = os.path.join(clean_dir, file_name)
match file_name:
case "index.rst": continue
case _:
print(f"Removing.. {file}")
os.remove(file)
#!/bin/bash
#
# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0
#
FILE_DIR=$(dirname -- "$(readlink -f -- "${0}")")
echo "Cleaning.. ${FILE_DIR}/source/source_documentation/*"
for f in "${FILE_DIR}"/source/source_documentation/*
do
case $f in
*/index.rst) true;;
*) echo "Removing.. ${f}"; rm "${f}";;
esac
done
......@@ -3,3 +3,4 @@ sphinx-rtd-theme>=0.4.3 #MIT
sphinxcontrib-apidoc>=0.3.0 #BSD
myst-parser>=2.0 # MIT
docutils>=0.17 # BSD
diagrams # MIT
Deployment / Infrastructure
----------------------------
Stingray is deployed as containers running on the station.
For each statistic type (XST, SST, BST) and each antenna field, one container instance is running.
Each container is receiving the statistics packets for a specific type and antenna field via TCP,
recording them in chunks of up to 5 minutes and writing the result JSON encoded to a S3 bucket.
The bucket is located on the stations MinIO instance and configured to replicate the individual objects
to the central MinIO instance. To reduce the risk of data loss the objects are stored on the station for 24 hours.
.. image:: infrastructure.png
......@@ -14,3 +14,4 @@ Contents:
readme
source_documentation/index
deployment
# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: Apache-2.0
from diagrams import Diagram, Cluster, Edge
from diagrams.aws.storage import SimpleStorageServiceS3BucketWithObjects
from diagrams.digitalocean.compute import Docker
from diagrams.onprem.compute import Server
with Diagram("Infrastructure", show=False, direction="LR"):
with Cluster("Central"):
central_minio = SimpleStorageServiceS3BucketWithObjects("MinIO")
with Cluster("Station"):
minio = SimpleStorageServiceS3BucketWithObjects("MinIO")
sd = Server("statistics devices")
statistics_container = Docker("statistics writers")
sd >> Edge(label="statistics packets") >> statistics_container >> Edge(label="write blocks of 5m") >> minio
minio >> Edge(label="replicate") >> central_minio
docs/source/infrastructure.png

51.1 KiB

Source code documentation
=========================
.. toctree::
:maxdepth: 3
lofar_stingray
stingray:
sst:
lba: 5101
hba0: 5111
hba1: 5121
bst:
lba: 5103
hba0: 5113
hba1: 5123
xst:
lba: 5102
hba0: 5112
hba1: 5122
lba:
sst: 5101
xst: 5102
bst: 5103
hba0:
sst: 5111
xst: 5112
bst: 5113
hba1:
sst: 5121
xst: 5122
bst: 5123
......@@ -8,12 +8,12 @@ job "statistics" {
delay_function = "constant"
}
[[ range $st, $fields := $.stingray ]]
[[ range $af, $port := $fields ]]
group "stingray-[[ $st ]]-[[ $af ]]" {
count = 1
[[ range $af, $fields := $.stingray ]]
group "stingray-[[ $af ]]" {
count = 1
task "stingray-[[ $st ]]-[[ $af ]]" {
[[ range $st, $port := $fields ]]
task "stingray-[[ $af ]]-[[ $st ]]" {
driver = "docker"
config {
......@@ -37,7 +37,7 @@ job "statistics" {
memory = 512
}
}
[[ end ]]
}
[[ end ]]
[[ end ]]
}
......@@ -71,7 +71,7 @@ class Storage:
Rule(
ENABLED,
expiration=Expiration(days=1),
rule_filter=Filter(prefix="/"),
rule_filter=Filter(prefix=""),
)
]
),
......
importlib-metadata>=0.12, <5.0;python_version<"3.8"
numpy
minio~=7.1.17
lofar-station-client==0.18.3.dev0 # Apache 2
lofar-station-client>=0.18.3 # Apache 2
setuptools~=68.2.0
logfmter # MIT
......@@ -9,8 +9,6 @@ package = wheel
wheel_build_env = .pkg
setenv =
LANGUAGE=en_US
LC_ALL=en_US.UTF-8
PYTHONWARNINGS=default::DeprecationWarning
PIP_EXTRA_INDEX_URL={env:PIP_EXTRA_INDEX_URL:https://git.astron.nl/api/v4/projects/395/packages/pypi/simple}
deps =
......@@ -42,14 +40,12 @@ commands =
format: {envpython} -m black -v lofar_stingray tests
[testenv:docs]
allowlist_externals =
bash
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/docs/requirements.txt
changedir = {toxinidir}
commands =
bash docs/cleanup.sh
{envpython} docs/cleanup.py
sphinx-build -b html docs/source docs/build/html
[testenv:build]
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment