Skip to content
Snippets Groups Projects
Commit 76c9efd0 authored by Hannes Feldt's avatar Hannes Feldt
Browse files

Merge branch 'L2SS-1580-replicate-statistics-data' into 'main'

L2SS-1580: Automatically replicate statistics data from station to central

Closes L2SS-1580

See merge request !4
parents b45d2c88 5e8f6b00
No related branches found
No related tags found
1 merge request!4L2SS-1580: Automatically replicate statistics data from station to central
Pipeline #78597 passed with warnings
Pipeline: Stingray

#78598

    ......@@ -177,6 +177,9 @@ deploy_nomad:
    matrix:
    - STATION:
    - cs001
    COMPONENT:
    - stingray
    - stingray-bucket-replication
    environment:
    name: $STATION
    script:
    ......@@ -184,5 +187,6 @@ deploy_nomad:
    levant deploy \
    -address="http://${STATION}c.control.lofar:4646" \
    -var-file=infra/env.yaml \
    -var station="${STATION}" \
    -var image_tag="$CI_COMMIT_REF_SLUG" \
    infra/jobs/station/stingray.levant.nomad
    infra/jobs/station/${COMPONENT}.levant.nomad
    ......@@ -11,3 +11,10 @@ stingray:
    sst: 5121
    xst: 5122
    bst: 5123
    object_storage:
    mc:
    version: RELEASE.2024-03-09T06-43-06Z
    user:
    name: minioadmin
    pass: minioadmin
    job "statistics" {
    datacenters = ["stat"]
    type = "service"
    datacenters = ["stat"]
    type = "service"
    reschedule {
    unlimited = true
    delay = "10s"
    delay_function = "constant"
    reschedule {
    unlimited = true
    delay = "10s"
    delay_function = "constant"
    }
    group "replicate" {
    count = 1
    network {
    mode = "bridge"
    }
    [[ range $af, $fields := $.stingray ]]
    group "stingray-[[ $af ]]" {
    count = 1
    task "mc" {
    driver = "docker"
    config {
    image = "minio/mc:[[$.object_storage.mc.version]]"
    network {
    mode = "bridge"
    }
    entrypoint = ["mc", "mirror", "--watch", "local/statistics", "central/central-statistics"]
    [[ range $st, $port := $fields ]]
    task "stingray-[[ $af ]]-[[ $st ]]" {
    driver = "docker"
    mount {
    type = "bind"
    source = "local/mc"
    target = "/root/.mc"
    }
    }
    config {
    image = "git.astron.nl:5000/lofar2.0/stingray/stingray:[[$.image_tag]]"
    env {
    MINIO_ROOT_USER = "[[$.object_storage.user.name]]"
    MINIO_ROOT_PASSWORD = "[[$.object_storage.user.pass]]"
    }
    entrypoint = [
    "l2ss-stingray-record",
    "tcp://device-[[ $st ]].service.consul:[[ $port ]]",
    "s3://statistics/[[ $st ]]/[[ $af ]]"
    ]
    }
    resources {
    cpu = 10
    memory = 1024
    }
    env {
    MINIO_ROOT_USER = "minioadmin"
    MINIO_ROOT_PASSWORD = "minioadmin"
    }
    template {
    destination = "local/mc/config.json"
    change_mode = "noop"
    data = <<EOF
    {
    "aliases": {
    "local": {
    "url": "http://s3.service.consul:9000",
    "accessKey": "[[$.object_storage.user.name]]",
    "secretKey": "[[$.object_storage.user.name]]",
    "api": "s3v4",
    "path": "on"
    },
    "central": {
    "url": "http://s3.service.lofar-central.consul:9000",
    "accessKey": "[[$.object_storage.user.name]]",
    "secretKey": "[[$.object_storage.user.name]]",
    "api": "s3v4",
    "path": "on"
    }
    }
    }
    EOF
    }
    }
    }
    [[ range $af, $fields := $.stingray ]]
    group "stingray-[[ $af ]]" {
    count = 1
    resources {
    cpu = 10
    memory = 512
    }
    }
    [[ end ]]
    network {
    mode = "bridge"
    }
    [[ range $st, $port := $fields ]]
    task "stingray-[[ $af ]]-[[ $st ]]" {
    driver = "docker"
    config {
    image = "git.astron.nl:5000/lofar2.0/stingray/stingray:[[$.image_tag]]"
    entrypoint = [
    "l2ss-stingray-record",
    "tcp://device-[[ $st ]].service.consul:[[ $port ]]",
    "s3://statistics/[[$.station]]/[[ $st ]]/[[ $af ]]"
    ]
    }
    env {
    MINIO_ROOT_USER = "[[$.object_storage.user.name]]"
    MINIO_ROOT_PASSWORD = "[[$.object_storage.user.name]]"
    }
    resources {
    cpu = 10
    memory = 512
    }
    }
    [[ end ]]
    }
    [[ end ]]
    }
    0% Loading or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment