diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 00329539ce60e2d3c213c3c5c9cd0155c5e6add2..e0c1671ec5d67d3cd2100d100c5fb0d74ff2ec80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,7 +73,8 @@ wheel_packaging: tag="$CI_COMMIT_REF_SLUG" echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" fi - - apk add --update make bash docker-compose + - apk add --update make bash docker-compose python3 py3-pip + - pip install shyaml - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - touch ~/.Xauthority # Hack BASH_SOURCE into sourced files, docker its sh shell won't set this @@ -294,6 +295,8 @@ unit_test: - apk add --update make bash - apk add --update bind-tools - apk add --update postgresql14-client gzip socat + - apk add --update python3 py3-pip + - pip install shyaml - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - touch /root/.Xauthority # Hack BASH_SOURCE into sourced files, docker its sh shell won't set this @@ -314,7 +317,6 @@ unit_test: integration_test_docker: extends: .test_docker - allow_failure: true # until there is a machine that can properly run them script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag diff --git a/README.md b/README.md index 14972a0685098e3bdd032fb0e4a5dfe10a288e61..edddcae766666a599c22193af9a4ee50711dcaaf 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ Next change the version in the following places: # Release Notes +* 0.24.8 Push docker images to own registry and pull from there for deployments * 0.24.6 Bugfix: deploy tagged version instead of latest * 0.24.5 Bugfix: install 'tango' in Jupyter Lab again * 0.24.4 Apply calibration values in reverse order if subband frequencies are decreasing diff --git a/docker-compose/.env b/docker-compose/.env index 197682add5e07451141d2dd1c9cc2986a4544853..fdd79db3f7b864f865aaffbcb71346a51585bda2 100644 --- a/docker-compose/.env +++ b/docker-compose/.env @@ -1,5 +1,5 @@ -DOCKER_REGISTRY_HOST=artefact.skao.int -DOCKER_REGISTRY_USER=ska-tango-images +SKA_DOCKER_REGISTRY_HOST=artefact.skao.int +SKA_DOCKER_REGISTRY_USER=ska-tango-images LOCAL_DOCKER_REGISTRY_HOST=git.astron.nl:5000 LOCAL_DOCKER_REGISTRY_LOFAR=lofar2.0 LOCAL_DOCKER_REGISTRY_USER=lofar2.0/tango diff --git a/infra/dev/nomad/nomad.hcl b/infra/dev/nomad/nomad.hcl index 897d6d08dfc8ecf216690ae3b5596cbdf478128f..214e22257b1abf3e3f23598878f1cde92d75bae0 100644 --- a/infra/dev/nomad/nomad.hcl +++ b/infra/dev/nomad/nomad.hcl @@ -49,7 +49,7 @@ resource "container" "consul" { } image { - name = "hashicorp/consul:latest" + name = "git.astron.nl:5000/lofar2.0/tango/consul:latest" } command = [ diff --git a/infra/env.yaml b/infra/env.yaml index a9be0453158af4dd75c5312afd9d71cdf3a780d8..a3b3202358c3da9b5d7dadecb24d86e01729adae 100644 --- a/infra/env.yaml +++ b/infra/env.yaml @@ -6,6 +6,10 @@ tango: rest: version: 1.14.8 +registry: + astron: + url: git.astron.nl:5000/lofar2.0/tango + monitoring: db: version: 15.4 diff --git a/infra/jobs/station/Makefile b/infra/jobs/station/Makefile index 15bcff97bce249bc9ef432c2043b42d3e3bef57f..e145e6a681aa387879d908d0646d776f4a764162 100644 --- a/infra/jobs/station/Makefile +++ b/infra/jobs/station/Makefile @@ -12,7 +12,7 @@ ENV ?= ../../env.yaml render: pull $(JOBS) $(DEVICES) pull: - docker pull -q hashicorp/levant + docker pull -q git.astron.nl:5000/lofar2.0/tango/levant %.nomad: %.levant.nomad - docker run --rm -v $(realpath $(ENV)):/env.yaml -v $(realpath $(DIR_SRC)):/in -v $(realpath $(DIR_OUT)):/out hashicorp/levant render -var-file=/env.yaml -var image_tag="$(TAG)" -var station="$(STATION)" -out=/out/$@ /in/$< + docker run --rm -v $(realpath $(ENV)):/env.yaml -v $(realpath $(DIR_SRC)):/in -v $(realpath $(DIR_OUT)):/out git.astron.nl:5000/lofar2.0/tango/levant render -var-file=/env.yaml -var image_tag="$(TAG)" -var station="$(STATION)" -out=/out/$@ /in/$< diff --git a/infra/jobs/station/device-server.levant.nomad b/infra/jobs/station/device-server.levant.nomad index 4b313fa582a29a3ced6fedbfa358080e51d27813..98083ec80b6a339c3b47764ef59f8c0bfaea4dd6 100644 --- a/infra/jobs/station/device-server.levant.nomad +++ b/infra/jobs/station/device-server.levant.nomad @@ -45,7 +45,7 @@ job "device-servers" { driver = "docker" config { - image = "git.astron.nl:5000/lofar2.0/tango/lofar-device-base:[[ $.image_tag ]]" + image = "[[ $.registry.astron.url ]]/lofar-device-base:[[ $.image_tag ]]" ports = [ [[ range $port_name, $port := .ports]] "[[ $port_name ]]", diff --git a/infra/jobs/station/dsconfig.levant.nomad b/infra/jobs/station/dsconfig.levant.nomad index cd5eac570d88ecd36dbef909bbc6f6e061669e1b..ec12bb74640c29c40c7e5aba39022f769af402b1 100644 --- a/infra/jobs/station/dsconfig.levant.nomad +++ b/infra/jobs/station/dsconfig.levant.nomad @@ -13,7 +13,7 @@ job "dsconfig" { driver = "docker" config { - image = "git.astron.nl:5000/lofar2.0/tango/dsconfig:[[ .image_tag ]]" + image = "[[ .registry.astron.url ]]/dsconfig:[[ .image_tag ]]" mount { type = "bind" source = "local/dsconfig-update-settings.json" diff --git a/infra/jobs/station/ec-sim.levant.nomad b/infra/jobs/station/ec-sim.levant.nomad index 1880d16d280f3e2cd04b15a68f7ba9c840e8f5ea..ba2fa2e4c2b3526e00e0800682a397f621ac1b1c 100644 --- a/infra/jobs/station/ec-sim.levant.nomad +++ b/infra/jobs/station/ec-sim.levant.nomad @@ -22,7 +22,7 @@ job "simulators" { driver = "docker" config { - image = "git.astron.nl:5000/lofar2.0/tango/ec-sim:latest" + image = "[[.registry.astron.url]]/ec-sim:latest" ports = ["opcua"] } resources { diff --git a/infra/jobs/station/jupyter.levant.nomad b/infra/jobs/station/jupyter.levant.nomad index 81f553a5d303277eb08aecf4213b21e410ade95d..a836699f8081e9ef6f91701a6479c77d82397c51 100644 --- a/infra/jobs/station/jupyter.levant.nomad +++ b/infra/jobs/station/jupyter.levant.nomad @@ -34,7 +34,7 @@ job "jupyter" { } config { - image = "git.astron.nl:5000/lofar2.0/tango/jupyter-lab:[[.image_tag]]" + image = "[[.registry.astron.url]]/jupyter-lab:[[.image_tag]]" ports = ["jupyter"] mount { type = "bind" diff --git a/infra/jobs/station/logging.levant.nomad b/infra/jobs/station/logging.levant.nomad index 79a6dc0ebbed97beb4f7a4b92dbef2e21bfc5b12..636a38704394a2ae5f8c202f63f5e8b8639e5c0d 100644 --- a/infra/jobs/station/logging.levant.nomad +++ b/infra/jobs/station/logging.levant.nomad @@ -55,7 +55,7 @@ job "log-scraping" { task "vector" { driver = "docker" config { - image = "timberio/vector:0.32.1.custom.989ad14-distroless-static" + image = "[[.registry.astron.url]]/vector:0.32.1.custom.989ad14-distroless-static" ports = ["api"] } # docker socket volume mount diff --git a/infra/jobs/station/monitoring.levant.nomad b/infra/jobs/station/monitoring.levant.nomad index a9010f7477ff2ca104252385bd1196504d0a35af..a9030b057b5cf7031a237e3446555513028b5058 100644 --- a/infra/jobs/station/monitoring.levant.nomad +++ b/infra/jobs/station/monitoring.levant.nomad @@ -39,7 +39,7 @@ job "monitoring" { } config { - image = "postgres:[[.monitoring.db.version]]" + image = "[[.registry.astron.url]]/postgres:[[.monitoring.db.version]]" ports = ["postgres"] } @@ -97,7 +97,7 @@ job "monitoring" { driver = "docker" config { - image = "postgres:[[.monitoring.db.version]]" + image = "[[.registry.astron.url]]/postgres:[[.monitoring.db.version]]" command = "sh" args = ["-c", "while ! pg_isready -h localhost; do sleep 1; done"] } @@ -118,7 +118,7 @@ localhost:5432:grafana:postgres:password task "grafana" { driver = "docker" config { - image = "git.astron.nl:5000/lofar2.0/tango/grafana:[[.image_tag]]" + image = "[[.registry.astron.url]]/grafana:[[.image_tag]]" ports = ["http"] mount { type = "bind" @@ -237,7 +237,7 @@ localhost:5432:grafana:postgres:password } config { - image = "git.astron.nl:5000/lofar2.0/tango/prometheus:[[.image_tag]]" + image = "[[.registry.astron.url]]/prometheus:[[.image_tag]]" ports = ["prometheus"] args = [ "--config.file=/etc/prometheus/prometheus.yml", @@ -381,7 +381,7 @@ localhost:5432:grafana:postgres:password } config { - image = "git.astron.nl:5000/lofar2.0/tango/loki:[[.image_tag]]" + image = "[[.registry.astron.url]]/loki:[[.image_tag]]" ports = ["prometheus"] } diff --git a/infra/jobs/station/object-storage.levant.nomad b/infra/jobs/station/object-storage.levant.nomad index 00cd9701f4e92d2813cc6326b3b66888d37184ea..52acbca7e70d783127f09b9bbb5dd7ca4b74ab71 100644 --- a/infra/jobs/station/object-storage.levant.nomad +++ b/infra/jobs/station/object-storage.levant.nomad @@ -52,7 +52,7 @@ job "object-storage" { } config { - image = "minio/minio:[[.object_storage.minio.version]]" + image = "[[.registry.astron.url]]/minio:[[.object_storage.minio.version]]" ports = ["s3", "console"] command = "server" args = ["--console-address", ":9001", "/data"] @@ -74,7 +74,7 @@ job "object-storage" { task "vector" { driver = "docker" config { - image = "timberio/vector:0.32.1.custom.989ad14-distroless-static" + image = "[[.registry.astron.url]]/vector:0.32.1.custom.989ad14-distroless-static" ports = ["metrics"] } # Vector won't start unless the sinks(backends) configured are healthy diff --git a/infra/jobs/station/tango-prometheus-exporter.levant.nomad b/infra/jobs/station/tango-prometheus-exporter.levant.nomad index 6f65141cb3cf10f141bc945d245451d6ccc27986..4d613257a41fca13da5433ac8fc7f980aa4d30cf 100644 --- a/infra/jobs/station/tango-prometheus-exporter.levant.nomad +++ b/infra/jobs/station/tango-prometheus-exporter.levant.nomad @@ -21,7 +21,7 @@ job "tango-prometheus-exporter" { driver = "docker" config { - image = "git.astron.nl:5000/lofar2.0/tango/tango-prometheus-exporter:[[.image_tag]]" + image = "[[.registry.astron.url]]/tango-prometheus-exporter:[[.image_tag]]" ports = ["http"] command = "--config=/code/lofar2-policy.json" } @@ -54,7 +54,7 @@ job "tango-prometheus-exporter" { driver = "docker" config { - image = "git.astron.nl:5000/lofar2.0/tango/tango-prometheus-exporter:[[.image_tag]]" + image = "[[.registry.astron.url]]/tango-prometheus-exporter:[[.image_tag]]" ports = ["http"] command = "--config=/code/lofar2-fast-policy.json" } @@ -87,7 +87,7 @@ job "tango-prometheus-exporter" { driver = "docker" config { - image = "git.astron.nl:5000/lofar2.0/tango/tango-prometheus-exporter:[[.image_tag]]" + image = "[[.registry.astron.url]]tango-prometheus-exporter:[[.image_tag]]" ports = ["http"] command = "--config=/code/lofar2-slow-policy.json" } diff --git a/infra/jobs/station/tango.levant.nomad b/infra/jobs/station/tango.levant.nomad index 0003c4106306725bbe706fa22974d7aab20339c7..82a7ef1ae8b0fbc28e341e49b2cab88e70e5ef6c 100644 --- a/infra/jobs/station/tango.levant.nomad +++ b/infra/jobs/station/tango.levant.nomad @@ -48,7 +48,7 @@ job "tango" { } config { - image = "git.astron.nl:5000/lofar2.0/tango/tango-db:[[.tango.db.version]]" + image = "[[.registry.astron.url]]/tango-db:[[.tango.db.version]]" ports = ["mysql"] } @@ -103,7 +103,7 @@ job "tango" { driver = "docker" config { - image = "busybox" + image = "[[.registry.astron.url]]/busybox" command = "sh" args = ["-c", "while ! nc -z $MYSQL_HOST $MYSQL_PORT; do sleep 1; done"] } @@ -125,7 +125,7 @@ job "tango" { config { - image = "git.astron.nl:5000/lofar2.0/tango/tango-databaseds:[[.tango.databaseds.version]]" + image = "[[.registry.astron.url]]/tango-databaseds:[[.tango.databaseds.version]]" ports = ["tango"] entrypoint = [ "/usr/local/bin/DataBaseds", diff --git a/sbin/prepare_dev_env.sh b/sbin/prepare_dev_env.sh index d3dbc6094ef114ee9896a64ac23857b887a36d05..0d20b2ecfa3735b9d845f8ac3656e128fc145488 100755 --- a/sbin/prepare_dev_env.sh +++ b/sbin/prepare_dev_env.sh @@ -62,8 +62,8 @@ if [ "$(docker volume list | grep -c "$docker_volume")" = "0" ]; then docker volume create "$docker_volume" fi -docker pull -q bash -docker run --rm -i -v "$docker_volume":/mnt bash bash <<- EOM +docker pull -q git.astron.nl:5000/lofar2.0/tango/bash +docker run --rm -i -v "$docker_volume":/mnt git.astron.nl:5000/lofar2.0/tango/bash bash <<- EOM mkdir -p /mnt/volumes/tango-database mkdir -p /mnt/volumes/monitoring-postgresql-data mkdir -p /mnt/volumes/monitoring-loki-data diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index 3fc5527eb9defabd74258b8d9d2a06fd695eab06..83fc8543ade99ccc335e0113f22c37825e9f8795 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -52,8 +52,16 @@ fi # shellcheck disable=SC1090,SC1091 . "${LOFAR20_DIR}/docker-compose/.env" || exit 1 -# List of images and their tag -REMOTE_IMAGES=( +if [ -z "$(which shyaml)" ]; then + echo "Shyaml not found!, install using: 'pip install shyaml'" + exit 1 +fi + +POSTGRES_VERSION=$(shyaml get-value monitoring.db.version < "${LOFAR20_DIR}/infra/env.yaml") +MINIO_VERSION=$(shyaml get-value object_storage.minio.version < "${LOFAR20_DIR}/infra/env.yaml") + +# List of images and their tag from ska repository +REMOTE_SKA_IMAGES=( "tango-dsconfig:${TANGO_DSCONFIG_VERSION}" "tango-itango:${TANGO_ITANGO_VERSION}" "tango-cpp:${TANGO_CPP_VERSION}" @@ -63,13 +71,24 @@ REMOTE_IMAGES=( "tango-rest:${TANGO_REST_VERSION}" ) +# List of images and their tag from dockerhub +REMOTE_IMAGES=( + "vector 0.32.1.custom.989ad14-distroless-static timberio" + "postgres ${POSTGRES_VERSION}" + "levant latest hashicorp" + "consul latest hashicorp" + "minio ${MINIO_VERSION} minio" + "busybox latest" + "bash latest" +) + # Triple tuple of docker-compose names, image names and if necessary for # integration tests. # TODO(Corne): Have this list generated from the .yml files LOCAL_IMAGES=( "lofar-device-base lofar-device-base y" - "dsconfig dsconfig n" + "dsconfig dsconfig y" "ec-sim ec-sim y" @@ -115,9 +134,23 @@ if [ -z "${1+x}" ]; then echo "Pulling and retagging remote images" # Iterate over al the REMOTE_IMAGES and pull them from remote and push local - for image in "${REMOTE_IMAGES[@]}"; do - remote_url="${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-${image}" + for image in "${REMOTE_SKA_IMAGES[@]}"; do local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}" + remote_url="${SKA_DOCKER_REGISTRY_HOST}/${SKA_DOCKER_REGISTRY_USER}-${image}" + docker pull "${remote_url}" + docker tag "${remote_url}" "${local_url}" + docker push "${local_url}" + done + + for image in "${REMOTE_IMAGES[@]}"; do + # shellcheck disable=SC2086 + set -- $image + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${1}:${2}" + if [ -n "${3+x}" ]; then + remote_url="${3}/${1}:${2}" + else + remote_url="${1}:${2}" + fi docker pull "${remote_url}" docker tag "${remote_url}" "${local_url}" docker push "${local_url}" diff --git a/tangostationcontrol/VERSION b/tangostationcontrol/VERSION index c9731ff41861f2d4122dc88c8c7a2df7132a9e06..1f1cd637cf56f241b332f83cc52ed1b190d31b27 100644 --- a/tangostationcontrol/VERSION +++ b/tangostationcontrol/VERSION @@ -1 +1 @@ -0.24.7 +0.24.8