L2SS-1637: Push all nomad dockerhub images to own registry instead

.gitlab-ci.yml

@@ -73,7 +73,8 @@ wheel_packaging:
         tag="$CI_COMMIT_REF_SLUG"
         echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
       fi
-    - apk add --update make bash docker-compose
+    - apk add --update make bash docker-compose python3 py3-pip
+    - pip install shyaml
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - touch ~/.Xauthority
     #    Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
@@ -294,6 +295,8 @@ unit_test:
     - apk add --update make bash
     - apk add --update bind-tools
     - apk add --update postgresql14-client gzip socat
+    - apk add --update python3 py3-pip
+    - pip install shyaml
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
     - touch /root/.Xauthority
     #    Hack BASH_SOURCE into sourced files, docker its sh shell won't set this
@@ -314,7 +317,6 @@ unit_test:
 
 integration_test_docker:
   extends: .test_docker
-  allow_failure: true # until there is a machine that can properly run them
   script:
     #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag

README.md

@@ -139,6 +139,7 @@ Next change the version in the following places:
 
 # Release Notes
 
+* 0.24.8 Push docker images to own registry and pull from there for deployments
 * 0.24.6 Bugfix: deploy tagged version instead of latest
 * 0.24.5 Bugfix: install 'tango' in Jupyter Lab again
 * 0.24.4 Apply calibration values in reverse order if subband frequencies are decreasing

docker-compose/.env

-DOCKER_REGISTRY_HOST=artefact.skao.int
-DOCKER_REGISTRY_USER=ska-tango-images
+SKA_DOCKER_REGISTRY_HOST=artefact.skao.int
+SKA_DOCKER_REGISTRY_USER=ska-tango-images
 LOCAL_DOCKER_REGISTRY_HOST=git.astron.nl:5000
 LOCAL_DOCKER_REGISTRY_LOFAR=lofar2.0
 LOCAL_DOCKER_REGISTRY_USER=lofar2.0/tango

infra/dev/nomad/nomad.hcl

@@ -49,7 +49,7 @@ resource "container" "consul" {
   }
 
   image {
-    name = "hashicorp/consul:latest"
+    name = "git.astron.nl:5000/lofar2.0/tango/consul:latest"
   }
 
   command = [

infra/env.yaml

@@ -6,6 +6,10 @@ tango:
   rest:
     version: 1.14.8
 
+registry:
+  astron:
+    url: git.astron.nl:5000/lofar2.0/tango
+
 monitoring:
   db:
     version: 15.4

infra/jobs/station/Makefile

@@ -12,7 +12,7 @@ ENV ?= ../../env.yaml
 render: pull $(JOBS) $(DEVICES)
 
 pull:
-	docker pull -q hashicorp/levant
+	docker pull -q git.astron.nl:5000/lofar2.0/tango/levant
 
 %.nomad: %.levant.nomad
-	docker run --rm -v $(realpath $(ENV)):/env.yaml -v $(realpath $(DIR_SRC)):/in -v $(realpath $(DIR_OUT)):/out hashicorp/levant render -var-file=/env.yaml -var image_tag="$(TAG)" -var station="$(STATION)" -out=/out/$@ /in/$<
+	docker run --rm -v $(realpath $(ENV)):/env.yaml -v $(realpath $(DIR_SRC)):/in -v $(realpath $(DIR_OUT)):/out git.astron.nl:5000/lofar2.0/tango/levant render -var-file=/env.yaml -var image_tag="$(TAG)" -var station="$(STATION)" -out=/out/$@ /in/$<

infra/jobs/station/device-server.levant.nomad

@@ -45,7 +45,7 @@ job "device-servers" {
             driver = "docker"
 
             config {
-                image = "git.astron.nl:5000/lofar2.0/tango/lofar-device-base:[[ $.image_tag ]]"
+                image = "[[ $.registry.astron.url ]]/lofar-device-base:[[ $.image_tag ]]"
                 ports = [
                     [[ range $port_name, $port := .ports]]
                       "[[ $port_name ]]",

infra/jobs/station/dsconfig.levant.nomad

@@ -13,7 +13,7 @@ job "dsconfig" {
       driver = "docker"
 
       config {
-        image = "git.astron.nl:5000/lofar2.0/tango/dsconfig:[[ .image_tag ]]"
+        image = "[[ .registry.astron.url ]]/dsconfig:[[ .image_tag ]]"
         mount {
           type   = "bind"
           source = "local/dsconfig-update-settings.json"

infra/jobs/station/ec-sim.levant.nomad

@@ -22,7 +22,7 @@ job "simulators" {
       driver = "docker"
 
       config {
-        image = "git.astron.nl:5000/lofar2.0/tango/ec-sim:latest"
+        image = "[[.registry.astron.url]]/ec-sim:latest"
         ports = ["opcua"]
       }
       resources {

infra/jobs/station/jupyter.levant.nomad

@@ -34,7 +34,7 @@ job "jupyter" {
       }
 
       config {
-        image = "git.astron.nl:5000/lofar2.0/tango/jupyter-lab:[[.image_tag]]"
+        image = "[[.registry.astron.url]]/jupyter-lab:[[.image_tag]]"
         ports = ["jupyter"]
         mount {
           type   = "bind"

infra/jobs/station/logging.levant.nomad

@@ -55,7 +55,7 @@ job "log-scraping" {
         task "vector" {
             driver = "docker"
             config {
-                image = "timberio/vector:0.32.1.custom.989ad14-distroless-static"
+                image = "[[.registry.astron.url]]/vector:0.32.1.custom.989ad14-distroless-static"
                 ports = ["api"]
             }
             # docker socket volume mount

infra/jobs/station/monitoring.levant.nomad

@@ -39,7 +39,7 @@ job "monitoring" {
       }
 
       config {
-        image = "postgres:[[.monitoring.db.version]]"
+        image = "[[.registry.astron.url]]/postgres:[[.monitoring.db.version]]"
         ports = ["postgres"]
       }
 
@@ -97,7 +97,7 @@ job "monitoring" {
       driver = "docker"
 
       config {
-        image   = "postgres:[[.monitoring.db.version]]"
+        image   = "[[.registry.astron.url]]/postgres:[[.monitoring.db.version]]"
         command = "sh"
         args    = ["-c", "while ! pg_isready -h localhost; do sleep 1; done"]
       }
@@ -118,7 +118,7 @@ localhost:5432:grafana:postgres:password
     task "grafana" {
       driver = "docker"
       config {
-        image = "git.astron.nl:5000/lofar2.0/tango/grafana:[[.image_tag]]"
+        image = "[[.registry.astron.url]]/grafana:[[.image_tag]]"
         ports = ["http"]
         mount {
           type   = "bind"
@@ -237,7 +237,7 @@ localhost:5432:grafana:postgres:password
       }
 
       config {
-        image = "git.astron.nl:5000/lofar2.0/tango/prometheus:[[.image_tag]]"
+        image = "[[.registry.astron.url]]/prometheus:[[.image_tag]]"
         ports = ["prometheus"]
         args  = [
           "--config.file=/etc/prometheus/prometheus.yml",
@@ -381,7 +381,7 @@ localhost:5432:grafana:postgres:password
       }
 
       config {
-        image = "git.astron.nl:5000/lofar2.0/tango/loki:[[.image_tag]]"
+        image = "[[.registry.astron.url]]/loki:[[.image_tag]]"
         ports = ["prometheus"]
       }
 

infra/jobs/station/object-storage.levant.nomad

@@ -52,7 +52,7 @@ job "object-storage" {
       }
 
       config {
-        image   = "minio/minio:[[.object_storage.minio.version]]"
+        image   = "[[.registry.astron.url]]/minio:[[.object_storage.minio.version]]"
         ports   = ["s3", "console"]
         command = "server"
         args    = ["--console-address", ":9001", "/data"]
@@ -74,7 +74,7 @@ job "object-storage" {
     task "vector" {
       driver = "docker"
       config {
-        image = "timberio/vector:0.32.1.custom.989ad14-distroless-static"
+        image = "[[.registry.astron.url]]/vector:0.32.1.custom.989ad14-distroless-static"
         ports = ["metrics"]
       }
       # Vector won't start unless the sinks(backends) configured are healthy

infra/jobs/station/tango-prometheus-exporter.levant.nomad

@@ -21,7 +21,7 @@ job "tango-prometheus-exporter" {
       driver = "docker"
 
       config {
-        image   = "git.astron.nl:5000/lofar2.0/tango/tango-prometheus-exporter:[[.image_tag]]"
+        image   = "[[.registry.astron.url]]/tango-prometheus-exporter:[[.image_tag]]"
         ports   = ["http"]
         command = "--config=/code/lofar2-policy.json"
       }
@@ -54,7 +54,7 @@ job "tango-prometheus-exporter" {
       driver = "docker"
 
       config {
-        image   = "git.astron.nl:5000/lofar2.0/tango/tango-prometheus-exporter:[[.image_tag]]"
+        image   = "[[.registry.astron.url]]/tango-prometheus-exporter:[[.image_tag]]"
         ports   = ["http"]
         command = "--config=/code/lofar2-fast-policy.json"
       }
@@ -87,7 +87,7 @@ job "tango-prometheus-exporter" {
       driver = "docker"
 
       config {
-        image   = "git.astron.nl:5000/lofar2.0/tango/tango-prometheus-exporter:[[.image_tag]]"
+        image   = "[[.registry.astron.url]]tango-prometheus-exporter:[[.image_tag]]"
         ports   = ["http"]
         command = "--config=/code/lofar2-slow-policy.json"
       }

infra/jobs/station/tango.levant.nomad

@@ -48,7 +48,7 @@ job "tango" {
       }
 
       config {
-        image = "git.astron.nl:5000/lofar2.0/tango/tango-db:[[.tango.db.version]]"
+        image = "[[.registry.astron.url]]/tango-db:[[.tango.db.version]]"
         ports = ["mysql"]
       }
 
@@ -103,7 +103,7 @@ job "tango" {
       driver = "docker"
 
       config {
-        image   = "busybox"
+        image   = "[[.registry.astron.url]]/busybox"
         command = "sh"
         args    = ["-c", "while ! nc -z $MYSQL_HOST $MYSQL_PORT; do sleep 1; done"]
       }
@@ -125,7 +125,7 @@ job "tango" {
 
 
       config {
-        image      = "git.astron.nl:5000/lofar2.0/tango/tango-databaseds:[[.tango.databaseds.version]]"
+        image      = "[[.registry.astron.url]]/tango-databaseds:[[.tango.databaseds.version]]"
         ports      = ["tango"]
         entrypoint = [
           "/usr/local/bin/DataBaseds",

sbin/prepare_dev_env.sh

@@ -62,8 +62,8 @@ if [ "$(docker volume list | grep -c "$docker_volume")" = "0" ]; then
   docker volume create "$docker_volume"
 fi
 
-docker pull -q bash
-docker run --rm -i -v "$docker_volume":/mnt bash bash  <<- EOM
+docker pull -q git.astron.nl:5000/lofar2.0/tango/bash
+docker run --rm -i -v "$docker_volume":/mnt git.astron.nl:5000/lofar2.0/tango/bash bash  <<- EOM
   mkdir -p /mnt/volumes/tango-database
   mkdir -p /mnt/volumes/monitoring-postgresql-data
   mkdir -p /mnt/volumes/monitoring-loki-data

sbin/tag_and_push_docker_image.sh

@@ -52,8 +52,16 @@ fi
 # shellcheck disable=SC1090,SC1091
 . "${LOFAR20_DIR}/docker-compose/.env" || exit 1
 
-# List of images and their tag
-REMOTE_IMAGES=(
+if [ -z "$(which shyaml)" ]; then
+  echo "Shyaml not found!, install using: 'pip install shyaml'"
+  exit 1
+fi
+
+POSTGRES_VERSION=$(shyaml get-value monitoring.db.version < "${LOFAR20_DIR}/infra/env.yaml")
+MINIO_VERSION=$(shyaml get-value object_storage.minio.version < "${LOFAR20_DIR}/infra/env.yaml")
+
+# List of images and their tag from ska repository
+REMOTE_SKA_IMAGES=(
   "tango-dsconfig:${TANGO_DSCONFIG_VERSION}"
   "tango-itango:${TANGO_ITANGO_VERSION}"
   "tango-cpp:${TANGO_CPP_VERSION}"
@@ -63,13 +71,24 @@ REMOTE_IMAGES=(
   "tango-rest:${TANGO_REST_VERSION}"
 )
 
+# List of images and their tag from dockerhub
+REMOTE_IMAGES=(
+  "vector 0.32.1.custom.989ad14-distroless-static timberio"
+  "postgres ${POSTGRES_VERSION}"
+  "levant latest hashicorp"
+  "consul latest hashicorp"
+  "minio ${MINIO_VERSION} minio"
+  "busybox latest"
+  "bash latest"
+)
+
 # Triple tuple of docker-compose names, image names and if necessary for
 # integration tests.
 # TODO(Corne): Have this list generated from the .yml files
 LOCAL_IMAGES=(
   "lofar-device-base lofar-device-base y"
 
-  "dsconfig dsconfig n"
+  "dsconfig dsconfig y"
 
   "ec-sim ec-sim y"
 
@@ -115,9 +134,23 @@ if [ -z "${1+x}" ]; then
   echo "Pulling and retagging remote images"
 
   # Iterate over al the REMOTE_IMAGES and pull them from remote and push local
-  for image in "${REMOTE_IMAGES[@]}"; do
-    remote_url="${DOCKER_REGISTRY_HOST}/${DOCKER_REGISTRY_USER}-${image}"
+  for image in "${REMOTE_SKA_IMAGES[@]}"; do
     local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}"
+    remote_url="${SKA_DOCKER_REGISTRY_HOST}/${SKA_DOCKER_REGISTRY_USER}-${image}"
+    docker pull "${remote_url}"
+    docker tag "${remote_url}" "${local_url}"
+    docker push "${local_url}"
+  done
+
+  for image in "${REMOTE_IMAGES[@]}"; do
+    # shellcheck disable=SC2086
+    set -- $image
+    local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${1}:${2}"
+    if [ -n "${3+x}" ]; then
+      remote_url="${3}/${1}:${2}"
+    else
+      remote_url="${1}:${2}"
+    fi
     docker pull "${remote_url}"
     docker tag "${remote_url}" "${local_url}"
     docker push "${local_url}"

tangostationcontrol/VERSION

-0.24.7
+0.24.8