L2SS-205: Programmatically only pull images needed for integration

.gitlab-ci.yml

@@ -66,25 +66,6 @@ docker_store_images_changes:
     refs:
       - tags
       - master
-# TODO(Corne): Remove this after first run
-docker_build_image_devices:
-  extends: .base_docker_images
-  script:
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apsct latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 latest
-    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst latest
 docker_build_image_all:
   extends: .base_docker_images
   only:
@@ -219,6 +200,96 @@ docker_build_image_unb2_sim:
   script:
 #    Do not remove 'bash' or statement will be ignored by primitive docker shell
     - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag
+docker_build_image_device_apsct:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-aspct.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-aspct $tag
+docker_build_image_device_apspu:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-apspu.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-apspu $tag
+docker_build_image_device_boot:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-boot.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-boot $tag
+docker_build_image_device_docker:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-docker.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-docker $tag
+docker_build_image_device_ovservation_control:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-observation_control.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-observation_control $tag
+docker_build_image_device_recv:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-recv.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-recv $tag
+docker_build_image_device_sdp:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-sdp.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sdp $tag
+docker_build_image_device_sst:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-sst.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-sst $tag
+docker_build_image_device_unb2:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-unb2.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-unb2 $tag
+docker_build_image_device_xst:
+  extends: .base_docker_images_except
+  only:
+    changes:
+      - docker-compose/device-xst.yml
+      - docker-compose/lofar-device-base/*
+  script:
+#    Do not remove 'bash' or statement will be ignored by primitive docker shell
+    - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh device-xst $tag
 newline_at_eof:
   stage: linting
   before_script:

sbin/tag_and_push_docker_image.sh

@@ -36,30 +36,33 @@ if [ -z "${1+x}" ]; then
   exit 0
 fi
 
-# Tuple of images and queries to detect changes
+# Triple tuple of docker-compose names, image names and if necessary for
+# integration tests.
 # TODO(Corne): Have this list generated from the .yml files
 LOCAL_IMAGES=(
-  "elk elk" "elk-configure-host elk-configure-host"
-  "lofar-device-base lofar-device-base"
-
-  "apsct-sim docker-compose_apsct-sim" "apspu-sim docker-compose_apspu-sim"
-  "recv-sim docker-compose_recv-sim" "sdptr-sim docker-compose_sdptr-sim"
-  "unb2-sim docker-compose_unb2-sim"
-
-  "device-apsct device-apsct" "device-apspu device-apspu"
-  "device-boot device-boot" "device-docker device-docker"
-  "device-observation_control device-observation_control"
-  "device-recv device-recv" "device-sdp device-sdp" "device-sst device-sst"
-  "device-unb2 device-unb2" "device-xst device-xst"
-
-  "itango docker-compose_itango"
+  "elk elk y" "elk-configure-host elk-configure-host y"
+  "lofar-device-base lofar-device-base y"
+
+  "apsct-sim docker-compose_apsct-sim y" "apspu-sim docker-compose_apspu-sim y"
+  "recv-sim docker-compose_recv-sim y" "sdptr-sim docker-compose_sdptr-sim y"
+  "unb2-sim docker-compose_unb2-sim y"
+
+  "device-apsct device-apsct y" "device-apspu device-apspu y"
+  "device-boot device-boot y" "device-docker device-docker y"
+  "device-observation_control device-observation_control y"
+  "device-recv device-recv y" "device-sdp device-sdp y"
+  "device-sst device-sst y" "device-unb2 device-unb2 y"
+  "device-xst device-xst y"
+
+  "itango docker-compose_itango y"
+
+  "grafana grafana n" "prometheus prometheus n"
+  "jupyter docker-compose_jupyter n"
+  "integration-test docker-compose_integration-test n"
+  "tango-prometheus-exporter docker-compose_tango-prometheus-exporter n"
 )
 
-    # "grafana grafana"
-   # "prometheus prometheus"
-     # "jupyter docker-compose_jupyter"
-  # "integration-test docker-compose_integration-test"
-  # "tango-prometheus-exporter docker-compose_tango-prometheus-exporter"
+
 
 # If first argument set run second stage, determine LOCAL_IMAGE to build and
 # push from the argument
@@ -119,16 +122,20 @@ if [ ! -z "${1+x}" ] && [ "${1}" == "pull" ]; then
       # Set, splits tuple into $1 and $2. this shadows previous variables
     # shellcheck disable=SC2086
     set -- $image
-    local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}"
-    # Pull images, at least one of the two images must succeed
-    echo "docker pull ${local_url}:${tag}"
-    docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || exit 1
-    # Ensure the images will have the same tags as generated by docker-compose
-    docker tag "${local_url}:${tag}" "${2}" || docker tag "${local_url}:latest" "${2}" || exit 1
+
+    # Only download images which are needed for integration test
+    if [ "${3}" == "y" ]; then
+      local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${2}"
+      # Pull images, at least one of the two images must succeed
+      echo "docker pull ${local_url}:${tag}"
+      docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" || exit 1
+      # Ensure the images will have the same tags as generated by docker-compose
+      docker tag "${local_url}:${tag}" "${2}" || docker tag "${local_url}:latest" "${2}" || exit 1
+    fi
   done
 
   exit 0
 fi
 
-# Someone nothing ran, that is an error do not fail silently
+# Somehow nothing ran, that is an error do not fail silently
 exit 1