diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4e0c05264b59489913cbd7117bee2f609eaf4a4..0f5ccfb103e99373d337dd0170faf78fb37f903d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,4 @@ -# TODO(Corne): Update this image to use our own registry once building -# images is in place. -image: artefact.skao.int/ska-tango-images-tango-itango:9.3.5 +image: git.astron.nl:5000/lofar2.0/tango/tango-itango:9.3.5 variables: GIT_SUBMODULE_STRATEGY: recursive PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -13,7 +11,7 @@ stages: # - linting # - static-analysis # - unit-tests -# - integration-tests + - integration-tests # - packaging - images .base_docker_images: @@ -26,19 +24,27 @@ stages: variables: DOCKER_TLS_CERTDIR: "/certs" before_script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then + tag="" + echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag=":$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi - apk add --update make bash docker-compose - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - touch /root/.Xauthority - # Hack BASH_SOURCE into sourced files, docker its sh shell won't set this +# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh - # source the lofarrc file and mask its non zero exit code +# source the lofarrc file and mask its non zero exit code - . bootstrap/etc/lofar20rc.sh || true - # Allow docker image script test to execute +# Allow docker image script to execute - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh .base_docker_store_images: extends: .base_docker_images script: - # Do not remove 'bash' or statement will be ignored by primitive docker shell +# Do not remove 'bash' or statem ent will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh docker_store_images_master_tag: extends: .base_docker_store_images @@ -63,9 +69,9 @@ docker_build_image_elk: - docker-compose/elk/* - docker-compose/elk-configure-host/* script: - # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag docker_build_image_lofar_device_base: extends: .base_docker_images only: @@ -73,8 +79,8 @@ docker_build_image_lofar_device_base: - docker-compose/lofar-device-base.yml - docker-compose/lofar-device-base/* script: - # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag docker_build_image_prometheus: extends: .base_docker_images only: @@ -82,8 +88,8 @@ docker_build_image_prometheus: - docker-compose/prometheus.yml - docker-compose/prometheus/* script: - # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag #docker_build_image_docker-compose_itango: # extends: .base_docker_images # only: @@ -99,8 +105,8 @@ docker_build_image_grafana: - docker-compose/grafana.yml - docker-compose/grafana/* script: - # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag docker_build_image_jupyter: extends: .base_docker_images only: @@ -108,8 +114,8 @@ docker_build_image_jupyter: - docker-compose/jupyter.yml - docker-compose/jupyter/* script: - # Do not remove 'bash' or statement will be ignored by primitive docker shell - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag #newline_at_eof: # stage: linting # before_script: @@ -149,35 +155,47 @@ docker_build_image_jupyter: # script: # - cd tangostationcontrol # - tox -e py37 -#integration_test_docker: -# stage: integration-tests -# image: docker:latest -# tags: -# - privileged -# services: -# - name: docker:dind -# variables: -# DOCKER_TLS_CERTDIR: "/certs" -# before_script: -# - apk add --update make bash docker-compose -# - apk add --update bind-tools -# - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY -# script: -# - touch /root/.Xauthority -## Hack BASH_SOURCE into sourced files, docker its sh shell won't set this -# - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh -## Hack HOSTNAME env variable into host.docker.internal, set in docker-compose -# - export HOSTNAME=host.docker.internal -## - export HOSTNAME=$(hostname -i) -## - export HOSTNAME=$(cat /run/systemd/netif/leases/2 | grep ^ADDRESS= | awk -F'=' '{print $2}') -## source the lofarrc file and mask its non zero exit code -# - . bootstrap/etc/lofar20rc.sh || true -## TANGO_HOST must be unset our databaseds will be unreachable -# - unset TANGO_HOST -## Allow integration test to execute -# - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh -## Do not remove 'bash' or statement will be ignored by primitive docker shell -# - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh +integration_test_docker: + stage: integration-tests + image: docker:latest + tags: + - privileged + services: + - name: docker:dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + before_script: + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then + tag="" + echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag=":$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - apk add --update make bash docker-compose + - apk add --update bind-tools + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +# Allow docker image script to execute + - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh pull $tag + script: + - touch /root/.Xauthority +# Hack BASH_SOURCE into sourced files, docker its sh shell won't set this + - export BASH_SOURCE=$(pwd)/bootstrap/etc/lofar20rc.sh +# Hack HOSTNAME env variable into host.docker.internal, set in docker-compose + - export HOSTNAME=host.docker.internal +# - export HOSTNAME=$(hostname -i) +# - export HOSTNAME=$(cat /run/systemd/netif/leases/2 | grep ^ADDRESS= | awk -F'=' '{print $2}') +# source the lofarrc file and mask its non zero exit code + - . bootstrap/etc/lofar20rc.sh || true +# TANGO_HOST must be unset our databaseds will be unreachable + - unset TANGO_HOST +# Allow integration test to execute + - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh #wheel_packaging: # stage: packaging # artifacts: diff --git a/sbin/tag_and_push_docker_image.sh b/sbin/tag_and_push_docker_image.sh index bebc84d225da139ec4e5995313413be4a147d631..5898e6de3100e292c96f74f16d106e06174388ee 100755 --- a/sbin/tag_and_push_docker_image.sh +++ b/sbin/tag_and_push_docker_image.sh @@ -1,11 +1,5 @@ #!/bin/bash -e -# Tag and push which image version? -DOCKER_TAG=latest - -# Change to git tag or git hash if no tag -VERSION=$(date +"%Y-%M-%d") - if [ -z "${LOFAR20_DIR+x}" ]; then echo "LOFAR20_DIR not set, did you forget to source lofar20rc.sh?" exit 1 @@ -25,6 +19,8 @@ REMOTE_IMAGES=( # If first argument of bash script not set run first stage if [ -z "${1+x}" ]; then + echo "Pulling and retagging remote images" + # Iterate over al the REMOTE_IMAGES and pull them from remote and push local for image in "${REMOTE_IMAGES[@]}"; do # Set, splits tuple into $1 and $2 @@ -36,6 +32,8 @@ if [ -z "${1+x}" ]; then docker tag "${remote_url}" "${local_url}" docker push "${local_url}" done + + exit 0 fi # Tuple of images and queries to detect changes @@ -48,17 +46,57 @@ LOCAL_IMAGES=( # If first argument set run second stage, determine LOCAL_IMAGE to build and # push from the argument -if [ ! -z "${1+x}" ]; then +if [ ! -z "${1+x}" ] && [ "${1}" != "pull" ]; then + + # The second argument must pass the tag variable must be set + if [ -z "${2+x}" ]; then + echo "Error, second argument must pass tag variable" + exit 1 + fi + + # Set the tag variable + tag="${2}" cd "${LOFAR20_DIR}/docker-compose" || exit 1 + # Loop through images and find the specified one for image in "${LOCAL_IMAGES[@]}"; do if [ "${1}" == "${image}" ]; then + echo "Building image for ${image} container" + local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}" + + # If tag is not latest, than it is not a tagged master build and we can + # pull the latest image as cache. + if [ "${tag}" != "latest" ]; then + docker pull "${local_url}:latest" + fi + make build "${image}" - local_url="${LOCAL_DOCKER_REGISTRY_HOST}/${LOCAL_DOCKER_REGISTRY_USER}/${image}:latest" - docker tag "${image}" "${local_url}" - docker push "${local_url}" + docker tag "${image}" "${local_url}:${tag}" + docker push "${local_url}:${tag}" fi done + exit 0 +fi + +# Final stage, pull images for integration cache try special tag image first +# if it fails download latest instead +if [ -z "${1+x}" ] && [ "${1}" == "pull" ]; then + echo "Pulling images for integration test cache" + + # The second argument must pass the tag variable must be set + if [ -z "${2+x}" ]; then + echo "Error, second argument must pass tag variable" + exit 1 + fi + + # Set the tag variable + tag="${2}" + + for image in "${LOCAL_IMAGES[@]}"; do + docker pull "${local_url}:${tag}" || docker pull "${local_url}:latest" + done + + exit 0 fi