Skip to content
Snippets Groups Projects
Commit 54eb2d76 authored by Corné Lukken's avatar Corné Lukken
Browse files

L2SS-205: Reenable other stages and create build all image job

parent 540ef1bf
No related branches found
No related tags found
1 merge request!183Docker image building & pushing with CI pipeline caching
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 135
− 75
View file @ 54eb2d76
......@@ -6,16 +6,15 @@ cache:
paths:
- .cache/pip
stages:
- remote_images
# - building
# - linting
# - static-analysis
# - unit-tests
- integration-tests
# - packaging
- images
- building
- linting
- static-analysis
- unit-tests
- integration-tests
- packaging
.base_docker_images:
stage: remote_images
stage: images
image: docker:latest
tags:
- privileged
......@@ -26,7 +25,7 @@ stages:
before_script:
- |
if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
tag=""
tag="latest"
echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
else
tag="$CI_COMMIT_REF_SLUG"
......@@ -41,6 +40,12 @@ stages:
- . bootstrap/etc/lofar20rc.sh || true
# Allow docker image script to execute
- chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh
.base_docker_images_except:
extends: .base_docker_images
except:
refs:
- tags
- master
.base_docker_store_images:
extends: .base_docker_images
script:
......@@ -61,22 +66,28 @@ docker_store_images_changes:
refs:
- tags
- master
# TODO(Corne): Remove this job after it has run once
docker_build_image_all:
extends: .base_docker_images
only:
refs:
- tags
- master
script:
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh integration-test $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh tango-prometheus-exporter $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag
docker_build_image_elk:
extends: .base_docker_images
extends: .base_docker_images_except
only:
changes:
- docker-compose/elk.yml
......@@ -87,7 +98,7 @@ docker_build_image_elk:
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag
docker_build_image_lofar_device_base:
extends: .base_docker_images
extends: .base_docker_images_except
only:
changes:
- docker-compose/lofar-device-base.yml
......@@ -96,16 +107,20 @@ docker_build_image_lofar_device_base:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag
docker_build_image_prometheus:
extends: .base_docker_images
extends: .base_docker_images_except
only:
changes:
- docker-compose/prometheus.yml
- docker-compose/prometheus/*
except:
refs:
- tags
- master
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag
docker_build_image_itango:
extends: .base_docker_images
extends: .base_docker_images_except
only:
changes:
- docker-compose/itango.yml
......@@ -113,7 +128,7 @@ docker_build_image_itango:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag
docker_build_image_grafana:
extends: .base_docker_images
extends: .base_docker_images_except
only:
changes:
- docker-compose/grafana.yml
......@@ -122,7 +137,7 @@ docker_build_image_grafana:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag
docker_build_image_jupyter:
extends: .base_docker_images
extends: .base_docker_images_except
only:
changes:
- docker-compose/jupyter.yml
......@@ -130,45 +145,90 @@ docker_build_image_jupyter:
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag
#newline_at_eof:
# stage: linting
# before_script:
# - pip3 install -r tangostationcontrol/test-requirements.txt
# script:
## TODO(Corne): Ignore shell files in submodules more cleanly
# - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker
#python_linting:
# stage: linting
# before_script:
# - sudo apt-get update
# - sudo apt-get install -y git
# script:
# - cd tangostationcontrol
# - tox -e pep8
#bandit:
# stage: static-analysis
# before_script:
# - sudo apt-get update
# - sudo apt-get install -y git
# script:
# - cd tangostationcontrol
# - tox -e bandit
#shellcheck:
# stage: static-analysis
# before_script:
# - sudo apt-get update
# - sudo apt-get install -y shellcheck
# script:
## TODO(Corne): Ignore shell files in submodules
# - shellcheck **/*.sh
#unit_test:
# stage: unit-tests
# before_script:
# - sudo apt-get update
# - sudo apt-get install -y git
# script:
# - cd tangostationcontrol
# - tox -e py37
docker_build_image_apsct_sim:
extends: .base_docker_images_except
only:
changes:
- docker-compose/aspct-sim.yml
- docker-compose/pypcc-sim-base/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim $tag
docker_build_image_apspu_sim:
extends: .base_docker_images_except
only:
changes:
- docker-compose/apspu-sim.yml
- docker-compose/pypcc-sim-base/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim $tag
docker_build_image_recv_sim:
extends: .base_docker_images_except
only:
changes:
- docker-compose/recv-sim.yml
- docker-compose/pypcc-sim-base/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim $tag
docker_build_image_sdptr_sim:
extends: .base_docker_images_except
only:
changes:
- docker-compose/sdptr-sim.yml
- docker-compose/sdptr-sim/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim $tag
docker_build_image_unb2_sim:
extends: .base_docker_images_except
only:
changes:
- docker-compose/unb2-sim.yml
- docker-compose/pypcc-sim-base/*
script:
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag
newline_at_eof:
stage: linting
before_script:
- pip3 install -r tangostationcontrol/test-requirements.txt
script:
# TODO(Corne): Ignore shell files in submodules more cleanly
- flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker
python_linting:
stage: linting
before_script:
- sudo apt-get update
- sudo apt-get install -y git
script:
- cd tangostationcontrol
- tox -e pep8
bandit:
stage: static-analysis
before_script:
- sudo apt-get update
- sudo apt-get install -y git
script:
- cd tangostationcontrol
- tox -e bandit
shellcheck:
stage: static-analysis
before_script:
- sudo apt-get update
- sudo apt-get install -y shellcheck
script:
# TODO(Corne): Ignore shell files in submodules
- shellcheck **/*.sh
unit_test:
stage: unit-tests
before_script:
- sudo apt-get update
- sudo apt-get install -y git
script:
- cd tangostationcontrol
- tox -e py37
integration_test_docker:
stage: integration-tests
image: docker:latest
......@@ -181,7 +241,7 @@ integration_test_docker:
before_script:
- |
if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then
tag=""
tag="latest"
echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
else
tag="$CI_COMMIT_REF_SLUG"
......@@ -210,16 +270,16 @@ integration_test_docker:
- chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh
# Do not remove 'bash' or statement will be ignored by primitive docker shell
- bash $CI_PROJECT_DIR/sbin/run_integration_test.sh
#wheel_packaging:
# stage: packaging
# artifacts:
# paths:
# - tangostationcontrol/dist/*.whl
# before_script:
# - sudo apt-get update
# - sudo apt-get install -y git
# - pip3 install -r tangostationcontrol/test-requirements.txt
# - pip3 install -r docker-compose/itango/lofar-requirements.txt
# script:
# - cd tangostationcontrol
# - python setup.py bdist_wheel
\ No newline at end of file
wheel_packaging:
stage: packaging
artifacts:
paths:
- tangostationcontrol/dist/*.whl
before_script:
- sudo apt-get update
- sudo apt-get install -y git
- pip3 install -r tangostationcontrol/test-requirements.txt
- pip3 install -r docker-compose/itango/lofar-requirements.txt
script:
- cd tangostationcontrol
- python setup.py bdist_wheel
\ No newline at end of file
bootstrap/etc/lofar20rc.sh
+ 1
− 0
View file @ 54eb2d76
......@@ -16,6 +16,7 @@ if [ ! -f "${LOFAR20_DIR}/.git/hooks/post-checkout" ]; then
alias git="cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-checkout; cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-merge; unalias git; git"
fi
# CI_BUILD_ID does not exist see https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
if [ ! -z ${CI_BUILD_ID+x} ]; then
export CONTAINER_NAME_PREFIX=${CI_BUILD_ID}-
elif [ ! -z ${CI_JOB_ID+x} ]; then
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment