diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e20e0c143bc2ee5158de3e79651f5bf71a0fa652..27e4bd0c585336a8bb19d97f4b73d3cc62c64486 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,16 +6,15 @@ cache: paths: - .cache/pip stages: - - remote_images -# - building -# - linting -# - static-analysis -# - unit-tests - - integration-tests -# - packaging - images + - building + - linting + - static-analysis + - unit-tests + - integration-tests + - packaging .base_docker_images: - stage: remote_images + stage: images image: docker:latest tags: - privileged @@ -26,7 +25,7 @@ stages: before_script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then - tag="" + tag="latest" echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" else tag="$CI_COMMIT_REF_SLUG" @@ -41,6 +40,12 @@ stages: - . bootstrap/etc/lofar20rc.sh || true # Allow docker image script to execute - chmod u+x $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh +.base_docker_images_except: + extends: .base_docker_images + except: + refs: + - tags + - master .base_docker_store_images: extends: .base_docker_images script: @@ -61,22 +66,28 @@ docker_store_images_changes: refs: - tags - master -# TODO(Corne): Remove this job after it has run once docker_build_image_all: extends: .base_docker_images + only: + refs: + - tags + - master script: - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag +# Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh integration-test $tag - - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh tango-prometheus-exporter $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim $tag + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag docker_build_image_elk: - extends: .base_docker_images + extends: .base_docker_images_except only: changes: - docker-compose/elk.yml @@ -87,7 +98,7 @@ docker_build_image_elk: - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk $tag - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh elk-configure-host $tag docker_build_image_lofar_device_base: - extends: .base_docker_images + extends: .base_docker_images_except only: changes: - docker-compose/lofar-device-base.yml @@ -96,16 +107,20 @@ docker_build_image_lofar_device_base: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh lofar-device-base $tag docker_build_image_prometheus: - extends: .base_docker_images + extends: .base_docker_images_except only: changes: - docker-compose/prometheus.yml - docker-compose/prometheus/* + except: + refs: + - tags + - master script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh prometheus $tag docker_build_image_itango: - extends: .base_docker_images + extends: .base_docker_images_except only: changes: - docker-compose/itango.yml @@ -113,7 +128,7 @@ docker_build_image_itango: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh itango $tag docker_build_image_grafana: - extends: .base_docker_images + extends: .base_docker_images_except only: changes: - docker-compose/grafana.yml @@ -122,7 +137,7 @@ docker_build_image_grafana: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh grafana $tag docker_build_image_jupyter: - extends: .base_docker_images + extends: .base_docker_images_except only: changes: - docker-compose/jupyter.yml @@ -130,45 +145,90 @@ docker_build_image_jupyter: script: # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh jupyter $tag -#newline_at_eof: -# stage: linting -# before_script: -# - pip3 install -r tangostationcontrol/test-requirements.txt -# script: -## TODO(Corne): Ignore shell files in submodules more cleanly -# - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker -#python_linting: -# stage: linting -# before_script: -# - sudo apt-get update -# - sudo apt-get install -y git -# script: -# - cd tangostationcontrol -# - tox -e pep8 -#bandit: -# stage: static-analysis -# before_script: -# - sudo apt-get update -# - sudo apt-get install -y git -# script: -# - cd tangostationcontrol -# - tox -e bandit -#shellcheck: -# stage: static-analysis -# before_script: -# - sudo apt-get update -# - sudo apt-get install -y shellcheck -# script: -## TODO(Corne): Ignore shell files in submodules -# - shellcheck **/*.sh -#unit_test: -# stage: unit-tests -# before_script: -# - sudo apt-get update -# - sudo apt-get install -y git -# script: -# - cd tangostationcontrol -# - tox -e py37 +docker_build_image_apsct_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/aspct-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apsct-sim $tag +docker_build_image_apspu_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/apspu-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh apspu-sim $tag +docker_build_image_recv_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/recv-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh recv-sim $tag +docker_build_image_sdptr_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/sdptr-sim.yml + - docker-compose/sdptr-sim/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh sdptr-sim $tag +docker_build_image_unb2_sim: + extends: .base_docker_images_except + only: + changes: + - docker-compose/unb2-sim.yml + - docker-compose/pypcc-sim-base/* + script: +# Do not remove 'bash' or statement will be ignored by primitive docker shell + - bash $CI_PROJECT_DIR/sbin/tag_and_push_docker_image.sh unb2-sim $tag +newline_at_eof: + stage: linting + before_script: + - pip3 install -r tangostationcontrol/test-requirements.txt + script: +# TODO(Corne): Ignore shell files in submodules more cleanly + - flake8 --filename *.sh,*.conf,*.md,*.yml --select=W292 --exclude docker-compose/tango-prometheus-exporter,.tox,.egg-info,docker +python_linting: + stage: linting + before_script: + - sudo apt-get update + - sudo apt-get install -y git + script: + - cd tangostationcontrol + - tox -e pep8 +bandit: + stage: static-analysis + before_script: + - sudo apt-get update + - sudo apt-get install -y git + script: + - cd tangostationcontrol + - tox -e bandit +shellcheck: + stage: static-analysis + before_script: + - sudo apt-get update + - sudo apt-get install -y shellcheck + script: +# TODO(Corne): Ignore shell files in submodules + - shellcheck **/*.sh +unit_test: + stage: unit-tests + before_script: + - sudo apt-get update + - sudo apt-get install -y git + script: + - cd tangostationcontrol + - tox -e py37 integration_test_docker: stage: integration-tests image: docker:latest @@ -181,7 +241,7 @@ integration_test_docker: before_script: - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" && -z "$CI_COMMIT_TAG" ]]; then - tag="" + tag="latest" echo "Running on tagged default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" else tag="$CI_COMMIT_REF_SLUG" @@ -210,16 +270,16 @@ integration_test_docker: - chmod u+x $CI_PROJECT_DIR/sbin/run_integration_test.sh # Do not remove 'bash' or statement will be ignored by primitive docker shell - bash $CI_PROJECT_DIR/sbin/run_integration_test.sh -#wheel_packaging: -# stage: packaging -# artifacts: -# paths: -# - tangostationcontrol/dist/*.whl -# before_script: -# - sudo apt-get update -# - sudo apt-get install -y git -# - pip3 install -r tangostationcontrol/test-requirements.txt -# - pip3 install -r docker-compose/itango/lofar-requirements.txt -# script: -# - cd tangostationcontrol -# - python setup.py bdist_wheel \ No newline at end of file +wheel_packaging: + stage: packaging + artifacts: + paths: + - tangostationcontrol/dist/*.whl + before_script: + - sudo apt-get update + - sudo apt-get install -y git + - pip3 install -r tangostationcontrol/test-requirements.txt + - pip3 install -r docker-compose/itango/lofar-requirements.txt + script: + - cd tangostationcontrol + - python setup.py bdist_wheel \ No newline at end of file diff --git a/bootstrap/etc/lofar20rc.sh b/bootstrap/etc/lofar20rc.sh index 6e4a5c9bc8d6a78c1b61cca02159ee01291d3805..4b9d806d819816a86c5fea3ab8eb59135d8edcfc 100755 --- a/bootstrap/etc/lofar20rc.sh +++ b/bootstrap/etc/lofar20rc.sh @@ -16,6 +16,7 @@ if [ ! -f "${LOFAR20_DIR}/.git/hooks/post-checkout" ]; then alias git="cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-checkout; cp ${LOFAR20_DIR}/bin/update_submodules.sh ${LOFAR20_DIR}/.git/hooks/post-merge; unalias git; git" fi +# CI_BUILD_ID does not exist see https://docs.gitlab.com/ee/ci/variables/predefined_variables.html if [ ! -z ${CI_BUILD_ID+x} ]; then export CONTAINER_NAME_PREFIX=${CI_BUILD_ID}- elif [ ! -z ${CI_JOB_ID+x} ]; then