stages:
- prepare-base
- prepare
- build
- unit_test
- integration_test
- dockerize
- deploy
include:
- template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'
variables:
FASTLANE: "false"
#
# PREPARE BASE STAGE
#
prepare_ci_base_docker_image:
stage: prepare-base
script:
- docker build -t ci_base:$CI_COMMIT_SHORT_SHA -f Docker/lofar-ci/Dockerfile_ci_base .
#
# PREPARE STAGE
#
prepare_ci_sas_docker_image:
stage: prepare
script:
- docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_sas:$CI_COMMIT_SHORT_SHA -f Docker/lofar-ci/Dockerfile_ci_sas .
prepare_ci_lta_docker_image:
stage: prepare
script:
- docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_lta:$CI_COMMIT_SHORT_SHA -f Docker/lofar-ci/Dockerfile_ci_lta .
prepare_ci_mac_docker_image:
stage: prepare
script:
- docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_mac:$CI_COMMIT_SHORT_SHA -f Docker/lofar-ci/Dockerfile_ci_mac .
#
# BUILD STAGE
#
build_TMSS:
stage: build
image: ci_sas:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=TMSS
- echo "Building $PACKAGE..."
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false ../..
- make -j 12
- make install
dependencies:
- prepare_ci_sas_docker_image
artifacts:
expire_in: 6 hours
paths:
- build/gnucxx11_opt
build_RAServices:
stage: build
image: ci_sas:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=RAServices
- echo "Building $PACKAGE..."
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false ../..
- make -j 12
- make install
dependencies:
- prepare_ci_sas_docker_image
artifacts:
expire_in: 6 hours
paths:
- build/gnucxx11_opt
build_LTAIngest:
stage: build
image: ci_lta:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=LTAIngest
- echo "Building $PACKAGE..."
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DUSE_LOG4CPLUS=false ../..
- make -j 12
- make install
dependencies:
- prepare_ci_lta_docker_image
artifacts:
expire_in: 6 hours
paths:
- build/gnucxx11_opt
build_MCU_MAC:
stage: build
image: ci_mac:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=MCU_MAC
- echo "Building $PACKAGE..."
- mkdir -p build/gnucxx11_opt
- cd build/gnucxx11_opt
- cmake -DBUILD_PACKAGES=$PACKAGE -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.16/ -DBLITZ_ROOT_DIR=/opt/blitz/ -DCASACORE_ROOT_DIR=/opt/casacore/ -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
- make -j 12
- make DESTDIR=${CI_BUILDS_DIR}/install install
- cd ${CI_BUILDS_DIR}/install/opt/lofar
- tar --ignore-failed-read --exclude=include -czf MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
- curl --insecure --upload-file MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar -u upload:upload https://support.astron.nl/nexus/content/repositories/branches/nl/astron/lofar/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.x86_64.ztar
dependencies:
- prepare_ci_mac_docker_image
artifacts:
expire_in: 6 hours
paths:
- build/gnucxx11_opt
#
# UNIT TEST STAGE
#
unit_test_TMSS:
stage: unit_test
image: ci_sas:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=TMSS
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
dependencies:
- build_TMSS
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
rules:
- if: '$FASTLANE == "false"'
unit_test_RAServices:
stage: unit_test
image: ci_sas:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=RAServices
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_RAServices
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
rules:
- if: '$FASTLANE == "false"'
unit_test_LTAIngest:
stage: unit_test
image: ci_lta:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=LTAIngest
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_LTAIngest
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
rules:
- if: '$FASTLANE == "false"'
unit_test_MCU_MAC:
stage: unit_test
image: ci_mac:$CI_COMMIT_SHORT_SHA
script:
- echo "Starting Qpid server..." && qpidd &
- PACKAGE=MCU_MAC
- echo "Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_MCU_MAC
artifacts:
name: unit-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
# allow_failure: true # allow failure for now, so MAC_MCU failure does not block this pipeline and we can deploy TMSS. TODO: fix docker_mac environment and services so the tests pass.
rules:
- if: '$FASTLANE == "false"'
#
# DOCKERIZE
#
dockerize_TMSS:
stage: dockerize
script:
- cd build/gnucxx11_opt
- ls *
- docker build --build-arg SAS_VERSION=$CI_COMMIT_SHORT_SHA -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss .
- cd ../..
- cd SAS/TMSS/test/oidc/docker-test-mozilla-django-oidc
- docker build -t tmss_testprovider:$CI_COMMIT_SHORT_SHA -f dockerfiles/oidc_testprovider .
- docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
- docker tag tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
- docker push nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
- docker tag tmss_testprovider:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
- docker push nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
- docker logout $CI_NEXUS_REGISTRY
dependencies:
- build_TMSS
#
# INTEGRATION TEST STAGE
#
integration_test_TMSS:
stage: integration_test
image: ci_sas:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=TMSS
- echo "Integration Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_TMSS
artifacts:
name: integration-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
rules:
- if: '$FASTLANE == "false"'
integration_test_RAServices:
stage: integration_test
image: ci_sas:$CI_COMMIT_SHORT_SHA
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
script:
- PACKAGE=RAServices
- echo "Integration Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
dependencies:
- build_RAServices
artifacts:
name: integration-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
rules:
- if: '$FASTLANE == "false"'
integration_test_LTAIngest:
stage: integration_test
image: ci_lta:$CI_COMMIT_SHORT_SHA
script:
- PACKAGE=LTAIngest
- echo "Integration Testing $PACKAGE..."
- cd build/gnucxx11_opt
- SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
services:
- rabbitmq:latest
variables:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
dependencies:
- build_LTAIngest
artifacts:
name: integration-test-report
when: always
paths:
- build/gnucxx11_opt/Testing/Temporary/LastTest.log
rules:
- if: '$FASTLANE == "false"'
#
# DEPLOY STAGE
#
deploy-tmss-test:
stage: deploy
before_script:
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan scu199.control.lofar >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- cd SAS/TMSS
- ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml down"
- scp docker-compose-scu199.yml lofarsys@scu199.control.lofar:~/
- ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
- ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA"
- ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
- ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
- ssh lofarsys@scu199.control.lofar "docker-compose -f docker-compose-scu199.yml up -d"
dependencies:
- dockerize_TMSS
when: manual
deploy-tmss-ua:
stage: deploy
before_script:
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan tmss-ua.control.lofar >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- cd SAS/TMSS
- ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml down"
- scp docker-compose-ua.yml lofarsys@tmss-ua.control.lofar:~/
- ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
- ssh lofarsys@tmss-ua.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA"
- ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
- ssh lofarsys@tmss-ua.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
- ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml up -d"
dependencies:
- dockerize_TMSS
when: manual
only:
- "master"
deploy-tmss-dockerhub:
stage: deploy
script:
- docker login -u $CI_DOCKERHUB_USERNAME -p $CI_DOCKERHUB_PASSWORD
- docker tag tmss_django:$CI_COMMIT_SHORT_SHA lofar/tmss:master-$CI_COMMIT_SHORT_SHA
- docker push lofar/tmss:master-$CI_COMMIT_SHORT_SHA
- docker logout
dependencies:
- dockerize_TMSS
when: manual
only:
- "master"
deploy-MCU_MAC-test:
stage: deploy
before_script:
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan mcu199.control.lofar >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- ssh lofarsys@mcu199.control.lofar "MAC_install -b ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA} -v ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
when: manual