diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ffafe1839926cac13ad3b6ad1e9a66d2907cde2..dcc0073f044180c9805c0cb7951ae98094c78d5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,13 +5,56 @@ stages: - unit_test - integration_test - dockerize - - deploy + - deploy-test + - deploy-prod include: - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' variables: FASTLANE: "false" + VERSION: "0.0.0" + +set_version: + stage: .pre + script: + - if [[ $CI_COMMIT_TAG =~ "^LOFAR-Release-[0-9]_[0-9]_[0-9]" ]]; then TMP=${CI_COMMIT_TAG//LOFAR-Release-/}; VERSION=${TMP//_/.}; fi + - echo "BUILDING $VERSION" + +# +# This script snippet needs a PACKAGE variable to define which package to build. +# It also uses the global VERSION variable. +# +.build_mac_package: &build_mac_package + - echo "Building $PACKAGE..." + - mkdir -p build/gnucxx11_opt + - cd build/gnucxx11_opt + - cmake -DBUILD_PACKAGES=$PACKAGE -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.16/ -DBLITZ_ROOT_DIR=/opt/blitz/ -DCASACORE_ROOT_DIR=/opt/casacore/ -DCMAKE_INSTALL_PREFIX=/opt/lofar ../.. + - make -j $(nproc) + - make DESTDIR=${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install install + - cd ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install/opt/lofar + - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar * + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE}/${VERSION}/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"' + +# +# This script snippet needs a LOFAR_TARGET variable to define the SSH host to connect to. +# +.prepare_ssh: &prepare_ssh + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' + - eval $(ssh-agent -s) + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - ssh-keyscan $LOFAR_TARGET >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + +# +# This script snipper needs a PACKAGE, a LOFAR_TARGET, and a LOFAR_USER variable to mac install a package to a target +# It also uses the global VERSION variable +# +.mac_install_package: &mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "cd ~/incoming; wget --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar" + - ssh $LOFAR_USER@$LOFAR_TARGET "MAC_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}" # # PREPARE BASE STAGE @@ -84,20 +127,77 @@ build_LTAIngest: paths: - build/gnucxx11_opt -build_MCU_MAC: +build_LTAIngestTransfer: stage: build - image: ci_mac:$CI_COMMIT_SHORT_SHA + image: ci_lta:$CI_COMMIT_SHORT_SHA script: - - PACKAGE=MCU_MAC + - PACKAGE="LTAIngestTransfer" - echo "Building $PACKAGE..." - mkdir -p build/gnucxx11_opt - cd build/gnucxx11_opt - - cmake -DBUILD_PACKAGES=$PACKAGE -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.16/ -DBLITZ_ROOT_DIR=/opt/blitz/ -DCASACORE_ROOT_DIR=/opt/casacore/ -DCMAKE_INSTALL_PREFIX=/opt/lofar ../.. + - cmake -DBUILD_PACKAGES="$PACKAGE" -DUSE_LOG4CPLUS=false ../.. - make -j $(nproc) - - make DESTDIR=${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install install - - cd ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install/opt/lofar - - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar * - - curl --insecure --upload-file MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar -u upload:upload https://support.astron.nl/nexus/content/repositories/branches/nl/astron/lofar/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.x86_64.ztar + - make install + interruptible: true + needs: + - prepare_ci_lta_docker_image + artifacts: + expire_in: 6 hours + paths: + - build/gnucxx11_opt + +build_MCU_MAC: + stage: build + image: ci_mac:$CI_COMMIT_SHORT_SHA + variables: + PACKAGE: MCU_MAC + script: + - *build_mac_package + interruptible: true + needs: + - prepare_ci_mac_docker_image + artifacts: + expire_in: 6 hours + paths: + - build/gnucxx11_opt + +build_LCU_MAC: + stage: build + image: ci_mac:$CI_COMMIT_SHORT_SHA + variables: + PACKAGE: "LCU_MAC" + script: + - *build_mac_package + interruptible: true + needs: + - prepare_ci_mac_docker_image + artifacts: + expire_in: 6 hours + paths: + - build/gnucxx11_opt + +build_CCU_MAC: + stage: build + image: ci_mac:$CI_COMMIT_SHORT_SHA + variables: + PACKAGE: "CCU_MAC" + script: + - *build_mac_package + interruptible: true + needs: + - prepare_ci_mac_docker_image + artifacts: + expire_in: 6 hours + paths: + - build/gnucxx11_opt + +build_ST_MAC: + stage: build + image: ci_mac:$CI_COMMIT_SHORT_SHA + variables: + PACKAGE: "ST_MAC" + script: + - *build_mac_package interruptible: true needs: - prepare_ci_mac_docker_image @@ -160,6 +260,31 @@ unit_test_LTAIngest: rules: - if: '$FASTLANE == "false"' +unit_test_LTAIngestTransfer: + stage: unit_test + image: ci_lta:$CI_COMMIT_SHORT_SHA + script: + - PACKAGE=LTAIngestTransfer + - echo "Testing $PACKAGE..." + - cd build/gnucxx11_opt + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true ctest + interruptible: true + services: + - rabbitmq:latest + variables: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq. + needs: + - build_LTAIngestTransfer + artifacts: + name: unit-test-report + when: always + paths: + - build/gnucxx11_opt/Testing/Temporary/LastTest.log + rules: + - if: '$FASTLANE == "false"' + unit_test_MCU_MAC: stage: unit_test image: ci_mac:$CI_COMMIT_SHORT_SHA @@ -183,7 +308,6 @@ unit_test_MCU_MAC: when: always paths: - build/gnucxx11_opt/Testing/Temporary/LastTest.log -# allow_failure: true # allow failure for now, so MAC_MCU failure does not block this pipeline and we can deploy TMSS. TODO: fix docker_mac environment and services so the tests pass. rules: - if: '$FASTLANE == "false"' @@ -268,42 +392,62 @@ integration_test_LTAIngest: rules: - if: '$FASTLANE == "false"' +integration_test_LTAIngestTransfer: + stage: integration_test + image: ci_lta:$CI_COMMIT_SHORT_SHA + script: + - PACKAGE=LTAIngestTransfer + - echo "Integration Testing $PACKAGE..." + - cd build/gnucxx11_opt + - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest + interruptible: true + services: + - rabbitmq:latest + variables: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq. + needs: + - build_LTAIngestTransfer + - unit_test_LTAIngestTransfer + artifacts: + name: integration-test-report + when: always + paths: + - build/gnucxx11_opt/Testing/Temporary/LastTest.log + rules: + - if: '$FASTLANE == "false"' + # -# DEPLOY STAGE +# DEPLOY TEST STAGE # deploy-tmss-test: - stage: deploy + stage: deploy-test + variables: + LOFAR_TARGET: "scu199.control.lofar" + LOFAR_USER: "lofarsys" before_script: - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' - - eval $(ssh-agent -s) - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - ssh-keyscan scu199.control.lofar >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - script: - - ssh lofarsys@scu199.control.lofar "supervisorctl -u user -p 123 stop TMSS:*" - - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA" - - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA" - - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest" - - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest" - - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest" - - ssh lofarsys@scu199.control.lofar "supervisorctl -u user -p 123 start TMSS:*" + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p 123 $SUPERVISOR_PASSWORD TMSS:*" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest" + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*" needs: - dockerize_TMSS + allow_failure: true when: manual deploy-tmss-ua: - stage: deploy + stage: deploy-test + variables: + LOFAR_TARGET: "tmss-ua.control.lofar" before_script: - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' - - eval $(ssh-agent -s) - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - ssh-keyscan tmss-ua.control.lofar >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts + - *prepare_ssh script: - cd SAS/TMSS - ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml down" @@ -317,35 +461,248 @@ deploy-tmss-ua: - dockerize_TMSS - job: integration_test_SCU artifacts: false + allow_failure: true when: manual only: - "master" +deploy-MCU_MAC-test: + stage: deploy-test + variables: + LOFAR_TARGET: "mcu199.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: "MCU_MAC" + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0" + - *mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6" + needs: + - unit_test_MCU_MAC + allow_failure: true + when: manual + +deploy-CCU_MAC-test: + stage: deploy-test + variables: + LOFAR_TARGET: "ccu199.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: "CCU_MAC" + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0" + - *mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6" + needs: + - build_CCU_MAC + allow_failure: true + when: manual + +deploy-SCU-test: + stage: deploy-test + variables: + LOFAR_TARGET: "scu199.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: "SCU" + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all" + - *mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all" + needs: + - integration_test_SCU + allow_failure: true + when: manual + +# +# deploy-prod stage +# + deploy-tmss-dockerhub: - stage: deploy + stage: deploy-prod script: - docker login -u $CI_DOCKERHUB_USERNAME -p $CI_DOCKERHUB_PASSWORD - docker tag tmss_django:$CI_COMMIT_SHORT_SHA lofar/tmss:master-$CI_COMMIT_SHORT_SHA - docker push lofar/tmss:master-$CI_COMMIT_SHORT_SHA - docker logout + environment: + name: production needs: - dockerize_TMSS - job: integration_test_SCU artifacts: false + allow_failure: true when: manual -deploy-MCU_MAC-test: - stage: deploy +deploy-MCU_MAC-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "mcu001.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: "MCU_MAC" before_script: - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' - - eval $(ssh-agent -s) - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - ssh-keyscan mcu199.control.lofar >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - script: - - ssh lofarsys@mcu199.control.lofar "MAC_install -b ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA} -v ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0" + - *mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6" + environment: + name: production needs: - unit_test_MCU_MAC + allow_failure: true + when: manual + only: + - tags + +deploy-CCU_MAC-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "ccu001.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: "CCU_MAC" + before_script: + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0" + - *prepare_ssh + - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6" + script: + - *mac_install_package + environment: + name: production + needs: + - build_CCU_MAC + allow_failure: true + when: manual + only: + - tags + +deploy-SCU-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "scu001.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: "SCU" + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all" + - *mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all" + environment: + name: production + needs: + - integration_test_SCU + allow_failure: true + when: manual + only: + - tags + +deploy-tmss-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "scu001.control.lofar" + LOFAR_USER: "lofarsys" + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop TMSS:*" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest" + - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest" + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*" + environment: + name: production + needs: + - dockerize_TMSS + allow_failure: true + when: manual + only: + - tags + +deploy-LTAIngest-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "lexar003.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: LTAIngest + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all" + - *mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all" + environment: + name: production + needs: + - integration_test_LTAIngest + allow_failure: true when: manual + only: + - tags + +deploy-LTAIngestTransfer-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "lexar004.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: LTAIngestTransfer + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all" + - *mac_install_package + - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all" + environment: + name: production + needs: + - integration_test_LTAIngestTransfer + allow_failure: true + when: manual + only: + - tags + +deploy-lcu-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "lcuhead.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: LCU_MAC + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'cd ~/incoming; wget --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar'" + - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'swlevel 0'" + - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'MAC_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}'" + environment: + name: production + needs: + - build_LCU_MAC + allow_failure: true + when: manual + only: + - tags + +deploy-station-test-prod: + stage: deploy-prod + variables: + LOFAR_TARGET: "lcuhead.control.lofar" + LOFAR_USER: "lofarsys" + PACKAGE: ST_MAC + before_script: + - *prepare_ssh + script: + - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'cd ~/incoming; wget --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar'" + - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'ST_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}'" + environment: + name: production + needs: + - build_ST_MAC + allow_failure: true + when: manual + only: + - tags