stages:
  - prepare-base
  - prepare
  - build
  - unit_test
  - integration_test
  - regression_test
  - dockerize
  - deploy-test
  - deploy-prod
  - quality-control

include:
  - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'

variables:
  VERSION: "0.0.0"
  USE_PYTHON: "true"
  CEP_HEADNODE: "head.cep4.control.lofar"
  MARK_PIPELINES_LATEST:
    description: "Use deployed pipelines as default in production?"
    value: "false"
    options:
      - "false"
      - "true"

#
# This script snippet sets the VERSION argument to the correct version based on the LOFAR-Release tag.
# Non LOFAR-Release tag builds will have the 0.0.0 version.
#
.set_version: &set_version
    - if echo $CI_COMMIT_TAG | grep -Eq "^LOFAR-Release-[0-9]+_[0-9]+_[0-9]+"; then TMP=${CI_COMMIT_TAG//LOFAR-Release-/}; VERSION=${TMP//_/.}; fi
    - echo "BUILDING $VERSION"

#
# This script snippet needs a PACKAGE variable to define which package to build.
# It also uses the global VERSION variable.
#
.build_mac_package: &build_mac_package
  - echo "Building $PACKAGE..."
  - mkdir -p build/gnucxx11_opt
  - cd build/gnucxx11_opt
  - /usr/local/bin/cmake -DBUILD_PACKAGES=$PACKAGE -DUSE_PYTHON=$USE_PYTHON -DUSE_UNITTEST++=OFF -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.16/ -DBLITZ_ROOT_DIR=/opt/blitz/ -DCASACORE_ROOT_DIR=/opt/casacore/ -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
  - make -j $(nproc)
  - make DESTDIR=${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install install
  - cd ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install/opt/lofar
  - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
  - *set_version
  - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE}/${VERSION}/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"'

#
# This script snippet needs a LOFAR_TARGET variable to define the SSH host to connect to.
#
.prepare_ssh: &prepare_ssh
  - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
  - eval $(ssh-agent -s)
  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - ssh-keyscan $LOFAR_TARGET >> ~/.ssh/known_hosts
  - chmod 644 ~/.ssh/known_hosts

#
# This script snipper needs a PACKAGE, a LOFAR_TARGET, and a LOFAR_USER variable to mac install a package to a target
# It also uses the global VERSION variable
#
.mac_install_package: &mac_install_package
  - *set_version
  - ssh $LOFAR_USER@$LOFAR_TARGET "mkdir -p ~/incoming && cd ~/incoming && wget --no-check-certificate --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"
  - ssh $LOFAR_USER@$LOFAR_TARGET "MAC_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}"

#
# PREPARE BASE STAGE
#
prepare_ci_base_docker_image:
  stage: prepare-base
  script:
    - docker build -t ci_base:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_base .
  interruptible: true

prepare_ci_base_ubuntu_docker_image:
  stage: prepare-base
  script:
    - docker build -t ci_base_ubuntu:$CI_COMMIT_SHORT_SHA -f Docker/lofar-ci/Dockerfile_ci_base_ubuntu .
  interruptible: true

#
# PREPARE STAGE
#
prepare_ci_scu_docker_image:
  stage: prepare
  needs:
    - prepare_ci_base_docker_image
  script:
    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_scu:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_scu .
  interruptible: true

prepare_ci_tmss_docker_image:
  stage: prepare
  needs:
    - prepare_ci_base_ubuntu_docker_image
  script:
    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_tmss:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_tmss .
  interruptible: true

prepare_ci_lta_docker_image:
  stage: prepare
  needs:
    - prepare_ci_scu_docker_image
  script:
    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_lta:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_lta .
  interruptible: true

prepare_ci_mac_docker_image:
  stage: prepare
  script:
    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_mac:$CI_COMMIT_SHORT_SHA -f  Docker/lofar-ci/Dockerfile_ci_mac .
  interruptible: true

#
# BUILD STAGE
#

build_SCU:
  stage: build
  image: ci_scu:$CI_COMMIT_SHORT_SHA
  variables:
    PACKAGE: SCU
  script:
    - echo "Building SCU..."
    - mkdir -p build/gnucxx11_opt
    - mkdir -p build/gnucxx11_opt/install
    - cd build/gnucxx11_opt
    - /usr/local/bin/cmake -DBUILD_PACKAGES="SCU" -DUSE_PYTHON=$USE_PYTHON -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
    - make -j $(nproc)
    - make DESTDIR=install install
    - cd install/opt/lofar
    - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
    - *set_version
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE}/${VERSION}/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"'
  interruptible: true
  needs:
    - prepare_ci_scu_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_TMSS:
  stage: build
  image: ci_tmss:$CI_COMMIT_SHORT_SHA
  variables:
    PACKAGE: TMSS
  script:
    - echo "Building TMSS..."
    - mkdir -p build/gnucxx11_opt
    - mkdir -p build/gnucxx11_opt/install
    - cd build/gnucxx11_opt
    - cmake -DBUILD_PACKAGES="TMSS" -DUSE_PYTHON=$USE_PYTHON -DCASACORE_ROOT_DIR=/opt/casacore/ -DCASAREST_ROOT_DIR=/opt/casarest/ -DUSE_LOG4CPLUS=false -DHAVE_BFD=0 -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
    - make -j $(nproc)
    - make DESTDIR=install install
    - cd install/opt/lofar
    - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
    - *set_version
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE}/${VERSION}/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"'
  interruptible: true
  needs:
    - prepare_ci_tmss_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_LTAIngest:
  stage: build
  image: ci_lta:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE="LTAIngest"
    - echo "Building $PACKAGE..."
    - mkdir -p build/gnucxx11_opt
    - mkdir -p build/gnucxx11_opt/install
    - cd build/gnucxx11_opt
    - /usr/local/bin/cmake -DBUILD_PACKAGES="$PACKAGE TMSSClient" -DUSE_PYTHON=$USE_PYTHON -DUSE_LOG4CPLUS=false -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
    - make -j $(nproc)
    - make DESTDIR=install install
    - cd install/opt/lofar
    - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
    - *set_version
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE}/${VERSION}/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"'
  interruptible: true
  needs:
    - prepare_ci_lta_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_MCU_MAC:
  stage: build
  image: ci_mac:$CI_COMMIT_SHORT_SHA
  variables:
    PACKAGE: MCU_MAC
  script:
    - *build_mac_package
  interruptible: true
  needs:
    - prepare_ci_mac_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_LCU_MAC:
  stage: build
  image: ci_mac:$CI_COMMIT_SHORT_SHA
  variables:
    PACKAGE: "LCU_MAC"
  script:
    - *build_mac_package
  interruptible: true
  needs:
    - prepare_ci_mac_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_CCU_MAC:
  stage: build
  image: ci_mac:$CI_COMMIT_SHORT_SHA
  variables:
    PACKAGE: "CCU_MAC"
  script:
    - *build_mac_package
  interruptible: true
  needs:
    - prepare_ci_mac_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_ST_MAC:
  stage: build
  image: ci_mac:$CI_COMMIT_SHORT_SHA
  variables:
    PACKAGE: "ST_MAC"
  script:
    - *build_mac_package
  interruptible: true
  needs:
    - prepare_ci_mac_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_Docker:
  stage: build
  image: ci_mac:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE="Docker" # required to create Dockerfiles from the Dockerfile templates
    - echo "Building $PACKAGE..."
    - mkdir -p build/gnucxx11_opt
    - cd build/gnucxx11_opt
    - /usr/local/bin/cmake -DBUILD_PACKAGES="$PACKAGE" ../..
    - make -j $(nproc)
    - make install  # this puts Dockerfile and dependencies like rc files in a place that can be used as build context
  interruptible: true
  needs:
    - prepare_ci_mac_docker_image
  artifacts:
    expire_in: 6 hours
    paths:
      - build/gnucxx11_opt

build_lofar_base:
  stage: build
  script:
    - CPU_OPTIMISATION="haswell"   # also supported: "x86-64" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
    - docker build --build-arg CPU_OPTIMISATION=$CPU_OPTIMISATION -t lofar-base:$CI_COMMIT_SHORT_SHA Docker/lofar-base
  interruptible: true
  when: manual # this takes a long time to build and the image is humongous. So let's not do that on every commit.

build_lofar_pipeline:
  stage: build
  script:
    - CPU_OPTIMISATION="haswell"   # also supported: "x86-64" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
    - docker build --build-arg LOFAR_VERSION=$CI_COMMIT_REF_NAME --build-arg LOFAR_BASE_IMAGE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA Docker/lofar-pipeline
    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
    - docker tag ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA
    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA
    - docker logout $CI_NEXUS_REGISTRY
  interruptible: true
  needs:
    - build_lofar_base

build_dynspec:
  stage: build
  script:
    - CPU_OPTIMISATION="haswell"   # also supported: "x86-64" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
    - docker build -t ci_dynspec:$CI_COMMIT_SHORT_SHA Docker/dynspec
    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
    - docker tag ci_dynspec:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA
    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA
    - docker logout $CI_NEXUS_REGISTRY
  interruptible: true
  when: manual # this takes a long time to build and the image is humongous. So let's not do that on every commit.

build_lofar_pulp:
  stage: build
  script:
    - docker build -t ci_pulp:$CI_COMMIT_SHORT_SHA Docker/pulp-base
    - docker build --build-arg BASE_VERSION=$CI_COMMIT_SHORT_SHA --build-arg LOFAR_TAG=$CI_COMMIT_REF_NAME -t ci_lofar_pulp:$CI_COMMIT_SHORT_SHA Docker/lofar-pulp
    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
    - docker tag ci_lofar_pulp:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA
    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA
    - docker logout $CI_NEXUS_REGISTRY
  interruptible: true
  when: manual # pulp docker images take a long time to build. It does not/hardly depend on currently developed lofar code. So an automatic build for each lofar commit is not needed.

build_lofar_tbbwriter_control:
  stage: build
  script:
    - CPU_OPTIMISATION="x86-64"   # also supported: "haswell" and "skylake". Do we need to support switching build variants/optimizations in an interactive way?
    - cd build/gnucxx11_opt
    - docker build --build-arg LOFAR_BASE_IMAGE_VERSION=$CI_COMMIT_SHORT_SHA -t ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA installed/share/docker/lofar-tbbwriter-control
    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
    - docker tag ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA
    - docker push $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA
    - docker logout $CI_NEXUS_REGISTRY
  interruptible: true
  needs:
    - build_Docker
    - build_lofar_base
  when: manual # this takes a long time to build and the image is humongous. So let's not do that on every commit.

#
# UNIT TEST STAGE
#

unit_test_SCU:
  stage: unit_test
  image: ci_scu:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=SCU
    - echo "Testing $PACKAGE..."
    - cd build/gnucxx11_opt
    - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true SKIP_REGRESSION_TESTS=true ctest
  interruptible: true
  services:
    - rabbitmq:latest
  variables:
    RABBITMQ_DEFAULT_USER: guest
    RABBITMQ_DEFAULT_PASS: guest
    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
  needs:
    - build_SCU
  artifacts:
    name: unit-test-report
    when: always
    paths:
      - build/gnucxx11_opt/Testing/Temporary/LastTest.log

unit_test_MCU_MAC:
  stage: unit_test
  image: ci_mac:$CI_COMMIT_SHORT_SHA
  script:
    - echo "Starting Qpid server..." && qpidd &
    - PACKAGE=MCU_MAC
    - echo "Testing $PACKAGE..."
    - cd build/gnucxx11_opt
    - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true SKIP_REGRESSION_TESTS=true ctest
  interruptible: true
  services:
    - rabbitmq:latest
  variables:
    RABBITMQ_DEFAULT_USER: guest
    RABBITMQ_DEFAULT_PASS: guest
    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
  needs:
    - build_MCU_MAC
  artifacts:
    name: unit-test-report
    when: always
    paths:
      - build/gnucxx11_opt/Testing/Temporary/LastTest.log

integration_test_TMSS_Frontend:
  stage: integration_test
  image: ci_tmss:$CI_COMMIT_SHORT_SHA
  allow_failure: true
  script:
    - PACKAGE=TMSSFrontend
    - echo "Testing $PACKAGE..."
    - cd build/gnucxx11_opt/SAS/TMSS/frontend/tmss_webapp
    - echo 'copying schemas ...'
    - npm run prepareTemplateSchemas
    - nohup npm start &
    - sleep 30
    - cd build
    - npm run test:ci
    - npx kill-port 3000
    - npm run cleanTemplateSchemas
  needs:
    - build_TMSS
  coverage: /All files[^|]*\|[^|]*\s+([\d\.]+)/
  artifacts:
    name: front-end-junit-report
    when: always
    paths:
      - build/gnucxx11_opt/SAS/TMSS/frontend/tmss_webapp/coverage/
    reports:
      junit:
        -  build/gnucxx11_opt/SAS/TMSS/frontend/tmss_webapp/junit.xml
      coverage_report:
        coverage_format: cobertura
        path:  build/gnucxx11_opt/SAS/TMSS/frontend/tmss_webapp/coverage/cobertura-coverage.xml

#
# DOCKERIZE
#

dockerize_TMSS:
  stage: dockerize
  allow_failure: true
  script:
    - cd build/gnucxx11_opt
    - docker build --build-arg TMSS_VERSION=$CI_COMMIT_SHORT_SHA --build-arg TMSS_COPY_DIR=install/opt/lofar -t tmss_django:$CI_COMMIT_SHORT_SHA -f docker/Dockerfile-tmss .
    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
    - docker tag tmss_django:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/tmss_django:$CI_COMMIT_SHORT_SHA
    - docker push $CI_NEXUS_REGISTRY_LOCATION/tmss_django:$CI_COMMIT_SHORT_SHA
    - docker logout $CI_NEXUS_REGISTRY
  interruptible: true
  needs:
    - job: build_TMSS
      artifacts: true

#
# INTEGRATION TEST STAGE
#

integration_test_SCU:
  stage: integration_test
  image: ci_scu:$CI_COMMIT_SHORT_SHA
  services:
    - rabbitmq:latest
  variables:
    RABBITMQ_DEFAULT_USER: guest
    RABBITMQ_DEFAULT_PASS: guest
    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
  script:
    - PACKAGE=SCU
    - echo "Integration Testing $PACKAGE..."
    - cd build/gnucxx11_opt
    - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true SKIP_REGRESSION_TESTS=true ctest
  interruptible: true
  needs:
    - build_SCU
    # - unit_test_SCU
  artifacts:
    name: integration-test-report
    when: always
    paths:
      - build/gnucxx11_opt/Testing/Temporary/LastTest.log

integration_test_LTAIngest:
  stage: integration_test
  image: ci_lta:$CI_COMMIT_SHORT_SHA
  script:
    - PACKAGE=LTAIngest
    - echo "Integration Testing $PACKAGE..."
    - cd build/gnucxx11_opt
    - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=false SKIP_REGRESSION_TESTS=true ctest
  interruptible: true
  services:
    - rabbitmq:latest
  variables:
    RABBITMQ_DEFAULT_USER: guest
    RABBITMQ_DEFAULT_PASS: guest
    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
  needs:
    - build_LTAIngest
  artifacts:
    name: integration-test-report
    when: always
    paths:
      - build/gnucxx11_opt/Testing/Temporary/LastTest.log

unit_and_integration_and_regression_test_TMSS:
  # for speed reasons, we just run all TMSS related unit/integration&regression tests in one job
  stage: integration_test
  image: ci_tmss:$CI_COMMIT_SHORT_SHA
  services:
    - rabbitmq:latest
  variables:
    RABBITMQ_DEFAULT_USER: guest
    RABBITMQ_DEFAULT_PASS: guest
    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
  before_script:
    - python3 --version
    - pip3 --version
    - pip3 list
  script:
    - PACKAGE=TMSS
    - echo "Unit & Integration Testing $PACKAGE..."
    - cd build/gnucxx11_opt
    - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=false SKIP_REGRESSION_TESTS=false ctest --output-on-failure
  interruptible: true
  needs:
    - build_TMSS
  artifacts:
    name: integration-test-report
    when: always
    paths:
      - build/gnucxx11_opt/Testing/Temporary/LastTest.log

multi_project_integration_test_TMSS_COBALT_STATION:
  stage: integration_test
  variables:
    LOFAR_VERSION: $CI_COMMIT_SHA
    COBALT_VERSION: main
    TANGO_VERSION: master
  trigger:
    project: lofar2.0/multi-project-integration
    strategy: depend

#
# REGRESSION TEST STAGE
#

regression_test_SCU:
  stage: regression_test
  image: ci_scu:$CI_COMMIT_SHORT_SHA
  services:
    - rabbitmq:latest
  variables:
    RABBITMQ_DEFAULT_USER: guest
    RABBITMQ_DEFAULT_PASS: guest
    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
  script:
    - PACKAGE=SCU
    - echo "Regression Testing $PACKAGE..."
    - cd build/gnucxx11_opt
    - SKIP_PYTHON_COVERAGE=true SKIP_REGRESSION_TESTS=false SKIP_UNIT_TESTS=true SKIP_INTEGRATION_TESTS=true ctest
  interruptible: true
  needs:
    - build_SCU
    # - unit_test_SCU
  artifacts:
    name: regression-test-report
    when: always
    paths:
      - build/gnucxx11_opt/Testing/Temporary/LastTest.log

#
# DEPLOY TEST STAGE
#

deploy-MCU_MAC-test:
  stage: deploy-test
  variables:
    LOFAR_TARGET: "mcu199.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: "MCU_MAC"
  before_script:
    - *prepare_ssh
  script:
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 0; if [ $? -eq 0 ]; then exit 0; else exit 1; fi'
    - *mac_install_package
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 6; if [ $? -eq 6 ]; then exit 0; else exit 1; fi'
  environment:
    name: MCU-TEST
  needs: 
    - job: unit_test_MCU_MAC
      artifacts: false
  allow_failure: true
  when: manual

deploy-CCU_MAC-test:
  stage: deploy-test
  variables:
    LOFAR_TARGET: "ccu199.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: "CCU_MAC"
  before_script:
    - *prepare_ssh
  script:
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 0; if [ $? -eq 0 ]; then exit 0; else exit 1; fi'
    - *mac_install_package
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 6; if [ $? -eq 6 ]; then exit 0; else exit 1; fi'
  environment:
    name: CCU-TEST
  needs: 
    - job: build_CCU_MAC
      artifacts: false
  allow_failure: true
  when: manual

deploy-SCU-test:
  stage: deploy-test
  variables:
    LOFAR_TARGET: "scu199.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: "SCU"
  before_script:
    - *prepare_ssh
  script:
    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all"
    - *mac_install_package
    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all"
  environment:
    name: test
  needs: 
    - job: integration_test_SCU
      artifacts: false
  allow_failure: true
  when: manual

#
# deploy-prod stage
#

deploy-tmss-dockerhub:
  stage: deploy-prod
  script:
    - docker login -u $CI_DOCKERHUB_USERNAME -p $CI_DOCKERHUB_PASSWORD
    - docker tag tmss_django:$CI_COMMIT_SHORT_SHA lofar/tmss:master-$CI_COMMIT_SHORT_SHA
    - docker push lofar/tmss:master-$CI_COMMIT_SHORT_SHA
    - docker logout
  environment:
    name: dockerhub
  needs:
    - dockerize_TMSS
    - job: integration_test_SCU
      artifacts: false
  allow_failure: true
  when: manual

deploy-tmss-prod:
  stage: deploy-prod
  variables:
    LOFAR_USER: "lofarsys"
    LOFAR_TARGET: "lcs129.control.lofar"
    SOURCE_IMAGE: "${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA"
  environment:
    name: production-lcs129
  before_script:
    - *prepare_ssh
  script:
    - apk update
    - apk add rsync
    - cd SAS/TMSS/deploy
    - cp ${TMSS_DEPLOY_LCS129_PRODUCTION} environment
    - chmod u+x environment
    - source environment
    - sh -e ./generate-env.sh
    - rm environment
    - rsync -aAXv --chmod=700 ./ ${LOFAR_USER}@${LOFAR_TARGET}:~/.lofar/tmss
    - rm -rf .env; rm -rf env; rm -rf app/.env
    - ssh $LOFAR_USER@$LOFAR_TARGET "cd .lofar/tmss; /localhome/lofarsys/bin/docker-compose build; /localhome/lofarsys/bin/docker-compose up -d; /localhome/lofarsys/bin/docker-compose logs db_migrate"
  after_script:
    - ssh $LOFAR_USER@$LOFAR_TARGET "cd .lofar/tmss; rm -rf .env; rm -rf env; rm -rf app/.env"
  needs:
    - job: dockerize_TMSS
      artifacts: false
    - job: unit_and_integration_and_regression_test_TMSS
      artifacts: false
  allow_failure: true
  when: manual
  only:
    - tags

deploy-MCU_MAC-prod:
  stage: deploy-prod
  variables:
    LOFAR_TARGET: "mcu001.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: "MCU_MAC"
  before_script:
    - *prepare_ssh
  script:
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 0; if [ $? -eq 0 ]; then exit 0; else exit 1; fi'
    - *mac_install_package
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 6; if [ $? -eq 6 ]; then exit 0; else exit 1; fi'
  environment:
    name: MCU-PROD
  needs: 
    - job: unit_test_MCU_MAC
      artifacts: false
  allow_failure: true
  when: manual
  only:
    - tags

deploy-CCU_MAC-prod:
  stage: deploy-prod
  variables:
    LOFAR_TARGET: "ccu001.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: "CCU_MAC"
  before_script:
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 0; if [ $? -eq 0 ]; then exit 0; else exit 1; fi'
    - *prepare_ssh
    - ssh $LOFAR_USER@$LOFAR_TARGET 'swlevel 6; if [ $? -eq 6 ]; then exit 0; else exit 1; fi'
  script:
    - *mac_install_package
  environment:
    name: CCU-PROD
  needs: 
    - job: build_CCU_MAC
      artifacts: false
  allow_failure: true
  when: manual
  only:
    - tags

deploy-lofar-pipeline-prod:
  stage: deploy-prod
  variables:
    CEP_PARTITION: "cpu"
  before_script:
    # needed so that gitlab-runner can login at cep4
    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
  script:
    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
    - echo "Deploying on ${NODES}"
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA"
    # the pipeline is built to be run as lofar-pipeline:$CI_COMMIT_REF_NAME
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pipeline:$CI_COMMIT_SHORT_SHA lofar-pipeline:$CI_COMMIT_REF_NAME"
    - if $MARK_PIPELINES_LATEST; then ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag lofar-pipeline:$CI_COMMIT_REF_NAME lofar-pipeline:latest"; done
  environment:
    name: production
  needs:
    - build_lofar_pipeline
  when: manual
  only:
    - tags

deploy-dynspec-prod:
  stage: deploy-prod
  variables:
    CEP_PARTITION: "cpu"
  before_script:
    # needed so that gitlab-runner can login at cep4
    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
  script:
    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
    - echo "Deploying on ${NODES}"
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA"
    # the pipeline is built to be run as dynspec:latest!!
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_dynspec:$CI_COMMIT_SHORT_SHA dynspec:latest"
  environment:
    name: production
  needs:
    - build_dynspec
  when: manual
  only:
    - tags

deploy-lofar-pulp-prod:
  stage: deploy-prod
  variables:
    CEP_PARTITION: "cpu"
  before_script:
    # needed so that gitlab-runner can login at cep4
    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
  script:
    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
    - echo "Deploying on ${NODES}"
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA"
    # the pulp pipeline is built to be run as lofar-pulp:$CI_COMMIT_REF_NAME through setting the LOFAR_TAG in build_lofar_pulp_docker_image
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_pulp:$CI_COMMIT_SHORT_SHA lofar-pulp:$CI_COMMIT_REF_NAME"
  environment:
    name: production
  needs:
    - build_lofar_pulp
  when: manual
  only:
    - tags

deploy-tbbwriter-control-prod:
  stage: deploy-prod
  variables:
    CEP_PARTITION: "cpu"
  before_script:
    # needed so that gitlab-runner can login at cep4
    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - ssh-keyscan ${CEP_HEADNODE} >> ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
  script:
    - NODES=$(ssh lofarsys@${CEP_HEADNODE} sinfo -p ${CEP_PARTITION} -h -o %N)
    - echo "Deploying on ${NODES}"
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker pull $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA"
    # the tbbwriter pipeline is built to be run as lofar-tbbwriter-control:$CI_COMMIT_REF_NAME
    - ssh lofarsys@${CEP_HEADNODE} "clush -S -P -w ${NODES} -w head[01-02] docker tag $CI_NEXUS_REGISTRY_LOCATION/ci_lofar_tbbwriter_control:$CI_COMMIT_SHORT_SHA lofar-tbbwriter-control:$CI_COMMIT_REF_NAME"
  environment:
    name: production
  needs:
    - build_lofar_tbbwriter_control
  when: manual
  only:
    - tags

deploy-SCU-prod:
  stage: deploy-prod
  variables:
    LOFAR_TARGET: "scu001.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: "SCU"
  before_script:
    - *prepare_ssh
  script:
    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all"
    - *mac_install_package
    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all"
  environment:
    name: production
  needs: 
    - job: integration_test_SCU
      artifacts: false
  allow_failure: true
  when: manual
  only:
    - tags


deploy-LTAIngest-prod:
  stage: deploy-prod
  variables:
    LOFAR_TARGET: "lexar003.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: LTAIngest
  before_script:
    - *prepare_ssh
  script:
    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all"
    - *mac_install_package
    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all"
  environment:
    name: production
  needs: 
    - job: integration_test_LTAIngest
      artifacts: false
  allow_failure: true
  when: manual
  only:
    - tags

deploy-lcu-prod:
  stage: deploy-prod
  variables:
    LOFAR_TARGET: "lcuhead.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: LCU_MAC
  before_script:
    - *prepare_ssh
  script:
    - *set_version
    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'cd ~/incoming; wget --no-check-certificate --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar'"
    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'swlevel 0; if [ $? -eq 0 ]; then exit 0; else exit 1; fi'"
    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'MAC_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}'"
  environment:
    name: production
  needs:
    - job: build_LCU_MAC
      artifacts: false
  allow_failure: true
  when: manual
  only:
    - tags

deploy-station-test-prod:
  stage: deploy-prod
  variables:
    LOFAR_TARGET: "lcuhead.control.lofar"
    LOFAR_USER: "lofarsys"
    PACKAGE: ST_MAC
  before_script:
    - *prepare_ssh
  script:
    - *set_version
    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'cd ~/incoming; wget --no-check-certificate --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar'"
    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'ST_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}'"
  environment:
    name: production
  needs:
    - job: build_ST_MAC
      artifacts: false
  allow_failure: true
  when: manual
  only:
    - tags

#
# Quality Control
#

.sonarcloud_check:
  stage: quality-control
  image:
    name: sonarsource/sonar-scanner-cli:latest
    entrypoint: [ "" ]
  variables:
    SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache
    GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
  cache:
    key: "${CI_JOB_NAME}"
    paths:
      - .sonar/cache

sonarcloud_check_TMSS_Frontend:
  extends: .sonarcloud_check
  allow_failure: true # for main branch, after all issues have been resolved, disable this
  needs:
    - integration_test_TMSS_Frontend
  script:
    #Run with the location of the sonar-project.properties for that specific project. The basic properties file is generated by sonarcloud when setting up your project
    - echo 'Checking contents of tms_webapp..'
    - ls -al SAS/TMSS/frontend/tmss_webapp
    - echo 'checking restored build output artifacts'
    - ls -al build/gnucxx11_opt/SAS/TMSS/frontend/tmss_webapp/coverage
    - sonar-scanner -Dproject.settings=SAS/TMSS/frontend/tmss_webapp/sonar-project.properties

package_check_TMSS_Frontend:
  stage: quality-control
  image: ci_tmss:$CI_COMMIT_SHORT_SHA
  allow_failure: true
  script:
    - PACKAGE=TMSSFrontend
    - cd build/gnucxx11_opt/SAS/TMSS/frontend/tmss_webapp
    - echo 'checking npm dependencies'
    - npx npm-check-updates  | tee dependency-check.txt
    - ! grep "Run npx npm-check-updates -u to upgrade package.json" dependency-check.txt #grep returns a boolean w.r.t. found/not found; ensuring the pipeline fails when there are packages to update
  needs:
    - build_TMSS
  except:
    - merge_requests

security_check_TMSS_Frontend:
  stage: quality-control
  image: ci_tmss:$CI_COMMIT_SHORT_SHA
  allow_failure: true #after all upgrades/updates, disable this again
  script:
    - PACKAGE=TMSSFrontend
    - cd build/gnucxx11_opt/SAS/TMSS/frontend/tmss_webapp
    - echo 'checking npm dependencies for security issues'
    - npm audit --production
  needs:
    - build_TMSS
  except:
    - merge_requests
