Merge branch 'SW-973' into 'master'

Resolve SW-973 Closes SW-973 See merge request !478

Merge branch 'SW-973' into 'master'
2a8e08a0 · Auke Klazema · 420eb8ce · 75d790a8 · 2a8e08a0
Commit 2a8e08a0 authored 3 years ago by Auke Klazema
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,13 +5,56 @@ stages:
  - unit_test
  - integration_test
  - dockerize
-  - deploy
+  - deploy-test
+  - deploy-prod
 include:
  - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'
 variables:
  FASTLANE: "false"
+  VERSION: "0.0.0"
+set_version:
+  stage: .pre
+  script:
+    - if [[ $CI_COMMIT_TAG =~ "^LOFAR-Release-[0-9]_[0-9]_[0-9]" ]]; then TMP=${CI_COMMIT_TAG//LOFAR-Release-/}; VERSION=${TMP//_/.}; fi
+    - echo "BUILDING $VERSION"
+#
+# This script snippet needs a PACKAGE variable to define which package to build.
+# It also uses the global VERSION variable.
+#
+.build_mac_package: &build_mac_package
+  - echo "Building $PACKAGE..."
+  - mkdir -p build/gnucxx11_opt
+  - cd build/gnucxx11_opt
+  - cmake -DBUILD_PACKAGES=$PACKAGE -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.16/ -DBLITZ_ROOT_DIR=/opt/blitz/ -DCASACORE_ROOT_DIR=/opt/casacore/ -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
+  - make -j $(nproc)
+  - make DESTDIR=${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install install
+  - cd ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install/opt/lofar
+  - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
+  - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file ${PACKAGE}_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE}/${VERSION}/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"'
+#
+# This script snippet needs a LOFAR_TARGET variable to define the SSH host to connect to.
+#
+.prepare_ssh: &prepare_ssh
+  - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+  - eval $(ssh-agent -s)
+  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+  - mkdir -p ~/.ssh
+  - chmod 700 ~/.ssh
+  - ssh-keyscan $LOFAR_TARGET >> ~/.ssh/known_hosts
+  - chmod 644 ~/.ssh/known_hosts
+#
+# This script snipper needs a PACKAGE, a LOFAR_TARGET, and a LOFAR_USER variable to mac install a package to a target
+# It also uses the global VERSION variable
+#
+.mac_install_package: &mac_install_package
+  - ssh $LOFAR_USER@$LOFAR_TARGET "cd ~/incoming; wget --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar"
+  - ssh $LOFAR_USER@$LOFAR_TARGET "MAC_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}"
 #
 # PREPARE BASE STAGE
@@ -84,20 +127,77 @@ build_LTAIngest:
    paths:
      - build/gnucxx11_opt
-build_MCU_MAC:
+build_LTAIngestTransfer:
  stage: build
-  image: ci_mac:$CI_COMMIT_SHORT_SHA
+  image: ci_lta:$CI_COMMIT_SHORT_SHA
  script:
-    - PACKAGE=MCU_MAC
+    - PACKAGE="LTAIngestTransfer"
    - echo "Building $PACKAGE..."
    - mkdir -p build/gnucxx11_opt
    - cd build/gnucxx11_opt
-    - cmake -DBUILD_PACKAGES=$PACKAGE -DWINCC_ROOT_DIR=/opt/WinCC_OA/3.16/ -DBLITZ_ROOT_DIR=/opt/blitz/ -DCASACORE_ROOT_DIR=/opt/casacore/ -DCMAKE_INSTALL_PREFIX=/opt/lofar ../..
+    - cmake -DBUILD_PACKAGES="$PACKAGE" -DUSE_LOG4CPLUS=false ../..
    - make -j $(nproc)
-    - make DESTDIR=${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install install
+    - make install
-    - cd ${CI_BUILDS_DIR}/${CI_COMMIT_SHORT_SHA}/install/opt/lofar
+  interruptible: true
-    - tar --ignore-failed-read --exclude=include --exclude="*.ztar" -czf MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar *
+  needs:
-    - curl --insecure --upload-file MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.ztar -u upload:upload https://support.astron.nl/nexus/content/repositories/branches/nl/astron/lofar/${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}/MCU_MAC_${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}.x86_64.ztar
+    - prepare_ci_lta_docker_image
+  artifacts:
+    expire_in: 6 hours
+    paths:
+      - build/gnucxx11_opt
+build_MCU_MAC:
+  stage: build
+  image: ci_mac:$CI_COMMIT_SHORT_SHA
+  variables:
+    PACKAGE: MCU_MAC
+  script:
+    - *build_mac_package
+  interruptible: true
+  needs:
+    - prepare_ci_mac_docker_image
+  artifacts:
+    expire_in: 6 hours
+    paths:
+      - build/gnucxx11_opt
+build_LCU_MAC:
+  stage: build
+  image: ci_mac:$CI_COMMIT_SHORT_SHA
+  variables:
+    PACKAGE: "LCU_MAC"
+  script:
+    - *build_mac_package
+  interruptible: true
+  needs:
+    - prepare_ci_mac_docker_image
+  artifacts:
+    expire_in: 6 hours
+    paths:
+      - build/gnucxx11_opt
+build_CCU_MAC:
+  stage: build
+  image: ci_mac:$CI_COMMIT_SHORT_SHA
+  variables:
+    PACKAGE: "CCU_MAC"
+  script:
+    - *build_mac_package
+  interruptible: true
+  needs:
+    - prepare_ci_mac_docker_image
+  artifacts:
+    expire_in: 6 hours
+    paths:
+      - build/gnucxx11_opt
+build_ST_MAC:
+  stage: build
+  image: ci_mac:$CI_COMMIT_SHORT_SHA
+  variables:
+    PACKAGE: "ST_MAC"
+  script:
+    - *build_mac_package
  interruptible: true
  needs:
    - prepare_ci_mac_docker_image
@@ -160,6 +260,31 @@ unit_test_LTAIngest:
  rules:
    - if: '$FASTLANE == "false"'
+unit_test_LTAIngestTransfer:
+  stage: unit_test
+  image: ci_lta:$CI_COMMIT_SHORT_SHA
+  script:
+    - PACKAGE=LTAIngestTransfer
+    - echo "Testing $PACKAGE..."
+    - cd build/gnucxx11_opt
+    - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=true ctest
+  interruptible: true
+  services:
+    - rabbitmq:latest
+  variables:
+    RABBITMQ_DEFAULT_USER: guest
+    RABBITMQ_DEFAULT_PASS: guest
+    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
+  needs:
+    - build_LTAIngestTransfer
+  artifacts:
+    name: unit-test-report
+    when: always
+    paths:
+      - build/gnucxx11_opt/Testing/Temporary/LastTest.log
+  rules:
+    - if: '$FASTLANE == "false"'
 unit_test_MCU_MAC:
  stage: unit_test
  image: ci_mac:$CI_COMMIT_SHORT_SHA
@@ -183,7 +308,6 @@ unit_test_MCU_MAC:
    when: always
    paths:
      - build/gnucxx11_opt/Testing/Temporary/LastTest.log
-#  allow_failure: true # allow failure for now, so MAC_MCU failure does not block this pipeline and we can deploy TMSS. TODO: fix docker_mac environment and services so the tests pass.
  rules:
    - if: '$FASTLANE == "false"'
@@ -268,42 +392,62 @@ integration_test_LTAIngest:
  rules:
    - if: '$FASTLANE == "false"'
+integration_test_LTAIngestTransfer:
+  stage: integration_test
+  image: ci_lta:$CI_COMMIT_SHORT_SHA
+  script:
+    - PACKAGE=LTAIngestTransfer
+    - echo "Integration Testing $PACKAGE..."
+    - cd build/gnucxx11_opt
+    - SKIP_PYTHON_COVERAGE=true SKIP_INTEGRATION_TESTS=false SKIP_UNIT_TESTS=true ctest
+  interruptible: true
+  services:
+    - rabbitmq:latest
+  variables:
+    RABBITMQ_DEFAULT_USER: guest
+    RABBITMQ_DEFAULT_PASS: guest
+    LOFAR_DEFAULT_BROKER: 'rabbitmq' # override default 'localhost' which does not work for CI service rabbitmq.
+  needs:
+    - build_LTAIngestTransfer
+    - unit_test_LTAIngestTransfer
+  artifacts:
+    name: integration-test-report
+    when: always
+    paths:
+      - build/gnucxx11_opt/Testing/Temporary/LastTest.log
+  rules:
+    - if: '$FASTLANE == "false"'
 #
-# DEPLOY STAGE
+# DEPLOY TEST STAGE
 #
 deploy-tmss-test:
-  stage: deploy
+  stage: deploy-test
+  variables:
+    LOFAR_TARGET: "scu199.control.lofar"
+    LOFAR_USER: "lofarsys"
  before_script:
-    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - *prepare_ssh
-    - eval $(ssh-agent -s)
+  script:
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p 123 $SUPERVISOR_PASSWORD TMSS:*"
-    - mkdir -p ~/.ssh
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
-    - chmod 700 ~/.ssh
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA"
-    - ssh-keyscan scu199.control.lofar >> ~/.ssh/known_hosts
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest"
-    - chmod 644 ~/.ssh/known_hosts
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest"
-  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest"
-    - ssh lofarsys@scu199.control.lofar "supervisorctl -u user -p 123 stop TMSS:*"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*"
-    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
-    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA"
-    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest"
-    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest"
-    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest"
-    - ssh lofarsys@scu199.control.lofar "supervisorctl -u user -p 123 start TMSS:*"
  needs:
    - dockerize_TMSS
+  allow_failure: true
  when: manual
 deploy-tmss-ua:
-  stage: deploy
+  stage: deploy-test
+  variables:
+    LOFAR_TARGET: "tmss-ua.control.lofar"
  before_script:
-    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - *prepare_ssh
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - ssh-keyscan tmss-ua.control.lofar >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
  script:
    - cd SAS/TMSS
    - ssh lofarsys@tmss-ua.control.lofar "docker-compose -f docker-compose-ua.yml down"
@@ -317,35 +461,248 @@ deploy-tmss-ua:
    - dockerize_TMSS
    - job: integration_test_SCU
      artifacts: false
+  allow_failure: true
  when: manual
  only:
    - "master"
+deploy-MCU_MAC-test:
+  stage: deploy-test
+  variables:
+    LOFAR_TARGET: "mcu199.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: "MCU_MAC"
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0"
+    - *mac_install_package
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6"
+  needs: 
+    - unit_test_MCU_MAC
+  allow_failure: true
+  when: manual
+deploy-CCU_MAC-test:
+  stage: deploy-test
+  variables:
+    LOFAR_TARGET: "ccu199.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: "CCU_MAC"
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0"
+    - *mac_install_package
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6"
+  needs: 
+    - build_CCU_MAC
+  allow_failure: true
+  when: manual
+deploy-SCU-test:
+  stage: deploy-test
+  variables:
+    LOFAR_TARGET: "scu199.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: "SCU"
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all"
+    - *mac_install_package
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all"
+  needs: 
+    - integration_test_SCU
+  allow_failure: true
+  when: manual
+#
+# deploy-prod stage
+#
 deploy-tmss-dockerhub:
-  stage: deploy
+  stage: deploy-prod
  script:
    - docker login -u $CI_DOCKERHUB_USERNAME -p $CI_DOCKERHUB_PASSWORD
    - docker tag tmss_django:$CI_COMMIT_SHORT_SHA lofar/tmss:master-$CI_COMMIT_SHORT_SHA
    - docker push lofar/tmss:master-$CI_COMMIT_SHORT_SHA
    - docker logout
+  environment:
+    name: production
  needs:
    - dockerize_TMSS
    - job: integration_test_SCU
      artifacts: false
+  allow_failure: true
  when: manual
-deploy-MCU_MAC-test:
+deploy-MCU_MAC-prod:
-  stage: deploy
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "mcu001.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: "MCU_MAC"
  before_script:
-    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - *prepare_ssh
-    - eval $(ssh-agent -s)
+  script:
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0"
-    - mkdir -p ~/.ssh
+    - *mac_install_package
-    - chmod 700 ~/.ssh
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6"
-    - ssh-keyscan mcu199.control.lofar >> ~/.ssh/known_hosts
+  environment:
-    - chmod 644 ~/.ssh/known_hosts
+    name: production
-  script:
-    - ssh lofarsys@mcu199.control.lofar "MAC_install -b ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA} -v ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
  needs: 
    - unit_test_MCU_MAC
+  allow_failure: true
+  when: manual
+  only:
+    - tags
+deploy-CCU_MAC-prod:
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "ccu001.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: "CCU_MAC"
+  before_script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 0"
+    - *prepare_ssh
+    - ssh $LOFAR_USER@$LOFAR_TARGET "swlevel 6"
+  script:
+    - *mac_install_package
+  environment:
+    name: production
+  needs: 
+    - build_CCU_MAC
+  allow_failure: true
+  when: manual
+  only:
+    - tags
+deploy-SCU-prod:
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "scu001.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: "SCU"
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all"
+    - *mac_install_package
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all"
+  environment:
+    name: production
+  needs: 
+    - integration_test_SCU
+  allow_failure: true
+  when: manual
+  only:
+    - tags
+deploy-tmss-prod:
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "scu001.control.lofar"
+    LOFAR_USER: "lofarsys"
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop TMSS:*"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start TMSS:*"
+  environment:
+    name: production
+  needs:
+    - dockerize_TMSS
+  allow_failure: true
+  when: manual
+  only:
+    - tags
+deploy-LTAIngest-prod:
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "lexar003.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: LTAIngest
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all"
+    - *mac_install_package
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all"
+  environment:
+    name: production
+  needs: 
+    - integration_test_LTAIngest
+  allow_failure: true
  when: manual
+  only:
+    - tags
+deploy-LTAIngestTransfer-prod:
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "lexar004.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: LTAIngestTransfer
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD stop all"
+    - *mac_install_package
+    - ssh $LOFAR_USER@$LOFAR_TARGET "supervisorctl -u $SUPERVISOR_USER -p $SUPERVISOR_PASSWORD start all"
+  environment:
+    name: production
+  needs: 
+    - integration_test_LTAIngestTransfer
+  allow_failure: true
+  when: manual
+  only:
+    - tags
+deploy-lcu-prod:
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "lcuhead.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: LCU_MAC
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'cd ~/incoming; wget --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar'"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'swlevel 0'"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'MAC_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}'"
+  environment:
+    name: production
+  needs:
+    - build_LCU_MAC
+  allow_failure: true
+  when: manual
+  only:
+    - tags
+deploy-station-test-prod:
+  stage: deploy-prod
+  variables:
+    LOFAR_TARGET: "lcuhead.control.lofar"
+    LOFAR_USER: "lofarsys"
+    PACKAGE: ST_MAC
+  before_script:
+    - *prepare_ssh
+  script:
+    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'cd ~/incoming; wget --backups=1 --user=$MACINSTALL_USER --password=$MACINSTALL_PASSWORD https://git.astron.nl/api/v4/projects/6/packages/generic/$PACKAGE/$VERSION/${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar'"
+    - ssh $LOFAR_USER@$LOFAR_TARGET "lcurun -s today 'ST_install -f ${PACKAGE}_${CI_COMMIT_REF_NAME}_0.x86_64.ztar -v ${CI_COMMIT_REF_NAME}'"
+  environment:
+    name: production
+  needs:
+    - build_ST_MAC
+  allow_failure: true
+  when: manual
+  only:
+    - tags