image: docker:latest

workflow:
  rules:
    # don't create a pipeline if its a commit pipeline, on a branch and that branch has open merge requests.
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - when: always

variables:
  TEST_HBA_DATASET_NAME: "test_data.tar.gz"
  TEST_LBA_DATASET_NAME: "test_data_lba.tar.gz"
  CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
  CALIBRATOR_LBA_RESULTS_NAME: 'results_calibrator_lba.tar.gz'
  TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
  TARGET_LBA_RESULTS_NAME: "results_target_lba.tar.gz"
  TARGET_HBA_SELFCAL_RESULTS_NAME: "results_target_selfcal.tar.gz"
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  BUILD_DOCKER_IMAGE: "0"

stages:
  - versioning
  - build
  - tests
  - docs
  - deploy

before_script:
  - mkdir workdir
  - mkdir logs

after_script:
  - echo "All done"

versioning:
  stage: versioning
  image: bitnami/git
  script:
    # Unshallowing ensures that 'git describe' works
    - git fetch --unshallow
    - ./Docker/fetch_latest_commits.sh | tee commits.txt > versions.env
    - echo LINC_VERSION=$(git describe --tags --always) >> versions.env
    # Use hash of commits to determine version of base image (and rebuild if necessary)
    - echo INTEGRATION_BASE_IMAGE=${CI_REGISTRY_IMAGE}/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
    - echo INTEGRATION_IMAGE=${CI_REGISTRY_IMAGE}/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
    - cat versions.env
  artifacts:
    reports:
      dotenv: versions.env

# Docker login
.prepare:
  stage: build
  needs: ["versioning"]
  image: docker:latest
  tags:
    - dind
  before_script:
    - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY

# Create and push the integration image to the gitlab registry, if it does not exist.
build_base:
  stage: build
  extends: .prepare
  script:
    - |
      if ! docker manifest inspect $INTEGRATION_BASE_IMAGE > /dev/null || [ "$BUILD_DOCKER_IMAGE" = "1" ]; then
        if [ "$BUILD_DOCKER_IMAGE" = "1" ] ; then
          DOCKER_CACHE_PARAMETERS="--no-cache"
        else
          DOCKER_CACHE_PARAMETERS=""
        fi
        docker build --tag $INTEGRATION_BASE_IMAGE ${DOCKER_CACHE_PARAMETERS} \
          --build-arg LOFARSTMAN_COMMIT \
          --build-arg DYSCO_COMMIT \
          --build-arg IDG_COMMIT \
          --build-arg SAGECAL_COMMIT \
          --build-arg AOFLAGGER_COMMIT \
          --build-arg LOFARBEAM_COMMIT \
          --build-arg EVERYBEAM_COMMIT \
          --build-arg DP3_COMMIT \
          --build-arg WSCLEAN_COMMIT \
          -f Docker/Dockerfile-base .
        docker push $INTEGRATION_BASE_IMAGE
      fi
  tags:
    - das6

install_linc:
  stage: build
  needs: ["versioning","build_base"]
  image: docker:latest
  tags:
    - dind
  extends: .prepare
  script:
    - |
      if [ "$BUILD_DOCKER_IMAGE" = "1" ] ; then
        DOCKER_CACHE_PARAMETERS="--no-cache"
      else
        DOCKER_CACHE_PARAMETERS=""
      fi
      docker build --build-arg BASE_TAG=$INTEGRATION_BASE_IMAGE --tag $INTEGRATION_IMAGE ${DOCKER_CACHE_PARAMETERS} -f Docker/Dockerfile-linc .
    - docker push $INTEGRATION_IMAGE

download_data:
  needs: ["versioning","install_linc"]
  image: $INTEGRATION_IMAGE
  stage: build
  script:
    - mkdir data && cd data
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TEST_LBA_DATASET_NAME -O $TEST_LBA_DATASET_NAME && tar xfz $TEST_LBA_DATASET_NAME && rm -f $TEST_LBA_DATASET_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$CALIBRATOR_HBA_RESULTS_NAME -O $CALIBRATOR_HBA_RESULTS_NAME && tar xfz $CALIBRATOR_HBA_RESULTS_NAME && rm -f $CALIBRATOR_HBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$CALIBRATOR_LBA_RESULTS_NAME -O $CALIBRATOR_LBA_RESULTS_NAME && tar xfz $CALIBRATOR_LBA_RESULTS_NAME && rm -f $CALIBRATOR_LBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_RESULTS_NAME -O $TARGET_HBA_RESULTS_NAME && tar xfz $TARGET_HBA_RESULTS_NAME && rm -f $TARGET_HBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_LBA_RESULTS_NAME -O $TARGET_LBA_RESULTS_NAME && tar xfz $TARGET_LBA_RESULTS_NAME && rm -f $TARGET_LBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_SELFCAL_RESULTS_NAME -O $TARGET_HBA_SELFCAL_RESULTS_NAME && tar xfz $TARGET_HBA_SELFCAL_RESULTS_NAME && rm -f $TARGET_HBA_SELFCAL_RESULTS_NAME
  artifacts:
    paths:
    - data

validate_scripts:
  stage: tests
  # Versioning required for $INTEGRATION_IMAGE
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - |
      errors=0
      for file in $(find steps -name "*.cwl")
      do
        cwltool --validate $file || ((errors++))
      done
      ((errors == 0))

blsmooth:
  stage: tests
  # Versioning required for $INTEGRATION_IMAGE
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json

find_skymodel_cal:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json

check_ateam_separation:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json

run_hba_calibrator:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
    - test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_calibrator
  after_script:
    - find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
  artifacts:
    paths:
      - hba_calibrator_logs.tar.gz
    when: on_failure

run_hba_target:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
    - test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_target
  after_script:
    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
    - find /builds/RD/LINC/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
  artifacts:
    paths:
      - hba_target_logs.tar.gz
      - inspection.tar.gz
      - cal_solutions.tar.gz
    when: on_failure

run_hba_target_selfcal:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
    - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results /builds/RD/LINC/data/results_target_selfcal
  after_script:
    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_selfcal_logs.tar.gz --null -T -
    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
    - find /builds/RD/LINC/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
  artifacts:
    paths:
      - hba_target_selfcal_logs.tar.gz
      - inspection.tar.gz
      - cal_solutions.tar.gz
    when: on_failure

run_lba_calibrator:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
    - test_jobs/check_workflow_results.py results /builds/RD/LINC/data/results_calibrator_lba
  after_script:
    - find /tmp/run_lba_calibrator -name "*.log" -print0 | tar czf lba_calibrator_logs.tar.gz --null -T -
  artifacts:
    paths:
      - lba_calibrator_logs.tar.gz
    when: on_failure

run_lba_target:
  stage: tests
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
    - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results /builds/RD/LINC/data/results_target_lba
  after_script:
    - find /tmp/run_lba_target -name "*.log" -print0 | tar czf lba_target_logs.tar.gz --null -T -
    - find /tmp/run_lba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
    - find /builds/RD/LINC/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
  artifacts:
    paths:
      - lba_target_logs.tar.gz
      - inspection.tar.gz
      - cal_solutions.tar.gz
    when: on_failure

build_doc:
  stage: docs
  needs: ["versioning", "download_data"]
  image: $INTEGRATION_IMAGE
  before_script:
    - apt-get update
    - apt-get install -y make
    - pip install sphinx
  script:
    - cd docs
    - make html
  artifacts:
    paths:
      - docs/build/html
  rules:
    # Only add job for commits to master or on merge request events
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE != "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - changes:
        - docs/**/*

# Deploy to docker hub
deploy_docker:
  stage: deploy
  needs: ["versioning", "download_data"]
  image: docker:latest
  tags:
    - dind
  before_script:
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
  script:
    - docker pull $INTEGRATION_IMAGE
    - docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_VERSION
    - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
    - docker push astronrd/linc:$LINC_VERSION
    - docker push astronrd/linc:latest
  rules:
    # Only run on master
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: always

# Deploy a release version
deploy_release:
  stage: deploy
  needs: ["versioning", "download_data"]
  image: docker:latest
  tags:
    - dind
  before_script:
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
    - apk update && apk add git
  script:
    - echo -n "$CI_COMMIT_BRANCH" | sed -e "s/^releases\//export RELEASE=/" > version && source version
    - docker pull $INTEGRATION_IMAGE
    - docker tag $INTEGRATION_IMAGE astronrd/linc:$RELEASE
    - docker push astronrd/linc:$RELEASE
    # Update docker image in cwl steps by tagged version
    - sed -i "s/astronrd\/linc/astronrd\/linc:$RELEASE/g" steps/*.cwl
    # Create and push Git tag
    - git add steps/*.cwl
    - git config user.email $GITLAB_USER_EMAIL
    - git config user.name $GITLAB_USER_NAME
    # Skip CI on this commit and tag
    - git commit -m "[SKIP CI] Replace latest with tag $RELEASE"
    - git tag -a $RELEASE -m "[SKIP CI] Version $RELEASE created by gitlab-ci build"
    - git push --all
  rules:
    # TODO: check if pushed tag triggers Pipeline
    # Only run on release branches
    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
      when: always

deploy_docker_tag_stable:
  stage: deploy
  needs: ["versioning", "download_data"]
  when: manual
  image: docker:latest
  tags:
    - dind
  before_script:
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin
  script:
    - docker pull $INTEGRATION_IMAGE
    - docker tag $INTEGRATION_IMAGE astronrd/linc:stable
    - docker push astronrd/linc:stable
