.gitlab-ci.yml

workflow:
  rules:
    # Don't create a pipeline if commit is on a branch with open merge requests
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - when: always

variables:
  TEST_HBA_DATASET_NAME: "test_data.tar.gz"
  TEST_LBA_DATASET_NAME: "test_data_lba.tar.gz"
  CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
  CALIBRATOR_LBA_RESULTS_NAME: 'results_calibrator_lba.tar.gz'
  TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
  TARGET_LBA_RESULTS_NAME: "results_target_lba.tar.gz"
  TARGET_HBA_SELFCAL_RESULTS_NAME: "results_target_selfcal.tar.gz"
  TARGET_HBA_VLBI_RESULTS_NAME: "results_target_VLBI.tar.gz"
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  BUILD_DOCKER_IMAGE: "0"
  GIT_STRATEGY: clone  # ensure every job starts with a pristine working copy
  GIT_DEPTH: 0  # do not do shallow clones, needed for `git describe --tags`

stages:
  - initialize
  - versioning
  - build
  - install
  - prepare_tests
  - run_tests
  - docs
  - deploy
  - finalize

.release:
  before_script:
    # Make sure release branch is checked out when building a release
    - export RELEASE=$(echo -n $CI_COMMIT_BRANCH | sed -n 's,^releases/,,p')
    - |
      if test -n "$RELEASE"
      then
        echo "Checking out branch '$CI_COMMIT_BRANCH'"
        git checkout $CI_COMMIT_BRANCH
      fi

.versioning:
  image: python
  before_script:
    - !reference [.release, before_script]
    - pip install setuptools_scm
  script:
    # Determine the LINC version, and the docker image tag. If RELEASE is set,
    # check if it adheres to the PEP-440 naming conventions, and if so, set the LINC
    # version string.  If RELEASE is not set, then determine the LINC version string
    # using `setuptools_scm`. For the docker image tag, replace all the characters
    # in the LINC version string that are not allowed in a tag string with a dash.
    - |
      if test -n "$RELEASE"
      then
        LINC_VERSION=$(python -c 'import os; from packaging.version import Version; print(Version(os.getenv("RELEASE")))')
      else
        LINC_VERSION=$(python -m setuptools_scm)
      fi
    - LINC_TAG=${LINC_VERSION//[^[:alnum:]_.-]/-}
    - echo "LINC_VERSION=$LINC_VERSION" >> versions.env
    - echo "LINC_TAG=$LINC_TAG" >> versions.env
  artifacts:
    reports:
      dotenv: versions.env

.setup_git:
  image: bitnami/git
  before_script:
    # Use a token that has git push permissions. Token must be created in the
    # GitLab GUI; a project variable must be created containing the token.
    # See file `DEVELOPER.md` for details.
    - !reference [.release, before_script]
    - git config user.email $GITLAB_USER_EMAIL
    - git config user.name $GITLAB_USER_NAME
    - git remote set-url origin https://oauth2:$GIT_PUSH_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git

.setup_docker:
  stage: build
  image: docker
  tags:
    - dind
  before_script:
    - !reference [.release, before_script]
    - echo "Logging in as $CI_REGISTRY_USER @ $CI_REGISTRY"
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY

.run_tests:
  stage: run_tests
  image: $INTEGRATION_IMAGE
  before_script:
    - !reference [.release, before_script]
    - mkdir -p workdir
    - mkdir -p logs

.deploy:
  stage: deploy
  extends: .setup_docker
  tags:
    - dind
  before_script:
    - !reference [.setup_docker, before_script]
    - echo "Logging in as $DH_REGISTRY_USER @ DockerHub"
    - echo $DH_REGISTRY_PASSWORD | docker login -u $DH_REGISTRY_USER --password-stdin

.docs:
  stage: docs
  image: $INTEGRATION_IMAGE

.finalize:
  stage: finalize
  extends: .setup_git


### Stage: initialize

prepare_release:
  stage: initialize
  image: python
  extends: .setup_git
  rules:
    # Run this job when current branch is a release branch
    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
      when: always
  artifacts:
    # Create a marker file if release tag already exists. The presence of this
    # file is used in the `rollback_release` job, to decide that the existing
    # tag should not be removed.
    paths:
      - .tag.exists
    untracked: true
    when: on_failure
  before_script:
    - !reference [.versioning, before_script]
    # When building a release, bail out if release tag already exists
    - |
      if git ls-remote --tags --exit-code origin $RELEASE > /dev/null
      then
        echo "*** Release '$RELEASE' already exists. Bailing out! ***"
        touch .tag.exists
        exit 1
      fi
    - !reference [.setup_git, before_script]
  script:
    # Use .versioning:script to get LINC_TAG
    - !reference [.versioning, script]
    - echo "Updating dockerPull URI to use image tag '$LINC_TAG'"
    - sed -ri "/dockerPull/s,['\"]?(astronrd/linc).*,\1:$LINC_TAG," steps/*.cwl
    - git add -u steps/*.cwl
    # Only commit if there are changes
    - |
      if test -n "$(git status -uno --porcelain)"
      then
        git commit -m "Tag $RELEASE added to dockerPull URI in CWL steps (by GitLab CI)"
      fi
    # Tag current revision
    - git tag -a $RELEASE -m "Git tag $RELEASE created (by GitLab CI)"
    # Skip CI on this push
    - git push --follow-tags -o ci.skip


### Stage: versioning

versioning:
  stage: versioning
  extends: .versioning
  script:
    - !reference [.versioning, script]
    - ./Docker/fetch_commit_hashes.sh | tee commits.txt >> versions.env
    # Use hash of commits to determine version of base image (and rebuild if necessary)
    - echo INTEGRATION_BASE_IMAGE=$CI_REGISTRY_IMAGE/integration_base:$(sha256sum commits.txt | cut -d " " -f 1) >> versions.env
    - echo INTEGRATION_IMAGE=$CI_REGISTRY_IMAGE/integration_full:$(git log -n 1 --pretty=format:%H) >> versions.env
    - cat versions.env


### Stage: build

build_base:
  stage: build
  extends: .setup_docker
  script:
    - |
      if ! docker manifest inspect $INTEGRATION_BASE_IMAGE > /dev/null || [ "$BUILD_DOCKER_IMAGE" = "1" ]; then
        if [ "$BUILD_DOCKER_IMAGE" = "1" ] ; then
          DOCKER_CACHE_PARAMETERS="--no-cache"
        else
          DOCKER_CACHE_PARAMETERS=""
        fi
        docker build \
          $DOCKER_CACHE_PARAMETERS \
          --build-arg CASACORE_COMMIT \
          --build-arg LOFARSTMAN_COMMIT \
          --build-arg IDG_COMMIT \
          --build-arg AOFLAGGER_COMMIT \
          --build-arg EVERYBEAM_COMMIT \
          --build-arg SAGECAL_COMMIT \
          --build-arg DP3_COMMIT \
          --build-arg WSCLEAN_COMMIT \
          --build-arg PYTHONCASACORE_COMMIT \
          --build-arg LINC_VERSION \
          --file Docker/Dockerfile-base \
          --tag $INTEGRATION_BASE_IMAGE \
          .
        docker push $INTEGRATION_BASE_IMAGE
      fi


### Stage: install

install_linc:
  stage: install
  extends: .setup_docker
  script:
    - |
      if [ "$BUILD_DOCKER_IMAGE" = "1" ] ; then
        DOCKER_CACHE_PARAMETERS="--no-cache"
      else
        DOCKER_CACHE_PARAMETERS=""
      fi
      docker build \
        $DOCKER_CACHE_PARAMETERS \
        --build-arg BASE_IMAGE=$INTEGRATION_BASE_IMAGE \
        --build-arg LINC_VERSION=$LINC_VERSION \
        --file Docker/Dockerfile-linc \
        --tag $INTEGRATION_IMAGE \
        .
    - docker push $INTEGRATION_IMAGE


### Stage: prepare_tests

download_data:
  image: $INTEGRATION_IMAGE
  stage: prepare_tests
  script:
    - mkdir data && cd data
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TEST_LBA_DATASET_NAME -O $TEST_LBA_DATASET_NAME && tar xfz $TEST_LBA_DATASET_NAME && rm -f $TEST_LBA_DATASET_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$CALIBRATOR_HBA_RESULTS_NAME -O $CALIBRATOR_HBA_RESULTS_NAME && tar xfz $CALIBRATOR_HBA_RESULTS_NAME && rm -f $CALIBRATOR_HBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$CALIBRATOR_LBA_RESULTS_NAME -O $CALIBRATOR_LBA_RESULTS_NAME && tar xfz $CALIBRATOR_LBA_RESULTS_NAME && rm -f $CALIBRATOR_LBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_RESULTS_NAME -O $TARGET_HBA_RESULTS_NAME && tar xfz $TARGET_HBA_RESULTS_NAME && rm -f $TARGET_HBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_LBA_RESULTS_NAME -O $TARGET_LBA_RESULTS_NAME && tar xfz $TARGET_LBA_RESULTS_NAME && rm -f $TARGET_LBA_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_SELFCAL_RESULTS_NAME -O $TARGET_HBA_SELFCAL_RESULTS_NAME && tar xfz $TARGET_HBA_SELFCAL_RESULTS_NAME && rm -f $TARGET_HBA_SELFCAL_RESULTS_NAME
    - wget -nv https://support.astron.nl/software/ci_data/linc/$TARGET_HBA_VLBI_RESULTS_NAME -O $TARGET_HBA_VLBI_RESULTS_NAME && tar xfz $TARGET_HBA_VLBI_RESULTS_NAME && rm -f $TARGET_HBA_VLBI_RESULTS_NAME
  artifacts:
    paths:
      - data


### Stage: run_tests

validate_scripts:
  extends: .run_tests
  script:
    - |
      errors=0
      for file in $(find steps -name "*.cwl")
      do
        cwltool --validate $file || ((errors++))
      done
      ((errors == 0))

blsmooth:
  extends: .run_tests
  script:
    - cwltool --no-container steps/blsmooth.cwl test_jobs/blsmooth.json

find_skymodel_cal:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json

check_ateam_separation:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json

run_hba_calibrator:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator
  after_script:
    - find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
  artifacts:
    paths:
      - hba_calibrator_logs.tar.gz
    when: on_failure

run_hba_target:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target
  after_script:
    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
  artifacts:
    paths:
      - hba_target_logs.tar.gz
      - inspection.tar.gz
      - cal_solutions.tar.gz
    when: on_failure

run_hba_target_selfcal:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target_selfcal.json
    - test_jobs/check_workflow_results.py --skip_soltabs TGSSphase_final results $CI_PROJECT_DIR/data/results_target_selfcal
  after_script:
    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_selfcal_logs.tar.gz --null -T -
    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
  artifacts:
    paths:
      - hba_target_selfcal_logs.tar.gz
      - inspection.tar.gz
      - cal_solutions.tar.gz
    when: on_failure

run_hba_target_vlbi:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target_VLBI.cwl test_jobs/HBA_target.json
    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_target_VLBI
  after_script:
    - find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_vlbi_logs.tar.gz --null -T -
    - find /tmp/run_hba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
  artifacts:
    paths:
      - hba_target_vlbi_logs.tar.gz
      - inspection.tar.gz
      - cal_solutions.tar.gz
    when: on_failure

run_lba_calibrator:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_calibrator workflows/LBA_calibrator.cwl test_jobs/LBA_calibrator.json
    - test_jobs/check_workflow_results.py results $CI_PROJECT_DIR/data/results_calibrator_lba
  after_script:
    - find /tmp/run_lba_calibrator -name "*.log" -print0 | tar czf lba_calibrator_logs.tar.gz --null -T -
  artifacts:
    paths:
      - lba_calibrator_logs.tar.gz
    when: on_failure

run_lba_target:
  extends: .run_tests
  script:
    - cwltool --no-container --preserve-environment PATH --preserve-environment LINC_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_lba_target/ workflows/LBA_target.cwl test_jobs/LBA_target.json
    - test_jobs/check_workflow_results.py --skip_soltabs GSMtec_final results $CI_PROJECT_DIR/data/results_target_lba
  after_script:
    - find /tmp/run_lba_target -name "*.log" -print0 | tar czf lba_target_logs.tar.gz --null -T -
    - find /tmp/run_lba_target -name "*.png" -print0 | tar czf inspection.tar.gz --null -T -
    - find $CI_PROJECT_DIR/results/ -name "cal_solutions.h5" -print0 | tar czf cal_solutions.tar.gz --null -T -
  artifacts:
    paths:
      - lba_target_logs.tar.gz
      - inspection.tar.gz
      - cal_solutions.tar.gz
    when: on_failure


### Stage: docs

build_doc:
  extends: .docs
  before_script:
    - apt-get update
    - apt-get install -y make
    - pip install sphinx
  script:
    - cd docs
    - make html
  artifacts:
    paths:
      - docs/build/html
  rules:
    # Only add job for commits to the default branch or on merge request events
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE != "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - changes:
        - docs/**/*


### Stage: deploy

deploy_docker:
  stage: deploy
  extends: .deploy
  script:
    - echo "Deploying to DockerHub, using tag '$LINC_TAG'"
    - docker pull $INTEGRATION_IMAGE
    - docker tag $INTEGRATION_IMAGE astronrd/linc:$LINC_TAG
    - docker tag $INTEGRATION_IMAGE astronrd/linc:latest
    - docker push astronrd/linc:$LINC_TAG
    - docker push astronrd/linc:latest
  rules:
    # Run on the default branch or on a release branch
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
      when: on_success

deploy_docker_tag_stable:
  stage: deploy
  extends: .deploy
  when: manual
  script:
    - docker pull $INTEGRATION_IMAGE
    - docker tag $INTEGRATION_IMAGE astronrd/linc:stable
    - docker push astronrd/linc:stable


### Stage: finalize

rollback_release:
  stage: finalize
  extends: .finalize
  rules:
    # Run this job if the pipeline fails, to undo changes made in prepare_release.
    # We only care about removing the tag; other changes can remain.
    - if: '$CI_COMMIT_BRANCH =~ /^releases//'
      when: on_failure
  script:
    - echo "Rolling back release $RELEASE"
    - |
      if test -f .tag.exists
      then
        echo "*** Not removing existing tag '$RELEASE'! ***"
      else
        git push origin -d $RELEASE
      fi

finalize_release:
  stage: finalize
  extends: .finalize
  rules:
    # Run this job if the pipeline succeeds, to create a versioned release.
    # A versioned release is a release whose branch/tag name matches a string
    # that looks like "v<major-version>[.<minor_version>][...]".
    # We want to ensure that the release tag also exists on the default branch,
    # so that `git describe --tags` yields the desired output. Hence, we must
    # undo the changes made in the `prepare_release` job, before merging to
    # the default branch.
    - if: '$CI_COMMIT_BRANCH =~ /^releases\/v[0-9]+(\.[0-9]+)*/'
      when: on_success
  script:
    - echo "Finalizing release $RELEASE"
    # Update dockerPull image URI in CWL steps by removing version tag
    - sed -ri "/dockerPull/s,(astronrd/linc).*,\1," steps/*.cwl
    - git add -u steps/*.cwl
    # Only commit if there are changes
    - |
      if test -n "$(git status -uno --porcelain)"
      then
        git commit -m "Tag removed from dockerPull URI in CWL steps (by GitLab CI)"
      fi
    # Skip CI on this push
    - git push -o ci.skip
    # Next switch to the default branch and make sure it's up to date
    - git checkout $CI_DEFAULT_BRANCH
    - git pull
    - echo "Merging changes from $CI_COMMIT_BRANCH into $CI_DEFAULT_BRANCH"
    # Merge release branch into the default branch
    - git merge $CI_COMMIT_BRANCH -m"Merged release branch into default branch (by Gitlab CI)"
    # Skip CI on this push
    - git push -o ci.skip