Add CI/CD

.gitignore

+.vscode
\ No newline at end of file

.gitlab-ci.yml

+variables:
+  EVERYBEAM_TAG: master
+  DP3_TAG: master
+  AOFLAGGER_TAG: master
+
+
+stages:
+  - build
+  - configure
+  - run-performance-tests
+  - collect-results
+
+.load-build:
+  before_script:
+  - module load spack/13.1.0 
+  - source /var/scratch/gitlab-runner/spack/share/spack/setup-env.sh
+  - spack env activate pipeline-metrics-collector
+                
+.load-run:
+  before_script:
+  - module load spack/13.1.0 
+  - source /var/scratch/gitlab-runner/spack/share/spack/setup-env.sh
+  - spack env activate pipeline-metrics-collector
+  - export CASARCFILES=$PWD/casarc
+  - "echo 'measures.directory: ${PWD}/measures' > casarc"
+  
+build-everybeam:
+  extends: [.load-build]
+  stage: build
+  tags:
+    - das6-gpu
+  script:
+  - scripts/build.sh https://git.astron.nl/RD/EveryBeam everybeam ${EVERYBEAM_TAG}
+  artifacts:
+    paths:
+      - everybeam-installed
+
+build-dp3:
+  extends: [.load-build]
+  needs:
+    - build-everybeam
+  stage: build
+  tags:
+    - das6-gpu
+  script:
+  - scripts/configure.sh everybeam
+  - module use modules
+  - module load everybeam/cicd
+  - scripts/build.sh https://git.astron.nl/RD/DP3 dp3 ${DP3_TAG}
+  artifacts:
+    paths:
+      - dp3-installed
+
+download-measures:
+  extends: [.load-build]
+  tags: 
+    - das6-gpu
+  stage: configure
+  script:
+    - mkdir measures
+    - wget ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar
+    - cd measures && tar -xvf ../WSRT_Measures.ztar
+  artifacts:
+    paths:
+      - measures
+      - casarc
+  
+dp3-performance:
+  extends: [.load-run]
+  needs:
+    - build-everybeam
+    - build-dp3
+    - download-measures
+  stage: run-performance-tests
+  tags:
+    - das6-gpu
+  script:
+  - scripts/configure.sh everybeam
+  - scripts/configure.sh dp3
+  - module use modules
+  - module load everybeam/cicd dp3/cicd
+  - scripts/run_test.sh $PWD integration_tests/dp3_ddecal.sh
+  - python3 scripts/parse_dp3_output.py stdout.log dp3_ddecal_timings.yaml
+  artifacts:
+    paths:
+      - "*.log"
+      - "*.yaml"
\ No newline at end of file

integration_tests/ddecal.parset

+avg.type=bdaaverager 
+avg.minchannels=1 
+avg.frequencybase=0.0 
+avg.maxinterval=75 
+avg.timebase=20000.0 
+steps=[solve] 
+solve.applycal.fastphase.correction=phase000 
+solve.applycal.parmdb=fast_phases.h5parm 
+solve.applycal.steps=[fastphase] 
+solve.beammode=array_factor 
+solve.datause=dual 
+solve.directions=[Patch_1014,Patch_1049,Patch_1051,Patch_1119,Patch_1122,Patch_1208,Patch_1372,Patch_1501,Patch_160,Patch_1669,Patch_18,Patch_2180,Patch_2357,Patch_2360,Patch_2409,Patch_2414,Patch_293,Patch_334,Patch_339,Patch_401,Patch_418,Patch_430,Patch_444,Patch_448,Patch_453,Patch_468,Patch_476,Patch_493,Patch_511,Patch_590,Patch_625,Patch_644,Patch_646,Patch_664,Patch_69,Patch_760,Patch_762,Patch_788,Patch_790,Patch_82,Patch_895,Patch_904,Patch_917,Patch_922,Patch_930,Patch_937,Patch_941,Patch_942,Patch_971,Patch_992] 
+solve.h5parm=slow_gain_separate_152.h5parm 
+solve.llssolver=qr 
+solve.maxiter=150 
+solve.mode=diagonal 
+solve.nchan=15 
+solve.onebeamperpatch=False 
+solve.parallelbaselines=False 
+solve.propagatesolutions=True 
+solve.sagecalpredict=False 
+solve.smoothnessconstraint=3000000.0 
+solve.solint=75 
+solve.solutions_per_direction=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] 
+solve.solveralgorithm=directioniterative 
+solve.solverlbfgs.dof=200.0 
+solve.solverlbfgs.iter=4 
+solve.solverlbfgs.minibatches=1 
+solve.sourcedb=calibration_skymodel.txt 
+solve.stepsigma=0.1 
+solve.stepsize=0.02 
+solve.tolerance=0.005 
+solve.type=ddecal 
+solve.usebeammodel=True 
+solve.uvlambdamin=150 
\ No newline at end of file

integration_tests/dp3_ddecal.sh

+#!/bin/bash
+set -e
+
+
+
+DATA_DIR=/var/scratch/mancini/wp5/andre_test/Testset
+SCRIPT_DIR=$(dirname $0)
+LOCAL_DIR=/local/gitlab-runner/
+rsync -av --progress ${DATA_DIR}/*.h5parm ${DATA_DIR}/epoch_5026988680.9055605_concatenated.ms ${DATA_DIR}/calibration_skymodel.txt ${LOCAL_DIR}/
+cp ${SCRIPT_DIR}/ddecal.parset ${LOCAL_DIR}
+
+cd ${LOCAL_DIR}
+
+echo EXECUTING COMMAND:
+echo DP3 ddecal.parset msin.datacolumn=DATA\
+     msin=epoch_5026988680.9055605_concatenated.ms \
+     msin.nchan=50 \
+     msin.ntimes=15 \
+     msin.startchan=100 \
+     msin.starttime=05Mar2018/22:47:54.161 \
+     numthreads=40 \
+     verbosity=verbose \
+     time_logging=true \
+     msout=.
+     
+
+DP3 ddecal.parset msin.datacolumn=DATA\
+     msin=epoch_5026988680.9055605_concatenated.ms \
+     msin.nchan=50 \
+     msin.ntimes=15 \
+     msin.startchan=100 \
+     msin.starttime=05Mar2018/22:47:54.161 \
+     numthreads=40 \
+     verbosity=verbose \
+     time_logging=true \
+     msout=.
+     
\ No newline at end of file

modules/dp3/cicd

+#%Module1.0
+
+
+set root "{DP3_PATH}"
+prepend-path LIBRARY_PATH $root/lib
+prepend-path PATH $root/bin
+prepend-path LD_LIBRARY_PATH $root/lib
+prepend-path LD_LIBRARY_PATH $root/lib64
+prepend-path CPATH $root/include
+
+prepend-path CMAKE_PREFIX_PATH $root/
+setenv DP3_ROOT $root

modules/everybeam/cicd

+#%Module1.0
+
+
+set root "{EVERYBEAM_PATH}"
+prepend-path LIBRARY_PATH $root/lib
+prepend-path PATH $root/bin
+prepend-path LD_LIBRARY_PATH $root/lib
+prepend-path LD_LIBRARY_PATH $root/lib64
+prepend-path CPATH $root/include
+
+prepend-path CMAKE_PREFIX_PATH $root/
+setenv EveryBeam_ROOT $root

scripts/build.sh

+#!/bin/bash
+set -e
+
+REPO_URL=$1
+REPO_NAME=$2
+REPO_TAG=$3
+
+shift 3
+EXTRA_OPTIONS=$@
+
+REPO_BUILD=${REPO_NAME}-build
+REPO_INSTALLED=${REPO_NAME}-installed
+
+CURRENT_DIR=${PWD}
+
+export OPENBLAS_NUM_THREADS=1
+
+if [ -d ${REPO_NAME} ]; 
+then
+    cd ${REPO_NAME}
+    git reset --hard 
+    git submodule init
+    git submodule update
+else
+    git clone ${REPO_URL} ${REPO_NAME}
+    cd ${REPO_NAME}
+    git submodule init
+    git submodule update
+fi
+
+git checkout ${REPO_TAG}
+
+cd ${CURRENT_DIR}
+
+cmake -B ${REPO_BUILD} ${REPO_NAME} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=${REPO_INSTALLED} ${EXTRA_OPTIONS}
+make install -C ${REPO_BUILD} -j 5

scripts/configure.sh

+#!/bin/bash
+set -e
+
+CURRENT_DIR=$PWD
+PACKAGE_NAME=$1
+PACKAGE_INSTALLED_PATH=$1-installed
+PACKAGE_MODULE_PATH=modules/$1/cicd
+
+sed -i 's#{'${PACKAGE_NAME^^}'_PATH}#'${PWD}'/'${PACKAGE_NAME,,}'-installed#' modules/${PACKAGE_NAME,,}/cicd

scripts/parse_dp3_output.py

+import subprocess
+from argparse import ArgumentParser
+import re
+import yaml
+
+def parse_args():
+    parser = ArgumentParser(description="Run DP3 test process and parse output")
+    parser.add_argument("dp3_output") 
+    parser.add_argument("parsed_timings")
+    return parser.parse_args()
+
+def trim_white_spaces(content):
+    return content.rstrip("\n")
+
+def read_file(dp3_stdout):
+    with open(dp3_stdout, "r") as f_stream:
+        return trim_white_spaces(f_stream.read())
+
+def is_description(string):
+    return "of" in string
+
+def parse_dp3_output(dp3_output):
+    # Parse Total 
+    # es.
+    # Total DP3 time     116.12 real     2303.97 user       46.42 system
+    result = re.search(r"Total DP3 time\s+(\d+\.\d+)\s+real\s+(\d+\.\d+)\s+user\s+(\d+\.\d+)\s+system+", dp3_output)
+    real, user, system = map(float, result.groups())
+    timings = {"total": {"real": real, "user": user, "system": system}, "steps":[]}
+    detailed_timings_output = dp3_output[result.end(0):].lstrip("\n").rstrip("\n")
+    last_step = None
+
+    # Parse sub steps
+    # es.
+    # 2024-Nov-07 15:46:45.468694    84.6% (   98  s) DDECal solve.
+    # 2024-Nov-07 15:46:45.468696            58.5% (   57  s) of it spent in predict
+    # 2024-Nov-07 15:46:45.468697            38.7% (   38  s) of it spent in estimating gains and computing residuals
+    # 2024-Nov-07 15:46:45.468699             0.0% (    5 ms) of it spent in writing gain solutions to disk
+    # 2024-Nov-07 15:46:45.468701           Substeps taken:
+    for line in detailed_timings_output.split("\n"):
+        line_match = re.search("(\d+.\d+)%\s\(\s+(\d+)\s+(\w+)\) (.*)", line)
+        if not line_match:
+            continue
+
+        percentage, timing, timing_unit, name_or_description = line_match.groups()
+        if not is_description(name_or_description):
+            step_name = name_or_description
+            last_step = {"name": step_name, 
+                         "percentage": percentage, 
+                         "timing": timing,
+                         "timing_unit": timing_unit,
+                         "sub_steps": []}
+
+            timings["steps"].append(last_step)
+        else:
+            sub_step_name = name_or_description.split("in ")[-1]
+            last_step["sub_steps"].append(
+                {
+                    "name": sub_step_name, 
+                    "percentage": percentage, 
+                    "timing": timing,
+                    "timing_unit": timing_unit
+                 }
+            )
+    return timings
+    
+def main():
+    args = parse_args()
+
+    parsed_timings = parse_dp3_output(read_file(args.dp3_output))
+    with open(args.parsed_timings, "w") as timing_file:
+        timing_file.write(yaml.dump(parsed_timings))
+        print(yaml.dump(parsed_timings))
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

scripts/run_test.sh

+#!/bin/bash
+set -e
+
+WORK_DIR=$1
+shift 1
+cmd=$@
+CURRENT_DIR=$PWD
+LOG_DIR=$CURRENT_DIR
+export OPENBLAS_NUM_THREADS=1
+
+cd $WORK_DIR
+($cmd | tee $LOG_DIR/stdout.log) |& tee $LOG_DIR/stderr.log
+cd $CURRENT_DIR
\ No newline at end of file

spack-env/spack.yaml

+spack:
+  # add package specs to the `specs` list
+  specs:
+  - casacore+python+hdf5 build_type=RelWithDebInfo
+  - python
+  - hdf5+cxx
+  - aoflagger
+  - boost+program_options
+  - openblas threads=pthreads
+  - py-pyyaml
+  - py-matplotlib
+  - py-numpy
+  - py-seaborn
+  - py-pandas
+  - cuda@12.6.0
+  - gcc@13.1.0
+  view: true
+  concretizer:
+    unify: true
+  repos:
+  - /var/scratch/gitlab-runner/spack-repos/ska-sdp-spack
+  compilers:
+  - compiler:
+      spec: gcc@=13.1.0
+      paths:
+        cc: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/gcc
+        cxx: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/g++
+        f77: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/gfortran
+        fc: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/gfortran
+      flags: {}
+      operating_system: rocky8
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []