diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..600d2d33badf45cc068e01d2e3c837e11c417bc4
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+.vscode
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..68930d805853691ae4b259066143ed811f3fdc4d
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,87 @@
+variables:
+ EVERYBEAM_TAG: master
+ DP3_TAG: master
+ AOFLAGGER_TAG: master
+
+
+stages:
+ - build
+ - configure
+ - run-performance-tests
+ - collect-results
+
+.load-build:
+ before_script:
+ - module load spack/13.1.0
+ - source /var/scratch/gitlab-runner/spack/share/spack/setup-env.sh
+ - spack env activate pipeline-metrics-collector
+
+.load-run:
+ before_script:
+ - module load spack/13.1.0
+ - source /var/scratch/gitlab-runner/spack/share/spack/setup-env.sh
+ - spack env activate pipeline-metrics-collector
+ - export CASARCFILES=$PWD/casarc
+ - "echo 'measures.directory: ${PWD}/measures' > casarc"
+
+build-everybeam:
+ extends: [.load-build]
+ stage: build
+ tags:
+ - das6-gpu
+ script:
+ - scripts/build.sh https://git.astron.nl/RD/EveryBeam everybeam ${EVERYBEAM_TAG}
+ artifacts:
+ paths:
+ - everybeam-installed
+
+build-dp3:
+ extends: [.load-build]
+ needs:
+ - build-everybeam
+ stage: build
+ tags:
+ - das6-gpu
+ script:
+ - scripts/configure.sh everybeam
+ - module use modules
+ - module load everybeam/cicd
+ - scripts/build.sh https://git.astron.nl/RD/DP3 dp3 ${DP3_TAG}
+ artifacts:
+ paths:
+ - dp3-installed
+
+download-measures:
+ extends: [.load-build]
+ tags:
+ - das6-gpu
+ stage: configure
+ script:
+ - mkdir measures
+ - wget ftp://ftp.astron.nl/outgoing/Measures/WSRT_Measures.ztar
+ - cd measures && tar -xvf ../WSRT_Measures.ztar
+ artifacts:
+ paths:
+ - measures
+ - casarc
+
+dp3-performance:
+ extends: [.load-run]
+ needs:
+ - build-everybeam
+ - build-dp3
+ - download-measures
+ stage: run-performance-tests
+ tags:
+ - das6-gpu
+ script:
+ - scripts/configure.sh everybeam
+ - scripts/configure.sh dp3
+ - module use modules
+ - module load everybeam/cicd dp3/cicd
+ - scripts/run_test.sh $PWD integration_tests/dp3_ddecal.sh
+ - python3 scripts/parse_dp3_output.py stdout.log dp3_ddecal_timings.yaml
+ artifacts:
+ paths:
+ - "*.log"
+ - "*.yaml"
\ No newline at end of file
diff --git a/integration_tests/ddecal.parset b/integration_tests/ddecal.parset
new file mode 100644
index 0000000000000000000000000000000000000000..2a4b825353bf93f5ba130aff460d358227e6f869
--- /dev/null
+++ b/integration_tests/ddecal.parset
@@ -0,0 +1,35 @@
+avg.type=bdaaverager
+avg.minchannels=1
+avg.frequencybase=0.0
+avg.maxinterval=75
+avg.timebase=20000.0
+steps=[solve]
+solve.applycal.fastphase.correction=phase000
+solve.applycal.parmdb=fast_phases.h5parm
+solve.applycal.steps=[fastphase]
+solve.beammode=array_factor
+solve.datause=dual
+solve.directions=[Patch_1014,Patch_1049,Patch_1051,Patch_1119,Patch_1122,Patch_1208,Patch_1372,Patch_1501,Patch_160,Patch_1669,Patch_18,Patch_2180,Patch_2357,Patch_2360,Patch_2409,Patch_2414,Patch_293,Patch_334,Patch_339,Patch_401,Patch_418,Patch_430,Patch_444,Patch_448,Patch_453,Patch_468,Patch_476,Patch_493,Patch_511,Patch_590,Patch_625,Patch_644,Patch_646,Patch_664,Patch_69,Patch_760,Patch_762,Patch_788,Patch_790,Patch_82,Patch_895,Patch_904,Patch_917,Patch_922,Patch_930,Patch_937,Patch_941,Patch_942,Patch_971,Patch_992]
+solve.h5parm=slow_gain_separate_152.h5parm
+solve.llssolver=qr
+solve.maxiter=150
+solve.mode=diagonal
+solve.nchan=15
+solve.onebeamperpatch=False
+solve.parallelbaselines=False
+solve.propagatesolutions=True
+solve.sagecalpredict=False
+solve.smoothnessconstraint=3000000.0
+solve.solint=75
+solve.solutions_per_direction=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+solve.solveralgorithm=directioniterative
+solve.solverlbfgs.dof=200.0
+solve.solverlbfgs.iter=4
+solve.solverlbfgs.minibatches=1
+solve.sourcedb=calibration_skymodel.txt
+solve.stepsigma=0.1
+solve.stepsize=0.02
+solve.tolerance=0.005
+solve.type=ddecal
+solve.usebeammodel=True
+solve.uvlambdamin=150
\ No newline at end of file
diff --git a/integration_tests/dp3_ddecal.sh b/integration_tests/dp3_ddecal.sh
new file mode 100755
index 0000000000000000000000000000000000000000..aed5552d7228fdb4629bc81419e930d37aa70105
--- /dev/null
+++ b/integration_tests/dp3_ddecal.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e
+
+
+
+DATA_DIR=/var/scratch/mancini/wp5/andre_test/Testset
+SCRIPT_DIR=$(dirname $0)
+LOCAL_DIR=/local/gitlab-runner/
+rsync -av --progress ${DATA_DIR}/*.h5parm ${DATA_DIR}/epoch_5026988680.9055605_concatenated.ms ${DATA_DIR}/calibration_skymodel.txt ${LOCAL_DIR}/
+cp ${SCRIPT_DIR}/ddecal.parset ${LOCAL_DIR}
+
+cd ${LOCAL_DIR}
+
+echo EXECUTING COMMAND:
+echo DP3 ddecal.parset msin.datacolumn=DATA\
+ msin=epoch_5026988680.9055605_concatenated.ms \
+ msin.nchan=50 \
+ msin.ntimes=15 \
+ msin.startchan=100 \
+ msin.starttime=05Mar2018/22:47:54.161 \
+ numthreads=40 \
+ verbosity=verbose \
+ time_logging=true \
+ msout=.
+
+
+DP3 ddecal.parset msin.datacolumn=DATA\
+ msin=epoch_5026988680.9055605_concatenated.ms \
+ msin.nchan=50 \
+ msin.ntimes=15 \
+ msin.startchan=100 \
+ msin.starttime=05Mar2018/22:47:54.161 \
+ numthreads=40 \
+ verbosity=verbose \
+ time_logging=true \
+ msout=.
+
\ No newline at end of file
diff --git a/modules/dp3/cicd b/modules/dp3/cicd
new file mode 100644
index 0000000000000000000000000000000000000000..4f79ec77e0ac291383bd886ee6713594297d46e6
--- /dev/null
+++ b/modules/dp3/cicd
@@ -0,0 +1,12 @@
+#%Module1.0
+
+
+set root "{DP3_PATH}"
+prepend-path LIBRARY_PATH $root/lib
+prepend-path PATH $root/bin
+prepend-path LD_LIBRARY_PATH $root/lib
+prepend-path LD_LIBRARY_PATH $root/lib64
+prepend-path CPATH $root/include
+
+prepend-path CMAKE_PREFIX_PATH $root/
+setenv DP3_ROOT $root
diff --git a/modules/everybeam/cicd b/modules/everybeam/cicd
new file mode 100644
index 0000000000000000000000000000000000000000..5796baa5bf582fdef34420a6cebef3e085e867b8
--- /dev/null
+++ b/modules/everybeam/cicd
@@ -0,0 +1,12 @@
+#%Module1.0
+
+
+set root "{EVERYBEAM_PATH}"
+prepend-path LIBRARY_PATH $root/lib
+prepend-path PATH $root/bin
+prepend-path LD_LIBRARY_PATH $root/lib
+prepend-path LD_LIBRARY_PATH $root/lib64
+prepend-path CPATH $root/include
+
+prepend-path CMAKE_PREFIX_PATH $root/
+setenv EveryBeam_ROOT $root
diff --git a/scripts/build.sh b/scripts/build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..561fceb60f469317c0ee8e3a04805fac32f11127
--- /dev/null
+++ b/scripts/build.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+REPO_URL=$1
+REPO_NAME=$2
+REPO_TAG=$3
+
+shift 3
+EXTRA_OPTIONS=$@
+
+REPO_BUILD=${REPO_NAME}-build
+REPO_INSTALLED=${REPO_NAME}-installed
+
+CURRENT_DIR=${PWD}
+
+export OPENBLAS_NUM_THREADS=1
+
+if [ -d ${REPO_NAME} ];
+then
+ cd ${REPO_NAME}
+ git reset --hard
+ git submodule init
+ git submodule update
+else
+ git clone ${REPO_URL} ${REPO_NAME}
+ cd ${REPO_NAME}
+ git submodule init
+ git submodule update
+fi
+
+git checkout ${REPO_TAG}
+
+cd ${CURRENT_DIR}
+
+cmake -B ${REPO_BUILD} ${REPO_NAME} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=${REPO_INSTALLED} ${EXTRA_OPTIONS}
+make install -C ${REPO_BUILD} -j 5
diff --git a/scripts/configure.sh b/scripts/configure.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f0089bdd0eea574d69f68b6749429dea68dfc9f3
--- /dev/null
+++ b/scripts/configure.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+CURRENT_DIR=$PWD
+PACKAGE_NAME=$1
+PACKAGE_INSTALLED_PATH=$1-installed
+PACKAGE_MODULE_PATH=modules/$1/cicd
+
+sed -i 's#{'${PACKAGE_NAME^^}'_PATH}#'${PWD}'/'${PACKAGE_NAME,,}'-installed#' modules/${PACKAGE_NAME,,}/cicd
diff --git a/scripts/parse_dp3_output.py b/scripts/parse_dp3_output.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc1af16867c3624a333d36f31dff3f87ba26334a
--- /dev/null
+++ b/scripts/parse_dp3_output.py
@@ -0,0 +1,75 @@
+import subprocess
+from argparse import ArgumentParser
+import re
+import yaml
+
+def parse_args():
+ parser = ArgumentParser(description="Run DP3 test process and parse output")
+ parser.add_argument("dp3_output")
+ parser.add_argument("parsed_timings")
+ return parser.parse_args()
+
+def trim_white_spaces(content):
+ return content.rstrip("\n")
+
+def read_file(dp3_stdout):
+ with open(dp3_stdout, "r") as f_stream:
+ return trim_white_spaces(f_stream.read())
+
+def is_description(string):
+ return "of" in string
+
+def parse_dp3_output(dp3_output):
+ # Parse Total
+ # es.
+ # Total DP3 time 116.12 real 2303.97 user 46.42 system
+ result = re.search(r"Total DP3 time\s+(\d+\.\d+)\s+real\s+(\d+\.\d+)\s+user\s+(\d+\.\d+)\s+system+", dp3_output)
+ real, user, system = map(float, result.groups())
+ timings = {"total": {"real": real, "user": user, "system": system}, "steps":[]}
+ detailed_timings_output = dp3_output[result.end(0):].lstrip("\n").rstrip("\n")
+ last_step = None
+
+ # Parse sub steps
+ # es.
+ # 2024-Nov-07 15:46:45.468694 84.6% ( 98 s) DDECal solve.
+ # 2024-Nov-07 15:46:45.468696 58.5% ( 57 s) of it spent in predict
+ # 2024-Nov-07 15:46:45.468697 38.7% ( 38 s) of it spent in estimating gains and computing residuals
+ # 2024-Nov-07 15:46:45.468699 0.0% ( 5 ms) of it spent in writing gain solutions to disk
+ # 2024-Nov-07 15:46:45.468701 Substeps taken:
+ for line in detailed_timings_output.split("\n"):
+ line_match = re.search("(\d+.\d+)%\s\(\s+(\d+)\s+(\w+)\) (.*)", line)
+ if not line_match:
+ continue
+
+ percentage, timing, timing_unit, name_or_description = line_match.groups()
+ if not is_description(name_or_description):
+ step_name = name_or_description
+ last_step = {"name": step_name,
+ "percentage": percentage,
+ "timing": timing,
+ "timing_unit": timing_unit,
+ "sub_steps": []}
+
+ timings["steps"].append(last_step)
+ else:
+ sub_step_name = name_or_description.split("in ")[-1]
+ last_step["sub_steps"].append(
+ {
+ "name": sub_step_name,
+ "percentage": percentage,
+ "timing": timing,
+ "timing_unit": timing_unit
+ }
+ )
+ return timings
+
+def main():
+ args = parse_args()
+
+ parsed_timings = parse_dp3_output(read_file(args.dp3_output))
+ with open(args.parsed_timings, "w") as timing_file:
+ timing_file.write(yaml.dump(parsed_timings))
+ print(yaml.dump(parsed_timings))
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/scripts/run_test.sh b/scripts/run_test.sh
new file mode 100755
index 0000000000000000000000000000000000000000..4eb482a0152e4db0f6d4d93fe21c59aa4b3cef32
--- /dev/null
+++ b/scripts/run_test.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+WORK_DIR=$1
+shift 1
+cmd=$@
+CURRENT_DIR=$PWD
+LOG_DIR=$CURRENT_DIR
+export OPENBLAS_NUM_THREADS=1
+
+cd $WORK_DIR
+($cmd | tee $LOG_DIR/stdout.log) |& tee $LOG_DIR/stderr.log
+cd $CURRENT_DIR
\ No newline at end of file
diff --git a/spack-env/spack.yaml b/spack-env/spack.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f8242627be26ccf81c9cf53bc7fe37baec150363
--- /dev/null
+++ b/spack-env/spack.yaml
@@ -0,0 +1,35 @@
+spack:
+ # add package specs to the `specs` list
+ specs:
+ - casacore+python+hdf5 build_type=RelWithDebInfo
+ - python
+ - hdf5+cxx
+ - aoflagger
+ - boost+program_options
+ - openblas threads=pthreads
+ - py-pyyaml
+ - py-matplotlib
+ - py-numpy
+ - py-seaborn
+ - py-pandas
+ - cuda@12.6.0
+ - gcc@13.1.0
+ view: true
+ concretizer:
+ unify: true
+ repos:
+ - /var/scratch/gitlab-runner/spack-repos/ska-sdp-spack
+ compilers:
+ - compiler:
+ spec: gcc@=13.1.0
+ paths:
+ cc: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/gcc
+ cxx: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/g++
+ f77: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/gfortran
+ fc: /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-13.1.0/gcc-13.1.0-hocevpdwueflvrbbsfu7zcjo7lz5hguo/bin/gfortran
+ flags: {}
+ operating_system: rocky8
+ target: x86_64
+ modules: []
+ environment: {}
+ extra_rpaths: []