diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb0910c78b936e21e90981eb760c3c52335d8e4f..68930d805853691ae4b259066143ed811f3fdc4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,13 +10,13 @@ stages: - run-performance-tests - collect-results -.load_build: +.load-build: before_script: - module load spack/13.1.0 - source /var/scratch/gitlab-runner/spack/share/spack/setup-env.sh - spack env activate pipeline-metrics-collector -.load_run: +.load-run: before_script: - module load spack/13.1.0 - source /var/scratch/gitlab-runner/spack/share/spack/setup-env.sh @@ -25,7 +25,7 @@ stages: - "echo 'measures.directory: ${PWD}/measures' > casarc" build-everybeam: - extends: [.load_build] + extends: [.load-build] stage: build tags: - das6-gpu @@ -36,7 +36,7 @@ build-everybeam: - everybeam-installed build-dp3: - extends: [.load_build] + extends: [.load-build] needs: - build-everybeam stage: build @@ -52,7 +52,7 @@ build-dp3: - dp3-installed download-measures: - extends: [.load_build] + extends: [.load-build] tags: - das6-gpu stage: configure @@ -65,8 +65,8 @@ download-measures: - measures - casarc -dp3_performance: - extends: [.load_run] +dp3-performance: + extends: [.load-run] needs: - build-everybeam - build-dp3 diff --git a/integration_tests/ddecal.parset b/integration_tests/ddecal.parset new file mode 100644 index 0000000000000000000000000000000000000000..f75a884f6ecbeb83283eba18830f61c730537972 --- /dev/null +++ b/integration_tests/ddecal.parset @@ -0,0 +1,35 @@ +avg.type=bdaaverager +avg.minchannels=1 +avg.frequencybase=0.0 +avg.maxinterval=75 +avg.timebase=20000.0 +steps=[solve] +solve.applycal.fastphase.correction=phase000 +solve.applycal.parmdb=fast_phases.h5parm +solve.applycal.steps=[fastphase] +solve.beammode=array_factor +solve.datause=dual +solve.directions=[Patch_1014,Patch_1049,Patch_1051,Patch_1119,Patch_1122,Patch_1208,Patch_1372,Patch_1501,Patch_160,Patch_1669,Patch_18,Patch_2180,Patch_2357,Patch_2360,Patch_2409,Patch_2414,Patch_293,Patch_334,Patch_339,Patch_401,Patch_418,Patch_430,Patch_444,Patch_448,Patch_453,Patch_468,Patch_476,Patch_493,Patch_511,Patch_590,Patch_625,Patch_644,Patch_646,Patch_664,Patch_69,Patch_760,Patch_762,Patch_788,Patch_790,Patch_82,Patch_895,Patch_904,Patch_917,Patch_922,Patch_930,Patch_937,Patch_941,Patch_942,Patch_971,Patch_992] +solve.h5parm=slow_gain_separate_152.h5parm +solve.llssolver=qr +solve.maxiter=150 +solve.mode=diagonal +solve.nchan=15 +solve.onebeamperpatch=False +solve.parallelbaselines=False +solve.propagatesolutions=True +solve.sagecalpredict=False +solve.smoothnessconstraint=3000000.0 +solve.solint=75 +solve.solutions_per_direction=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] +solve.solveralgorithm=directioniterative +solve.solverlbfgs.dof=200.0 +solve.solverlbfgs.iter=4 +solve.solverlbfgs.minibatches=1 +solve.sourcedb=/var/scratch/mancini/wp5/andre_test/Testset/calibration_skymodel.txt +solve.stepsigma=0.1 +solve.stepsize=0.02 +solve.tolerance=0.005 +solve.type=ddecal +solve.usebeammodel=True +solve.uvlambdamin=150 \ No newline at end of file diff --git a/integration_tests/dp3_ddecal.sh b/integration_tests/dp3_ddecal.sh index 9e530d104318e653cdd51071b4852ac20e28f8bf..ac2b0dbb26a1d0fbb28c0a01b041ccb1bc3d4661 100755 --- a/integration_tests/dp3_ddecal.sh +++ b/integration_tests/dp3_ddecal.sh @@ -1,9 +1,12 @@ #!/bin/bash set -e -cp /var/scratch/mancini/wp5/andre_test/Testset/*.h5parm . + +DATA_DIR=/var/scratch/mancini/wp5/andre_test/Testset + +cp ${DATA_DIR}/*.h5parm . DP3 msin.datacolumn=DATA\ - msin=/var/scratch/mancini/wp5/andre_test/Testset/epoch_5026988680.9055605_concatenated.ms \ + msin=${DATA_DIR}/epoch_5026988680.9055605_concatenated.ms \ msin.nchan=50 \ msin.ntimes=15 \ msin.startchan=100 \ @@ -11,38 +14,5 @@ DP3 msin.datacolumn=DATA\ numthreads=40 \ verbosity=verbose \ time_logging=true \ - msout= avg.type=bdaaverager \ - avg.minchannels=1 \ - avg.frequencybase=0.0 \ - avg.maxinterval=75 \ - avg.timebase=20000.0 \ - steps=[solve] \ - solve.applycal.fastphase.correction=phase000 \ - solve.applycal.parmdb=fast_phases.h5parm \ - solve.applycal.steps=[fastphase] \ - solve.beammode=array_factor \ - solve.datause=dual \ - solve.directions=[Patch_1014,Patch_1049,Patch_1051,Patch_1119,Patch_1122,Patch_1208,Patch_1372,Patch_1501,Patch_160,Patch_1669,Patch_18,Patch_2180,Patch_2357,Patch_2360,Patch_2409,Patch_2414,Patch_293,Patch_334,Patch_339,Patch_401,Patch_418,Patch_430,Patch_444,Patch_448,Patch_453,Patch_468,Patch_476,Patch_493,Patch_511,Patch_590,Patch_625,Patch_644,Patch_646,Patch_664,Patch_69,Patch_760,Patch_762,Patch_788,Patch_790,Patch_82,Patch_895,Patch_904,Patch_917,Patch_922,Patch_930,Patch_937,Patch_941,Patch_942,Patch_971,Patch_992] \ - solve.h5parm=slow_gain_separate_152.h5parm \ - solve.llssolver=qr \ - solve.maxiter=150 \ - solve.mode=diagonal \ - solve.nchan=15 \ - solve.onebeamperpatch=False \ - solve.parallelbaselines=False \ - solve.propagatesolutions=True \ - solve.sagecalpredict=False \ - solve.smoothnessconstraint=3000000.0 \ - solve.solint=75 \ - solve.solutions_per_direction=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1] \ - solve.solveralgorithm=directioniterative \ - solve.solverlbfgs.dof=200.0 \ - solve.solverlbfgs.iter=4 \ - solve.solverlbfgs.minibatches=1 \ - solve.sourcedb=/var/scratch/mancini/wp5/andre_test/Testset/calibration_skymodel.txt \ - solve.stepsigma=0.1 \ - solve.stepsize=0.02 \ - solve.tolerance=0.005 \ - solve.type=ddecal \ - solve.usebeammodel=True \ - solve.uvlambdamin=150 \ \ No newline at end of file + msout=. + ddecal.parset \ No newline at end of file diff --git a/modules/dp3/cicd b/modules/dp3/cicd index 13ded09f35dba551f8a398f932d8f373bbe6303b..4f79ec77e0ac291383bd886ee6713594297d46e6 100644 --- a/modules/dp3/cicd +++ b/modules/dp3/cicd @@ -8,5 +8,5 @@ prepend-path LD_LIBRARY_PATH $root/lib prepend-path LD_LIBRARY_PATH $root/lib64 prepend-path CPATH $root/include -prepend-path CMAKE_PREFIX_PATH $root/. +prepend-path CMAKE_PREFIX_PATH $root/ setenv DP3_ROOT $root diff --git a/modules/everybeam/cicd b/modules/everybeam/cicd index ba7f139cbded30330a36250f99f6777dd551d83c..5796baa5bf582fdef34420a6cebef3e085e867b8 100644 --- a/modules/everybeam/cicd +++ b/modules/everybeam/cicd @@ -8,5 +8,5 @@ prepend-path LD_LIBRARY_PATH $root/lib prepend-path LD_LIBRARY_PATH $root/lib64 prepend-path CPATH $root/include -prepend-path CMAKE_PREFIX_PATH $root/. +prepend-path CMAKE_PREFIX_PATH $root/ setenv EveryBeam_ROOT $root diff --git a/scripts/build.sh b/scripts/build.sh index da23fe18834bc55cb4757da09a87e75ea814b9f4..99060fb8bd10c68ac5bb88d38a0e238c14c34b2c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -28,5 +28,5 @@ git checkout ${REPO_TAG} cd ${CURRENT_DIR} -cmake -B ${REPO_BUILD} ${REPO_NAME} -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=${REPO_INSTALLED} ${EXTRA_OPTIONS} +cmake -B ${REPO_BUILD} ${REPO_NAME} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=${REPO_INSTALLED} ${EXTRA_OPTIONS} make install -C ${REPO_BUILD} -j 5 diff --git a/scripts/parse_dp3_output.py b/scripts/parse_dp3_output.py index 48b8f5cbc86b23a08058eca8dc1f6fdf44f729be..09d4af2ca3acff49ad70b33fa2828b2b96cbe3bf 100644 --- a/scripts/parse_dp3_output.py +++ b/scripts/parse_dp3_output.py @@ -21,6 +21,7 @@ def is_description(string): def parse_dp3_output(dp3_output): # Parse Total + # es. # Total DP3 time 116.12 real 2303.97 user 46.42 system result = re.search(r"Total DP3 time\s+(\d+\.\d+)\s+real\s+(\d+\.\d+)\s+user\s+(\d+\.\d+)\s+system+", dp3_output) real, user, system = map(float, result.groups()) @@ -28,6 +29,14 @@ def parse_dp3_output(dp3_output): print(result.end(0)) detailed_timings_output = dp3_output[result.end(0):].lstrip("\n").rstrip("\n") last_step = None + + # Parse sub steps + # es. + # 2024-Nov-07 15:46:45.468694 84.6% ( 98 s) DDECal solve. + # 2024-Nov-07 15:46:45.468696 58.5% ( 57 s) of it spent in predict + # 2024-Nov-07 15:46:45.468697 38.7% ( 38 s) of it spent in estimating gains and computing residuals + # 2024-Nov-07 15:46:45.468699 0.0% ( 5 ms) of it spent in writing gain solutions to disk + # 2024-Nov-07 15:46:45.468701 Substeps taken: for line in detailed_timings_output.split("\n"): line_match = re.search("(\d+.\d+)%\s\(\s+(\d+)\s+(\w+)\) (.*)", line) print(line, line_match) diff --git a/scripts/run_test.sh b/scripts/run_test.sh index 90540792b6413a2b1cb6256e8505ec1a47cabf5f..4eb482a0152e4db0f6d4d93fe21c59aa4b3cef32 100755 --- a/scripts/run_test.sh +++ b/scripts/run_test.sh @@ -9,5 +9,5 @@ LOG_DIR=$CURRENT_DIR export OPENBLAS_NUM_THREADS=1 cd $WORK_DIR -($cmd | tee $LOG_DIR/stdout.log) 3>&1 1>&2 2>&3 | tee $LOG_DIR/stderr.log +($cmd | tee $LOG_DIR/stdout.log) |& tee $LOG_DIR/stderr.log cd $CURRENT_DIR \ No newline at end of file