From cf8affc49dce4c1759b4dd5db494604fd9efdbfc Mon Sep 17 00:00:00 2001
From: David rafferty <rafferty@strw.leidenuniv.nl>
Date: Mon, 6 Sep 2021 07:57:39 +0000
Subject: [PATCH] Resolve RAP-218: fix the target CI pipeline
Former-commit-id: 5347baaec21bb86cbdfb0cca8990b89bcef588ac
---
.gitlab-ci.yml | 11 ++++++++---
lofar-cwl/steps/ms_concat.cwl | 2 +-
steps/aoflag.cwl | 4 ++--
test_jobs/HBA_target.json | 1 +
test_jobs/check_workflow_results.py | 14 ++++++++------
workflows/prefactor_calibrator.cwl | 4 ++--
6 files changed, 22 insertions(+), 14 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6c18e444..27535806 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: lofareosc/prefactor3-cwl:latest
variables:
- TEST_HBA_DATASET_NAME: "test_data_30subbands_12timeslots.tar.gz"
+ TEST_HBA_DATASET_NAME: "test_data.tar.gz"
CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -58,13 +58,18 @@ check_ateam_separation:
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
-run_hba_full:
+run_hba_calibrator:
stage: test_full
allow_failure: true
needs: ["download_data"]
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
- - rm -rf results
+
+run_hba_target:
+ stage: test_full
+ allow_failure: true
+ needs: ["download_data"]
+ script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_target.cwl test_jobs/HBA_target.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
diff --git a/lofar-cwl/steps/ms_concat.cwl b/lofar-cwl/steps/ms_concat.cwl
index 5b39b048..87c36a77 100644
--- a/lofar-cwl/steps/ms_concat.cwl
+++ b/lofar-cwl/steps/ms_concat.cwl
@@ -70,4 +70,4 @@ requirements:
- class: InplaceUpdateRequirement
inplaceUpdate: true
- class: InlineJavascriptRequirement
-stdout: ms_concat.log
\ No newline at end of file
+stdout: ms_concat.log
diff --git a/steps/aoflag.cwl b/steps/aoflag.cwl
index 04e259d1..c2b3ec13 100644
--- a/steps/aoflag.cwl
+++ b/steps/aoflag.cwl
@@ -133,7 +133,7 @@ requirements:
- entry: $(inputs.concat_additional_ms)
writable: true
- entryname: script.sh
- entry: |
+ entry: |
#!/bin/bash
directories=\$(ls -d out*)
echo \$directories
@@ -143,4 +143,4 @@ requirements:
aoflagger \$1 \$2 \$3 \$4 \$concat
stdout: aoflag.log
-stderr: aoflag_err.log
\ No newline at end of file
+stderr: aoflag_err.log
diff --git a/test_jobs/HBA_target.json b/test_jobs/HBA_target.json
index 7be2dd11..30777f0c 100644
--- a/test_jobs/HBA_target.json
+++ b/test_jobs/HBA_target.json
@@ -82,6 +82,7 @@
}
],
"avg_timeresolution_concat": 4,
+ "ncores": 12,
"cal_solutions": {
"class": "File",
"path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_values/cal_solutions.h5"
diff --git a/test_jobs/check_workflow_results.py b/test_jobs/check_workflow_results.py
index 721b61f4..44c8327d 100755
--- a/test_jobs/check_workflow_results.py
+++ b/test_jobs/check_workflow_results.py
@@ -24,11 +24,13 @@ def main(results_path, control_path):
"""
# Check that all expected output files are present
dcmp = dircmp(results_path, control_path)
- if len(dcmp.left_only) > 0:
- print('ERROR: The following files are present in the output but not in the control: {}'.format(dcmp.left_only))
- sys.exit(1)
- if len(dcmp.right_only) > 0:
- print('ERROR: The following files are present in the control but not in the output: {}'.format(dcmp.right_only))
+ if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0:
+ if len(dcmp.left_only) > 0:
+ print('ERROR: The following files are present in the output but not in the '
+ 'control: {}'.format(dcmp.left_only))
+ if len(dcmp.right_only) > 0:
+ print('ERROR: The following files are present in the control but not in the '
+ 'output: {}'.format(dcmp.right_only))
sys.exit(1)
# Check that the calibration solutions match the control ones
@@ -43,7 +45,7 @@ def main(results_path, control_path):
if not np.allclose(check_soltab.val, control_soltab.val, rtol=1e-03,
atol=1e-03, equal_nan=True):
print('ERROR: Val array of soltab {} of solset {} does not match '
- 'the control'.format(check_soltab.anme, check_solset.name))
+ 'the control'.format(check_soltab.name, check_solset.name))
sys.exit(1)
diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl
index 13e75238..d6820b3e 100644
--- a/workflows/prefactor_calibrator.cwl
+++ b/workflows/prefactor_calibrator.cwl
@@ -204,7 +204,7 @@ outputs:
- id: inspection
linkMerge: merge_flattened
outputSource:
- - prep/check_Ateam_separation.png
+ - prep/check_Ateam_separation.png
- pa/inspection
- fr/inspection
- bp/inspection
@@ -531,4 +531,4 @@ steps:
'sbg:y': 0
requirements:
- class: SubworkflowFeatureRequirement
- - class: ScatterFeatureRequirement
\ No newline at end of file
+ - class: ScatterFeatureRequirement
--
GitLab