diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c18e4449e4dd4da294039d5af203f4fcbd19f97..2753580617913c7c130365b8fbd19ac7b5a3c940 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ image: lofareosc/prefactor3-cwl:latest variables: - TEST_HBA_DATASET_NAME: "test_data_30subbands_12timeslots.tar.gz" + TEST_HBA_DATASET_NAME: "test_data.tar.gz" CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz" TARGET_HBA_RESULTS_NAME: "results_target.tar.gz" PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -58,13 +58,18 @@ check_ateam_separation: script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json -run_hba_full: +run_hba_calibrator: stage: test_full allow_failure: true needs: ["download_data"] script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator - - rm -rf results + +run_hba_target: + stage: test_full + allow_failure: true + needs: ["download_data"] + script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_target.cwl test_jobs/HBA_target.json - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target diff --git a/lofar-cwl/steps/ms_concat.cwl b/lofar-cwl/steps/ms_concat.cwl index 5b39b048bce5218a1e69a2b2cc0f57bb08ccd10e..87c36a77627c1b8891fb8e0d3bbb0a5dd0e0a874 100644 --- a/lofar-cwl/steps/ms_concat.cwl +++ b/lofar-cwl/steps/ms_concat.cwl @@ -70,4 +70,4 @@ requirements: - class: InplaceUpdateRequirement inplaceUpdate: true - class: InlineJavascriptRequirement -stdout: ms_concat.log \ No newline at end of file +stdout: ms_concat.log diff --git a/steps/aoflag.cwl b/steps/aoflag.cwl index 04e259d17c0da229b39770ccdaf2af9928cf864f..c2b3ec1363572b56420c8b050fcfff87d4cd5001 100644 --- a/steps/aoflag.cwl +++ b/steps/aoflag.cwl @@ -133,7 +133,7 @@ requirements: - entry: $(inputs.concat_additional_ms) writable: true - entryname: script.sh - entry: | + entry: | #!/bin/bash directories=\$(ls -d out*) echo \$directories @@ -143,4 +143,4 @@ requirements: aoflagger \$1 \$2 \$3 \$4 \$concat stdout: aoflag.log -stderr: aoflag_err.log \ No newline at end of file +stderr: aoflag_err.log diff --git a/test_jobs/HBA_target.json b/test_jobs/HBA_target.json index 7be2dd11b014e9d77f3fc086bae818f39ce7032c..30777f0ce2576e9822a7fa7f79730840e08209ea 100644 --- a/test_jobs/HBA_target.json +++ b/test_jobs/HBA_target.json @@ -82,6 +82,7 @@ } ], "avg_timeresolution_concat": 4, + "ncores": 12, "cal_solutions": { "class": "File", "path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_values/cal_solutions.h5" diff --git a/test_jobs/check_workflow_results.py b/test_jobs/check_workflow_results.py index 721b61f49d486fd04bdc5e1431571830b4e3c841..44c8327d321eb2d5772e239e009be57f47f7a1fe 100755 --- a/test_jobs/check_workflow_results.py +++ b/test_jobs/check_workflow_results.py @@ -24,11 +24,13 @@ def main(results_path, control_path): """ # Check that all expected output files are present dcmp = dircmp(results_path, control_path) - if len(dcmp.left_only) > 0: - print('ERROR: The following files are present in the output but not in the control: {}'.format(dcmp.left_only)) - sys.exit(1) - if len(dcmp.right_only) > 0: - print('ERROR: The following files are present in the control but not in the output: {}'.format(dcmp.right_only)) + if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0: + if len(dcmp.left_only) > 0: + print('ERROR: The following files are present in the output but not in the ' + 'control: {}'.format(dcmp.left_only)) + if len(dcmp.right_only) > 0: + print('ERROR: The following files are present in the control but not in the ' + 'output: {}'.format(dcmp.right_only)) sys.exit(1) # Check that the calibration solutions match the control ones @@ -43,7 +45,7 @@ def main(results_path, control_path): if not np.allclose(check_soltab.val, control_soltab.val, rtol=1e-03, atol=1e-03, equal_nan=True): print('ERROR: Val array of soltab {} of solset {} does not match ' - 'the control'.format(check_soltab.anme, check_solset.name)) + 'the control'.format(check_soltab.name, check_solset.name)) sys.exit(1) diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl index 13e75238777ccdae31755c0100a81bc46bb6b40f..d6820b3e35927f8a0732202645786a1eb83866ed 100644 --- a/workflows/prefactor_calibrator.cwl +++ b/workflows/prefactor_calibrator.cwl @@ -204,7 +204,7 @@ outputs: - id: inspection linkMerge: merge_flattened outputSource: - - prep/check_Ateam_separation.png + - prep/check_Ateam_separation.png - pa/inspection - fr/inspection - bp/inspection @@ -531,4 +531,4 @@ steps: 'sbg:y': 0 requirements: - class: SubworkflowFeatureRequirement - - class: ScatterFeatureRequirement \ No newline at end of file + - class: ScatterFeatureRequirement