Merge branch 'RAP-218' into 'master'

Resolve RAP-218: fix the target CI pipeline

Closes RAP-218

See merge request eosc/prefactor3-cwl!36

Former-commit-id: 55031624

.gitlab-ci.yml

 image: lofareosc/prefactor3-cwl:latest
 
 variables:
-  TEST_HBA_DATASET_NAME: "test_data_30subbands_12timeslots.tar.gz"
+  TEST_HBA_DATASET_NAME: "test_data.tar.gz"
   CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
   TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -58,13 +58,18 @@ check_ateam_separation:
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
 
-run_hba_full:
+run_hba_calibrator:
   stage: test_full
   allow_failure: true
   needs: ["download_data"]
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
-    - rm -rf results
+
+run_hba_target:
+  stage: test_full
+  allow_failure: true
+  needs: ["download_data"]
+  script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_target.cwl test_jobs/HBA_target.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target

test_jobs/HBA_target.json

@@ -82,6 +82,7 @@
         }
     ],
     "avg_timeresolution_concat": 4,
+    "ncores": 12,
     "cal_solutions": {
         "class": "File",
         "path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_values/cal_solutions.h5"

test_jobs/check_workflow_results.py

@@ -24,11 +24,13 @@ def main(results_path, control_path):
     """
     # Check that all expected output files are present
     dcmp = dircmp(results_path, control_path)
+    if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0:
         if len(dcmp.left_only) > 0:
-        print('ERROR: The following files are present in the output but not in the control: {}'.format(dcmp.left_only))
-        sys.exit(1)
+            print('ERROR: The following files are present in the output but not in the '
+                  'control: {}'.format(dcmp.left_only))
         if len(dcmp.right_only) > 0:
-        print('ERROR: The following files are present in the control but not in the output: {}'.format(dcmp.right_only))
+            print('ERROR: The following files are present in the control but not in the '
+                  'output: {}'.format(dcmp.right_only))
         sys.exit(1)
 
     # Check that the calibration solutions match the control ones
@@ -43,7 +45,7 @@ def main(results_path, control_path):
             if not np.allclose(check_soltab.val, control_soltab.val, rtol=1e-03,
                                atol=1e-03, equal_nan=True):
                 print('ERROR: Val array of soltab {} of solset {} does not match '
-                      'the control'.format(check_soltab.anme, check_solset.name))
+                      'the control'.format(check_soltab.name, check_solset.name))
                 sys.exit(1)