Resolve RAP-218: fix the target CI pipeline

Former-commit-id: 5347baae
Former-commit-id: 85059414

.gitlab-ci.yml

 image: lofareosc/prefactor3-cwl:latest
 
 variables:
-  TEST_HBA_DATASET_NAME: "test_data_30subbands_12timeslots.tar.gz"
+  TEST_HBA_DATASET_NAME: "test_data.tar.gz"
   CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
   TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
@@ -58,13 +58,18 @@ check_ateam_separation:
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
 
-run_hba_full:
+run_hba_calibrator:
   stage: test_full
   allow_failure: true
   needs: ["download_data"]
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
-    - rm -rf results
+
+run_hba_target:
+  stage: test_full
+  allow_failure: true
+  needs: ["download_data"]
+  script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_target.cwl test_jobs/HBA_target.json
     - test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target

lofar-cwl/steps/ms_concat.cwl

@@ -70,4 +70,4 @@ requirements:
   - class: InplaceUpdateRequirement
     inplaceUpdate: true
   - class: InlineJavascriptRequirement
-stdout: ms_concat.log
\ No newline at end of file
+stdout: ms_concat.log

steps/aoflag.cwl

@@ -133,7 +133,7 @@ requirements:
       - entry: $(inputs.concat_additional_ms)
         writable: true
       - entryname: script.sh
-        entry: | 
+        entry: |
           #!/bin/bash
           directories=\$(ls -d out*)
           echo \$directories
@@ -143,4 +143,4 @@ requirements:
           aoflagger \$1 \$2 \$3 \$4 \$concat
 
 stdout: aoflag.log
-stderr: aoflag_err.log
\ No newline at end of file
+stderr: aoflag_err.log

test_jobs/HBA_target.json

@@ -82,6 +82,7 @@
         }
     ],
     "avg_timeresolution_concat": 4,
+    "ncores": 12,
     "cal_solutions": {
         "class": "File",
         "path": "/builds/eosc/prefactor3-cwl/data/results_calibrator/cal_values/cal_solutions.h5"

test_jobs/check_workflow_results.py

@@ -24,11 +24,13 @@ def main(results_path, control_path):
     """
     # Check that all expected output files are present
     dcmp = dircmp(results_path, control_path)
-    if len(dcmp.left_only) > 0:
-        print('ERROR: The following files are present in the output but not in the control: {}'.format(dcmp.left_only))
-        sys.exit(1)
-    if len(dcmp.right_only) > 0:
-        print('ERROR: The following files are present in the control but not in the output: {}'.format(dcmp.right_only))
+    if len(dcmp.left_only) > 0 or len(dcmp.right_only) > 0:
+        if len(dcmp.left_only) > 0:
+            print('ERROR: The following files are present in the output but not in the '
+                  'control: {}'.format(dcmp.left_only))
+        if len(dcmp.right_only) > 0:
+            print('ERROR: The following files are present in the control but not in the '
+                  'output: {}'.format(dcmp.right_only))
         sys.exit(1)
 
     # Check that the calibration solutions match the control ones
@@ -43,7 +45,7 @@ def main(results_path, control_path):
             if not np.allclose(check_soltab.val, control_soltab.val, rtol=1e-03,
                                atol=1e-03, equal_nan=True):
                 print('ERROR: Val array of soltab {} of solset {} does not match '
-                      'the control'.format(check_soltab.anme, check_solset.name))
+                      'the control'.format(check_soltab.name, check_solset.name))
                 sys.exit(1)
 
 

workflows/prefactor_calibrator.cwl

@@ -204,7 +204,7 @@ outputs:
   - id: inspection
     linkMerge: merge_flattened
     outputSource:
-      - prep/check_Ateam_separation.png 
+      - prep/check_Ateam_separation.png
       - pa/inspection
       - fr/inspection
       - bp/inspection
@@ -531,4 +531,4 @@ steps:
     'sbg:y': 0
 requirements:
   - class: SubworkflowFeatureRequirement
-  - class: ScatterFeatureRequirement
\ No newline at end of file
+  - class: ScatterFeatureRequirement