Add initial version of calibrator workflow ci test

Former-commit-id: 89e3c3ae [formerly 96282492]
Former-commit-id: 1cb42061
Former-commit-id: 22134bf7

.gitlab-ci.yml

@@ -2,12 +2,17 @@ image: lofareosc/prefactor3-cwl:latest
 
 variables:
   TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
+  TEST_HBA_DATASET_NAME: "test_data_30subbands_12timeslots.tar.gz"
+  CALIBRATOR_HBA_RESULTS_NAME: "results_calibrator.tar.gz"
+  TARGET_HBA_RESULTS_NAME: "results_target.tar.gz"
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
 
 stages:
   - download_data
   - validate_cwl_scripts
   - test_steps
+  - test_calibrator
+  - test_target
 
 before_script:
   - mkdir workdir
@@ -20,7 +25,10 @@ download_data:
   stage: download_data
   script:
     - mkdir data && cd data
-    - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/$TEST_DATASET_NAME -O $TEST_DATASET_NAME && tar -xf $TEST_DATASET_NAME && rm -f $TEST_DATASET_NAME
+    - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/$TEST_DATASET_NAME -O $TEST_DATASET_NAME && tar -xf $TEST_DATASET_NAME && rm -f $TEST_DATA_NAME
+    - wget -q www.astron.nl/citt/ci_data/prefactor/$TEST_HBA_DATASET_NAME -O $TEST_HBA_DATASET_NAME && tar -xfz $TEST_HBA_DATASET_NAME && rm -f $TEST_HBA_DATASET_NAME
+    - wget -q www.astron.nl/citt/ci_data/prefactor/$CALIBRATOR_HBA_RESULTS_NAME -O $CALIBRATOR_HBA_RESULTS_NAME && tar -xfz $CALIBRATOR_HBA_RESULTS_NAME && rm -f $CALIBRATOR_HBA_RESULTS_NAME
+    - wget -q www.astron.nl/citt/ci_data/prefactor/$TARGET_HBA_RESULTS_NAME -O $TARGET_HBA_RESULTS_NAME && tar -xfz $TARGET_HBA_RESULTS_NAME && rm -f $TARGET_HBA_RESULTS_NAME
     - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/example.h5 -O example.h5
     - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/A-Team_lowres.sourcedb -O A-Team_lowres.sourcedb
   artifacts:
@@ -51,4 +59,12 @@ check_ateam_separation:
   allow_failure: true
   needs: ["download_data"]
   script:
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
\ No newline at end of file
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
+
+run_hba_calibrator:
+  stage: test_calibrator
+  allow_failure: true
+  needs: ["download_data"]
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+    - test_jobs/check_workflow_results.py results ../data/results_calibrator

test_jobs/HBA_calibrator.json

 {
-    "msin" : [
-        {"class": "Directory", "path": "../data/L570745_SB000_uv_first10.MS"},
-        {"class": "Directory", "path": "../data/L570745_SB001_uv_first10.MS"},
-        {"class": "Directory", "path": "../data/L570745_SB002_uv_first10.MS"}        
+    "msin": [
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB000_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB001_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB002_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB003_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB004_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB005_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB006_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB007_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB008_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB009_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB010_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB011_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB012_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB013_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB014_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB015_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB016_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB017_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB018_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB019_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB020_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB021_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB022_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB023_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB024_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB025_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB026_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB027_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB028_uv_12timeslots.MS"
+        },
+        {
+            "class": "Directory",
+            "path": "../data/L667520_SB029_uv_12timeslots.MS"
+        },
     ],
-    "raw_data": false,
-    "demix": false,
-} 
+    "A-Team_skymodel": {
+        "class": "File",
+        "path": "/usr/local/share/prefactor/skymodels/Ateam_LBA_CC.skymodel"
+    },
+    "calibrator_path_skymodel": {
+        "class": "Directory",
+        "path": "/usr/local/share/prefactor/skymodels"
+    }
+}