Merge branch 'RAP-165' into 'master'

Resolve RAP-165

Closes RAP-165

See merge request eosc/prefactor3-cwl!28

Former-commit-id: 1a3bfc3a
Former-commit-id: f677e39f

.gitlab-ci.yml

-image: lofareosc/prefactor-ci:latest
+image: lofareosc/prefactor3-cwl:latest
 
 variables:
   TEST_DATASET_NAME: "L570745_uv_first10.MS.tar.xz"
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
 
 stages:
-  - build_docker
-  - push_docker
+  - download_data
+  - test_prefactor
   - validate_cwl_scripts
   - test_steps
   - integration_test
@@ -18,45 +18,52 @@ before_script:
 after_script:
   - echo "All done"
 
-build_docker:
-  image: docker:latest
-  stage: build_docker
-  only:
-    - tags
+download_data:
+  stage: download_data
   script:
-    - apk add git subversion git-lfs bash
-    - cd Docker
-    - bash build_docker.sh
+    - mkdir data && cd data
+    - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/$TEST_DATASET_NAME -O $TEST_DATASET_NAME && tar -xf $TEST_DATASET_NAME && rm -f $TEST_DATASET_NAME
+    - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/example.h5 -O example.h5
+    - wget -q https://git.astron.nl/eosc/prefactor3-cwl/-/raw/master/test_data/A-Team_lowres.sourcedb -O A-Team_lowres.sourcedb
+  artifacts:
+    paths:
+    - data
 
-push_docker_tags:
-  image: docker:latest
-  allow_failure: true
-  stage: push_docker
-  only:
-    - tags
+test_prefactor:
+  stage: test_prefactor
+  needs: ["download_data"]
   script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker tag lofareosc/prefactor-ci:latest $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
-    - docker push $CI_REGISTRY/prefactor-ci:$CI_COMMIT_REF_NAME
+     - cd data
+     - ls -l
 
 validate_scripts:
   stage: validate_cwl_scripts
   script:
     - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
 
+blsmooth:
+  stage: test_steps
+  allow_failure: true
+  needs: ["download_data"]
+  script:
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
+
 find_skymodel_cal:
   stage: test_steps
   allow_failure: true
+  needs: ["download_data"]
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
 
-blsmooth:
+check_ateam_separation:
   stage: test_steps
   allow_failure: true
+  needs: ["download_data"]
   script:
-    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/check_ateam_separation.cwl test_jobs/check_ateam_separation.json
 
-pol_align:
+prefactor_calibrator:
   stage: integration_test
+  needs: ["download_data"]
   script:
-    - cwltool --no-container subworkflow/pol_align.cwl test_jobs/pol_align.json
+    - cwltool --no-container workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json

lofar-cwl/steps/Ateamclipper.cwl

@@ -4,8 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: check_ateam_separation
 baseCommand:
-  - python3
-  - /usr/local/bin/Ateamclipper.py
+  - Ateamclipper.py
 inputs:
   - id: msin
     type:

steps/add_missing_stations.cwl

@@ -5,8 +5,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: add_missing_stations
 baseCommand:
-  - python3
-  - /usr/local/bin/add_missing_stations.py
+  - add_missing_stations.py
 inputs:
   - default: output.h5
     id: h5parm

steps/aoflag.cwl

@@ -140,7 +140,6 @@ requirements:
           mkdir -pv workdir
           cp -r \$directories workdir/. && cd workdir
           concat=out.MS_[0-9*]
-          echo \$1 \$2 \$3 \$4 \$concat
           aoflagger \$1 \$2 \$3 \$4 \$concat
 
 stdout: aoflag.log

steps/blsmooth.cwl

@@ -3,8 +3,7 @@ cwlVersion: v1.0
 id: blsmooth
 label: BLsmooth
 baseCommand:
-  - python3
-  - /usr/local/bin/BLsmooth.py
+  - BLsmooth.py
 inputs:
   - id: msin
     type: Directory

steps/check_ateam_separation.cwl

@@ -2,8 +2,7 @@ class: CommandLineTool
 cwlVersion: v1.0
 id: check_ateam_separation
 baseCommand:
-  - python3
-  - /usr/local/bin/check_Ateam_separation.py
+  - check_Ateam_separation.py
 inputs:
   - id: ms
     type:

steps/createRMh5parm.cwl

@@ -3,8 +3,7 @@ cwlVersion: v1.0
 id: createRMh5parm
 label: createRMh5parm
 baseCommand:
-  - python3
-  - /usr/local/bin/createRMh5parm.py
+  - createRMh5parm.py
 inputs:
   - id: msin
     type:

steps/directory_listing.cwl

@@ -47,7 +47,7 @@ arguments:
     valueFrom: d
 hints:
   - class: DockerRequirement
-    dockerPull: 'lofareosc/prefactor-ci:master'
+    dockerPull: lofareosc/prefactor3-cwl
+stdout: out_file
 requirements:
   - class: InlineJavascriptRequirement
-stdout: out_file

steps/h5parm_pointingname.cwl

@@ -5,8 +5,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: h5parm_pointingname
 baseCommand:
-  - python3
-  - /usr/local/bin/h5parm_pointingname.py
+  - h5parm_pointingname.py
 inputs:
   - format: 'lofar:#H5Parm'
     id: h5parmFile

steps/plot_Ateamclipper.cwl

@@ -4,8 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: plot_Ateamclipper
 baseCommand:
-  - python3
-  - /usr/local/bin/plot_Ateamclipper.py
+  - plot_Ateamclipper.py
 inputs:
   - id: clipper_output
     type: File

steps/structure_function.cwl

@@ -5,8 +5,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: structure_function
 baseCommand:
-  - python3
-  - /usr/local/bin/getStructure_from_phases.py
+  - getStructure_from_phases.py
 inputs:
   - format: 'lofar:#H5Parm'
     id: h5parmFile

steps/transfer_solutions.cwl

@@ -4,8 +4,7 @@ $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 id: transfer_solutions
 baseCommand:
-  - python3
-  - /usr/local/bin/transfer_solutions.py
+  - transfer_solutions.py
 inputs:
   - default: output.h5
     id: h5parm

test_jobs/HBA_calibrator.json

+{
+    "msin" : [
+        {"class": "Directory", "path": "../data/L570745_SB000_uv_first10.MS"},
+        {"class": "Directory", "path": "../data/L570745_SB001_uv_first10.MS"},
+        {"class": "Directory", "path": "../data/L570745_SB002_uv_first10.MS"}        
+    ],
+    "raw_data": false,
+    "demix": false,
+} 

test_jobs/blsmooth.json

 {
     "msin": {
             "class": "Directory",
-            "path": "/data/L570745_SB003_uv_first10.MS"
+            "path": "../data/L570745_SB003_uv_first10.MS"
         },
     "do_smooth": true
 }

test_jobs/check_ateam_separation.json

+{
+  "ms": {
+    "class": "Directory",
+    "path": "../data/L570745_SB001_uv_first10.MS"
+  }
+}

test_jobs/find_skymodel_cal.json

 {
     "msin": {
         "class": "Directory",
-        "path": "/data/L570745_SB000_uv_first10.MS"
+        "path": "../data/L570745_SB000_uv_first10.MS"
     },
     "skymodels": {
         "class": "Directory",
-        "path": "/data/skymodels"
+        "path": "/usr/local/share/prefactor/skymodels"
     }
 }

test_jobs/integration_test_prefactor_calibrator.json

-{
-    "msin" : [
-        {"class": "Directory", "path": "/data/L570745_SB000_uv_first10.MS"},
-        {"class": "Directory", "path": "/data/L570745_SB001_uv_first10.MS"},
-        {"class": "Directory", "path": "/data/L570745_SB002_uv_first10.MS"}        
-    ],
-    "raw_data": false,
-    "demix": false,
-    "skymodels": {
-        "class": "Directory",
-        "path": "/data/skymodels"
-    },
-    "A-Team_sky_model": "/data/skymodels/Ateam_LBA_CC.skymodel"
-} 

test_jobs/pol_align.json

@@ -2,6 +2,6 @@
   "input_h5parm": {
     "class": "File",
     "format": "lofar:#H5Parm",
-    "path": "/data/example.h5"
+    "path": "../data/example.h5"
   }
 }