First prefactor pipeline and integration fixes

.gitlab-ci.yml

@@ -25,7 +25,7 @@ validate_scripts:
   script:
     - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
 
-test_DPPP_step:
+test_steps:
   stage: test
   script:
     - cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
@@ -39,3 +39,7 @@ test_DPPP_step:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
+  allow_failure: true
+
+integration_test: 
+    - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json

prefactor_calibrator.cwl

+class: Workflow
+cwlVersion: v1.0
+id: prefactor_calibrator
+label: prefactor_calibrator
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+inputs:
+  - id: msin
+    type: 'Directory[]'
+    'sbg:x': -659.0313110351562
+    'sbg:y': -522.6085205078125
+  - id: raw_data
+    type: boolean
+    'sbg:x': -620
+    'sbg:y': -683
+  - id: demix
+    type: boolean
+    'sbg:x': -617
+    'sbg:y': -347
+outputs:
+  - id: demix_parmdb
+    outputSource:
+      - ndppp_prep_cal/demix_parmdb
+    type: ['Directory[]', Directory, 'null']
+    'sbg:x': -67.85011291503906
+    'sbg:y': -356.76287841796875
+  - id: output_ms
+    outputSource:
+      - aoflagger/output_ms
+    type: 'Directory[]'
+    'sbg:x': 219.203125
+    'sbg:y': -564.4959716796875
+steps:
+  - id: ndppp_prep_cal
+    in:
+      - id: baselines_to_flag
+        default: []
+      - id: elevation_to_flag
+        default: 0deg..20deg
+      - id: min_amplitude_to_flag
+        default: 1.e-30
+      - id: memoryperc
+        default: 20
+      - id: raw_data
+        source: raw_data
+      - id: demix
+        source: demix
+      - id: msin
+        linkMerge: merge_flattened
+        source:
+          - msin
+    out:
+      - id: msout
+      - id: demix_parmdb
+    run: subworkflow/ndppp_prep_cal.cwl
+    label: ndppp_prep_cal
+    scatter:
+      - msin
+    'sbg:x': -359
+    'sbg:y': -519
+  - id: aoflagger
+    in:
+      - id: msin
+        source:
+          - ndppp_prep_cal/msout
+    out:
+      - id: output_ms
+    run: steps/AOFlagger.cwl
+    label: AOFlagger
+    'sbg:x': -74
+    'sbg:y': -597
+requirements:
+  - class: SubworkflowFeatureRequirement
+  - class: ScatterFeatureRequirement

steps/DP3.DemixerStepGenerator.cwl

@@ -65,7 +65,7 @@ inputs:
       averaging, it has to fit integrally.
   - id: skymodel
     default: sky
-    type: [File, Directory]
+    type: [File, Directory, string]
     doc: The name of the SourceDB to use (i.e., the output of makesourcedb).
   - id: instrumentmodel
     default: instrument

steps/DirSelector.cwl

@@ -17,13 +17,21 @@ inputs:
       (e.g. 'instrumentmodel')
 outputs:
   - id: output
-    type: Directory
+    type: [Directory, 'Directory[]']
 
 expression: |
   ${
     var step_name = inputs.step_name
     var key_name = inputs.key_name
-    return {'output': inputs.inputs[step_name][key_name]}
+    
+    if(inputs.inputs.hasOwnProperty(step_name)){
+       var inputs_per_step = inputs.inputs[step_name]
+       if(inputs_per_step.hasOwnProperty(key_name)){
+         return {'output': inputs_per_step[key_name]}   
+       }
+    }
+    return {'output': []}   
+    
   }
 label: DirSelector
 

subworkflow/ndppp_prep_cal.cwl

@@ -176,6 +176,9 @@ steps:
         source: parsetgenerator/output_parset
       - id: msin
         source: msin
+      - id: msout_name
+        source: msin
+        valueFrom: $("out_"+self.basename)
       - id: secondary_files
         source:
           - parsetgenerator/input_files
@@ -198,13 +201,17 @@ steps:
       - id: inputs
         source: dppp/secondary_output_directories
       - id: step_name
+        default: demixstepgenerator
         valueFrom: demixstepgenerator
       - id: key_name
         valueFrom: instrumentmodel
     out:
       - id: output
     run: ../steps/DirSelector.cwl
+    label: DirSelector
     'sbg:x': 2838.8115234375
     'sbg:y': -151.54710388183594
 requirements:
   - class: SubworkflowFeatureRequirement
+  - class: StepInputExpressionRequirement
+  - class: InlineJavascriptRequirement

test_jobs/integration_test_prefactor_calibrator.json

+{
+    "msin" : [
+        {"class": "Directory", "path": "/data/L570745_SB000_uv_first10.MS"},
+        {"class": "Directory", "path": "/data/L570745_SB001_uv_first10.MS"},
+        {"class": "Directory", "path": "/data/L570745_SB002_uv_first10.MS"}        
+    ],
+    "raw_data": false,
+    "demix": false
+}