workflows/linc_target/concat.cwl

@@ -21,6 +21,9 @@ inputs:
   - id: chunkduration
     type: float?
     default: 0.0
+  - id: execute
+    type: boolean?
+    default: true
 outputs:
   - id: flagged_fraction_dict
     outputSource:

workflows/linc_target/finalize.cwl

@@ -113,7 +113,7 @@ steps:
         default: target
       - id: soltab_in
         source: gsmcal_step
-        valueFrom: $(self+'000')
+        valueFrom: $(self == "phase" || self == "scalarphase" ? 'phase000':self+'000')
       - id: soltab_out
         source:
           - skymodel_source

workflows/linc_target/gsmcal.cwl

@@ -99,9 +99,7 @@ outputs:
   - id: outh5parm
     outputSource:
       - h5parm_collector/outh5parm
-      - h5parm_collector_hbaselfcal/outh5parm
     type: File
-    pickValue: the_only_non_null
   - id: outsolutions
     outputSource:
       - selfcal_target/outsolutions
@@ -128,8 +126,8 @@ outputs:
       - losoto_plot_Pd2/output_plots
       - losoto_plot_tec/output_plots
       - plot_unflagged/output_imag
-      - image_target/image
       - selfcal_target/inspection
+      - selfcal_target_hba/image_before
     type: File[]
     linkMerge: merge_flattened
     pickValue: all_non_null
@@ -516,170 +514,31 @@ steps:
       - msin
     when: $(!inputs.execute || inputs.selfcal_strategy == 'HBA')
 
-  # START image target field
-  - id: h5parm_collector_skysolve
-    in:
-      - id: h5parmFiles
-        source:
-          - calibrate_target/outh5parm
-        pickValue: all_non_null
-        linkMerge: merge_flattened
-      - id: squeeze
-        default: true
-      - id: verbose
-        default: true
-      - id: clobber
-        default: true
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-    out:
-      - id: outh5parm
-      - id: log
-    run: ../../steps/H5ParmCollector.cwl
-    label: h5parm_collector_skysolve
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-
-  - id: apply_skysolve
+  - id: selfcal_target_hba
     in:
       - id: max_dp3_threads
         source: max_dp3_threads
       - id: msin
         source: calibrate_target/msout
-      - id: msin_datacolumn
-        default: DATA
-      - id: parmdb
-        source: h5parm_collector_skysolve/outh5parm
-      - id: msout_datacolumn
-        default: CORRECTED_DATA
-      - id: storagemanager
-        default: Dysco
-      - id: databitrate
-        default: 0
-      - id: solset
-        default: sol000
-      - id: correction
-        default: phase000
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-    out:
-      - id: msout
-      - id: logfile
-    run: ../../steps/applycal.cwl
-    scatter:
-      - msin
-    label: apply_skysolve
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-
-  - id: image_target
-    in:
-      - id: msin
-        source: apply_skysolve/msout
-      - id: image_scale
-        default: 1.5asec
-      - id: auto_threshold
-        default: 0.5
-      - id: image_size
-        source: selfcal_hba_imsize
-      - id: niter
-        default: 285000
-      - id: nmiter
-        default: 12
-      - id: multiscale
-        default: true
-      - id: use-wgridder
-        default: true
-      - id: mgain
-        default: 0.8
-      - id: parallel-reordering
-        default: 4
-      - id: channels-out
-        default: 6
-      - id: join-channels
-        default: True
-      - id: fit-spectral-pol
-        default: 3
-      - id: weighting
-        default: briggs -1.5
-      - id: minuv-l
-        default: 80
-      - id: tempdir
+      - id: preapply_h5parm
+        source: calibrate_target/outh5parm
+      - id: wsclean_tmpdir
         source: wsclean_tmpdir
-      - id: image_name
-        default: image_before_selfcal
-      - id: no_model_update
-        default: false
-      # This can be turned on if we do a separate predict after, not sure if that would be faster.
-      # - id: baseline_averaging
-      #   default: 0.4908738521234052
-      - id: save_source_list
-        default: true
       - id: execute
         source: selfcal
       - id: selfcal_strategy
         source: selfcal_strategy
-    out:
-      - id: msout
-      - id: dirty_image
-      - id: image
-      - id: logfile
-      - id: sourcelist
-    run: ../../steps/wsclean.cwl
-    label: image_target
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-    # END image target field
-
-  # START HBA selfcal
-  - id: selfcal_target_hba
-    in:
-      - id: max_dp3_threads
-        source: max_dp3_threads
-      - id: msin
-        source:
-          - image_target/msout
-          # This is a little hacky, but CWL seems to try to create the scatter
-          # even if the step isn't run at all. The scatter is kept to be compatible with the other steps,
-          # despite there being only one MS.
-          - calibrate_target/msout
-        pickValue: first_non_null
-      # - id: skymodel
-      #   source: image_target/sourcelist
-      - id: do_smooth
-        default: false
-      - id: propagatesolutions
-        source: propagatesolutions
       - id: gsmcal_step
         source: gsmcal_step
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-      - id: model_column
-        default: ["MODEL_DATA"]
-      - id: nchan
-        default: 1
-      - id: smoothnessconstraint
-        default: 5000000
-      - id: smoothnessreffrequency
-        default: 120000000
-      - id: beammode
-        default: array_factor
-      - id: uvlambdamin
-        default: 200
+      - id: selfcal_hba_imsize
+        source: selfcal_hba_imsize
     out:
       - id: msout
-      - id: BLsmooth.log
-      - id: gaincal.log
       - id: outh5parm
-    run: ./calib_targ.cwl
-    label: calibrate_target
-    scatter:
-      - msin
+      - id: image_before
+    run: ./selfcal_targ_hba.cwl
+    label: selfcal_target_hba
     when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
-  # END HBA selfcal
 
   - id: selfcal_target
     in:
@@ -760,29 +619,6 @@ steps:
       - id: log
     run: ../../steps/H5ParmCollector.cwl
     label: H5parm_collector
-    when: $(!inputs.execute || (inputs.execute && inputs.selfcal_strategy != 'HBA'))
-
-  - id: h5parm_collector_hbaselfcal
-    in:
-      - id: h5parmFiles
-        source:
-          - selfcal_target_hba/outh5parm
-      - id: squeeze
-        default: true
-      - id: verbose
-        default: true
-      - id: clobber
-        default: true
-      - id: execute
-        source: selfcal
-      - id: selfcal_strategy
-        source: selfcal_strategy
-    out:
-      - id: outh5parm
-      - id: log
-    run: ../../steps/H5ParmCollector.cwl
-    label: h5parm_collector_hbaselfcal
-    when: $(inputs.execute && inputs.selfcal_strategy == 'HBA')
 
   - id: plot_unflagged
     in:
@@ -800,9 +636,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/tec000
       - id: axesInPlot
@@ -834,9 +667,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot
@@ -872,9 +702,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot
@@ -909,9 +736,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot
@@ -949,9 +773,6 @@ steps:
       - id: input_h5parm
         source:
           - h5parm_collector/outh5parm
-          - h5parm_collector_hbaselfcal/outh5parm
-        pickValue: the_only_non_null
-        linkMerge: merge_nested
       - id: soltab
         default: sol000/phase000
       - id: axesInPlot

workflows/linc_target/selfcal_targ_hba.cwl

+class: Workflow
+cwlVersion: v1.2
+id: selfcal_target
+label: selfcal_target
+inputs:
+  - id: max_dp3_threads
+    type: int?
+  - id: msin
+    type: Directory[]
+  - id: propagatesolutions
+    type: boolean?
+    default: true
+  - id: preapply_h5parm
+    type: File[]
+  - id: gsmcal_step
+    type: string?
+    default: 'phaseonly'
+  - id: selfcal_hba_imsize
+    type: int[]?
+    default: [20000,20000]
+  - id: wsclean_tmpdir
+    type: string
+  - id: execute
+    type: boolean?
+    default: true
+  - id: selfcal_strategy
+    type: string?
+    default: 'HBA'
+outputs:
+  - id: msout
+    outputSource:
+      - selfcal_target_hba/msout
+    type: Directory[]
+  - id: outh5parm
+    outputSource:
+      - selfcal_target_hba/outh5parm
+    type: File[]
+  - id: image_before
+    outputSource:
+      - image_target/image
+    type: File
+steps:
+  # START image target field
+  - id: h5parm_collector_skysolve
+    in:
+      - id: h5parmFiles
+        source: preapply_h5parm
+        linkMerge: merge_flattened
+      - id: squeeze
+        default: true
+      - id: verbose
+        default: true
+      - id: clobber
+        default: true
+    out:
+      - id: outh5parm
+      - id: log
+    run: ../../steps/H5ParmCollector.cwl
+    label: h5parm_collector_skysolve
+
+  - id: apply_skysolve
+    in:
+      - id: max_dp3_threads
+        source: max_dp3_threads
+      - id: msin
+        source: msin
+      - id: msin_datacolumn
+        default: DATA
+      - id: parmdb
+        source: h5parm_collector_skysolve/outh5parm
+      - id: msout_datacolumn
+        default: CORRECTED_DATA
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        default: 0
+      - id: solset
+        default: sol000
+      - id: correction
+        default: phase000
+    out:
+      - id: msout
+      - id: logfile
+    run: ../../steps/applycal.cwl
+    scatter:
+      - msin
+    label: apply_skysolve
+
+  - id: image_target
+    in:
+      - id: msin
+        source: apply_skysolve/msout
+      - id: image_scale
+        default: 1.5asec
+      - id: auto_threshold
+        default: 0.5
+      - id: image_size
+        source: selfcal_hba_imsize
+      - id: niter
+        default: 285000
+      - id: nmiter
+        default: 12
+      - id: multiscale
+        default: true
+      - id: use-wgridder
+        default: true
+      - id: mgain
+        default: 0.8
+      - id: parallel-reordering
+        default: 4
+      - id: channels-out
+        default: 6
+      - id: join-channels
+        default: True
+      - id: fit-spectral-pol
+        default: 3
+      - id: weighting
+        default: briggs -1.5
+      - id: minuv-l
+        default: 80
+      - id: tempdir
+        source: wsclean_tmpdir
+      - id: image_name
+        default: image_before_selfcal
+      - id: no_model_update
+        default: false
+      # This can be turned on if we do a separate predict after, not sure if that would be faster.
+      # - id: baseline_averaging
+      #   default: 0.4908738521234052
+      - id: save_source_list
+        default: true
+    out:
+      - id: msout
+      - id: dirty_image
+      - id: image
+      - id: logfile
+      - id: sourcelist
+    run: ../../steps/wsclean.cwl
+    label: image_target
+  # END image target field
+
+  # START HBA selfcal
+  - id: selfcal_target_hba
+    in:
+      - id: max_dp3_threads
+        source: max_dp3_threads
+      - id: msin
+        source: image_target/msout
+      # - id: skymodel
+      #   source: image_target/sourcelist
+      - id: do_smooth
+        default: false
+      - id: propagatesolutions
+        source: propagatesolutions
+      - id: gsmcal_step
+        source: gsmcal_step
+      - id: model_column
+        default: ["MODEL_DATA"]
+      - id: nchan
+        default: 1
+      - id: smoothnessconstraint
+        default: 5000000
+      - id: smoothnessreffrequency
+        default: 120000000
+      - id: beammode
+        default: array_factor
+      - id: uvlambdamin
+        default: 200
+    out:
+      - id: msout
+      - id: BLsmooth.log
+      - id: gaincal.log
+      - id: outh5parm
+    run: ./calib_targ.cwl
+    label: calibrate_target
+    scatter:
+      - msin
+  # END HBA selfcal
+requirements:
+  - class: ScatterFeatureRequirement
+  - class: SubworkflowFeatureRequirement
+  - class: InplaceUpdateRequirement
+    inplaceUpdate: true