add diagnostic imaging (RAP-170)

Former-commit-id: 846e9834
Former-commit-id: 361419c5

lofar-cwl/steps/average.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: average
+baseCommand:
+  - DPPP
+inputs:
+  - id: msin
+    type: Directory
+    inputBinding:
+      position: 0
+      prefix: msin=
+      separate: false
+    doc: Input Measurement Set
+  - id: msout_name
+    type: string
+    inputBinding:
+      prefix: msout=
+      separate: false
+      shellQuote: false
+      position: 0
+  - id: msin_datacolumn
+    type: string
+    inputBinding:
+      prefix: msin.datacolumn=
+      separate: false
+      shellQuote: false
+      position: 0
+    doc: Input data Column
+    default: DATA
+  - id: writefullresflag
+    type: boolean
+    inputBinding:
+      prefix: msout.writefullresflag=True
+      shellQuote: false
+      position: 0
+    default: false
+  - id: overwrite
+    type: boolean
+    inputBinding:
+      prefix: msout.overwrite=True
+      shellQuote: false
+      position: 0
+    default: false
+  - id: storagemanager
+    type: string
+    inputBinding:
+      prefix: msout.storagemanager=
+      separate: false
+      shellQuote: false
+      position: 0
+    default: ''
+  - id: databitrate
+    type: int?
+    inputBinding:
+      prefix: msout.storagemanager.databitrate=
+      separate: false
+      shellQuote: false
+      position: 0
+  - id: avg_timestep
+    type: int?
+    inputBinding:
+      prefix: avg.timestep=
+      separate: false
+      shellQuote: false
+      position: 0
+    default: 1
+  - id: avg_freqstep
+    type: int?
+    inputBinding:
+      prefix: avg.freqstep=
+      separate: false
+      shellQuote: false
+      position: 0
+    default: 1
+outputs:
+  - id: msout
+    doc: Output Measurement Set
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msout_name)
+  - id: logfile
+    type: 'File[]'
+    outputBinding:
+      glob: average*.log
+arguments:
+  - 'steps=[avg]'
+  - msin.orderms=False
+  - avg.type=average
+requirements:
+  - class: ShellCommandRequirement
+hints:
+  - class: DockerRequirement
+    dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+stdout: average.log
+stderr: average_err.log

lofar-cwl/steps/dpppconcat.cwl

@@ -88,7 +88,7 @@ inputs:
       shellQuote: false
       position: 0
   - id: missingdata
-    type: boolean
+    type: boolean?
     inputBinding:
       prefix: msin.missingdata=True
       separate: false

lofar-cwl/steps/utils.js

@@ -27,4 +27,10 @@ function concatenate_path(object_list){
     object_list.forEach(function (x, index, arr) {arr[index] = x.path;});
     return '[' + object_list.join(',') + ']'
     
+}
+
+function concatenate_path_wsclean(object_list){
+    object_list.forEach(function (x, index, arr) {arr[index] = x.path;});
+    return object_list.join(' ')
+    
 }
\ No newline at end of file

lofar-cwl/steps/wsclean.cwl

@@ -6,43 +6,196 @@ id: wsclean
 baseCommand:
   - wsclean
 inputs:
-  - default:
+  - id: msin
+    type: 'Directory[]'
+    inputBinding:
+      position: 2
+      shellQuote: false
+      itemSeparator: ','
+      valueFrom: $(concatenate_path_wsclean(self))
+  - id: image_size
+    default:
       - 1000
       - 1000
-    id: image_size
     type: 'int[]'
     inputBinding:
       position: 1
+      shellQuote: false
       prefix: '-size'
-  - default: 0.01
-    id: image_scale
-    type: float
+  - id: image_scale
+    default: '36asec'
+    type: string?
     inputBinding:
       position: 1
       prefix: '-scale'
-      valueFrom: $(self)deg
-  - default: image
-    id: image_name
+      shellQuote: false
+  - id: niter
+    default: 10000
+    type: int?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-niter'
+  - id: nmiter
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-nmiter'
+  - id: auto_threshold
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      shellQuote: false
+      position: 1
+      prefix: '-auto-threshold'
+  - id: multiscale
+    default: false
+    type: 
+      - boolean?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-multiscale'
+  - id: mgain
+    default: false
+    type: 
+      - boolean?
+      - float?
+    inputBinding:
+      shellQuote: false
+      position: 1
+      prefix: '-mgain'
+  - id: ncpu
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-j'
+  - id: parallel-gridding
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-parallel-gridding'
+  - id: parallel-deconvolution
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-parallel-deconvolution'
+  - id: parallel-reordering
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-parallel-reordering'
+  - id: channels-out
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-channels-out'
+  - id: join-channels
+    default: false
+    type: boolean?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-join-channels'
+  - id: taper-gaussian
+    default: false
+    type: 
+      - boolean?
+      - string?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-taper-gaussian'
+  - id: weighting
+    default: false
+    type: 
+      - boolean?
+      - string?
+    inputBinding:
+      position: 1
+      shellQuote: false
+      prefix: '-weight'
+  - id: maxuvw-m
+    default: false
+    type: 
+      - boolean?
+      - int?
+    inputBinding:
+      position: 1
+      prefix: '-maxuvw-m'
+      shellQuote: false
+  - id: tempdir
+    type: string?
+    default: '/tmp/'
+    inputBinding:
+      position: 1
+      prefix: '-temp-dir'
+      shellQuote: false
+  - id: model_update
+    default: true
+    type: boolean
+    inputBinding:
+      position: 1
+      prefix: '-no-update-model-required'
+      shellQuote: false
+  - id: image_name
+    default: 'image'
     type: string
     inputBinding:
       position: 1
       prefix: '-name'
-  - id: msin
-    type: Directory
-    inputBinding:
-      position: 2
+      shellQuote: false
 outputs:
   - id: dirty_image
     type: File
     outputBinding:
-      glob: $(inputs.image_name)-dirty.fits
+      glob: $(inputs.image_name)-MFS-dirty.fits
   - id: image
     type: File
     outputBinding:
-      glob: $(inputs.image_name)-image.fits
+      glob: $(inputs.image_name)-MFS-image.fits
+  - id: logfile
+    type: 'File[]'
+    outputBinding:
+      glob: 'wsclean*.log'
 label: WSClean
 hints:
   - class: DockerRequirement
     dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+  - class: InitialWorkDirRequirement
+    listing:
+      - $(inputs.msin)
 requirements:
+  - class: ShellCommandRequirement
   - class: InlineJavascriptRequirement
+    expressionLib:
+      - { $include: 'utils.js' }
+
+stdout: wsclean.log
+stderr: wsclean_err.log
\ No newline at end of file

steps/sort_times_into_freqGroups.cwl

@@ -59,9 +59,11 @@ arguments:
 
     filenames  = output['filenames']
     groupnames = output['groupnames']
+    total_bandwidth = output['total_bandwidth']
     
     cwl_output = {}
     cwl_output['groupnames'] = groupnames
+    cwl_output['total_bandwidth'] = total_bandwidth
 
     with open('./filenames.json', 'w') as fp:
         json.dump(filenames, fp)
@@ -80,6 +82,12 @@ outputs:
         loadContents: true
         glob: 'out.json'
         outputEval: $(JSON.parse(self[0].contents).groupnames)
+  - id: total_bandwidth
+    type: int
+    outputBinding:
+        loadContents: true
+        glob: 'out.json'
+        outputEval: $(JSON.parse(self[0].contents).total_bandwidth)
   - id: logfile
     type: File?
     outputBinding:

workflows/prefactor_target.cwl

@@ -428,6 +428,7 @@ steps:
       - id: msout
       - id: outh5parm
       - id: bad_antennas
+      - id: total_bandwidth
       - id: outh5parm_logfile
       - id: inspection
       - id: logfiles
@@ -456,6 +457,8 @@ steps:
         source: compression_bitrate
       - id: skymodel_source
         source: skymodel_source
+      - id: total_bandwidth
+        source: gsmcal/total_bandwidth
     out:
       - id: msout
       - id: solutions

workflows/prefactor_target/finalize.cwl

@@ -41,6 +41,10 @@ inputs:
     type: string
     'sbg:x': -1000
     'sbg:y': 400
+  - id: total_bandwidth
+    type: int
+    'sbg:x': -1000
+    'sbg:y': 500
 outputs:
   - id: msout
     outputSource:
@@ -59,6 +63,7 @@ outputs:
       - concat_logfiles_applygsm/output
       - concat_logfiles_solutions/output
       - concat_logfiles_structure/output
+      - concat_logfiles_wsclean/output
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 1300
@@ -66,6 +71,7 @@ outputs:
   - id: inspection
     outputSource:
       - structure_function/structure_plot
+      - wsclean/image
     type: 'File[]?'
     linkMerge: merge_flattened
     'sbg:x': 1300
@@ -75,12 +81,12 @@ steps:
     in:
       - id: h5parm
         source: input_h5parm
-      - id: solset
-        default: 'sol000'
-      - id: refsolset
-        default: 'target'
       - id: refh5parm
         source: insolutions
+      - id: solset
+        default: sol000
+      - id: refsolset
+        default: target
       - id: soltab_in
         source: gsmcal_step
         valueFrom: $(self+'000')
@@ -100,32 +106,32 @@ steps:
     label: add_missing_stations
     'sbg:x': -200
     'sbg:y': -300
-
   - id: apply_gsmcal
     in:
       - id: msin
         source: msin
-      - id: msin_datacolumn
-        default: DATA
       - id: msout_name
-        source: msin
-        valueFrom: $(self.nameroot+'_pre-cal.ms')
         linkMerge: merge_flattened
+        source:
+          - msin
+        valueFrom: $(self.nameroot+'_pre-cal.ms')
+      - id: msin_datacolumn
+        default: DATA
       - id: parmdb
         source: write_solutions/outh5parm
       - id: msout_datacolumn
         default: DATA
-      - id: storagemanager
-        default: "Dysco"
-      - id: databitrate
-        source: compression_bitrate
       - id: correction
         source:
           - skymodel_source
           - gsmcal_step
         valueFrom: $(self.join(''))
       - id: solset
-        default: 'target'
+        default: target
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        source: compression_bitrate
     out:
       - id: msout
       - id: logfile
@@ -137,6 +143,80 @@ steps:
     scatterMethod: flat_crossproduct
     'sbg:x': 400
     'sbg:y': 0
+  - id: average
+    in:
+      - id: msin
+        source: apply_gsmcal/msout
+      - id: msout_name
+        linkMerge: merge_flattened
+        source:
+          - apply_gsmcal/msout
+        valueFrom: $(self.nameroot+'_wsclean.ms')
+      - id: msin_datacolumn
+        default: DATA
+      - id: msout_datacolumn
+        default: DATA
+      - id: overwrite
+        default: false
+      - id: storagemanager
+        default: Dysco
+      - id: avg_timestep
+        default: 2
+      - id: avg_freqstep
+        default: 2
+    out:
+      - id: msout
+      - id: logfile
+    run: ../../lofar-cwl/steps/average.cwl
+    label: average
+    scatter:
+      - msin
+    'sbg:x': 600
+    'sbg:y': -200
+  - id: wsclean
+    in:
+      - id: msin
+        source:
+          - average/msout
+      - id: image_scale
+        default: 15asec
+      - id: niter
+        default: 1000000
+      - id: nmiter
+        default: 5
+      - id: multiscale
+        default: true
+      - id: mgain
+        default: 0.8
+      - id: ncpu
+        default: 4
+      - id: parallel-gridding
+        default: 1
+      - id: parallel-deconvolution
+        default: 1500
+      - id: parallel-reordering
+        default: 4
+      - id: channels-out
+        source: total_bandwidth
+        valueFrom: $(Math.round(self/1000000))
+      - id: join-channels
+        default: true
+      - id: taper-gaussian
+        default: 40asec
+      - id: weighting
+        default: briggs -0.5
+      - id: maxuvw-m
+        default: 20000
+      - id: image_name
+        source: get_targetname/targetname
+    out:
+      - id: dirty_image
+      - id: image
+      - id: logfile
+    run: ../../lofar-cwl/steps/wsclean.cwl
+    label: wsclean
+    'sbg:x': 800
+    'sbg:y': -200
   - id: merge_array_files
     in:
       - id: input
@@ -148,7 +228,6 @@ steps:
     label: merge_array_files
     'sbg:x': 600
     'sbg:y': 0
-
   - id: write_solutions
     in:
       - id: h5parmFile
@@ -179,10 +258,8 @@ steps:
     in:
       - id: h5parmFile
         source: write_solutions/outh5parm
-      - id: outsolset
-        default: 'target'
       - id: solsetName
-        default: 'target'
+        default: target
       - id: pointing
         source: get_targetname/targetname
     out:
@@ -197,7 +274,7 @@ steps:
       - id: h5parmFile
         source: write_solutions/outh5parm
       - id: solset
-        default: 'target'
+        default: target
       - id: soltab
         source:
           - skymodel_source
@@ -260,11 +337,26 @@ steps:
     label: concat_logfiles_structure
     'sbg:x': 950
     'sbg:y': 500
+  - id: concat_logfiles_wsclean
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source:
+          - wsclean/logfile
+      - id: file_prefix
+        default: wsclean
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_wsclean
+    'sbg:x': 1100
+    'sbg:y': 500
   - id: get_targetname
     in:
       - id: msin
         linkMerge: merge_flattened
-        source: msin
+        source:
+          - msin
     out:
       - id: targetname
       - id: logfile
@@ -273,7 +365,7 @@ steps:
     'sbg:x': 200
     'sbg:y': 200
 requirements:
-  - class: SubworkflowFeatureRequirement
-  - class: ScatterFeatureRequirement
   - class: StepInputExpressionRequirement
-  - class: InlineJavascriptRequirement
\ No newline at end of file
+  - class: InlineJavascriptRequirement
+  - class: ScatterFeatureRequirement
+  - class: MultipleInputFeatureRequirement

workflows/prefactor_target/gsmcal.cwl

@@ -120,6 +120,12 @@ outputs:
     linkMerge: merge_flattened
     'sbg:x': 1900
     'sbg:y': 1000
+  - id: total_bandwidth
+    outputSource:
+      - sort_times_into_freqGroups/total_bandwidth
+    type: int
+    'sbg:x': 1900
+    'sbg:y': 1200
 steps:
   - id: identifybadantennas
     in:
@@ -166,6 +172,7 @@ steps:
     out:
       - id: filenames
       - id: groupnames
+      - id: total_bandwidth
       - id: logfile
     run: ../../steps/sort_times_into_freqGroups.cwl
     label: sorttimesintofreqGroups