diff --git a/lofar-cwl/steps/average.cwl b/lofar-cwl/steps/average.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..ea43e98bf2d2a7ff97d055000f92c73f8e33fa43
--- /dev/null
+++ b/lofar-cwl/steps/average.cwl
@@ -0,0 +1,97 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: average
+baseCommand:
+ - DPPP
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
+ - id: msout_name
+ type: string
+ inputBinding:
+ prefix: msout=
+ separate: false
+ shellQuote: false
+ position: 0
+ - id: msin_datacolumn
+ type: string
+ inputBinding:
+ prefix: msin.datacolumn=
+ separate: false
+ shellQuote: false
+ position: 0
+ doc: Input data Column
+ default: DATA
+ - id: writefullresflag
+ type: boolean
+ inputBinding:
+ prefix: msout.writefullresflag=True
+ shellQuote: false
+ position: 0
+ default: false
+ - id: overwrite
+ type: boolean
+ inputBinding:
+ prefix: msout.overwrite=True
+ shellQuote: false
+ position: 0
+ default: false
+ - id: storagemanager
+ type: string
+ inputBinding:
+ prefix: msout.storagemanager=
+ separate: false
+ shellQuote: false
+ position: 0
+ default: ''
+ - id: databitrate
+ type: int?
+ inputBinding:
+ prefix: msout.storagemanager.databitrate=
+ separate: false
+ shellQuote: false
+ position: 0
+ - id: avg_timestep
+ type: int?
+ inputBinding:
+ prefix: avg.timestep=
+ separate: false
+ shellQuote: false
+ position: 0
+ default: 1
+ - id: avg_freqstep
+ type: int?
+ inputBinding:
+ prefix: avg.freqstep=
+ separate: false
+ shellQuote: false
+ position: 0
+ default: 1
+outputs:
+ - id: msout
+ doc: Output Measurement Set
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msout_name)
+ - id: logfile
+ type: 'File[]'
+ outputBinding:
+ glob: average*.log
+arguments:
+ - 'steps=[avg]'
+ - msin.orderms=False
+ - avg.type=average
+requirements:
+ - class: ShellCommandRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+stdout: average.log
+stderr: average_err.log
diff --git a/lofar-cwl/steps/dpppconcat.cwl b/lofar-cwl/steps/dpppconcat.cwl
index 0fca5fbf8d277303827072dc9f920174deb98f48..40d3f3e422ffd35b3ab8ec1a0dbd24c7ac006b02 100644
--- a/lofar-cwl/steps/dpppconcat.cwl
+++ b/lofar-cwl/steps/dpppconcat.cwl
@@ -88,7 +88,7 @@ inputs:
shellQuote: false
position: 0
- id: missingdata
- type: boolean
+ type: boolean?
inputBinding:
prefix: msin.missingdata=True
separate: false
diff --git a/lofar-cwl/steps/utils.js b/lofar-cwl/steps/utils.js
index 711c00ab03dfc1af230f3e891dde819da5775431..b84024d7fe1004eac8db199fa6962b8eac152841 100644
--- a/lofar-cwl/steps/utils.js
+++ b/lofar-cwl/steps/utils.js
@@ -27,4 +27,10 @@ function concatenate_path(object_list){
object_list.forEach(function (x, index, arr) {arr[index] = x.path;});
return '[' + object_list.join(',') + ']'
+}
+
+function concatenate_path_wsclean(object_list){
+ object_list.forEach(function (x, index, arr) {arr[index] = x.path;});
+ return object_list.join(' ')
+
}
\ No newline at end of file
diff --git a/lofar-cwl/steps/wsclean.cwl b/lofar-cwl/steps/wsclean.cwl
index 24af8bd95512ffd5e615a6381c187f61216b3c1a..96b56e937468bbd44f275656a4ab9510eb8ba828 100644
--- a/lofar-cwl/steps/wsclean.cwl
+++ b/lofar-cwl/steps/wsclean.cwl
@@ -6,43 +6,196 @@ id: wsclean
baseCommand:
- wsclean
inputs:
- - default:
+ - id: msin
+ type: 'Directory[]'
+ inputBinding:
+ position: 2
+ shellQuote: false
+ itemSeparator: ','
+ valueFrom: $(concatenate_path_wsclean(self))
+ - id: image_size
+ default:
- 1000
- 1000
- id: image_size
type: 'int[]'
inputBinding:
position: 1
+ shellQuote: false
prefix: '-size'
- - default: 0.01
- id: image_scale
- type: float
+ - id: image_scale
+ default: '36asec'
+ type: string?
inputBinding:
position: 1
prefix: '-scale'
- valueFrom: $(self)deg
- - default: image
- id: image_name
+ shellQuote: false
+ - id: niter
+ default: 10000
+ type: int?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-niter'
+ - id: nmiter
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-nmiter'
+ - id: auto_threshold
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ shellQuote: false
+ position: 1
+ prefix: '-auto-threshold'
+ - id: multiscale
+ default: false
+ type:
+ - boolean?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-multiscale'
+ - id: mgain
+ default: false
+ type:
+ - boolean?
+ - float?
+ inputBinding:
+ shellQuote: false
+ position: 1
+ prefix: '-mgain'
+ - id: ncpu
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-j'
+ - id: parallel-gridding
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-parallel-gridding'
+ - id: parallel-deconvolution
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-parallel-deconvolution'
+ - id: parallel-reordering
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-parallel-reordering'
+ - id: channels-out
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-channels-out'
+ - id: join-channels
+ default: false
+ type: boolean?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-join-channels'
+ - id: taper-gaussian
+ default: false
+ type:
+ - boolean?
+ - string?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-taper-gaussian'
+ - id: weighting
+ default: false
+ type:
+ - boolean?
+ - string?
+ inputBinding:
+ position: 1
+ shellQuote: false
+ prefix: '-weight'
+ - id: maxuvw-m
+ default: false
+ type:
+ - boolean?
+ - int?
+ inputBinding:
+ position: 1
+ prefix: '-maxuvw-m'
+ shellQuote: false
+ - id: tempdir
+ type: string?
+ default: '/tmp/'
+ inputBinding:
+ position: 1
+ prefix: '-temp-dir'
+ shellQuote: false
+ - id: model_update
+ default: true
+ type: boolean
+ inputBinding:
+ position: 1
+ prefix: '-no-update-model-required'
+ shellQuote: false
+ - id: image_name
+ default: 'image'
type: string
inputBinding:
position: 1
prefix: '-name'
- - id: msin
- type: Directory
- inputBinding:
- position: 2
+ shellQuote: false
outputs:
- id: dirty_image
type: File
outputBinding:
- glob: $(inputs.image_name)-dirty.fits
+ glob: $(inputs.image_name)-MFS-dirty.fits
- id: image
type: File
outputBinding:
- glob: $(inputs.image_name)-image.fits
+ glob: $(inputs.image_name)-MFS-image.fits
+ - id: logfile
+ type: 'File[]'
+ outputBinding:
+ glob: 'wsclean*.log'
label: WSClean
hints:
- class: DockerRequirement
dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+ - class: InitialWorkDirRequirement
+ listing:
+ - $(inputs.msin)
requirements:
+ - class: ShellCommandRequirement
- class: InlineJavascriptRequirement
+ expressionLib:
+ - { $include: 'utils.js' }
+
+stdout: wsclean.log
+stderr: wsclean_err.log
\ No newline at end of file
diff --git a/steps/sort_times_into_freqGroups.cwl b/steps/sort_times_into_freqGroups.cwl
index c2dffe021efe70adbb703250dba0c80a6130e48f..3b6ebf150459f1502d086ab367be30e8c97d2090 100644
--- a/steps/sort_times_into_freqGroups.cwl
+++ b/steps/sort_times_into_freqGroups.cwl
@@ -59,9 +59,11 @@ arguments:
filenames = output['filenames']
groupnames = output['groupnames']
+ total_bandwidth = output['total_bandwidth']
cwl_output = {}
cwl_output['groupnames'] = groupnames
+ cwl_output['total_bandwidth'] = total_bandwidth
with open('./filenames.json', 'w') as fp:
json.dump(filenames, fp)
@@ -80,6 +82,12 @@ outputs:
loadContents: true
glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).groupnames)
+ - id: total_bandwidth
+ type: int
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents).total_bandwidth)
- id: logfile
type: File?
outputBinding:
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index b9850932c79e56a1f187a968b07166f018ce112a..21bc03c6967833b2a63e6df4cf392112979aa15d 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -428,6 +428,7 @@ steps:
- id: msout
- id: outh5parm
- id: bad_antennas
+ - id: total_bandwidth
- id: outh5parm_logfile
- id: inspection
- id: logfiles
@@ -456,6 +457,8 @@ steps:
source: compression_bitrate
- id: skymodel_source
source: skymodel_source
+ - id: total_bandwidth
+ source: gsmcal/total_bandwidth
out:
- id: msout
- id: solutions
diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl
index 93346d0285251d714b0db13f4035d71f1edcb1e0..8543d90c5056dcd1b70cf07c58e95181295861c0 100644
--- a/workflows/prefactor_target/finalize.cwl
+++ b/workflows/prefactor_target/finalize.cwl
@@ -41,6 +41,10 @@ inputs:
type: string
'sbg:x': -1000
'sbg:y': 400
+ - id: total_bandwidth
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': 500
outputs:
- id: msout
outputSource:
@@ -59,6 +63,7 @@ outputs:
- concat_logfiles_applygsm/output
- concat_logfiles_solutions/output
- concat_logfiles_structure/output
+ - concat_logfiles_wsclean/output
type: 'File[]'
linkMerge: merge_flattened
'sbg:x': 1300
@@ -66,6 +71,7 @@ outputs:
- id: inspection
outputSource:
- structure_function/structure_plot
+ - wsclean/image
type: 'File[]?'
linkMerge: merge_flattened
'sbg:x': 1300
@@ -75,12 +81,12 @@ steps:
in:
- id: h5parm
source: input_h5parm
- - id: solset
- default: 'sol000'
- - id: refsolset
- default: 'target'
- id: refh5parm
source: insolutions
+ - id: solset
+ default: sol000
+ - id: refsolset
+ default: target
- id: soltab_in
source: gsmcal_step
valueFrom: $(self+'000')
@@ -100,32 +106,32 @@ steps:
label: add_missing_stations
'sbg:x': -200
'sbg:y': -300
-
- id: apply_gsmcal
in:
- id: msin
source: msin
- - id: msin_datacolumn
- default: DATA
- id: msout_name
- source: msin
- valueFrom: $(self.nameroot+'_pre-cal.ms')
linkMerge: merge_flattened
+ source:
+ - msin
+ valueFrom: $(self.nameroot+'_pre-cal.ms')
+ - id: msin_datacolumn
+ default: DATA
- id: parmdb
source: write_solutions/outh5parm
- id: msout_datacolumn
default: DATA
- - id: storagemanager
- default: "Dysco"
- - id: databitrate
- source: compression_bitrate
- id: correction
source:
- skymodel_source
- gsmcal_step
valueFrom: $(self.join(''))
- id: solset
- default: 'target'
+ default: target
+ - id: storagemanager
+ default: Dysco
+ - id: databitrate
+ source: compression_bitrate
out:
- id: msout
- id: logfile
@@ -137,6 +143,80 @@ steps:
scatterMethod: flat_crossproduct
'sbg:x': 400
'sbg:y': 0
+ - id: average
+ in:
+ - id: msin
+ source: apply_gsmcal/msout
+ - id: msout_name
+ linkMerge: merge_flattened
+ source:
+ - apply_gsmcal/msout
+ valueFrom: $(self.nameroot+'_wsclean.ms')
+ - id: msin_datacolumn
+ default: DATA
+ - id: msout_datacolumn
+ default: DATA
+ - id: overwrite
+ default: false
+ - id: storagemanager
+ default: Dysco
+ - id: avg_timestep
+ default: 2
+ - id: avg_freqstep
+ default: 2
+ out:
+ - id: msout
+ - id: logfile
+ run: ../../lofar-cwl/steps/average.cwl
+ label: average
+ scatter:
+ - msin
+ 'sbg:x': 600
+ 'sbg:y': -200
+ - id: wsclean
+ in:
+ - id: msin
+ source:
+ - average/msout
+ - id: image_scale
+ default: 15asec
+ - id: niter
+ default: 1000000
+ - id: nmiter
+ default: 5
+ - id: multiscale
+ default: true
+ - id: mgain
+ default: 0.8
+ - id: ncpu
+ default: 4
+ - id: parallel-gridding
+ default: 1
+ - id: parallel-deconvolution
+ default: 1500
+ - id: parallel-reordering
+ default: 4
+ - id: channels-out
+ source: total_bandwidth
+ valueFrom: $(Math.round(self/1000000))
+ - id: join-channels
+ default: true
+ - id: taper-gaussian
+ default: 40asec
+ - id: weighting
+ default: briggs -0.5
+ - id: maxuvw-m
+ default: 20000
+ - id: image_name
+ source: get_targetname/targetname
+ out:
+ - id: dirty_image
+ - id: image
+ - id: logfile
+ run: ../../lofar-cwl/steps/wsclean.cwl
+ label: wsclean
+ 'sbg:x': 800
+ 'sbg:y': -200
- id: merge_array_files
in:
- id: input
@@ -148,7 +228,6 @@ steps:
label: merge_array_files
'sbg:x': 600
'sbg:y': 0
-
- id: write_solutions
in:
- id: h5parmFile
@@ -179,10 +258,8 @@ steps:
in:
- id: h5parmFile
source: write_solutions/outh5parm
- - id: outsolset
- default: 'target'
- id: solsetName
- default: 'target'
+ default: target
- id: pointing
source: get_targetname/targetname
out:
@@ -197,7 +274,7 @@ steps:
- id: h5parmFile
source: write_solutions/outh5parm
- id: solset
- default: 'target'
+ default: target
- id: soltab
source:
- skymodel_source
@@ -260,11 +337,26 @@ steps:
label: concat_logfiles_structure
'sbg:x': 950
'sbg:y': 500
+ - id: concat_logfiles_wsclean
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - wsclean/logfile
+ - id: file_prefix
+ default: wsclean
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_wsclean
+ 'sbg:x': 1100
+ 'sbg:y': 500
- id: get_targetname
in:
- id: msin
linkMerge: merge_flattened
- source: msin
+ source:
+ - msin
out:
- id: targetname
- id: logfile
@@ -273,7 +365,7 @@ steps:
'sbg:x': 200
'sbg:y': 200
requirements:
- - class: SubworkflowFeatureRequirement
- - class: ScatterFeatureRequirement
- class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
\ No newline at end of file
+ - class: InlineJavascriptRequirement
+ - class: ScatterFeatureRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/workflows/prefactor_target/gsmcal.cwl b/workflows/prefactor_target/gsmcal.cwl
index 07d6adb16fd05ab1d974cb522eace5e359d01fed..48bb1606ad5123afbf31e314372d37ecca7c8121 100644
--- a/workflows/prefactor_target/gsmcal.cwl
+++ b/workflows/prefactor_target/gsmcal.cwl
@@ -120,6 +120,12 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1900
'sbg:y': 1000
+ - id: total_bandwidth
+ outputSource:
+ - sort_times_into_freqGroups/total_bandwidth
+ type: int
+ 'sbg:x': 1900
+ 'sbg:y': 1200
steps:
- id: identifybadantennas
in:
@@ -166,6 +172,7 @@ steps:
out:
- id: filenames
- id: groupnames
+ - id: total_bandwidth
- id: logfile
run: ../../steps/sort_times_into_freqGroups.cwl
label: sorttimesintofreqGroups