diff --git a/lofar-cwl/steps/DP3.ApplyBeamStepGenerator.cwl b/lofar-cwl/steps/DP3.ApplyBeamStepGenerator.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..3e0c9fde9da2eb2b577339fc8bfde8c36e00c6c8
--- /dev/null
+++ b/lofar-cwl/steps/DP3.ApplyBeamStepGenerator.cwl
@@ -0,0 +1,104 @@
+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: applybeam_step_generator
+
+inputs:
+ - id: steps
+ type: Any[]?
+ default: []
+ - id: step_name
+ type: string
+ doc: unique name for the step
+ default: applybeam
+
+ - id: direction
+ type: string?
+ doc: A RA/Dec value specifying in what direction to correct the beam. See phaseshift.phasecenter for syntax. If empty, the beam is corrected in the direction of the current phase center.
+ - id: onebeamperpatch
+ type: string?
+ doc: |
+ Compute the beam only for the center of each patch
+ (saves computation time, but you should set this to false for large
+ patches).
+ In the ApplyBeam step, this setting does not make sense
+ (but it does if the applybeam is part of predict, ddecal, gaincal,
+ h5parmpredict, etc.).
+ Generally, FALSE is the right setting for this option.
+ The default has changed to false in a recent (Nov 2018) version.
+ - id: usechannelfreq
+ type: boolean?
+ doc: |
+ Compute the beam for each channel of the measurement set separately.
+ This is useful for merged / concatenated measurement sets.
+ For raw LOFAR data you should set it to false,
+ so that the beam will be formed as in the station hardware.
+ Also, setting it to false is faster.
+
+ - id: updateweights
+ type: boolean?
+ doc: |
+ Update the weights column, in a way consistent with
+ the weights being inverse proportional to the autocorrelations
+ (e.g. if 'autoweights' was used before).
+
+ - id: invert
+ type: boolean?
+ doc: |
+ Invert the corrections, to correct the data. Default is true.
+ If you want to corrupt the data, set it to 'false'
+
+ - id: beammode
+
+ type:
+ type: enum
+ symbols:
+ - array_factor
+ - element
+ - default
+ doc: |
+ Beam mode to apply, can be “array_factor”, “element” or “default”.
+ Default is to apply both the element beam and the array factor.
+
+
+outputs:
+ - id: augmented_steps
+ outputSource:
+ - DP3_GenericStep/augmented_steps
+ type: Any[]
+steps:
+ - id: DP3_GenericStep
+ in:
+ - id: step_type
+ default: 'applybeam'
+ - id: step_id
+ source: step_name
+ - id: steps
+ source: steps
+ - id: parameters
+ valueFrom: $(inputs)
+
+ - id: direction
+ source: direction
+ - id: onebeamperpatch
+ source: onebeamperpatch
+ - id: usechannelfreq
+ source: usechannelfreq
+ - id: updateweights
+ source: updateweights
+ - id: invert
+ source: invert
+ - id: beammode
+ source: beammode
+
+
+#-------------------------------
+
+ out:
+ - augmented_steps
+ run: ../steps/DP3.GenericStep.cwl
+requirements:
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/lofar-cwl/steps/applybeam.cwl b/lofar-cwl/steps/applybeam.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..6c7baccf77c4f812aa23614c963102453ede20e2
--- /dev/null
+++ b/lofar-cwl/steps/applybeam.cwl
@@ -0,0 +1,93 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: applycal
+baseCommand:
+ - DPPP
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
+ - default: DATA
+ id: msin_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msin.datacolumn=
+ separate: false
+ doc: Input data Column
+ - id: msout_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msout.datacolumn=
+ separate: false
+ doc: Output data column
+ - default: applybeam
+ id: type
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: applybeam.type=
+ separate: false
+ doc: >
+ Type of correction to perform. When using H5Parm, this is for now the name
+ of the soltab; the type will be deduced from the metadata in that soltab,
+ except for full Jones, in which case correction should be 'fulljones'.
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding:
+ prefix: msout.storagemanager=
+ - id: updateweights
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: -applybeam.updateweights=
+ separate: false
+ - id: beammode
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: -applybeam.beammode=
+ separate: false
+ - id: usechannelfreq
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: -applybeam.usechannelfreq=
+ separate: false
+ - id: invert
+ type: boolean?
+ inputBinding:
+ position: 0
+ prefix: -applybeam.invert=
+ separate: false
+outputs:
+ - id: msout
+ doc: Output Measurement Set
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msin.basename)
+ - id: logfile
+ type: File
+ outputBinding:
+ glob: applycal_$(inputs.type).log
+stdout: applycal_$(inputs.type).log
+arguments:
+ - 'steps=[applybeam]'
+ - msout=.
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
+ - class: InlineJavascriptRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
diff --git a/lofar-cwl/steps/applycal.cwl b/lofar-cwl/steps/applycal.cwl
index b9364ea0e24c8b3b5e03f4ffafaaca01ce6f3e11..49b7af68095d2dc5c452ecbf273dad43e25ffe7f 100644
--- a/lofar-cwl/steps/applycal.cwl
+++ b/lofar-cwl/steps/applycal.cwl
@@ -5,9 +5,6 @@ $namespaces:
id: applycal
baseCommand:
- DPPP
-arguments:
- - steps=[applycal]
- - msout=.
inputs:
- id: msin
type: Directory
@@ -26,32 +23,62 @@ inputs:
doc: Input data Column
- id: parmdb
type: File
- doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
inputBinding:
+ position: 0
prefix: applycal.parmdb=
separate: false
- - id: correction
- default: gain
+ doc: >-
+ Path of parmdb in which the parameters are stored. This can also be an
+ H5Parm file, in that case the filename has to end in '.h5'
+ - id: msout_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msout.datacolumn=
+ separate: false
+ doc: Output data column
+ - default: gain
+ id: correction
+ type: string
+ inputBinding:
+ position: 0
+ prefix: applycal.correction=
+ separate: false
+ doc: >
+ Type of correction to perform. When using H5Parm, this is for now the name
+ of the soltab; the type will be deduced from the metadata in that soltab,
+ except for full Jones, in which case correction should be 'fulljones'.
+ - id: storagemanager
type: string
- doc: |
- Type of correction to perform. When using H5Parm, this is for now the name of the soltab; the type will be deduced from the metadata in that soltab, except for full Jones, in which case correction should be 'fulljones'.
+ default: dysco
+ inputBinding:
+ prefix: msout.storagemanager=
+ - id: updateweights
+ type: boolean?
inputBinding:
- prefix: applycal.correction=
- separate: false
-
+ position: 0
+ prefix: -applycal.updateweights=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
- glob: $(inputs.msin.basename)
-
+ glob: $(inputs.msin.basename)
+ - id: logfile
+ type: File
+ outputBinding:
+ glob: applycal_$(inputs.correction).log
+stdout: applycal_$(inputs.correction).log
+arguments:
+ - 'steps=[applycal]'
+ - msout=.
requirements:
- class: InitialWorkDirRequirement
listing:
- - entry: $(inputs.msin)
- writable: true
+ - entry: $(inputs.msin)
+ writable: true
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
- dockerPull: lofareosc/lofar-pipeline-ci:latest
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
diff --git a/subworkflow/apply_calibrate_pa.cwl b/subworkflow/apply_calibrate_pa.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..f13f928286ab2c6b280015a1e29b4542839a6f29
--- /dev/null
+++ b/subworkflow/apply_calibrate_pa.cwl
@@ -0,0 +1,146 @@
+class: Workflow
+cwlVersion: v1.0
+id: apply_calibrate
+label: apply_calibrate
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msin
+ type: Directory
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: do_smooth
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: flagunconverged
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: propagatesolutions
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: input_h5parm
+ type: File
+ 'sbg:x': -1000
+ 'sbg:y': 300
+outputs:
+ - id: msout
+ outputSource:
+ - calib_cal/msout
+ type: Directory
+ 'sbg:x': 1000
+ 'sbg:y': -200
+ - id: BLsmooth.log
+ outputSource:
+ - BLsmooth/logfile
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -100
+ - id: apply_cal.log
+ outputSource:
+ - applyPA/logfile
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': 0
+ - id: calib_cal.log
+ outputSource:
+ - calib_cal/logfile
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': 100
+ - id: outh5parm
+ outputSource:
+ - calib_cal/h5parm
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': 200
+ - id: applybeam.log
+ outputSource:
+ - applybeam/logfile
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': 300
+steps:
+ - id: applyPA
+ in:
+ - id: msin
+ source: msin
+ - id: msin_datacolumn
+ default: DATA
+ - id: parmdb
+ source: input_h5parm
+ - id: msout_datacolumn
+ default: CORRECTED_DATA
+ - id: correction
+ default: polalign
+ out:
+ - id: msout
+ - id: logfile
+ run: ../lofar-cwl/steps/applycal.cwl
+ label: applyPA
+ 'sbg:x': -400
+ 'sbg:y': 0
+ - id: applybeam
+ in:
+ - id: msin_datacolumn
+ default: CORRECTED_DATA
+ - id: msout_datacolumn
+ default: CORRECTED_DATA
+ - id: updateweights
+ default: true
+ - id: invert
+ default: true
+ - id: beammode
+ default: element
+ - id: usechannelfreq
+ default: false
+ - id: msin
+ source: applyPA/msout
+ - id: type
+ default: applybeam
+ out:
+ - id: msout
+ - id: logfile
+ run: ../lofar-cwl/steps/applybeam.cwl
+ label: applybeam
+ 'sbg:x': -200
+ 'sbg:y': 0
+ - id: BLsmooth
+ in:
+ - id: msin
+ source: applybeam/msout
+ - id: do_smooth
+ source: do_smooth
+ - id: in_column_name
+ default: CORRECTED_DATA
+ out:
+ - id: msout
+ - id: logfile
+ run: ../steps/blsmooth.cwl
+ label: BLsmooth
+ 'sbg:x': 0
+ 'sbg:y': 0
+ - id: calib_cal
+ in:
+ - id: msin
+ source: BLsmooth/msout
+ - id: msin_datacolumn
+ default: SMOOTHED_DATA
+ - id: msin_modelcolum
+ default: MODEL_DATA
+ - id: flagunconverged
+ source: flagunconverged
+ - id: propagate_solutions
+ source: propagatesolutions
+ - id: mode
+ default: rotation+diagonal
+ out:
+ - id: msout
+ - id: h5parm
+ - id: logfile
+ run: ../lofar-cwl/steps/ddecal.cwl
+ 'sbg:x': 500
+ 'sbg:y': 0
+requirements: []
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index fda22fbe5aa6569ea2f81d85348b9fca6f728542..cd03d8c1139e5bccafc49a0518fcd7542cb2610d 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -307,7 +307,7 @@ steps:
- id: logfiles
- id: msout
- id: parset
- - id: instrument.h5
+ - id: solutions
- id: inspection
run: ./prefactor_calibrator.cwl
label: prefactor_calibrator
@@ -345,7 +345,7 @@ steps:
in:
- id: log_files
source:
- - prefactor/instrument.h5
+ - prefactor/solutions
- id: sub_directory_name
default: results/cal_values
out:
diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl
index 04316e702363619af96f6bb15a7e6f2304034cb8..083d96830bdc5cf675ad30a7c4795396b2ac8f22 100644
--- a/workflows/prefactor_calibrator.cwl
+++ b/workflows/prefactor_calibrator.cwl
@@ -200,9 +200,9 @@ outputs:
type: File[]
'sbg:x': 1000
'sbg:y': -600
- - id: instrument.h5
+ - id: solutions
outputSource:
- - pa/output_h5parm
+ - pa/solutions
type: File
'sbg:x': 1000
'sbg:y': -500
@@ -277,12 +277,12 @@ steps:
- id: A-Team_skymodel
source: A-Team_skymodel
out:
- - id: instrument.h5
+ - id: outh5parm
- id: logfiles
+ - id: outh5parm_logfile
- id: check_Ateam_separation.png
- id: msout
- id: parset
- - id: sourcedb
run: ./prefactor_calibrator/prep.cwl
label: prep
'sbg:x': 0
@@ -294,12 +294,21 @@ steps:
- prep/msout
- id: h5parm
source:
- - prep/instrument.h5
+ - prep/outh5parm
- id: refant
source: refant
+ - id: inh5parm_logfile
+ source:
+ - prep/outh5parm_logfile
+ - id: flagunconverged
+ source: flagunconverged
+ - id: propagatesolutions
+ source: propagatesolutions
+ - id: do_smooth
+ source: do_smooth
out:
- id: msout
- - id: output_h5parm
+ - id: solutions
- id: inspection
- id: logfiles
run: ./prefactor_calibrator/pa.cwl
diff --git a/workflows/prefactor_calibrator/pa.cwl b/workflows/prefactor_calibrator/pa.cwl
index c2951b3d2032dce1e788195715c94e62eb636168..9bdac6585831b6cc9209d0a8b966e9d0bfeb90bd 100644
--- a/workflows/prefactor_calibrator/pa.cwl
+++ b/workflows/prefactor_calibrator/pa.cwl
@@ -5,6 +5,16 @@ label: PA
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
+ - id: flagunconverged
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: propagatesolutions
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': -200
- id: msin
type: 'Directory[]'
'sbg:x': -1000
@@ -17,10 +27,25 @@ inputs:
type: string?
'sbg:x': -1000
'sbg:y': 100
+ - id: inh5parm_logfile
+ type: File
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: do_smooth
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': 300
outputs:
+ - id: outh5parm
+ outputSource:
+ - h5parm_collector/outh5parm
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -200
- id: msout
outputSource:
- - msin
+ - apply_calibrate_pa/msout
type: 'Directory[]'
'sbg:x': 1000
'sbg:y': -100
@@ -37,7 +62,7 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': 0
- - id: output_h5parm
+ - id: solutions
outputSource:
- write_solutions/outh5parm
type: File
@@ -46,10 +71,20 @@ outputs:
- id: logfiles
outputSource:
- concat_logfiles_pa/output
+ - concat_logfiles_calib/output
+ - concat_logfiles_blsmooth/output
+ - concat_logfiles_beam/output
+ - concat_logfiles_apply/output
type: 'File[]'
linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': 200
+ - id: outh5parm_logfile
+ outputSource:
+ - h5parm_collector/log
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': 300
steps:
- id: pol_align
in:
@@ -271,6 +306,7 @@ steps:
- id: file_list
linkMerge: merge_flattened
source:
+ - inh5parm_logfile
- pol_align/logfiles
- losoto_plot_P3/logfile
- losoto_plot_Pd/logfile
@@ -291,14 +327,13 @@ steps:
- id: write_solutions
in:
- id: h5parmFiles
- source:
- - pol_align/output_h5parm
- - id: outh5parmname
- default: cal_solutions.h5
+ source: pol_align/output_h5parm
- id: outsolset
default: calibrator
- id: insoltab
default: polalign
+ - id: outh5parmname
+ default: cal_solutions.h5
- id: squeeze
default: true
- id: verbose
@@ -310,6 +345,102 @@ steps:
label: write_solutions
'sbg:x': 0
'sbg:y': 500
+ - id: apply_calibrate_pa
+ in:
+ - id: msin
+ source: msin
+ - id: do_smooth
+ source: do_smooth
+ - id: flagunconverged
+ source: flagunconverged
+ - id: propagatesolutions
+ source: propagatesolutions
+ - id: input_h5parm
+ source: write_solutions/outh5parm
+ out:
+ - id: msout
+ - id: BLsmooth.log
+ - id: apply_cal.log
+ - id: calib_cal.log
+ - id: outh5parm
+ - id: applybeam.log
+ run: ../../subworkflow/apply_calibrate_pa.cwl
+ label: apply_calibrate_pa
+ scatter:
+ - msin
+ 'sbg:x': 500
+ 'sbg:y': 0
+ - id: concat_logfiles_blsmooth
+ in:
+ - id: file_list
+ source:
+ - apply_calibrate_pa/BLsmooth.log
+ - id: file_prefix
+ default: blsmooth_PA
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_blsmooth
+ 'sbg:x': 750
+ 'sbg:y': 300
+ - id: concat_logfiles_beam
+ in:
+ - id: file_list
+ source:
+ - apply_calibrate_pa/applybeam.log
+ - id: file_prefix
+ default: applybeam_PA
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_beam
+ 'sbg:x': 750
+ 'sbg:y': 400
+ - id: concat_logfiles_apply
+ in:
+ - id: file_list
+ source:
+ - apply_calibrate_pa/apply_cal.log
+ - id: file_prefix
+ default: apply_cal_PA
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_apply
+ 'sbg:x': 750
+ 'sbg:y': 600
+ - id: concat_logfiles_calib
+ in:
+ - id: file_list
+ source:
+ - apply_calibrate_pa/calib_cal.log
+ - id: file_prefix
+ default: calib_cal_PA
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_calib
+ 'sbg:x': 750
+ 'sbg:y': 800
+ - id: h5parm_collector
+ in:
+ - id: h5parmFiles
+ source:
+ - apply_calibrate_pa/outh5parm
+ - id: squeeze
+ default: true
+ - id: verbose
+ default: true
+ - id: clobber
+ default: true
+ out:
+ - id: outh5parm
+ - id: log
+ run: ../../lofar-cwl/steps/H5ParmCollector.cwl
+ label: H5parm_collector
+ 'sbg:x': 750
+ 'sbg:y': -300
requirements:
- class: SubworkflowFeatureRequirement
+ - class: ScatterFeatureRequirement
- class: MultipleInputFeatureRequirement
diff --git a/workflows/prefactor_calibrator/prep.cwl b/workflows/prefactor_calibrator/prep.cwl
index 538c2be33fdcb0e7da1ba1fb51c40a7f57ca0e59..950f70d62a80b18f55f6e84ce916f7b705c28326 100644
--- a/workflows/prefactor_calibrator/prep.cwl
+++ b/workflows/prefactor_calibrator/prep.cwl
@@ -110,7 +110,7 @@ inputs:
'sbg:x': -1000
'sbg:y': 1500
outputs:
- - id: instrument.h5
+ - id: outh5parm
outputSource:
- h5parm_collector/outh5parm
type: File
@@ -134,12 +134,12 @@ outputs:
type: 'File[]'
'sbg:x': 1000
'sbg:y': 200
- - id: sourcedb
+ - id: outh5parm_logfile
outputSource:
- - make_sourcedb/sourcedb
+ - h5parm_collector/log
type: File
'sbg:x': 1000
- 'sbg:y': 500
+ 'sbg:y': 650
- id: logfiles
linkMerge: merge_flattened
outputSource:
@@ -319,7 +319,7 @@ steps:
source:
- predict_calibrate/BLsmooth.log
- id: file_prefix
- default: blsmooth
+ default: blsmooth_prep
out:
- id: output
run: ../../steps/concatenate_files.cwl
@@ -345,7 +345,7 @@ steps:
source:
- predict_calibrate/calib_cal.log
- id: file_prefix
- default: calib_cal
+ default: calib_cal_prep
out:
- id: output
run: ../../steps/concatenate_files.cwl