Commit c40929f6 authored by mancini's avatar mancini

Merge branch 'add_bandpass' into 'master'

Add bandpass

See merge request !18
parents 326d9b47 23580c7c
Pipeline #756 passed with stages
in 8 minutes and 39 seconds
#!/usr/bin/env cwl-runner
class: Workflow
cwlVersion: v1.0
id: applybeam_step_generator
inputs:
- id: steps
type: Any[]?
default: []
- id: step_name
type: string
doc: unique name for the step
default: applybeam
- id: direction
type: string?
doc: A RA/Dec value specifying in what direction to correct the beam. See phaseshift.phasecenter for syntax. If empty, the beam is corrected in the direction of the current phase center.
- id: onebeamperpatch
type: string?
doc: |
Compute the beam only for the center of each patch
(saves computation time, but you should set this to false for large
patches).
In the ApplyBeam step, this setting does not make sense
(but it does if the applybeam is part of predict, ddecal, gaincal,
h5parmpredict, etc.).
Generally, FALSE is the right setting for this option.
The default has changed to false in a recent (Nov 2018) version.
- id: usechannelfreq
type: boolean?
doc: |
Compute the beam for each channel of the measurement set separately.
This is useful for merged / concatenated measurement sets.
For raw LOFAR data you should set it to false,
so that the beam will be formed as in the station hardware.
Also, setting it to false is faster.
- id: updateweights
type: boolean?
doc: |
Update the weights column, in a way consistent with
the weights being inverse proportional to the autocorrelations
(e.g. if 'autoweights' was used before).
- id: invert
type: boolean?
doc: |
Invert the corrections, to correct the data. Default is true.
If you want to corrupt the data, set it to 'false'
- id: beammode
type:
type: enum
symbols:
- array_factor
- element
- default
doc: |
Beam mode to apply, can be “array_factor”, “element” or “default”.
Default is to apply both the element beam and the array factor.
outputs:
- id: augmented_steps
outputSource:
- DP3_GenericStep/augmented_steps
type: Any[]
steps:
- id: DP3_GenericStep
in:
- id: step_type
default: 'applybeam'
- id: step_id
source: step_name
- id: steps
source: steps
- id: parameters
valueFrom: $(inputs)
- id: direction
source: direction
- id: onebeamperpatch
source: onebeamperpatch
- id: usechannelfreq
source: usechannelfreq
- id: updateweights
source: updateweights
- id: invert
source: invert
- id: beammode
source: beammode
#-------------------------------
out:
- augmented_steps
run: ../steps/DP3.GenericStep.cwl
requirements:
- class: StepInputExpressionRequirement
- class: InlineJavascriptRequirement
- class: MultipleInputFeatureRequirement
......@@ -5,9 +5,6 @@ $namespaces:
id: applycal
baseCommand:
- DPPP
arguments:
- steps=[applycal]
- msout=.
inputs:
- id: msin
type: Directory
......@@ -26,32 +23,51 @@ inputs:
doc: Input data Column
- id: parmdb
type: File
doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
inputBinding:
position: 0
prefix: applycal.parmdb=
separate: false
- id: correction
default: gain
doc: >-
Path of parmdb in which the parameters are stored. This can also be an
H5Parm file, in that case the filename has to end in '.h5'
- id: msout_datacolumn
type: string
inputBinding:
position: 0
prefix: msout.datacolumn=
separate: false
doc: Output data column
- default: gain
id: correction
type: string
doc: |
Type of correction to perform. When using H5Parm, this is for now the name of the soltab; the type will be deduced from the metadata in that soltab, except for full Jones, in which case correction should be 'fulljones'.
inputBinding:
prefix: applycal.correction=
separate: false
position: 0
prefix: applycal.correction=
separate: false
doc: >
Type of correction to perform. When using H5Parm, this is for now the name
of the soltab; the type will be deduced from the metadata in that soltab,
except for full Jones, in which case correction should be 'fulljones'.
- id: updateweights
type: boolean?
inputBinding:
position: 0
prefix: applycal.updateweights=True
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
glob: $(inputs.msin.basename)
glob: $(inputs.msin.basename)
arguments:
- 'steps=[applycal]'
- msout=.
requirements:
- class: InitialWorkDirRequirement
listing:
- entry: $(inputs.msin)
writable: true
- entry: $(inputs.msin)
writable: true
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
dockerPull: lofareosc/lofar-pipeline-ci:latest
dockerPull: 'lofareosc/lofar-pipeline-ci:latest'
class: Workflow
cwlVersion: 'sbg:draft-2'
id: applybeam
label: applybeam
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
- type:
- 'null'
- boolean
id: '#usechannelfreq'
'sbg:includeInPorts': true
'sbg:x': -491.39886474609375
'sbg:y': -275.5
- type:
- 'null'
- boolean
id: '#updateweights'
'sbg:includeInPorts': true
'sbg:x': -595.3988647460938
'sbg:y': -140.5
- type:
- 'null'
- string
id: '#onebeamperpatch'
'sbg:includeInPorts': true
'sbg:x': -609.3988647460938
'sbg:y': -22.5
- type:
- 'null'
- boolean
id: '#invert'
'sbg:includeInPorts': true
'sbg:x': -568.3988647460938
'sbg:y': 102.5
- type:
- 'null'
- string
id: '#direction'
'sbg:includeInPorts': true
'sbg:x': -472.39886474609375
'sbg:y': 211.5
- type:
- type: enum
symbols:
- array_factor
- element
- default
name: ''
id: '#beammode'
'sbg:includeInPorts': true
'sbg:x': -455
'sbg:y': 378
- type:
- Directory
id: '#msin'
'sbg:includeInPorts': true
'sbg:x': -203
'sbg:y': 424
outputs:
- id: '#msout'
source:
- '#dp3_execute.msout'
type:
- Directory
'sbg:x': 429.60113525390625
'sbg:y': 267.5
steps:
- id: '#applybeam_step_generator'
inputs:
- id: '#applybeam_step_generator.direction'
source: '#direction'
- id: '#applybeam_step_generator.onebeamperpatch'
source: '#onebeamperpatch'
- id: '#applybeam_step_generator.usechannelfreq'
source: '#usechannelfreq'
- id: '#applybeam_step_generator.updateweights'
source: '#updateweights'
- id: '#applybeam_step_generator.invert'
source: '#invert'
- id: '#applybeam_step_generator.beammode'
source: '#beammode'
outputs:
- id: '#applybeam_step_generator.augmented_steps'
run: ../lofar-cwl/steps/DP3.ApplyBeamStepGenerator.cwl
'sbg:x': -30
'sbg:y': 115
- id: '#dp3_execute'
inputs:
- id: '#dp3_execute.msin'
source: '#msin'
- id: '#dp3_execute.steps'
source:
- '#applybeam_step_generator.augmented_steps'
outputs:
- id: '#dp3_execute.secondary_output_files'
- id: '#dp3_execute.secondary_output_directories'
- id: '#dp3_execute.msout'
run: ../lofar-cwl/steps/DP3.Execute.cwl
label: DP3.Execute
'sbg:x': 208
'sbg:y': 127
doc: ''
class: CommandLineTool
cwlVersion: v1.0
$namespaces:
sbg: 'https://www.sevenbridges.com/'
id: transfer_solutions
baseCommand:
- transfer_solutions.py
inputs:
- 'sbg:toolDefaultValue': output.h5
id: h5parm_name
type: string?
inputBinding:
position: 1
doc: H5parm to which the solutions should be transferred.
- id: refh5parm
type: File
inputBinding:
position: 0
prefix: '--refh5parm'
doc: Name of the h5parm from which the solutions should be transferred.
- id: insolset
type: string?
inputBinding:
position: 0
prefix: '--insolset'
doc: Name of the input h5parm solution set
- id: outsolset
type: string?
inputBinding:
position: 0
prefix: '--outsolset'
doc: Name of the output h5parm solution set
- id: insoltab
type: string?
inputBinding:
position: 0
prefix: '--insoltab'
doc: Name of the input h5parm solution set
- id: outsoltab
type: string?
inputBinding:
position: 0
prefix: '--outsoltab'
doc: Name of the output h5parm solution set
- id: antenna
type: string?
inputBinding:
position: 0
prefix: '--antenna'
doc: Regular expression of antenna solutions to be transferred
outputs:
- id: output
type: File?
outputBinding:
glob: $(inputs.h5parm_name)
label: transfer_solutions
hints:
- class: DockerRequirement
dockerPull: 'lofareosc/prefactor:latest'
requirements:
- class: InlineJavascriptRequirement
class: Workflow
cwlVersion: v1.0
id: apply_calibration
label: apply_calibration
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
- id: parmdb
type: File
'sbg:x': -604.3988647460938
'sbg:y': -199.5
- id: msin
type: Directory
'sbg:x': -679.001220703125
'sbg:y': 36
outputs:
- id: msout
outputSource:
- apply_FR/msout
type: Directory
'sbg:x': 551.6011352539062
'sbg:y': -4.501007080078125
steps:
- id: apply_PA
in:
- id: msin
source: msin
- id: msin_datacolumn
default: DATA
- id: parmdb
source: parmdb
- id: msout_datacolumn
default: CORRECTED_DATA
- id: correction
default: polalign
out:
- id: msout
run: ../lofar-cwl/steps/applycal.cwl
'sbg:x': -358
'sbg:y': 4
- id: apply_bandpass
in:
- id: msin
source: apply_PA/msout
- id: msin_datacolumn
default: CORRECTED_DATA
- id: parmdb
source: parmdb
- id: msout_datacolumn
default: CORRECTED_DATA
- id: correction
default: bandpass
- id: updateweights
default: true
out:
- id: msout
run: ../lofar-cwl/steps/applycal.cwl
'sbg:x': -133
'sbg:y': -14
- id: apply_FR
in:
- id: msin
source: applybeam/msout
- id: msin_datacolumn
default: CORRECTED_DATA
- id: parmdb
source: parmdb
- id: msout_datacolumn
default: CORRECTED_DATA
- id: correction
default: faraday
out:
- id: msout
run: ../lofar-cwl/steps/applycal.cwl
'sbg:x': 287
'sbg:y': -36
- id: applybeam
in:
- id: updateweights
default: true
- id: invert
default: true
- id: beammode
default: element
- id: msin
source: apply_bandpass/msout
out:
- id: msout
run: ../steps/applybeam.cwl
label: applybeam
'sbg:x': 62
'sbg:y': -15
requirements:
- class: SubworkflowFeatureRequirement
class: Workflow
cwlVersion: v1.0
id: bandpass
label: bandpass
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
- id: msin
type: 'Directory[]'
'sbg:x': -66.62995910644531
'sbg:y': -47.43605041503906
- id: do_smooth
type: boolean
'sbg:x': -30.103496551513672
'sbg:y': 168.6453857421875
- id: propagate_solutions
type: boolean
'sbg:x': 81.81513214111328
'sbg:y': -313.3733215332031
- id: flagunconverged
type: boolean
'sbg:x': 172.986328125
'sbg:y': 244.30821228027344
- id: skipInternational
type: string?
'sbg:x': 600.0303344726562
'sbg:y': -411.65814208984375
- id: max2interpolate
type: int?
'sbg:x': 472.91082763671875
'sbg:y': -275.4460144042969
- id: bandpass_freqresolution
type: string
'sbg:x': 479.5017395019531
'sbg:y': 199.0994415283203
- id: avg_freqresolution
type: string?
'sbg:x': 565.18359375
'sbg:y': 390.2358093261719
- id: ampRange
type: 'float[]?'
'sbg:x': 773.8956909179688
'sbg:y': 471.523681640625
- id: outh5parmname
type: string
'sbg:x': 805.8613891601562
'sbg:y': -296.09796142578125
- id: antenna_to_transfer
type: string?
'sbg:x': 985.4193725585938
'sbg:y': 183.84898376464844
outputs:
- id: output
outputSource:
- transfer_solutions/output
type: File?
'sbg:x': 1310.1014404296875
'sbg:y': -65.6321029663086
steps:
- id: smooth_corrected
in:
- id: msin
source: msin
- id: do_smooth
source: do_smooth
- id: in_column_name
default: CORRECTED_DATA
- id: out_column
default: SMOOTHED_DATA
- id: restore
default: true
out:
- id: msout
run: ../steps/blsmooth.cwl
label: smooth_corrected
scatter:
- msin
'sbg:x': 143.50439453125
'sbg:y': -49.544036865234375
- id: calib_cal2
in:
- id: msin
source: smooth_corrected/msout
- id: msin_datacolumn
default: SMOOTHED_DATA
- id: output_name_h5parm
default: calibration.h5
- id: propagate_solutions
source: propagate_solutions
- id: flagunconverged
source: flagunconverged
out:
- id: msout
- id: h5parm
run: ../lofar-cwl/steps/calib_rot_diag.cwl
scatter:
- msin
'sbg:x': 347.1034851074219
'sbg:y': -49.969146728515625
- id: h5imp_cal_bandpass
in:
- id: h5parmFiles
source:
- calib_cal2/h5parm
- id: insolset
default: bandpass
- id: squeeze
default: true
- id: verbose
default: true
- id: clobber
default: true
out:
- id: outh5parm
- id: log
run: ../lofar-cwl/steps/H5ParmCollector.cwl
label: h5imp_cal_bandpass
scatter:
- h5parmFiles
'sbg:x': 553.6541748046875
'sbg:y': -29.645381927490234
- id: h5exp_cal_bandpass
in:
- id: h5parmFiles
source:
- losoto_bandpass/output_h5parm
- id: insolset
default: bandpass
- id: outh5parmname
source: outh5parmname
- id: verbose
default: true
- id: clobber
default: true
out:
- id: outh5parm
- id: log
run: ../lofar-cwl/steps/H5ParmCollector.cwl
label: h5exp_cal_bandpass
'sbg:x': 943
'sbg:y': -45
- id: transfer_solutions
in:
- id: refh5parm
source: h5exp_cal_bandpass/outh5parm
- id: insolset
default: calibrator
- id: outsolset
default: calibrator
- id: insoltab
default: bandpass
- id: outsoltab
default: bandpass
- id: antenna
source: antenna_to_transfer
out:
- id: output
run: ../steps/transfer_solutions.cwl
label: transfer_solutions
'sbg:x': 1132
'sbg:y': -62
- id: losoto_bandpass
in:
- id: ampRange
source:
- ampRange
- id: skipInternational
source: skipInternational
- id: input_h5parm
source: h5imp_cal_bandpass/outh5parm
- id: max2interpolate
source: max2interpolate
- id: bandpass_freqresolution
source: bandpass_freqresolution
- id: avg_freqresolution
source: avg_freqresolution
out:
- id: output_h5parm
run: ./losoto_bandpass.cwl
label: losoto_bandpass
'sbg:x': 747.5215454101562
'sbg:y': -46.37216567993164
requirements:
- class: SubworkflowFeatureRequirement
- class: ScatterFeatureRequirement
......@@ -137,6 +137,8 @@ steps:
default: CORRECTED_DATA
- id: parmdb
source: h5exp_cal_FR/outh5parm
- id: msout_datacolumn
default: CORRECTED_DATA
- id: correction
default: rotationmeasure000
out:
......
class: Workflow
cwlVersion: v1.0
id: losoto_bandpass
label: losoto_bandpass
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
- id: ampRange
type: 'float[]?'
'sbg:x': 730.283935546875
'sbg:y': 152.7667236328125
- id: skipInternational
type: string?
'sbg:x': 742.662841796875
'sbg:y': -221.30563354492188
- id: input_h5parm
type: File
'sbg:x': 512.96240234375
'sbg:y': -41.42390060424805
- id: max2interpolate
type: int?
'sbg:x': 1227
'sbg:y': -278
- id: bandpass_freqresolution
type: string
'sbg:x': 1399
'sbg:y': -354
- id: avg_freqresolution
type: string?
'sbg:x': 1286.2171630859375
'sbg:y': 127.25900268554688
outputs:
- id: output_h5parm
outputSource:
- smoothb/output_h5parm
type: File
'sbg:x': 1798
'sbg:y': -47
steps:
- id: duplicateAbkp
in:
- id: input_h5parm
source: input_h5parm
- id: soltab
default: sol000/amplitude000
- id: soltabOut
default: amplitudeOrig000
out:
- id: output_h5parm
run: ../lofar-cwl/steps/LoSoTo.Duplicate.cwl
'sbg:x': 642
'sbg:y': -48
- id: losoto_flag
in:
- id: input_h5parm
source: duplicateAbkp/output_h5parm
- id: soltab
default: sol000/amplitude000
- id: axesToFlag
default:
- time
- freq
- id: order
default:
- 100
- 40
- id: maxCycles
default: 1
- id: maxRms
default: 5
- id: mode
default: smooth
out:
- id: output_h5parm
run: ../lofar-cwl/steps/LoSoTo.Flag.cwl
label: flag
'sbg:x': 764
'sbg:y': -47
- id: flagbp
in:
- id: input_h5parm
source: losoto_flag/output_h5parm
- id: soltab
default: sol000/amplitude000
- id: mode
default: bandpass
- id: ampRange
source:
- ampRange
- id: skipInternational
source: skipInternational
out:
- id: output_h5parm
run: ../lofar-cwl/steps/LoSoTo.FlagStation.cwl
'sbg:x': 873
'sbg:y': -55
- id: flagextend
in:
- id: input_h5parm
source: flagbp/output_h5parm
- id: soltab
default: sol000/amplitude000
- id: axesToExt
default:
- time
- freq
- id: size
default:
- 200
- 80
- id: percent
default: 50
- id: maxCycles
default: 2
out:
- id: output_h5parm
run: ../lofar-cwl/steps/LoSoTo.Flagextend.cwl
'sbg:x': 1005
'sbg:y': -51
- id: merge
in:
- id: input_h5parm
source: flagextend/output_h5parm
- id: soltab
default: sol000/phase000
- id: mode
default: copy
- id: soltabImport
default: amplitude000
out:
- id: output_h5parm
run: ../lofar-cwl/steps/LoSoTo.Reweight.cwl
'sbg:x': 1121
'sbg:y': -55
- id: smooth
in:
- id: input_h5parm
source: merge/output_h5parm
- id: soltab
default: sol000/amplitude000
- id: axesToSmooth
default:
- time
- id: mode