Commit 49188986 authored by mancini's avatar mancini

Squashed 'lofar-cwl/' changes from 56850c7d..110e4b15

110e4b15 Fix a space to many in shell parameters
42cbd7c9 Fix type
c5ab49c3 Add storagemanager parameter
1f48b152 Add input/output column
50bef375 Add parameters to step
0829830d Merge branch 'losoto_clocktec' into 'master'
170940e2 Add LoSoTo clocktec step
38b40fb4 Add parameters
83d14c79 Fix test
9280bde0 Add parameter to select the calibration mode
28d4fe18 Add parameter
13e37700 Merge
b2961ca6 Add output column parameter
93610759 Merge branch 'ApplyBeam' into 'master'
eb7281cc Add ApplyBeam
faf58af0 Add baseline parameter to DPPP
e553b493 Fix applycal
a178c7e5 Rename ddecal

git-subtree-dir: lofar-cwl
git-subtree-split: 110e4b15
parent 56850c7d
......@@ -132,6 +132,13 @@ losoto_clip:
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json
losoto_clocktec:
stage: test_steps
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.ClockTec.cwl test_jobs/losoto_clocktec.json
losoto_flagextend:
stage: test_steps
allow_failure: true
......
......@@ -42,7 +42,7 @@ inputs:
available memory too much affects flagging accuracy; in general try to use
at least 10 GB of memory.
- id: memorymax
type: double
type: float
default: 0
doc: >-
Maximum amount of memory (in GB) to use. ⇐0 means no maximum. As stated
......@@ -56,7 +56,7 @@ inputs:
memory to use. Note that the time window can be extended with an overlap
on the left and right side to minimize possible boundary effects.
- id: overlapperc
type: double
type: float
default: 0
doc: >-
If >0, percentage of time window to be added to the left and right side
......
#!/usr/bin/env cwl-runner
class: Workflow
cwlVersion: v1.0
id: applybeam_step_generator
inputs:
- id: steps
type: Any[]?
default: []
- id: step_name
type: string
doc: unique name for the step
default: applybeam
- id: direction
type: string?
doc: A RA/Dec value specifying in what direction to correct the beam. See phaseshift.phasecenter for syntax. If empty, the beam is corrected in the direction of the current phase center.
- id: onebeamperpatch
type: string?
doc: |
Compute the beam only for the center of each patch
(saves computation time, but you should set this to false for large
patches).
In the ApplyBeam step, this setting does not make sense
(but it does if the applybeam is part of predict, ddecal, gaincal,
h5parmpredict, etc.).
Generally, FALSE is the right setting for this option.
The default has changed to false in a recent (Nov 2018) version.
- id: usechannelfreq
type: boolean?
doc: |
Compute the beam for each channel of the measurement set separately.
This is useful for merged / concatenated measurement sets.
For raw LOFAR data you should set it to false,
so that the beam will be formed as in the station hardware.
Also, setting it to false is faster.
- id: updateweights
type: boolean?
doc: |
Update the weights column, in a way consistent with
the weights being inverse proportional to the autocorrelations
(e.g. if 'autoweights' was used before).
- id: invert
type: boolean?
doc: |
Invert the corrections, to correct the data. Default is true.
If you want to corrupt the data, set it to 'false'
- id: beammode
type:
type: enum
symbols:
- array_factor
- element
- default
doc: |
Beam mode to apply, can be “array_factor”, “element” or “default”.
Default is to apply both the element beam and the array factor.
outputs:
- id: augmented_steps
outputSource:
- DP3_GenericStep/augmented_steps
type: Any[]
steps:
- id: DP3_GenericStep
in:
- id: step_type
default: 'applybeam'
- id: step_id
source: step_name
- id: steps
source: steps
- id: parameters
valueFrom: $(inputs)
- id: direction
source: direction
- id: onebeamperpatch
source: onebeamperpatch
- id: usechannelfreq
source: usechannelfreq
- id: updateweights
source: updateweights
- id: invert
source: invert
- id: beammode
source: beammode
#-------------------------------
out:
- augmented_steps
run: ../steps/DP3.GenericStep.cwl
requirements:
- class: StepInputExpressionRequirement
- class: InlineJavascriptRequirement
- class: MultipleInputFeatureRequirement
......@@ -7,8 +7,8 @@ $namespaces:
inputs:
- id: msout_name
type: string
'sbg:x': -425.39886474609375
'sbg:y': 127
'sbg:x': -418
'sbg:y': 187
- id: msin
type: Directory
'sbg:x': -254.39886474609375
......@@ -20,8 +20,16 @@ inputs:
- id: autoweight
type: boolean
default: true
'sbg:x': -113
'sbg:y': 290
'sbg:x': -42
'sbg:y': 305
- id: output_column
type: string?
'sbg:x': -513
'sbg:y': 104
- id: input_column
type: string?
'sbg:x': -248.0078125
'sbg:y': 363
outputs:
- id: secondary_output_files
outputSource:
......@@ -76,6 +84,10 @@ steps:
source: generic_step/output_directory_names
- id: autoweight
source: autoweight
- id: output_column
source: output_column
- id: input_column
source: input_column
out:
- id: msout
- id: secondary_output_files
......
......@@ -37,11 +37,40 @@ inputs:
- id: output_directory_names
type: Any
doc: Expected output file names
- id: autoweight
- default: true
id: autoweight
type: boolean
default: true
inputBinding:
prefix: -msin.autoweight=True
position: 0
prefix: '-msin.autoweight=True'
- default: "dysco"
id: storagemanager
type: string
inputBinding:
position: 0
prefix: '-msout.storagemanager='
separate: false
- default: ''
id: baseline
type: string
inputBinding:
position: 0
prefix: '-msin.baseline='
separate: false
- default: DATA
id: output_column
type: string?
inputBinding:
position: 0
prefix: '-msout.datacolumn='
separate: false
- default: DATA
id: input_column
type: string?
inputBinding:
position: 0
prefix: '-msin.datacolumn='
separate: false
outputs:
- id: msout
doc: Output Measurement Set
......
#!/usr/bin/env cwl-runner
class: CommandLineTool
cwlVersion: v1.0
id: losoto_clocktec
$namespaces:
lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
doc: |
Separate phase solutions into Clock and TEC.
The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'tec3rd'.
requirements:
InlineJavascriptRequirement:
expressionLib:
- { $include: utils.js}
InitialWorkDirRequirement:
listing:
- entryname: 'parset.config'
entry: $(get_losoto_config('CLOCKTEC').join('\n'))
- entryname: $(inputs.input_h5parm.basename)
entry: $(inputs.input_h5parm)
writable: true
baseCommand: "losoto"
arguments:
- $(inputs.input_h5parm.basename)
- parset.config
hints:
DockerRequirement:
dockerPull: lofareosc/lofar-pipeline-ci:latest
inputs:
- id: input_h5parm
type: File
format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
- id: clocksoltabOut
doc: output soltab name for clock
type: string?
- id: tecsoltabOut
doc: output soltab name for tec
type: string?
- id: offsetsoltabOut
doc: output soltab name for phase offset
type: string?
- id: tec3rdsoltabOut
doc: output soltab name for tec3rd offset
type: string?
- id: flagBadChannels
type: boolean?
doc: Detect and remove bad channel before fitting, by default True.
- id: flagCut
type: float?
- id: chi2cut
type: float?
- id: combinePol
type: boolean?
doc: |
Find a combined polarization solution, by default False.
- id: removePhaseWraps
type: boolean?
doc: |
Detect and remove phase wraps, by default True.
- id: fit3rdorder
type: boolean?
doc: |
Fit a 3rd order ionospheric ocmponent (usefult <40 MHz). By default False.
- id: circular
type: boolean?
doc: |
Assume circular polarization with FR not removed. By default False.
- id: reverse
type: boolean?
doc:
Reverse the time axis. By default False.
- id: invertOffset
type: boolean?
doc: |
Invert (reverse the sign of) the phase offsets. By default False. Set to True
if you want to use them with the residuals operation.
outputs:
- id: output_h5parm
type: File
format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
$schema:
- https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
......@@ -5,8 +5,6 @@ $namespaces:
id: applycal
baseCommand:
- DPPP
arguments:
- steps=[applycal]
inputs:
- id: msin
type: Directory
......@@ -25,41 +23,56 @@ inputs:
doc: Input data Column
- id: parmdb
type: File
doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
inputBinding:
prefix: applycal.parmdb
- id: correction
default: gain
type:
type: enum
symbols:
- gain
- tec
- clock
- rotationangle
- rotation
- scalarphase
- scalaramplitude
- rotationmeasure
- fulljones
doc: |
Type of correction to perform. When using H5Parm, this is for now the name of the soltab; the type will be deduced from the metadata in that soltab, except for full Jones, in which case correction should be 'fulljones'.
position: 0
prefix: applycal.parmdb=
separate: false
doc: >-
Path of parmdb in which the parameters are stored. This can also be an
H5Parm file, in that case the filename has to end in '.h5'
- id: msout_datacolumn
type: string
inputBinding:
position: 0
prefix: msout.datacolumn=
separate: false
doc: Output data column
- default: gain
id: correction
type: string
inputBinding:
position: 0
prefix: applycal.correction=
separate: false
doc: >
Type of correction to perform. When using H5Parm, this is for now the name
of the soltab; the type will be deduced from the metadata in that soltab,
except for full Jones, in which case correction should be 'fulljones'.
- id: storagemanager
type: string
default: dysco
inputBinding
prefix: msout.storagemanager=
- id: updateweights
type: boolean?
inputBinding:
prefix: applycal.correction=
position: 0
prefix: -applycal.updateweights=True
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
glob: $(inputs.msin.basename)
glob: $(inputs.msin.basename)
arguments:
- 'steps=[applycal]'
- msout=.
requirements:
- class: InitialWorkDirRequirement
listing:
- entry: $(inputs.msin)
writable: true
- entry: $(inputs.msin)
writable: true
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
dockerPull: lofareosc/lofar-pipeline-ci:latest
dockerPull: 'lofareosc/lofar-pipeline-ci:latest'
......@@ -2,7 +2,7 @@
class: CommandLineTool
cwlVersion: v1.0
id: ddecal
id: calib_rot_diag
baseCommand: [DPPP]
requirements:
......@@ -14,7 +14,6 @@ hints:
arguments:
- steps=[ddecal]
- ddecal.mode=rotation+diagonal
- ddecal.uvlambdamin=300
- ddecal.maxiter=50
- ddecal.nchan=1
......@@ -30,38 +29,38 @@ inputs:
type: Directory?
doc: Input Measurement Set
inputBinding:
prefix: msin=
separate: false
prefix: msin=
separate: false
- id: msin_datacolumn
type: string
default: DATA
doc: Input data Column
inputBinding:
prefix: msin.datacolumn=
separate: false
prefix: msin.datacolumn=
separate: false
- id: msin_modelcolum
type: string
default: MODEL_DATA
doc: Model data Column
inputBinding:
prefix: msin.modelcolumn=
separate: false
prefix: msin.modelcolumn=
separate: false
- id: output_name_h5parm
type: string
default: instrument.h5
inputBinding:
prefix: ddecal.h5parm=
separate: false
prefix: ddecal.h5parm=
separate: false
- id: msout_name
type: string
doc: Output Measurement Set
default: out.MS
inputBinding:
prefix: msout=
separate: false
prefix: msout=
separate: false
#--------------------
- id: propagate_solutions
......@@ -91,6 +90,31 @@ inputs:
and mode=rotation+diagonal.
inputBinding:
prefix: flagdivergedonly=True
- id: storagemanager
type: string
default: dysco
inputBinding
prefix: msout.storagemanager=
- id: mode
type:
type: enum
symbols:
- scalarcomplexgain
- scalarphase
- scalaramplitude
- tec
- tecandphase
- fulljones
- diagonal
- phaseonly
- amplitudeonly
- rotation
- rotation+diagonal
doc: |
Type of constraint to apply. Options are
inputBinding:
prefix: ddecal.mode=
separate: false
outputs:
- id: msout
doc: Output Measurement Set
......
......@@ -51,6 +51,11 @@ inputs:
inputBinding:
prefix: gaincal.usebeammodel=true
separate: false
- id: storagemanager
type: string
default: dysco
inputBinding
prefix: msout.storagemanager=
- id: solint
type: int
default: 1
......
......@@ -21,6 +21,11 @@ inputs:
prefix: msin.datacolumn=
separate: false
doc: Input data Column
- id: storagemanager
type: string
default: dysco
inputBinding
prefix: msout.storagemanager=
- default: MODEL_DATA
id: msout_datacolumn
type: string
......
......@@ -4,5 +4,6 @@
"path": "/data/L570745_SB001_uv_MODEL.MS"
},
"msin_datacolumn": "DATA",
"usemodelcolumn": true
"usemodelcolumn": true,
"mode": "diagonal"
}
{
"input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
"soltab": "sol000/phase000",
"ncpu": 1
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment