From 49188986e636a39a97646f0acb3ec318cebdec1e Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Tue, 10 Mar 2020 10:35:58 +0100
Subject: [PATCH] Squashed 'lofar-cwl/' changes from 56850c7..110e4b1
110e4b1 Fix a space to many in shell parameters
42cbd7c Fix type
c5ab49c Add storagemanager parameter
1f48b15 Add input/output column
50bef37 Add parameters to step
0829830 Merge branch 'losoto_clocktec' into 'master'
170940e Add LoSoTo clocktec step
38b40fb Add parameters
83d14c7 Fix test
9280bde Add parameter to select the calibration mode
28d4fe1 Add parameter
13e3770 Merge
b2961ca Add output column parameter
9361075 Merge branch 'ApplyBeam' into 'master'
eb7281c Add ApplyBeam
faf58af Add baseline parameter to DPPP
e553b49 Fix applycal
a178c7e Rename ddecal
git-subtree-dir: lofar-cwl
git-subtree-split: 110e4b15beb8d5457746f7f1ce2f689c211f5336
---
.gitlab-ci.yml | 7 ++
steps/DP3.AOFlaggerStepGenerator.cwl | 4 +-
steps/DP3.ApplyBeamStepGenerator.cwl | 104 +++++++++++++++++++++++++++
steps/DP3.Execute.cwl | 20 ++++--
steps/DPPP.cwl | 35 ++++++++-
steps/LoSoTo.ClockTec.cwl | 95 ++++++++++++++++++++++++
steps/applycal.cwl | 67 ++++++++++-------
steps/ddecal.cwl | 48 +++++++++----
steps/gaincal.cwl | 5 ++
steps/predict.cwl | 5 ++
test_jobs/ddecal.json | 3 +-
test_jobs/losoto_clocktec.json | 5 ++
12 files changed, 349 insertions(+), 49 deletions(-)
create mode 100755 steps/DP3.ApplyBeamStepGenerator.cwl
create mode 100644 steps/LoSoTo.ClockTec.cwl
create mode 100644 test_jobs/losoto_clocktec.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 00f9bb80..09d6ebc7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -132,6 +132,13 @@ losoto_clip:
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json
+
+losoto_clocktec:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.ClockTec.cwl test_jobs/losoto_clocktec.json
+
losoto_flagextend:
stage: test_steps
allow_failure: true
diff --git a/steps/DP3.AOFlaggerStepGenerator.cwl b/steps/DP3.AOFlaggerStepGenerator.cwl
index 2a70f602..cc270197 100755
--- a/steps/DP3.AOFlaggerStepGenerator.cwl
+++ b/steps/DP3.AOFlaggerStepGenerator.cwl
@@ -42,7 +42,7 @@ inputs:
available memory too much affects flagging accuracy; in general try to use
at least 10 GB of memory.
- id: memorymax
- type: double
+ type: float
default: 0
doc: >-
Maximum amount of memory (in GB) to use. ⇐0 means no maximum. As stated
@@ -56,7 +56,7 @@ inputs:
memory to use. Note that the time window can be extended with an overlap
on the left and right side to minimize possible boundary effects.
- id: overlapperc
- type: double
+ type: float
default: 0
doc: >-
If >0, percentage of time window to be added to the left and right side
diff --git a/steps/DP3.ApplyBeamStepGenerator.cwl b/steps/DP3.ApplyBeamStepGenerator.cwl
new file mode 100755
index 00000000..3e0c9fde
--- /dev/null
+++ b/steps/DP3.ApplyBeamStepGenerator.cwl
@@ -0,0 +1,104 @@
+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: applybeam_step_generator
+
+inputs:
+ - id: steps
+ type: Any[]?
+ default: []
+ - id: step_name
+ type: string
+ doc: unique name for the step
+ default: applybeam
+
+ - id: direction
+ type: string?
+ doc: A RA/Dec value specifying in what direction to correct the beam. See phaseshift.phasecenter for syntax. If empty, the beam is corrected in the direction of the current phase center.
+ - id: onebeamperpatch
+ type: string?
+ doc: |
+ Compute the beam only for the center of each patch
+ (saves computation time, but you should set this to false for large
+ patches).
+ In the ApplyBeam step, this setting does not make sense
+ (but it does if the applybeam is part of predict, ddecal, gaincal,
+ h5parmpredict, etc.).
+ Generally, FALSE is the right setting for this option.
+ The default has changed to false in a recent (Nov 2018) version.
+ - id: usechannelfreq
+ type: boolean?
+ doc: |
+ Compute the beam for each channel of the measurement set separately.
+ This is useful for merged / concatenated measurement sets.
+ For raw LOFAR data you should set it to false,
+ so that the beam will be formed as in the station hardware.
+ Also, setting it to false is faster.
+
+ - id: updateweights
+ type: boolean?
+ doc: |
+ Update the weights column, in a way consistent with
+ the weights being inverse proportional to the autocorrelations
+ (e.g. if 'autoweights' was used before).
+
+ - id: invert
+ type: boolean?
+ doc: |
+ Invert the corrections, to correct the data. Default is true.
+ If you want to corrupt the data, set it to 'false'
+
+ - id: beammode
+
+ type:
+ type: enum
+ symbols:
+ - array_factor
+ - element
+ - default
+ doc: |
+ Beam mode to apply, can be “array_factor”, “element” or “default”.
+ Default is to apply both the element beam and the array factor.
+
+
+outputs:
+ - id: augmented_steps
+ outputSource:
+ - DP3_GenericStep/augmented_steps
+ type: Any[]
+steps:
+ - id: DP3_GenericStep
+ in:
+ - id: step_type
+ default: 'applybeam'
+ - id: step_id
+ source: step_name
+ - id: steps
+ source: steps
+ - id: parameters
+ valueFrom: $(inputs)
+
+ - id: direction
+ source: direction
+ - id: onebeamperpatch
+ source: onebeamperpatch
+ - id: usechannelfreq
+ source: usechannelfreq
+ - id: updateweights
+ source: updateweights
+ - id: invert
+ source: invert
+ - id: beammode
+ source: beammode
+
+
+#-------------------------------
+
+ out:
+ - augmented_steps
+ run: ../steps/DP3.GenericStep.cwl
+requirements:
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.Execute.cwl b/steps/DP3.Execute.cwl
index e1f3bbfd..c46e5066 100644
--- a/steps/DP3.Execute.cwl
+++ b/steps/DP3.Execute.cwl
@@ -7,8 +7,8 @@ $namespaces:
inputs:
- id: msout_name
type: string
- 'sbg:x': -425.39886474609375
- 'sbg:y': 127
+ 'sbg:x': -418
+ 'sbg:y': 187
- id: msin
type: Directory
'sbg:x': -254.39886474609375
@@ -20,8 +20,16 @@ inputs:
- id: autoweight
type: boolean
default: true
- 'sbg:x': -113
- 'sbg:y': 290
+ 'sbg:x': -42
+ 'sbg:y': 305
+ - id: output_column
+ type: string?
+ 'sbg:x': -513
+ 'sbg:y': 104
+ - id: input_column
+ type: string?
+ 'sbg:x': -248.0078125
+ 'sbg:y': 363
outputs:
- id: secondary_output_files
outputSource:
@@ -76,6 +84,10 @@ steps:
source: generic_step/output_directory_names
- id: autoweight
source: autoweight
+ - id: output_column
+ source: output_column
+ - id: input_column
+ source: input_column
out:
- id: msout
- id: secondary_output_files
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
index 17005aba..b7f2dd56 100755
--- a/steps/DPPP.cwl
+++ b/steps/DPPP.cwl
@@ -37,11 +37,40 @@ inputs:
- id: output_directory_names
type: Any
doc: Expected output file names
- - id: autoweight
+ - default: true
+ id: autoweight
type: boolean
- default: true
inputBinding:
- prefix: -msin.autoweight=True
+ position: 0
+ prefix: '-msin.autoweight=True'
+ - default: "dysco"
+ id: storagemanager
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-msout.storagemanager='
+ separate: false
+ - default: ''
+ id: baseline
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-msin.baseline='
+ separate: false
+ - default: DATA
+ id: output_column
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: '-msout.datacolumn='
+ separate: false
+ - default: DATA
+ id: input_column
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: '-msin.datacolumn='
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
diff --git a/steps/LoSoTo.ClockTec.cwl b/steps/LoSoTo.ClockTec.cwl
new file mode 100644
index 00000000..a328b6be
--- /dev/null
+++ b/steps/LoSoTo.ClockTec.cwl
@@ -0,0 +1,95 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_clocktec
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+ Separate phase solutions into Clock and TEC.
+ The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'tec3rd'.
+
+requirements:
+ InlineJavascriptRequirement:
+ expressionLib:
+ - { $include: utils.js}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: $(get_losoto_config('CLOCKTEC').join('\n'))
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/lofar-pipeline-ci:latest
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: clocksoltabOut
+ doc: output soltab name for clock
+ type: string?
+ - id: tecsoltabOut
+ doc: output soltab name for tec
+ type: string?
+ - id: offsetsoltabOut
+ doc: output soltab name for phase offset
+ type: string?
+ - id: tec3rdsoltabOut
+ doc: output soltab name for tec3rd offset
+ type: string?
+ - id: flagBadChannels
+ type: boolean?
+ doc: Detect and remove bad channel before fitting, by default True.
+ - id: flagCut
+ type: float?
+ - id: chi2cut
+ type: float?
+ - id: combinePol
+ type: boolean?
+ doc: |
+ Find a combined polarization solution, by default False.
+ - id: removePhaseWraps
+ type: boolean?
+ doc: |
+ Detect and remove phase wraps, by default True.
+ - id: fit3rdorder
+ type: boolean?
+ doc: |
+ Fit a 3rd order ionospheric ocmponent (usefult <40 MHz). By default False.
+ - id: circular
+ type: boolean?
+ doc: |
+ Assume circular polarization with FR not removed. By default False.
+ - id: reverse
+ type: boolean?
+ doc:
+ Reverse the time axis. By default False.
+ - id: invertOffset
+ type: boolean?
+ doc: |
+ Invert (reverse the sign of) the phase offsets. By default False. Set to True
+ if you want to use them with the residuals operation.
+
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/applycal.cwl b/steps/applycal.cwl
index 08b76e94..5a62a211 100644
--- a/steps/applycal.cwl
+++ b/steps/applycal.cwl
@@ -5,8 +5,6 @@ $namespaces:
id: applycal
baseCommand:
- DPPP
-arguments:
- - steps=[applycal]
inputs:
- id: msin
type: Directory
@@ -25,41 +23,56 @@ inputs:
doc: Input data Column
- id: parmdb
type: File
- doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
inputBinding:
- prefix: applycal.parmdb
- - id: correction
- default: gain
- type:
- type: enum
- symbols:
- - gain
- - tec
- - clock
- - rotationangle
- - rotation
- - scalarphase
- - scalaramplitude
- - rotationmeasure
- - fulljones
- doc: |
- Type of correction to perform. When using H5Parm, this is for now the name of the soltab; the type will be deduced from the metadata in that soltab, except for full Jones, in which case correction should be 'fulljones'.
+ position: 0
+ prefix: applycal.parmdb=
+ separate: false
+ doc: >-
+ Path of parmdb in which the parameters are stored. This can also be an
+ H5Parm file, in that case the filename has to end in '.h5'
+ - id: msout_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msout.datacolumn=
+ separate: false
+ doc: Output data column
+ - default: gain
+ id: correction
+ type: string
+ inputBinding:
+ position: 0
+ prefix: applycal.correction=
+ separate: false
+ doc: >
+ Type of correction to perform. When using H5Parm, this is for now the name
+ of the soltab; the type will be deduced from the metadata in that soltab,
+ except for full Jones, in which case correction should be 'fulljones'.
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
+ - id: updateweights
+ type: boolean?
inputBinding:
- prefix: applycal.correction=
-
+ position: 0
+ prefix: -applycal.updateweights=True
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
- glob: $(inputs.msin.basename)
-
+ glob: $(inputs.msin.basename)
+arguments:
+ - 'steps=[applycal]'
+ - msout=.
requirements:
- class: InitialWorkDirRequirement
listing:
- - entry: $(inputs.msin)
- writable: true
+ - entry: $(inputs.msin)
+ writable: true
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
- dockerPull: lofareosc/lofar-pipeline-ci:latest
+ dockerPull: 'lofareosc/lofar-pipeline-ci:latest'
diff --git a/steps/ddecal.cwl b/steps/ddecal.cwl
index 7268170c..3eddf9ef 100644
--- a/steps/ddecal.cwl
+++ b/steps/ddecal.cwl
@@ -2,7 +2,7 @@
class: CommandLineTool
cwlVersion: v1.0
-id: ddecal
+id: calib_rot_diag
baseCommand: [DPPP]
requirements:
@@ -14,7 +14,6 @@ hints:
arguments:
- steps=[ddecal]
- - ddecal.mode=rotation+diagonal
- ddecal.uvlambdamin=300
- ddecal.maxiter=50
- ddecal.nchan=1
@@ -30,38 +29,38 @@ inputs:
type: Directory?
doc: Input Measurement Set
inputBinding:
- prefix: msin=
- separate: false
+ prefix: msin=
+ separate: false
- id: msin_datacolumn
type: string
default: DATA
doc: Input data Column
inputBinding:
- prefix: msin.datacolumn=
- separate: false
+ prefix: msin.datacolumn=
+ separate: false
- id: msin_modelcolum
type: string
default: MODEL_DATA
doc: Model data Column
inputBinding:
- prefix: msin.modelcolumn=
- separate: false
+ prefix: msin.modelcolumn=
+ separate: false
- id: output_name_h5parm
type: string
default: instrument.h5
inputBinding:
- prefix: ddecal.h5parm=
- separate: false
+ prefix: ddecal.h5parm=
+ separate: false
- id: msout_name
type: string
doc: Output Measurement Set
default: out.MS
inputBinding:
- prefix: msout=
- separate: false
+ prefix: msout=
+ separate: false
#--------------------
- id: propagate_solutions
@@ -91,6 +90,31 @@ inputs:
and mode=rotation+diagonal.
inputBinding:
prefix: flagdivergedonly=True
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
+ - id: mode
+ type:
+ type: enum
+ symbols:
+ - scalarcomplexgain
+ - scalarphase
+ - scalaramplitude
+ - tec
+ - tecandphase
+ - fulljones
+ - diagonal
+ - phaseonly
+ - amplitudeonly
+ - rotation
+ - rotation+diagonal
+ doc: |
+ Type of constraint to apply. Options are
+ inputBinding:
+ prefix: ddecal.mode=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
diff --git a/steps/gaincal.cwl b/steps/gaincal.cwl
index e017373e..d57b7128 100644
--- a/steps/gaincal.cwl
+++ b/steps/gaincal.cwl
@@ -51,6 +51,11 @@ inputs:
inputBinding:
prefix: gaincal.usebeammodel=true
separate: false
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
- id: solint
type: int
default: 1
diff --git a/steps/predict.cwl b/steps/predict.cwl
index d9facbb8..d785f108 100644
--- a/steps/predict.cwl
+++ b/steps/predict.cwl
@@ -21,6 +21,11 @@ inputs:
prefix: msin.datacolumn=
separate: false
doc: Input data Column
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
- default: MODEL_DATA
id: msout_datacolumn
type: string
diff --git a/test_jobs/ddecal.json b/test_jobs/ddecal.json
index 13a89c1f..66f549e1 100644
--- a/test_jobs/ddecal.json
+++ b/test_jobs/ddecal.json
@@ -4,5 +4,6 @@
"path": "/data/L570745_SB001_uv_MODEL.MS"
},
"msin_datacolumn": "DATA",
- "usemodelcolumn": true
+ "usemodelcolumn": true,
+ "mode": "diagonal"
}
diff --git a/test_jobs/losoto_clocktec.json b/test_jobs/losoto_clocktec.json
new file mode 100644
index 00000000..3632d7f1
--- /dev/null
+++ b/test_jobs/losoto_clocktec.json
@@ -0,0 +1,5 @@
+{
+ "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+ "soltab": "sol000/phase000",
+ "ncpu": 1
+}
--
GitLab