diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 00f9bb80b6bd773cc6aad4b5d4ce99ad77e1c8a9..09d6ebc7e831c7368b9f8869052a226f5b43b84a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -132,6 +132,13 @@ losoto_clip:
allow_failure: true
script:
- cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json
+
+losoto_clocktec:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.ClockTec.cwl test_jobs/losoto_clocktec.json
+
losoto_flagextend:
stage: test_steps
allow_failure: true
diff --git a/steps/DP3.AOFlaggerStepGenerator.cwl b/steps/DP3.AOFlaggerStepGenerator.cwl
index 2a70f602771d7e5e669ba46ee395550807bb379d..cc2701971c822351d3f40a916eb6978749852fe4 100755
--- a/steps/DP3.AOFlaggerStepGenerator.cwl
+++ b/steps/DP3.AOFlaggerStepGenerator.cwl
@@ -42,7 +42,7 @@ inputs:
available memory too much affects flagging accuracy; in general try to use
at least 10 GB of memory.
- id: memorymax
- type: double
+ type: float
default: 0
doc: >-
Maximum amount of memory (in GB) to use. â‡0 means no maximum. As stated
@@ -56,7 +56,7 @@ inputs:
memory to use. Note that the time window can be extended with an overlap
on the left and right side to minimize possible boundary effects.
- id: overlapperc
- type: double
+ type: float
default: 0
doc: >-
If >0, percentage of time window to be added to the left and right side
diff --git a/steps/DP3.ApplyBeamStepGenerator.cwl b/steps/DP3.ApplyBeamStepGenerator.cwl
new file mode 100755
index 0000000000000000000000000000000000000000..3e0c9fde9da2eb2b577339fc8bfde8c36e00c6c8
--- /dev/null
+++ b/steps/DP3.ApplyBeamStepGenerator.cwl
@@ -0,0 +1,104 @@
+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: applybeam_step_generator
+
+inputs:
+ - id: steps
+ type: Any[]?
+ default: []
+ - id: step_name
+ type: string
+ doc: unique name for the step
+ default: applybeam
+
+ - id: direction
+ type: string?
+ doc: A RA/Dec value specifying in what direction to correct the beam. See phaseshift.phasecenter for syntax. If empty, the beam is corrected in the direction of the current phase center.
+ - id: onebeamperpatch
+ type: string?
+ doc: |
+ Compute the beam only for the center of each patch
+ (saves computation time, but you should set this to false for large
+ patches).
+ In the ApplyBeam step, this setting does not make sense
+ (but it does if the applybeam is part of predict, ddecal, gaincal,
+ h5parmpredict, etc.).
+ Generally, FALSE is the right setting for this option.
+ The default has changed to false in a recent (Nov 2018) version.
+ - id: usechannelfreq
+ type: boolean?
+ doc: |
+ Compute the beam for each channel of the measurement set separately.
+ This is useful for merged / concatenated measurement sets.
+ For raw LOFAR data you should set it to false,
+ so that the beam will be formed as in the station hardware.
+ Also, setting it to false is faster.
+
+ - id: updateweights
+ type: boolean?
+ doc: |
+ Update the weights column, in a way consistent with
+ the weights being inverse proportional to the autocorrelations
+ (e.g. if 'autoweights' was used before).
+
+ - id: invert
+ type: boolean?
+ doc: |
+ Invert the corrections, to correct the data. Default is true.
+ If you want to corrupt the data, set it to 'false'
+
+ - id: beammode
+
+ type:
+ type: enum
+ symbols:
+ - array_factor
+ - element
+ - default
+ doc: |
+ Beam mode to apply, can be “array_factorâ€, “element†or “defaultâ€.
+ Default is to apply both the element beam and the array factor.
+
+
+outputs:
+ - id: augmented_steps
+ outputSource:
+ - DP3_GenericStep/augmented_steps
+ type: Any[]
+steps:
+ - id: DP3_GenericStep
+ in:
+ - id: step_type
+ default: 'applybeam'
+ - id: step_id
+ source: step_name
+ - id: steps
+ source: steps
+ - id: parameters
+ valueFrom: $(inputs)
+
+ - id: direction
+ source: direction
+ - id: onebeamperpatch
+ source: onebeamperpatch
+ - id: usechannelfreq
+ source: usechannelfreq
+ - id: updateweights
+ source: updateweights
+ - id: invert
+ source: invert
+ - id: beammode
+ source: beammode
+
+
+#-------------------------------
+
+ out:
+ - augmented_steps
+ run: ../steps/DP3.GenericStep.cwl
+requirements:
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/steps/DP3.Execute.cwl b/steps/DP3.Execute.cwl
index e1f3bbfd445ec81cfb820afd4fc8a52800d402be..c46e50662b5ee48c157f8f1e2af0e8b7b3ee3719 100644
--- a/steps/DP3.Execute.cwl
+++ b/steps/DP3.Execute.cwl
@@ -7,8 +7,8 @@ $namespaces:
inputs:
- id: msout_name
type: string
- 'sbg:x': -425.39886474609375
- 'sbg:y': 127
+ 'sbg:x': -418
+ 'sbg:y': 187
- id: msin
type: Directory
'sbg:x': -254.39886474609375
@@ -20,8 +20,16 @@ inputs:
- id: autoweight
type: boolean
default: true
- 'sbg:x': -113
- 'sbg:y': 290
+ 'sbg:x': -42
+ 'sbg:y': 305
+ - id: output_column
+ type: string?
+ 'sbg:x': -513
+ 'sbg:y': 104
+ - id: input_column
+ type: string?
+ 'sbg:x': -248.0078125
+ 'sbg:y': 363
outputs:
- id: secondary_output_files
outputSource:
@@ -76,6 +84,10 @@ steps:
source: generic_step/output_directory_names
- id: autoweight
source: autoweight
+ - id: output_column
+ source: output_column
+ - id: input_column
+ source: input_column
out:
- id: msout
- id: secondary_output_files
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
index 17005aba9681c9e66fb293bd574331935a4cedca..b7f2dd56a92913e3adbbcea5370d4a3b978d2a98 100755
--- a/steps/DPPP.cwl
+++ b/steps/DPPP.cwl
@@ -37,11 +37,40 @@ inputs:
- id: output_directory_names
type: Any
doc: Expected output file names
- - id: autoweight
+ - default: true
+ id: autoweight
type: boolean
- default: true
inputBinding:
- prefix: -msin.autoweight=True
+ position: 0
+ prefix: '-msin.autoweight=True'
+ - default: "dysco"
+ id: storagemanager
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-msout.storagemanager='
+ separate: false
+ - default: ''
+ id: baseline
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-msin.baseline='
+ separate: false
+ - default: DATA
+ id: output_column
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: '-msout.datacolumn='
+ separate: false
+ - default: DATA
+ id: input_column
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: '-msin.datacolumn='
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
diff --git a/steps/LoSoTo.ClockTec.cwl b/steps/LoSoTo.ClockTec.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..a328b6bef5d6e5fe25a636b50417220397c5f460
--- /dev/null
+++ b/steps/LoSoTo.ClockTec.cwl
@@ -0,0 +1,95 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_clocktec
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+ Separate phase solutions into Clock and TEC.
+ The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'tec3rd'.
+
+requirements:
+ InlineJavascriptRequirement:
+ expressionLib:
+ - { $include: utils.js}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: $(get_losoto_config('CLOCKTEC').join('\n'))
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/lofar-pipeline-ci:latest
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: clocksoltabOut
+ doc: output soltab name for clock
+ type: string?
+ - id: tecsoltabOut
+ doc: output soltab name for tec
+ type: string?
+ - id: offsetsoltabOut
+ doc: output soltab name for phase offset
+ type: string?
+ - id: tec3rdsoltabOut
+ doc: output soltab name for tec3rd offset
+ type: string?
+ - id: flagBadChannels
+ type: boolean?
+ doc: Detect and remove bad channel before fitting, by default True.
+ - id: flagCut
+ type: float?
+ - id: chi2cut
+ type: float?
+ - id: combinePol
+ type: boolean?
+ doc: |
+ Find a combined polarization solution, by default False.
+ - id: removePhaseWraps
+ type: boolean?
+ doc: |
+ Detect and remove phase wraps, by default True.
+ - id: fit3rdorder
+ type: boolean?
+ doc: |
+ Fit a 3rd order ionospheric ocmponent (usefult <40 MHz). By default False.
+ - id: circular
+ type: boolean?
+ doc: |
+ Assume circular polarization with FR not removed. By default False.
+ - id: reverse
+ type: boolean?
+ doc:
+ Reverse the time axis. By default False.
+ - id: invertOffset
+ type: boolean?
+ doc: |
+ Invert (reverse the sign of) the phase offsets. By default False. Set to True
+ if you want to use them with the residuals operation.
+
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/applycal.cwl b/steps/applycal.cwl
index 08b76e9418d1397f798917abcac4f31a6397050b..5a62a2119adb8792c0d86d1708cf717757672cf9 100644
--- a/steps/applycal.cwl
+++ b/steps/applycal.cwl
@@ -5,8 +5,6 @@ $namespaces:
id: applycal
baseCommand:
- DPPP
-arguments:
- - steps=[applycal]
inputs:
- id: msin
type: Directory
@@ -25,41 +23,56 @@ inputs:
doc: Input data Column
- id: parmdb
type: File
- doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
inputBinding:
- prefix: applycal.parmdb
- - id: correction
- default: gain
- type:
- type: enum
- symbols:
- - gain
- - tec
- - clock
- - rotationangle
- - rotation
- - scalarphase
- - scalaramplitude
- - rotationmeasure
- - fulljones
- doc: |
- Type of correction to perform. When using H5Parm, this is for now the name of the soltab; the type will be deduced from the metadata in that soltab, except for full Jones, in which case correction should be 'fulljones'.
+ position: 0
+ prefix: applycal.parmdb=
+ separate: false
+ doc: >-
+ Path of parmdb in which the parameters are stored. This can also be an
+ H5Parm file, in that case the filename has to end in '.h5'
+ - id: msout_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msout.datacolumn=
+ separate: false
+ doc: Output data column
+ - default: gain
+ id: correction
+ type: string
+ inputBinding:
+ position: 0
+ prefix: applycal.correction=
+ separate: false
+ doc: >
+ Type of correction to perform. When using H5Parm, this is for now the name
+ of the soltab; the type will be deduced from the metadata in that soltab,
+ except for full Jones, in which case correction should be 'fulljones'.
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
+ - id: updateweights
+ type: boolean?
inputBinding:
- prefix: applycal.correction=
-
+ position: 0
+ prefix: -applycal.updateweights=True
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
- glob: $(inputs.msin.basename)
-
+ glob: $(inputs.msin.basename)
+arguments:
+ - 'steps=[applycal]'
+ - msout=.
requirements:
- class: InitialWorkDirRequirement
listing:
- - entry: $(inputs.msin)
- writable: true
+ - entry: $(inputs.msin)
+ writable: true
- class: InlineJavascriptRequirement
hints:
- class: DockerRequirement
- dockerPull: lofareosc/lofar-pipeline-ci:latest
+ dockerPull: 'lofareosc/lofar-pipeline-ci:latest'
diff --git a/steps/ddecal.cwl b/steps/ddecal.cwl
index 7268170ca6f13e8e9a95ad741f2d868827a3ea27..3eddf9ef67016053829920325c18bd45ceb4f470 100644
--- a/steps/ddecal.cwl
+++ b/steps/ddecal.cwl
@@ -2,7 +2,7 @@
class: CommandLineTool
cwlVersion: v1.0
-id: ddecal
+id: calib_rot_diag
baseCommand: [DPPP]
requirements:
@@ -14,7 +14,6 @@ hints:
arguments:
- steps=[ddecal]
- - ddecal.mode=rotation+diagonal
- ddecal.uvlambdamin=300
- ddecal.maxiter=50
- ddecal.nchan=1
@@ -30,38 +29,38 @@ inputs:
type: Directory?
doc: Input Measurement Set
inputBinding:
- prefix: msin=
- separate: false
+ prefix: msin=
+ separate: false
- id: msin_datacolumn
type: string
default: DATA
doc: Input data Column
inputBinding:
- prefix: msin.datacolumn=
- separate: false
+ prefix: msin.datacolumn=
+ separate: false
- id: msin_modelcolum
type: string
default: MODEL_DATA
doc: Model data Column
inputBinding:
- prefix: msin.modelcolumn=
- separate: false
+ prefix: msin.modelcolumn=
+ separate: false
- id: output_name_h5parm
type: string
default: instrument.h5
inputBinding:
- prefix: ddecal.h5parm=
- separate: false
+ prefix: ddecal.h5parm=
+ separate: false
- id: msout_name
type: string
doc: Output Measurement Set
default: out.MS
inputBinding:
- prefix: msout=
- separate: false
+ prefix: msout=
+ separate: false
#--------------------
- id: propagate_solutions
@@ -91,6 +90,31 @@ inputs:
and mode=rotation+diagonal.
inputBinding:
prefix: flagdivergedonly=True
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
+ - id: mode
+ type:
+ type: enum
+ symbols:
+ - scalarcomplexgain
+ - scalarphase
+ - scalaramplitude
+ - tec
+ - tecandphase
+ - fulljones
+ - diagonal
+ - phaseonly
+ - amplitudeonly
+ - rotation
+ - rotation+diagonal
+ doc: |
+ Type of constraint to apply. Options are
+ inputBinding:
+ prefix: ddecal.mode=
+ separate: false
outputs:
- id: msout
doc: Output Measurement Set
diff --git a/steps/gaincal.cwl b/steps/gaincal.cwl
index e017373e0bfe4a6ef68930d4f380dbfbc8cb65e7..d57b71284e6dbbb7c8068f505c3d0b00b97d1868 100644
--- a/steps/gaincal.cwl
+++ b/steps/gaincal.cwl
@@ -51,6 +51,11 @@ inputs:
inputBinding:
prefix: gaincal.usebeammodel=true
separate: false
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
- id: solint
type: int
default: 1
diff --git a/steps/predict.cwl b/steps/predict.cwl
index d9facbb83d6430dcde76caf49490e711afe14952..d785f10865aa797e555d7690459619a753ba4783 100644
--- a/steps/predict.cwl
+++ b/steps/predict.cwl
@@ -21,6 +21,11 @@ inputs:
prefix: msin.datacolumn=
separate: false
doc: Input data Column
+ - id: storagemanager
+ type: string
+ default: dysco
+ inputBinding
+ prefix: msout.storagemanager=
- default: MODEL_DATA
id: msout_datacolumn
type: string
diff --git a/test_jobs/ddecal.json b/test_jobs/ddecal.json
index 13a89c1f35db2da2d1be6966d5f213619360e514..66f549e1dea0814a6e94949caa8b12615853fe16 100644
--- a/test_jobs/ddecal.json
+++ b/test_jobs/ddecal.json
@@ -4,5 +4,6 @@
"path": "/data/L570745_SB001_uv_MODEL.MS"
},
"msin_datacolumn": "DATA",
- "usemodelcolumn": true
+ "usemodelcolumn": true,
+ "mode": "diagonal"
}
diff --git a/test_jobs/losoto_clocktec.json b/test_jobs/losoto_clocktec.json
new file mode 100644
index 0000000000000000000000000000000000000000..3632d7f13c8786836692e86a0a59a0cc1c9c96f0
--- /dev/null
+++ b/test_jobs/losoto_clocktec.json
@@ -0,0 +1,5 @@
+{
+ "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+ "soltab": "sol000/phase000",
+ "ncpu": 1
+}