Squashed 'lofar-cwl/' changes from 56850c7d..110e4b15

110e4b15 Fix a space to many in shell parameters
42cbd7c9 Fix type
c5ab49c3 Add storagemanager parameter
1f48b152 Add input/output column
50bef375 Add parameters to step
0829830d Merge branch 'losoto_clocktec' into 'master'
170940e2 Add LoSoTo clocktec step
38b40fb4 Add parameters
83d14c79 Fix test
9280bde0 Add parameter to select the calibration mode
28d4fe18 Add parameter
13e37700 Merge
b2961ca6 Add output column parameter
93610759 Merge branch 'ApplyBeam' into 'master'
eb7281cc Add ApplyBeam
faf58af0 Add baseline parameter to DPPP
e553b493 Fix applycal
a178c7e5 Rename ddecal

git-subtree-dir: lofar-cwl
git-subtree-split: 110e4b15

.gitlab-ci.yml

@@ -132,6 +132,13 @@ losoto_clip:
   allow_failure: true
   script:
     - cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json
+
+losoto_clocktec:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.ClockTec.cwl test_jobs/losoto_clocktec.json
+
 losoto_flagextend:
   stage: test_steps
   allow_failure: true

steps/DP3.AOFlaggerStepGenerator.cwl

@@ -42,7 +42,7 @@ inputs:
       available memory too much affects flagging accuracy; in general try to use
       at least 10 GB of memory.
   - id: memorymax
-    type: double
+    type: float
     default: 0
     doc: >-
       Maximum amount of memory (in GB) to use. ⇐0 means no maximum. As stated
@@ -56,7 +56,7 @@ inputs:
       memory to use. Note that the time window can be extended with an overlap
       on the left and right side to minimize possible boundary effects.
   - id: overlapperc
-    type: double
+    type: float
     default: 0
     doc: >-
       If >0, percentage of time window to be added to the left and right side

steps/DP3.ApplyBeamStepGenerator.cwl

+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: applybeam_step_generator
+
+inputs:
+  - id: steps
+    type: Any[]?
+    default: []
+  - id: step_name
+    type: string
+    doc: unique name for the step
+    default: applybeam
+
+  - id: direction
+    type: string?
+    doc: A RA/Dec value specifying in what direction to correct the beam. See phaseshift.phasecenter for syntax. If empty, the beam is corrected in the direction of the current phase center. 
+  - id: onebeamperpatch
+    type: string?
+    doc: |
+      Compute the beam only for the center of each patch
+      (saves computation time, but you should set this to false for large
+      patches). 
+      In the ApplyBeam step, this setting does not make sense 
+      (but it does if the applybeam is part of predict, ddecal, gaincal,
+      h5parmpredict, etc.). 
+      Generally, FALSE is the right setting for this option. 
+      The default has changed to false in a recent (Nov 2018) version. 
+  - id: usechannelfreq
+    type: boolean?
+    doc: |
+      Compute the beam for each channel of the measurement set separately.
+      This is useful for merged / concatenated measurement sets.
+      For raw LOFAR data you should set it to false,
+      so that the beam will be formed as in the station hardware.
+      Also, setting it to false is faster.
+      
+  - id: updateweights
+    type: boolean?
+    doc: |
+      Update the weights column, in a way consistent with
+      the weights being inverse proportional to the autocorrelations
+      (e.g. if 'autoweights' was used before).
+
+  - id: invert
+    type: boolean?
+    doc: |
+      Invert the corrections, to correct the data. Default is true.
+      If you want to corrupt the data, set it to 'false'
+     
+  - id: beammode
+    
+    type:
+      type: enum
+      symbols:
+       - array_factor
+       - element
+       - default
+    doc: |
+      Beam mode to apply, can be “array_factor”, “element” or “default”.
+      Default is to apply both the element beam and the array factor. 
+    
+    
+outputs:
+  - id: augmented_steps
+    outputSource:
+      - DP3_GenericStep/augmented_steps
+    type: Any[]
+steps:
+  - id: DP3_GenericStep
+    in:
+      - id: step_type
+        default: 'applybeam'
+      - id: step_id
+        source: step_name
+      - id: steps
+        source: steps
+      - id: parameters
+        valueFrom: $(inputs)
+
+      - id: direction
+        source: direction
+      - id: onebeamperpatch
+        source: onebeamperpatch
+      - id: usechannelfreq
+        source: usechannelfreq
+      - id: updateweights
+        source: updateweights
+      - id: invert
+        source: invert
+      - id: beammode
+        source: beammode
+
+
+#-------------------------------
+
+    out:
+      - augmented_steps
+    run: ../steps/DP3.GenericStep.cwl
+requirements:
+  - class: StepInputExpressionRequirement
+  - class: InlineJavascriptRequirement
+  - class: MultipleInputFeatureRequirement

steps/DP3.Execute.cwl

@@ -7,8 +7,8 @@ $namespaces:
 inputs:
   - id: msout_name
     type: string
-    'sbg:x': -425.39886474609375
-    'sbg:y': 127
+    'sbg:x': -418
+    'sbg:y': 187
   - id: msin
     type: Directory
     'sbg:x': -254.39886474609375
@@ -20,8 +20,16 @@ inputs:
   - id: autoweight
     type: boolean
     default: true
-    'sbg:x': -113
-    'sbg:y': 290
+    'sbg:x': -42
+    'sbg:y': 305
+  - id: output_column
+    type: string?
+    'sbg:x': -513
+    'sbg:y': 104
+  - id: input_column
+    type: string?
+    'sbg:x': -248.0078125
+    'sbg:y': 363
 outputs:
   - id: secondary_output_files
     outputSource:
@@ -76,6 +84,10 @@ steps:
         source: generic_step/output_directory_names
       - id: autoweight
         source: autoweight
+      - id: output_column
+        source: output_column
+      - id: input_column
+        source: input_column
     out:
       - id: msout
       - id: secondary_output_files

steps/DPPP.cwl

@@ -37,11 +37,40 @@ inputs:
   - id: output_directory_names
     type: Any
     doc: Expected output file names
-  - id: autoweight
+  - default: true
+    id: autoweight
     type: boolean
-    default: true
     inputBinding:
-      prefix: -msin.autoweight=True
+      position: 0
+      prefix: '-msin.autoweight=True'
+  - default: "dysco"
+    id: storagemanager
+    type: string
+    inputBinding:
+      position: 0
+      prefix: '-msout.storagemanager='
+      separate: false
+  - default: ''
+    id: baseline
+    type: string
+    inputBinding:
+      position: 0
+      prefix: '-msin.baseline='
+      separate: false
+  - default: DATA
+    id: output_column
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '-msout.datacolumn='
+      separate: false
+  - default: DATA
+    id: input_column
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: '-msin.datacolumn='
+      separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set

steps/LoSoTo.ClockTec.cwl

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_clocktec
+
+$namespaces:
+  lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+  Separate phase solutions into Clock and TEC.
+  The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'tec3rd'.
+
+requirements:
+  InlineJavascriptRequirement:
+    expressionLib:
+      - { $include: utils.js}
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: 'parset.config'
+        entry: $(get_losoto_config('CLOCKTEC').join('\n'))
+
+      - entryname: $(inputs.input_h5parm.basename)
+        entry: $(inputs.input_h5parm)
+        writable: true
+
+baseCommand: "losoto"
+
+arguments:
+  - $(inputs.input_h5parm.basename)
+  - parset.config
+
+hints:
+  DockerRequirement:
+    dockerPull: lofareosc/lofar-pipeline-ci:latest
+
+inputs:
+  - id: input_h5parm
+    type: File
+    format: lofar:#H5Parm
+  - id: soltab
+    type: string
+    doc: "Solution table"
+  - id: clocksoltabOut
+    doc: output soltab name for clock
+    type: string?
+  - id: tecsoltabOut
+    doc: output soltab name for tec
+    type: string?
+  - id: offsetsoltabOut
+    doc: output soltab name for phase offset
+    type: string?
+  - id: tec3rdsoltabOut
+    doc: output soltab name for tec3rd offset
+    type: string?
+  - id: flagBadChannels
+    type: boolean?
+    doc: Detect and remove bad channel before fitting, by default True.
+  - id: flagCut
+    type: float?
+  - id: chi2cut
+    type: float?
+  - id: combinePol
+    type: boolean?
+    doc: |
+      Find a combined polarization solution, by default False.
+  - id: removePhaseWraps
+    type: boolean?
+    doc: |
+      Detect and remove phase wraps, by default True.
+  - id: fit3rdorder
+    type: boolean?
+    doc: |
+      Fit a 3rd order ionospheric ocmponent (usefult <40 MHz). By default False.
+  - id: circular
+    type: boolean?
+    doc: |
+      Assume circular polarization with FR not removed. By default False.
+  - id: reverse
+    type: boolean?
+    doc:
+      Reverse the time axis. By default False.
+  - id: invertOffset
+    type: boolean?
+    doc: |
+      Invert (reverse the sign of) the phase offsets. By default False. Set to True
+      if you want to use them with the residuals operation.
+
+outputs:
+  - id: output_h5parm
+    type: File
+    format: lofar:#H5Parm
+    outputBinding:
+      glob: $(inputs.input_h5parm.basename)
+$schema:
+  - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

steps/applycal.cwl

@@ -5,8 +5,6 @@ $namespaces:
 id: applycal
 baseCommand:
   - DPPP
-arguments:
-  - steps=[applycal]
 inputs:
   - id: msin
     type: Directory
@@ -25,41 +23,56 @@ inputs:
     doc: Input data Column
   - id: parmdb
     type: File
-    doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
     inputBinding:
-      prefix: applycal.parmdb
-  - id: correction
-    default: gain
-    type:
-      type: enum
-      symbols:
-        - gain
-        - tec
-        - clock
-        - rotationangle
-        - rotation
-        - scalarphase
-        - scalaramplitude
-        - rotationmeasure
-        - fulljones
-    doc: | 
-        Type of correction to perform. When using H5Parm, this is for now the name of the soltab; the type will be deduced from the metadata in that soltab, except for full Jones, in which case correction should be 'fulljones'.
+      position: 0
+      prefix: applycal.parmdb=
+      separate: false
+    doc: >-
+      Path of parmdb in which the parameters are stored. This can also be an
+      H5Parm file, in that case the filename has to end in '.h5'
+  - id: msout_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout.datacolumn=
+      separate: false
+    doc: Output data column
+  - default: gain
+    id: correction
+    type: string
+    inputBinding:
+      position: 0
+      prefix: applycal.correction=
+      separate: false
+    doc: >
+      Type of correction to perform. When using H5Parm, this is for now the name
+      of the soltab; the type will be deduced from the metadata in that soltab,
+      except for full Jones, in which case correction should be 'fulljones'.
+  - id: storagemanager
+    type: string
+    default: dysco
+    inputBinding
+      prefix: msout.storagemanager=
+  - id: updateweights
+    type: boolean?
     inputBinding:
-        prefix: applycal.correction=
-  
+      position: 0
+      prefix: -applycal.updateweights=True
 outputs:
   - id: msout
     doc: Output Measurement Set
     type: Directory
     outputBinding:
-        glob: $(inputs.msin.basename)
-      
+      glob: $(inputs.msin.basename)
+arguments:
+  - 'steps=[applycal]'
+  - msout=.
 requirements:
   - class: InitialWorkDirRequirement
     listing:
-        - entry: $(inputs.msin)
-          writable: true
+      - entry: $(inputs.msin)
+        writable: true
   - class: InlineJavascriptRequirement
 hints:
   - class: DockerRequirement
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: 'lofareosc/lofar-pipeline-ci:latest'

steps/ddecal.cwl

@@ -2,7 +2,7 @@
 
 class: CommandLineTool
 cwlVersion: v1.0
-id: ddecal
+id: calib_rot_diag
 baseCommand: [DPPP]
 
 requirements:
@@ -14,7 +14,6 @@ hints:
 
 arguments:
   - steps=[ddecal]
-  - ddecal.mode=rotation+diagonal
   - ddecal.uvlambdamin=300
   - ddecal.maxiter=50
   - ddecal.nchan=1
@@ -30,38 +29,38 @@ inputs:
     type: Directory?
     doc: Input Measurement Set
     inputBinding:
-        prefix: msin=
-        separate: false
+      prefix: msin=
+      separate: false
 
   - id: msin_datacolumn
     type: string
     default: DATA
     doc: Input data Column
     inputBinding:
-        prefix: msin.datacolumn=
-        separate: false
+      prefix: msin.datacolumn=
+      separate: false
 
   - id: msin_modelcolum
     type: string
     default: MODEL_DATA
     doc: Model data Column
     inputBinding:
-        prefix: msin.modelcolumn=
-        separate: false
+      prefix: msin.modelcolumn=
+      separate: false
 
   - id: output_name_h5parm
     type: string
     default: instrument.h5
     inputBinding:
-        prefix: ddecal.h5parm=
-        separate: false
+      prefix: ddecal.h5parm=
+      separate: false
   - id: msout_name
     type: string
     doc: Output Measurement Set
     default: out.MS
     inputBinding:
-        prefix: msout=
-        separate: false
+      prefix: msout=
+      separate: false
 
 #--------------------
   - id: propagate_solutions
@@ -91,6 +90,31 @@ inputs:
       and mode=rotation+diagonal.
     inputBinding:
       prefix: flagdivergedonly=True
+  - id: storagemanager
+    type: string
+    default: dysco
+    inputBinding
+      prefix: msout.storagemanager=
+  - id: mode
+    type:
+      type: enum
+      symbols:
+        - scalarcomplexgain
+        - scalarphase
+        - scalaramplitude
+        - tec
+        - tecandphase
+        - fulljones
+        - diagonal
+        - phaseonly
+        - amplitudeonly
+        - rotation
+        - rotation+diagonal
+    doc: |
+      Type of constraint to apply. Options are
+    inputBinding:
+      prefix: ddecal.mode=
+      separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set

steps/gaincal.cwl

@@ -51,6 +51,11 @@ inputs:
     inputBinding:
         prefix: gaincal.usebeammodel=true
         separate: false
+  - id: storagemanager
+    type: string
+    default: dysco
+    inputBinding
+      prefix: msout.storagemanager=
   - id: solint
     type: int
     default: 1

steps/predict.cwl

@@ -21,6 +21,11 @@ inputs:
       prefix: msin.datacolumn=
       separate: false
     doc: Input data Column
+  - id: storagemanager
+    type: string
+    default: dysco
+    inputBinding
+      prefix: msout.storagemanager=  
   - default: MODEL_DATA
     id: msout_datacolumn
     type: string

test_jobs/ddecal.json

@@ -4,5 +4,6 @@
     "path": "/data/L570745_SB001_uv_MODEL.MS"
   },
   "msin_datacolumn": "DATA", 
-  "usemodelcolumn": true
+  "usemodelcolumn": true,
+  "mode": "diagonal"
 }

test_jobs/losoto_clocktec.json

+{
+  "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+  "soltab":  "sol000/phase000",
+  "ncpu": 1
+}