finish PA step

lofar-cwl/steps/DP3.ApplyBeamStepGenerator.cwl

+#!/usr/bin/env cwl-runner
+
+class: Workflow
+cwlVersion: v1.0
+id: applybeam_step_generator
+
+inputs:
+  - id: steps
+    type: Any[]?
+    default: []
+  - id: step_name
+    type: string
+    doc: unique name for the step
+    default: applybeam
+
+  - id: direction
+    type: string?
+    doc: A RA/Dec value specifying in what direction to correct the beam. See phaseshift.phasecenter for syntax. If empty, the beam is corrected in the direction of the current phase center. 
+  - id: onebeamperpatch
+    type: string?
+    doc: |
+      Compute the beam only for the center of each patch
+      (saves computation time, but you should set this to false for large
+      patches). 
+      In the ApplyBeam step, this setting does not make sense 
+      (but it does if the applybeam is part of predict, ddecal, gaincal,
+      h5parmpredict, etc.). 
+      Generally, FALSE is the right setting for this option. 
+      The default has changed to false in a recent (Nov 2018) version. 
+  - id: usechannelfreq
+    type: boolean?
+    doc: |
+      Compute the beam for each channel of the measurement set separately.
+      This is useful for merged / concatenated measurement sets.
+      For raw LOFAR data you should set it to false,
+      so that the beam will be formed as in the station hardware.
+      Also, setting it to false is faster.
+      
+  - id: updateweights
+    type: boolean?
+    doc: |
+      Update the weights column, in a way consistent with
+      the weights being inverse proportional to the autocorrelations
+      (e.g. if 'autoweights' was used before).
+
+  - id: invert
+    type: boolean?
+    doc: |
+      Invert the corrections, to correct the data. Default is true.
+      If you want to corrupt the data, set it to 'false'
+     
+  - id: beammode
+    
+    type:
+      type: enum
+      symbols:
+       - array_factor
+       - element
+       - default
+    doc: |
+      Beam mode to apply, can be “array_factor”, “element” or “default”.
+      Default is to apply both the element beam and the array factor. 
+    
+    
+outputs:
+  - id: augmented_steps
+    outputSource:
+      - DP3_GenericStep/augmented_steps
+    type: Any[]
+steps:
+  - id: DP3_GenericStep
+    in:
+      - id: step_type
+        default: 'applybeam'
+      - id: step_id
+        source: step_name
+      - id: steps
+        source: steps
+      - id: parameters
+        valueFrom: $(inputs)
+
+      - id: direction
+        source: direction
+      - id: onebeamperpatch
+        source: onebeamperpatch
+      - id: usechannelfreq
+        source: usechannelfreq
+      - id: updateweights
+        source: updateweights
+      - id: invert
+        source: invert
+      - id: beammode
+        source: beammode
+
+
+#-------------------------------
+
+    out:
+      - augmented_steps
+    run: ../steps/DP3.GenericStep.cwl
+requirements:
+  - class: StepInputExpressionRequirement
+  - class: InlineJavascriptRequirement
+  - class: MultipleInputFeatureRequirement

lofar-cwl/steps/applybeam.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: applycal
+baseCommand:
+  - DPPP
+inputs:
+  - id: msin
+    type: Directory
+    inputBinding:
+      position: 0
+      prefix: msin=
+      separate: false
+    doc: Input Measurement Set
+  - default: DATA
+    id: msin_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msin.datacolumn=
+      separate: false
+    doc: Input data Column
+  - id: msout_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout.datacolumn=
+      separate: false
+    doc: Output data column
+  - default: applybeam
+    id: type
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: applybeam.type=
+      separate: false
+    doc: >
+      Type of correction to perform. When using H5Parm, this is for now the name
+      of the soltab; the type will be deduced from the metadata in that soltab,
+      except for full Jones, in which case correction should be 'fulljones'.
+  - id: storagemanager
+    type: string
+    default: dysco
+    inputBinding:
+      prefix: msout.storagemanager=
+  - id: updateweights
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: -applybeam.updateweights=
+      separate: false
+  - id: beammode
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: -applybeam.beammode=
+      separate: false
+  - id: usechannelfreq
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: -applybeam.usechannelfreq=
+      separate: false
+  - id: invert
+    type: boolean?
+    inputBinding:
+      position: 0
+      prefix: -applybeam.invert=
+      separate: false
+outputs:
+  - id: msout
+    doc: Output Measurement Set
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msin.basename)
+  - id: logfile
+    type: File
+    outputBinding:
+      glob: applycal_$(inputs.type).log    
+stdout: applycal_$(inputs.type).log
+arguments:
+  - 'steps=[applybeam]'
+  - msout=.
+requirements:
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+  - class: InlineJavascriptRequirement
+hints:
+  - class: DockerRequirement
+    dockerPull: 'lofareosc/prefactor:HBAcalibrator'

lofar-cwl/steps/applycal.cwl

@@ -5,9 +5,6 @@ $namespaces:
 id: applycal
 baseCommand:
   - DPPP
-arguments:
-  - steps=[applycal]
-  - msout=.
 inputs:
   - id: msin
     type: Directory
@@ -26,32 +23,62 @@ inputs:
     doc: Input data Column
   - id: parmdb
     type: File
-    doc: Path of parmdb in which the parameters are stored. This can also be an H5Parm file, in that case the filename has to end in '.h5'
     inputBinding:
+      position: 0
       prefix: applycal.parmdb=
       separate: false
-  - id: correction
-    default: gain
+    doc: >-
+      Path of parmdb in which the parameters are stored. This can also be an
+      H5Parm file, in that case the filename has to end in '.h5'
+  - id: msout_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout.datacolumn=
+      separate: false
+    doc: Output data column
+  - default: gain
+    id: correction
+    type: string
+    inputBinding:
+      position: 0
+      prefix: applycal.correction=
+      separate: false
+    doc: >
+      Type of correction to perform. When using H5Parm, this is for now the name
+      of the soltab; the type will be deduced from the metadata in that soltab,
+      except for full Jones, in which case correction should be 'fulljones'.
+  - id: storagemanager
     type: string
-    doc: | 
-        Type of correction to perform. When using H5Parm, this is for now the name of the soltab; the type will be deduced from the metadata in that soltab, except for full Jones, in which case correction should be 'fulljones'.
+    default: dysco
+    inputBinding:
+      prefix: msout.storagemanager=
+  - id: updateweights
+    type: boolean?
     inputBinding:
-        prefix: applycal.correction=
-        separate: false
-  
+      position: 0
+      prefix: -applycal.updateweights=
+      separate: false
 outputs:
   - id: msout
     doc: Output Measurement Set
     type: Directory
     outputBinding:
-        glob: $(inputs.msin.basename)
-      
+      glob: $(inputs.msin.basename)
+  - id: logfile
+    type: File
+    outputBinding:
+      glob: applycal_$(inputs.correction).log   
+stdout: applycal_$(inputs.correction).log
+arguments:
+  - 'steps=[applycal]'
+  - msout=.
 requirements:
   - class: InitialWorkDirRequirement
     listing:
-        - entry: $(inputs.msin)
-          writable: true
+      - entry: $(inputs.msin)
+        writable: true
   - class: InlineJavascriptRequirement
 hints:
   - class: DockerRequirement
-    dockerPull: lofareosc/lofar-pipeline-ci:latest
+    dockerPull: 'lofareosc/prefactor:HBAcalibrator'

subworkflow/apply_calibrate_pa.cwl

+class: Workflow
+cwlVersion: v1.0
+id: apply_calibrate
+label: apply_calibrate
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+inputs:
+  - id: msin
+    type: Directory
+    'sbg:x': -1000
+    'sbg:y': -200
+  - id: do_smooth
+    type: boolean
+    'sbg:x': -1000
+    'sbg:y': -100
+  - id: flagunconverged
+    type: boolean
+    'sbg:x': -1000
+    'sbg:y': 100
+  - id: propagatesolutions
+    type: boolean
+    'sbg:x': -1000
+    'sbg:y': 200
+  - id: input_h5parm
+    type: File
+    'sbg:x': -1000
+    'sbg:y': 300
+outputs:
+  - id: msout
+    outputSource:
+      - calib_cal/msout
+    type: Directory
+    'sbg:x': 1000
+    'sbg:y': -200
+  - id: BLsmooth.log
+    outputSource:
+      - BLsmooth/logfile
+    type: File
+    'sbg:x': 1000
+    'sbg:y': -100
+  - id: apply_cal.log
+    outputSource:
+      - applyPA/logfile
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 0
+  - id: calib_cal.log
+    outputSource:
+      - calib_cal/logfile
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 100
+  - id: outh5parm
+    outputSource:
+      - calib_cal/h5parm
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 200
+  - id: applybeam.log
+    outputSource:
+      - applybeam/logfile
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 300
+steps:
+  - id: applyPA
+    in:
+      - id: msin
+        source: msin
+      - id: msin_datacolumn
+        default: DATA
+      - id: parmdb
+        source: input_h5parm
+      - id: msout_datacolumn
+        default: CORRECTED_DATA
+      - id: correction
+        default: polalign
+    out:
+      - id: msout
+      - id: logfile
+    run: ../lofar-cwl/steps/applycal.cwl
+    label: applyPA
+    'sbg:x': -400
+    'sbg:y': 0
+  - id: applybeam
+    in:
+      - id: msin_datacolumn
+        default: CORRECTED_DATA
+      - id: msout_datacolumn
+        default: CORRECTED_DATA
+      - id: updateweights
+        default: true
+      - id: invert
+        default: true
+      - id: beammode
+        default: element
+      - id: usechannelfreq
+        default: false
+      - id: msin
+        source: applyPA/msout
+      - id: type
+        default: applybeam
+    out:
+      - id: msout
+      - id: logfile
+    run: ../lofar-cwl/steps/applybeam.cwl
+    label: applybeam
+    'sbg:x': -200
+    'sbg:y': 0
+  - id: BLsmooth
+    in:
+      - id: msin
+        source: applybeam/msout
+      - id: do_smooth
+        source: do_smooth
+      - id: in_column_name
+        default: CORRECTED_DATA
+    out:
+      - id: msout
+      - id: logfile
+    run: ../steps/blsmooth.cwl
+    label: BLsmooth
+    'sbg:x': 0
+    'sbg:y': 0
+  - id: calib_cal
+    in:
+      - id: msin
+        source: BLsmooth/msout
+      - id: msin_datacolumn
+        default: SMOOTHED_DATA
+      - id: msin_modelcolum
+        default: MODEL_DATA
+      - id: flagunconverged
+        source: flagunconverged
+      - id: propagate_solutions
+        source: propagatesolutions
+      - id: mode
+        default: rotation+diagonal
+    out:
+      - id: msout
+      - id: h5parm
+      - id: logfile
+    run: ../lofar-cwl/steps/ddecal.cwl
+    'sbg:x': 500
+    'sbg:y': 0
+requirements: []

workflows/HBA_calibrator.cwl

@@ -307,7 +307,7 @@ steps:
       - id: logfiles
       - id: msout
       - id: parset
-      - id: instrument.h5
+      - id: solutions
       - id: inspection
     run: ./prefactor_calibrator.cwl
     label: prefactor_calibrator
@@ -345,7 +345,7 @@ steps:
     in:
       - id: log_files
         source:
-          - prefactor/instrument.h5
+          - prefactor/solutions
       - id: sub_directory_name
         default: results/cal_values
     out:

workflows/prefactor_calibrator.cwl

@@ -200,9 +200,9 @@ outputs:
     type: File[]
     'sbg:x': 1000
     'sbg:y': -600
-  - id: instrument.h5
+  - id: solutions
     outputSource:
-      - pa/output_h5parm
+      - pa/solutions
     type: File
     'sbg:x': 1000
     'sbg:y': -500
@@ -277,12 +277,12 @@ steps:
       - id: A-Team_skymodel
         source: A-Team_skymodel
     out:
-      - id: instrument.h5
+      - id: outh5parm
       - id: logfiles
+      - id: outh5parm_logfile
       - id: check_Ateam_separation.png
       - id: msout
       - id: parset
-      - id: sourcedb
     run: ./prefactor_calibrator/prep.cwl
     label: prep
     'sbg:x': 0
@@ -294,12 +294,21 @@ steps:
           - prep/msout
       - id: h5parm
         source:
-          - prep/instrument.h5
+          - prep/outh5parm
       - id: refant
         source: refant
+      - id: inh5parm_logfile
+        source:
+          - prep/outh5parm_logfile
+      - id: flagunconverged
+        source: flagunconverged
+      - id: propagatesolutions
+        source: propagatesolutions
+      - id: do_smooth
+        source: do_smooth
     out:
       - id: msout
-      - id: output_h5parm
+      - id: solutions
       - id: inspection
       - id: logfiles
     run: ./prefactor_calibrator/pa.cwl

workflows/prefactor_calibrator/pa.cwl

@@ -5,6 +5,16 @@ label: PA
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
+  - id: flagunconverged
+    type: boolean?
+    default: false
+    'sbg:x': -1000
+    'sbg:y': -300
+  - id: propagatesolutions
+    type: boolean?
+    default: true
+    'sbg:x': -1000
+    'sbg:y': -200
   - id: msin
     type: 'Directory[]'
     'sbg:x': -1000
@@ -17,10 +27,25 @@ inputs:
     type: string?
     'sbg:x': -1000
     'sbg:y': 100
+  - id: inh5parm_logfile
+    type: File
+    'sbg:x': -1000
+    'sbg:y': 200
+  - id: do_smooth
+    type: boolean?
+    default: false
+    'sbg:x': -1000
+    'sbg:y': 300
 outputs:
+  - id: outh5parm
+    outputSource:
+      - h5parm_collector/outh5parm
+    type: File
+    'sbg:x': 1000
+    'sbg:y': -200
   - id: msout
     outputSource:
-      - msin
+      - apply_calibrate_pa/msout
     type: 'Directory[]'
     'sbg:x': 1000
     'sbg:y': -100
@@ -37,7 +62,7 @@ outputs:
     linkMerge: merge_flattened
     'sbg:x': 1000
     'sbg:y': 0
-  - id: output_h5parm
+  - id: solutions
     outputSource:
       - write_solutions/outh5parm
     type: File
@@ -46,10 +71,20 @@ outputs:
   - id: logfiles
     outputSource:
       - concat_logfiles_pa/output
+      - concat_logfiles_calib/output
+      - concat_logfiles_blsmooth/output
+      - concat_logfiles_beam/output
+      - concat_logfiles_apply/output
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 1000
     'sbg:y': 200
+  - id: outh5parm_logfile
+    outputSource:
+      - h5parm_collector/log
+    type: File
+    'sbg:x': 1000
+    'sbg:y': 300
 steps:
   - id: pol_align
     in:
@@ -271,6 +306,7 @@ steps:
       - id: file_list
         linkMerge: merge_flattened
         source:
+          - inh5parm_logfile
           - pol_align/logfiles
           - losoto_plot_P3/logfile
           - losoto_plot_Pd/logfile
@@ -291,14 +327,13 @@ steps:
   - id: write_solutions
     in:
       - id: h5parmFiles
-        source:
-          - pol_align/output_h5parm
-      - id: outh5parmname
-        default: cal_solutions.h5
+        source: pol_align/output_h5parm
       - id: outsolset
         default: calibrator
       - id: insoltab
         default: polalign
+      - id: outh5parmname
+        default: cal_solutions.h5
       - id: squeeze
         default: true
       - id: verbose
@@ -310,6 +345,102 @@ steps:
     label: write_solutions
     'sbg:x': 0
     'sbg:y': 500
+  - id: apply_calibrate_pa
+    in:
+      - id: msin
+        source: msin
+      - id: do_smooth
+        source: do_smooth
+      - id: flagunconverged
+        source: flagunconverged
+      - id: propagatesolutions
+        source: propagatesolutions
+      - id: input_h5parm
+        source: write_solutions/outh5parm
+    out:
+      - id: msout
+      - id: BLsmooth.log
+      - id: apply_cal.log
+      - id: calib_cal.log
+      - id: outh5parm
+      - id: applybeam.log
+    run: ../../subworkflow/apply_calibrate_pa.cwl
+    label: apply_calibrate_pa
+    scatter:
+      - msin
+    'sbg:x': 500
+    'sbg:y': 0
+  - id: concat_logfiles_blsmooth
+    in:
+      - id: file_list
+        source:
+          - apply_calibrate_pa/BLsmooth.log
+      - id: file_prefix
+        default: blsmooth_PA
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_blsmooth
+    'sbg:x': 750
+    'sbg:y': 300
+  - id: concat_logfiles_beam
+    in:
+      - id: file_list
+        source:
+          - apply_calibrate_pa/applybeam.log
+      - id: file_prefix
+        default: applybeam_PA
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_beam
+    'sbg:x': 750
+    'sbg:y': 400
+  - id: concat_logfiles_apply
+    in:
+      - id: file_list
+        source:
+          - apply_calibrate_pa/apply_cal.log
+      - id: file_prefix
+        default: apply_cal_PA
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_apply
+    'sbg:x': 750
+    'sbg:y': 600
+  - id: concat_logfiles_calib
+    in:
+      - id: file_list
+        source:
+          - apply_calibrate_pa/calib_cal.log
+      - id: file_prefix
+        default: calib_cal_PA
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_calib
+    'sbg:x': 750
+    'sbg:y': 800
+  - id: h5parm_collector
+    in:
+      - id: h5parmFiles
+        source:
+          - apply_calibrate_pa/outh5parm
+      - id: squeeze
+        default: true
+      - id: verbose
+        default: true
+      - id: clobber
+        default: true
+    out:
+      - id: outh5parm
+      - id: log
+    run: ../../lofar-cwl/steps/H5ParmCollector.cwl
+    label: H5parm_collector
+    'sbg:x': 750
+    'sbg:y': -300
 requirements:
   - class: SubworkflowFeatureRequirement
+  - class: ScatterFeatureRequirement
   - class: MultipleInputFeatureRequirement

workflows/prefactor_calibrator/prep.cwl

@@ -110,7 +110,7 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1500
 outputs:
-  - id: instrument.h5
+  - id: outh5parm
     outputSource:
       - h5parm_collector/outh5parm
     type: File
@@ -134,12 +134,12 @@ outputs:
     type: 'File[]'
     'sbg:x': 1000
     'sbg:y': 200
-  - id: sourcedb
+  - id: outh5parm_logfile
     outputSource:
-      - make_sourcedb/sourcedb
+      - h5parm_collector/log
     type: File
     'sbg:x': 1000
-    'sbg:y': 500
+    'sbg:y': 650
   - id: logfiles
     linkMerge: merge_flattened
     outputSource:
@@ -319,7 +319,7 @@ steps:
         source:
           - predict_calibrate/BLsmooth.log
       - id: file_prefix
-        default: blsmooth
+        default: blsmooth_prep
     out:
       - id: output
     run: ../../steps/concatenate_files.cwl
@@ -345,7 +345,7 @@ steps:
         source:
           - predict_calibrate/calib_cal.log
       - id: file_prefix
-        default: calib_cal
+        default: calib_cal_prep
     out:
       - id: output
     run: ../../steps/concatenate_files.cwl