add Ateamclipper

Former-commit-id: d4debaa8

lofar-cwl/steps/Ateamclipper.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: check_ateam_separation
+baseCommand:
+  - python3
+  - /usr/local/bin/Ateamclipper.py
+inputs:
+  - id: msin
+    type:
+      - Directory
+      - type: array
+        items: Directory
+    inputBinding:
+      position: 0
+    doc: Input measurement set
+outputs:
+  - id: msout
+    doc: Output MS
+    type: File?
+    outputBinding:
+      glob: $(inputs.msin)
+  - id: logfile
+    type: 'File[]?'
+    outputBinding:
+      glob: Ateamclipper.log
+label: Ateamclipper
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+  - class: InlineJavascriptRequirement
+stdout: Ateamclipper.log

lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl

@@ -38,6 +38,8 @@ inputs:
        - fulljones
        - polalign
        - bandpass
+       - RMextract
+       - phaseOrig
     doc: |
       Type of correction to perform, can be one of 'gain', 'tec', 'clock',
       '(common)rotationangle' / 'rotation', '(common)scalarphase',

lofar-cwl/steps/filter_predict.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: predict
+baseCommand:
+  - DPPP
+inputs:
+  - id: msin
+    type: Directory
+    inputBinding:
+      position: 0
+      prefix: msin=
+      separate: false
+    doc: Input Measurement Set
+  - default: DATA
+    id: msin_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msin.datacolumn=
+      separate: false
+    doc: Input data Column
+  - default: MODEL_DATA
+    id: msout_datacolumn
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout.datacolumn=
+      separate: false
+  - id: sources_db
+    type: File
+    inputBinding:
+      position: 0
+      prefix: predict.sourcedb=
+      separate: false
+  - default: null
+    id: sources
+    type: 'string[]'
+    inputBinding:
+      position: 0
+      prefix: predict.sources=
+      separate: false
+      itemSeparator: ','
+      valueFrom: '[$(self)]'
+  - default: false
+    id: usebeammodel
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: predict.usebeammodel=True
+  - default: false
+    id: onebeamperpatch
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: predict.onebeamperpatch=True
+  - default: null
+    id: filter_baselines
+    type: string
+    inputBinding:
+      position: 0
+      prefix: filter.baseline=
+      separate: false
+      valueFrom: '$(self)'
+  - default: false
+    id: filter_remove
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: filter.remove=True
+  - id: writefullresflag
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.writefullresflag=True
+  - id: overwrite
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.overwrite=True
+  - id: storagemanager
+    type: string
+    default: ""
+    inputBinding:
+       prefix: msout.storagemanager=
+       separate: false
+  - id: databitrate
+    type: int?
+    inputBinding:
+       prefix: msout.storagemanager.databitrate=
+       separate: false
+outputs:
+  - id: msout
+    doc: Output Measurement Set
+    type: Directory
+    outputBinding:
+      glob: $(inputs.msin.basename)
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'predict_cal*.log'
+arguments:
+  - 'steps=[filter,predict]'
+  - predict.beammode=array_factor
+  - predict.usechannelfreq=False
+  - msout=.
+requirements:
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+  - class: InlineJavascriptRequirement
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: filter_predict.log
+stderr: filter_predict_err.log
\ No newline at end of file

subworkflow/ndppp_prep_targ.cwl

@@ -53,7 +53,7 @@ inputs:
     type: int
     'sbg:x': 1465.0855712890625
     'sbg:y': 450.8930358886719
-  - id: demix_baseline
+  - id: process_baselines_target
     type: string
     'sbg:x': 1585.496826171875
     'sbg:y': 588.0062866210938
@@ -97,25 +97,41 @@ inputs:
     type: string
     'sbg:x': 1880
     'sbg:y': -380
+  - id: updateweights
+    type: boolean
+    'sbg:x': 2234.90185546875
+    'sbg:y': -393.8929748535156
 outputs:
-  - id: logfile
+  - id: prep_logfile
     outputSource:
       - concat_logfiles_dppp/output
-    type: File
-    'sbg:x': 4982.51416015625
-    'sbg:y': 121.597900390625
+    type: File?
+    'sbg:x': 5500
+    'sbg:y': 100
+  - id: predict_logfile
+    outputSource:
+      - concat_logfiles_predict/output
+    type: File?
+    'sbg:x': 5500
+    'sbg:y': 0
+  - id: clipper_logfile
+    outputSource:
+      - concat_logfiles_clipper/output
+    type: File?
+    'sbg:x': 5500
+    'sbg:y': -100
   - id: msout
     outputSource:
-      - dp3_execute/msout
+      - predict/msout
     type: Directory
-    'sbg:x': 4884.83251953125
-    'sbg:y': -232.5366973876953
+    'sbg:x': 5502.318359375
+    'sbg:y': -279.49835205078125
   - id: parset
     outputSource:
       - dp3_execute/parset
     type: File
-    'sbg:x': 4862.033203125
-    'sbg:y': -541.10693359375
+    'sbg:x': 5427.662109375
+    'sbg:y': -765.5377197265625
 steps:
   - id: flagbaseline
     in:
@@ -228,7 +244,7 @@ steps:
       - id: step_name
         default: demixer
       - id: baseline
-        source: demix_baseline
+        source: process_baselines_target
       - id: demixtimestep
         source: demix_timestep
       - id: demixfreqstep
@@ -278,6 +294,32 @@ steps:
     label: concat_logfiles_dppp
     'sbg:x': 4543.36376953125
     'sbg:y': -6.564670085906982
+  - id: concat_logfiles_predict
+    in:
+      - id: file_list
+        source:
+          - predict/logfile
+      - id: file_prefix
+        default: predict_targ
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_predict
+    'sbg:x': 4700
+    'sbg:y': -6.564670085906982
+  - id: concat_logfiles_clipper
+    in:
+      - id: file_list
+        source:
+          - Ateamclipper/logfile
+      - id: file_prefix
+        default: Ateamclipper
+    out:
+      - id: output
+    run: ../steps/concatenate_files.cwl
+    label: concat_logfiles_clipper
+    'sbg:x': 4900
+    'sbg:y': -6.564670085906982
   - id: dp3_execute
     in:
       - id: msout_name
@@ -323,6 +365,8 @@ steps:
         default: apply_pa_1
       - id: parmdb
         source: parmdb
+      - id: solset
+        default: calibrator
       - id: correction
         default: polalign
     out:
@@ -339,6 +383,8 @@ steps:
         default: apply_bandpass_1
       - id: parmdb
         source: parmdb
+      - id: solset
+        default: calibrator
       - id: correction
         default: bandpass
     out:
@@ -363,11 +409,57 @@ steps:
         source: apply_phase_correction
       - id: parmdb
         source: parmdb
+      - id: updateweights
+        source: updateweights
     out:
       - id: selected_steps
     run: ./prep_target_applycal.cwl
     label: prep_target_applycal
     'sbg:x': 2728.697998046875
     'sbg:y': -18.151018142700195
+  - id: predict
+    in:
+      - id: msin
+        source: dp3_execute/msout
+      - id: msin_datacolumn
+        default: DATA
+      - id: msout_datacolumn
+        default: MODEL_DATA
+      - id: sources_db
+        source: skymodel
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        default: 0
+      - id: filter_baselines
+        source: process_baselines_target
+      - id: sources
+        default: 
+          - 'VirA_4_patch'
+          - 'CygAGG'
+          - 'CasA_4_patch'
+          - 'TauAGG'
+      - id: usebeammodel
+        default: true
+      - id: onebeamperpatch
+        default: true
+    out:
+      - id: msout
+      - id: logfile
+    run: ../lofar-cwl/steps/filter_predict.cwl
+    'sbg:x': 4500
+    'sbg:y': -344.5472106933594
+  - id: Ateamclipper
+    in:
+      - id: msin
+        source: predict/msout
+    out:
+      - id: msout
+      - id: logfile
+    run: ../lofar-cwl/steps/Ateamclipper.cwl
+    'sbg:x': 4700
+    'sbg:y': -344.5472106933594
 requirements:
   - class: SubworkflowFeatureRequirement
+  - class: StepInputExpressionRequirement
+  - class: InlineJavascriptRequirement

subworkflow/prep_target_applycal.cwl

@@ -18,13 +18,13 @@ inputs:
     'sbg:x': 1990.8035888671875
     'sbg:y': -230.75811767578125
   - id: apply_rm_correction
-    type: boolean
-    'sbg:x': 2273.170166015625
-    'sbg:y': -251.86672973632812
-  - id: apply_beam_correction
     type: boolean
     'sbg:x': 2502.2421875
     'sbg:y': -225.68701171875
+  - id: apply_beam_correction
+    type: boolean
+    'sbg:x': 2273.170166015625
+    'sbg:y': -251.86672973632812
   - id: apply_phase_correction
     type: boolean
     'sbg:x': 2750.948974609375
@@ -33,6 +33,10 @@ inputs:
     type: File
     'sbg:x': 2007.115234375
     'sbg:y': 685.0719604492188
+  - id: updateweights
+    type: boolean
+    'sbg:x': 2167.219970703125
+    'sbg:y': 422.6792297363281
 outputs:
   - id: selected_steps
     outputSource:
@@ -50,6 +54,8 @@ steps:
         default: apply_clock
       - id: parmdb
         source: parmdb
+      - id: solset
+        default: calibrator
       - id: correction
         default: clock
     out:
@@ -66,6 +72,8 @@ steps:
         default: apply_tec
       - id: parmdb
         source: parmdb
+      - id: solset
+        default: calibrator
       - id: correction
         default: tec
     out:
@@ -77,11 +85,13 @@ steps:
     in:
       - id: steps
         source:
-          - apply_beam/selected_steps
+          - apply_rm/selected_steps
       - id: step_name
         default: apply_phase
       - id: parmdb
         source: parmdb
+      - id: solset
+        default: calibrator
       - id: correction
         default: phaseOrig
     out:
@@ -93,32 +103,40 @@ steps:
     in:
       - id: steps
         source:
-          - apply_tec/selected_steps
+          - apply_beam/selected_steps
       - id: step_name
         default: apply_rm
       - id: parmdb
         source: parmdb
+      - id: solset
+        default: target
       - id: correction
         default: RMextract
     out:
       - id: augmented_steps
     run: ../lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl
-    'sbg:x': 2321
-    'sbg:y': 285.3694763183594
+    'sbg:x': 2545.4775390625
+    'sbg:y': 291.2240295410156
   - id: applybeam
     in:
       - id: steps
         source:
-          - apply_rm/selected_steps
+          - apply_tec/selected_steps
       - id: step_name
         default: apply_beam
+      - id: usechannelfreq
+        default: true
+      - id: updateweights
+        source: updateweights
+      - id: invert
+        default: true
       - id: beammode
         default: default
     out:
       - id: augmented_steps
     run: ../lofar-cwl/steps/DP3.ApplyBeamStepGenerator.cwl
-    'sbg:x': 2545.4775390625
-    'sbg:y': 291.2240295410156
+    'sbg:x': 2321
+    'sbg:y': 285.3694763183594
   - id: apply_beam
     in:
       - id: select_a
@@ -129,13 +147,13 @@ steps:
       - id: b_steps
         default: []
         source:
-          - apply_rm/selected_steps
+          - apply_tec/selected_steps
     out:
       - id: selected_steps
     run: ../lofar-cwl/steps/DP3.Selector.cwl
     label: apply_beam
-    'sbg:x': 2648.432373046875
-    'sbg:y': 167.062255859375
+    'sbg:x': 2441.14599609375
+    'sbg:y': 167.0730438232422
   - id: apply_rm
     in:
       - id: select_a
@@ -146,13 +164,13 @@ steps:
       - id: b_steps
         default: []
         source:
-          - apply_tec/selected_steps
+          - apply_beam/selected_steps
     out:
       - id: selected_steps
     run: ../lofar-cwl/steps/DP3.Selector.cwl
     label: apply_rm?
-    'sbg:x': 2441.14599609375
-    'sbg:y': 167.0730438232422
+    'sbg:x': 2648.432373046875
+    'sbg:y': 167.062255859375
   - id: apply_clock
     in:
       - id: select_a
@@ -197,7 +215,7 @@ steps:
       - id: b_steps
         default: []
         source:
-          - apply_beam/selected_steps
+          - apply_rm/selected_steps
     out:
       - id: selected_steps
     run: ../lofar-cwl/steps/DP3.Selector.cwl

workflows/prefactor_target/prep.cwl

@@ -208,6 +208,8 @@ outputs:
       - losoto_plot_RM/logfile
       - compare_station_list/logfile
       - concat_logfiles_prep_targ/output
+      - concat_logfiles_predict_targ/output
+      - concat_logfiles_clipper_targ/output
     type: 'File[]'
     linkMerge: merge_flattened
     'sbg:x': 1000
@@ -347,7 +349,7 @@ steps:
       - id: demix_freqstep
         default: 1
         source: demix_freqstep
-      - id: demix_baseline
+      - id: process_baselines_target
         default: '*'
         source: process_baselines_target
       - id: target_source
@@ -377,9 +379,13 @@ steps:
         default: true
       - id: filter_baselines
         source: compare_station_list/filter_out
+      - id: updateweights
+        source: updateweights
     out:
       - id: msout
-      - id: logfile
+      - id: prep_logfile
+      - id: predict_logfile
+      - id: clipper_logfile
       - id: parset
     run: ../../subworkflow/ndppp_prep_targ.cwl
     label: ndppp_prep_target
@@ -391,14 +397,40 @@ steps:
     in:
       - id: file_list
         linkMerge: merge_flattened
-        source: dppp_prep_target/logfile
+        source: dppp_prep_target/prep_logfile
       - id: file_prefix
         default: ndppp_prep_targ
     out:
       - id: output
     run: ../../steps/concatenate_files.cwl
-    label: concat_logfiles_prep_targ
+    label: concat_logfiles_prep_target
     'sbg:x': 250
     'sbg:y': 300
+  - id: concat_logfiles_predict_targ
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source: dppp_prep_target/predict_logfile
+      - id: file_prefix
+        default: predict_targ
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_predict_targ
+    'sbg:x': 250
+    'sbg:y': 400
+  - id: concat_logfiles_clipper_targ
+    in:
+      - id: file_list
+        linkMerge: merge_flattened
+        source: dppp_prep_target/clipper_logfile
+      - id: file_prefix
+        default: Ateamclipper
+    out:
+      - id: output
+    run: ../../steps/concatenate_files.cwl
+    label: concat_logfiles_clipper_targ
+    'sbg:x': 250
+    'sbg:y': 500
 requirements:
   - class: ScatterFeatureRequirement
\ No newline at end of file