From 0d884a33ff09a8e887ca07ed78a249aa5cfc6439 Mon Sep 17 00:00:00 2001
From: Alexander Drabent <alex@tls-tautenburg.de>
Date: Wed, 12 Aug 2020 11:47:07 +0200
Subject: [PATCH] add Ateamclipper
Former-commit-id: 199a82fc15f02debd7ad4c74799e7fd2e4931536 [formerly d4debaa8c145f717e66797632309eaa86d0e1b44]
Former-commit-id: e7631788a7e9de2ae03a30a11d0f6f43f95311c9
Former-commit-id: 6a838be3df2248a761787bf43aa9a156ed497a55
---
lofar-cwl/steps/Ateamclipper.cwl | 33 +++++
lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl | 2 +
lofar-cwl/steps/filter_predict.cwl | 118 ++++++++++++++++++
subworkflow/ndppp_prep_targ.cwl | 114 +++++++++++++++--
subworkflow/prep_target_applycal.cwl | 54 +++++---
workflows/prefactor_target/prep.cwl | 40 +++++-
6 files changed, 328 insertions(+), 33 deletions(-)
create mode 100755 lofar-cwl/steps/Ateamclipper.cwl
create mode 100644 lofar-cwl/steps/filter_predict.cwl
diff --git a/lofar-cwl/steps/Ateamclipper.cwl b/lofar-cwl/steps/Ateamclipper.cwl
new file mode 100755
index 00000000..e4739b78
--- /dev/null
+++ b/lofar-cwl/steps/Ateamclipper.cwl
@@ -0,0 +1,33 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: check_ateam_separation
+baseCommand:
+ - python3
+ - /usr/local/bin/Ateamclipper.py
+inputs:
+ - id: msin
+ type:
+ - Directory
+ - type: array
+ items: Directory
+ inputBinding:
+ position: 0
+ doc: Input measurement set
+outputs:
+ - id: msout
+ doc: Output MS
+ type: File?
+ outputBinding:
+ glob: $(inputs.msin)
+ - id: logfile
+ type: 'File[]?'
+ outputBinding:
+ glob: Ateamclipper.log
+label: Ateamclipper
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor:HBAcalibrator
+ - class: InlineJavascriptRequirement
+stdout: Ateamclipper.log
diff --git a/lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl b/lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl
index 284487d1..2850b977 100755
--- a/lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl
+++ b/lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl
@@ -38,6 +38,8 @@ inputs:
- fulljones
- polalign
- bandpass
+ - RMextract
+ - phaseOrig
doc: |
Type of correction to perform, can be one of 'gain', 'tec', 'clock',
'(common)rotationangle' / 'rotation', '(common)scalarphase',
diff --git a/lofar-cwl/steps/filter_predict.cwl b/lofar-cwl/steps/filter_predict.cwl
new file mode 100644
index 00000000..458db456
--- /dev/null
+++ b/lofar-cwl/steps/filter_predict.cwl
@@ -0,0 +1,118 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: predict
+baseCommand:
+ - DPPP
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
+ - default: DATA
+ id: msin_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msin.datacolumn=
+ separate: false
+ doc: Input data Column
+ - default: MODEL_DATA
+ id: msout_datacolumn
+ type: string
+ inputBinding:
+ position: 0
+ prefix: msout.datacolumn=
+ separate: false
+ - id: sources_db
+ type: File
+ inputBinding:
+ position: 0
+ prefix: predict.sourcedb=
+ separate: false
+ - default: null
+ id: sources
+ type: 'string[]'
+ inputBinding:
+ position: 0
+ prefix: predict.sources=
+ separate: false
+ itemSeparator: ','
+ valueFrom: '[$(self)]'
+ - default: false
+ id: usebeammodel
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: predict.usebeammodel=True
+ - default: false
+ id: onebeamperpatch
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: predict.onebeamperpatch=True
+ - default: null
+ id: filter_baselines
+ type: string
+ inputBinding:
+ position: 0
+ prefix: filter.baseline=
+ separate: false
+ valueFrom: '$(self)'
+ - default: false
+ id: filter_remove
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: filter.remove=True
+ - id: writefullresflag
+ type: boolean
+ default: false
+ inputBinding:
+ prefix: msout.writefullresflag=True
+ - id: overwrite
+ type: boolean
+ default: false
+ inputBinding:
+ prefix: msout.overwrite=True
+ - id: storagemanager
+ type: string
+ default: ""
+ inputBinding:
+ prefix: msout.storagemanager=
+ separate: false
+ - id: databitrate
+ type: int?
+ inputBinding:
+ prefix: msout.storagemanager.databitrate=
+ separate: false
+outputs:
+ - id: msout
+ doc: Output Measurement Set
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msin.basename)
+ - id: logfile
+ type: File[]
+ outputBinding:
+ glob: 'predict_cal*.log'
+arguments:
+ - 'steps=[filter,predict]'
+ - predict.beammode=array_factor
+ - predict.usechannelfreq=False
+ - msout=.
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
+ - class: InlineJavascriptRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: filter_predict.log
+stderr: filter_predict_err.log
\ No newline at end of file
diff --git a/subworkflow/ndppp_prep_targ.cwl b/subworkflow/ndppp_prep_targ.cwl
index 0938b0ac..58e91c17 100644
--- a/subworkflow/ndppp_prep_targ.cwl
+++ b/subworkflow/ndppp_prep_targ.cwl
@@ -53,7 +53,7 @@ inputs:
type: int
'sbg:x': 1465.0855712890625
'sbg:y': 450.8930358886719
- - id: demix_baseline
+ - id: process_baselines_target
type: string
'sbg:x': 1585.496826171875
'sbg:y': 588.0062866210938
@@ -97,25 +97,41 @@ inputs:
type: string
'sbg:x': 1880
'sbg:y': -380
+ - id: updateweights
+ type: boolean
+ 'sbg:x': 2234.90185546875
+ 'sbg:y': -393.8929748535156
outputs:
- - id: logfile
+ - id: prep_logfile
outputSource:
- concat_logfiles_dppp/output
- type: File
- 'sbg:x': 4982.51416015625
- 'sbg:y': 121.597900390625
+ type: File?
+ 'sbg:x': 5500
+ 'sbg:y': 100
+ - id: predict_logfile
+ outputSource:
+ - concat_logfiles_predict/output
+ type: File?
+ 'sbg:x': 5500
+ 'sbg:y': 0
+ - id: clipper_logfile
+ outputSource:
+ - concat_logfiles_clipper/output
+ type: File?
+ 'sbg:x': 5500
+ 'sbg:y': -100
- id: msout
outputSource:
- - dp3_execute/msout
+ - predict/msout
type: Directory
- 'sbg:x': 4884.83251953125
- 'sbg:y': -232.5366973876953
+ 'sbg:x': 5502.318359375
+ 'sbg:y': -279.49835205078125
- id: parset
outputSource:
- dp3_execute/parset
type: File
- 'sbg:x': 4862.033203125
- 'sbg:y': -541.10693359375
+ 'sbg:x': 5427.662109375
+ 'sbg:y': -765.5377197265625
steps:
- id: flagbaseline
in:
@@ -228,7 +244,7 @@ steps:
- id: step_name
default: demixer
- id: baseline
- source: demix_baseline
+ source: process_baselines_target
- id: demixtimestep
source: demix_timestep
- id: demixfreqstep
@@ -278,6 +294,32 @@ steps:
label: concat_logfiles_dppp
'sbg:x': 4543.36376953125
'sbg:y': -6.564670085906982
+ - id: concat_logfiles_predict
+ in:
+ - id: file_list
+ source:
+ - predict/logfile
+ - id: file_prefix
+ default: predict_targ
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_predict
+ 'sbg:x': 4700
+ 'sbg:y': -6.564670085906982
+ - id: concat_logfiles_clipper
+ in:
+ - id: file_list
+ source:
+ - Ateamclipper/logfile
+ - id: file_prefix
+ default: Ateamclipper
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_clipper
+ 'sbg:x': 4900
+ 'sbg:y': -6.564670085906982
- id: dp3_execute
in:
- id: msout_name
@@ -323,6 +365,8 @@ steps:
default: apply_pa_1
- id: parmdb
source: parmdb
+ - id: solset
+ default: calibrator
- id: correction
default: polalign
out:
@@ -339,6 +383,8 @@ steps:
default: apply_bandpass_1
- id: parmdb
source: parmdb
+ - id: solset
+ default: calibrator
- id: correction
default: bandpass
out:
@@ -363,11 +409,57 @@ steps:
source: apply_phase_correction
- id: parmdb
source: parmdb
+ - id: updateweights
+ source: updateweights
out:
- id: selected_steps
run: ./prep_target_applycal.cwl
label: prep_target_applycal
'sbg:x': 2728.697998046875
'sbg:y': -18.151018142700195
+ - id: predict
+ in:
+ - id: msin
+ source: dp3_execute/msout
+ - id: msin_datacolumn
+ default: DATA
+ - id: msout_datacolumn
+ default: MODEL_DATA
+ - id: sources_db
+ source: skymodel
+ - id: storagemanager
+ default: Dysco
+ - id: databitrate
+ default: 0
+ - id: filter_baselines
+ source: process_baselines_target
+ - id: sources
+ default:
+ - 'VirA_4_patch'
+ - 'CygAGG'
+ - 'CasA_4_patch'
+ - 'TauAGG'
+ - id: usebeammodel
+ default: true
+ - id: onebeamperpatch
+ default: true
+ out:
+ - id: msout
+ - id: logfile
+ run: ../lofar-cwl/steps/filter_predict.cwl
+ 'sbg:x': 4500
+ 'sbg:y': -344.5472106933594
+ - id: Ateamclipper
+ in:
+ - id: msin
+ source: predict/msout
+ out:
+ - id: msout
+ - id: logfile
+ run: ../lofar-cwl/steps/Ateamclipper.cwl
+ 'sbg:x': 4700
+ 'sbg:y': -344.5472106933594
requirements:
- class: SubworkflowFeatureRequirement
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
diff --git a/subworkflow/prep_target_applycal.cwl b/subworkflow/prep_target_applycal.cwl
index a763ad8b..e3968609 100644
--- a/subworkflow/prep_target_applycal.cwl
+++ b/subworkflow/prep_target_applycal.cwl
@@ -18,13 +18,13 @@ inputs:
'sbg:x': 1990.8035888671875
'sbg:y': -230.75811767578125
- id: apply_rm_correction
- type: boolean
- 'sbg:x': 2273.170166015625
- 'sbg:y': -251.86672973632812
- - id: apply_beam_correction
type: boolean
'sbg:x': 2502.2421875
'sbg:y': -225.68701171875
+ - id: apply_beam_correction
+ type: boolean
+ 'sbg:x': 2273.170166015625
+ 'sbg:y': -251.86672973632812
- id: apply_phase_correction
type: boolean
'sbg:x': 2750.948974609375
@@ -33,6 +33,10 @@ inputs:
type: File
'sbg:x': 2007.115234375
'sbg:y': 685.0719604492188
+ - id: updateweights
+ type: boolean
+ 'sbg:x': 2167.219970703125
+ 'sbg:y': 422.6792297363281
outputs:
- id: selected_steps
outputSource:
@@ -50,6 +54,8 @@ steps:
default: apply_clock
- id: parmdb
source: parmdb
+ - id: solset
+ default: calibrator
- id: correction
default: clock
out:
@@ -66,6 +72,8 @@ steps:
default: apply_tec
- id: parmdb
source: parmdb
+ - id: solset
+ default: calibrator
- id: correction
default: tec
out:
@@ -77,11 +85,13 @@ steps:
in:
- id: steps
source:
- - apply_beam/selected_steps
+ - apply_rm/selected_steps
- id: step_name
default: apply_phase
- id: parmdb
source: parmdb
+ - id: solset
+ default: calibrator
- id: correction
default: phaseOrig
out:
@@ -93,32 +103,40 @@ steps:
in:
- id: steps
source:
- - apply_tec/selected_steps
+ - apply_beam/selected_steps
- id: step_name
default: apply_rm
- id: parmdb
source: parmdb
+ - id: solset
+ default: target
- id: correction
default: RMextract
out:
- id: augmented_steps
run: ../lofar-cwl/steps/DP3.ApplyCalStepGenerator.cwl
- 'sbg:x': 2321
- 'sbg:y': 285.3694763183594
+ 'sbg:x': 2545.4775390625
+ 'sbg:y': 291.2240295410156
- id: applybeam
in:
- id: steps
source:
- - apply_rm/selected_steps
+ - apply_tec/selected_steps
- id: step_name
default: apply_beam
+ - id: usechannelfreq
+ default: true
+ - id: updateweights
+ source: updateweights
+ - id: invert
+ default: true
- id: beammode
default: default
out:
- id: augmented_steps
run: ../lofar-cwl/steps/DP3.ApplyBeamStepGenerator.cwl
- 'sbg:x': 2545.4775390625
- 'sbg:y': 291.2240295410156
+ 'sbg:x': 2321
+ 'sbg:y': 285.3694763183594
- id: apply_beam
in:
- id: select_a
@@ -129,13 +147,13 @@ steps:
- id: b_steps
default: []
source:
- - apply_rm/selected_steps
+ - apply_tec/selected_steps
out:
- id: selected_steps
run: ../lofar-cwl/steps/DP3.Selector.cwl
label: apply_beam
- 'sbg:x': 2648.432373046875
- 'sbg:y': 167.062255859375
+ 'sbg:x': 2441.14599609375
+ 'sbg:y': 167.0730438232422
- id: apply_rm
in:
- id: select_a
@@ -146,13 +164,13 @@ steps:
- id: b_steps
default: []
source:
- - apply_tec/selected_steps
+ - apply_beam/selected_steps
out:
- id: selected_steps
run: ../lofar-cwl/steps/DP3.Selector.cwl
label: apply_rm?
- 'sbg:x': 2441.14599609375
- 'sbg:y': 167.0730438232422
+ 'sbg:x': 2648.432373046875
+ 'sbg:y': 167.062255859375
- id: apply_clock
in:
- id: select_a
@@ -197,7 +215,7 @@ steps:
- id: b_steps
default: []
source:
- - apply_beam/selected_steps
+ - apply_rm/selected_steps
out:
- id: selected_steps
run: ../lofar-cwl/steps/DP3.Selector.cwl
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
index 2fd96332..9344d9cc 100644
--- a/workflows/prefactor_target/prep.cwl
+++ b/workflows/prefactor_target/prep.cwl
@@ -208,6 +208,8 @@ outputs:
- losoto_plot_RM/logfile
- compare_station_list/logfile
- concat_logfiles_prep_targ/output
+ - concat_logfiles_predict_targ/output
+ - concat_logfiles_clipper_targ/output
type: 'File[]'
linkMerge: merge_flattened
'sbg:x': 1000
@@ -347,7 +349,7 @@ steps:
- id: demix_freqstep
default: 1
source: demix_freqstep
- - id: demix_baseline
+ - id: process_baselines_target
default: '*'
source: process_baselines_target
- id: target_source
@@ -377,9 +379,13 @@ steps:
default: true
- id: filter_baselines
source: compare_station_list/filter_out
+ - id: updateweights
+ source: updateweights
out:
- id: msout
- - id: logfile
+ - id: prep_logfile
+ - id: predict_logfile
+ - id: clipper_logfile
- id: parset
run: ../../subworkflow/ndppp_prep_targ.cwl
label: ndppp_prep_target
@@ -391,14 +397,40 @@ steps:
in:
- id: file_list
linkMerge: merge_flattened
- source: dppp_prep_target/logfile
+ source: dppp_prep_target/prep_logfile
- id: file_prefix
default: ndppp_prep_targ
out:
- id: output
run: ../../steps/concatenate_files.cwl
- label: concat_logfiles_prep_targ
+ label: concat_logfiles_prep_target
'sbg:x': 250
'sbg:y': 300
+ - id: concat_logfiles_predict_targ
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source: dppp_prep_target/predict_logfile
+ - id: file_prefix
+ default: predict_targ
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_predict_targ
+ 'sbg:x': 250
+ 'sbg:y': 400
+ - id: concat_logfiles_clipper_targ
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source: dppp_prep_target/clipper_logfile
+ - id: file_prefix
+ default: Ateamclipper
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_clipper_targ
+ 'sbg:x': 250
+ 'sbg:y': 500
requirements:
- class: ScatterFeatureRequirement
\ No newline at end of file
--
GitLab