diff --git a/lofar-cwl/steps/Ateamclipper.cwl b/lofar-cwl/steps/Ateamclipper.cwl
index 32b08a2af4f6fc862da62ec90e51cff84636a19b..0276008ef13c5626bf3edadb912c896eb9ed603e 100755
--- a/lofar-cwl/steps/Ateamclipper.cwl
+++ b/lofar-cwl/steps/Ateamclipper.cwl
@@ -20,11 +20,15 @@ outputs:
doc: Output MS
type: Directory
outputBinding:
- glob: $(inputs.msin)
+ glob: $(inputs.msin.basename)
- id: logfile
type: 'File[]'
outputBinding:
glob: Ateamclipper.log
+ - id: output
+ type: File
+ outputBinding:
+ glob: Ateamclipper.txt
label: Ateamclipper
hints:
- class: InitialWorkDirRequirement
diff --git a/lofar-cwl/steps/filter_predict.cwl b/lofar-cwl/steps/filter_predict.cwl
index 458db4562e115537b3e2666627168bf95255a013..35e182e29facec80f1313e2e824279a74008fa03 100644
--- a/lofar-cwl/steps/filter_predict.cwl
+++ b/lofar-cwl/steps/filter_predict.cwl
@@ -42,7 +42,7 @@ inputs:
prefix: predict.sources=
separate: false
itemSeparator: ','
- valueFrom: '[$(self)]'
+ valueFrom: "[$(self.join(','))]"
- default: false
id: usebeammodel
type: boolean
@@ -99,7 +99,7 @@ outputs:
- id: logfile
type: File[]
outputBinding:
- glob: 'predict_cal*.log'
+ glob: 'filter_predict*.log'
arguments:
- 'steps=[filter,predict]'
- predict.beammode=array_factor
diff --git a/steps/identify_bad_antennas.cwl b/steps/identify_bad_antennas.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..a4cb554ed344c28119c6938e329e50d992ff1114
--- /dev/null
+++ b/steps/identify_bad_antennas.cwl
@@ -0,0 +1,59 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: identify_bad_antennas
+baseCommand:
+ - python3
+inputs:
+ - id: msin
+ type: Directory[]
+ doc: MS to compare with
+ inputBinding:
+ position: 0
+ - id: filter
+ type: string?
+ default: '*&'
+ doc: Filter these baselines for the comparison
+
+label: identifyBadAntennas.py
+arguments:
+ - '-c'
+ - |
+ import sys
+ import json
+ from identifyBadAntennas import main as identifyBadAntennas
+
+ mss = sys.argv[1:]
+ filter = '$(inputs.filter)'
+
+ output = identifyBadAntennas(mss, filter)
+
+ filter_out = output['filter']
+ cwl_output = {"filter_out": filter_out}
+
+ with open('./out.json', 'w') as fp:
+ json.dump(cwl_output, fp)
+outputs:
+ - id: filter_out
+ type: string
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: $(JSON.parse(self[0].contents).filter_out)
+ - id: logfile
+ type: File?
+ outputBinding:
+ glob: identifyBadAntennas.log
+
+requirements:
+ - class: InlineJavascriptRequirement
+
+hints:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: identifyBadAntennas.log
\ No newline at end of file
diff --git a/steps/plot_Ateamclipper.cwl b/steps/plot_Ateamclipper.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..6d5f88253ec404b3418b2e5934ce7b3240d8be7b
--- /dev/null
+++ b/steps/plot_Ateamclipper.cwl
@@ -0,0 +1,28 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: plot_Ateamclipper
+baseCommand:
+ - python3
+ - /usr/local/bin/plot_Ateamclipper.py
+inputs:
+ - id: clipper_output
+ type: File
+ inputBinding:
+ position: 1
+ - id: outputimage
+ type: string?
+ default: Ateamclipper.png
+ inputBinding:
+ position: 2
+outputs:
+ - id: output_imag
+ doc: Output image
+ type: File
+ outputBinding:
+ glob: $(inputs.outputimage)
+label: plot_Ateamclipper
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor:HBAcalibrator
diff --git a/subworkflow/ndppp_prep_targ.cwl b/subworkflow/ndppp_prep_targ.cwl
index 58e91c177114cdaa1f12c41b3fa89e2ce297f506..7b3cefdccbb1139b98a2d19d201568851c790369 100644
--- a/subworkflow/ndppp_prep_targ.cwl
+++ b/subworkflow/ndppp_prep_targ.cwl
@@ -126,6 +126,12 @@ outputs:
type: Directory
'sbg:x': 5502.318359375
'sbg:y': -279.49835205078125
+ - id: clipper_output
+ outputSource:
+ - Ateamclipper/output
+ type: File
+ 'sbg:x': 5502.318359375
+ 'sbg:y': -500
- id: parset
outputSource:
- dp3_execute/parset
@@ -435,10 +441,10 @@ steps:
source: process_baselines_target
- id: sources
default:
- - 'VirA_4_patch'
- - 'CygAGG'
- - 'CasA_4_patch'
- - 'TauAGG'
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
- id: usebeammodel
default: true
- id: onebeamperpatch
@@ -456,6 +462,7 @@ steps:
out:
- id: msout
- id: logfile
+ - id: output
run: ../lofar-cwl/steps/Ateamclipper.cwl
'sbg:x': 4700
'sbg:y': -344.5472106933594
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index 1aa9e6bca32d907d6b4e1e3fa19f5d738e8e4e71..e6d97096708490ebce16cba309bdeaac75aab245 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -10,7 +10,7 @@ inputs:
'sbg:x': -1000
'sbg:y': -1700
- id: cal_solutions
- type: 'File'
+ type: File
'sbg:x': -1000
'sbg:y': -1600
- id: refant
@@ -94,7 +94,7 @@ inputs:
type: boolean
default: false
'sbg:x': -1000
- 'sbg:y': 0
+ 'sbg:y': 0
- id: apply_clock
type: boolean
default: true
@@ -122,7 +122,7 @@ inputs:
'sbg:y': 500
- id: gsmcal_step
type: string?
- default: 'phase'
+ default: phase
'sbg:x': -1000
'sbg:y': 600
- id: updateweights
@@ -170,12 +170,12 @@ inputs:
'sbg:y': 1500
- id: skymodel_source
type: string?
- default: 'TGSS'
+ default: TGSS
'sbg:x': -1000
'sbg:y': 1600
- id: avg_timeresolution
type: float?
- default: 4.
+ default: 4
'sbg:x': -1000
'sbg:y': 1700
- id: avg_freqresolution
@@ -185,7 +185,7 @@ inputs:
'sbg:y': 1800
- id: avg_timeresolution_concat
type: float?
- default: 8.
+ default: 8
'sbg:x': -1000
'sbg:y': 1900
- id: avg_freqresolution_concat
@@ -199,9 +199,7 @@ inputs:
'sbg:x': -1000
'sbg:y': 2100
- id: reference_stationSB
- type:
- - int?
- - string?
+ type: string?
default: ''
'sbg:x': -1000
'sbg:y': 1800
@@ -212,7 +210,7 @@ inputs:
'sbg:y': 1900
- id: ionex_prefix
type: string?
- default: 'CODG'
+ default: CODG
'sbg:x': -1000
'sbg:y': 2000
- id: proxy_server
@@ -242,10 +240,10 @@ inputs:
'sbg:y': 2500
outputs:
- id: inspection
- linkMerge: merge_flattened
outputSource:
- prep/inspection
- type: File[]
+ type: 'File[]'
+ linkMerge: merge_flattened
'sbg:x': 2000
'sbg:y': -600
- id: solutions
@@ -256,13 +254,14 @@ outputs:
'sbg:y': -500
- id: msout
outputSource:
- - prep/msout
+ - concat/msout
type: 'Directory[]'
'sbg:x': 2000
'sbg:y': 0
- id: logfiles
outputSource:
- prep/logfiles
+ - concat/logfiles
type: 'File[]'
linkMerge: merge_flattened
'sbg:x': 2000
@@ -275,14 +274,26 @@ steps:
- msin
- id: cal_solutions
source: cal_solutions
+ - id: flag_baselines
+ source:
+ - flag_baselines
+ - id: process_baselines_target
+ source: process_baselines_target
- id: filter_baselines
source: filter_baselines
- id: raw_data
source: raw_data
- id: propagatesolutions
source: propagatesolutions
- - id: updateweights
- source: updateweights
+ - id: demix_sources
+ source:
+ - demix_sources
+ - id: demix_target
+ source: demix_target
+ - id: demix_freqstep
+ source: demix_freqstep
+ - id: demix_timestep
+ source: demix_timestep
- id: demix
source: demix
- id: apply_tec
@@ -297,32 +308,20 @@ steps:
source: apply_beam
- id: clipATeam
source: clipATeam
+ - id: updateweights
+ source: updateweights
- id: max_dppp_threads
source: max_dppp_threads
- id: memoryperc
source: memoryperc
- - id: flag_baselines
- source:
- - flag_baselines
- - id: avg_timeresolution
- source: avg_timeresolution
- - id: avg_freqresolution
- source: avg_freqresolution
- - id: process_baselines_target
- source: process_baselines_target
- - id: demix_timestep
- source: demix_timestep
- - id: demix_freqstep
- source: demix_freqstep
- - id: demix_target
- source: demix_target
- - id: demix_sources
- source:
- - demix_sources
- id: min_separation
source: min_separation
- id: A-Team_skymodel
source: A-Team_skymodel
+ - id: avg_timeresolution
+ source: avg_timeresolution
+ - id: avg_freqresolution
+ source: avg_freqresolution
- id: ionex_server
source: ionex_server
- id: ionex_prefix
@@ -339,12 +338,26 @@ steps:
source: proxy_pass
out:
- id: outh5parm
- - id: logfiles
- id: inspection
- id: msout
+ - id: logfiles
run: ./prefactor_target/prep.cwl
label: prep
'sbg:x': 0
'sbg:y': 0
+ - id: concat
+ in:
+ - id: msin
+ source:
+ - prep/msout
+ - id: filter_baselines
+ source: process_baselines_target
+ out:
+ - id: msout
+ - id: logfiles
+ run: ./prefactor_target/concat.cwl
+ label: concat
+ 'sbg:x': 500
+ 'sbg:y': 0
requirements:
- class: SubworkflowFeatureRequirement
diff --git a/workflows/prefactor_target/concat.cwl b/workflows/prefactor_target/concat.cwl
index 3faf8dbb4f11839314fe40720c4f8e39031317bb..d8978090f678cdbbadbddce4a8d9ad22185bb7c2 100644
--- a/workflows/prefactor_target/concat.cwl
+++ b/workflows/prefactor_target/concat.cwl
@@ -1,7 +1,7 @@
class: Workflow
cwlVersion: v1.0
-id: prep
-label: prep
+id: concat
+label: concat
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
@@ -13,369 +13,35 @@ inputs:
type: string
'sbg:x': -1000
'sbg:y': -800
- - id: raw_data
- type: boolean
- 'sbg:x': -1000
- 'sbg:y': -700
- - id: propagatesolutions
- type: boolean
- 'sbg:x': -1000
- 'sbg:y': -600
- - id: flagunconverged
- type: boolean
- 'sbg:x': -1000
- 'sbg:y': -500
- - id: demix
- type: boolean
- 'sbg:x': -1000
- 'sbg:y': -400
- - id: max_dppp_threads
- type: int?
- default: 10
- 'sbg:x': -1000
- 'sbg:y': -300
- - id: memoryperc
- type: int
- 'sbg:x': -1000
- 'sbg:y': -200
- - id: flag_baselines
- type: 'string[]'
- 'sbg:x': -1000
- 'sbg:y': -100
- - id: avg_timeresolution
- type: float
- 'sbg:x': -1000
- 'sbg:y': 0
- - id: avg_freqresolution
- type: string
- 'sbg:x': -1000
- 'sbg:y': 100
- - id: process_baselines_cal
- type: string
- 'sbg:x': -1000
- 'sbg:y': 200
- - id: demix_timestep
- type: int
- 'sbg:x': -1000
- 'sbg:y': 300
- - id: demix_freqstep
- type: int
- 'sbg:x': -1000
- 'sbg:y': 400
- - id: demix_target
- type: string
- 'sbg:x': -1000
- 'sbg:y': 500
- - id: demix_sources
- type: 'string[]'
- 'sbg:x': -1000
- 'sbg:y': 600
- - id: min_length
- type: int?
- 'sbg:x': -1000
- 'sbg:y': 700
- - id: overhead
- type: float?
- 'sbg:x': -1000
- 'sbg:y': 800
- - id: min_separation
- type: int?
- 'sbg:x': -1000
- 'sbg:y': 900
- - id: do_smooth
- type: boolean?
- default: false
- 'sbg:x': -1000
- 'sbg:y': 1000
- - id: max_separation_arcmin
- type: float?
- 'sbg:x': -1000
- 'sbg:y': 1100
- - id: calibrator_path_skymodel
- type: Directory?
- 'sbg:x': -1000
- 'sbg:y': 1200
- - id: A-Team_skymodel
- type: File?
- 'sbg:x': -1000
- 'sbg:y': 1300
- - id: elevation
- type: string
- default: 0deg..20deg
- 'sbg:x': -1000
- 'sbg:y': 1400
- - id: amplmin
- type: float
- default: 1.e-30
- 'sbg:x': -1000
- 'sbg:y': 1500
outputs:
- - id: outh5parm
- outputSource:
- - h5parm_collector/outh5parm
- type: File
- 'sbg:x': 1000
- 'sbg:y': -500
- - id: check_Ateam_separation.png
- outputSource:
- - check_ateam_separation/output_imag
- type: File
- 'sbg:x': 1000
- 'sbg:y': -300
- id: msout
outputSource:
- - predict_calibrate/msout
+ - msin
type: 'Directory[]'
'sbg:x': 1000
'sbg:y': 0
- - id: parset
- outputSource:
- - ndppp_prep_cal/parset
- type: 'File[]'
- 'sbg:x': 1000
- 'sbg:y': 200
- - id: calibrator_name
- outputSource:
- - find_skymodel_cal/model_name
- type: string
- 'sbg:x': 1000
- 'sbg:y': 400
- - id: outh5parm_logfile
- outputSource:
- - h5parm_collector/log
- type: File[]
- 'sbg:x': 1000
- 'sbg:y': 650
- id: logfiles
linkMerge: merge_flattened
outputSource:
- - concat_logfiles_calib/output
- - concat_logfiles_predict/output
- - concat_logfiles_blsmooth/output
- - make_sourcedb/log
- - find_skymodel_cal/logfile
- - concat_logfiles_prep_cal/output
- - make_sourcedb_ateam/log
- - check_ateam_separation/logfile
+ - identifybadantennas/logfile
type: 'File[]'
'sbg:x': 1000
'sbg:y': 800
steps:
- - id: ndppp_prep_cal
+ - id: identifybadantennas
in:
- - id: baselines_to_flag
- default: []
- source:
- - flag_baselines
- - id: elevation_to_flag
- source: elevation
- - id: min_amplitude_to_flag
- source: amplmin
- - id: memoryperc
- default: 20
- source: memoryperc
- - id: raw_data
- default: false
- source: raw_data
- - id: demix
- default: false
- source: demix
- id: msin
- linkMerge: merge_flattened
source:
- msin
- - id: msin_baseline
- default: '*'
+ - id: filter
source: filter_baselines
- - id: skymodel
- source: make_sourcedb_ateam/sourcedb
- - id: timeresolution
- default: 1
- source: avg_timeresolution
- - id: freqresolution
- default: 12.21kHz
- source: avg_freqresolution
- - id: demix_timestep
- default: 1
- source: demix_timestep
- - id: demix_freqstep
- default: 1
- source: demix_freqstep
- - id: demix_baseline
- default: '*'
- source: process_baselines_cal
- - id: target_source
- source: demix_target
- - id: ntimechunk
- default: 10
- source: max_dppp_threads
- - id: subtract_sources
- source:
- - demix_sources
- out:
- - id: msout
- - id: logfile
- - id: parset
- run: ../../subworkflow/ndppp_prep_cal.cwl
- label: ndppp_prep_cal
- scatter:
- - msin
- 'sbg:x': 0
- 'sbg:y': 0
- - id: predict_calibrate
- in:
- - id: msin
- linkMerge: merge_flattened
- source:
- - ndppp_prep_cal/msout
- - id: do_smooth
- source: do_smooth
- - id: sourcedb
- source: make_sourcedb/sourcedb
- - id: flagunconverged
- source: flagunconverged
- - id: propagatesolutions
- source: propagatesolutions
- out:
- - id: msout
- - id: BLsmooth.log
- - id: predict_cal.log
- - id: calib_cal.log
- - id: outh5parm
- run: ../../subworkflow/predict_calibrate.cwl
- label: predict_calibrate
- scatter:
- - msin
- 'sbg:x': 500
- 'sbg:y': 0
- - id: check_ateam_separation
- in:
- - id: ms
- source:
- - msin
- - id: min_separation
- source: min_separation
out:
- - id: output_imag
+ - id: filter_out
- id: logfile
- run: ../../lofar-cwl/steps/check_ateam_separation.cwl
- label: check_Ateam_separation
+ run: ../../steps/identify_bad_antennas.cwl
+ label: identifybadantennas
'sbg:x': 0
'sbg:y': -300
- - id: make_sourcedb_ateam
- in:
- - id: sky_model
- source: A-Team_skymodel
- - id: logname
- default: make_sourcedb_ateam.log
- out:
- - id: sourcedb
- - id: log
- run: ../../lofar-cwl/steps/makesourcedb.cwl
- label: make_sourcedb_ateam
- 'sbg:x': -500
- 'sbg:y': 300
- - id: find_skymodel_cal
- in:
- - id: msin
- source:
- - msin
- - id: skymodels
- source: calibrator_path_skymodel
- - id: max_separation_arcmin
- source: max_separation_arcmin
- out:
- - id: output_models
- - id: model_name
- - id: logfile
- run: ../../steps/find_skymodel_cal.cwl
- label: find_skymodel_cal
- 'sbg:x': -500
- 'sbg:y': 500
- - id: make_sourcedb
- in:
- - id: sky_model
- source: find_skymodel_cal/output_models
- - id: output_file_name
- default: calibrator.sourcedb
- out:
- - id: sourcedb
- - id: log
- run: ../../lofar-cwl/steps/makesourcedb.cwl
- label: make_sourcedb
- 'sbg:x': 0
- 'sbg:y': 500
- - id: concat_logfiles_prep_cal
- in:
- - id: file_list
- linkMerge: merge_flattened
- source: ndppp_prep_cal/logfile
- - id: file_prefix
- default: ndppp_prep_cal
- out:
- - id: output
- run: ../../steps/concatenate_files.cwl
- label: concat_logfiles_prep_cal
- 'sbg:x': 250
- 'sbg:y': 300
- - id: concat_logfiles_blsmooth
- in:
- - id: file_list
- source:
- - predict_calibrate/BLsmooth.log
- - id: file_prefix
- default: blsmooth_prep
- out:
- - id: output
- run: ../../steps/concatenate_files.cwl
- label: concat_logfiles_blsmooth
- 'sbg:x': 750
- 'sbg:y': 300
- - id: concat_logfiles_predict
- in:
- - id: file_list
- linkMerge: merge_flattened
- source:
- - predict_calibrate/predict_cal.log
- - id: file_prefix
- default: predict_cal
- out:
- - id: output
- run: ../../steps/concatenate_files.cwl
- label: concat_logfiles_predict
- 'sbg:x': 750
- 'sbg:y': 500
- - id: concat_logfiles_calib
- in:
- - id: file_list
- source:
- - predict_calibrate/calib_cal.log
- - id: file_prefix
- default: calib_cal_prep
- out:
- - id: output
- run: ../../steps/concatenate_files.cwl
- label: concat_logfiles_calib
- 'sbg:x': 750
- 'sbg:y': 800
- - id: h5parm_collector
- in:
- - id: h5parmFiles
- source:
- - predict_calibrate/outh5parm
- - id: squeeze
- default: true
- - id: verbose
- default: true
- - id: clobber
- default: true
- out:
- - id: outh5parm
- - id: log
- run: ../../lofar-cwl/steps/H5ParmCollector.cwl
- label: H5parm_collector
- 'sbg:x': 750
- 'sbg:y': -300
requirements:
- class: SubworkflowFeatureRequirement
- class: ScatterFeatureRequirement
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
index 9344d9cc3c4cb2aac12d89dc95845d44f16d0ecf..25ae65475d0bd0881153258bb080a7e1d0c60d2b 100644
--- a/workflows/prefactor_target/prep.cwl
+++ b/workflows/prefactor_target/prep.cwl
@@ -190,6 +190,7 @@ outputs:
outputSource:
- check_ateam_separation/output_imag
- losoto_plot_RM/output_plots
+ - plot_Ateamclipper/output_imag
type: 'File[]?'
linkMerge: merge_flattened
'sbg:x': 1000
@@ -386,6 +387,7 @@ steps:
- id: prep_logfile
- id: predict_logfile
- id: clipper_logfile
+ - id: clipper_output
- id: parset
run: ../../subworkflow/ndppp_prep_targ.cwl
label: ndppp_prep_target
@@ -393,6 +395,31 @@ steps:
- msin
'sbg:x': 0
'sbg:y': 0
+ - id: plot_Ateamclipper
+ in:
+ - id: clipper_output
+ source: concat_logfiles_clipper_output/output
+ out:
+ - id: output_imag
+ run: ../../steps/plot_Ateamclipper.cwl
+ label: concat_logfiles_clipper_output
+ 'sbg:x': 600
+ 'sbg:y': 200
+ - id: concat_logfiles_clipper_output
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source: dppp_prep_target/clipper_output
+ - id: file_prefix
+ default: Ateamclipper
+ - id: file_suffix
+ default: txt
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_clipper_output
+ 'sbg:x': 400
+ 'sbg:y': 200
- id: concat_logfiles_prep_targ
in:
- id: file_list
@@ -404,7 +431,7 @@ steps:
- id: output
run: ../../steps/concatenate_files.cwl
label: concat_logfiles_prep_target
- 'sbg:x': 250
+ 'sbg:x': 400
'sbg:y': 300
- id: concat_logfiles_predict_targ
in:
@@ -417,7 +444,7 @@ steps:
- id: output
run: ../../steps/concatenate_files.cwl
label: concat_logfiles_predict_targ
- 'sbg:x': 250
+ 'sbg:x': 400
'sbg:y': 400
- id: concat_logfiles_clipper_targ
in:
@@ -430,7 +457,7 @@ steps:
- id: output
run: ../../steps/concatenate_files.cwl
label: concat_logfiles_clipper_targ
- 'sbg:x': 250
+ 'sbg:x': 400
'sbg:y': 500
requirements:
- class: ScatterFeatureRequirement
\ No newline at end of file