Commit 0abf7d39 authored by alex's avatar alex

add wide-band flagging, limit cal outputs

parent ef21e782
Pipeline #10294 failed with stages
in 11 minutes and 7 seconds
......@@ -118,4 +118,10 @@ outputs:
type: Directory[]
outputBinding:
outputEval: $(inputs.msin)
- id: logfile
type: File?
outputBinding:
glob: aoflag.log
label: AOFlagger
stdout: aoflag.log
\ No newline at end of file
......@@ -77,7 +77,7 @@ outputs:
stdout: applycal_$(inputs.correction).log
stderr: applycal_$(inputs.correction)_err.log
arguments:
- 'steps=[applycal]'
- 'steps=[applycal, count]'
- msout=.
requirements:
- class: InitialWorkDirRequirement
......
class: CommandLineTool
cwlVersion: v1.0
cwlVersion: v1.1
$namespaces:
sbg: 'https://www.sevenbridges.com/'
id: ms_concat
baseCommand:
- python3
- /usr/local/bin/concat_MS_CWL.py
- /bin/bash
- script.sh
inputs:
- default: 50
id: min_length
......@@ -16,6 +16,7 @@ inputs:
doc: Minimum amount of subbands to concatenate in frequency.
- id: overhead
type: float?
default: 0.8
inputBinding:
position: 0
prefix: '--overhead'
......@@ -24,9 +25,9 @@ inputs:
of data to be concatenated.
- id: msin
type: 'Directory[]'
inputBinding:
position: 1
shellQuote: false
# inputBinding:
# position: 1
# shellQuote: false
- default: out.MS
id: msout
type: string
......@@ -34,18 +35,24 @@ inputs:
position: 2
outputs:
- id: concat_meta_ms
type: 'Directory[]'
type: 'Directory'
outputBinding:
glob: |
$(inputs.msout)_[0-9*]
workdir/$(inputs.msout)_[0-9*]
- id: concat_additional_ms
type: 'Directory[]'
type: 'Directory'
outputBinding:
glob: $(inputs.msout)_*_CONCAT
glob: |
workdir/$(inputs.msout)_*_CONCAT
- id: ms_outs
type: 'Directory[]'
outputBinding:
outputEval: $(inputs.msin)
glob: 'workdir/out_*'
# outputEval: $(inputs.msin)
- id: logfile
type: File?
outputBinding:
glob: ms_concat.log
label: ms_concat
requirements:
- class: ShellCommandRequirement
......@@ -55,4 +62,34 @@ requirements:
listing:
- entry: $(inputs.msin)
writable: true
- entryname: script.sh
entry: |
#!/bin/bash
directories=\$(ls -d out_*)
#echo \$directories
mkdir -pv workdir
cp -r \$directories workdir/. && cd workdir
python3 /usr/local/bin/concat_MS_CWL.py \$1 \$2 \$3 \$4 \$directories \$5
- class: InplaceUpdateRequirement
inplaceUpdate: true
- class: InlineJavascriptRequirement
stdout: ms_concat.log
#
# mkdir -pv dirs && cp -rv \$directories dirs/. && cd dirs
# directories=\$(ls -d out_*)
#
#- entryname: script.sh
# entry: |
# #!/bin/bash
#
# ls
#- python3
#- /usr/local/bin/concat_MS_CWL.py
# directories=$(ls out_*)
# python3 /usr/local/bin/concat_MS_CWL.py $@ $directories
......@@ -5,6 +5,7 @@ $namespaces:
id: findRefAnt_join
baseCommand:
- python3
- script.py
inputs:
- id: flagged_fraction_dict
type: string[]?
......@@ -16,50 +17,6 @@ inputs:
doc: Filter these baselines for the comparison
label: findRefAnt_join
arguments:
- '-c'
- |
import sys
import json
import re
import ast
flagged_fraction_dict_list = $(inputs.flagged_fraction_dict)
filter_station = '$(inputs.filter_station)'
flagged_fraction_data = {}
for flagged_fraction_dict in flagged_fraction_dict_list:
entry = ast.literal_eval(flagged_fraction_dict)
antennas = entry.keys()
selected_stations = [ station_name for station_name in antennas if re.match(filter_station, station_name) ]
if len(selected_stations) == 0:
raiseError('No stations left after filtering.')
for antenna in selected_stations:
try:
flagged_fraction_data[antenna].append(float(entry[antenna]))
except KeyError:
flagged_fraction_data[antenna] = [float(entry[antenna])]
flagged_fraction_list = []
sorted_stations = sorted(flagged_fraction_data.keys())
for antenna in sorted_stations:
flagged_fraction = sum(flagged_fraction_data[antenna]) / len(flagged_fraction_data[antenna])
flagged_fraction_list.append(flagged_fraction)
try:
flagged_fraction_data[flagged_fraction].append(antenna)
except KeyError:
flagged_fraction_data[flagged_fraction] = [antenna]
min_flagged_fraction = min(flagged_fraction_list)
refant = flagged_fraction_data[min_flagged_fraction][0]
print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
cwl_output = {'refant': str(refant)}
with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp)
outputs:
- id: refant
......@@ -75,6 +32,53 @@ outputs:
requirements:
- class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement
listing:
- entryname: input.json
entry: $(inputs.flagged_fraction_dict)
- entryname: script.py
entry: |
import sys
import json
import re
import ast
with open('input.json', 'r') as f_stream:
flagged_fraction_dict_list = json.load(f_stream)
filter_station = '$(inputs.filter_station)'
flagged_fraction_data = {}
for flagged_fraction_dict in flagged_fraction_dict_list:
entry = ast.literal_eval(flagged_fraction_dict)
antennas = entry.keys()
selected_stations = [ station_name for station_name in antennas if re.match(filter_station, station_name) ]
if len(selected_stations) == 0:
raiseError('No stations left after filtering.')
for antenna in selected_stations:
try:
flagged_fraction_data[antenna].append(float(entry[antenna]))
except KeyError:
flagged_fraction_data[antenna] = [float(entry[antenna])]
flagged_fraction_list = []
sorted_stations = sorted(flagged_fraction_data.keys())
for antenna in sorted_stations:
flagged_fraction = sum(flagged_fraction_data[antenna]) / len(flagged_fraction_data[antenna])
flagged_fraction_list.append(flagged_fraction)
try:
flagged_fraction_data[flagged_fraction].append(antenna)
except KeyError:
flagged_fraction_data[flagged_fraction] = [antenna]
min_flagged_fraction = min(flagged_fraction_list)
refant = flagged_fraction_data[min_flagged_fraction][0]
print('Selected station ' + str(refant) + ' as reference antenna. Fraction of flagged data is ' + '{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
cwl_output = {'refant': str(refant)}
with open('./out.json', 'w') as fp:
json.dump(cwl_output, fp)
stdout: findRefAnt.log
stderr: findRefAnt_err.log
\ No newline at end of file
stderr: findRefAnt_err.log
......@@ -32,12 +32,6 @@ outputs:
type: File
'sbg:x': 1000
'sbg:y': -300
- id: msout
outputSource:
- calib_cal/msout
type: Directory
'sbg:x': 1000
'sbg:y': -200
- id: BLsmooth.log
outputSource:
- BLsmooth/logfile
......@@ -205,7 +199,6 @@ steps:
- id: mode
default: diagonal
out:
- id: msout
- id: h5parm
- id: logfile
run: ../lofar-cwl/steps/ddecal.cwl
......
......@@ -34,23 +34,11 @@ inputs:
'sbg:x': -1000
'sbg:y': 300
outputs:
- id: msout_old
- id: msout
outputSource:
- dpppconcat/msout
type: Directory
'sbg:x': 1000
'sbg:y': -200
- id: unflagged_fraction
outputSource:
- check_unflagged_fraction/unflagged_fraction
type: float
'sbg:x': 1000
'sbg:y': -100
- id: msout
outputSource:
- check_unflagged_fraction/msout
type: 'Directory[]'
'sbg:x': 1000
'sbg:y': 0
- id: dpppconcat.log
outputSource:
......@@ -58,12 +46,6 @@ outputs:
type: File
'sbg:x': 1000
'sbg:y': 100
- id: check_unflagged_fraction.log
outputSource:
- concat_logfiles_unflagged/output
type: File
'sbg:x': 1000
'sbg:y': 200
steps:
- id: filter_ms_group
in:
......@@ -116,20 +98,6 @@ steps:
label: dpppconcat
'sbg:x': -200
'sbg:y': 0
- id: check_unflagged_fraction
in:
- id: msin
source: dpppconcat/msout
- id: min_fraction
source: min_unflagged_fraction
out:
- id: msout
- id: unflagged_fraction
- id: logfile
run: ../steps/check_unflagged_fraction.cwl
label: check_unflagged_fraction
'sbg:x': 100
'sbg:y': 0
- id: concat_logfiles_dpppconcat
in:
- id: file_list
......@@ -143,17 +111,5 @@ steps:
label: concat_logfiles_dpppconcat
'sbg:x': 600
'sbg:y': 200
- id: concat_logfiles_unflagged
in:
- id: file_list
source:
- check_unflagged_fraction/logfile
- id: file_prefix
default: check_unflagged_fraction
out:
- id: output
run: ../steps/concatenate_files.cwl
label: concat_logfiles_unflagged
'sbg:x': 600
'sbg:y': 400
requirements: []
......@@ -201,12 +201,6 @@ inputs:
'sbg:x': -1000
'sbg:y': 1900
outputs:
- id: logs
outputSource:
- save_logfiles/log_dir
type: Directory
'sbg:x': 1500
'sbg:y': -100
- id: results
outputSource:
- save_results/log_dir
......@@ -356,7 +350,7 @@ steps:
source:
- save_solutions/log_dir
- save_inspection/log_dir
- prefactor/msout
- save_logfiles/log_dir
- id: sub_directory_name
default: results
out:
......
......@@ -218,18 +218,6 @@ outputs:
type: File
'sbg:x': 2000
'sbg:y': -500
- id: msout
outputSource:
- bp/msout
type: 'Directory[]'
'sbg:x': 2000
'sbg:y': 0
- id: parset
outputSource:
- ion/parsets
type: 'File[]'
'sbg:x': 2000
'sbg:y': 200
- id: logfiles
outputSource:
- prep/logfiles
......@@ -456,7 +444,6 @@ steps:
- id: calibrator_name
source: prep/calibrator_name
out:
- id: msout
- id: outsolutions
- id: inspection
- id: logfiles
......@@ -492,7 +479,6 @@ steps:
- id: outsolutions
- id: inspection
- id: logfiles
- id: parsets
run: ./prefactor_calibrator/ion.cwl
label: ion
'sbg:x': 1600
......
......@@ -56,7 +56,7 @@ inputs:
'sbg:x': -1000
'sbg:y': 300
- id: inh5parm_logfile
type: File[]
type: 'File[]'
'sbg:x': -1000
'sbg:y': 400
- id: do_smooth
......@@ -87,12 +87,6 @@ outputs:
type: File
'sbg:x': 1000
'sbg:y': -200
- id: msout
outputSource:
- apply_calibrate_bp/msout
type: 'Directory[]'
'sbg:x': 1000
'sbg:y': -100
- id: inspection
outputSource:
- losoto_plot_A1/output_plots
......@@ -127,18 +121,19 @@ outputs:
- id: outh5parm_logfile
outputSource:
- h5parm_collector/log
type: File[]
type: 'File[]'
'sbg:x': 1000
'sbg:y': 300
steps:
- id: bandpass
in:
- id: ampRange
source: ampRange
- id: input_h5parm
source: h5parm
source:
- ampRange
- id: skipInternational
source: skipInternational
- id: input_h5parm
source: h5parm
- id: max2interpolate
source: max2interpolate
- id: bandpass_freqresolution
......@@ -171,6 +166,7 @@ steps:
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_A1
'sbg:x': -200
......@@ -194,6 +190,7 @@ steps:
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_A2
'sbg:x': -200
......@@ -217,6 +214,7 @@ steps:
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_B1
'sbg:x': -200
......@@ -234,18 +232,19 @@ steps:
default: ant
- id: axisInCol
default: pol
- id: plotFlag
default: true
- id: time.minmaxstep
default:
- 0
- 1e20
- 100000000000000000000
- 500000
- id: plotFlag
default: true
- id: prefix
default: bandpass_
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_B2
'sbg:x': -200
......@@ -261,18 +260,19 @@ steps:
- freq
- id: axisInCol
default: ant
- id: plotFlag
default: true
- id: time.minmaxstep
default:
- 0
- 1e20
- 100000000000000000000
- 500000
- id: plotFlag
default: true
- id: prefix
default: bandpass_
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_B3
'sbg:x': -200
......@@ -294,6 +294,8 @@ steps:
- id: prefix
default: bandpass_transfer_
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_B1_trans
......@@ -312,16 +314,18 @@ steps:
default: ant
- id: axisInCol
default: pol
- id: plotFlag
default: true
- id: time.minmaxstep
default:
- 0
- 1e20
- 100000000000000000000
- 500000
- id: plotFlag
default: true
- id: prefix
default: bandpass_transfer_
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_B2_trans
......@@ -338,16 +342,18 @@ steps:
- freq
- id: axisInCol
default: ant
- id: plotFlag
default: true
- id: time.minmaxstep
default:
- 0
- 1e20
- 100000000000000000000
- 500000
- id: plotFlag
default: true
- id: prefix
default: bbandpass_transfer_
out:
- id: output_plots
- id: logfile
- id: parset
run: ../../lofar-cwl/steps/LoSoTo.Plot.cwl
label: losoto_plot_B3_trans
......@@ -380,14 +386,12 @@ steps:
in:
- id: h5parmFile
source: bandpass/output_h5parm
- id: input_file
source: insolutions
- id: outsolset
default: calibrator
- id: insoltab
default: bandpass
- id: outh5parmname
default: cal_solutions.h5
- id: input_file
source: insolutions
- id: squeeze
default: true
- id: verbose
......@@ -412,14 +416,13 @@ steps:
- id: input_h5parm
source: transfer_solutions/outh5parm
out:
- id: msout
- id: apply_fr.log
- id: BLsmooth.log
- id: apply_cal.log
- id: apply_pa.log
- id: apply_fr.log
- id: calib_cal.log
- id: applybeam.log
- id: outh5parm
- id: apply_pa.log
- id: applybeam.log
run: ../../subworkflow/apply_calibrate_bp.cwl
label: apply_calibrate_bp
scatter:
......
......@@ -70,13 +70,6 @@ outputs:
linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': 200
- id: parsets
outputSource:
- clocktec/parset
type: 'File[]'
linkMerge: merge_flattened
'sbg:x': 1000
'sbg:y': 300
steps:
- id: clocktec
in:
......
......@@ -101,7 +101,7 @@ inputs:
'sbg:y': 1205
- id: elevation
type: string
default: 0deg..20deg
default: 0deg..15deg
'sbg:x': 254.234375
'sbg:y': 791
- id: amplmin
......@@ -149,6 +149,7 @@ outputs:
- id: logfiles
outputSource:
- concat_logfiles_calib/output
- ms_concat/logfile
- concat_logfiles_predict/output
- concat_logfiles_blsmooth/output
- make_sourcedb/log
......@@ -156,6 +157,7 @@ outputs:
- concat_logfiles_prep_cal/output
- make_sourcedb_ateam/log
- check_ateam_separation/logfile
- aoflag/logfile
type: 'File[]'
linkMerge: merge_flattened
'sbg:x': 1554.5106201171875
......@@ -169,7 +171,7 @@ steps:
out:
- id: output
run: ../../steps/selectfirstdirectory.cwl
label: select_ony_first
label: select_only_first
'sbg:x': -303.7861022949219
'sbg:y': 542.37646484375
- id: ndppp_prep_cal
......@@ -238,7 +240,7 @@ steps:
- id: msin
linkMerge: merge_flattened
source:
- ndppp_prep_cal/msout
- aoflag/output_ms
- id: do_smooth
source: do_smooth
- id: sourcedb
......@@ -259,6 +261,46 @@ steps:
- msin
'sbg:x': 1027.490966796875
'sbg:y': 749
- id: ms_concat
in:
- id: min_length
source: min_length
- id: overhead
source: overhead
- id: msin
linkMerge: merge_flattened
source: