Skip to content
Snippets Groups Projects
Commit e44ea0aa authored by alex's avatar alex
Browse files

update logging output

Former-commit-id: 3ffe3789 [formerly 38fe83c8]
Former-commit-id: f41d337d
Former-commit-id: bf85d7cf
parent 18ed43e2
No related branches found
No related tags found
No related merge requests found
...@@ -17,7 +17,7 @@ inputs: ...@@ -17,7 +17,7 @@ inputs:
type: string type: string
inputBinding: inputBinding:
position: 0 position: 0
prefix: msout=msout_name prefix: msout=
separate: false separate: false
doc: Name of output MS doc: Name of output MS
- default: DATA - default: DATA
...@@ -67,6 +67,7 @@ inputs: ...@@ -67,6 +67,7 @@ inputs:
default: "" default: ""
inputBinding: inputBinding:
prefix: msout.storagemanager= prefix: msout.storagemanager=
separate: false
- id: databitrate - id: databitrate
type: int? type: int?
inputBinding: inputBinding:
...@@ -85,7 +86,7 @@ outputs: ...@@ -85,7 +86,7 @@ outputs:
outputBinding: outputBinding:
glob: $(inputs.msin.basename) glob: $(inputs.msin.basename)
- id: logfile - id: logfile
type: File type: File[]
outputBinding: outputBinding:
glob: 'applycal_$(inputs.correction).log' glob: 'applycal_$(inputs.correction).log'
stdout: applycal_$(inputs.correction).log stdout: applycal_$(inputs.correction).log
......
...@@ -27,7 +27,6 @@ arguments: ...@@ -27,7 +27,6 @@ arguments:
from check_unflagged_fraction import main as check_unflagged_fraction from check_unflagged_fraction import main as check_unflagged_fraction
print(sys.argv)
ms = sys.argv[1] ms = sys.argv[1]
min_fraction = float($(inputs.min_fraction)) min_fraction = float($(inputs.min_fraction))
...@@ -35,11 +34,7 @@ arguments: ...@@ -35,11 +34,7 @@ arguments:
filename = os.path.basename(output['flagged']) filename = os.path.basename(output['flagged'])
unflagged_fraction = output['unflagged_fraction'] unflagged_fraction = output['unflagged_fraction']
print(os.getcwd())
print(filename)
print(filename.lstrip('out_'))
if filename != 'None': if filename != 'None':
shutil.move(filename, filename.lstrip('out_')) shutil.move(filename, filename.lstrip('out_'))
...@@ -61,9 +56,9 @@ outputs: ...@@ -61,9 +56,9 @@ outputs:
glob: 'out.json' glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).unflagged_fraction) outputEval: $(JSON.parse(self[0].contents).unflagged_fraction)
- id: logfile - id: logfile
type: File? type: File[]?
outputBinding: outputBinding:
glob: check_unflagged_fraction.log glob: 'check_unflagged_fraction*.log'
requirements: requirements:
- class: InlineJavascriptRequirement - class: InlineJavascriptRequirement
...@@ -75,4 +70,5 @@ requirements: ...@@ -75,4 +70,5 @@ requirements:
hints: hints:
- class: DockerRequirement - class: DockerRequirement
dockerPull: lofareosc/prefactor:HBAcalibrator dockerPull: lofareosc/prefactor:HBAcalibrator
stdout: check_unflagged_fraction.log stdout: check_unflagged_fraction.log
\ No newline at end of file stderr: check_unflagged_fraction_err.log
\ No newline at end of file
...@@ -51,9 +51,9 @@ outputs: ...@@ -51,9 +51,9 @@ outputs:
glob: 'out.json' glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).filter_out) outputEval: $(JSON.parse(self[0].contents).filter_out)
- id: logfile - id: logfile
type: File? type: File[]?
outputBinding: outputBinding:
glob: compareStationList.log glob: 'compareStationList*.log'
requirements: requirements:
- class: InlineJavascriptRequirement - class: InlineJavascriptRequirement
...@@ -61,4 +61,5 @@ requirements: ...@@ -61,4 +61,5 @@ requirements:
hints: hints:
DockerRequirement: DockerRequirement:
dockerPull: lofareosc/prefactor:HBAcalibrator dockerPull: lofareosc/prefactor:HBAcalibrator
stdout: compareStationList.log stdout: compareStationList.log
\ No newline at end of file stderr: compareStationList_err.log
\ No newline at end of file
id: merge_array_files
label: merge_array_files
class: ExpressionTool
cwlVersion: v1.0
inputs:
- id: input
type:
- type: array
items:
- type: array
items: File
outputs:
- id: output
type: File[]
expression: |
${
var out_file = []
for(var i=0; i<inputs.input.length; i++){
var item = inputs.input[i]
if(item != null){
out_file = out_file.concat(item)
}
}
return {'output': out_file}
}
requirements:
- class: InlineJavascriptRequirement
\ No newline at end of file
...@@ -24,19 +24,11 @@ arguments: ...@@ -24,19 +24,11 @@ arguments:
from plot_unflagged_fraction import main as plot_unflagged_fraction from plot_unflagged_fraction import main as plot_unflagged_fraction
print(sys.argv)
print(len(sys.argv))
center = int((len(sys.argv) - 1) / 2 + 1) center = int((len(sys.argv) - 1) / 2 + 1)
print(center)
mss = sys.argv[1:center] mss = sys.argv[1:center]
unflagged_fraction = sys.argv[center:] unflagged_fraction = sys.argv[center:]
print(mss)
print(unflagged_fraction)
print(os.getcwd() + '/unflagged_fraction.png')
output = plot_unflagged_fraction(ms_list = mss, frac_list = unflagged_fraction, outfile = os.getcwd() + '/unflagged_fraction.png') output = plot_unflagged_fraction(ms_list = mss, frac_list = unflagged_fraction, outfile = os.getcwd() + '/unflagged_fraction.png')
outputs: outputs:
...@@ -45,6 +37,11 @@ outputs: ...@@ -45,6 +37,11 @@ outputs:
type: File type: File
outputBinding: outputBinding:
glob: 'unflagged_fraction.png' glob: 'unflagged_fraction.png'
- id: logfile
type: File[]?
outputBinding:
glob: 'plot_unflagged_fraction*.log'
hints: hints:
- class: DockerRequirement - class: DockerRequirement
......
...@@ -6,7 +6,7 @@ $namespaces: ...@@ -6,7 +6,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/' sbg: 'https://www.sevenbridges.com/'
inputs: inputs:
- id: msin - id: msin
type: Directory[] type: 'Directory[]'
'sbg:x': -1000 'sbg:x': -1000
'sbg:y': -300 'sbg:y': -300
- id: group_id - id: group_id
...@@ -49,7 +49,7 @@ outputs: ...@@ -49,7 +49,7 @@ outputs:
- id: msout - id: msout
outputSource: outputSource:
- check_unflagged_fraction/msout - check_unflagged_fraction/msout
type: Directory[] type: 'Directory[]'
'sbg:x': 1000 'sbg:x': 1000
'sbg:y': 0 'sbg:y': 0
- id: dpppconcat.log - id: dpppconcat.log
...@@ -60,11 +60,10 @@ outputs: ...@@ -60,11 +60,10 @@ outputs:
'sbg:y': 100 'sbg:y': 100
- id: check_unflagged_fraction.log - id: check_unflagged_fraction.log
outputSource: outputSource:
- check_unflagged_fraction/logfile - concat_logfiles_unflagged/output
type: File type: File
'sbg:x': 1000 'sbg:x': 1000
'sbg:y': 200 'sbg:y': 200
steps: steps:
- id: filter_ms_group - id: filter_ms_group
in: in:
...@@ -73,7 +72,8 @@ steps: ...@@ -73,7 +72,8 @@ steps:
- id: groups_specification - id: groups_specification
source: groups_specification source: groups_specification
- id: measurement_sets - id: measurement_sets
source: msin source:
- msin
out: out:
- id: output - id: output
run: ../steps/filter_ms_group.cwl run: ../steps/filter_ms_group.cwl
...@@ -83,27 +83,28 @@ steps: ...@@ -83,27 +83,28 @@ steps:
- id: dpppconcat - id: dpppconcat
in: in:
- id: msin - id: msin
source: filter_ms_group/output source:
- filter_ms_group/output
- id: msout_name - id: msout_name
source: group_id source: group_id
- id: msin_datacolumn - id: msin_datacolumn
default: DATA default: DATA
- id: msout_datacolumn - id: msout_datacolumn
default: DATA default: DATA
- id: baseline
source: filter_baselines
- id: filter_baselines - id: filter_baselines
source: filter_baselines source: filter_baselines
- id: filter_remove - id: filter_remove
default: true default: true
- id: overwrite - id: overwrite
default: false default: false
- id: missingdata
default: True
- id: storagemanager - id: storagemanager
default: "Dysco" default: Dysco
- id: databitrate - id: databitrate
default: 0 default: 0
- id: missingdata
default: true
- id: baseline
source: filter_baselines
- id: avg_timeresolution - id: avg_timeresolution
source: avg_timeresolution_concat source: avg_timeresolution_concat
- id: avg_freqresolution - id: avg_freqresolution
...@@ -142,4 +143,17 @@ steps: ...@@ -142,4 +143,17 @@ steps:
label: concat_logfiles_dpppconcat label: concat_logfiles_dpppconcat
'sbg:x': 600 'sbg:x': 600
'sbg:y': 200 'sbg:y': 200
requirements: [] - id: concat_logfiles_unflagged
\ No newline at end of file in:
- id: file_list
source:
- check_unflagged_fraction/logfile
- id: file_prefix
default: check_unflagged_fraction
out:
- id: output
run: ../steps/concatenate_files.cwl
label: concat_logfiles_unflagged
'sbg:x': 600
'sbg:y': 400
requirements: []
...@@ -112,7 +112,7 @@ steps: ...@@ -112,7 +112,7 @@ steps:
- id: msout_datacolumn - id: msout_datacolumn
default: DATA default: DATA
- id: storagemanager - id: storagemanager
default: Dysco default: "Dysco"
- id: databitrate - id: databitrate
source: compression_bitrate source: compression_bitrate
- id: correction - id: correction
...@@ -124,7 +124,7 @@ steps: ...@@ -124,7 +124,7 @@ steps:
default: 'target' default: 'target'
- id: msout_name - id: msout_name
source: msin source: msin
valueFrom: $(self.basename+'_prep.ms') valueFrom: $(self.items.basename+'_pre-cal.ms')
out: out:
- id: msout - id: msout
- id: logfile - id: logfile
...@@ -134,6 +134,18 @@ steps: ...@@ -134,6 +134,18 @@ steps:
- msin - msin
'sbg:x': 400 'sbg:x': 400
'sbg:y': 0 'sbg:y': 0
- id: merge_array_files
in:
- id: input
source:
- apply_gsmcal/logfile
out:
- id: output
run: ../../steps/merge_array_files.cwl
label: merge_array_files
'sbg:x': 600
'sbg:y': 0
- id: write_solutions - id: write_solutions
in: in:
- id: h5parmFile - id: h5parmFile
...@@ -202,7 +214,7 @@ steps: ...@@ -202,7 +214,7 @@ steps:
in: in:
- id: file_list - id: file_list
source: source:
- apply_gsmcal/logfile - merge_array_files/output
- id: file_prefix - id: file_prefix
default: apply_gsmcal default: apply_gsmcal
out: out:
......
...@@ -103,7 +103,7 @@ outputs: ...@@ -103,7 +103,7 @@ outputs:
- concat_logfiles_calib/output - concat_logfiles_calib/output
- concat_logfiles_dpppconcat/output - concat_logfiles_dpppconcat/output
- concat_logfiles_blsmooth/output - concat_logfiles_blsmooth/output
- concat/check_unflagged_fraction.log - concat_logfiles_unflagged/output
type: 'File[]' type: 'File[]'
linkMerge: merge_flattened linkMerge: merge_flattened
'sbg:x': 1500 'sbg:x': 1500
...@@ -238,6 +238,21 @@ steps: ...@@ -238,6 +238,21 @@ steps:
label: concat_logfiles_losoto label: concat_logfiles_losoto
'sbg:x': 1200 'sbg:x': 1200
'sbg:y': 700 'sbg:y': 700
- id: concat_logfiles_unflagged
in:
- id: file_list
linkMerge: merge_flattened
source:
- concat/check_unflagged_fraction.log
- plot_unflagged/logfile
- id: file_prefix
default: check_unflagged_fraction
out:
- id: output
run: ../../steps/concatenate_files.cwl
label: concat_logfiles_unflagged
'sbg:x': 1200
'sbg:y': 900
- id: concat - id: concat
in: in:
- id: msin - id: msin
...@@ -326,6 +341,7 @@ steps: ...@@ -326,6 +341,7 @@ steps:
source: concat/unflagged_fraction source: concat/unflagged_fraction
out: out:
- id: output_imag - id: output_imag
- id: logfile
run: ../../steps/plot_unflagged.cwl run: ../../steps/plot_unflagged.cwl
label: plot_unflagged label: plot_unflagged
'sbg:x': 500 'sbg:x': 500
......
...@@ -205,7 +205,7 @@ outputs: ...@@ -205,7 +205,7 @@ outputs:
outputSource: outputSource:
- make_sourcedb_ateam/log - make_sourcedb_ateam/log
- check_ateam_separation/logfile - check_ateam_separation/logfile
- compare_station_list/logfile - concat_logfiles_stationlist/output
- concat_logfiles_RMextract/output - concat_logfiles_RMextract/output
- concat_logfiles_prep_targ/output - concat_logfiles_prep_targ/output
- concat_logfiles_predict_targ/output - concat_logfiles_predict_targ/output
...@@ -473,5 +473,19 @@ steps: ...@@ -473,5 +473,19 @@ steps:
label: concat_logfiles_RMextract label: concat_logfiles_RMextract
'sbg:x': 400 'sbg:x': 400
'sbg:y': 600 'sbg:y': 600
- id: concat_logfiles_stationlist
in:
- id: file_list
linkMerge: merge_flattened
source:
- compare_station_list/logfile
- id: file_prefix
default: compareStationList
out:
- id: output
run: ../../steps/concatenate_files.cwl
label: concat_logfiles_stationlist
'sbg:x': 400
'sbg:y': 700
requirements: requirements:
- class: ScatterFeatureRequirement - class: ScatterFeatureRequirement
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment