From f41d337d313b8adf00bde2b51aac1aab37200049 Mon Sep 17 00:00:00 2001
From: Alexander Drabent <alex@tls-tautenburg.de>
Date: Mon, 16 Nov 2020 15:33:49 +0100
Subject: [PATCH] update logging output
Former-commit-id: 38fe83c870a08bce02f07e66e109af447c76d1e1
---
lofar-cwl/steps/applytarget.cwl | 5 ++--
steps/check_unflagged_fraction.cwl | 14 ++++-----
steps/compare_station_list.cwl | 7 +++--
steps/merge_array_files.cwl | 31 ++++++++++++++++++++
steps/plot_unflagged.cwl | 13 ++++-----
subworkflow/concat.cwl | 38 +++++++++++++++++--------
workflows/prefactor_target/finalize.cwl | 18 ++++++++++--
workflows/prefactor_target/gsmcal.cwl | 18 +++++++++++-
workflows/prefactor_target/prep.cwl | 16 ++++++++++-
9 files changed, 121 insertions(+), 39 deletions(-)
create mode 100644 steps/merge_array_files.cwl
diff --git a/lofar-cwl/steps/applytarget.cwl b/lofar-cwl/steps/applytarget.cwl
index 883ab2ec..5cca1b7b 100644
--- a/lofar-cwl/steps/applytarget.cwl
+++ b/lofar-cwl/steps/applytarget.cwl
@@ -17,7 +17,7 @@ inputs:
type: string
inputBinding:
position: 0
- prefix: msout=msout_name
+ prefix: msout=
separate: false
doc: Name of output MS
- default: DATA
@@ -67,6 +67,7 @@ inputs:
default: ""
inputBinding:
prefix: msout.storagemanager=
+ separate: false
- id: databitrate
type: int?
inputBinding:
@@ -85,7 +86,7 @@ outputs:
outputBinding:
glob: $(inputs.msin.basename)
- id: logfile
- type: File
+ type: File[]
outputBinding:
glob: 'applycal_$(inputs.correction).log'
stdout: applycal_$(inputs.correction).log
diff --git a/steps/check_unflagged_fraction.cwl b/steps/check_unflagged_fraction.cwl
index e4031b5a..0d7d2a01 100644
--- a/steps/check_unflagged_fraction.cwl
+++ b/steps/check_unflagged_fraction.cwl
@@ -27,7 +27,6 @@ arguments:
from check_unflagged_fraction import main as check_unflagged_fraction
- print(sys.argv)
ms = sys.argv[1]
min_fraction = float($(inputs.min_fraction))
@@ -35,11 +34,7 @@ arguments:
filename = os.path.basename(output['flagged'])
unflagged_fraction = output['unflagged_fraction']
-
- print(os.getcwd())
- print(filename)
- print(filename.lstrip('out_'))
-
+
if filename != 'None':
shutil.move(filename, filename.lstrip('out_'))
@@ -61,9 +56,9 @@ outputs:
glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).unflagged_fraction)
- id: logfile
- type: File?
+ type: File[]?
outputBinding:
- glob: check_unflagged_fraction.log
+ glob: 'check_unflagged_fraction*.log'
requirements:
- class: InlineJavascriptRequirement
@@ -75,4 +70,5 @@ requirements:
hints:
- class: DockerRequirement
dockerPull: lofareosc/prefactor:HBAcalibrator
-stdout: check_unflagged_fraction.log
\ No newline at end of file
+stdout: check_unflagged_fraction.log
+stderr: check_unflagged_fraction_err.log
\ No newline at end of file
diff --git a/steps/compare_station_list.cwl b/steps/compare_station_list.cwl
index 96574806..21e18c5f 100644
--- a/steps/compare_station_list.cwl
+++ b/steps/compare_station_list.cwl
@@ -51,9 +51,9 @@ outputs:
glob: 'out.json'
outputEval: $(JSON.parse(self[0].contents).filter_out)
- id: logfile
- type: File?
+ type: File[]?
outputBinding:
- glob: compareStationList.log
+ glob: 'compareStationList*.log'
requirements:
- class: InlineJavascriptRequirement
@@ -61,4 +61,5 @@ requirements:
hints:
DockerRequirement:
dockerPull: lofareosc/prefactor:HBAcalibrator
-stdout: compareStationList.log
\ No newline at end of file
+stdout: compareStationList.log
+stderr: compareStationList_err.log
\ No newline at end of file
diff --git a/steps/merge_array_files.cwl b/steps/merge_array_files.cwl
new file mode 100644
index 00000000..c4c5c612
--- /dev/null
+++ b/steps/merge_array_files.cwl
@@ -0,0 +1,31 @@
+id: merge_array_files
+label: merge_array_files
+class: ExpressionTool
+
+cwlVersion: v1.0
+inputs:
+ - id: input
+ type:
+ - type: array
+ items:
+ - type: array
+ items: File
+outputs:
+ - id: output
+ type: File[]
+
+expression: |
+ ${
+ var out_file = []
+ for(var i=0; i<inputs.input.length; i++){
+ var item = inputs.input[i]
+ if(item != null){
+ out_file = out_file.concat(item)
+ }
+ }
+ return {'output': out_file}
+ }
+
+
+requirements:
+ - class: InlineJavascriptRequirement
\ No newline at end of file
diff --git a/steps/plot_unflagged.cwl b/steps/plot_unflagged.cwl
index 00640a1b..290636c5 100644
--- a/steps/plot_unflagged.cwl
+++ b/steps/plot_unflagged.cwl
@@ -24,19 +24,11 @@ arguments:
from plot_unflagged_fraction import main as plot_unflagged_fraction
- print(sys.argv)
- print(len(sys.argv))
-
center = int((len(sys.argv) - 1) / 2 + 1)
- print(center)
mss = sys.argv[1:center]
unflagged_fraction = sys.argv[center:]
- print(mss)
- print(unflagged_fraction)
- print(os.getcwd() + '/unflagged_fraction.png')
-
output = plot_unflagged_fraction(ms_list = mss, frac_list = unflagged_fraction, outfile = os.getcwd() + '/unflagged_fraction.png')
outputs:
@@ -45,6 +37,11 @@ outputs:
type: File
outputBinding:
glob: 'unflagged_fraction.png'
+ - id: logfile
+ type: File[]?
+ outputBinding:
+ glob: 'plot_unflagged_fraction*.log'
+
hints:
- class: DockerRequirement
diff --git a/subworkflow/concat.cwl b/subworkflow/concat.cwl
index c5732783..c7337131 100644
--- a/subworkflow/concat.cwl
+++ b/subworkflow/concat.cwl
@@ -6,7 +6,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
- id: msin
- type: Directory[]
+ type: 'Directory[]'
'sbg:x': -1000
'sbg:y': -300
- id: group_id
@@ -49,7 +49,7 @@ outputs:
- id: msout
outputSource:
- check_unflagged_fraction/msout
- type: Directory[]
+ type: 'Directory[]'
'sbg:x': 1000
'sbg:y': 0
- id: dpppconcat.log
@@ -60,11 +60,10 @@ outputs:
'sbg:y': 100
- id: check_unflagged_fraction.log
outputSource:
- - check_unflagged_fraction/logfile
+ - concat_logfiles_unflagged/output
type: File
'sbg:x': 1000
'sbg:y': 200
-
steps:
- id: filter_ms_group
in:
@@ -73,7 +72,8 @@ steps:
- id: groups_specification
source: groups_specification
- id: measurement_sets
- source: msin
+ source:
+ - msin
out:
- id: output
run: ../steps/filter_ms_group.cwl
@@ -83,27 +83,28 @@ steps:
- id: dpppconcat
in:
- id: msin
- source: filter_ms_group/output
+ source:
+ - filter_ms_group/output
- id: msout_name
source: group_id
- id: msin_datacolumn
default: DATA
- id: msout_datacolumn
default: DATA
- - id: baseline
- source: filter_baselines
- id: filter_baselines
source: filter_baselines
- id: filter_remove
default: true
- id: overwrite
default: false
- - id: missingdata
- default: True
- id: storagemanager
- default: "Dysco"
+ default: Dysco
- id: databitrate
default: 0
+ - id: missingdata
+ default: true
+ - id: baseline
+ source: filter_baselines
- id: avg_timeresolution
source: avg_timeresolution_concat
- id: avg_freqresolution
@@ -142,4 +143,17 @@ steps:
label: concat_logfiles_dpppconcat
'sbg:x': 600
'sbg:y': 200
-requirements: []
\ No newline at end of file
+ - id: concat_logfiles_unflagged
+ in:
+ - id: file_list
+ source:
+ - check_unflagged_fraction/logfile
+ - id: file_prefix
+ default: check_unflagged_fraction
+ out:
+ - id: output
+ run: ../steps/concatenate_files.cwl
+ label: concat_logfiles_unflagged
+ 'sbg:x': 600
+ 'sbg:y': 400
+requirements: []
diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl
index 4960e027..77837ecf 100644
--- a/workflows/prefactor_target/finalize.cwl
+++ b/workflows/prefactor_target/finalize.cwl
@@ -112,7 +112,7 @@ steps:
- id: msout_datacolumn
default: DATA
- id: storagemanager
- default: Dysco
+ default: "Dysco"
- id: databitrate
source: compression_bitrate
- id: correction
@@ -124,7 +124,7 @@ steps:
default: 'target'
- id: msout_name
source: msin
- valueFrom: $(self.basename+'_prep.ms')
+ valueFrom: $(self.items.basename+'_pre-cal.ms')
out:
- id: msout
- id: logfile
@@ -134,6 +134,18 @@ steps:
- msin
'sbg:x': 400
'sbg:y': 0
+ - id: merge_array_files
+ in:
+ - id: input
+ source:
+ - apply_gsmcal/logfile
+ out:
+ - id: output
+ run: ../../steps/merge_array_files.cwl
+ label: merge_array_files
+ 'sbg:x': 600
+ 'sbg:y': 0
+
- id: write_solutions
in:
- id: h5parmFile
@@ -202,7 +214,7 @@ steps:
in:
- id: file_list
source:
- - apply_gsmcal/logfile
+ - merge_array_files/output
- id: file_prefix
default: apply_gsmcal
out:
diff --git a/workflows/prefactor_target/gsmcal.cwl b/workflows/prefactor_target/gsmcal.cwl
index 58f5b1af..23c1541b 100644
--- a/workflows/prefactor_target/gsmcal.cwl
+++ b/workflows/prefactor_target/gsmcal.cwl
@@ -103,7 +103,7 @@ outputs:
- concat_logfiles_calib/output
- concat_logfiles_dpppconcat/output
- concat_logfiles_blsmooth/output
- - concat/check_unflagged_fraction.log
+ - concat_logfiles_unflagged/output
type: 'File[]'
linkMerge: merge_flattened
'sbg:x': 1500
@@ -238,6 +238,21 @@ steps:
label: concat_logfiles_losoto
'sbg:x': 1200
'sbg:y': 700
+ - id: concat_logfiles_unflagged
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - concat/check_unflagged_fraction.log
+ - plot_unflagged/logfile
+ - id: file_prefix
+ default: check_unflagged_fraction
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_unflagged
+ 'sbg:x': 1200
+ 'sbg:y': 900
- id: concat
in:
- id: msin
@@ -326,6 +341,7 @@ steps:
source: concat/unflagged_fraction
out:
- id: output_imag
+ - id: logfile
run: ../../steps/plot_unflagged.cwl
label: plot_unflagged
'sbg:x': 500
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
index 968bc3bd..1e1ebb52 100644
--- a/workflows/prefactor_target/prep.cwl
+++ b/workflows/prefactor_target/prep.cwl
@@ -205,7 +205,7 @@ outputs:
outputSource:
- make_sourcedb_ateam/log
- check_ateam_separation/logfile
- - compare_station_list/logfile
+ - concat_logfiles_stationlist/output
- concat_logfiles_RMextract/output
- concat_logfiles_prep_targ/output
- concat_logfiles_predict_targ/output
@@ -473,5 +473,19 @@ steps:
label: concat_logfiles_RMextract
'sbg:x': 400
'sbg:y': 600
+ - id: concat_logfiles_stationlist
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - compare_station_list/logfile
+ - id: file_prefix
+ default: compareStationList
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_stationlist
+ 'sbg:x': 400
+ 'sbg:y': 700
requirements:
- class: ScatterFeatureRequirement
\ No newline at end of file
--
GitLab