From c0eb4d11cb8049fbd8edf5273148dc0f1751509f Mon Sep 17 00:00:00 2001
From: Alexander Drabent <alex@tls-tautenburg.de>
Date: Wed, 8 Apr 2020 11:18:03 +0200
Subject: [PATCH] optimize logfile and inspection plot management
---
steps/bulk_rename.cwl | 10 +--
workflows/HBA_calibrator.cwl | 112 +++++++++++++++---------
workflows/prefactor_calibrator.cwl | 4 +-
workflows/prefactor_calibrator/prep.cwl | 4 +-
4 files changed, 77 insertions(+), 53 deletions(-)
diff --git a/steps/bulk_rename.cwl b/steps/bulk_rename.cwl
index d75af405..eaae4ab1 100644
--- a/steps/bulk_rename.cwl
+++ b/steps/bulk_rename.cwl
@@ -15,11 +15,12 @@ inputs:
type: string
- id: file_suffix
type: string?
+ default: log
outputs:
- id: output
- type: 'File[]'
+ type: File
outputBinding:
- glob: "tmp/$(inputs.file_prefix)*"
+ glob: "$(inputs.file_prefix).$(inputs.file_suffix)"
label: bulk_rename
requirements:
- class: InitialWorkDirRequirement
@@ -31,9 +32,6 @@ requirements:
FILE_LIST=("\${@}")
FILE_PREFIX=$(inputs.file_prefix)
FILE_SUFFIX=$(inputs.file_suffix === null ? '' : inputs.file_suffix)
- mkdir tmp
- for i in "\${!FILE_LIST[@]}"; do
- cp "\${FILE_LIST[\$i]}" "tmp/\${FILE_PREFIX}_\${i}\${FILE_SUFFIX}"
- done
+ cat "\${FILE_LIST[@]}" > "\$FILE_PREFIX.\$FILE_SUFFIX"
writable: false
- class: InlineJavascriptRequirement
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index 1263d33d..14b4f250 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -1,6 +1,6 @@
class: Workflow
cwlVersion: v1.0
-id: HBA_calibrator
+id: _h_b_a_calibrator
label: HBA_calibrator
$namespaces:
sbg: 'https://www.sevenbridges.com/'
@@ -73,7 +73,7 @@ inputs:
'sbg:y': -500
- id: maxStddev
type: float?
- default: -1.0
+ default: -1
'sbg:x': -1000
'sbg:y': -400
- id: solutions2transfer
@@ -160,7 +160,7 @@ inputs:
'sbg:y': 1200
- id: max_separation_arcmin
type: float?
- default: 1.0
+ default: 1
'sbg:x': -1000
'sbg:y': 1300
- id: calibrator_path_skymodel
@@ -192,63 +192,41 @@ inputs:
'sbg:x': -1000
'sbg:y': 1900
outputs:
- - id: check_Ateam_separation.png
- outputSource:
- - prefactor/check_Ateam_separation.png
- type: 'File[]?'
- 'sbg:x': 1000
- 'sbg:y': -300
- - id: check_Ateam_separation.log
- outputSource:
- - prefactor/check_Ateam_separation.log
- type: 'File[]?'
- 'sbg:x': 1000
- 'sbg:y': -200
- - id: make_sourcedb_ateam.log
- outputSource:
- - prefactor/make_sourcedb_ateam.log
- type: File?
- 'sbg:x': 1000
- 'sbg:y': -100
- id: msout
outputSource:
- prefactor/msout
type: 'Directory[]'
- 'sbg:x': 1000
+ 'sbg:x': 1500
+ 'sbg:y': -100
+ - id: logs
+ outputSource:
+ - save_logfiles/log_dir
+ type: Directory
+ 'sbg:x': 1500
'sbg:y': 0
- - id: logfile_prep_cal
+ - id: inspection
outputSource:
- - prefactor/logfile_prep_cal
- type: 'File[]?'
- 'sbg:x': 1000
+ - save_inspection/log_dir
+ type: Directory
+ 'sbg:x': 1500
'sbg:y': 100
- id: parset
outputSource:
- prefactor/parset
type: 'File[]'
- 'sbg:x': 1000
+ 'sbg:x': 1500
'sbg:y': 200
- - id: find_skymodel_cal.log
- outputSource:
- - prefactor/find_skymodel_cal.log
- type: File
- 'sbg:x': 1000
- 'sbg:y': 300
- - id: make_sourcedb.log
- outputSource:
- - prefactor/make_sourcedb.log
- type: File
- 'sbg:x': 1000
- 'sbg:y': 400
steps:
- id: prefactor
in:
- id: msin
- source: msin
+ source:
+ - msin
- id: refant
source: refant
- id: flag_baselines
- source: flag_baselines
+ source:
+ - flag_baselines
- id: process_baselines_cal
source: process_baselines_cal
- id: filter_baselines
@@ -260,7 +238,8 @@ steps:
- id: max2interpolate
source: max2interpolate
- id: ampRange
- source: ampRange
+ source:
+ - ampRange
- id: skip_international
source: skip_international
- id: raw_data
@@ -276,7 +255,8 @@ steps:
- id: antennas2transfer
source: antennas2transfer
- id: demix_sources
- source: demix_sources
+ source:
+ - demix_sources
- id: demix_target
source: demix_target
- id: demix_freqstep
@@ -330,5 +310,51 @@ steps:
label: prefactor_calibrator
'sbg:x': 0
'sbg:y': 0
+ - id: concat_logfiles
+ in:
+ - id: file_list
+ source:
+ - prefactor/logfile_prep_cal
+ - id: file_prefix
+ default: dppp_prep_cal
+ out:
+ - id: output
+ run: ./../steps/bulk_rename.cwl
+ label: concat_logfiles
+ 'sbg:x': 500
+ 'sbg:y': 1000
+ - id: save_logfiles
+ in:
+ - id: log_files
+ linkMerge: merge_flattened
+ source:
+ - prefactor/check_Ateam_separation.log
+ - prefactor/make_sourcedb_ateam.log
+ - prefactor/find_skymodel_cal.log
+ - prefactor/make_sourcedb.log
+ - concat_logfiles/output
+ - id: sub_directory_name
+ default: logs
+ out:
+ - id: log_dir
+ run: ./../steps/collectlog.cwl
+ label: save_logfiles
+ 'sbg:x': 1000
+ 'sbg:y': 1000
+ - id: save_inspection
+ in:
+ - id: log_files
+ linkMerge: merge_flattened
+ source:
+ - prefactor/check_Ateam_separation.png
+ - id: sub_directory_name
+ default: inspection
+ out:
+ - id: log_dir
+ run: ./../steps/collectlog.cwl
+ label: save_inspection
+ 'sbg:x': 1000
+ 'sbg:y': 1200
requirements:
- class: SubworkflowFeatureRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl
index 8b41ad80..264c90d2 100644
--- a/workflows/prefactor_calibrator.cwl
+++ b/workflows/prefactor_calibrator.cwl
@@ -195,13 +195,13 @@ outputs:
- id: check_Ateam_separation.png
outputSource:
- prep/check_Ateam_separation.png
- type: 'File[]?'
+ type: File
'sbg:x': 1000
'sbg:y': -300
- id: check_Ateam_separation.log
outputSource:
- prep/check_Ateam_separation.log
- type: 'File[]?'
+ type: File
'sbg:x': 1000
'sbg:y': -200
- id: make_sourcedb_ateam.log
diff --git a/workflows/prefactor_calibrator/prep.cwl b/workflows/prefactor_calibrator/prep.cwl
index 5e35d89c..a6040f56 100644
--- a/workflows/prefactor_calibrator/prep.cwl
+++ b/workflows/prefactor_calibrator/prep.cwl
@@ -104,13 +104,13 @@ outputs:
- id: check_Ateam_separation.png
outputSource:
- check_ateam_separation/output_imag
- type: 'File[]?'
+ type: File
'sbg:x': 1000
'sbg:y': -300
- id: check_Ateam_separation.log
outputSource:
- check_ateam_separation/logfile
- type: 'File[]?'
+ type: File
'sbg:x': 1000
'sbg:y': -200
- id: make_sourcedb_ateam.log
--
GitLab