optimize logfile and inspection plot management

steps/bulk_rename.cwl

@@ -15,11 +15,12 @@ inputs:
     type: string
   - id: file_suffix
     type: string?
+    default: log
 outputs:
   - id: output
-    type: 'File[]'
+    type: File
     outputBinding:
-      glob: "tmp/$(inputs.file_prefix)*"
+      glob: "$(inputs.file_prefix).$(inputs.file_suffix)"
 label: bulk_rename
 requirements:
   - class: InitialWorkDirRequirement
@@ -31,9 +32,6 @@ requirements:
           FILE_LIST=("\${@}")
           FILE_PREFIX=$(inputs.file_prefix)
           FILE_SUFFIX=$(inputs.file_suffix === null ? '' : inputs.file_suffix)
-          mkdir tmp
-          for i in "\${!FILE_LIST[@]}"; do 
-            cp "\${FILE_LIST[\$i]}" "tmp/\${FILE_PREFIX}_\${i}\${FILE_SUFFIX}"
-          done
+          cat "\${FILE_LIST[@]}" > "\$FILE_PREFIX.\$FILE_SUFFIX"
         writable: false
   - class: InlineJavascriptRequirement

workflows/HBA_calibrator.cwl

 class: Workflow
 cwlVersion: v1.0
-id: HBA_calibrator
+id: _h_b_a_calibrator
 label: HBA_calibrator
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
@@ -73,7 +73,7 @@ inputs:
     'sbg:y': -500
   - id: maxStddev
     type: float?
-    default: -1.0
+    default: -1
     'sbg:x': -1000
     'sbg:y': -400
   - id: solutions2transfer
@@ -160,7 +160,7 @@ inputs:
     'sbg:y': 1200
   - id: max_separation_arcmin
     type: float?
-    default: 1.0
+    default: 1
     'sbg:x': -1000
     'sbg:y': 1300
   - id: calibrator_path_skymodel
@@ -192,63 +192,41 @@ inputs:
     'sbg:x': -1000
     'sbg:y': 1900
 outputs:
-  - id: check_Ateam_separation.png
-    outputSource:
-      - prefactor/check_Ateam_separation.png
-    type: 'File[]?'
-    'sbg:x': 1000
-    'sbg:y': -300
-  - id: check_Ateam_separation.log
-    outputSource:
-      - prefactor/check_Ateam_separation.log
-    type: 'File[]?'
-    'sbg:x': 1000
-    'sbg:y': -200
-  - id: make_sourcedb_ateam.log
-    outputSource:
-      - prefactor/make_sourcedb_ateam.log
-    type: File?
-    'sbg:x': 1000
-    'sbg:y': -100
   - id: msout
     outputSource:
       - prefactor/msout
     type: 'Directory[]'
-    'sbg:x': 1000
+    'sbg:x': 1500
+    'sbg:y': -100
+  - id: logs
+    outputSource:
+      - save_logfiles/log_dir
+    type: Directory
+    'sbg:x': 1500
     'sbg:y': 0
-  - id: logfile_prep_cal
+  - id: inspection
     outputSource:
-      - prefactor/logfile_prep_cal
-    type: 'File[]?'
-    'sbg:x': 1000
+      - save_inspection/log_dir
+    type: Directory
+    'sbg:x': 1500
     'sbg:y': 100
   - id: parset
     outputSource:
       - prefactor/parset
     type: 'File[]'
-    'sbg:x': 1000
+    'sbg:x': 1500
     'sbg:y': 200
-  - id: find_skymodel_cal.log
-    outputSource:
-      - prefactor/find_skymodel_cal.log
-    type: File
-    'sbg:x': 1000
-    'sbg:y': 300
-  - id: make_sourcedb.log
-    outputSource:
-      - prefactor/make_sourcedb.log
-    type: File
-    'sbg:x': 1000
-    'sbg:y': 400
 steps:
   - id: prefactor
     in:
       - id: msin
-        source: msin
+        source:
+          - msin
       - id: refant
         source: refant
       - id: flag_baselines
-        source: flag_baselines
+        source:
+          - flag_baselines
       - id: process_baselines_cal
         source: process_baselines_cal
       - id: filter_baselines
@@ -260,7 +238,8 @@ steps:
       - id: max2interpolate
         source: max2interpolate
       - id: ampRange
-        source: ampRange
+        source:
+          - ampRange
       - id: skip_international
         source: skip_international
       - id: raw_data
@@ -276,7 +255,8 @@ steps:
       - id: antennas2transfer
         source: antennas2transfer
       - id: demix_sources
-        source: demix_sources
+        source:
+          - demix_sources
       - id: demix_target
         source: demix_target
       - id: demix_freqstep
@@ -330,5 +310,51 @@ steps:
     label: prefactor_calibrator
     'sbg:x': 0
     'sbg:y': 0
+  - id: concat_logfiles
+    in:
+      - id: file_list
+        source:
+          - prefactor/logfile_prep_cal
+      - id: file_prefix
+        default: dppp_prep_cal
+    out:
+      - id: output
+    run: ./../steps/bulk_rename.cwl
+    label: concat_logfiles
+    'sbg:x': 500
+    'sbg:y': 1000
+  - id: save_logfiles
+    in:
+      - id: log_files
+        linkMerge: merge_flattened
+        source:
+          - prefactor/check_Ateam_separation.log
+          - prefactor/make_sourcedb_ateam.log
+          - prefactor/find_skymodel_cal.log
+          - prefactor/make_sourcedb.log
+          - concat_logfiles/output
+      - id: sub_directory_name
+        default: logs
+    out:
+      - id: log_dir
+    run: ./../steps/collectlog.cwl
+    label: save_logfiles
+    'sbg:x': 1000
+    'sbg:y': 1000
+  - id: save_inspection
+    in:
+      - id: log_files
+        linkMerge: merge_flattened
+        source:
+          - prefactor/check_Ateam_separation.png
+      - id: sub_directory_name
+        default: inspection
+    out:
+      - id: log_dir
+    run: ./../steps/collectlog.cwl
+    label: save_inspection
+    'sbg:x': 1000
+    'sbg:y': 1200
 requirements:
   - class: SubworkflowFeatureRequirement
+  - class: MultipleInputFeatureRequirement

workflows/prefactor_calibrator.cwl

@@ -195,13 +195,13 @@ outputs:
   - id: check_Ateam_separation.png
     outputSource:
       - prep/check_Ateam_separation.png
-    type: 'File[]?'
+    type: File
     'sbg:x': 1000
     'sbg:y': -300
   - id: check_Ateam_separation.log
     outputSource:
       - prep/check_Ateam_separation.log
-    type: 'File[]?'
+    type: File
     'sbg:x': 1000
     'sbg:y': -200
   - id: make_sourcedb_ateam.log

workflows/prefactor_calibrator/prep.cwl

@@ -104,13 +104,13 @@ outputs:
   - id: check_Ateam_separation.png
     outputSource:
       - check_ateam_separation/output_imag
-    type: 'File[]?'
+    type: File
     'sbg:x': 1000
     'sbg:y': -300
   - id: check_Ateam_separation.log
     outputSource:
       - check_ateam_separation/logfile
-    type: 'File[]?'
+    type: File
     'sbg:x': 1000
     'sbg:y': -200
   - id: make_sourcedb_ateam.log