updated workflow and digitize step to use ScatterFeatureRequirement and also...

updated workflow and digitize step to use ScatterFeatureRequirement and also added extra step to combine stdout and stderr output to a single pipeline.log and pipeline_error.log files

steps/concatenate_files.cwl

+cwlVersion: v1.2
+class: CommandLineTool
+
+label: Concatenates a list of ascii files
+doc: |
+  This tool creates a file that contains the concatenated
+  content of the input ascii files
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entryname: concatenate.sh
+        writable: false
+        entry: |
+          set -e
+          files=("\${@}")
+          filename=$(inputs.output_filename)
+          cat "\${files[@]}" > $filename
+
+baseCommand:
+  - bash
+  - concatenate.sh
+
+inputs:
+  - id: files
+    type: File[]
+    inputBinding:
+      position: 0
+  - id: output_filename
+    type: string
+
+outputs:
+  - id: output
+    type: File
+    outputBinding:
+      glob: "$(inputs.output_filename)"

steps/digitize.cwl

@@ -18,7 +18,7 @@ arguments: ["--verbose"]
 
 inputs:
   - id: h5in
-    type: File[]
+    type: File
     inputBinding:
       position: 1
     doc: Input LOFAR BF .h5 files
@@ -45,7 +45,21 @@ outputs:
       glob: "$(inputs.output_dir)"
 
   - id: h5out
-    type: File[]
+    type: File
     doc: Output 8-bit .h5 files
     outputBinding:
-      glob: "$(inputs.output_dir)/*.h5"
+      glob: "$(inputs.output_dir)/$(inputs.h5in.basename)"
+#      glob: "$(inputs.output_dir)/*.h5"
+
+  - id: stdout_log
+    type: File[]
+    outputBinding:
+      glob: "*_stdout.log"
+
+  - id: stderr_log
+    type: File[]
+    outputBinding:
+      glob: "*_stderr.log"
+
+stdout: $(inputs.h5in.nameroot)_digitize_stdout.log
+stderr: $(inputs.h5in.nameroot)_digitize_stderr.log

workflows/pulp2-xxyy-8bit-requantisation.cwl

@@ -3,12 +3,14 @@ class: Workflow
 
 requirements:
   - class: InlineJavascriptRequirement
+  - class: ScatterFeatureRequirement
+  - class: StepInputExpressionRequirement
   
 inputs:
   - id: h5in
     type: File[]
-    secondaryFiles: ^.raw
-    doc: Input LOFAR BF .h5 files (input .raw files will be looked for as well)
+    secondaryFiles: [^.raw]
+    doc: Input LOFAR BF .h5 files
 
   - id: output_dir
     type: string
@@ -21,8 +23,8 @@ inputs:
     doc: Clip raw data above this many stddevs
 
 steps:
-  digitize:
-    run: ../steps/digitize.cwl
+  - id: digitize
+    run: digitize.cwl
     label: Converts 32-bit raw LOFAR BF XXYY data to 8-bit
     doc: |
       This tool only performs digitizes raw 32-bit LOFAR BF XXYY data to int8.
@@ -32,16 +34,57 @@ steps:
        h5in: h5in
        output_dir: output_dir
        nsigma: nsigma
-    out: [out_dir, h5out]
+    out: [out_dir, h5out, stdout_log, stderr_log]
+    scatter:
+      - h5in
+    scatterMethod: dotproduct
+
+  - id: concat_stdout_files
+    label: Combine stdout logfiles from digitize3.py
+    doc: |
+      Concatenate stdout files produced by digitize3.py into pipeline.log
+    run: concatenate_files.cwl
+    in:
+      - id: files
+        source: digitize/stdout_log
+        valueFrom: $(self.flat())
+      - id: output_filename
+        default: pipeline.log
+    out: [output]
+
+  - id: concat_stderr_files
+    label: Combine stderr logfiles from digitize3.py
+    doc: |
+      Concatenate stderr files produced by digitize3.py into pipeline.log
+    run: concatenate_files.cwl
+    in:
+      - id: files
+        source: digitize/stderr_log
+        valueFrom: $(self.flat())
+      - id: output_filename
+        default: pipeline_error.log
+    out: [output]
 
 outputs:
   - id: h5out
     type: File[]
     outputSource: digitize/h5out
-    secondaryFiles: ^.raw
-    doc: Output 8-bit HDF5 (both .h5 and .raw)
+    secondaryFiles: [^.raw]
+    doc: Output 8-bit HDF5
 
   - id: out_dir
-    type: Directory
+    type: Directory[]
     outputSource: digitize/out_dir
     doc: directory for output 8-bit .h5 and .raw files
+
+  - id: stdout_log
+    type: File
+    outputSource:
+      - concat_stdout_files/output
+    doc: Concatenated stdout logfiles
+
+  - id: stderr_log
+    type: File
+    outputSource:
+      - concat_stderr_files/output
+    doc: Concatenated stderr logfiles