diff --git a/steps/concatenate_files.cwl b/steps/concatenate_files.cwl new file mode 100644 index 0000000000000000000000000000000000000000..941a544cfb4072b57428f27591850f9856fbffc5 --- /dev/null +++ b/steps/concatenate_files.cwl @@ -0,0 +1,37 @@ +cwlVersion: v1.2 +class: CommandLineTool + +label: Concatenates a list of ascii files +doc: | + This tool creates a file that contains the concatenated + content of the input ascii files + +requirements: + - class: InlineJavascriptRequirement + - class: InitialWorkDirRequirement + listing: + - entryname: concatenate.sh + writable: false + entry: | + set -e + files=("\${@}") + filename=$(inputs.output_filename) + cat "\${files[@]}" > $filename + +baseCommand: + - bash + - concatenate.sh + +inputs: + - id: files + type: File[] + inputBinding: + position: 0 + - id: output_filename + type: string + +outputs: + - id: output + type: File + outputBinding: + glob: "$(inputs.output_filename)" diff --git a/steps/digitize.cwl b/steps/digitize.cwl index 5f5d19e4f5ad0322a56060e89b0a008268da61a2..444393b1a21cd886f68e0ed214ca88bd4f8e5bca 100644 --- a/steps/digitize.cwl +++ b/steps/digitize.cwl @@ -18,7 +18,7 @@ arguments: ["--verbose"] inputs: - id: h5in - type: File[] + type: File inputBinding: position: 1 doc: Input LOFAR BF .h5 files @@ -45,7 +45,21 @@ outputs: glob: "$(inputs.output_dir)" - id: h5out - type: File[] + type: File doc: Output 8-bit .h5 files outputBinding: - glob: "$(inputs.output_dir)/*.h5" + glob: "$(inputs.output_dir)/$(inputs.h5in.basename)" +# glob: "$(inputs.output_dir)/*.h5" + + - id: stdout_log + type: File[] + outputBinding: + glob: "*_stdout.log" + + - id: stderr_log + type: File[] + outputBinding: + glob: "*_stderr.log" + +stdout: $(inputs.h5in.nameroot)_digitize_stdout.log +stderr: $(inputs.h5in.nameroot)_digitize_stderr.log diff --git a/workflows/pulp2-xxyy-8bit-requantisation.cwl b/workflows/pulp2-xxyy-8bit-requantisation.cwl index 6df443d17ca61d9bf3fc86470bfb759b2351b4d3..b42f4a2567f7fad3d6224c9267e98a18feedb94c 100644 --- a/workflows/pulp2-xxyy-8bit-requantisation.cwl +++ b/workflows/pulp2-xxyy-8bit-requantisation.cwl @@ -3,12 +3,14 @@ class: Workflow requirements: - class: InlineJavascriptRequirement + - class: ScatterFeatureRequirement + - class: StepInputExpressionRequirement inputs: - id: h5in type: File[] - secondaryFiles: ^.raw - doc: Input LOFAR BF .h5 files (input .raw files will be looked for as well) + secondaryFiles: [^.raw] + doc: Input LOFAR BF .h5 files - id: output_dir type: string @@ -21,8 +23,8 @@ inputs: doc: Clip raw data above this many stddevs steps: - digitize: - run: ../steps/digitize.cwl + - id: digitize + run: digitize.cwl label: Converts 32-bit raw LOFAR BF XXYY data to 8-bit doc: | This tool only performs digitizes raw 32-bit LOFAR BF XXYY data to int8. @@ -32,16 +34,57 @@ steps: h5in: h5in output_dir: output_dir nsigma: nsigma - out: [out_dir, h5out] + out: [out_dir, h5out, stdout_log, stderr_log] + scatter: + - h5in + scatterMethod: dotproduct + + - id: concat_stdout_files + label: Combine stdout logfiles from digitize3.py + doc: | + Concatenate stdout files produced by digitize3.py into pipeline.log + run: concatenate_files.cwl + in: + - id: files + source: digitize/stdout_log + valueFrom: $(self.flat()) + - id: output_filename + default: pipeline.log + out: [output] + + - id: concat_stderr_files + label: Combine stderr logfiles from digitize3.py + doc: | + Concatenate stderr files produced by digitize3.py into pipeline.log + run: concatenate_files.cwl + in: + - id: files + source: digitize/stderr_log + valueFrom: $(self.flat()) + - id: output_filename + default: pipeline_error.log + out: [output] outputs: - id: h5out type: File[] outputSource: digitize/h5out - secondaryFiles: ^.raw - doc: Output 8-bit HDF5 (both .h5 and .raw) + secondaryFiles: [^.raw] + doc: Output 8-bit HDF5 - id: out_dir - type: Directory + type: Directory[] outputSource: digitize/out_dir doc: directory for output 8-bit .h5 and .raw files + + - id: stdout_log + type: File + outputSource: + - concat_stdout_files/output + doc: Concatenated stdout logfiles + + - id: stderr_log + type: File + outputSource: + - concat_stderr_files/output + doc: Concatenated stderr logfiles