diff --git a/.gitignore b/.gitignore index d457fdf4f388fa7061bc07ac7cd3d952690210ce..272dd922ea6920b4c91d4fbabf2e03edc888ce0e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ venv/ # Data *.ms *.MS + +# IDEs +.vscode/ diff --git a/steps/concatenate_files.cwl b/steps/concatenate_files.cwl new file mode 100644 index 0000000000000000000000000000000000000000..95dd9dec549d1758ff30114dc63473af13b86208 --- /dev/null +++ b/steps/concatenate_files.cwl @@ -0,0 +1,35 @@ +class: CommandLineTool +cwlVersion: v1.2 +baseCommand: + - sh + - concatenate.sh +label: Concatenates a list of files +doc: | + This tool creates a file that contains the concatenated + content of the inputs files + +inputs: + - id: files + type: File[] + inputBinding: + position: 0 + - id: output_filename + type: string + +outputs: + - id: output + type: File + outputBinding: + glob: "$(inputs.output_filename)" + +requirements: + - class: InlineJavascriptRequirement + - class: InitialWorkDirRequirement + listing: + - entryname: concatenate.sh + writable: false + entry: | + set -e + files=("\${@}") + filename=$(inputs.output_filename) + cat "\${files[@]}" > $filename diff --git a/steps/preprocess.cwl b/steps/preprocess.cwl index 5f9cab9bbda5f37b46c0b94d6db79ba813419b55..f682d08051273d48b4e496ad1c7ca23994433c93 100644 --- a/steps/preprocess.cwl +++ b/steps/preprocess.cwl @@ -187,6 +187,13 @@ outputs: outputBinding: glob: '$(inputs.msout_name == "." ? inputs.msin.basename : inputs.msout_name)' doc: Output MS + - id: logfiles + type: File[] + outputBinding: + glob: '*.log' + +stdout: $(inputs.msin.basename)_preprocess.log +stderr: $(inputs.msin.basename)_preprocess_err.log requirements: InlineJavascriptRequirement: {} diff --git a/workflows/pipeline.cwl b/workflows/pipeline.cwl index 55a8042d4cf1710282ec208edb80bdef634f0645..3c41844472b28beb81c60d00b410856010a480ab 100644 --- a/workflows/pipeline.cwl +++ b/workflows/pipeline.cwl @@ -100,6 +100,11 @@ outputs: outputSource: - preprocess/msout doc: List of pre-processed LOFAR MSs + - id: logfiles + type: File + outputSource: + - concat_logfiles/output + doc: Concatenated log files steps: - id: preprocess @@ -159,9 +164,22 @@ steps: source: dp3_checkparset - id: dp3_numthreads source: dp3_numthreads - out: [msout] + out: [msout, logfiles] + - id: concat_logfiles + label: Combine DP3 log files + doc: | + Concatenate logs produced by DP3 into pipeline.log + run: ../steps/concatenate_files.cwl + in: + - id: files + source: preprocess/logfiles + valueFrom: $(self.flat()) + - id: output_filename + default: pipeline.log + out: [output] requirements: - class: InlineJavascriptRequirement - class: ScatterFeatureRequirement - class: StepInputExpressionRequirement +