Merge branch 'add-pipeline-log' into 'main'

RAP-876 Add pipeline log

See merge request !7

.gitignore

@@ -6,3 +6,6 @@ venv/
 # Data
 *.ms
 *.MS
+
+# IDEs
+.vscode/

steps/concatenate_files.cwl

+class: CommandLineTool
+cwlVersion: v1.2
+baseCommand:
+  - sh
+  - concatenate.sh
+label: Concatenates a list of files
+doc: |
+  This tool creates a file that contains the concatenated 
+  content of the inputs files
+
+inputs:
+  - id: files
+    type: File[]
+    inputBinding:
+      position: 0
+  - id: output_filename
+    type: string
+
+outputs:
+  - id: output
+    type: File
+    outputBinding:
+      glob: "$(inputs.output_filename)"
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entryname: concatenate.sh
+        writable: false
+        entry: |
+          set -e
+          files=("\${@}")
+          filename=$(inputs.output_filename)
+          cat "\${files[@]}" > $filename

steps/preprocess.cwl

@@ -187,6 +187,13 @@ outputs:
     outputBinding:
       glob: '$(inputs.msout_name == "." ? inputs.msin.basename : inputs.msout_name)'
     doc: Output MS
+  - id: logfiles
+    type: File[]
+    outputBinding:
+      glob: '*.log'
+
+stdout: $(inputs.msin.basename)_preprocess.log
+stderr: $(inputs.msin.basename)_preprocess_err.log
 
 requirements:
   InlineJavascriptRequirement: {}

workflows/pipeline.cwl

@@ -100,6 +100,11 @@ outputs:
     outputSource:
       - preprocess/msout
     doc: List of pre-processed LOFAR MSs
+  - id: logfiles
+    type: File
+    outputSource:
+      - concat_logfiles/output
+    doc: Concatenated log files
 
 steps:
   - id: preprocess
@@ -159,9 +164,22 @@ steps:
         source: dp3_checkparset
       - id: dp3_numthreads
         source: dp3_numthreads
-    out: [msout]
+    out: [msout, logfiles]
+  - id: concat_logfiles
+    label: Combine DP3 log files
+    doc: |
+      Concatenate logs produced by DP3 into pipeline.log
+    run: ../steps/concatenate_files.cwl
+    in:
+      - id: files
+        source: preprocess/logfiles
+        valueFrom: $(self.flat())
+      - id: output_filename
+        default: pipeline.log
+    out: [output]
 
 requirements:
   - class: InlineJavascriptRequirement
   - class: ScatterFeatureRequirement
   - class: StepInputExpressionRequirement
+