Working workflow

compress_pipeline.cwl

@@ -5,73 +5,45 @@ label: compress_pipeline.cwl
 $namespaces:
   sbg: 'https://www.sevenbridges.com/'
 inputs:
-  - id: surl_list
-    type: File
-    doc: input surl file
-    'sbg:x': -334.23162841796875
-    'sbg:y': -27
-  - id: flagging_strategy
-    type:
-      type: enum
-      symbols:
-        - HBAdefault
-        - LBAdefault
-      name: flagging_strategy
-    'sbg:x': -30
-    'sbg:y': -321.8600769042969
   - id: flag_autocorrelation
     type: boolean?
-    'sbg:x': -242.8984375
-    'sbg:y': -154.86007690429688
+    'sbg:x': -41
+    'sbg:y': -212
+  - id: surl
+    type: string
+    'sbg:x': -229
+    'sbg:y': -29
 outputs:
   - id: msout
     outputSource:
       - dppp/msout
     type: Directory
     doc: Output Measurement Set
-    'sbg:x': 705.827392578125
-    'sbg:y': 48.75
+    'sbg:x': 756.0812377929688
+    'sbg:y': -155.634521484375
   - id: logfile
     outputSource:
       - dppp/logfile
     type: 'File[]'
-    'sbg:x': 606.827392578125
-    'sbg:y': 207.75
+    'sbg:x': 893.7462158203125
+    'sbg:y': 91.7055892944336
+  - id: flagging_metrics
+    outputSource:
+      - inspect_flagging_dataloss/output
+    type: File
+    'sbg:x': 581.6015625
+    'sbg:y': 217
 steps:
-  - id: read
-    in:
-      - id: surl_list
-        source: surl_list
-    out:
-      - id: surls
-    run: steps/read_surl_list.cwl
-    label: read_surl
-    'sbg:x': -184
-    'sbg:y': -19.5
-  - id: stage
-    in:
-      - id: input
-        source: read/surls
-    out:
-      - id: output
-    run: steps/stage.cwl
-    label: stage
-    scatter:
-      - input
-    'sbg:x': -46
-    'sbg:y': -2
   - id: surl_copy
     in:
       - id: surl
-        source: stage/output
+        source: surl
     out:
       - id: output
     run: steps/surl_copy.cwl
     label: surl_copy
-    scatter:
-      - surl
-    'sbg:x': 117.234375
-    'sbg:y': 53.5
+    'sbg:x': 14
+    'sbg:y': -28
   - id: untar
     in:
       - id: tar_file
@@ -80,10 +52,8 @@ steps:
       - id: uncompressed
     run: steps/untar.cwl
     label: untar
-    scatter:
-      - tar_file
-    'sbg:x': 263.296875
-    'sbg:y': 53.5
+    'sbg:x': 173
+    'sbg:y': -28
   - id: dppp
     in:
       - id: parset
@@ -92,7 +62,7 @@ steps:
         source: untar/uncompressed
       - id: msout_name
         source: untar/uncompressed
-        valueFrom: $("out_" + self.basename)
+        valueFrom: $("COMPRESSED_" + self.basename)
       - id: writefullresflag
         default: true
       - id: storagemanager
@@ -101,29 +71,32 @@ steps:
         default: 10
     out:
       - id: msout
-      - id: secondary_output_files
-      - id: secondary_output_directories
       - id: logfile
     run: steps/DPPP.cwl
-    scatter:
-      - msin
-      - msout_name
-    scatterMethod: flat_crossproduct
-    'sbg:x': 469.03125
-    'sbg:y': 32.5
+    'sbg:x': 471
+    'sbg:y': -75
   - id: define_parset
     in:
-      - id: flagging_strategy
-        source: flagging_strategy
       - id: flag_autocorrelation
         source: flag_autocorrelation
     out:
       - id: output
     run: steps/define_parset.cwl
     label: define_parset
-    'sbg:x': 154
-    'sbg:y': -136
+    'sbg:x': 196
+    'sbg:y': -227
+  - id: inspect_flagging_dataloss
+    in:
+      - id: input
+        source: untar/uncompressed
+    out:
+      - id: output
+    run: steps/inspect_flagging_dataloss.cwl
+    label: inspect_flagging_dataloss
+    'sbg:x': 362
+    'sbg:y': 91
 requirements: 
-  - class: ScatterFeatureRequirement
-  - class: InlineJavascriptRequirement
   - class: StepInputExpressionRequirement
+  - class: InlineJavascriptRequirement
+
+

steps/DPPP.cwl

@@ -25,18 +25,6 @@ inputs:
       prefix: msout=
       separate: false
     doc: Output Measurement Set
-  - id: secondary_files
-    type: 'File[]?'
-    doc: Secondary files needed for the step
-  - id: secondary_directories
-    type: 'Directory[]?'
-    doc: Secondary directories needed for the step
-  - id: output_file_names
-    type: Any
-    doc: Expected output file names
-  - id: output_directory_names
-    type: Any
-    doc: Expected output file names
   - default: false
     id: autoweight
     type: boolean
@@ -92,40 +80,6 @@ outputs:
     type: Directory
     outputBinding:
       glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
-  - id: secondary_output_files
-    doc: Secondary output files
-    type: Any
-    outputBinding:
-      outputEval: |-
-        ${
-          var output={}
-          for(var step_name in inputs.output_file_names){
-            var file_per_idx = inputs.output_file_names[step_name];
-            for(var file_idx in file_per_idx){
-                var file_name = file_per_idx[file_idx];
-
-                output[file_idx] = {'class':'File', 'path': file_name};
-            }
-          }
-          return output
-        }
-  - id: secondary_output_directories
-    doc: Secondary output directories
-    type: Any
-    outputBinding:
-      outputEval: |-
-        ${
-          var output={}
-          for(var step_name in inputs.output_directory_names){
-            var file_per_idx = inputs.output_directory_names[step_name];
-            for(var file_idx in file_per_idx){
-                var file_name = file_per_idx[file_idx];
-
-                output[file_idx] = {'class':'Directory', 'path': file_name};
-            }
-          }
-          return output
-        }
   - id: logfile
     type: File[]
     outputBinding:
@@ -141,6 +95,3 @@ requirements:
     listing:
       - entry: $(inputs.msin)
         writable: true
-      - entry: $(inputs.secondary_files)
-      - entry: $(inputs.secondary_directories)
-      

steps/define_parset.cwl

@@ -6,18 +6,9 @@ id: define_parset
 baseCommand:
   - cp
 inputs:
-  - id: flagging_strategy
-    type:
-      type: enum
-      symbols:
-        - HBAdefault
-        - LBAdefault
-      name: flagging_strategy
   - id: flag_autocorrelation
     type: boolean?
     default: true
-    inputBinding:
-      valueFrom: '$(self ? ''True'': ''False'')'
 outputs:
   - id: output
     type: File
@@ -39,7 +30,7 @@ requirements:
     listing:
       - entryname: input.parset
         entry: |+
-          steps=[flagedge,aoflag,flagamp,compress]
+          steps=[flagedge,aoflag,flagamp]
           #
           flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
           flagedge.type=preflagger
@@ -47,8 +38,8 @@ requirements:
           aoflag.keepstatistics=false
           aoflag.memoryperc=20
           aoflag.type=aoflagger
-          aoflag.autocorr=$(flag_autocorrelation)
-          aoflag.strategy=$(flagging_strategy)
+          aoflag.autocorr=$(inputs.flag_autocorrelation?'True':'False')
+          aoflag.strategy=/usr/local/share/aoflagger/strategies/lofar-default.lua
           #
           flagbaseline.type=preflagger
           flagbaseline.baseline=[]
@@ -58,5 +49,4 @@ requirements:
           #
           flagamp.amplmin=1e-30
           flagamp.type=preflagger
-        writable: false
   - class: InlineJavascriptRequirement

steps/inspect_compression_quality.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: inspect_compression_quality
+baseCommand: []
+inputs:
+  - id: before_compression
+    type: Directory
+    inputBinding:
+      shellQuote: false
+      position: 0
+  - id: after_compression
+    type: Directory
+    inputBinding:
+      shellQuote: false
+      position: 0
+outputs:
+  - id: output
+    type: 'File[]'
+label: inspect_compression_quality
+requirements:
+  - class: ShellCommandRequirement

steps/inspect_flagging_dataloss.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: inspect_flagging_dataloss
+baseCommand:
+  - python3
+  - script.py
+inputs:
+  - id: input
+    type: Directory?
+    inputBinding:
+      position: 0
+      shellQuote: false
+outputs:
+  - id: output
+    type: 'File'
+    outputBinding:
+      glob: metrics.json
+label: inspect_flagging_dataloss
+requirements:
+  - class: ShellCommandRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+     - entryname: script.py
+       entry: |
+         import sys
+         from casacore.tables import table
+         input_ms_path = sys.argv[1]
+         
+         
+         with open('metrics.json', 'w') as f_stream:
+             pass
+         
+  - class: DockerRequirement
+    dockerPull: 'lofareosc/lofar-pipeline:latest'