First commit

compress_pipeline.cwl

+class: Workflow
+cwlVersion: v1.0
+id: compress_pipeline_cwl
+label: compress_pipeline.cwl
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+inputs:
+  - id: surl_list
+    type: File
+    doc: input surl file
+    'sbg:x': -334.23162841796875
+    'sbg:y': -27
+  - id: flagging_strategy
+    type:
+      type: enum
+      symbols:
+        - HBAdefault
+        - LBAdefault
+      name: flagging_strategy
+    'sbg:x': -30
+    'sbg:y': -321.8600769042969
+  - id: flag_autocorrelation
+    type: boolean?
+    'sbg:x': -242.8984375
+    'sbg:y': -154.86007690429688
+outputs:
+  - id: msout
+    outputSource:
+      - dppp/msout
+    type: Directory
+    doc: Output Measurement Set
+    'sbg:x': 705.827392578125
+    'sbg:y': 48.75
+  - id: logfile
+    outputSource:
+      - dppp/logfile
+    type: 'File[]'
+    'sbg:x': 606.827392578125
+    'sbg:y': 207.75
+steps:
+  - id: read
+    in:
+      - id: surl_list
+        source: surl_list
+    out:
+      - id: surls
+    run: steps/read_surl_list.cwl
+    label: read_surl
+    'sbg:x': -184
+    'sbg:y': -19.5
+  - id: stage
+    in:
+      - id: input
+        source: read/surls
+    out:
+      - id: output
+    run: steps/stage.cwl
+    label: stage
+    scatter:
+      - input
+    'sbg:x': -46
+    'sbg:y': -2
+  - id: surl_copy
+    in:
+      - id: surl
+        source: stage/output
+    out:
+      - id: output
+    run: steps/surl_copy.cwl
+    label: surl_copy
+    scatter:
+      - surl
+    'sbg:x': 117.234375
+    'sbg:y': 53.5
+  - id: untar
+    in:
+      - id: tar_file
+        source: surl_copy/output
+    out:
+      - id: uncompressed
+    run: steps/untar.cwl
+    label: untar
+    scatter:
+      - tar_file
+    'sbg:x': 263.296875
+    'sbg:y': 53.5
+  - id: dppp
+    in:
+      - id: parset
+        source: define_parset/output
+      - id: msin
+        source: untar/uncompressed
+      - id: msout_name
+        source: untar/uncompressed
+        valueFrom: $("out_" + self.basename)
+      - id: writefullresflag
+        default: true
+      - id: storagemanager
+        default: Dysco
+      - id: databitrate
+        default: 10
+    out:
+      - id: msout
+      - id: secondary_output_files
+      - id: secondary_output_directories
+      - id: logfile
+    run: steps/DPPP.cwl
+    scatter:
+      - msin
+      - msout_name
+    scatterMethod: flat_crossproduct
+    'sbg:x': 469.03125
+    'sbg:y': 32.5
+  - id: define_parset
+    in:
+      - id: flagging_strategy
+        source: flagging_strategy
+      - id: flag_autocorrelation
+        source: flag_autocorrelation
+    out:
+      - id: output
+    run: steps/define_parset.cwl
+    label: define_parset
+    'sbg:x': 154
+    'sbg:y': -136
+requirements:
+  - class: ScatterFeatureRequirement
+  - class: InlineJavascriptRequirement
+  - class: StepInputExpressionRequirement

steps/DPPP.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: dppp
+baseCommand:
+  - DPPP
+inputs:
+  - id: parset
+    type: File?
+    inputBinding:
+      position: -1
+  - id: msin
+    type: Directory?
+    inputBinding:
+      position: 0
+      prefix: msin=
+      separate: false
+    doc: Input Measurement Set
+  - default: "."
+    id: msout_name
+    type: string
+    inputBinding:
+      position: 0
+      prefix: msout=
+      separate: false
+    doc: Output Measurement Set
+  - id: secondary_files
+    type: 'File[]?'
+    doc: Secondary files needed for the step
+  - id: secondary_directories
+    type: 'Directory[]?'
+    doc: Secondary directories needed for the step
+  - id: output_file_names
+    type: Any
+    doc: Expected output file names
+  - id: output_directory_names
+    type: Any
+    doc: Expected output file names
+  - default: false
+    id: autoweight
+    type: boolean
+    inputBinding:
+      position: 0
+      prefix: 'msin.autoweight=True'
+  - default: ''
+    id: baseline
+    type: string
+    inputBinding:
+      position: 0
+      prefix: 'msin.baseline='
+      separate: false
+  - default: DATA
+    id: output_column
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: 'msout.datacolumn='
+      separate: false
+  - default: DATA
+    id: input_column
+    type: string?
+    inputBinding:
+      position: 0
+      prefix: 'msin.datacolumn='
+      separate: false
+  - id: writefullresflag
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.writefullresflag=True
+  - id: overwrite
+    type: boolean
+    default: false
+    inputBinding:
+       prefix: msout.overwrite=True
+  - id: storagemanager
+    type: string
+    default: ""
+    inputBinding:
+       prefix: msout.storagemanager=
+       separate: false
+  - id: databitrate
+    type: int
+    default: 0
+    inputBinding:
+       prefix: msout.storagemanager.databitrate=
+       separate: false
+outputs:
+  - id: msout
+    doc: Output Measurement Set
+    type: Directory
+    outputBinding:
+      glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
+  - id: secondary_output_files
+    doc: Secondary output files
+    type: Any
+    outputBinding:
+      outputEval: |-
+        ${
+          var output={}
+          for(var step_name in inputs.output_file_names){
+            var file_per_idx = inputs.output_file_names[step_name];
+            for(var file_idx in file_per_idx){
+                var file_name = file_per_idx[file_idx];
+
+                output[file_idx] = {'class':'File', 'path': file_name};
+            }
+          }
+          return output
+        }
+  - id: secondary_output_directories
+    doc: Secondary output directories
+    type: Any
+    outputBinding:
+      outputEval: |-
+        ${
+          var output={}
+          for(var step_name in inputs.output_directory_names){
+            var file_per_idx = inputs.output_directory_names[step_name];
+            for(var file_idx in file_per_idx){
+                var file_name = file_per_idx[file_idx];
+
+                output[file_idx] = {'class':'Directory', 'path': file_name};
+            }
+          }
+          return output
+        }
+  - id: logfile
+    type: File[]
+    outputBinding:
+      glob: 'DPPP*.log'
+hints:
+  - class: DockerRequirement
+    dockerPull: lofareosc/prefactor:HBAcalibrator
+stdout: DPPP.log
+stderr: DPPP_err.log
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entry: $(inputs.msin)
+        writable: true
+      - entry: $(inputs.secondary_files)
+      - entry: $(inputs.secondary_directories)
+      

steps/define_parset.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: define_parset
+baseCommand:
+  - cp
+inputs:
+  - id: flagging_strategy
+    type:
+      type: enum
+      symbols:
+        - HBAdefault
+        - LBAdefault
+      name: flagging_strategy
+  - id: flag_autocorrelation
+    type: boolean?
+    default: true
+    inputBinding:
+      valueFrom: '$(self ? ''True'': ''False'')'
+outputs:
+  - id: output
+    type: File
+    outputBinding:
+      glob: preproc.parset
+label: define_parset
+arguments:
+  - prefix: ''
+    shellQuote: false
+    position: 0
+    valueFrom: input.parset
+  - prefix: ''
+    shellQuote: false
+    position: 0
+    valueFrom: preproc.parset
+requirements:
+  - class: ShellCommandRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entryname: input.parset
+        entry: |+
+          steps=[flagedge,aoflag,flagamp,compress]
+          #
+          flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
+          flagedge.type=preflagger
+          #
+          aoflag.keepstatistics=false
+          aoflag.memoryperc=20
+          aoflag.type=aoflagger
+          aoflag.autocorr=$(flag_autocorrelation)
+          aoflag.strategy=$(flagging_strategy)
+          #
+          flagbaseline.type=preflagger
+          flagbaseline.baseline=[]
+          #
+          flagelev.type=preflagger
+          flagelev.elevation=0deg..20deg
+          #
+          flagamp.amplmin=1e-30
+          flagamp.type=preflagger
+        writable: false
+  - class: InlineJavascriptRequirement

steps/read_surl_list.cwl

+class: ExpressionTool
+cwlVersion: v1.0
+id: read_surl_file
+inputs:
+  - id: surl_list
+    type: File
+    doc: input surl file
+    inputBinding:
+      loadContents: True
+outputs:
+  - id: surls
+    type: string[]
+
+expression: |
+  ${
+    var surl_list = inputs.surl_list.contents.split("\n");
+    surl_list = surl_list.filter(function (el) { return el != ''; } );
+    return {'surls': surl_list}
+  }
+label: ReadSurlList
+
+requirements:
+  - class: InlineJavascriptRequirement

steps/stage.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: stage
+baseCommand:
+  - gfal-legacy-bringonline
+inputs:
+  - id: input
+    type: string
+    inputBinding:
+      shellQuote: false
+      position: 0
+outputs:
+  - id: output
+    type: string
+    outputBinding:
+      outputEval: $(inputs.input)
+label: stage
+requirements:
+  - class: InlineJavascriptRequirement

steps/surl_copy.cwl

+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  sbg: 'https://www.sevenbridges.com/'
+id: surl_copy
+baseCommand:
+  - gfal-copy
+
+inputs:
+  - id: surl
+    type: string
+    inputBinding:
+      position: -1
+outputs:
+  - id: output
+    type: File
+    outputBinding:
+      glob: '*'
+doc: Download a file give a surl
+label: surl_copy
+arguments:
+  - .
+temporaryFailCodes:
+  - 70

steps/untar.cwl

+id: untar
+label: untar
+class: CommandLineTool
+cwlVersion: v1.0
+inputs: 
+  - id: tar_file
+    type: File
+    inputBinding:
+      position: 0
+    
+outputs: 
+  - id: uncompressed
+    type: Directory
+    outputBinding:
+      glob: 'out/*'
+baseCommand: 
+ - 'bash'
+ - 'untar.sh'
+doc: 'Untar a compressed file'
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: 'untar.sh' 
+        entry: |
+          #!/bin/bash
+          mkdir out
+          cd out
+          tar -xvf $1
+
+