Merge branch 'LoSoTo-Clip' into 'master'

LoSoTo clip: Resolves EOSC-21

See merge request eosc/lofar-cwl!3

.gitlab-ci.yml

@@ -105,6 +105,12 @@ losoto_reset:
   script:
     - cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json
 
+losoto_clip:
+  stage: test_steps
+  allow_failure: true
+  script:
+    - cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json
+
 losoto_residual:
   stage: test_steps
   allow_failure: true
@@ -178,11 +184,9 @@ gaincal:
   allow_failure: true
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/gaincal.cwl test_jobs/gaincal.json
-    
+
 predict:
   stage: test_steps
   allow_failure: true
   script:
     - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json
-
-

steps/LoSoTo.Clip.cwl

+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_clip
+
+$namespaces:
+  lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+  Clip solutions around the median by a factor specified by the user.
+  WEIGHT: flag compliant, putting weights into median is tricky
+
+requirements:
+  InlineJavascriptRequirement:
+    expressionLib:
+      - { $include: utils.js}
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: 'parset.config'
+        entry: $(get_losoto_config('CLIP').join('\n'))
+
+      - entryname: $(inputs.input_h5parm.basename)
+        entry: $(inputs.input_h5parm)
+        writable: true
+
+baseCommand: "losoto"
+
+arguments:
+  - $(inputs.input_h5parm.basename)
+  - parset.config
+
+hints:
+  DockerRequirement:
+    dockerPull: lofareosc/lofar-pipeline-ci:latest
+
+inputs:
+  - id: input_h5parm
+    type: File
+    format: lofar:#H5Parm
+  - id: soltab
+    type: string
+    doc: "Solution table"
+  - id: axesToClip
+    type: string[]
+    doc: axes along which to calculate the median (e.g. [time,freq])
+  - id: clipLevel
+    type: float?
+    doc: factor above/below median at which to clip, by default 5.
+  - id: log
+    type: boolean?
+    doc: clip is done in log10 space, by default False.
+  - id: mode
+    type: string?
+    doc: |
+      if "median" then flag at rms*clipLevel times away from the median, if it is "above",
+      then flag all values above clipLevel, if it is "below" then flag all values below clipLevel.
+      By default median.
+
+
+outputs:
+  - id: output_h5parm
+    type: File
+    format: lofar:#H5Parm
+    outputBinding:
+      glob: $(inputs.input_h5parm.basename)
+$schema:
+  - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl

steps/LoSoTo.Plot.cwl

@@ -11,22 +11,7 @@ doc: |
 requirements:
   InlineJavascriptRequirement:
     expressionLib:
-     - |
-        function get_config() {
-        var par = ['soltab = ' + inputs.soltab]
-        if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu);
-        par.push('[plot]')
-        par.push('operation = PLOT')
-        for(var field_name in inputs){
-            if(field_name === 'input_h5parm' ||
-               field_name === 'soltab' ||
-               field_name === 'ncpu') continue;
-            if(inputs[field_name] === null ||
-               inputs[field_name] === 'null') continue;
-            par.push(field_name+'='+inputs[field_name])
-        }
-        return par
-        }
+      - { $include: utils.js}
   InitialWorkDirRequirement:
     listing:
       - entryname: 'parset.config'

steps/utils.js

+function get_losoto_config(step_name) {
+    var par = ['soltab = ' + inputs.soltab]
+    if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu);
+    par.push("[" + step_name + "]")
+    par.push('operation=' + step_name)
+    for(var field_name in inputs){
+        if(field_name === 'input_h5parm' ||
+           field_name === 'soltab' ||
+           field_name === 'ncpu') continue;
+        if(inputs[field_name] === null ||
+           inputs[field_name] === 'null') continue;
+        par.push(field_name+'='+inputs[field_name])
+    }
+    return par
+}

test_jobs/losoto_clip.json

+{
+  "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+  "soltab":  "sol000/phase000",
+  "axesToClip": ["time"],
+  "ncpu": 1,
+  "log": true
+}