From 1394d6b518afdbb3147bdd131d8ba1c01513999e Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Fri, 10 Jan 2020 13:26:39 +0100
Subject: [PATCH] LoSoTo clip: Resolves EOSC-21
---
.gitlab-ci.yml | 10 ++++--
steps/LoSoTo.Clip.cwl | 67 ++++++++++++++++++++++++++++++++++++++
steps/LoSoTo.Plot.cwl | 17 +---------
steps/utils.js | 15 +++++++++
test_jobs/losoto_clip.json | 7 ++++
5 files changed, 97 insertions(+), 19 deletions(-)
create mode 100644 steps/LoSoTo.Clip.cwl
create mode 100644 steps/utils.js
create mode 100644 test_jobs/losoto_clip.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 265d982c..a03cf0d5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -105,6 +105,12 @@ losoto_reset:
script:
- cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json
+losoto_clip:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json
+
losoto_residual:
stage: test_steps
allow_failure: true
@@ -178,11 +184,9 @@ gaincal:
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/gaincal.cwl test_jobs/gaincal.json
-
+
predict:
stage: test_steps
allow_failure: true
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json
-
-
diff --git a/steps/LoSoTo.Clip.cwl b/steps/LoSoTo.Clip.cwl
new file mode 100644
index 00000000..867699da
--- /dev/null
+++ b/steps/LoSoTo.Clip.cwl
@@ -0,0 +1,67 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+id: losoto_clip
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+doc: |
+ Clip solutions around the median by a factor specified by the user.
+ WEIGHT: flag compliant, putting weights into median is tricky
+
+requirements:
+ InlineJavascriptRequirement:
+ expressionLib:
+ - { $include: utils.js}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'parset.config'
+ entry: $(get_losoto_config('CLIP').join('\n'))
+
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+
+baseCommand: "losoto"
+
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/lofar-pipeline-ci:latest
+
+inputs:
+ - id: input_h5parm
+ type: File
+ format: lofar:#H5Parm
+ - id: soltab
+ type: string
+ doc: "Solution table"
+ - id: axesToClip
+ type: string[]
+ doc: axes along which to calculate the median (e.g. [time,freq])
+ - id: clipLevel
+ type: float?
+ doc: factor above/below median at which to clip, by default 5.
+ - id: log
+ type: boolean?
+ doc: clip is done in log10 space, by default False.
+ - id: mode
+ type: string?
+ doc: |
+ if "median" then flag at rms*clipLevel times away from the median, if it is "above",
+ then flag all values above clipLevel, if it is "below" then flag all values below clipLevel.
+ By default median.
+
+
+outputs:
+ - id: output_h5parm
+ type: File
+ format: lofar:#H5Parm
+ outputBinding:
+ glob: $(inputs.input_h5parm.basename)
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Plot.cwl b/steps/LoSoTo.Plot.cwl
index e4145505..c4309d5f 100644
--- a/steps/LoSoTo.Plot.cwl
+++ b/steps/LoSoTo.Plot.cwl
@@ -11,22 +11,7 @@ doc: |
requirements:
InlineJavascriptRequirement:
expressionLib:
- - |
- function get_config() {
- var par = ['soltab = ' + inputs.soltab]
- if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu);
- par.push('[plot]')
- par.push('operation = PLOT')
- for(var field_name in inputs){
- if(field_name === 'input_h5parm' ||
- field_name === 'soltab' ||
- field_name === 'ncpu') continue;
- if(inputs[field_name] === null ||
- inputs[field_name] === 'null') continue;
- par.push(field_name+'='+inputs[field_name])
- }
- return par
- }
+ - { $include: utils.js}
InitialWorkDirRequirement:
listing:
- entryname: 'parset.config'
diff --git a/steps/utils.js b/steps/utils.js
new file mode 100644
index 00000000..27b21ab8
--- /dev/null
+++ b/steps/utils.js
@@ -0,0 +1,15 @@
+function get_losoto_config(step_name) {
+ var par = ['soltab = ' + inputs.soltab]
+ if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu);
+ par.push("[" + step_name + "]")
+ par.push('operation=' + step_name)
+ for(var field_name in inputs){
+ if(field_name === 'input_h5parm' ||
+ field_name === 'soltab' ||
+ field_name === 'ncpu') continue;
+ if(inputs[field_name] === null ||
+ inputs[field_name] === 'null') continue;
+ par.push(field_name+'='+inputs[field_name])
+ }
+ return par
+}
diff --git a/test_jobs/losoto_clip.json b/test_jobs/losoto_clip.json
new file mode 100644
index 00000000..fb2448f7
--- /dev/null
+++ b/test_jobs/losoto_clip.json
@@ -0,0 +1,7 @@
+{
+ "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
+ "soltab": "sol000/phase000",
+ "axesToClip": ["time"],
+ "ncpu": 1,
+ "log": true
+}
--
GitLab