diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 265d982c854beb24b37f3e25f001ea94b74dfc56..a03cf0d5b843253f918ef2467404eea0ace1d84b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,6 +105,12 @@ losoto_reset: script: - cwl-runner --no-container steps/LoSoTo.Reset.cwl test_jobs/losoto_reset.json +losoto_clip: + stage: test_steps + allow_failure: true + script: + - cwl-runner --no-container steps/LoSoTo.Clip.cwl test_jobs/losoto_clip.json + losoto_residual: stage: test_steps allow_failure: true @@ -178,11 +184,9 @@ gaincal: allow_failure: true script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/gaincal.cwl test_jobs/gaincal.json - + predict: stage: test_steps allow_failure: true script: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/predict.cwl test_jobs/predict.json - - diff --git a/steps/LoSoTo.Clip.cwl b/steps/LoSoTo.Clip.cwl new file mode 100644 index 0000000000000000000000000000000000000000..867699dac1e0c2c43eef644a1ebced30aa34d104 --- /dev/null +++ b/steps/LoSoTo.Clip.cwl @@ -0,0 +1,67 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.0 +id: losoto_clip + +$namespaces: + lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl +doc: | + Clip solutions around the median by a factor specified by the user. + WEIGHT: flag compliant, putting weights into median is tricky + +requirements: + InlineJavascriptRequirement: + expressionLib: + - { $include: utils.js} + InitialWorkDirRequirement: + listing: + - entryname: 'parset.config' + entry: $(get_losoto_config('CLIP').join('\n')) + + - entryname: $(inputs.input_h5parm.basename) + entry: $(inputs.input_h5parm) + writable: true + +baseCommand: "losoto" + +arguments: + - $(inputs.input_h5parm.basename) + - parset.config + +hints: + DockerRequirement: + dockerPull: lofareosc/lofar-pipeline-ci:latest + +inputs: + - id: input_h5parm + type: File + format: lofar:#H5Parm + - id: soltab + type: string + doc: "Solution table" + - id: axesToClip + type: string[] + doc: axes along which to calculate the median (e.g. [time,freq]) + - id: clipLevel + type: float? + doc: factor above/below median at which to clip, by default 5. + - id: log + type: boolean? + doc: clip is done in log10 space, by default False. + - id: mode + type: string? + doc: | + if "median" then flag at rms*clipLevel times away from the median, if it is "above", + then flag all values above clipLevel, if it is "below" then flag all values below clipLevel. + By default median. + + +outputs: + - id: output_h5parm + type: File + format: lofar:#H5Parm + outputBinding: + glob: $(inputs.input_h5parm.basename) +$schema: + - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl diff --git a/steps/LoSoTo.Plot.cwl b/steps/LoSoTo.Plot.cwl index e4145505e9595c28a1574e08223c702f772edbd0..c4309d5f8d0479ac41bc8bf7f91cdd6a34bab00c 100644 --- a/steps/LoSoTo.Plot.cwl +++ b/steps/LoSoTo.Plot.cwl @@ -11,22 +11,7 @@ doc: | requirements: InlineJavascriptRequirement: expressionLib: - - | - function get_config() { - var par = ['soltab = ' + inputs.soltab] - if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu); - par.push('[plot]') - par.push('operation = PLOT') - for(var field_name in inputs){ - if(field_name === 'input_h5parm' || - field_name === 'soltab' || - field_name === 'ncpu') continue; - if(inputs[field_name] === null || - inputs[field_name] === 'null') continue; - par.push(field_name+'='+inputs[field_name]) - } - return par - } + - { $include: utils.js} InitialWorkDirRequirement: listing: - entryname: 'parset.config' diff --git a/steps/utils.js b/steps/utils.js new file mode 100644 index 0000000000000000000000000000000000000000..27b21ab866c538d67828e46514205281a1dee5ff --- /dev/null +++ b/steps/utils.js @@ -0,0 +1,15 @@ +function get_losoto_config(step_name) { + var par = ['soltab = ' + inputs.soltab] + if (inputs.ncpu !== null) par.push('ncpu='+inputs.ncpu); + par.push("[" + step_name + "]") + par.push('operation=' + step_name) + for(var field_name in inputs){ + if(field_name === 'input_h5parm' || + field_name === 'soltab' || + field_name === 'ncpu') continue; + if(inputs[field_name] === null || + inputs[field_name] === 'null') continue; + par.push(field_name+'='+inputs[field_name]) + } + return par +} diff --git a/test_jobs/losoto_clip.json b/test_jobs/losoto_clip.json new file mode 100644 index 0000000000000000000000000000000000000000..fb2448f72937fae06e4fd69716f25f3f09ca5653 --- /dev/null +++ b/test_jobs/losoto_clip.json @@ -0,0 +1,7 @@ +{ + "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"}, + "soltab": "sol000/phase000", + "axesToClip": ["time"], + "ncpu": 1, + "log": true +}