From eb2a987dc0d3e5b5ea1962a66edb099107f2c05d Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Fri, 25 Oct 2019 16:18:44 +0200
Subject: [PATCH] Add blsmooth and refactored gitlab integration
---
.gitlab-ci.yml | 61 +++++++++++++++++++++---
steps/blsmooth.cwl | 87 +++++++++++++++++++++++++++++++++++
test_jobs/blsmooth.json | 6 +++
test_jobs/demo_prefactor.json | 6 +--
4 files changed, 151 insertions(+), 9 deletions(-)
create mode 100644 steps/blsmooth.cwl
create mode 100644 test_jobs/blsmooth.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dd589a17..4430f058 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,24 +25,73 @@ validate_scripts:
script:
- for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
-test_steps:
- stage: test
+average_step_generator:
+ stage: test_steps
allow_failure: true
script:
- cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
+ - ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
+
+aoflagger_step_generator:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwl-runner steps/DP3.AOFlaggerStepGenerator.cwl test_jobs/aoflagger_step_generator.json
+
+preflagger_step_generator:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwl-runner steps/DP3.PreflaggerStepGenerator.cwl test_jobs/preflagger_step_generator.json
+
+parset_selector:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwl-runner steps/DP3.ParsetSelector.cwl test_jobs/parset_selector.json
+
+file_selector:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwl-runner steps/FileSelector.cwl test_jobs/file_selector.json
- - ! cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator_error.json
- - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH test_workflow/flag_average.cwl --msin=/data/L570745_SB001_uv_first10.MS
+
+NDPPP:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/DPPP.cwl test_jobs/NDPPP.json
+
+aoflagger:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
+
+find_skymodel_cal:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
+
+interpolate_visibilities:
+ stage: test_steps
+ allow_failure: true
+ script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
+blsmooth:
+ stage: test_steps
+ allow_failure: true
+ script:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/blsmooth.cwl test_jobs/blsmooth.json
+
-integration_test:
- stage: test
+prefactor_calibrator:
+ stage: integration_test
script:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json
+
+flag_average:
+ stage: integration_test
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH test_workflow/flag_average.cwl --msin=/data/L570745_SB001_uv_first10.MS
diff --git a/steps/blsmooth.cwl b/steps/blsmooth.cwl
new file mode 100644
index 00000000..449e392b
--- /dev/null
+++ b/steps/blsmooth.cwl
@@ -0,0 +1,87 @@
+class: CommandLineTool
+cwlVersion: v1.0
+id: blsmooth
+label: BLsmooth
+arguments:
+ - '-S SMOOTH'
+baseCommand:
+ - BLsmooth.py
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 1
+ doc: Input measurement set
+ - default: 0.2
+ id: ionfactor
+ type: float
+ inputBinding:
+ position: 0
+ prefix: '-f'
+ doc: Gives an indication on how strong is the ionosphere
+ - default: 0.5
+ id: bscalefactor
+ type: float
+ inputBinding:
+ position: 0
+ prefix: '-s'
+ doc: Gives an indication on how the smoothing varies with
+ - default: DATA
+ id: in_column_name
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-i'
+ doc: Column name to smooth
+ - default: SMOOTHED_DATA
+ id: out_column
+ type: string
+ inputBinding:
+ position: 0
+ prefix: '-o'
+ doc: Output column
+ - default: false
+ id: weight
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-w'
+ doc: >-
+ Save the newly computed WEIGHT_SPECTRUM, this action permanently modify
+ the MS!
+ - default: false
+ id: restore
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-r'
+ doc: If WEIGHT_SPECTRUM_ORIG exists then restore it before smoothing
+ - default: false
+ id: nobackup
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-b'
+ doc: Do not backup the old WEIGHT_SPECTRUM in WEIGHT_SPECTRUM_ORIG
+ - default: false
+ id: onlyamp
+ type: boolean
+ inputBinding:
+ position: 0
+ prefix: '-a'
+ doc: Smooth only amplitudes
+outputs:
+ - id: msout
+ doc: MS set output
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msin.basename)
+
+requirements:
+- class: DockerRequirement
+ dockerPull: 'prefactor:latest'
+- class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
+- class: InlineJavascriptRequirement
diff --git a/test_jobs/blsmooth.json b/test_jobs/blsmooth.json
new file mode 100644
index 00000000..b2510c52
--- /dev/null
+++ b/test_jobs/blsmooth.json
@@ -0,0 +1,6 @@
+{
+ "msin": {
+ "class": "Directory",
+ "path": "/data/L570745_SB003_uv_first10.MS"
+ }
+}
diff --git a/test_jobs/demo_prefactor.json b/test_jobs/demo_prefactor.json
index d51872d1..ef2e0db9 100644
--- a/test_jobs/demo_prefactor.json
+++ b/test_jobs/demo_prefactor.json
@@ -2,15 +2,15 @@
"msin": [
{
"class": "Directory",
- "path": "/home/mmancini/git/prefactor3-cwl/test_data/L570745_SB003_uv_first10.MS"
+ "path": "/data/L570745_SB003_uv_first10.MS"
},
{
"class": "Directory",
- "path": "/home/mmancini/git/prefactor3-cwl/test_data/L570745_SB005_uv_first10.MS"
+ "path": "/data/L570745_SB005_uv_first10.MS"
},
{
"class": "Directory",
- "path": "/home/mmancini/git/prefactor3-cwl/test_data/L570745_SB008_uv_first10.MS"
+ "path": "/data/L570745_SB008_uv_first10.MS"
}
]
}
--
GitLab