From 622e0edaed4a84c02911a8090538e55bd3111782 Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Fri, 25 Oct 2019 09:49:36 +0200
Subject: [PATCH] First prefactor pipeline and integration fixes
Former-commit-id: 2d1a87523a7509cd86edace3abd2487a46c0e342
---
.gitlab-ci.yml | 6 +-
prefactor_calibrator.cwl | 74 +++++++++++++++++++
steps/DP3.DemixerStepGenerator.cwl | 2 +-
steps/DirSelector.cwl | 12 ++-
subworkflow/ndppp_prep_cal.cwl | 7 ++
...integration_test_prefactor_calibrator.json | 9 +++
6 files changed, 106 insertions(+), 4 deletions(-)
create mode 100644 prefactor_calibrator.cwl
create mode 100644 test_jobs/integration_test_prefactor_calibrator.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4449bc2f..a67f7dba 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ validate_scripts:
script:
- for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done
-test_DPPP_step:
+test_steps:
stage: test
script:
- cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json
@@ -39,3 +39,7 @@ test_DPPP_step:
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json
- cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json
+ allow_failure: true
+
+integration_test:
+ - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json
diff --git a/prefactor_calibrator.cwl b/prefactor_calibrator.cwl
new file mode 100644
index 00000000..d049b55d
--- /dev/null
+++ b/prefactor_calibrator.cwl
@@ -0,0 +1,74 @@
+class: Workflow
+cwlVersion: v1.0
+id: prefactor_calibrator
+label: prefactor_calibrator
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msin
+ type: 'Directory[]'
+ 'sbg:x': -659.0313110351562
+ 'sbg:y': -522.6085205078125
+ - id: raw_data
+ type: boolean
+ 'sbg:x': -620
+ 'sbg:y': -683
+ - id: demix
+ type: boolean
+ 'sbg:x': -617
+ 'sbg:y': -347
+outputs:
+ - id: demix_parmdb
+ outputSource:
+ - ndppp_prep_cal/demix_parmdb
+ type: ['Directory[]', Directory, 'null']
+ 'sbg:x': -67.85011291503906
+ 'sbg:y': -356.76287841796875
+ - id: output_ms
+ outputSource:
+ - aoflagger/output_ms
+ type: 'Directory[]'
+ 'sbg:x': 219.203125
+ 'sbg:y': -564.4959716796875
+steps:
+ - id: ndppp_prep_cal
+ in:
+ - id: baselines_to_flag
+ default: []
+ - id: elevation_to_flag
+ default: 0deg..20deg
+ - id: min_amplitude_to_flag
+ default: 1.e-30
+ - id: memoryperc
+ default: 20
+ - id: raw_data
+ source: raw_data
+ - id: demix
+ source: demix
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - msin
+ out:
+ - id: msout
+ - id: demix_parmdb
+ run: subworkflow/ndppp_prep_cal.cwl
+ label: ndppp_prep_cal
+ scatter:
+ - msin
+ 'sbg:x': -359
+ 'sbg:y': -519
+ - id: aoflagger
+ in:
+ - id: msin
+ source:
+ - ndppp_prep_cal/msout
+ out:
+ - id: output_ms
+ run: steps/AOFlagger.cwl
+ label: AOFlagger
+ 'sbg:x': -74
+ 'sbg:y': -597
+requirements:
+ - class: SubworkflowFeatureRequirement
+ - class: ScatterFeatureRequirement
diff --git a/steps/DP3.DemixerStepGenerator.cwl b/steps/DP3.DemixerStepGenerator.cwl
index d1a1ea50..c11ced07 100755
--- a/steps/DP3.DemixerStepGenerator.cwl
+++ b/steps/DP3.DemixerStepGenerator.cwl
@@ -65,7 +65,7 @@ inputs:
averaging, it has to fit integrally.
- id: skymodel
default: sky
- type: [File, Directory]
+ type: [File, Directory, string]
doc: The name of the SourceDB to use (i.e., the output of makesourcedb).
- id: instrumentmodel
default: instrument
diff --git a/steps/DirSelector.cwl b/steps/DirSelector.cwl
index 136a6ce0..219bf744 100644
--- a/steps/DirSelector.cwl
+++ b/steps/DirSelector.cwl
@@ -17,13 +17,21 @@ inputs:
(e.g. 'instrumentmodel')
outputs:
- id: output
- type: Directory
+ type: [Directory, 'Directory[]']
expression: |
${
var step_name = inputs.step_name
var key_name = inputs.key_name
- return {'output': inputs.inputs[step_name][key_name]}
+
+ if(inputs.inputs.hasOwnProperty(step_name)){
+ var inputs_per_step = inputs.inputs[step_name]
+ if(inputs_per_step.hasOwnProperty(key_name)){
+ return {'output': inputs_per_step[key_name]}
+ }
+ }
+ return {'output': []}
+
}
label: DirSelector
diff --git a/subworkflow/ndppp_prep_cal.cwl b/subworkflow/ndppp_prep_cal.cwl
index ebe32a2f..8226f13a 100644
--- a/subworkflow/ndppp_prep_cal.cwl
+++ b/subworkflow/ndppp_prep_cal.cwl
@@ -176,6 +176,9 @@ steps:
source: parsetgenerator/output_parset
- id: msin
source: msin
+ - id: msout_name
+ source: msin
+ valueFrom: $("out_"+self.basename)
- id: secondary_files
source:
- parsetgenerator/input_files
@@ -198,13 +201,17 @@ steps:
- id: inputs
source: dppp/secondary_output_directories
- id: step_name
+ default: demixstepgenerator
valueFrom: demixstepgenerator
- id: key_name
valueFrom: instrumentmodel
out:
- id: output
run: ../steps/DirSelector.cwl
+ label: DirSelector
'sbg:x': 2838.8115234375
'sbg:y': -151.54710388183594
requirements:
- class: SubworkflowFeatureRequirement
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
diff --git a/test_jobs/integration_test_prefactor_calibrator.json b/test_jobs/integration_test_prefactor_calibrator.json
new file mode 100644
index 00000000..8def27f2
--- /dev/null
+++ b/test_jobs/integration_test_prefactor_calibrator.json
@@ -0,0 +1,9 @@
+{
+ "msin" : [
+ {"class": "Directory", "path": "/data/L570745_SB000_uv_first10.MS"},
+ {"class": "Directory", "path": "/data/L570745_SB001_uv_first10.MS"},
+ {"class": "Directory", "path": "/data/L570745_SB002_uv_first10.MS"}
+ ],
+ "raw_data": false,
+ "demix": false
+}
--
GitLab