diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4449bc2f4112e3ddb43abfc5a3409ee59c21d648..a67f7dba44185526ca3d24ad1a1d65b83305d059 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ validate_scripts: script: - for file in `find . -name 'steps/*.cwl'`; do cwltool --validate $file; done -test_DPPP_step: +test_steps: stage: test script: - cwl-runner steps/DP3.AveragerStepGenerator.cwl test_jobs/averager_step_generator.json @@ -39,3 +39,7 @@ test_DPPP_step: - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH steps/AOFlagger.cwl test_jobs/aoflagger.json - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/find_skymodel_cal.cwl test_jobs/find_skymodel_cal.json - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH steps/interpolate_visibilities.cwl test_jobs/interpolate_visibilities.json + allow_failure: true + +integration_test: + - cwltool --no-container --preserve-environment LD_LIBRARY_PATH --preserve-environment PATH --preserve-environment PYTHONPATH prefactor_calibrator.cwl test_jobs/integration_test_prefactor_calibrator.json diff --git a/prefactor_calibrator.cwl b/prefactor_calibrator.cwl new file mode 100644 index 0000000000000000000000000000000000000000..d049b55d4a85e2f1d926493c4f2e7b0d7aa96f8e --- /dev/null +++ b/prefactor_calibrator.cwl @@ -0,0 +1,74 @@ +class: Workflow +cwlVersion: v1.0 +id: prefactor_calibrator +label: prefactor_calibrator +$namespaces: + sbg: 'https://www.sevenbridges.com/' +inputs: + - id: msin + type: 'Directory[]' + 'sbg:x': -659.0313110351562 + 'sbg:y': -522.6085205078125 + - id: raw_data + type: boolean + 'sbg:x': -620 + 'sbg:y': -683 + - id: demix + type: boolean + 'sbg:x': -617 + 'sbg:y': -347 +outputs: + - id: demix_parmdb + outputSource: + - ndppp_prep_cal/demix_parmdb + type: ['Directory[]', Directory, 'null'] + 'sbg:x': -67.85011291503906 + 'sbg:y': -356.76287841796875 + - id: output_ms + outputSource: + - aoflagger/output_ms + type: 'Directory[]' + 'sbg:x': 219.203125 + 'sbg:y': -564.4959716796875 +steps: + - id: ndppp_prep_cal + in: + - id: baselines_to_flag + default: [] + - id: elevation_to_flag + default: 0deg..20deg + - id: min_amplitude_to_flag + default: 1.e-30 + - id: memoryperc + default: 20 + - id: raw_data + source: raw_data + - id: demix + source: demix + - id: msin + linkMerge: merge_flattened + source: + - msin + out: + - id: msout + - id: demix_parmdb + run: subworkflow/ndppp_prep_cal.cwl + label: ndppp_prep_cal + scatter: + - msin + 'sbg:x': -359 + 'sbg:y': -519 + - id: aoflagger + in: + - id: msin + source: + - ndppp_prep_cal/msout + out: + - id: output_ms + run: steps/AOFlagger.cwl + label: AOFlagger + 'sbg:x': -74 + 'sbg:y': -597 +requirements: + - class: SubworkflowFeatureRequirement + - class: ScatterFeatureRequirement diff --git a/steps/DP3.DemixerStepGenerator.cwl b/steps/DP3.DemixerStepGenerator.cwl index d1a1ea5072a8e9ea4b12e3d0edb95ff6607742cf..c11ced07db6f7bfe7a71ffbaf778380e184ff824 100755 --- a/steps/DP3.DemixerStepGenerator.cwl +++ b/steps/DP3.DemixerStepGenerator.cwl @@ -65,7 +65,7 @@ inputs: averaging, it has to fit integrally. - id: skymodel default: sky - type: [File, Directory] + type: [File, Directory, string] doc: The name of the SourceDB to use (i.e., the output of makesourcedb). - id: instrumentmodel default: instrument diff --git a/steps/DirSelector.cwl b/steps/DirSelector.cwl index 136a6ce0a3986aae1edee1d8234e7756a6b77f38..219bf744ab9a81513c31a2a8adc788d844a05f0a 100644 --- a/steps/DirSelector.cwl +++ b/steps/DirSelector.cwl @@ -17,13 +17,21 @@ inputs: (e.g. 'instrumentmodel') outputs: - id: output - type: Directory + type: [Directory, 'Directory[]'] expression: | ${ var step_name = inputs.step_name var key_name = inputs.key_name - return {'output': inputs.inputs[step_name][key_name]} + + if(inputs.inputs.hasOwnProperty(step_name)){ + var inputs_per_step = inputs.inputs[step_name] + if(inputs_per_step.hasOwnProperty(key_name)){ + return {'output': inputs_per_step[key_name]} + } + } + return {'output': []} + } label: DirSelector diff --git a/subworkflow/ndppp_prep_cal.cwl b/subworkflow/ndppp_prep_cal.cwl index ebe32a2f869b79c3c054d7e39b577dc82d9eef37..8226f13af15327ea024f44426c2b93494dda5061 100644 --- a/subworkflow/ndppp_prep_cal.cwl +++ b/subworkflow/ndppp_prep_cal.cwl @@ -176,6 +176,9 @@ steps: source: parsetgenerator/output_parset - id: msin source: msin + - id: msout_name + source: msin + valueFrom: $("out_"+self.basename) - id: secondary_files source: - parsetgenerator/input_files @@ -198,13 +201,17 @@ steps: - id: inputs source: dppp/secondary_output_directories - id: step_name + default: demixstepgenerator valueFrom: demixstepgenerator - id: key_name valueFrom: instrumentmodel out: - id: output run: ../steps/DirSelector.cwl + label: DirSelector 'sbg:x': 2838.8115234375 'sbg:y': -151.54710388183594 requirements: - class: SubworkflowFeatureRequirement + - class: StepInputExpressionRequirement + - class: InlineJavascriptRequirement diff --git a/test_jobs/integration_test_prefactor_calibrator.json b/test_jobs/integration_test_prefactor_calibrator.json new file mode 100644 index 0000000000000000000000000000000000000000..8def27f27ac6bcfafb6c743bf80bebd67c3bb250 --- /dev/null +++ b/test_jobs/integration_test_prefactor_calibrator.json @@ -0,0 +1,9 @@ +{ + "msin" : [ + {"class": "Directory", "path": "/data/L570745_SB000_uv_first10.MS"}, + {"class": "Directory", "path": "/data/L570745_SB001_uv_first10.MS"}, + {"class": "Directory", "path": "/data/L570745_SB002_uv_first10.MS"} + ], + "raw_data": false, + "demix": false +}