From a79ad4cb74e7851314325b180957d8694155140b Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Wed, 16 Oct 2019 14:53:53 +0200
Subject: [PATCH] Refactor and add File/Dir Selector
---
steps/DP3.ParsetGenerator.cwl | 160 +++++++++++----------
steps/FileDirSelector.cwl | 24 ++++
test_jobs/parset_generator_file_input.json | 22 +++
3 files changed, 130 insertions(+), 76 deletions(-)
create mode 100644 steps/FileDirSelector.cwl
create mode 100644 test_jobs/parset_generator_file_input.json
diff --git a/steps/DP3.ParsetGenerator.cwl b/steps/DP3.ParsetGenerator.cwl
index 6f374ad8..3b759498 100644
--- a/steps/DP3.ParsetGenerator.cwl
+++ b/steps/DP3.ParsetGenerator.cwl
@@ -54,87 +54,95 @@ inputs:
type: Any[]?
outputs:
- - id: output_parset
- doc: Parset output file
- type: File
- outputBinding:
- glob: output_parset
- - id: input_files
- doc: additional input files
- type: File[]
- outputBinding:
- outputEval: |-
- ${
- var outfiles = []
- for(let step in inputs.parameters_input){
- step = inputs.parameters_input[step]
- for(let parameter in step['parameters']){
- parameter = step['parameters'][parameter]
-
- if (parameter != null && parameter.class =='File'){
- outfiles.push(parameter)
- }
+ - id: output_parset
+ doc: Parset output file
+ type: File
+ outputBinding:
+ glob: output_parset
+ - id: input_files
+ doc: additional input files
+ type: File[]
+ outputBinding:
+ outputEval: |-
+ ${
+ var outfiles = []
+ for(let step in inputs.parameters_input){
+ step = inputs.parameters_input[step]
+ for(let parameter in step['parameters']){
+ parameter = step['parameters'][parameter]
+
+ if (parameter != null && parameter.class =='File'){
+ outfiles.push(parameter)
}
}
- return outfiles
- }
- - id: input_directories
- doc: additional input directory
- type: Directory[]
- outputBinding:
- outputEval: |-
- ${
- var outdirs = []
- for(let step in inputs.parameters_input){
- step = inputs.parameters_input[step]
- for(let parameter in step['parameters']){
- parameter = step['parameters'][parameter]
- if (parameter != null && parameter.class=='Directory'){
- outdirs.push(parameter)
- }
+ }
+ return outfiles
+ }
+ - id: input_directories
+ doc: additional input directory
+ type: Directory[]
+ outputBinding:
+ outputEval: |
+ ${
+ var outdirs = []
+ for(let step in inputs.parameters_input){
+ step = inputs.parameters_input[step]
+ for(let parameter in step['parameters']){
+ parameter = step['parameters'][parameter]
+ if (parameter != null && parameter.class=='Directory'){
+ outdirs.push(parameter)
}
}
- return outdirs
- }
- - id: output_files
- doc: expected output files
- type: Any
- outputBinding:
- outputEval: |-
- ${
- var out_names = {}
- for(let step in inputs.parameters_input){
- step = inputs.parameters_input[step]
-
- if(!Object.hasOwnProperty("output_files")) continue;
-
- for(var index in step['output_files']){
- var file_id = step['output_files'][index]
- var file_name = step['output_files'][file_id]
- out_names[file_id] = file_name
- }
+ }
+ return outdirs
+ }
+
+ - id: output_files
+ doc: expected output files
+ type: Any
+ outputBinding:
+ outputEval: |
+ ${
+ var out_names = {}
+ for(var step_idx in inputs.parameters_input){
+ var step = inputs.parameters_input[step_idx]
+ var step_id = step['step_id']
+ var step_dirs = {}
+ if(!step.hasOwnProperty("output_files")) continue;
+
+ var step_files = {}
+
+ for(var index in step['output_files']){
+ var file_id = index
+ var file_name = step['output_files'][file_id]
+ step_files[file_id] = file_name
}
- return out_names
+
+ out_names[step_id] = step_files
+ }
+ return out_names
+ }
+ - id: output_directories
+ doc: expected output directories
+ type: Any
+ outputBinding:
+ outputEval: |-
+ ${
+ var out_names = {}
+ for(var step_idx in inputs.parameters_input){
+ var step = inputs.parameters_input[step_idx]
+ var step_id = step['step_id']
+ var step_dirs = {}
+ if(!step.hasOwnProperty("output_directories")) continue;
+
+ for(var index in step['output_directories']){
+ var file_id = index
+ var file_name = step['output_directories'][file_id]
+ step_dirs[file_id] = file_name
}
- - id: output_directories
- doc: expected output directories
- type: Any
- outputBinding:
- outputEval: |-
- ${
- var out_names = {}
- for(let step in inputs.parameters_input){
- step = inputs.parameters_input[step]
-
- if(!Object.hasOwnProperty("output_directories")) continue;
-
- for(var index in step['output_directories']){
- var file_id = step['output_directories'][index]
- var file_name = step['output_directories'][file_id]
- out_names[file_id] = file_name
- }
- }
- return out_names
- }
+ out_names[step_id] = step_dirs
+ }
+ return out_names
+ }
stdout:
output_parset
diff --git a/steps/FileDirSelector.cwl b/steps/FileDirSelector.cwl
new file mode 100644
index 00000000..fc1aaf99
--- /dev/null
+++ b/steps/FileDirSelector.cwl
@@ -0,0 +1,24 @@
+class: ExpressionTool
+cwlVersion: v1.0
+id: parsetselector
+inputs:
+ - id: inputs
+ type: Any
+ doc: input files
+ - id: step_name
+ type: string
+ doc: 'if true selects parset_a, files_a if false selects parset_b files_b'
+ - id: type
+ type: string
+outputs:
+ - id: output
+ type: [File, Directory]
+
+expression: |
+ $(
+ {'output': inputs.inputs[step_name][type]}
+ )
+label: ParsetSelector
+
+requirements:
+ - class: InlineJavascriptRequirement
diff --git a/test_jobs/parset_generator_file_input.json b/test_jobs/parset_generator_file_input.json
new file mode 100644
index 00000000..c29a5a08
--- /dev/null
+++ b/test_jobs/parset_generator_file_input.json
@@ -0,0 +1,22 @@
+{
+ "parameters_input": [{
+ "step_type": "averager",
+ "step_id": "average",
+ "output_files": {
+ "parmdb": "fily"
+ },
+ "output_directories": {
+ "parmdb": "diry"
+ },
+ "parameters": {
+ "timestep": 1,
+ "freqstep": 1,
+ "strategy": {
+ "class": "File",
+ "location": "file:///etc/hostname",
+ "size": 0,
+ "nameext": ""
+ }
+ }
+}]
+}
--
GitLab