From 6d64a32d952a6b9c960d7bf262f28fed1a8fa9f3 Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Mon, 9 Dec 2019 14:35:18 +0100
Subject: [PATCH] Defining ontologies for H5Parm.
Resolve EOSC-41.
Former-commit-id: 9ee0c01ab0d0d71bdba9e37a7bbfc53c71a7c482
---
steps/DP3.DemixerStepGenerator.cwl | 2 +-
steps/DirSelector.cwl | 8 +++---
steps/H5ParmCollector.cwl | 10 ++++++-
steps/LoSoTo.Abs.cwl | 8 ++++++
steps/LoSoTo.Duplicate.cwl | 10 ++++++-
steps/LoSoTo.Norm.cwl | 8 ++++++
steps/LoSoTo.Plot.cwl | 7 +++++
steps/LoSoTo.Replicateonaxis.cwl | 8 ++++++
steps/LoSoTo.Reset.cwl | 8 ++++++
steps/LoSoTo.Residual.cwl | 9 +++++-
steps/RadioDatatypes.yaml | 8 ++++++
steps/blsmooth.cwl | 4 +--
steps/check_ateam_separation.cwl | 2 +-
steps/ddecal.cwl | 14 +++++++---
steps/find_skymodel_cal.cwl | 2 +-
steps/msinfo.cwl | 40 +++++++++++++++++++++++++++
steps/wsclean.cwl | 4 +--
test_jobs/h5parm_collector.json | 3 +-
test_jobs/losoto_abs.json | 3 +-
test_jobs/losoto_duplicate.json | 3 +-
test_jobs/losoto_norm.json | 3 +-
test_jobs/losoto_plot.json | 2 +-
test_jobs/losoto_replicateonaxis.json | 3 +-
test_jobs/losoto_reset.json | 3 +-
test_jobs/losoto_residual.json | 3 +-
25 files changed, 149 insertions(+), 26 deletions(-)
create mode 100644 steps/RadioDatatypes.yaml
create mode 100644 steps/msinfo.cwl
diff --git a/steps/DP3.DemixerStepGenerator.cwl b/steps/DP3.DemixerStepGenerator.cwl
index caaa14ce..c88bdd17 100755
--- a/steps/DP3.DemixerStepGenerator.cwl
+++ b/steps/DP3.DemixerStepGenerator.cwl
@@ -138,7 +138,7 @@ steps:
source: steps
- id: output_files
default: ["instrumentmodel"]
-
+
- id: parameters
valueFrom: $(inputs)
- id: step_id
diff --git a/steps/DirSelector.cwl b/steps/DirSelector.cwl
index 219bf744..d5bd0403 100644
--- a/steps/DirSelector.cwl
+++ b/steps/DirSelector.cwl
@@ -23,15 +23,15 @@ expression: |
${
var step_name = inputs.step_name
var key_name = inputs.key_name
-
+
if(inputs.inputs.hasOwnProperty(step_name)){
var inputs_per_step = inputs.inputs[step_name]
if(inputs_per_step.hasOwnProperty(key_name)){
- return {'output': inputs_per_step[key_name]}
+ return {'output': inputs_per_step[key_name]}
}
}
- return {'output': []}
-
+ return {'output': []}
+
}
label: DirSelector
diff --git a/steps/H5ParmCollector.cwl b/steps/H5ParmCollector.cwl
index fe6ba474..e2e33e20 100755
--- a/steps/H5ParmCollector.cwl
+++ b/steps/H5ParmCollector.cwl
@@ -2,12 +2,18 @@ class: CommandLineTool
cwlVersion: v1.0
$namespaces:
sbg: 'https://www.sevenbridges.com/'
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+
id: h5parm_collector
+
baseCommand:
- H5parm_collector.py
inputs:
- id: h5parmFiles
type: 'File[]'
+ format: lofar:#H5Parm
inputBinding:
position: 0
doc: List of h5parm files
@@ -23,7 +29,7 @@ inputs:
type: string
doc: Output h5parm name
- id: squeeze
- type: boolean
+ type: boolean
default: false
inputBinding:
position: 0
@@ -47,6 +53,7 @@ outputs:
- id: outh5parm
doc: Output h5parm
type: File
+ format: lofar:#H5Parm
outputBinding:
glob: $(inputs.outh5parmname)
- id: log
@@ -60,3 +67,4 @@ hints:
stdout: $(inputs.outh5parmname)-parm_collector_output.log
requirements:
- class: InlineJavascriptRequirement
+
diff --git a/steps/LoSoTo.Abs.cwl b/steps/LoSoTo.Abs.cwl
index ed419077..43f5c28c 100755
--- a/steps/LoSoTo.Abs.cwl
+++ b/steps/LoSoTo.Abs.cwl
@@ -4,6 +4,8 @@ class: CommandLineTool
cwlVersion: v1.0
id: losoto_abs
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
doc: |
Take absolute value. Needed before smooth if amplitudes are negative!
WEIGHT: no need to be weight compliant
@@ -33,6 +35,7 @@ hints:
inputs:
- id: input_h5parm
type: File
+ format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
@@ -40,5 +43,10 @@ inputs:
outputs:
- id: output_h5parm
type: File
+ format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+
+
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Duplicate.cwl b/steps/LoSoTo.Duplicate.cwl
index 777f6a6b..9111602f 100644
--- a/steps/LoSoTo.Duplicate.cwl
+++ b/steps/LoSoTo.Duplicate.cwl
@@ -35,14 +35,22 @@ hints:
inputs:
- id: input_h5parm
type: File
+ format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
- - id: soltabOut
+ - id: soltabOut
type: string?
doc: Output table name. By default choose next available from table type.
outputs:
- id: output_h5parm
type: File
+ format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Norm.cwl b/steps/LoSoTo.Norm.cwl
index 3a6c00db..f60ee4c4 100644
--- a/steps/LoSoTo.Norm.cwl
+++ b/steps/LoSoTo.Norm.cwl
@@ -36,6 +36,7 @@ hints:
inputs:
- id: input_h5parm
type: File
+ format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
@@ -49,5 +50,12 @@ inputs:
outputs:
- id: output_h5parm
type: File
+ format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Plot.cwl b/steps/LoSoTo.Plot.cwl
index c87748e4..e5d5cfb0 100644
--- a/steps/LoSoTo.Plot.cwl
+++ b/steps/LoSoTo.Plot.cwl
@@ -49,6 +49,7 @@ hints:
inputs:
- id: input_h5parm
type: File
+ format: lofar:#H5Parm
- id: soltab
type: string[]
doc: "Tabs to plot"
@@ -139,3 +140,9 @@ outputs:
type: File[]
outputBinding:
glob: "$(inputs.prefix)*.png"
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Replicateonaxis.cwl b/steps/LoSoTo.Replicateonaxis.cwl
index 5a5bc2ae..c676467d 100644
--- a/steps/LoSoTo.Replicateonaxis.cwl
+++ b/steps/LoSoTo.Replicateonaxis.cwl
@@ -47,6 +47,7 @@ hints:
inputs:
- id: input_h5parm
type: File
+ format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
@@ -65,5 +66,12 @@ inputs:
outputs:
- id: output_h5parm
type: File
+ format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Reset.cwl b/steps/LoSoTo.Reset.cwl
index 149587c9..99594d12 100644
--- a/steps/LoSoTo.Reset.cwl
+++ b/steps/LoSoTo.Reset.cwl
@@ -34,6 +34,7 @@ hints:
inputs:
- id: input_h5parm
type: File
+ format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
@@ -45,5 +46,12 @@ inputs:
outputs:
- id: output_h5parm
type: File
+ format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/LoSoTo.Residual.cwl b/steps/LoSoTo.Residual.cwl
index e8aed596..4bc566cd 100644
--- a/steps/LoSoTo.Residual.cwl
+++ b/steps/LoSoTo.Residual.cwl
@@ -2,7 +2,7 @@
class: CommandLineTool
cwlVersion: v1.0
-id: losoto_abs
+id: losoto_residual
doc: Subtract/divide two tables or a clock/tec/tec3rd/rm from a phase.
@@ -34,6 +34,7 @@ hints:
inputs:
- id: input_h5parm
type: File
+ format: lofar:#H5Parm
- id: soltab
type: string
doc: "Solution table"
@@ -48,5 +49,11 @@ inputs:
outputs:
- id: output_h5parm
type: File
+ format: lofar#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/RadioDatatypes.yaml b/steps/RadioDatatypes.yaml
new file mode 100644
index 00000000..0a167954
--- /dev/null
+++ b/steps/RadioDatatypes.yaml
@@ -0,0 +1,8 @@
+type: array
+name: MeasurementSet
+label: The measurement set data location
+items:
+ - name: main_directory
+ type: Directory
+ inputBinding:
+ position: 0
diff --git a/steps/blsmooth.cwl b/steps/blsmooth.cwl
index 9d0c5dbb..d139c594 100644
--- a/steps/blsmooth.cwl
+++ b/steps/blsmooth.cwl
@@ -20,9 +20,9 @@ inputs:
- id: do_smooth
type: boolean
doc: 'If true performs smoothing'
- inputBinding:
+ inputBinding:
prefix: '--smooth=SMOOTH'
-
+
- default: 0.5
id: bscalefactor
type: float
diff --git a/steps/check_ateam_separation.cwl b/steps/check_ateam_separation.cwl
index a7bcaf7e..977d9f86 100755
--- a/steps/check_ateam_separation.cwl
+++ b/steps/check_ateam_separation.cwl
@@ -4,7 +4,7 @@ $namespaces:
sbg: 'https://www.sevenbridges.com/'
id: check_ateam_separation
baseCommand:
- - python3
+ - python3
- /usr/local/bin/check_Ateam_separation.py
inputs:
- id: ms
diff --git a/steps/ddecal.cwl b/steps/ddecal.cwl
index 17830c4c..c534d536 100644
--- a/steps/ddecal.cwl
+++ b/steps/ddecal.cwl
@@ -7,7 +7,7 @@ baseCommand: [DPPP]
requirements:
InlineJavascriptRequirement: {}
-
+
hints:
DockerRequirement:
dockerPull: lofareosc/prefactor-ci:master
@@ -23,7 +23,7 @@ arguments:
- ddecal.flagdivergedonly=True
- ddecal.tolerance=1.e-3
- ddecal.usemodelcolumn=True
-
+
inputs:
- id: msin
@@ -48,7 +48,7 @@ inputs:
inputBinding:
prefix: msin.modelcolumn=
separate: false
-
+
- id: output_name_h5parm
type: string
default: instrument.h5
@@ -62,7 +62,7 @@ inputs:
inputBinding:
prefix: msout=
separate: false
-
+
#--------------------
- id: propagate_solutions
type: boolean
@@ -101,5 +101,11 @@ outputs:
- id: h5parm
doc: Filename of output H5Parm (to be read by e.g. losoto)
type: File
+ format: lofar:#H5Parm
outputBinding:
glob: $(inputs.output_name_h5parm)
+
+$namespaces:
+ lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+$schema:
+ - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
diff --git a/steps/find_skymodel_cal.cwl b/steps/find_skymodel_cal.cwl
index 2d7af826..51a1dc4a 100644
--- a/steps/find_skymodel_cal.cwl
+++ b/steps/find_skymodel_cal.cwl
@@ -26,7 +26,7 @@ arguments:
from unittest.mock import MagicMock
sys.modules['lofarpipe.support.data_map'] = MagicMock()
-
+
from find_skymodel_cal import main as find_skymodel
import json
diff --git a/steps/msinfo.cwl b/steps/msinfo.cwl
new file mode 100644
index 00000000..0c7e2f5b
--- /dev/null
+++ b/steps/msinfo.cwl
@@ -0,0 +1,40 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.1
+id: msinfo
+
+requirements:
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: msinfo.py
+ entry: |
+ import sys
+ print(sys.argv)
+ - class: SchemaDefRequirement
+ types:
+ - $import: RadioDatatypes.yaml
+hints:
+ DockerRequirement:
+ dockerPull: lofareosc/prefactor-ci:master
+
+baseCommand:
+ - python3
+ - msinfo.py
+
+inputs:
+ data:
+ type: RadioDatatypes.yaml#MeasurementSet
+ inputBinding:
+ position: 0
+
+stdout: std.output
+
+outputs:
+ info:
+ type: File
+ outputBinding:
+ glob: std.output
+label: msinfo
+
diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl
index c4740821..2bb63f20 100644
--- a/steps/wsclean.cwl
+++ b/steps/wsclean.cwl
@@ -20,14 +20,14 @@ inputs:
- id: image_name
default: image
type: string
- inputBinding:
+ inputBinding:
position: 1
prefix: -name
- id: msin
type: Directory
inputBinding:
position: 2
-outputs:
+outputs:
dirty_image:
type: File
outputBinding:
diff --git a/test_jobs/h5parm_collector.json b/test_jobs/h5parm_collector.json
index 958493f2..42c60bb8 100644
--- a/test_jobs/h5parm_collector.json
+++ b/test_jobs/h5parm_collector.json
@@ -2,7 +2,8 @@
"h5parmFiles": [
{
"class": "File",
- "path": "/data/example.h5"
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
}
]
}
diff --git a/test_jobs/losoto_abs.json b/test_jobs/losoto_abs.json
index ff1cfa0f..a283362c 100644
--- a/test_jobs/losoto_abs.json
+++ b/test_jobs/losoto_abs.json
@@ -1,7 +1,8 @@
{
"input_h5parm": {
"class": "File",
- "path": "/data/example.h5"
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
},
"soltab": "sol000/amplitude000"
}
diff --git a/test_jobs/losoto_duplicate.json b/test_jobs/losoto_duplicate.json
index ff1cfa0f..a283362c 100644
--- a/test_jobs/losoto_duplicate.json
+++ b/test_jobs/losoto_duplicate.json
@@ -1,7 +1,8 @@
{
"input_h5parm": {
"class": "File",
- "path": "/data/example.h5"
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
},
"soltab": "sol000/amplitude000"
}
diff --git a/test_jobs/losoto_norm.json b/test_jobs/losoto_norm.json
index fad1ae92..d6d68994 100644
--- a/test_jobs/losoto_norm.json
+++ b/test_jobs/losoto_norm.json
@@ -1,7 +1,8 @@
{
"input_h5parm": {
"class": "File",
- "path": "/data/example.h5"
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
},
"soltab": "sol000/amplitude000",
"axesToNorm": ["time"]
diff --git a/test_jobs/losoto_plot.json b/test_jobs/losoto_plot.json
index f4ed8634..1316e959 100644
--- a/test_jobs/losoto_plot.json
+++ b/test_jobs/losoto_plot.json
@@ -1,5 +1,5 @@
{
- "input_h5parm": {"class": "File", "path": "/data/example.h5"},
+ "input_h5parm": {"class": "File", "path": "/data/example.h5", "format": "lofar:#H5Parm"},
"soltab": ["sol000/phase000"],
"axesInPlot": ["time"],
"ncpu": 1
diff --git a/test_jobs/losoto_replicateonaxis.json b/test_jobs/losoto_replicateonaxis.json
index c450feeb..0abfc35d 100644
--- a/test_jobs/losoto_replicateonaxis.json
+++ b/test_jobs/losoto_replicateonaxis.json
@@ -1,7 +1,8 @@
{
"input_h5parm": {
"class": "File",
- "path": "/data/example.h5"
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
},
"soltab": "sol000/amplitude000",
"axisReplicate": "time",
diff --git a/test_jobs/losoto_reset.json b/test_jobs/losoto_reset.json
index ff1cfa0f..2d773afb 100644
--- a/test_jobs/losoto_reset.json
+++ b/test_jobs/losoto_reset.json
@@ -1,7 +1,8 @@
{
"input_h5parm": {
"class": "File",
- "path": "/data/example.h5"
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
},
"soltab": "sol000/amplitude000"
}
diff --git a/test_jobs/losoto_residual.json b/test_jobs/losoto_residual.json
index 86b97d57..4a56f1c6 100644
--- a/test_jobs/losoto_residual.json
+++ b/test_jobs/losoto_residual.json
@@ -1,7 +1,8 @@
{
"input_h5parm": {
"class": "File",
- "path": "/data/example.h5"
+ "path": "/data/example.h5",
+ "format": "lofar:#H5Parm"
},
"soltab": "sol000/amplitude000",
"soltabsToSub": ["amplitude000"]
--
GitLab