diff --git a/compress_pipeline.cwl b/compress_pipeline.cwl
index 8d2026787aab53c62fde2f7a85ad8c8ecf63bc68..c17ebdf446411085442c64d8bce10ae5dafab0ef 100644
--- a/compress_pipeline.cwl
+++ b/compress_pipeline.cwl
@@ -5,73 +5,45 @@ label: compress_pipeline.cwl
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
- - id: surl_list
- type: File
- doc: input surl file
- 'sbg:x': -334.23162841796875
- 'sbg:y': -27
- - id: flagging_strategy
- type:
- type: enum
- symbols:
- - HBAdefault
- - LBAdefault
- name: flagging_strategy
- 'sbg:x': -30
- 'sbg:y': -321.8600769042969
- id: flag_autocorrelation
type: boolean?
- 'sbg:x': -242.8984375
- 'sbg:y': -154.86007690429688
+ 'sbg:x': -41
+ 'sbg:y': -212
+ - id: surl
+ type: string
+ 'sbg:x': -229
+ 'sbg:y': -29
outputs:
- id: msout
outputSource:
- dppp/msout
type: Directory
doc: Output Measurement Set
- 'sbg:x': 705.827392578125
- 'sbg:y': 48.75
+ 'sbg:x': 756.0812377929688
+ 'sbg:y': -155.634521484375
- id: logfile
outputSource:
- dppp/logfile
type: 'File[]'
- 'sbg:x': 606.827392578125
- 'sbg:y': 207.75
+ 'sbg:x': 893.7462158203125
+ 'sbg:y': 91.7055892944336
+ - id: flagging_metrics
+ outputSource:
+ - inspect_flagging_dataloss/output
+ type: File
+ 'sbg:x': 581.6015625
+ 'sbg:y': 217
steps:
- - id: read
- in:
- - id: surl_list
- source: surl_list
- out:
- - id: surls
- run: steps/read_surl_list.cwl
- label: read_surl
- 'sbg:x': -184
- 'sbg:y': -19.5
- - id: stage
- in:
- - id: input
- source: read/surls
- out:
- - id: output
- run: steps/stage.cwl
- label: stage
- scatter:
- - input
- 'sbg:x': -46
- 'sbg:y': -2
- id: surl_copy
in:
- id: surl
- source: stage/output
+ source: surl
out:
- id: output
run: steps/surl_copy.cwl
label: surl_copy
- scatter:
- - surl
- 'sbg:x': 117.234375
- 'sbg:y': 53.5
+ 'sbg:x': 14
+ 'sbg:y': -28
- id: untar
in:
- id: tar_file
@@ -80,10 +52,8 @@ steps:
- id: uncompressed
run: steps/untar.cwl
label: untar
- scatter:
- - tar_file
- 'sbg:x': 263.296875
- 'sbg:y': 53.5
+ 'sbg:x': 173
+ 'sbg:y': -28
- id: dppp
in:
- id: parset
@@ -92,7 +62,7 @@ steps:
source: untar/uncompressed
- id: msout_name
source: untar/uncompressed
- valueFrom: $("out_" + self.basename)
+ valueFrom: $("COMPRESSED_" + self.basename)
- id: writefullresflag
default: true
- id: storagemanager
@@ -101,29 +71,32 @@ steps:
default: 10
out:
- id: msout
- - id: secondary_output_files
- - id: secondary_output_directories
- id: logfile
run: steps/DPPP.cwl
- scatter:
- - msin
- - msout_name
- scatterMethod: flat_crossproduct
- 'sbg:x': 469.03125
- 'sbg:y': 32.5
+ 'sbg:x': 471
+ 'sbg:y': -75
- id: define_parset
in:
- - id: flagging_strategy
- source: flagging_strategy
- id: flag_autocorrelation
source: flag_autocorrelation
out:
- id: output
run: steps/define_parset.cwl
label: define_parset
- 'sbg:x': 154
- 'sbg:y': -136
-requirements:
- - class: ScatterFeatureRequirement
- - class: InlineJavascriptRequirement
+ 'sbg:x': 196
+ 'sbg:y': -227
+ - id: inspect_flagging_dataloss
+ in:
+ - id: input
+ source: untar/uncompressed
+ out:
+ - id: output
+ run: steps/inspect_flagging_dataloss.cwl
+ label: inspect_flagging_dataloss
+ 'sbg:x': 362
+ 'sbg:y': 91
+requirements:
- class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
+
+
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
index 8cd5a42291642598cd7fc2884a491b254246c476..86dbfce9a89dab195c996920360734dfc7009f61 100755
--- a/steps/DPPP.cwl
+++ b/steps/DPPP.cwl
@@ -25,18 +25,6 @@ inputs:
prefix: msout=
separate: false
doc: Output Measurement Set
- - id: secondary_files
- type: 'File[]?'
- doc: Secondary files needed for the step
- - id: secondary_directories
- type: 'Directory[]?'
- doc: Secondary directories needed for the step
- - id: output_file_names
- type: Any
- doc: Expected output file names
- - id: output_directory_names
- type: Any
- doc: Expected output file names
- default: false
id: autoweight
type: boolean
@@ -92,40 +80,6 @@ outputs:
type: Directory
outputBinding:
glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
- - id: secondary_output_files
- doc: Secondary output files
- type: Any
- outputBinding:
- outputEval: |-
- ${
- var output={}
- for(var step_name in inputs.output_file_names){
- var file_per_idx = inputs.output_file_names[step_name];
- for(var file_idx in file_per_idx){
- var file_name = file_per_idx[file_idx];
-
- output[file_idx] = {'class':'File', 'path': file_name};
- }
- }
- return output
- }
- - id: secondary_output_directories
- doc: Secondary output directories
- type: Any
- outputBinding:
- outputEval: |-
- ${
- var output={}
- for(var step_name in inputs.output_directory_names){
- var file_per_idx = inputs.output_directory_names[step_name];
- for(var file_idx in file_per_idx){
- var file_name = file_per_idx[file_idx];
-
- output[file_idx] = {'class':'Directory', 'path': file_name};
- }
- }
- return output
- }
- id: logfile
type: File[]
outputBinding:
@@ -141,6 +95,3 @@ requirements:
listing:
- entry: $(inputs.msin)
writable: true
- - entry: $(inputs.secondary_files)
- - entry: $(inputs.secondary_directories)
-
diff --git a/steps/define_parset.cwl b/steps/define_parset.cwl
index a84eaf64f3db4d366b22de7f35bcbfc3d55afc81..4e550cd5cf6bbd3a986f9c1911eba19d6838d278 100644
--- a/steps/define_parset.cwl
+++ b/steps/define_parset.cwl
@@ -6,18 +6,9 @@ id: define_parset
baseCommand:
- cp
inputs:
- - id: flagging_strategy
- type:
- type: enum
- symbols:
- - HBAdefault
- - LBAdefault
- name: flagging_strategy
- id: flag_autocorrelation
type: boolean?
default: true
- inputBinding:
- valueFrom: '$(self ? ''True'': ''False'')'
outputs:
- id: output
type: File
@@ -39,7 +30,7 @@ requirements:
listing:
- entryname: input.parset
entry: |+
- steps=[flagedge,aoflag,flagamp,compress]
+ steps=[flagedge,aoflag,flagamp]
#
flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
flagedge.type=preflagger
@@ -47,8 +38,8 @@ requirements:
aoflag.keepstatistics=false
aoflag.memoryperc=20
aoflag.type=aoflagger
- aoflag.autocorr=$(flag_autocorrelation)
- aoflag.strategy=$(flagging_strategy)
+ aoflag.autocorr=$(inputs.flag_autocorrelation?'True':'False')
+ aoflag.strategy=/usr/local/share/aoflagger/strategies/lofar-default.lua
#
flagbaseline.type=preflagger
flagbaseline.baseline=[]
@@ -58,5 +49,4 @@ requirements:
#
flagamp.amplmin=1e-30
flagamp.type=preflagger
- writable: false
- class: InlineJavascriptRequirement
diff --git a/steps/inspect_compression_quality.cwl b/steps/inspect_compression_quality.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..699b9e27c00f6ceabad0f368dba75c58925f375d
--- /dev/null
+++ b/steps/inspect_compression_quality.cwl
@@ -0,0 +1,23 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: inspect_compression_quality
+baseCommand: []
+inputs:
+ - id: before_compression
+ type: Directory
+ inputBinding:
+ shellQuote: false
+ position: 0
+ - id: after_compression
+ type: Directory
+ inputBinding:
+ shellQuote: false
+ position: 0
+outputs:
+ - id: output
+ type: 'File[]'
+label: inspect_compression_quality
+requirements:
+ - class: ShellCommandRequirement
diff --git a/steps/inspect_flagging_dataloss.cwl b/steps/inspect_flagging_dataloss.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..fbccfd81087d8f4b90d3bcc809128d6413b112a9
--- /dev/null
+++ b/steps/inspect_flagging_dataloss.cwl
@@ -0,0 +1,36 @@
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+id: inspect_flagging_dataloss
+baseCommand:
+ - python3
+ - script.py
+inputs:
+ - id: input
+ type: Directory?
+ inputBinding:
+ position: 0
+ shellQuote: false
+outputs:
+ - id: output
+ type: 'File'
+ outputBinding:
+ glob: metrics.json
+label: inspect_flagging_dataloss
+requirements:
+ - class: ShellCommandRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: script.py
+ entry: |
+ import sys
+ from casacore.tables import table
+ input_ms_path = sys.argv[1]
+
+
+ with open('metrics.json', 'w') as f_stream:
+ pass
+
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/lofar-pipeline:latest'