diff --git a/compress_pipeline.cwl b/compress_pipeline.cwl
index cd5d8c0c86beb6844bc72e53a391554b4423d245..8d9de30a7779bf831b14890c1f23bdd5204f08a9 100644
--- a/compress_pipeline.cwl
+++ b/compress_pipeline.cwl
@@ -1,155 +1,169 @@
-class: Workflow
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.2
+class: Workflow
label: compress_pipeline
+requirements:
+- class: StepInputExpressionRequirement
+- class: InlineJavascriptRequirement
+- class: MultipleInputFeatureRequirement
+
inputs:
- - id: flag_autocorrelation
- type: boolean?
+- id: flag_autocorrelation
+ type: boolean?
+- id: is_long_baseline
+ type: boolean?
+- id: msin
+ type: Directory
+
+outputs:
+- id: inspect
+ type: File
+ outputSource:
+ - inspect_step/inspect_file
+- id: logfile
+ type: File[]
+ outputSource:
+ - dppp/logfile
+- id: compressed
+ type: File
+ outputSource:
+ - compress/compressed
+- id: ingest
+ type: Any
+ outputSource:
+ - format_ingest/ingest
+- id: applied_fixes
+ type: string[]
+ outputSource:
+ - identify_issues/issue_list
+- id: unfixable_issues
+ type: string[]
+ outputSource:
+ - identify_issues/unfixable_issues
+- id: is_raw
+ type: boolean
+
+steps:
+- id: identify_issues
+ in:
+ - id: msin
+ source: msin
- id: is_long_baseline
- type: boolean?
+ source: is_long_baseline
+ run: steps/identify_issues.cwl
+ out:
+ - issue_list
+ - is_raw
+ - unfixable_issues
+ - msout
+- id: fix_ai_2013
+ in:
- id: msin
- type: Directory
-outputs:
- - id: inspect
- outputSource:
- - inspect_step/inspect_file
- type: 'File'
+ source: identify_issues/msout
+ - id: apply
+ valueFrom: $(self.includes("FIX_ANTENNA_TABLE"))
+ source: identify_issues/issue_list
+ run: steps/fix_antenna_information_2013.cwl
+ out:
+ - id: msout
+- id: fix_weight_issue
+ in:
+ - id: msin
+ source:
+ - fix_ai_2013/msout
+ - id: apply
+ valueFrom: $(self.includes("FIX_WEIGHT_SPECTRUM"))
+ source: identify_issues/issue_list
+ run: steps/fix_weightissue_flagging.cwl
+ out:
+ - id: msout
+- id: fix_ai_2015
+ in:
+ - id: msin
+ source:
+ - fix_weight_issue/msout
+ - id: apply
+ valueFrom: $(self.includes("FIX_BROKEN_TILES"))
+ source: identify_issues/issue_list
+ run: steps/fix_antenna_information_2015.cwl
+ out:
+ - id: msout
+- id: fix_baselines
+ in:
+ - id: msin
+ source:
+ - fix_ai_2015/msout
+ - id: apply
+ valueFrom: $(self.includes("FIX_STATION_ADDER"))
+ source: identify_issues/issue_list
+ run: steps/fix_long_baselines.cwl
+ out:
+ - id: msout
+- id: extract_sip_meta
+ in:
+ - id: msin
+ source: dppp/msout
+ - id: compressed_file
+ source: compress/compressed
+ run: steps/extract_sip_meta.cwl
+ out:
+ - id: ingest
+- id: dppp
+ in:
+ - id: parset
+ source: define_parset/output
+ - id: msin
+ source:
+ - fix_baselines/msout
+ - id: msout_name
+ valueFrom: $("COMPRESSED_" + self.basename)
+ source: fix_baselines/msout
+ - id: writefullresflag
+ default: true
+ - id: storagemanager
+ default: Dysco
+ - id: databitrate
+ default: 10
+ run: steps/DPPP.cwl
+ out:
+ - id: msout
- id: logfile
- outputSource:
- - dppp/logfile
- type: File[]
- - id: compressed
- outputSource:
- - compress/compressed
- type: File
+- id: format_ingest
+ in:
+ - id: metadata
+ source: extract_sip_meta/ingest
+ - id: output_name
+ default: compressed
+ - id: file_name
+ valueFrom: $(self.basename)
+ source: compress/compressed
+ run: steps/format_ingest.cwl
+ out:
- id: ingest
- outputSource:
- - format_ingest/ingest
- type: Any
- - id: applied_fixes
- type: string[]
- outputSource:
- - identify_issues/issue_list
-
-steps:
- - id: identify_issues
- run: steps/identify_issues.cwl
- in:
- - id: msin
- source: msin
- - id: is_long_baseline
- source: is_long_baseline
- out:
- - issue_list
- - msout
- - id: fix_ai_2013
- run: steps/fix_antenna_information_2013.cwl
- in:
- - id: msin
- source: identify_issues/msout
- - id: apply
- source: identify_issues/issue_list
- valueFrom: $(self.includes("FIX_ANTENNA_TABLE"))
- out:
- - id: msout
- - id: fix_weight_issue
- run: steps/fix_weightissue_flagging.cwl
- in:
- - id: msin
- source:
- - fix_ai_2013/msout
- - id: apply
- source: identify_issues/issue_list
- valueFrom: $(self.includes("FIX_WEIGHT_SPECTRUM"))
- out:
- - id: msout
- - id: fix_ai_2015
- run: steps/fix_antenna_information_2015.cwl
- in:
- - id: msin
- source:
- - fix_weight_issue/msout
- - id: apply
- source: identify_issues/issue_list
- valueFrom: $(self.includes("FIX_BROKEN_TILES"))
- out:
- - id: msout
- - id: fix_baselines
- run: steps/fix_long_baselines.cwl
- in:
- - id: msin
- source:
- - fix_ai_2015/msout
- - id: apply
- source: identify_issues/issue_list
- valueFrom: $(self.includes("FIX_STATION_ADDER"))
- out:
- - id: msout
- - id: extract_sip_meta
- in:
- - id: msin
- source: dppp/msout
- - id: compressed_file
- source: compress/compressed
- out:
- - id: ingest
- run: steps/extract_sip_meta.cwl
- - id: dppp
- in:
- - id: parset
- source: define_parset/output
- - id: msin
- source:
- - fix_baselines/msout
- - id: msout_name
- source: fix_baselines/msout
- valueFrom: '$("COMPRESSED_" + self.basename)'
- - id: writefullresflag
- default: true
- - id: storagemanager
- default: Dysco
- - id: databitrate
- default: 10
- out:
- - id: msout
- - id: logfile
- run: steps/DPPP.cwl
- - id: format_ingest
- in:
- - id: metadata
- source: extract_sip_meta/ingest
- - id: output_name
- default: compressed
- - id: file_name
- source: compress/compressed
- valueFrom: $(self.basename)
- out:
- - id: ingest
- run: steps/format_ingest.cwl
- - id: define_parset
- in:
- - id: flag_autocorrelation
- source: flag_autocorrelation
- out:
- - id: output
- run: steps/define_parset.cwl
- label: define_parset
- - id: compress
- in:
- - id: directory
- source: dppp/msout
- out:
- - id: compressed
- run: steps/compress.cwl
- label: compress
- - id: inspect_step
- in:
- - id: msin
- source: dppp/msout
- out:
- - inspect_file
- run: steps/create_inspect_dataset.cwl
-requirements:
- - class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
- - class: MultipleInputFeatureRequirement
+- id: define_parset
+ label: define_parset
+ in:
+ - id: flag_autocorrelation
+ source: flag_autocorrelation
+ - id: is_raw
+ source: identify_issues/is_raw
+ run: steps/define_parset.cwl
+ out:
+ - id: output
+- id: compress
+ label: compress
+ in:
+ - id: directory
+ source: dppp/msout
+ run: steps/compress.cwl
+ out:
+ - id: compressed
+- id: inspect_step
+ in:
+ - id: msin
+ source: dppp/msout
+ run: steps/create_inspect_dataset.cwl
+ out:
+ - inspect_file
diff --git a/download_and_compress_pipeline.cwl b/download_and_compress_pipeline.cwl
index fb549c5ddf3cdee7bc9ee3f824fc6c1c01bd40bb..841bc0b56f7a83b3cf2988937c3728c4b7f98c65 100644
--- a/download_and_compress_pipeline.cwl
+++ b/download_and_compress_pipeline.cwl
@@ -1,94 +1,103 @@
-class: Workflow
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.2
+class: Workflow
label: download_and_compress_pipeline
+
+requirements:
+- class: ScatterFeatureRequirement
+- class: SubworkflowFeatureRequirement
+- class: StepInputExpressionRequirement
+- class: InlineJavascriptRequirement
+
inputs:
+- id: flag_autocorrelation
+ type: boolean?
+- id: surls
+ type: string[]
+- id: is_long_baseline
+ type: boolean
+
+outputs:
+- id: compressed
+ type: File[]
+ outputSource:
+ - compress/compressed
+- id: logfile
+ type:
+ type: array
+ items:
+ type: array
+ items: File
+ outputSource:
+ - compress/logfile
+- id: ingest
+ type: Any[]
+ outputSource:
+ - compress/ingest
+- id: quality
+ type: Any
+ outputSource:
+ - format_quality_metrics/formatted_quality
+- id: inspect
+ type: File
+ outputSource:
+ - combine/inspect_file
+
+steps:
+- id: fetch_data
+ label: fetch_data
+ in:
+ - id: surl_link
+ source: surls
+ scatter:
+ - surl_link
+ run: steps/fetch_data.cwl
+ out:
+ - id: uncompressed
+- id: compress
+ in:
+ - id: msin
+ source: fetch_data/uncompressed
- id: flag_autocorrelation
- type: boolean?
- - id: surls
- type: 'string[]'
+ source: flag_autocorrelation
- id: is_long_baseline
- type: boolean
-outputs:
+ source: is_long_baseline
+ scatter:
+ - msin
+ run: ./compress_pipeline.cwl
+ out:
- id: compressed
- outputSource:
- - compress/compressed
- type: File[]
+ - id: inspect
- id: logfile
- outputSource:
- - compress/logfile
- type:
- type: array
- items:
- type: array
- items: File
- id: ingest
- outputSource:
- - compress/ingest
- type: Any[]
- - id: quality
- type: Any
- outputSource:
- - format_quality_metrics/formatted_quality
+ - id: applied_fixes
+ - id: unfixable_issues
+- id: combine
+ in:
+ - id: inputs
+ source: compress/inspect
+ run: steps/combine_inspect_dataset.cwl
+ out:
+ - inspect_file
+- id: extract_metrics
+ in:
- id: inspect
- type: File
- outputSource:
- - combine/inspect_file
-steps:
- - id: fetch_data
- in:
- - id: surl_link
- source: surls
- out:
- - id: uncompressed
- run: steps/fetch_data.cwl
- label: fetch_data
- scatter:
- - surl_link
- - id: compress
- in:
- - id: msin
- source: fetch_data/uncompressed
- - id: flag_autocorrelation
- source: flag_autocorrelation
- - id: is_long_baseline
- source: is_long_baseline
- scatter:
- - msin
- out:
- - id: compressed
- - id: inspect
- - id: logfile
- - id: ingest
- - id: applied_fixes
- run: ./compress_pipeline.cwl
- - id: combine
- in:
- - id: inputs
- source: compress/inspect
- out:
- - inspect_file
- run: steps/combine_inspect_dataset.cwl
- - id: extract_metrics
- in:
- - id: inspect
- source: combine/inspect_file
- out:
- - id: plots
- - id: quality
- run: steps/extract_quality_metrics.cwl
- - id: format_quality_metrics
- in:
- - id: plots
- source: extract_metrics/plots
- - id: quality
- source: extract_metrics/quality
- - id: applied_fixes
- source: compress/applied_fixes
- run: steps/format_metrics.cwl
- out:
- - formatted_quality
-requirements:
- - class: ScatterFeatureRequirement
- - class: SubworkflowFeatureRequirement
- - class: StepInputExpressionRequirement
- - class: InlineJavascriptRequirement
+ source: combine/inspect_file
+ run: steps/extract_quality_metrics.cwl
+ out:
+ - id: plots
+ - id: quality
+- id: format_quality_metrics
+ in:
+ - id: plots
+ source: extract_metrics/plots
+ - id: quality
+ source: extract_metrics/quality
+ - id: applied_fixes
+ source: compress/applied_fixes
+ - id: unfixable_issues
+ source: compress/unfixable_issues
+ run: steps/format_metrics.cwl
+ out:
+ - formatted_quality
diff --git a/steps/define_parset.cwl b/steps/define_parset.cwl
index 14c7a831fb579b9e08a9517e07198cdf08d20355..0e7e549cdd85b9f251b4005192bbac3ecf23b04f 100644
--- a/steps/define_parset.cwl
+++ b/steps/define_parset.cwl
@@ -1,47 +1,56 @@
-class: CommandLineTool
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.1
-id: define_parset
-baseCommand:
- - cp
+class: CommandLineTool
+label: define_parset
+
+requirements:
+- class: ShellCommandRequirement
+- class: InitialWorkDirRequirement
+ listing:
+ - entryname: input.parset
+ entry: |
+ steps=[flagedge,flagelev,$('is_raw'? ',aoflag,flagamp': '')$(inputs.demix? ',demix': '')]
+ #
+ flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
+ flagedge.type=preflagger
+ #
+ aoflag.keepstatistics=false
+ aoflag.memoryperc=20
+ aoflag.type=aoflagger
+ aoflag.autocorr=$(inputs.flag_autocorrelation?'True':'False')
+ aoflag.strategy=/usr/local/share/aoflagger/strategies/lofar-default.lua
+ #
+ flagelev.type=preflagger
+ flagelev.elevation=0deg..20deg
+ #
+ flagamp.amplmin=1e-30
+ flagamp.type=preflagger
+- class: InlineJavascriptRequirement
+
inputs:
- - id: flag_autocorrelation
- type: boolean?
- default: true
+- id: flag_autocorrelation
+ type: boolean?
+ default: true
+- id: is_raw
+ type: boolean
+ default: true
+
outputs:
- - id: output
- type: File
- outputBinding:
- glob: preproc.parset
-label: define_parset
+- id: output
+ type: File
+ outputBinding:
+ glob: preproc.parset
+
+baseCommand:
+- cp
arguments:
- - prefix: ''
- shellQuote: false
- position: 0
- valueFrom: input.parset
- - prefix: ''
- shellQuote: false
- position: 0
- valueFrom: preproc.parset
-requirements:
- - class: ShellCommandRequirement
- - class: InitialWorkDirRequirement
- listing:
- - entryname: input.parset
- entry: |+
- steps=[flagedge,flagelev,aoflag,flagamp $(inputs.demix? ',demix': '')]
- #
- flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
- flagedge.type=preflagger
- #
- aoflag.keepstatistics=false
- aoflag.memoryperc=20
- aoflag.type=aoflagger
- aoflag.autocorr=$(inputs.flag_autocorrelation?'True':'False')
- aoflag.strategy=/usr/local/share/aoflagger/strategies/lofar-default.lua
- #
- flagelev.type=preflagger
- flagelev.elevation=0deg..20deg
- #
- flagamp.amplmin=1e-30
- flagamp.type=preflagger
- - class: InlineJavascriptRequirement
+- prefix: ''
+ position: 0
+ valueFrom: input.parset
+ shellQuote: false
+- prefix: ''
+ position: 0
+ valueFrom: preproc.parset
+ shellQuote: false
+id: define_parset
diff --git a/steps/format_metrics.cwl b/steps/format_metrics.cwl
index c19d8b2114bae4e4963e992f8d4c7c03c4766dc2..9920403d50bd7442b8704cfc1fff45b1a7a6885e 100644
--- a/steps/format_metrics.cwl
+++ b/steps/format_metrics.cwl
@@ -17,6 +17,12 @@ inputs:
items:
type: array
items: string
+- id: unfixable_issues
+ type:
+ type: array
+ items:
+ type: array
+ items: string
outputs:
- id: formatted_quality
@@ -29,6 +35,7 @@ expression: |
'summary': inputs.quality
};
result['summary']['applied_fixes'] = inputs.applied_fixes[0]
+ result['summary']['unfixable_issues'] = inputs.unfixable_issues[0]
return {'formatted_quality': result}
}
diff --git a/steps/identify_issues.cwl b/steps/identify_issues.cwl
index 1d834213f0fdcff3c578dcb6fbf59af4d5a7783b..04ab97001cc45ae11066d694495e511af26b77b2 100644
--- a/steps/identify_issues.cwl
+++ b/steps/identify_issues.cwl
@@ -1,44 +1,60 @@
-id: identify_issues
-label: Identify Known issues
-class: CommandLineTool
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.2
-hints:
- - class: DockerRequirement
- dockerPull: git.astron.nl:5000/ldv/ldv-images/lofar-legacy:latest
- - class: NetworkAccess
- networkAccess: true
+class: CommandLineTool
+label: Identify Known issues
+
+requirements:
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+ listing:
+ - entryname: $(inputs.msin.basename)
+ writable: true
+ entry: $(inputs.msin)
+
inputs:
- - id: msin
- type: Directory
- inputBinding:
- position: 1
- - id: is_long_baseline
- type: boolean?
- inputBinding:
- position: 2
- prefix: '--is_long_baseline'
-outputs:
- - id: issue_list
- type: string[]
- outputBinding:
- glob: output.txt
- loadContents: true
- outputEval: $(JSON.parse(self[0].contents))
+- id: msin
+ type: Directory
+ inputBinding:
+ position: 1
+- id: is_long_baseline
+ type: boolean?
+ inputBinding:
+ prefix: --is_long_baseline
+ position: 2
- - id: msout
- type: Directory
- outputBinding:
- glob: $(inputs.msin.basename)
+outputs:
+- id: issue_list
+ type: string[]
+ outputBinding:
+ glob: output.txt
+ outputEval: $(JSON.parse(self[0].contents)['fixable'])
+ loadContents: true
+- id: unfixable_issues
+ type: string[]
+ outputBinding:
+ glob: output.txt
+ outputEval: $(JSON.parse(self[0].contents)['unfixable_issues'])
+ loadContents: true
+- id: is_raw
+ type: boolean
+ outputBinding:
+ glob: output.txt
+ outputEval: $(JSON.parse(self[0].contents)['is_raw'])
+ loadContents: true
+- id: msout
+ type: Directory
+ outputBinding:
+ glob: $(inputs.msin.basename)
stdout: output.txt
baseCommand:
- - python3
- - /usr/local/bin/fix_common_ms_issues
+- python3
+- /usr/local/bin/fix_common_ms_issues
-requirements:
- - class: InlineJavascriptRequirement
- - class: InitialWorkDirRequirement
- listing:
- - entry: $(inputs.msin)
- writable: true
- entryname: $(inputs.msin.basename)
+hints:
+- class: DockerRequirement
+ dockerPull: git.astron.nl:5000/ldv/ldv-images/lofar-legacy:latest
+- class: NetworkAccess
+ networkAccess: true
+id: identify_issues