Add unfixable issues and flagging only if raw

102a9c12 · Mattia Mancini · 0fb3017b · 102a9c12 · 102a9c12 · 102a9c12
Commit 102a9c12 authored Aug 14, 2023 by Mattia Mancini
--- a/compress_pipeline.cwl
+++ b/compress_pipeline.cwl
-class: Workflow
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.2
+class: Workflow
 label: compress_pipeline
+requirements:
+- class: StepInputExpressionRequirement
+- class: InlineJavascriptRequirement
+- class: MultipleInputFeatureRequirement
 inputs:
 - id: flag_autocorrelation
  type: boolean?
@@ -9,80 +16,89 @@ inputs:
  type: boolean?
 - id: msin
  type: Directory
 outputs:
 - id: inspect
+  type: File
  outputSource:
  - inspect_step/inspect_file
-    type: 'File'
 - id: logfile
+  type: File[]
  outputSource:
  - dppp/logfile
-    type: File[]
 - id: compressed
+  type: File
  outputSource:
  - compress/compressed
-    type: File
 - id: ingest
+  type: Any
  outputSource:
  - format_ingest/ingest
-    type: Any
 - id: applied_fixes
  type: string[]
  outputSource:
  - identify_issues/issue_list
+- id: unfixable_issues
+  type: string[]
+  outputSource:
+  - identify_issues/unfixable_issues
+- id: is_raw
+  type: boolean
 steps:
 - id: identify_issues
-    run: steps/identify_issues.cwl
  in:
  - id: msin
    source: msin
  - id: is_long_baseline
    source: is_long_baseline
+  run: steps/identify_issues.cwl
  out:
  - issue_list
+  - is_raw
+  - unfixable_issues
  - msout
 - id: fix_ai_2013
-    run: steps/fix_antenna_information_2013.cwl
  in:
  - id: msin
    source: identify_issues/msout
  - id: apply
-        source: identify_issues/issue_list
    valueFrom: $(self.includes("FIX_ANTENNA_TABLE"))
+    source: identify_issues/issue_list
+  run: steps/fix_antenna_information_2013.cwl
  out:
  - id: msout
 - id: fix_weight_issue
-    run: steps/fix_weightissue_flagging.cwl
  in:
  - id: msin
    source:
    - fix_ai_2013/msout
  - id: apply
-        source: identify_issues/issue_list
    valueFrom: $(self.includes("FIX_WEIGHT_SPECTRUM"))
+    source: identify_issues/issue_list
+  run: steps/fix_weightissue_flagging.cwl
  out:
  - id: msout
 - id: fix_ai_2015
-    run: steps/fix_antenna_information_2015.cwl
  in:
  - id: msin
    source:
    - fix_weight_issue/msout
  - id: apply
-        source: identify_issues/issue_list
    valueFrom: $(self.includes("FIX_BROKEN_TILES"))
+    source: identify_issues/issue_list
+  run: steps/fix_antenna_information_2015.cwl
  out:
  - id: msout
 - id: fix_baselines
-    run: steps/fix_long_baselines.cwl
  in:
  - id: msin
    source:
    - fix_ai_2015/msout
  - id: apply
-        source: identify_issues/issue_list
    valueFrom: $(self.includes("FIX_STATION_ADDER"))
+    source: identify_issues/issue_list
+  run: steps/fix_long_baselines.cwl
  out:
  - id: msout
 - id: extract_sip_meta
@@ -91,9 +107,9 @@ steps:
    source: dppp/msout
  - id: compressed_file
    source: compress/compressed
+  run: steps/extract_sip_meta.cwl
  out:
  - id: ingest
-    run: steps/extract_sip_meta.cwl
 - id: dppp
  in:
  - id: parset
@@ -102,18 +118,18 @@ steps:
    source:
    - fix_baselines/msout
  - id: msout_name
+    valueFrom: $("COMPRESSED_" + self.basename)
    source: fix_baselines/msout
-        valueFrom: '$("COMPRESSED_" + self.basename)'
  - id: writefullresflag
    default: true
  - id: storagemanager
    default: Dysco
  - id: databitrate
    default: 10
+  run: steps/DPPP.cwl
  out:
  - id: msout
  - id: logfile
-    run: steps/DPPP.cwl
 - id: format_ingest
  in:
  - id: metadata
@@ -121,35 +137,33 @@ steps:
  - id: output_name
    default: compressed
  - id: file_name
-        source: compress/compressed
    valueFrom: $(self.basename)
+    source: compress/compressed
+  run: steps/format_ingest.cwl
  out:
  - id: ingest
-    run: steps/format_ingest.cwl
 - id: define_parset
+  label: define_parset
  in:
  - id: flag_autocorrelation
    source: flag_autocorrelation
+  - id: is_raw
+    source: identify_issues/is_raw
+  run: steps/define_parset.cwl
  out:
  - id: output
-    run: steps/define_parset.cwl
-    label: define_parset
 - id: compress
+  label: compress
  in:
  - id: directory
    source: dppp/msout
+  run: steps/compress.cwl
  out:
  - id: compressed
-    run: steps/compress.cwl
-    label: compress
 - id: inspect_step
  in:
  - id: msin
    source: dppp/msout
+  run: steps/create_inspect_dataset.cwl
  out:
  - inspect_file
-    run: steps/create_inspect_dataset.cwl
-requirements:
-  - class: StepInputExpressionRequirement
-  - class: InlineJavascriptRequirement
-  - class: MultipleInputFeatureRequirement
--- a/download_and_compress_pipeline.cwl
+++ b/download_and_compress_pipeline.cwl
-class: Workflow
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.2
+class: Workflow
 label: download_and_compress_pipeline
+requirements:
+- class: ScatterFeatureRequirement
+- class: SubworkflowFeatureRequirement
+- class: StepInputExpressionRequirement
+- class: InlineJavascriptRequirement
 inputs:
 - id: flag_autocorrelation
  type: boolean?
 - id: surls
-    type: 'string[]'
+  type: string[]
 - id: is_long_baseline
  type: boolean
 outputs:
 - id: compressed
+  type: File[]
  outputSource:
  - compress/compressed
-    type: File[]
 - id: logfile
-    outputSource:
-      - compress/logfile
  type:
    type: array
    items:
      type: array
      items: File
+  outputSource:
+  - compress/logfile
 - id: ingest
+  type: Any[]
  outputSource:
  - compress/ingest
-    type: Any[]
 - id: quality
  type: Any
  outputSource:
@@ -33,17 +43,18 @@ outputs:
  type: File
  outputSource:
  - combine/inspect_file
 steps:
 - id: fetch_data
+  label: fetch_data
  in:
  - id: surl_link
    source: surls
-    out:
-      - id: uncompressed
-    run: steps/fetch_data.cwl
-    label: fetch_data
  scatter:
  - surl_link
+  run: steps/fetch_data.cwl
+  out:
+  - id: uncompressed
 - id: compress
  in:
  - id: msin
@@ -54,28 +65,29 @@ steps:
    source: is_long_baseline
  scatter:
  - msin
+  run: ./compress_pipeline.cwl
  out:
  - id: compressed
  - id: inspect
  - id: logfile
  - id: ingest
  - id: applied_fixes
-    run: ./compress_pipeline.cwl
+  - id: unfixable_issues
 - id: combine
  in:
  - id: inputs
    source: compress/inspect
+  run: steps/combine_inspect_dataset.cwl
  out:
  - inspect_file
-    run: steps/combine_inspect_dataset.cwl
 - id: extract_metrics
  in:
  - id: inspect
    source: combine/inspect_file
+  run: steps/extract_quality_metrics.cwl
  out:
  - id: plots
  - id: quality
-    run: steps/extract_quality_metrics.cwl
 - id: format_quality_metrics
  in:
  - id: plots
@@ -84,11 +96,8 @@ steps:
    source: extract_metrics/quality
  - id: applied_fixes
    source: compress/applied_fixes
+  - id: unfixable_issues
+    source: compress/unfixable_issues
  run: steps/format_metrics.cwl
  out:
  - formatted_quality
-requirements:
-  - class: ScatterFeatureRequirement
-  - class: SubworkflowFeatureRequirement
-  - class: StepInputExpressionRequirement
-  - class: InlineJavascriptRequirement
--- a/steps/define_parset.cwl
+++ b/steps/define_parset.cwl
-class: CommandLineTool
+#!/usr/bin/env cwl-runner
 cwlVersion: v1.1
-id: define_parset
+class: CommandLineTool
-baseCommand:
-  - cp
-inputs:
-  - id: flag_autocorrelation
-    type: boolean?
-    default: true
-outputs:
-  - id: output
-    type: File
-    outputBinding:
-      glob: preproc.parset
 label: define_parset
-arguments:
-  - prefix: ''
-    shellQuote: false
-    position: 0
-    valueFrom: input.parset
-  - prefix: ''
-    shellQuote: false
-    position: 0
-    valueFrom: preproc.parset
 requirements:
 - class: ShellCommandRequirement
 - class: InitialWorkDirRequirement
  listing:
  - entryname: input.parset
-        entry: |+
+    entry: |
-          steps=[flagedge,flagelev,aoflag,flagamp $(inputs.demix? ',demix': '')]
+      steps=[flagedge,flagelev,$('is_raw'? ',aoflag,flagamp': '')$(inputs.demix? ',demix': '')]
      #
      flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
      flagedge.type=preflagger
@@ -45,3 +27,30 @@ requirements:
      flagamp.amplmin=1e-30
      flagamp.type=preflagger
 - class: InlineJavascriptRequirement
+inputs:
+- id: flag_autocorrelation
+  type: boolean?
+  default: true
+- id: is_raw
+  type: boolean
+  default: true
+outputs:
+- id: output
+  type: File
+  outputBinding:
+    glob: preproc.parset
+baseCommand:
+- cp
+arguments:
+- prefix: ''
+  position: 0
+  valueFrom: input.parset
+  shellQuote: false
+- prefix: ''
+  position: 0
+  valueFrom: preproc.parset
+  shellQuote: false
+id: define_parset
--- a/steps/format_metrics.cwl
+++ b/steps/format_metrics.cwl
@@ -17,6 +17,12 @@ inputs:
    items:
      type: array
      items: string
+- id: unfixable_issues
+  type:
+    type: array
+    items:
+      type: array
+      items: string
 outputs:
 - id: formatted_quality
@@ -29,6 +35,7 @@ expression: |
              'summary': inputs.quality
    };
    result['summary']['applied_fixes'] = inputs.applied_fixes[0]
+    result['summary']['unfixable_issues'] = inputs.unfixable_issues[0]
    return {'formatted_quality': result}
  } 
--- a/steps/identify_issues.cwl
+++ b/steps/identify_issues.cwl
-id: identify_issues
+#!/usr/bin/env cwl-runner
-label: Identify Known issues
-class: CommandLineTool
 cwlVersion: v1.2
-hints:
+class: CommandLineTool
-  - class: DockerRequirement
+label: Identify Known issues
-    dockerPull: git.astron.nl:5000/ldv/ldv-images/lofar-legacy:latest
-  - class: NetworkAccess
+requirements:
-    networkAccess: true
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+  listing:
+  - entryname: $(inputs.msin.basename)
+    writable: true
+    entry: $(inputs.msin)
 inputs:
 - id: msin
  type: Directory
@@ -15,16 +20,28 @@ inputs:
 - id: is_long_baseline
  type: boolean?
  inputBinding:
+    prefix: --is_long_baseline
    position: 2
-      prefix: '--is_long_baseline'
 outputs:
 - id: issue_list
  type: string[]
  outputBinding:
    glob: output.txt
+    outputEval: $(JSON.parse(self[0].contents)['fixable'])
+    loadContents: true
+- id: unfixable_issues
+  type: string[]
+  outputBinding:
+    glob: output.txt
+    outputEval: $(JSON.parse(self[0].contents)['unfixable_issues'])
+    loadContents: true
+- id: is_raw
+  type: boolean
+  outputBinding:
+    glob: output.txt
+    outputEval: $(JSON.parse(self[0].contents)['is_raw'])
    loadContents: true
-      outputEval: $(JSON.parse(self[0].contents))
 - id: msout
  type: Directory
  outputBinding:
@@ -35,10 +52,9 @@ baseCommand:
 - python3
 - /usr/local/bin/fix_common_ms_issues
-requirements:
+hints:
-  - class: InlineJavascriptRequirement
+- class: DockerRequirement
-  - class: InitialWorkDirRequirement
+  dockerPull: git.astron.nl:5000/ldv/ldv-images/lofar-legacy:latest
-    listing:
+- class: NetworkAccess
-      - entry: $(inputs.msin)
+  networkAccess: true
-        writable: true
+id: identify_issues
-        entryname: $(inputs.msin.basename)