Add unfixable issues and flagging only if raw

compress_pipeline.cwl

-class: Workflow
+#!/usr/bin/env cwl-runner
+
 cwlVersion: v1.2
+class: Workflow
 label: compress_pipeline
 
+requirements:
+- class: StepInputExpressionRequirement
+- class: InlineJavascriptRequirement
+- class: MultipleInputFeatureRequirement
+
 inputs:
 - id: flag_autocorrelation
   type: boolean?
@@ -9,80 +16,89 @@ inputs:
   type: boolean?
 - id: msin
   type: Directory
+
 outputs:
 - id: inspect
+  type: File
   outputSource:
   - inspect_step/inspect_file
-    type: 'File'
 - id: logfile
+  type: File[]
   outputSource:
   - dppp/logfile
-    type: File[]
 - id: compressed
+  type: File
   outputSource:
   - compress/compressed
-    type: File
 - id: ingest
+  type: Any
   outputSource:
   - format_ingest/ingest
-    type: Any
 - id: applied_fixes
   type: string[]
   outputSource:
   - identify_issues/issue_list
+- id: unfixable_issues
+  type: string[]
+  outputSource:
+  - identify_issues/unfixable_issues
+- id: is_raw
+  type: boolean
 
 steps:
 - id: identify_issues
-    run: steps/identify_issues.cwl
   in:
   - id: msin
     source: msin
   - id: is_long_baseline
     source: is_long_baseline
+  run: steps/identify_issues.cwl
   out:
   - issue_list
+  - is_raw
+  - unfixable_issues
   - msout
 - id: fix_ai_2013
-    run: steps/fix_antenna_information_2013.cwl
   in:
   - id: msin
     source: identify_issues/msout
   - id: apply
-        source: identify_issues/issue_list
     valueFrom: $(self.includes("FIX_ANTENNA_TABLE"))
+    source: identify_issues/issue_list
+  run: steps/fix_antenna_information_2013.cwl
   out:
   - id: msout
 - id: fix_weight_issue
-    run: steps/fix_weightissue_flagging.cwl
   in:
   - id: msin
     source:
     - fix_ai_2013/msout
   - id: apply
-        source: identify_issues/issue_list
     valueFrom: $(self.includes("FIX_WEIGHT_SPECTRUM"))
+    source: identify_issues/issue_list
+  run: steps/fix_weightissue_flagging.cwl
   out:
   - id: msout
 - id: fix_ai_2015
-    run: steps/fix_antenna_information_2015.cwl
   in:
   - id: msin
     source:
     - fix_weight_issue/msout
   - id: apply
-        source: identify_issues/issue_list
     valueFrom: $(self.includes("FIX_BROKEN_TILES"))
+    source: identify_issues/issue_list
+  run: steps/fix_antenna_information_2015.cwl
   out:
   - id: msout
 - id: fix_baselines
-    run: steps/fix_long_baselines.cwl
   in:
   - id: msin
     source:
     - fix_ai_2015/msout
   - id: apply
-        source: identify_issues/issue_list
     valueFrom: $(self.includes("FIX_STATION_ADDER"))
+    source: identify_issues/issue_list
+  run: steps/fix_long_baselines.cwl
   out:
   - id: msout
 - id: extract_sip_meta
@@ -91,9 +107,9 @@ steps:
     source: dppp/msout
   - id: compressed_file
     source: compress/compressed
+  run: steps/extract_sip_meta.cwl
   out:
   - id: ingest
-    run: steps/extract_sip_meta.cwl
 - id: dppp
   in:
   - id: parset
@@ -102,18 +118,18 @@ steps:
     source:
     - fix_baselines/msout
   - id: msout_name
+    valueFrom: $("COMPRESSED_" + self.basename)
     source: fix_baselines/msout
-        valueFrom: '$("COMPRESSED_" + self.basename)'
   - id: writefullresflag
     default: true
   - id: storagemanager
     default: Dysco
   - id: databitrate
     default: 10
+  run: steps/DPPP.cwl
   out:
   - id: msout
   - id: logfile
-    run: steps/DPPP.cwl
 - id: format_ingest
   in:
   - id: metadata
@@ -121,35 +137,33 @@ steps:
   - id: output_name
     default: compressed
   - id: file_name
-        source: compress/compressed
     valueFrom: $(self.basename)
+    source: compress/compressed
+  run: steps/format_ingest.cwl
   out:
   - id: ingest
-    run: steps/format_ingest.cwl
 - id: define_parset
+  label: define_parset
   in:
   - id: flag_autocorrelation
     source: flag_autocorrelation
+  - id: is_raw
+    source: identify_issues/is_raw
+  run: steps/define_parset.cwl
   out:
   - id: output
-    run: steps/define_parset.cwl
-    label: define_parset
 - id: compress
+  label: compress
   in:
   - id: directory
     source: dppp/msout
+  run: steps/compress.cwl
   out:
   - id: compressed
-    run: steps/compress.cwl
-    label: compress
 - id: inspect_step
   in:
   - id: msin
     source: dppp/msout
+  run: steps/create_inspect_dataset.cwl
   out:
   - inspect_file
-    run: steps/create_inspect_dataset.cwl
-requirements:
-  - class: StepInputExpressionRequirement
-  - class: InlineJavascriptRequirement
-  - class: MultipleInputFeatureRequirement

download_and_compress_pipeline.cwl

-class: Workflow
+#!/usr/bin/env cwl-runner
+
 cwlVersion: v1.2
+class: Workflow
 label: download_and_compress_pipeline
+
+requirements:
+- class: ScatterFeatureRequirement
+- class: SubworkflowFeatureRequirement
+- class: StepInputExpressionRequirement
+- class: InlineJavascriptRequirement
+
 inputs:
 - id: flag_autocorrelation
   type: boolean?
 - id: surls
-    type: 'string[]'
+  type: string[]
 - id: is_long_baseline
   type: boolean
+
 outputs:
 - id: compressed
+  type: File[]
   outputSource:
   - compress/compressed
-    type: File[]
 - id: logfile
-    outputSource:
-      - compress/logfile
   type:
     type: array
     items:
       type: array
       items: File
+  outputSource:
+  - compress/logfile
 - id: ingest
+  type: Any[]
   outputSource:
   - compress/ingest
-    type: Any[]
 - id: quality
   type: Any
   outputSource:
@@ -33,17 +43,18 @@ outputs:
   type: File
   outputSource:
   - combine/inspect_file
+
 steps:
 - id: fetch_data
+  label: fetch_data
   in:
   - id: surl_link
     source: surls
-    out:
-      - id: uncompressed
-    run: steps/fetch_data.cwl
-    label: fetch_data
   scatter:
   - surl_link
+  run: steps/fetch_data.cwl
+  out:
+  - id: uncompressed
 - id: compress
   in:
   - id: msin
@@ -54,28 +65,29 @@ steps:
     source: is_long_baseline
   scatter:
   - msin
+  run: ./compress_pipeline.cwl
   out:
   - id: compressed
   - id: inspect
   - id: logfile
   - id: ingest
   - id: applied_fixes
-    run: ./compress_pipeline.cwl
+  - id: unfixable_issues
 - id: combine
   in:
   - id: inputs
     source: compress/inspect
+  run: steps/combine_inspect_dataset.cwl
   out:
   - inspect_file
-    run: steps/combine_inspect_dataset.cwl
 - id: extract_metrics
   in:
   - id: inspect
     source: combine/inspect_file
+  run: steps/extract_quality_metrics.cwl
   out:
   - id: plots
   - id: quality
-    run: steps/extract_quality_metrics.cwl
 - id: format_quality_metrics
   in:
   - id: plots
@@ -84,11 +96,8 @@ steps:
     source: extract_metrics/quality
   - id: applied_fixes
     source: compress/applied_fixes
+  - id: unfixable_issues
+    source: compress/unfixable_issues
   run: steps/format_metrics.cwl
   out:
   - formatted_quality
-requirements:
-  - class: ScatterFeatureRequirement
-  - class: SubworkflowFeatureRequirement
-  - class: StepInputExpressionRequirement
-  - class: InlineJavascriptRequirement

steps/define_parset.cwl

-class: CommandLineTool
+#!/usr/bin/env cwl-runner
+
 cwlVersion: v1.1
-id: define_parset
-baseCommand:
-  - cp
-inputs:
-  - id: flag_autocorrelation
-    type: boolean?
-    default: true
-outputs:
-  - id: output
-    type: File
-    outputBinding:
-      glob: preproc.parset
+class: CommandLineTool
 label: define_parset
-arguments:
-  - prefix: ''
-    shellQuote: false
-    position: 0
-    valueFrom: input.parset
-  - prefix: ''
-    shellQuote: false
-    position: 0
-    valueFrom: preproc.parset
+
 requirements:
 - class: ShellCommandRequirement
 - class: InitialWorkDirRequirement
   listing:
   - entryname: input.parset
-        entry: |+
-          steps=[flagedge,flagelev,aoflag,flagamp $(inputs.demix? ',demix': '')]
+    entry: |
+      steps=[flagedge,flagelev,$('is_raw'? ',aoflag,flagamp': '')$(inputs.demix? ',demix': '')]
       #
       flagedge.chan=[0..nchan/32-1,31*nchan/32..nchan-1]
       flagedge.type=preflagger
@@ -45,3 +27,30 @@ requirements:
       flagamp.amplmin=1e-30
       flagamp.type=preflagger
 - class: InlineJavascriptRequirement
+
+inputs:
+- id: flag_autocorrelation
+  type: boolean?
+  default: true
+- id: is_raw
+  type: boolean
+  default: true
+
+outputs:
+- id: output
+  type: File
+  outputBinding:
+    glob: preproc.parset
+
+baseCommand:
+- cp
+arguments:
+- prefix: ''
+  position: 0
+  valueFrom: input.parset
+  shellQuote: false
+- prefix: ''
+  position: 0
+  valueFrom: preproc.parset
+  shellQuote: false
+id: define_parset

steps/format_metrics.cwl

@@ -17,6 +17,12 @@ inputs:
     items:
       type: array
       items: string
+- id: unfixable_issues
+  type:
+    type: array
+    items:
+      type: array
+      items: string
 
 outputs:
 - id: formatted_quality
@@ -29,6 +35,7 @@ expression: |
               'summary': inputs.quality
     };
     result['summary']['applied_fixes'] = inputs.applied_fixes[0]
+    result['summary']['unfixable_issues'] = inputs.unfixable_issues[0]
     return {'formatted_quality': result}
       
   } 

steps/identify_issues.cwl

-id: identify_issues
-label: Identify Known issues
-class: CommandLineTool
+#!/usr/bin/env cwl-runner
+
 cwlVersion: v1.2
-hints:
-  - class: DockerRequirement
-    dockerPull: git.astron.nl:5000/ldv/ldv-images/lofar-legacy:latest
-  - class: NetworkAccess
-    networkAccess: true
+class: CommandLineTool
+label: Identify Known issues
+
+requirements:
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+  listing:
+  - entryname: $(inputs.msin.basename)
+    writable: true
+    entry: $(inputs.msin)
+
 inputs:
 - id: msin
   type: Directory
@@ -15,16 +20,28 @@ inputs:
 - id: is_long_baseline
   type: boolean?
   inputBinding:
+    prefix: --is_long_baseline
     position: 2
-      prefix: '--is_long_baseline'
+
 outputs:
 - id: issue_list
   type: string[]
   outputBinding:
     glob: output.txt
+    outputEval: $(JSON.parse(self[0].contents)['fixable'])
+    loadContents: true
+- id: unfixable_issues
+  type: string[]
+  outputBinding:
+    glob: output.txt
+    outputEval: $(JSON.parse(self[0].contents)['unfixable_issues'])
+    loadContents: true
+- id: is_raw
+  type: boolean
+  outputBinding:
+    glob: output.txt
+    outputEval: $(JSON.parse(self[0].contents)['is_raw'])
     loadContents: true
-      outputEval: $(JSON.parse(self[0].contents))
-
 - id: msout
   type: Directory
   outputBinding:
@@ -35,10 +52,9 @@ baseCommand:
 - python3
 - /usr/local/bin/fix_common_ms_issues
 
-requirements:
-  - class: InlineJavascriptRequirement
-  - class: InitialWorkDirRequirement
-    listing:
-      - entry: $(inputs.msin)
-        writable: true
-        entryname: $(inputs.msin.basename)
+hints:
+- class: DockerRequirement
+  dockerPull: git.astron.nl:5000/ldv/ldv-images/lofar-legacy:latest
+- class: NetworkAccess
+  networkAccess: true
+id: identify_issues