Add quality metrics generation

compress_pipeline.cwl

@@ -10,7 +10,7 @@ inputs:
 outputs:
   - id: inspect
     outputSource:
-      - inspect_flagging_dataloss/output
+      - inspect_step/inspect_file
     type: 'File'
   - id: logfile
     outputSource:
@@ -24,14 +24,6 @@ outputs:
     outputSource:
       - format_ingest/ingest
     type: Any
-  - id: uv_plot
-    type: File
-    outputSource:
-      - plot_uvw_coverage/uv_plot
-  - id: uv_coverage
-    type: File
-    outputSource:
-      - inspect_flagging_dataloss/flags_output
 steps:
   - id: identify_issues
     run: steps/identify_issues.cwl
@@ -133,23 +125,6 @@ steps:
       - id: output
     run: steps/define_parset.cwl
     label: define_parset
-  - id: inspect_flagging_dataloss
-    in:
-      - id: input
-        source:
-          - fix_baselines/msout
-    out:
-      - id: output
-      - id: flags_output
-    run: steps/inspect_flagging_dataloss.cwl
-    label: inspect_flagging_dataloss
-  - id: plot_uvw_coverage
-    in:
-      - id: input_file
-        source: inspect_flagging_dataloss/flags_output
-    out:
-      - id: uv_plot
-    run: steps/plot_uvw_coverage.cwl
   - id: compress
     in:
       - id: directory
@@ -158,6 +133,13 @@ steps:
       - id: compressed
     run: steps/compress.cwl
     label: compress
+  - id: inspect_step
+    in: 
+      - id: msin
+        source: dppp/msout
+    out:
+      - inspect_file
+    run: steps/create_inspect_dataset.cwl
 requirements:
   - class: StepInputExpressionRequirement
   - class: InlineJavascriptRequirement

download_and_compress_pipeline.cwl

@@ -12,10 +12,6 @@ outputs:
     outputSource:
       - compress/compressed
     type: File[]
-  - id: inspect
-    outputSource:
-      - compress/inspect
-    type: File[]
   - id: logfile
     outputSource:
       - compress/logfile
@@ -28,14 +24,14 @@ outputs:
     outputSource:
       - compress/ingest
     type: Any[]
-  - id: uv_coverage
-    type: File[]
+  - id: quality
+    type: Any
     outputSource:
-      - compress/uv_coverage
-  - id: uv_plot
-    type: File[]
+      - format_quality_metrics/formatted_quality
+  - id: inspect
+    type: File
     outputSource:
-      - compress/uv_plot
+      - combine/inspect_file
 steps:
   - id: fetch_data
     in:
@@ -60,9 +56,31 @@ steps:
       - id: inspect
       - id: logfile
       - id: ingest
-      - id: uv_coverage
-      - id: uv_plot
     run: ./compress_pipeline.cwl
+  - id: combine
+    in:
+    - id: inputs
+      source: compress/inspect
+    out:
+    - inspect_file
+    run: steps/combine_inspect_dataset.cwl
+  - id: extract_metrics
+    in: 
+    - id: inspect
+      source: combine/inspect_file
+    out:
+    - id: plots
+    - id: quality
+    run: steps/extract_quality_metrics.cwl
+  - id: format_quality_metrics
+    in: 
+    - id: plots
+      source: extract_metrics/plots
+    - id: quality
+      source: extract_metrics/quality
+    run: steps/format_metrics.cwl
+    out:
+    - formatted_quality
 requirements:
   - class: ScatterFeatureRequirement
   - class: SubworkflowFeatureRequirement

steps/combine_inspect_dataset.cwl

+cwlVersion: 'v1.2'
+class: CommandLineTool
+baseCommand: 
+  - inspect_utils.py
+  - join
+
+requirements:
+  - class: InlineJavascriptRequirement
+hints:
+  - class: DockerRequirement
+    dockerPull: git.astron.nl:5000/ldv/lofar_quality:latest
+  - class: NetworkAccess
+    networkAccess: true
+inputs: 
+- id: inputs
+  type: File[]
+  inputBinding:
+    position: 1
+- id: output_filename
+  type: string?
+  default: inspect.h5
+  inputBinding:
+    position: 2
+outputs: 
+- id: inspect_file
+  type: File
+  outputBinding: 
+    glob: $(inputs.output_filename)
\ No newline at end of file

steps/create_inspect_dataset.cwl

+cwlVersion: 'v1.2'
+class: CommandLineTool
+baseCommand: 
+  - inspect_utils.py
+  - create
+
+requirements:
+  - class: InlineJavascriptRequirement
+hints:
+  - class: DockerRequirement
+    dockerPull: git.astron.nl:5000/ldv/lofar_quality:latest
+  - class: NetworkAccess
+    networkAccess: true
+inputs: 
+- id: msin
+  type: Directory
+  inputBinding: 
+    position: 1
+- id: inspect_file_name
+  type: string?
+  default: 'inspect.h5'
+  inputBinding:
+    position: 2
+outputs: 
+- id: inspect_file
+  type: File
+  outputBinding: 
+    glob: $(inputs.inspect_file_name)
\ No newline at end of file

steps/extract_quality_metrics.cwl

+cwlVersion: 'v1.2'
+class: CommandLineTool
+baseCommand: 
+  - bash
+  - script.sh
+
+requirements:
+  - class: InlineJavascriptRequirement
+  - class: InitialWorkDirRequirement
+    listing:
+      - entryname: script.sh
+        entry: | 
+          #!/bin/bash
+          inspect_utils.py plot $(inputs.inspect.path) plots
+          inspect_utils.py inspect $(inputs.inspect.path) quality.json
+hints:
+  - class: DockerRequirement
+    dockerPull: git.astron.nl:5000/ldv/lofar_quality:latest
+  - class: NetworkAccess
+    networkAccess: true
+inputs: 
+- id: inspect
+  type: File
+outputs: 
+- id: plots
+  type: File[]
+  outputBinding: 
+    glob: plots/*.png
+- id: quality
+  type: Any
+  outputBinding:
+    glob: quality.json
+    loadContents: true
+    outputEval: $(JSON.parse(self[0].contents))
\ No newline at end of file

steps/format_metrics.cwl

+cwlVersion: 'v1.2'
+class: ExpressionTool
+expression: |
+  ${
+    var sensitivityScore = inputs.quality.dutch_array_incomplete ? 1 : 0;
+    sensitivityScore += inputs.quality.dutch_array_high_data_loss ? 0.5 : 0;
+    sensitivityScore += inputs.quality.dutch_array_flag_data_loss ? 0.5 : 0;
+    sensitivityScore += inputs.quality.full_array_incomplete ? 1 : 0;
+    sensitivityScore += inputs.quality.full_array_missing_important_pair ? 0.5 : 0;
+    sensitivityScore += inputs.quality.full_array_incomplete_is ? 0.5 : 0;
+    var uvCoverageScore = sensitivityScore;
+
+    uvCoverageScore += inputs.quality.fill_array_missing_is_pair ? 1 : 0;
+    uvCoverageScore += inputs.quality.dutch_array_missing_important_pair ? 1 : 0;
+    uvCoverageScore += inputs.quality.full_array_missing_important_pair ? 1 : 0;
+    
+    sensitivityScore += inputs.quality.high_flagging ? 1 : 0;
+    
+    var observingConditions =  inputs.quality.sun_interference;
+    observingConditions +=  inputs.quality.moon_interference;
+    observingConditions +=  inputs.quality.jupiter_interference;
+    observingConditions +=  1 - inputs.quality.elevation_score;
+    sensitivityScore += observingConditions;
+    
+    var result = {
+              'uv-coverage': uvCoverageScore,
+              'sensitivity': sensitivityScore,
+              'observing-conditions': observingConditions,
+              'plots': inputs.plots,
+              'details': inputs.quality
+    };
+    return {'formatted_quality': result}
+      
+  } 
+requirements:
+- class: InlineJavascriptRequirement
+inputs:
+- id: plots
+  type: File[]
+- id: quality
+  type: Any
+outputs:
+- id: formatted_quality
+  type: Any

workflows/inspect_workflow.cwl

+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+- id: ms
+  type: Directory[]
+
+outputs:
+- id: inspect
+  type: File
+  outputSource:
+    combine/inspect_file
+- id: quality
+  type: Any
+  outputSource:
+    format_metrics/formatted_quality
+
+requirements:
+- class: ScatterFeatureRequirement
+steps:
+- id: create
+  in: 
+  - id: msin
+    source: ms
+  out:
+  - id: inspect_file
+  scatter: msin
+  run: ../steps/create_inspect_dataset.cwl
+- id: combine
+  in:
+  - id: inputs
+    source: create/inspect_file
+  out:
+  - id: inspect_file
+  run: ../steps/combine_inspect_dataset.cwl
+- id: extract
+  in: 
+  - id: inspect
+    source: combine/inspect_file
+  out:
+   - id: quality
+   - id: plots
+  run: ../steps/extract_quality_metrics.cwl
+- id: format_metrics
+  in:
+  - id: quality
+    source: extract/quality
+  - id: plots
+    source: extract/plots
+  out:
+  - formatted_quality
+  run: ../steps/format_metrics.cwl
\ No newline at end of file