From 9eb098933a253266fb2d528e7f81fca80df3cf55 Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Fri, 1 Apr 2022 12:08:38 +0200
Subject: [PATCH] Add quality metrics generation
---
compress_pipeline.cwl | 36 +++++---------------
download_and_compress_pipeline.cwl | 42 ++++++++++++++++-------
steps/combine_inspect_dataset.cwl | 28 ++++++++++++++++
steps/create_inspect_dataset.cwl | 28 ++++++++++++++++
steps/extract_quality_metrics.cwl | 34 +++++++++++++++++++
steps/format_metrics.cwl | 44 ++++++++++++++++++++++++
workflows/inspect_workflow.cwl | 54 ++++++++++++++++++++++++++++++
7 files changed, 227 insertions(+), 39 deletions(-)
create mode 100644 steps/combine_inspect_dataset.cwl
create mode 100644 steps/create_inspect_dataset.cwl
create mode 100644 steps/extract_quality_metrics.cwl
create mode 100644 steps/format_metrics.cwl
create mode 100644 workflows/inspect_workflow.cwl
diff --git a/compress_pipeline.cwl b/compress_pipeline.cwl
index e76a0a3..192fd98 100644
--- a/compress_pipeline.cwl
+++ b/compress_pipeline.cwl
@@ -10,7 +10,7 @@ inputs:
outputs:
- id: inspect
outputSource:
- - inspect_flagging_dataloss/output
+ - inspect_step/inspect_file
type: 'File'
- id: logfile
outputSource:
@@ -24,14 +24,6 @@ outputs:
outputSource:
- format_ingest/ingest
type: Any
- - id: uv_plot
- type: File
- outputSource:
- - plot_uvw_coverage/uv_plot
- - id: uv_coverage
- type: File
- outputSource:
- - inspect_flagging_dataloss/flags_output
steps:
- id: identify_issues
run: steps/identify_issues.cwl
@@ -133,23 +125,6 @@ steps:
- id: output
run: steps/define_parset.cwl
label: define_parset
- - id: inspect_flagging_dataloss
- in:
- - id: input
- source:
- - fix_baselines/msout
- out:
- - id: output
- - id: flags_output
- run: steps/inspect_flagging_dataloss.cwl
- label: inspect_flagging_dataloss
- - id: plot_uvw_coverage
- in:
- - id: input_file
- source: inspect_flagging_dataloss/flags_output
- out:
- - id: uv_plot
- run: steps/plot_uvw_coverage.cwl
- id: compress
in:
- id: directory
@@ -158,7 +133,14 @@ steps:
- id: compressed
run: steps/compress.cwl
label: compress
+ - id: inspect_step
+ in:
+ - id: msin
+ source: dppp/msout
+ out:
+ - inspect_file
+ run: steps/create_inspect_dataset.cwl
requirements:
- class: StepInputExpressionRequirement
- class: InlineJavascriptRequirement
- - class: MultipleInputFeatureRequirement
+ - class: MultipleInputFeatureRequirement
\ No newline at end of file
diff --git a/download_and_compress_pipeline.cwl b/download_and_compress_pipeline.cwl
index 9614352..fed9961 100644
--- a/download_and_compress_pipeline.cwl
+++ b/download_and_compress_pipeline.cwl
@@ -12,10 +12,6 @@ outputs:
outputSource:
- compress/compressed
type: File[]
- - id: inspect
- outputSource:
- - compress/inspect
- type: File[]
- id: logfile
outputSource:
- compress/logfile
@@ -28,14 +24,14 @@ outputs:
outputSource:
- compress/ingest
type: Any[]
- - id: uv_coverage
- type: File[]
+ - id: quality
+ type: Any
outputSource:
- - compress/uv_coverage
- - id: uv_plot
- type: File[]
+ - format_quality_metrics/formatted_quality
+ - id: inspect
+ type: File
outputSource:
- - compress/uv_plot
+ - combine/inspect_file
steps:
- id: fetch_data
in:
@@ -60,9 +56,31 @@ steps:
- id: inspect
- id: logfile
- id: ingest
- - id: uv_coverage
- - id: uv_plot
run: ./compress_pipeline.cwl
+ - id: combine
+ in:
+ - id: inputs
+ source: compress/inspect
+ out:
+ - inspect_file
+ run: steps/combine_inspect_dataset.cwl
+ - id: extract_metrics
+ in:
+ - id: inspect
+ source: combine/inspect_file
+ out:
+ - id: plots
+ - id: quality
+ run: steps/extract_quality_metrics.cwl
+ - id: format_quality_metrics
+ in:
+ - id: plots
+ source: extract_metrics/plots
+ - id: quality
+ source: extract_metrics/quality
+ run: steps/format_metrics.cwl
+ out:
+ - formatted_quality
requirements:
- class: ScatterFeatureRequirement
- class: SubworkflowFeatureRequirement
diff --git a/steps/combine_inspect_dataset.cwl b/steps/combine_inspect_dataset.cwl
new file mode 100644
index 0000000..538a796
--- /dev/null
+++ b/steps/combine_inspect_dataset.cwl
@@ -0,0 +1,28 @@
+cwlVersion: 'v1.2'
+class: CommandLineTool
+baseCommand:
+ - inspect_utils.py
+ - join
+
+requirements:
+ - class: InlineJavascriptRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: git.astron.nl:5000/ldv/lofar_quality:latest
+ - class: NetworkAccess
+ networkAccess: true
+inputs:
+- id: inputs
+ type: File[]
+ inputBinding:
+ position: 1
+- id: output_filename
+ type: string?
+ default: inspect.h5
+ inputBinding:
+ position: 2
+outputs:
+- id: inspect_file
+ type: File
+ outputBinding:
+ glob: $(inputs.output_filename)
\ No newline at end of file
diff --git a/steps/create_inspect_dataset.cwl b/steps/create_inspect_dataset.cwl
new file mode 100644
index 0000000..4254400
--- /dev/null
+++ b/steps/create_inspect_dataset.cwl
@@ -0,0 +1,28 @@
+cwlVersion: 'v1.2'
+class: CommandLineTool
+baseCommand:
+ - inspect_utils.py
+ - create
+
+requirements:
+ - class: InlineJavascriptRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: git.astron.nl:5000/ldv/lofar_quality:latest
+ - class: NetworkAccess
+ networkAccess: true
+inputs:
+- id: msin
+ type: Directory
+ inputBinding:
+ position: 1
+- id: inspect_file_name
+ type: string?
+ default: 'inspect.h5'
+ inputBinding:
+ position: 2
+outputs:
+- id: inspect_file
+ type: File
+ outputBinding:
+ glob: $(inputs.inspect_file_name)
\ No newline at end of file
diff --git a/steps/extract_quality_metrics.cwl b/steps/extract_quality_metrics.cwl
new file mode 100644
index 0000000..c47e38a
--- /dev/null
+++ b/steps/extract_quality_metrics.cwl
@@ -0,0 +1,34 @@
+cwlVersion: 'v1.2'
+class: CommandLineTool
+baseCommand:
+ - bash
+ - script.sh
+
+requirements:
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: script.sh
+ entry: |
+ #!/bin/bash
+ inspect_utils.py plot $(inputs.inspect.path) plots
+ inspect_utils.py inspect $(inputs.inspect.path) quality.json
+hints:
+ - class: DockerRequirement
+ dockerPull: git.astron.nl:5000/ldv/lofar_quality:latest
+ - class: NetworkAccess
+ networkAccess: true
+inputs:
+- id: inspect
+ type: File
+outputs:
+- id: plots
+ type: File[]
+ outputBinding:
+ glob: plots/*.png
+- id: quality
+ type: Any
+ outputBinding:
+ glob: quality.json
+ loadContents: true
+ outputEval: $(JSON.parse(self[0].contents))
\ No newline at end of file
diff --git a/steps/format_metrics.cwl b/steps/format_metrics.cwl
new file mode 100644
index 0000000..c2eed30
--- /dev/null
+++ b/steps/format_metrics.cwl
@@ -0,0 +1,44 @@
+cwlVersion: 'v1.2'
+class: ExpressionTool
+expression: |
+ ${
+ var sensitivityScore = inputs.quality.dutch_array_incomplete ? 1 : 0;
+ sensitivityScore += inputs.quality.dutch_array_high_data_loss ? 0.5 : 0;
+ sensitivityScore += inputs.quality.dutch_array_flag_data_loss ? 0.5 : 0;
+ sensitivityScore += inputs.quality.full_array_incomplete ? 1 : 0;
+ sensitivityScore += inputs.quality.full_array_missing_important_pair ? 0.5 : 0;
+ sensitivityScore += inputs.quality.full_array_incomplete_is ? 0.5 : 0;
+ var uvCoverageScore = sensitivityScore;
+
+ uvCoverageScore += inputs.quality.fill_array_missing_is_pair ? 1 : 0;
+ uvCoverageScore += inputs.quality.dutch_array_missing_important_pair ? 1 : 0;
+ uvCoverageScore += inputs.quality.full_array_missing_important_pair ? 1 : 0;
+
+ sensitivityScore += inputs.quality.high_flagging ? 1 : 0;
+
+ var observingConditions = inputs.quality.sun_interference;
+ observingConditions += inputs.quality.moon_interference;
+ observingConditions += inputs.quality.jupiter_interference;
+ observingConditions += 1 - inputs.quality.elevation_score;
+ sensitivityScore += observingConditions;
+
+ var result = {
+ 'uv-coverage': uvCoverageScore,
+ 'sensitivity': sensitivityScore,
+ 'observing-conditions': observingConditions,
+ 'plots': inputs.plots,
+ 'details': inputs.quality
+ };
+ return {'formatted_quality': result}
+
+ }
+requirements:
+- class: InlineJavascriptRequirement
+inputs:
+- id: plots
+ type: File[]
+- id: quality
+ type: Any
+outputs:
+- id: formatted_quality
+ type: Any
diff --git a/workflows/inspect_workflow.cwl b/workflows/inspect_workflow.cwl
new file mode 100644
index 0000000..b334322
--- /dev/null
+++ b/workflows/inspect_workflow.cwl
@@ -0,0 +1,54 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.2
+class: Workflow
+
+inputs:
+- id: ms
+ type: Directory[]
+
+outputs:
+- id: inspect
+ type: File
+ outputSource:
+ combine/inspect_file
+- id: quality
+ type: Any
+ outputSource:
+ format_metrics/formatted_quality
+
+requirements:
+- class: ScatterFeatureRequirement
+steps:
+- id: create
+ in:
+ - id: msin
+ source: ms
+ out:
+ - id: inspect_file
+ scatter: msin
+ run: ../steps/create_inspect_dataset.cwl
+- id: combine
+ in:
+ - id: inputs
+ source: create/inspect_file
+ out:
+ - id: inspect_file
+ run: ../steps/combine_inspect_dataset.cwl
+- id: extract
+ in:
+ - id: inspect
+ source: combine/inspect_file
+ out:
+ - id: quality
+ - id: plots
+ run: ../steps/extract_quality_metrics.cwl
+- id: format_metrics
+ in:
+ - id: quality
+ source: extract/quality
+ - id: plots
+ source: extract/plots
+ out:
+ - formatted_quality
+ run: ../steps/format_metrics.cwl
\ No newline at end of file
--
GitLab