diff --git a/compress_pipeline.cwl b/compress_pipeline.cwl index 2f065dda29e254409c2ddffb9f84a79a6faef97f..3c0385fc9572e656abd37c114733e3ec504173e7 100644 --- a/compress_pipeline.cwl +++ b/compress_pipeline.cwl @@ -27,11 +27,11 @@ outputs: outputSource: - dppp/logfile - id: compressed - type: File + type: File? outputSource: - compress/compressed - id: ingest - type: Any + type: Any? outputSource: - format_ingest/ingest - id: applied_fixes @@ -42,6 +42,9 @@ outputs: type: string[] outputSource: - identify_issues/unfixable_issues +- id: keep_input + type: boolean + outputSource: identify_issues/skip_compress steps: - id: identify_issues @@ -56,6 +59,7 @@ steps: - is_raw - unfixable_issues - msout + - skip_compress - id: fix_ai_2013 in: - id: msin @@ -105,6 +109,9 @@ steps: source: dppp/msout - id: compressed_file source: compress/compressed + - id: skip + source: identify_issues/skip_compress + when: $(inputs.skip == false) run: steps/extract_sip_meta.cwl out: - id: ingest @@ -135,8 +142,11 @@ steps: - id: output_name default: compressed - id: file_name - valueFrom: $(self.basename) + valueFrom: '$(self ? self.basename : "skipped_so_no_file")' source: compress/compressed + - id: skip + source: identify_issues/skip_compress + when: $(inputs.skip == false) run: steps/format_ingest.cwl out: - id: ingest @@ -155,6 +165,9 @@ steps: in: - id: directory source: dppp/msout + - id: skip + source: identify_issues/skip_compress + when: $(inputs.skip == false) run: steps/compress.cwl out: - id: compressed @@ -162,6 +175,9 @@ steps: in: - id: msin source: dppp/msout + - id: inspect_file_name + source: dppp/msout + valueFrom: $(self.basename + "_inspect.h5") run: steps/create_inspect_dataset.cwl out: - inspect_file diff --git a/download_and_compress_pipeline.cwl b/download_and_compress_pipeline.cwl index 841bc0b56f7a83b3cf2988937c3728c4b7f98c65..4eaa44464aa2909c14d8ff40500b6a990abb48d3 100644 --- a/download_and_compress_pipeline.cwl +++ b/download_and_compress_pipeline.cwl @@ -16,7 +16,7 @@ inputs: - id: surls type: string[] - id: is_long_baseline - type: boolean + type: boolean? outputs: - id: compressed @@ -39,6 +39,10 @@ outputs: type: Any outputSource: - format_quality_metrics/formatted_quality +- id: control + type: Any + outputSource: + - format_quality_metrics/control - id: inspect type: File outputSource: @@ -71,6 +75,7 @@ steps: - id: inspect - id: logfile - id: ingest + - id: keep_input - id: applied_fixes - id: unfixable_issues - id: combine @@ -94,6 +99,10 @@ steps: source: extract_metrics/plots - id: quality source: extract_metrics/quality + - id: msin + source: fetch_data/uncompressed + - id: keep_input + source: compress/keep_input - id: applied_fixes source: compress/applied_fixes - id: unfixable_issues @@ -101,3 +110,4 @@ steps: run: steps/format_metrics.cwl out: - formatted_quality + - control diff --git a/steps/format_metrics.cwl b/steps/format_metrics.cwl index 9920403d50bd7442b8704cfc1fff45b1a7a6885e..fe3538c9011db505a33879fddadf89104f8fa4c2 100644 --- a/steps/format_metrics.cwl +++ b/steps/format_metrics.cwl @@ -11,6 +11,10 @@ inputs: type: File[] - id: quality type: Any +- id: msin + type: Directory[] +- id: keep_input + type: boolean[] - id: applied_fixes type: type: array @@ -27,15 +31,33 @@ inputs: outputs: - id: formatted_quality type: Any +- id: control + type: Any + expression: | ${ - var result = { - 'plots': inputs.plots, - 'summary': inputs.quality + var quality = { + 'plots': inputs.plots, + 'summary': inputs.quality + }; + + quality['summary']['applied_fixes'] = inputs.applied_fixes[0]; + quality['summary']['unfixable_issues'] = inputs.unfixable_issues[0]; + + var control = { + 'keep_input': inputs.msin.map((file, index) => { + if (inputs.keep_input[index]) { + return { + surl: file.location + }; + } + return null; + }).filter(item => item !== null) + }; + + return { + 'formatted_quality': quality, + 'control': control }; - result['summary']['applied_fixes'] = inputs.applied_fixes[0] - result['summary']['unfixable_issues'] = inputs.unfixable_issues[0] - return {'formatted_quality': result} - - } + } diff --git a/steps/identify_issues.cwl b/steps/identify_issues.cwl index e9fea34b2b3c66a9635e67f5895f696445c83e06..5815522e1f70ddec9e07e15d8c5ad4d440c8cbef 100644 --- a/steps/identify_issues.cwl +++ b/steps/identify_issues.cwl @@ -42,6 +42,12 @@ outputs: glob: output.txt outputEval: $(JSON.parse(self[0].contents)['is_raw']) loadContents: true +- id: skip_compress + type: boolean + outputBinding: + glob: output.txt + outputEval: $(JSON.parse(self[0].contents)['skip_compress']) + loadContents: true - id: msout type: Directory outputBinding: