Merge branch 'do_not_compress_sun_targets' into 'master'

Don't compress sun targets

See merge request !3

compress_pipeline.cwl

@@ -27,11 +27,11 @@ outputs:
   outputSource:
   - dppp/logfile
 - id: compressed
-  type: File
+  type: File?
   outputSource:
   - compress/compressed
 - id: ingest
-  type: Any
+  type: Any?
   outputSource:
   - format_ingest/ingest
 - id: applied_fixes
@@ -42,6 +42,9 @@ outputs:
   type: string[]
   outputSource:
   - identify_issues/unfixable_issues
+- id: keep_input
+  type: boolean
+  outputSource: identify_issues/skip_compress
 
 steps:
 - id: identify_issues
@@ -56,6 +59,7 @@ steps:
   - is_raw
   - unfixable_issues
   - msout
+  - skip_compress
 - id: fix_ai_2013
   in:
   - id: msin
@@ -105,6 +109,9 @@ steps:
     source: dppp/msout
   - id: compressed_file
     source: compress/compressed
+  - id: skip
+    source: identify_issues/skip_compress
+  when: $(inputs.skip == false)
   run: steps/extract_sip_meta.cwl
   out:
   - id: ingest
@@ -135,8 +142,11 @@ steps:
   - id: output_name
     default: compressed
   - id: file_name
-    valueFrom: $(self.basename)
+    valueFrom: '$(self ? self.basename : "skipped_so_no_file")'
     source: compress/compressed
+  - id: skip
+    source: identify_issues/skip_compress
+  when: $(inputs.skip == false)
   run: steps/format_ingest.cwl
   out:
   - id: ingest
@@ -155,6 +165,9 @@ steps:
   in:
   - id: directory
     source: dppp/msout
+  - id: skip
+    source: identify_issues/skip_compress
+  when: $(inputs.skip == false)
   run: steps/compress.cwl
   out:
   - id: compressed
@@ -162,6 +175,9 @@ steps:
   in:
   - id: msin
     source: dppp/msout
+  - id: inspect_file_name
+    source: dppp/msout
+    valueFrom: $(self.basename + "_inspect.h5")
   run: steps/create_inspect_dataset.cwl
   out:
   - inspect_file

download_and_compress_pipeline.cwl

@@ -16,7 +16,7 @@ inputs:
 - id: surls
   type: string[]
 - id: is_long_baseline
-  type: boolean
+  type: boolean?
 
 outputs:
 - id: compressed
@@ -39,6 +39,10 @@ outputs:
   type: Any
   outputSource:
   - format_quality_metrics/formatted_quality
+- id: control
+  type: Any
+  outputSource:
+  - format_quality_metrics/control
 - id: inspect
   type: File
   outputSource:
@@ -71,6 +75,7 @@ steps:
   - id: inspect
   - id: logfile
   - id: ingest
+  - id: keep_input
   - id: applied_fixes
   - id: unfixable_issues
 - id: combine
@@ -94,6 +99,10 @@ steps:
     source: extract_metrics/plots
   - id: quality
     source: extract_metrics/quality
+  - id: msin
+    source: fetch_data/uncompressed
+  - id: keep_input
+    source: compress/keep_input
   - id: applied_fixes
     source: compress/applied_fixes
   - id: unfixable_issues
@@ -101,3 +110,4 @@ steps:
   run: steps/format_metrics.cwl
   out:
   - formatted_quality
+  - control

steps/format_metrics.cwl

@@ -11,6 +11,10 @@ inputs:
   type: File[]
 - id: quality
   type: Any
+- id: msin
+  type: Directory[]
+- id: keep_input
+  type: boolean[]
 - id: applied_fixes
   type:
     type: array
@@ -27,15 +31,33 @@ inputs:
 outputs:
 - id: formatted_quality
   type: Any
+- id: control
+  type: Any
+
 expression: |
   ${
 
-    var result = {
+    var quality = {
       'plots': inputs.plots,
       'summary': inputs.quality
     };
-    result['summary']['applied_fixes'] = inputs.applied_fixes[0]
-    result['summary']['unfixable_issues'] = inputs.unfixable_issues[0]
-    return {'formatted_quality': result}
 
+    quality['summary']['applied_fixes'] = inputs.applied_fixes[0];
+    quality['summary']['unfixable_issues'] = inputs.unfixable_issues[0];
+
+    var control = {
+      'keep_input': inputs.msin.map((file, index) => {
+        if (inputs.keep_input[index]) {
+          return {
+            surl: file.location
+          };
+        }
+        return null;
+      }).filter(item => item !== null)
+    };
+
+    return {
+      'formatted_quality': quality,
+      'control': control
+    };
   }

steps/identify_issues.cwl

@@ -42,6 +42,12 @@ outputs:
     glob: output.txt
     outputEval: $(JSON.parse(self[0].contents)['is_raw'])
     loadContents: true
+- id: skip_compress
+  type: boolean
+  outputBinding:
+    glob: output.txt
+    outputEval: $(JSON.parse(self[0].contents)['skip_compress'])
+    loadContents: true
 - id: msout
   type: Directory
   outputBinding: