From 7e8409cb2f94951507141a3cb5349449eb395a4e Mon Sep 17 00:00:00 2001
From: mancini <mancini@astron.nl>
Date: Wed, 14 Apr 2021 11:24:59 +0200
Subject: [PATCH] Modify workflow to skip one copy step

---
 compress_pipeline.cwl               | 26 ++++++++------------------
 steps/DPPP.cwl                      |  6 ------
 steps/{untar.cwl => fetch_data.cwl} | 17 ++++++++++-------
 steps/stage.cwl                     | 21 ---------------------
 steps/surl_copy.cwl                 | 24 ------------------------
 5 files changed, 18 insertions(+), 76 deletions(-)
 rename steps/{untar.cwl => fetch_data.cwl} (52%)
 delete mode 100644 steps/stage.cwl
 delete mode 100644 steps/surl_copy.cwl

diff --git a/compress_pipeline.cwl b/compress_pipeline.cwl
index 7a1717a..89d7345 100644
--- a/compress_pipeline.cwl
+++ b/compress_pipeline.cwl
@@ -25,34 +25,24 @@ outputs:
       - compress/compressed
     type: File[]
 steps:
-  - id: surl_copy
+  - id: fetch_data
     in:
-      - id: surl
+      - id: surl_link
         source: surls
-    out:
-      - id: output
-    run: steps/surl_copy.cwl
-    label: surl_copy
-    scatter:
-      - surl
-  - id: untar
-    in:
-      - id: tar_file
-        source: surl_copy/output
     out:
       - id: uncompressed
-    run: steps/untar.cwl
-    label: untar
+    run: steps/fetch_data.cwl
+    label: fetch_data
     scatter:
-      - tar_file
+      - surl_link
   - id: dppp
     in:
       - id: parset
         source: define_parset/output
       - id: msin
-        source: untar/uncompressed
+        source: fetch_data/uncompressed
       - id: msout_name
-        source: untar/uncompressed
+        source: fetch_data/uncompressed
         valueFrom: '$("COMPRESSED_" + self[0].basename)'
       - id: writefullresflag
         default: true
@@ -77,7 +67,7 @@ steps:
   - id: inspect_flagging_dataloss
     in:
       - id: input
-        source: untar/uncompressed
+        source: fetch_data/uncompressed
     out:
       - id: output
     run: steps/inspect_flagging_dataloss.cwl
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
index 3930e63..d8a9c60 100755
--- a/steps/DPPP.cwl
+++ b/steps/DPPP.cwl
@@ -1,7 +1,5 @@
 class: CommandLineTool
 cwlVersion: v1.0
-$namespaces:
-  sbg: 'https://www.sevenbridges.com/'
 id: dppp
 baseCommand:
   - DPPP
@@ -91,7 +89,3 @@ stdout: DPPP.log
 stderr: DPPP_err.log
 requirements:
   - class: InlineJavascriptRequirement
-  - class: InitialWorkDirRequirement
-    listing:
-      - entry: $(inputs.msin)
-        writable: true
diff --git a/steps/untar.cwl b/steps/fetch_data.cwl
similarity index 52%
rename from steps/untar.cwl
rename to steps/fetch_data.cwl
index 55d02a9..10e7825 100644
--- a/steps/untar.cwl
+++ b/steps/fetch_data.cwl
@@ -1,10 +1,10 @@
-id: untar
-label: untar
+id: fetchdata
+label: fetch_data
 class: CommandLineTool
 cwlVersion: v1.0
 inputs: 
-  - id: tar_file
-    type: File
+  - id: surl_link
+    type: string
     inputBinding:
       position: 0
     
@@ -15,16 +15,19 @@ outputs:
       glob: 'out/*'
 baseCommand: 
  - 'bash'
- - 'untar.sh'
+ - 'fetch.sh'
 doc: 'Untar a compressed file'
 requirements:
+  InlineJavascriptRequirement: {}
   InitialWorkDirRequirement:
     listing:
-      - entryname: 'untar.sh' 
+      - entryname: 'fetch.sh' 
         entry: |
           #!/bin/bash
           mkdir out
           cd out
-          tar -xvf $1
+          turl=`echo $1 | awk '{gsub("srm://srm.grid.sara.nl[:0-9]*","gsiftp://gridftp.grid.sara.nl"); print}'`
+          echo "Downloading $turl"
+          globus-url-copy $turl - | tar -xvf -
 
     
diff --git a/steps/stage.cwl b/steps/stage.cwl
deleted file mode 100644
index 0cc4266..0000000
--- a/steps/stage.cwl
+++ /dev/null
@@ -1,21 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
-  sbg: 'https://www.sevenbridges.com/'
-id: stage
-baseCommand:
-  - gfal-legacy-bringonline
-inputs:
-  - id: input
-    type: string
-    inputBinding:
-      shellQuote: false
-      position: 0
-outputs:
-  - id: output
-    type: string
-    outputBinding:
-      outputEval: $(inputs.input)
-label: stage
-requirements:
-  - class: InlineJavascriptRequirement
diff --git a/steps/surl_copy.cwl b/steps/surl_copy.cwl
deleted file mode 100644
index 62223e3..0000000
--- a/steps/surl_copy.cwl
+++ /dev/null
@@ -1,24 +0,0 @@
-class: CommandLineTool
-cwlVersion: v1.0
-$namespaces:
-  sbg: 'https://www.sevenbridges.com/'
-id: surl_copy
-baseCommand:
-  - gfal-copy
-
-inputs:
-  - id: surl
-    type: string
-    inputBinding:
-      position: -1
-outputs:
-  - id: output
-    type: File
-    outputBinding:
-      glob: '*'
-doc: Download a file give a surl
-label: surl_copy
-arguments:
-  - .
-temporaryFailCodes:
-  - 70
-- 
GitLab