From a79fdefd6e280d9fcc4a31bb95c469b820ada354 Mon Sep 17 00:00:00 2001
From: Mattia Mancini <mancini@astron.nl>
Date: Mon, 5 Sep 2022 15:12:19 +0000
Subject: [PATCH] Add ldv workflow
---
steps/compress.cwl | 25 ++++++
steps/fetch_data.cwl | 32 ++++++++
steps/fetch_file.cwl | 34 ++++++++
steps/list_directory.cwl | 21 +++++
workflows/ldv_prefactor_calibrator.cwl | 63 +++++++++++++++
workflows/ldv_prefactor_target.cwl | 105 +++++++++++++++++++++++++
6 files changed, 280 insertions(+)
create mode 100644 steps/compress.cwl
create mode 100644 steps/fetch_data.cwl
create mode 100644 steps/fetch_file.cwl
create mode 100644 steps/list_directory.cwl
create mode 100644 workflows/ldv_prefactor_calibrator.cwl
create mode 100644 workflows/ldv_prefactor_target.cwl
diff --git a/steps/compress.cwl b/steps/compress.cwl
new file mode 100644
index 00000000..2051082a
--- /dev/null
+++ b/steps/compress.cwl
@@ -0,0 +1,25 @@
+id: compress
+label: compress
+class: CommandLineTool
+cwlVersion: v1.2
+inputs:
+ - id: directory
+ type: Directory
+outputs:
+ - id: compressed
+ type: File
+ outputBinding:
+ glob: 'out/*.tar'
+baseCommand:
+ - 'bash'
+ - 'tar.sh'
+doc: 'Create archive from directory'
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: $(inputs.directory)
+ - entryname: 'tar.sh'
+ entry: |
+ #!/bin/bash
+ mkdir out
+ tar -h -cvf out/$(inputs.directory.basename).tar $(inputs.directory.basename)
diff --git a/steps/fetch_data.cwl b/steps/fetch_data.cwl
new file mode 100644
index 00000000..4dbb4675
--- /dev/null
+++ b/steps/fetch_data.cwl
@@ -0,0 +1,32 @@
+id: fetchdata
+label: fetch_data
+class: CommandLineTool
+cwlVersion: v1.2
+inputs:
+ - id: surl_link
+ type: string
+ inputBinding:
+ position: 0
+
+outputs:
+ - id: uncompressed
+ type: Directory
+ outputBinding:
+ glob: 'out/*'
+baseCommand:
+ - 'bash'
+ - 'fetch.sh'
+doc: 'Fetch a file from surl and uncompresses it'
+requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'fetch.sh'
+ entry: |
+ #!/bin/bash
+ mkdir out
+ cd out
+ turl=`echo $1 | awk '{gsub("srm://srm.grid.sara.nl[:0-9]*","gsiftp://gridftp.grid.sara.nl"); print}'`
+ echo "Downloading $turl"
+ globus-url-copy $turl - | tar -xvf -
+
+
diff --git a/steps/fetch_file.cwl b/steps/fetch_file.cwl
new file mode 100644
index 00000000..aad47dda
--- /dev/null
+++ b/steps/fetch_file.cwl
@@ -0,0 +1,34 @@
+id: fetchfile
+label: fetch_file
+class: CommandLineTool
+cwlVersion: v1.2
+inputs:
+ - id: surl_link
+ type: string
+ inputBinding:
+ position: 0
+
+outputs:
+ - id: downloaded
+ type: File
+ outputBinding:
+ glob: 'out/*'
+baseCommand:
+ - 'bash'
+ - 'fetch.sh'
+doc: 'Fetch a file from surl'
+requirements:
+ InlineJavascriptRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: 'fetch.sh'
+ entry: |
+ #!/bin/bash
+ mkdir out
+ cd out
+ turl=`echo $1 | awk '{gsub("srm://srm.grid.sara.nl[:0-9]*","gsiftp://gridftp.grid.sara.nl"); print}'`
+ filename=`echo $1 | awk '{sub(".*/", "", $1); print $0}'`
+ echo "Downloading $turl"
+ globus-url-copy $turl file://$PWD/$filename
+
+
diff --git a/steps/list_directory.cwl b/steps/list_directory.cwl
new file mode 100644
index 00000000..83f72948
--- /dev/null
+++ b/steps/list_directory.cwl
@@ -0,0 +1,21 @@
+id: listdirectory
+label: list_directory
+class: ExpressionTool
+
+cwlVersion: v1.2
+inputs:
+ - id: input
+ type: Directory
+ loadListing: shallow_listing
+outputs:
+ - id: output
+ type: Directory[]
+
+expression: |
+ ${
+ return {'output': inputs.input.listing}
+ }
+
+
+requirements:
+ - class: InlineJavascriptRequirement
diff --git a/workflows/ldv_prefactor_calibrator.cwl b/workflows/ldv_prefactor_calibrator.cwl
new file mode 100644
index 00000000..32892237
--- /dev/null
+++ b/workflows/ldv_prefactor_calibrator.cwl
@@ -0,0 +1,63 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+- class: ScatterFeatureRequirement
+- class: SubworkflowFeatureRequirement
+
+inputs:
+- id: surls
+ type: string[]
+
+outputs:
+- id: solutions
+ type: File
+ outputSource:
+ - prefactor_calibrator/solutions
+- id: summary
+ type: File
+ outputSource:
+ - prefactor_calibrator/summary
+- id: inspection_plots
+ type: File
+ outputSource:
+ - compress_inspection_plots/compressed
+- id: log_files
+ type: File
+ outputSource:
+ - compress_logs/compressed
+steps:
+- id: fetch_data
+ in:
+ - id: surl_link
+ source: surls
+ scatter: surl_link
+ run: ../steps/fetch_data.cwl
+ out:
+ - id: uncompressed
+- id: prefactor_calibrator
+ in:
+ - id: msin
+ source: fetch_data/uncompressed
+ run: HBA_calibrator.cwl
+ out:
+ - id: solutions
+ - id: summary
+ - id: inspection_plots
+ - id: log_files
+- id: compress_inspection_plots
+ in:
+ - id: directory
+ source: prefactor_calibrator/inspection_plots
+ out:
+ - id: compressed
+ run: ../steps/compress.cwl
+- id: compress_logs
+ in:
+ - id: directory
+ source: prefactor_calibrator/log_files
+ out:
+ - id: compressed
+ run: ../steps/compress.cwl
\ No newline at end of file
diff --git a/workflows/ldv_prefactor_target.cwl b/workflows/ldv_prefactor_target.cwl
new file mode 100644
index 00000000..fdd1b4c4
--- /dev/null
+++ b/workflows/ldv_prefactor_target.cwl
@@ -0,0 +1,105 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+ - class: ScatterFeatureRequirement
+ - class: SubworkflowFeatureRequirement
+
+inputs:
+ - id: surls
+ type: string[]
+ - id: solutions
+ type: string
+ - id: avg_timeresolution_concat
+ type: int?
+ default: 4
+ - id: avg_freqresolution_concat
+ type: string?
+ default: 48.82kHz
+
+outputs:
+ - id: cal_solutions
+ type: File
+ outputSource:
+ - prefactor_target/solutions
+ - id: summary
+ type: File
+ outputSource:
+ - prefactor_target/summary
+ - id: calibrated_data
+ type: File[]
+ outputSource:
+ - compress_calibrated_data/compressed
+ - id: inspection_plots
+ type: File
+ outputSource:
+ - compress_inspection_plots/compressed
+ - id: log_files
+ type: File
+ outputSource:
+ - compress_logs/compressed
+steps:
+ - id: fetch_data
+ in:
+ - id: surl_link
+ source: surls
+ scatter: surl_link
+ run: ../steps/fetch_data.cwl
+ out:
+ - id: uncompressed
+ - id: fetch_solutions
+ in:
+ - id: surl_link
+ source: solutions
+ run: ../steps/fetch_file.cwl
+ out:
+ - id: downloaded
+ - id: prefactor_target
+ in:
+ - id: msin
+ source: fetch_data/uncompressed
+ - id: cal_solutions
+ source: fetch_solutions/downloaded
+ - id: avg_timeresolution_concat
+ source: avg_timeresolution_concat
+ - id: avg_freqresolution_concat
+ source: avg_freqresolution_concat
+ run: HBA_target.cwl
+ out:
+ - id: solutions
+ - id: calibrated_data
+ - id: summary
+ - id: inspection_plots
+ - id: log_files
+ - id: compress_inspection_plots
+ in:
+ - id: directory
+ source: prefactor_target/inspection_plots
+ out:
+ - id: compressed
+ run: ../steps/compress.cwl
+
+ - id: split_calibrated_data
+ in:
+ - id: input
+ source: prefactor_target/calibrated_data
+ out:
+ - id: output
+ run: ../steps/list_directory.cwl
+ - id: compress_calibrated_data
+ in:
+ - id: directory
+ source: split_calibrated_data/output
+ out:
+ - id: compressed
+ run: ../steps/compress.cwl
+ scatter: directory
+ - id: compress_logs
+ in:
+ - id: directory
+ source: prefactor_target/log_files
+ out:
+ - id: compressed
+ run: ../steps/compress.cwl
\ No newline at end of file
--
GitLab