diff --git a/steps/format_ingest.cwl b/steps/format_ingest.cwl
index c2b9f620a3ab28782f4a2f8dac7b4834d8739102..4bce6a315f941bdab2dd51c6d40894e02ecc6a1d 100644
--- a/steps/format_ingest.cwl
+++ b/steps/format_ingest.cwl
@@ -1,23 +1,28 @@
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.2
class: ExpressionTool
+
+requirements:
+- class: InlineJavascriptRequirement
+
inputs:
- - id: metadata
- type: Any
- - id: file_content
- type: File
- - id: output_name
- type: string
- - id: file_name
- type: string
- - id: md5sum
- type: string
- - id: filesize
- type: long
+- id: metadata
+ type: Any?
+- id: file_content
+ type: File
+- id: output_name
+ type: string
+- id: file_name
+ type: string
+- id: md5sum
+ type: string
+- id: filesize
+ type: long
+
outputs:
- - id: ingest
- type: Any
-requirements:
- - class: InlineJavascriptRequirement
+- id: ingest
+ type: Any
expression: |
${
@@ -31,7 +36,12 @@ expression: |
"dataProductType": "Pulsar pipeline output",
"storageWriterVersion": "Unknown"
}
- var pipelineRun = inputs.metadata
+ if(inputs.metadata){
+ var pipelineRun = inputs.metadata
+ }else{
+ var pipelineRun = {}
+ }
+
pipelineRun['fileContent'] = inputs.file_content
return { "ingest": {
"path": inputs.output_name,
diff --git a/steps/format_quality.cwl b/steps/format_quality.cwl
index c2149fc9a7446243f9eb7ec8005b08d9015a1dfa..168a55c2e2a641d86143544e1c6c7bad5deb69ff 100644
--- a/steps/format_quality.cwl
+++ b/steps/format_quality.cwl
@@ -1,46 +1,51 @@
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.2
class: ExpressionTool
+
+requirements:
+- class: InlineJavascriptRequirement
+
inputs:
- - id: summary_file
- type: File
- - id: plots
- type:
+- id: summary_file
+ type: File?
+- id: plots
+ type:
+ - type: array
+ items:
type: array
- items:
- type: array
- items: File
- - id: summaries
- type: Any[]
+ items: File
+- id: summaries
+ type: Any[]
-expression: |
- ${
- var plots = []
- for (var p_idx in inputs.plots) {
- plots = plots.concat(inputs.plots[p_idx])
- }
-
- plots.push(inputs.summary_file)
-
- var summary = {}
- for (var s_idx in inputs.summaries) {
- summary[inputs.summaries[s_idx]['input_name']] = inputs.summaries[s_idx]
- }
-
- return {
- "quality": {
- "plots": plots,
- "details": {},
- "summary": summary,
- "sensitivity": "N/A",
- "uv-coverage": "N/A",
- "observing-conditions": "N/A"
- }
-
- }
- }
outputs:
- - id: quality
- type: Any
+- id: quality
+ type: Any
+expression: |
+ ${
+ var plots = []
+ for (var p_idx in inputs.plots) {
+ plots = plots.concat(inputs.plots[p_idx])
+ }
-requirements:
- - class: InlineJavascriptRequirement
+ if(inputs.summary_file){
+ plots.push(inputs.summary_file);
+ }
+
+ var summary = {}
+ for (var s_idx in inputs.summaries) {
+ summary[inputs.summaries[s_idx]['input_name']] = inputs.summaries[s_idx]
+ }
+
+ return {
+ "quality": {
+ "plots": plots,
+ "details": {},
+ "summary": summary,
+ "sensitivity": "N/A",
+ "uv-coverage": "N/A",
+ "observing-conditions": "N/A"
+ }
+
+ }
+ }
diff --git a/steps/requantize.cwl b/steps/requantize.cwl
index 3c9de32efd0b39d1b31fff82d206afac98db1946..f10d6ccf98d7015eae84658bc2e52b0a06697ca8 100644
--- a/steps/requantize.cwl
+++ b/steps/requantize.cwl
@@ -7,8 +7,8 @@ requirements:
- class: InlineJavascriptRequirement
- class: InitialWorkDirRequirement
listing:
- - writable: True
- entry: $(inputs.bf_tar_archive)
+ - writable: true
+ entry: $(inputs.bf_tar_archive)
inputs:
- id: bf_tar_archive
@@ -21,37 +21,36 @@ outputs:
doc: output tar archive (modified input tar archive)
type: File
outputBinding:
- glob:
- - $(inputs.bf_tar_archive.basename)
+ glob:
+ - $(inputs.bf_tar_archive.basename)
- id: file_content
doc: file_content
type: File
outputBinding:
- glob: "*_filecontent.txt"
+ glob: '*_filecontent.txt'
- id: summary
doc: summary
type: Any
outputBinding:
- glob: "*-ldv-summary.json"
- loadContents: True
+ glob: '*-ldv-summary.json'
outputEval: $(JSON.parse(self[0].contents))
+ loadContents: true
- id: plots
type: File[]
outputBinding:
glob:
- - "*.png"
- - "*stokes/SAP*/BEAM*/*.png"
- - "*.rfifind.log"
+ - '*/*/SAP*/BEAM*/*.png'
+ - '*/*.png'
- id: summary_file
doc: summary_file
type: File?
outputBinding:
- glob:
- - "*stokes/SAP*/BEAM*/*_ldv_psrfits_requantisation.log"
+ glob:
+ - '*stokes/SAP*/BEAM*/*_ldv_psrfits_requantisation.log'
baseCommand:
- psrfits_requantisation
-
+
hints:
- class: DockerRequirement
dockerPull: git.astron.nl:5000/ldv/bf_double_tgz:latest
diff --git a/workflow/download_and_run_bf_remove.cwl b/workflow/download_and_run_bf_remove.cwl
index c4fecd62a50c1cf4d23b25aefee4f332079a2cd0..24078c6f46c6d4bffbc22b68de391f6e54352769 100644
--- a/workflow/download_and_run_bf_remove.cwl
+++ b/workflow/download_and_run_bf_remove.cwl
@@ -1,13 +1,23 @@
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.2
class: Workflow
+
+requirements:
+- class: ScatterFeatureRequirement
+- class: SubworkflowFeatureRequirement
+
inputs:
- - id: surls
- type: string[]
- - id: pulp_log_folder
- doc: Pulp Log Folder
- type: Directory
- default: {"class": "Directory", "path": "/project/ldv/Data/beamformed/pulp-logs/"}
-outputs:
+- id: surls
+ type: string[]
+- id: pulp_log_folder
+ doc: Pulp Log Folder
+ type: Directory
+ default:
+ class: Directory
+ path: /project/ldv/Data/beamformed/pulp-logs/
+
+outputs:
- id: tar_archive
type: File[]
outputSource: bf_process/tar_archive
@@ -20,38 +30,39 @@ outputs:
- id: quality
type: Any
outputSource: format_quality/quality
+
steps:
- id: fetch_data
in:
- id: surl_link
source: surls
- out:
- - id: tar_archive
scatter: surl_link
run: ../steps/fetch_data.cwl
+ out:
+ - id: tar_archive
- id: bf_process
- in:
+ in:
- id: bf_tar_archive
source: fetch_data/tar_archive
- id: pulp_log_folder
source: pulp_log_folder
+ scatter: bf_tar_archive
+ run: ./bf_remove_double_tgz.cwl
out:
- id: tar_archive
- id: ingest
- id: summary
- id: plots
- id: summary_file
- run: ./bf_remove_double_tgz.cwl
- scatter: bf_tar_archive
- id: join_contents
in:
- id: summary_files
source: bf_process/summary_file
- id: metadata
source: bf_process/summary
+ run: ../steps/join_contents.cwl
out:
- id: summary_file
- run: ../steps/join_contents.cwl
- id: format_quality
in:
- id: summary_file
@@ -60,10 +71,6 @@ steps:
source: bf_process/plots
- id: summaries
source: bf_process/summary
+ run: ../steps/format_quality.cwl
out:
- id: quality
- run: ../steps/format_quality.cwl
-
-requirements:
-- class: ScatterFeatureRequirement
-- class: SubworkflowFeatureRequirement
diff --git a/workflow/download_and_run_psrfits_requantisation.cwl b/workflow/download_and_run_psrfits_requantisation.cwl
index 55ba4b4f4afe33d09156019ee4f3f46dc8629421..367a6c2f190edfd0b2c59dfa836c655e743bb3a6 100644
--- a/workflow/download_and_run_psrfits_requantisation.cwl
+++ b/workflow/download_and_run_psrfits_requantisation.cwl
@@ -15,13 +15,19 @@ inputs:
outputs:
- id: tar_archive
type: File[]
- outputSource: requantise/bf_archive
+ outputSource: requantise/tar_archive
- id: summary
type: Any[]
outputSource: requantise/summary
- id: summary_file
type: File[]
- outputSource: requantise/file_content
+ outputSource: requantise/summary_file
+- id: ingest
+ type: Any
+ outputSource: requantise/ingest
+- id: quality
+ type: Any
+ outputSource: format_quality/quality
steps:
- id: fetch_data
@@ -37,8 +43,30 @@ steps:
- id: bf_tar_archive
source: fetch_data/tar_archive
scatter: bf_tar_archive
- run: ../steps/requantize.cwl
+ run: psrfits_requantisation.cwl
out:
- - id: file_content
+ - id: summary_file
- id: summary
- - id: bf_archive
+ - id: tar_archive
+ - id: ingest
+ - id: plots
+- id: join_contents
+ in:
+ - id: summary_files
+ source: requantise/summary_file
+ - id: metadata
+ source: requantise/summary
+ run: ../steps/join_contents.cwl
+ out:
+ - id: summary_file
+- id: format_quality
+ in:
+ - id: summary_file
+ source: join_contents/summary_file
+ - id: plots
+ source: requantise/plots
+ - id: summaries
+ source: requantise/summary
+ run: ../steps/format_quality.cwl
+ out:
+ - id: quality
diff --git a/workflow/psrfits_requantisation.cwl b/workflow/psrfits_requantisation.cwl
index ca8bbd1e9f12b70b7040123aefbb2102db9ab0ae..6b47c82e046c1263e01f21083bc43dbfc9c4129a 100644
--- a/workflow/psrfits_requantisation.cwl
+++ b/workflow/psrfits_requantisation.cwl
@@ -1,3 +1,5 @@
+#!/usr/bin/env cwl-runner
+
cwlVersion: v1.2
class: Workflow
@@ -6,28 +8,60 @@ requirements:
- class: InlineJavascriptRequirement
inputs:
- - id: bf_tar_archive
- doc: Tar archive with the BeamFormed dataset
- type: File
+- id: bf_tar_archive
+ doc: Tar archive with the BeamFormed dataset
+ type: File
outputs:
- - id: tar_archive
- type: File
- outputSource: requantise/bf_archive
- - id: summary
- type: Any
- outputSource: requantise/summary
- - id: summary_file
- type: File?
- outputSource: requantise/file_content
+- id: tar_archive
+ type: File
+ outputSource: requantise/bf_archive
+- id: summary
+ type: Any
+ outputSource: requantise/summary
+- id: summary_file
+ type: File
+ outputSource: requantise/file_content
+- id: plots
+ type: File[]
+ outputSource: requantise/plots
+- id: ingest
+ type: Any
+ outputSource: format_ingest/ingest
steps:
- id: requantise
- in:
+ in:
- id: bf_tar_archive
source: bf_tar_archive
+ run: ../steps/requantize.cwl
out:
- id: file_content
- id: summary
- id: bf_archive
- run: ../steps/requantize.cwl
+ - id: plots
+
+- id: compute_md5sum
+ run: ../steps/computemd5.cwl
+ in:
+ - id: archive
+ source: requantise/bf_archive
+ out:
+ - id: md5sum
+- id: format_ingest
+ run: ../steps/format_ingest.cwl
+ in:
+ - id: file_content
+ source: requantise/file_content
+ - id: output_name
+ default: tar_archive
+ - id: md5sum
+ source: compute_md5sum/md5sum
+ - id: file_name
+ source: requantise/bf_archive
+ valueFrom: $(self.basename)
+ - id: filesize
+ source: requantise/bf_archive
+ valueFrom: $(self.size)
+ out:
+ - id: ingest
\ No newline at end of file