diff --git a/compress_pipeline.cwl b/compress_pipeline.cwl
index 89d734590baee950b4f48b5ad6fa5bf21181bc06..74d63485d72a2cf2f6e2f65b5db6b22ac4e4f67a 100644
--- a/compress_pipeline.cwl
+++ b/compress_pipeline.cwl
@@ -5,45 +5,44 @@ label: compress_pipeline.cwl
inputs:
- id: flag_autocorrelation
type: boolean?
- - id: surls
- type: 'string[]'
+ - id: msin
+ type: Directory
outputs:
- - id: output
+ - id: inspect
outputSource:
- inspect_flagging_dataloss/output
- type: 'File[]'
+ type: 'File'
- id: logfile
outputSource:
- dppp/logfile
- type:
- type: array
- items:
- items: File
- type: array
+ type: File[]
- id: compressed
outputSource:
- compress/compressed
- type: File[]
+ type: File
+ - id: ingest
+ outputSource:
+ - format_ingest/ingest
+ type: Any
steps:
- - id: fetch_data
+ - id: extract_sip_meta
in:
- - id: surl_link
- source: surls
+ - id: msin
+ source: dppp/msout
+ - id: compressed_file
+ source: compress/compressed
out:
- - id: uncompressed
- run: steps/fetch_data.cwl
- label: fetch_data
- scatter:
- - surl_link
+ - id: ingest
+ run: steps/extract_sip_meta.cwl
- id: dppp
in:
- id: parset
source: define_parset/output
- id: msin
- source: fetch_data/uncompressed
+ source: msin
- id: msout_name
- source: fetch_data/uncompressed
- valueFrom: '$("COMPRESSED_" + self[0].basename)'
+ source: msin
+ valueFrom: '$("COMPRESSED_" + self.basename)'
- id: writefullresflag
default: true
- id: storagemanager
@@ -54,8 +53,15 @@ steps:
- id: msout
- id: logfile
run: steps/DPPP.cwl
- scatter:
- - msin
+ - id: format_ingest
+ in:
+ - id: metadata
+ source: extract_sip_meta/ingest
+ - id: output_name
+ default: compressed
+ out:
+ - id: ingest
+ run: steps/format_ingest.cwl
- id: define_parset
in:
- id: flag_autocorrelation
@@ -67,13 +73,11 @@ steps:
- id: inspect_flagging_dataloss
in:
- id: input
- source: fetch_data/uncompressed
+ source: msin
out:
- id: output
run: steps/inspect_flagging_dataloss.cwl
label: inspect_flagging_dataloss
- scatter:
- - input
- id: compress
in:
- id: directory
@@ -82,9 +86,6 @@ steps:
- id: compressed
run: steps/compress.cwl
label: compress
- scatter:
- - directory
requirements:
- - class: ScatterFeatureRequirement
- class: StepInputExpressionRequirement
- class: InlineJavascriptRequirement
diff --git a/download_and_compress_pipeline.cwl b/download_and_compress_pipeline.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..a90a90fe70e8dd2bf579aded81c58b8b4118b055
--- /dev/null
+++ b/download_and_compress_pipeline.cwl
@@ -0,0 +1,60 @@
+class: Workflow
+cwlVersion: v1.0
+id: compress_pipeline_cwl
+label: compress_pipeline.cwl
+inputs:
+ - id: flag_autocorrelation
+ type: boolean?
+ - id: surls
+ type: 'string[]'
+outputs:
+ - id: compressed
+ outputSource:
+ - compress/compressed
+ type: File[]
+ - id: inspect
+ outputSource:
+ - compress/inspect
+ type: File[]
+ - id: logfile
+ outputSource:
+ - compress/logfile
+ type:
+ type: array
+ items:
+ type: array
+ items: File
+ - id: ingest
+ outputSource:
+ - compress/inspect
+ type: Any[]
+steps:
+ - id: fetch_data
+ in:
+ - id: surl_link
+ source: surls
+ out:
+ - id: uncompressed
+ run: steps/fetch_data.cwl
+ label: fetch_data
+ scatter:
+ - surl_link
+ - id: compress
+ in:
+ - id: msin
+ source: fetch_data/uncompressed
+ - id: flag_autocorrelation
+ source: flag_autocorrelation
+ scatter:
+ - msin
+ out:
+ - id: compressed
+ - id: inspect
+ - id: logfile
+ - id: ingest
+ run: ./compress_pipeline.cwl
+requirements:
+ - class: ScatterFeatureRequirement
+ - class: SubworkflowFeatureRequirement
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
diff --git a/steps/DPPP.cwl b/steps/DPPP.cwl
index d8a9c604fd0e843fb84a1c952797a03efe17ca50..aa1aab4fb6808f04f607b547230450eef5b49cd0 100755
--- a/steps/DPPP.cwl
+++ b/steps/DPPP.cwl
@@ -84,7 +84,7 @@ outputs:
glob: 'DPPP*.log'
hints:
- class: DockerRequirement
- dockerPull: lofareosc/lofar-pipeline:latest
+ dockerPull: astronsdc/lofar-ms-software:latest
stdout: DPPP.log
stderr: DPPP_err.log
requirements:
diff --git a/steps/extract_sip_meta.cwl b/steps/extract_sip_meta.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..e21882259afd2d444754942e207fd47163d7130c
--- /dev/null
+++ b/steps/extract_sip_meta.cwl
@@ -0,0 +1,76 @@
+class: CommandLineTool
+cwlVersion: v1.0
+id: inspect_flagging_dataloss
+baseCommand:
+ - bash
+ - script.sh
+inputs:
+ - id: msin
+ type: Directory
+ inputBinding:
+ position: 0
+ - id: compressed_file
+ type: File
+ inputBinding:
+ position: 1
+outputs:
+ - id: ingest
+ type: Any
+ outputBinding:
+ glob: metadata.json
+ loadContents: True
+ outputEval: |
+ ${
+ return JSON.parse(self[0].contents)
+ }
+label: inspect_flagging_dataloss
+hints:
+ - class: DockerRequirement
+ dockerPull: 'astronsdc/lofar-ms-software:latest'
+
+requirements:
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: pipeline_info.json
+ entry: |
+ {
+ "pipelineRun": {
+ "_type": "AveragingPipeline",
+ "demixing": "false",
+ "pipelineName": "compress_pipeline",
+ "strategyName": "",
+ "pipelineVersion": "v01.23",
+ "strategyDescription": "",
+ "timeIntegrationStep": "1",
+ "flagAutoCorrelations": "true",
+ "frequencyIntegrationStep": "1",
+ "numberOfCorrelatedDataProducts": "1"
+ }
+ }
+ - entryname: dataproduct.json
+ entry: |
+ {
+ "dataProduct":
+ {
+ "fileName": $(inputs.compressed_file.basename),
+ "size": $(inputs.compressed_file.size)
+ }
+ }
+ - entryname: script.sh
+ entry: |
+ md5=`md5sum $2 | awk '{ print $1 }'`
+ cat > md5_sum.json << EOF
+ {
+ "dataProduct":
+ {
+ "fileName": $(inputs.compressed_file.basename),
+ "size": $(inputs.compressed_file.size),
+ "md5checksum": $md5
+ }
+ }
+ EOF
+ lofar_sip_from_ms.py --json metadata.json $1
+ lofar_sip_merge.py metadata.json pipeline_info.json --json metadata.json
+ lofar_sip_merge.py metadata.json dataproduct.json --json metadata.json
+ lofar_sip_merge.py metadata.json md5_sum.json --json metadata.json
diff --git a/steps/format_ingest.cwl b/steps/format_ingest.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..073e6e8e0c7ff6328f10e3e8da7c93ea33cce78a
--- /dev/null
+++ b/steps/format_ingest.cwl
@@ -0,0 +1,22 @@
+cwlVersion: v1.2
+class: ExpressionTool
+inputs:
+ - id: metadata
+ type: Any
+
+ - id: output_name
+ type: string
+
+outputs:
+ - id: ingest
+ type: Any
+requirements:
+ - class: InlineJavascriptRequirement
+expression: |
+ ${
+ return { "ingest": {
+ "path": inputs.output_name,
+ "metadata": inputs.metadata
+ }
+ }
+ }
diff --git a/steps/inspect_flagging_dataloss.cwl b/steps/inspect_flagging_dataloss.cwl
index fbccfd81087d8f4b90d3bcc809128d6413b112a9..5668af87fa46797782cf9787818d35c5e61cd6f3 100644
--- a/steps/inspect_flagging_dataloss.cwl
+++ b/steps/inspect_flagging_dataloss.cwl
@@ -8,7 +8,7 @@ baseCommand:
- script.py
inputs:
- id: input
- type: Directory?
+ type: Directory
inputBinding:
position: 0
shellQuote: false
@@ -27,10 +27,10 @@ requirements:
import sys
from casacore.tables import table
input_ms_path = sys.argv[1]
-
-
+
+
with open('metrics.json', 'w') as f_stream:
pass
-
+
- class: DockerRequirement
dockerPull: 'lofareosc/lofar-pipeline:latest'