From 491ff022d40c06348dcaa99e4f5d2f8c696780f0 Mon Sep 17 00:00:00 2001
From: Alexander Drabent <alex@tls-tautenburg.de>
Date: Mon, 12 Oct 2020 09:28:48 +0200
Subject: [PATCH] finish first version target pipeline
Former-commit-id: 7350e7bf360c9621a5621db755dbbf67850039d3
Former-commit-id: 1381a821e03cc48b969e3ad902306bd8e486ad81
---
lofar-cwl/steps/applytarget.cwl | 7 +++++++
lofar-cwl/steps/dpppconcat.cwl | 11 +++++++----
steps/add_missing_stations.cwl | 12 +++++++++++-
steps/filter_ms_group.cwl | 7 ++++---
steps/find_skymodel_target.cwl | 2 +-
steps/h5parmcat.cwl | 4 ++--
steps/identify_bad_antennas.cwl | 6 +++---
subworkflow/ndppp_prep_targ.cwl | 1 +
workflows/prefactor_target/finalize.cwl | 3 +++
9 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/lofar-cwl/steps/applytarget.cwl b/lofar-cwl/steps/applytarget.cwl
index cb0f8114..883ab2ec 100644
--- a/lofar-cwl/steps/applytarget.cwl
+++ b/lofar-cwl/steps/applytarget.cwl
@@ -55,6 +55,13 @@ inputs:
Type of correction to perform. When using H5Parm, this is for now the name
of the soltab; the type will be deduced from the metadata in that soltab,
except for full Jones, in which case correction should be 'fulljones'.
+ - default: 'sol000'
+ id: solset
+ type: string
+ inputBinding:
+ position: 0
+ prefix: applycal.solset=
+ separate: false
- id: storagemanager
type: string
default: ""
diff --git a/lofar-cwl/steps/dpppconcat.cwl b/lofar-cwl/steps/dpppconcat.cwl
index b7e20eb0..e4570f06 100644
--- a/lofar-cwl/steps/dpppconcat.cwl
+++ b/lofar-cwl/steps/dpppconcat.cwl
@@ -9,11 +9,11 @@ inputs:
- id: msin
type: 'Directory[]'
inputBinding:
+ position: 0
prefix: msin=
separate: false
itemSeparator: ','
- shellQuote: false
- position: 0
+ valueFrom: $(concatenate_path(self))
doc: Input Measurement Set
- id: msout_name
type: string
@@ -44,7 +44,7 @@ inputs:
inputBinding:
prefix: filter.baseline=
separate: false
- shellQuote: false
+ shellQuote: true
position: 0
valueFrom: $(self)
default: null
@@ -96,7 +96,7 @@ inputs:
inputBinding:
prefix: msin.baseline=
separate: false
- shellQuote: false
+ shellQuote: true
position: 0
default: '*'
- id: avg_timeresolution
@@ -128,9 +128,12 @@ outputs:
arguments:
- 'steps=[filter,avg]'
- msout.orderms=False
+ - avg.type=average
requirements:
- class: ShellCommandRequirement
- class: InlineJavascriptRequirement
+ expressionLib:
+ - { $include: 'utils.js' }
hints:
- class: DockerRequirement
dockerPull: 'lofareosc/prefactor:HBAcalibrator'
diff --git a/steps/add_missing_stations.cwl b/steps/add_missing_stations.cwl
index ea4ecec1..d2258e17 100644
--- a/steps/add_missing_stations.cwl
+++ b/steps/add_missing_stations.cwl
@@ -1,6 +1,7 @@
class: CommandLineTool
cwlVersion: v1.0
$namespaces:
+ lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
sbg: 'https://www.sevenbridges.com/'
id: add_missing_stations
baseCommand:
@@ -17,13 +18,14 @@ inputs:
type: File
inputBinding:
position: 0
- prefix: '--refh5parm'
+ prefix: '--refh5'
doc: External H5parm from which the full list of antennas is used from.
- id: solset
type: string?
inputBinding:
position: 0
prefix: '--solset'
+ default: 'sol000'
doc: Input calibration solutions
- id: refsolset
type: string?
@@ -31,35 +33,41 @@ inputs:
position: 0
prefix: '--refsolset'
doc: Input calibration solutions of the reference h5parm file
+ default: 'sol000'
- id: soltab_in
type: string?
inputBinding:
position: 0
prefix: '--soltab_in'
doc: Input solution table
+ default: 'phase000'
- id: soltab_out
type: string?
inputBinding:
position: 0
prefix: '--soltab_out'
doc: Output solution table (has to be different from input solution table)
+ default: 'GSMphase'
- id: filter
type: string?
inputBinding:
position: 0
prefix: '--filter'
doc: Filter these antenna string from the processing
+ default: '[CR]S*&'
- id: bad_antennas
type: string?
inputBinding:
position: 0
prefix: '--bad_antennas'
doc: Antenna string to be processed
+ default: '[CR]S*&'
outputs:
- id: outh5parm
type: File
outputBinding:
glob: $(inputs.h5parm.basename)
+ format: 'lofar:#H5Parm'
- id: log
type: File[]
outputBinding:
@@ -77,3 +85,5 @@ requirements:
listing:
- entry: $(inputs.h5parm)
writable: true
+$schema:
+ - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/steps/filter_ms_group.cwl b/steps/filter_ms_group.cwl
index 2d3b5417..d51dcf69 100644
--- a/steps/filter_ms_group.cwl
+++ b/steps/filter_ms_group.cwl
@@ -34,12 +34,13 @@ requirements:
entry: |
import sys
import json
- import os.path as path
+ import os
+
group_id = "$(inputs.group_id)"
json_file = sys.argv[1]
ms_list = sys.argv[2:]
- ms_by_name = { ms.split(path.sep)[-1]:
+ ms_by_name = { ms.split(os.path.sep)[-1]:
{'class':'Directory', 'path': ms} for ms in ms_list}
output_file = 'selected_ms.json'
@@ -47,7 +48,7 @@ requirements:
with open(json_file, 'r') as f_stream:
selected_ms = json.load(f_stream)[group_id]
print(selected_ms, ms_by_name)
- selected_ms = [ms_by_name[ms_name] for ms_name in selected_ms]
+ selected_ms = [ms_by_name[os.path.basename(ms_name)] for ms_name in selected_ms]
with open(output_file, 'w') as f_stream:
diff --git a/steps/find_skymodel_target.cwl b/steps/find_skymodel_target.cwl
index e7aa0b88..36191f1c 100644
--- a/steps/find_skymodel_target.cwl
+++ b/steps/find_skymodel_target.cwl
@@ -52,7 +52,7 @@ arguments:
output = download_skymodel_target(mss, SkymodelPath, Radius, DoDownload, Source)
- if SkymodelPath != "target.skymodel":
+ if "target.skymodel" not in SkymodelPath:
shutil.copyfile(SkymodelPath, "target.skymodel")
outputs:
diff --git a/steps/h5parmcat.cwl b/steps/h5parmcat.cwl
index f14f7e38..2770fc82 100644
--- a/steps/h5parmcat.cwl
+++ b/steps/h5parmcat.cwl
@@ -90,7 +90,7 @@ requirements:
hints:
- class: DockerRequirement
dockerPull: lofareosc/prefactor:HBAcalibrator
-stdout: parm_collector_output.log
-stderr: parm_collector_output_err.log
+stdout: h5parm_collector_output.log
+stderr: h5parm_collector_output_err.log
$schema:
- 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/steps/identify_bad_antennas.cwl b/steps/identify_bad_antennas.cwl
index a4cb554e..3bdc91f2 100644
--- a/steps/identify_bad_antennas.cwl
+++ b/steps/identify_bad_antennas.cwl
@@ -48,12 +48,12 @@ outputs:
requirements:
- class: InlineJavascriptRequirement
-
-hints:
- class: InitialWorkDirRequirement
listing:
- entry: $(inputs.msin)
writable: true
+hints:
- class: DockerRequirement
dockerPull: lofareosc/prefactor:HBAcalibrator
-stdout: identifyBadAntennas.log
\ No newline at end of file
+stdout: identifyBadAntennas.log
+stderr: identifyBadAntennas_err.log
\ No newline at end of file
diff --git a/subworkflow/ndppp_prep_targ.cwl b/subworkflow/ndppp_prep_targ.cwl
index 09f71d9c..45d923d9 100644
--- a/subworkflow/ndppp_prep_targ.cwl
+++ b/subworkflow/ndppp_prep_targ.cwl
@@ -472,3 +472,4 @@ steps:
requirements:
- class: SubworkflowFeatureRequirement
- class: InlineJavascriptRequirement
+ - class: StepInputExpressionRequirement
diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl
index 7864e566..d660be8d 100644
--- a/workflows/prefactor_target/finalize.cwl
+++ b/workflows/prefactor_target/finalize.cwl
@@ -72,6 +72,7 @@ steps:
source: insolutions
- id: soltab_in
source: gsmcal_step
+ valueFrom: $(self+'000')
- id: soltab_out
default: 'TGSSphase'
- id: filter
@@ -171,3 +172,5 @@ steps:
requirements:
- class: SubworkflowFeatureRequirement
- class: ScatterFeatureRequirement
+ - class: StepInputExpressionRequirement
+ - class: InlineJavascriptRequirement
\ No newline at end of file
--
GitLab