From 19e41085d1fa4f38f91ab01ec80ab609eaee8f98 Mon Sep 17 00:00:00 2001
From: Alexander Drabent <alex@tls-tautenburg.de>
Date: Mon, 15 Jun 2020 17:53:12 +0200
Subject: [PATCH] add target pipeline
Former-commit-id: 37fa460b1074c2d29424483bb2db697f529f8af4
---
lofar-cwl/steps/AOFlagger.cwl | 2 +-
lofar-cwl/steps/DPPP.cwl | 42 +-
lofar-cwl/steps/LoSoTo.PrefactorBandpass.cwl | 152 ++++---
lofar-cwl/steps/LoSoTo.PrefactorXYoffset.cwl | 94 +++--
lofar-cwl/steps/calib_rot_diag.cwl | 130 +++---
lofar-cwl/steps/gaincal.cwl | 160 ++++----
lofar-cwl/steps/interpolate_visibilities.cwl | 40 +-
lofar-cwl/steps/wsclean.cwl | 31 +-
workflows/HBA_calibrator.cwl | 7 +-
workflows/HBA_target.cwl | 393 +++++++++++++++++++
workflows/prefactor_target.cwl | 319 ++++++++++++++-
workflows/prefactor_target/concat.cwl | 381 ++++++++++++++++++
workflows/prefactor_target/gsmcal.cwl | 381 ++++++++++++++++++
workflows/prefactor_target/prep.cwl | 307 +++++++++++++++
workflows/prefactor_target/prep_gsmcal.cwl | 381 ++++++++++++++++++
15 files changed, 2521 insertions(+), 299 deletions(-)
create mode 100644 workflows/HBA_target.cwl
create mode 100644 workflows/prefactor_target/concat.cwl
create mode 100644 workflows/prefactor_target/gsmcal.cwl
create mode 100644 workflows/prefactor_target/prep.cwl
create mode 100644 workflows/prefactor_target/prep_gsmcal.cwl
diff --git a/lofar-cwl/steps/AOFlagger.cwl b/lofar-cwl/steps/AOFlagger.cwl
index b2e3eff7..eac08767 100644
--- a/lofar-cwl/steps/AOFlagger.cwl
+++ b/lofar-cwl/steps/AOFlagger.cwl
@@ -13,7 +13,7 @@ requirements:
hints:
DockerRequirement:
- dockerPull: lofareosc/lofar-pipeline-ci:latest
+ dockerPull: lofareosc/prefactor:HBAcalibrator
baseCommand:
- aoflagger
diff --git a/lofar-cwl/steps/DPPP.cwl b/lofar-cwl/steps/DPPP.cwl
index 5dc50b4c..b15ffd78 100755
--- a/lofar-cwl/steps/DPPP.cwl
+++ b/lofar-cwl/steps/DPPP.cwl
@@ -17,7 +17,7 @@ inputs:
prefix: msin=
separate: false
doc: Input Measurement Set
- - default: out.MS
+ - default: "."
id: msout_name
type: string
inputBinding:
@@ -37,16 +37,32 @@ inputs:
- id: output_directory_names
type: Any
doc: Expected output file names
- - id: autoweight
+ - default: false
+ id: autoweight
type: boolean
- default: true
inputBinding:
- prefix: msin.autoweight=True
- - id: baseline
+ position: 0
+ prefix: 'msin.autoweight=True'
+ - default: ''
+ id: baseline
type: string
- default: ""
inputBinding:
- prefix: msin.baseline=
+ position: 0
+ prefix: 'msin.baseline='
+ separate: false
+ - default: DATA
+ id: output_column
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: 'msout.datacolumn='
+ separate: false
+ - default: DATA
+ id: input_column
+ type: string?
+ inputBinding:
+ position: 0
+ prefix: 'msin.datacolumn='
separate: false
- id: writefullresflag
type: boolean
@@ -60,7 +76,7 @@ inputs:
prefix: msout.overwrite=True
- id: storagemanager
type: string
- default: "Dysco"
+ default: ""
inputBinding:
prefix: msout.storagemanager=
separate: false
@@ -75,7 +91,7 @@ outputs:
doc: Output Measurement Set
type: Directory
outputBinding:
- glob: '$(inputs.msout_name=="."?inputs.msin:inputs.msout_name)'
+ glob: '$(inputs.msout_name=="." ? inputs.msin.basename : inputs.msout_name)'
- id: secondary_output_files
doc: Secondary output files
type: Any
@@ -116,8 +132,12 @@ outputs:
glob: 'DPPP*.log'
hints:
- class: DockerRequirement
- dockerPull: lofareosc/lofar-pipeline
+ dockerPull: lofareosc/prefactor:HBAcalibrator
stdout: DPPP.log
stderr: DPPP_err.log
requirements:
- - class: InlineJavascriptRequirement
\ No newline at end of file
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entry: $(inputs.msin)
+ writable: true
diff --git a/lofar-cwl/steps/LoSoTo.PrefactorBandpass.cwl b/lofar-cwl/steps/LoSoTo.PrefactorBandpass.cwl
index c81efdab..021adf8b 100644
--- a/lofar-cwl/steps/LoSoTo.PrefactorBandpass.cwl
+++ b/lofar-cwl/steps/LoSoTo.PrefactorBandpass.cwl
@@ -1,101 +1,131 @@
-#!/usr/bin/env cwl-runner
-
class: CommandLineTool
cwlVersion: v1.0
-id: losoto_prefactor_bandpass
-
$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-doc: |
- This operation for LoSoTo implements the Prefactor bandpass operation
- WEIGHT: flag-only compliant, no need for weight
-
-
-requirements:
- InlineJavascriptRequirement:
- expressionLib:
- - { $include: utils.js}
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: $(get_losoto_config('PREFACTOR_BANDPASS').join('\n'))
-
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/lofar-pipeline-ci:latest
-
+ lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
+ sbg: 'https://www.sevenbridges.com/'
+id: losoto_prefactor_bandpass
+baseCommand:
+ - losoto
inputs:
- - id: input_h5parm
+ - format: 'lofar:#H5Parm'
+ id: input_h5parm
type: File
- format: lofar:#H5Parm
- id: soltab
type: string
- doc: "Solution table"
+ doc: Solution table
- id: chanWidth
type: string?
- doc: |
- the width of each channel in the data from which solutions were obtained. Can be
- either a string like "48kHz" or a float in Hz. If interpolate = True, chanWidth
- must be specified
+ doc: >
+ the width of each channel in the data from which solutions were obtained.
+ Can be
+
+ either a string like "48kHz" or a float in Hz. If interpolate = True,
+ chanWidth
+ must be specified
- id: BadSBList
type: string?
doc: a list of bad subbands that will be flagged
-
- id: outSoltabName
type: string?
- doc: |
- Name of the output bandpass soltab. An existing soltab with this name will be
+ doc: >
+ Name of the output bandpass soltab. An existing soltab with this name will
+ be
+
overwritten
- - id: interpolate
+ - default: false
+ id: interpolate
type: boolean?
- default: false
- doc: |
- If True, interpolate to a regular frequency grid and then smooth, ignoring bad
- subbands. If False, neither interpolation nor smoothing is done and the output
- frequency grid is the same as the input one. If interpolate = True, chanWidth
+ doc: >
+ If True, interpolate to a regular frequency grid and then smooth, ignoring
+ bad
+
+ subbands. If False, neither interpolation nor smoothing is done and the
+ output
+
+ frequency grid is the same as the input one. If interpolate = True,
+ chanWidth
+
must be specified
- id: removeTimeAxis
type: boolean?
- doc: |
- If True, the time axis of the output bandpass soltab is removed by doing a median
+ doc: >
+ If True, the time axis of the output bandpass soltab is removed by doing a
+ median
+
over time. If False, the output time grid is the same as the input one
- id: autoFlag
type: boolean?
- doc: If True, automatically flag bad frequencies and stations
-
+ doc: 'If True, automatically flag bad frequencies and stations'
- id: nSigma
type: float?
- doc: Number of sigma for autoFlagging. Amplitudes outside of nSigma*stddev are flagged
-
+ doc: >-
+ Number of sigma for autoFlagging. Amplitudes outside of nSigma*stddev are
+ flagged
- id: maxFlaggedFraction
type: float?
- doc: |
- Maximum allowable fraction of flagged frequencies for autoFlagging. Stations with
+ doc: >
+ Maximum allowable fraction of flagged frequencies for autoFlagging.
+ Stations with
+
higher fractions will be completely flagged
- id: maxStddev
type: float?
doc: Maximum allowable standard deviation for autoFlagging
-
- id: ncpu
type: int?
doc: Number of CPUs to use during autoFlagging (0 = all)
-
outputs:
- id: output_h5parm
type: File
- format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+ format: 'lofar:#H5Parm'
+doc: |
+ This operation for LoSoTo implements the Prefactor bandpass operation
+ WEIGHT: flag-only compliant, no need for weight
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: parset.config
+ entry: $(get_losoto_config('PREFACTOR_BANDPASS').join('\n'))
+ writable: false
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+ - class: InlineJavascriptRequirement
+ expressionLib:
+ - |
+ function get_losoto_config(step_name) {
+ var par = ['soltab = ' + inputs.soltab]
+ if (inputs.ncpu !== null && inputs.ncpu !== undefined) par.push('ncpu='+inputs.ncpu);
+ console.log(inputs, par)
+ par.push("[" + step_name + "]")
+ par.push('operation=' + step_name)
+ for(var field_name in inputs){
+ if(field_name === 'input_h5parm' ||
+ field_name === 'soltab' ||
+ field_name === 'ncpu' ||
+ field_name === 'execute') continue;
+
+ if(inputs[field_name] === null ||
+ inputs[field_name] === 'null') continue;
+
+ if(inputs[field_name]["class"] !== undefined &&
+ (inputs[field_name]["class"] ==="Directory" ||
+ inputs[field_name]["class"] ==="File")){
+ par.push(field_name+'='+inputs[field_name].path)
+ } else {
+ par.push(field_name+'='+inputs[field_name])
+ }
+ }
+ return par
+ }
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+ - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/lofar-cwl/steps/LoSoTo.PrefactorXYoffset.cwl b/lofar-cwl/steps/LoSoTo.PrefactorXYoffset.cwl
index 49e072c3..2fd707f6 100644
--- a/lofar-cwl/steps/LoSoTo.PrefactorXYoffset.cwl
+++ b/lofar-cwl/steps/LoSoTo.PrefactorXYoffset.cwl
@@ -1,53 +1,73 @@
-#!/usr/bin/env cwl-runner
-
class: CommandLineTool
cwlVersion: v1.0
-id: losoto_prefactor_xyoffset
-
$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
-
-requirements:
- InlineJavascriptRequirement:
- expressionLib:
- - { $include: utils.js}
- InitialWorkDirRequirement:
- listing:
- - entryname: 'parset.config'
- entry: $(get_losoto_config('PREFACTOR_XYOFFSET').join('\n'))
-
- - entryname: $(inputs.input_h5parm.basename)
- entry: $(inputs.input_h5parm)
- writable: true
-
-baseCommand: "losoto"
-
-arguments:
- - $(inputs.input_h5parm.basename)
- - parset.config
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/lofar-pipeline-ci:latest
-
+ lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
+ sbg: 'https://www.sevenbridges.com/'
+id: losoto_prefactor_xyoffset
+baseCommand:
+ - losoto
inputs:
- - id: input_h5parm
+ - format: 'lofar:#H5Parm'
+ id: input_h5parm
type: File
- format: lofar:#H5Parm
- id: soltab
type: string
- doc: "Solution table"
+ doc: Solution table
- id: chanWidth
type: string
- doc: |
- the width of each channel in the data from which solutions were obtained. Can be
+ doc: >
+ the width of each channel in the data from which solutions were obtained.
+ Can be
+
either a string like "48kHz" or a float in Hz.
-
outputs:
- id: output_h5parm
type: File
- format: lofar:#H5Parm
outputBinding:
glob: $(inputs.input_h5parm.basename)
+ format: 'lofar:#H5Parm'
+arguments:
+ - $(inputs.input_h5parm.basename)
+ - parset.config
+requirements:
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: parset.config
+ entry: $(get_losoto_config('PREFACTOR_XYOFFSET').join('\n'))
+ writable: false
+ - entryname: $(inputs.input_h5parm.basename)
+ entry: $(inputs.input_h5parm)
+ writable: true
+ - class: InlineJavascriptRequirement
+ expressionLib:
+ - |
+ function get_losoto_config(step_name) {
+ var par = ['soltab = ' + inputs.soltab]
+ if (inputs.ncpu !== null && inputs.ncpu !== undefined) par.push('ncpu='+inputs.ncpu);
+ console.log(inputs, par)
+ par.push("[" + step_name + "]")
+ par.push('operation=' + step_name)
+ for(var field_name in inputs){
+ if(field_name === 'input_h5parm' ||
+ field_name === 'soltab' ||
+ field_name === 'ncpu' ||
+ field_name === 'execute') continue;
+
+ if(inputs[field_name] === null ||
+ inputs[field_name] === 'null') continue;
+
+ if(inputs[field_name]["class"] !== undefined &&
+ (inputs[field_name]["class"] ==="Directory" ||
+ inputs[field_name]["class"] ==="File")){
+ par.push(field_name+'='+inputs[field_name].path)
+ } else {
+ par.push(field_name+'='+inputs[field_name])
+ }
+ }
+ return par
+ }
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+ - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/lofar-cwl/steps/calib_rot_diag.cwl b/lofar-cwl/steps/calib_rot_diag.cwl
index 66e57cd8..f57c1a88 100644
--- a/lofar-cwl/steps/calib_rot_diag.cwl
+++ b/lofar-cwl/steps/calib_rot_diag.cwl
@@ -1,85 +1,71 @@
-#!/usr/bin/env cwl-runner
-
class: CommandLineTool
cwlVersion: v1.0
+$namespaces:
+ lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
+ sbg: 'https://www.sevenbridges.com/'
id: calib_rot_diag
-baseCommand: [DPPP]
-
-requirements:
- InlineJavascriptRequirement: {}
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/lofar-pipeline-ci:latest
-
-arguments:
- - steps=[ddecal]
- - ddecal.mode=rotation+diagonal
- - ddecal.uvlambdamin=300
- - ddecal.maxiter=50
- - ddecal.nchan=1
- - ddecal.solint=1
- - ddecal.propagateconvergedonly=True
- - ddecal.flagdivergedonly=True
- - ddecal.tolerance=1.e-3
- - ddecal.usemodelcolumn=True
-
-
+baseCommand:
+ - DPPP
inputs:
- id: msin
type: Directory?
- doc: Input Measurement Set
inputBinding:
- prefix: msin=
- separate: false
-
- - id: msin_datacolumn
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
+ - default: DATA
+ id: msin_datacolumn
type: string
- default: DATA
- doc: Input data Column
inputBinding:
- prefix: msin.datacolumn=
- separate: false
-
- - id: msin_modelcolum
+ position: 0
+ prefix: msin.datacolumn=
+ separate: false
+ doc: Input data Column
+ - default: MODEL_DATA
+ id: msin_modelcolum
type: string
- default: MODEL_DATA
- doc: Model data Column
inputBinding:
- prefix: msin.modelcolumn=
- separate: false
-
- - id: output_name_h5parm
+ position: 0
+ prefix: msin.modelcolumn=
+ separate: false
+ doc: Model data Column
+ - default: instrument.h5
+ id: output_name_h5parm
type: string
- default: instrument.h5
inputBinding:
- prefix: ddecal.h5parm=
- separate: false
- - id: msout_name
+ position: 0
+ prefix: ddecal.h5parm=
+ separate: false
+ - default: out.MS
+ id: msout_name
type: string
- doc: Output Measurement Set
- default: out.MS
inputBinding:
- prefix: msout=
- separate: false
-
-#--------------------
- - id: propagate_solutions
+ position: 0
+ prefix: msout=
+ separate: false
+ doc: Output Measurement Set
+ - default: true
+ id: propagate_solutions
type: boolean
- default: true
inputBinding:
+ position: 0
prefix: propagatesolutions=True
- - id: flagunconverged
+ - default: false
+ id: flagunconverged
type: boolean
- default: false
+ inputBinding:
+ position: 0
+ prefix: flagdivergedonly=True
doc: |
Flag unconverged solutions (i.e., those from solves that did not converge
within maxiter iterations).
+ - default: false
+ id: flagdivergedonly
+ type: boolean
inputBinding:
+ position: 0
prefix: flagdivergedonly=True
- - id: flagdivergedonly
- default: false
- type: boolean
doc: |
Flag only the unconverged solutions for which divergence was detected.
At the moment, this option is effective only for rotation+diagonal
@@ -89,23 +75,33 @@ inputs:
If divergence for any one station is detected, all stations are flagged
for that solution interval. Only effective when flagunconverged=true
and mode=rotation+diagonal.
- inputBinding:
- prefix: flagdivergedonly=True
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
- glob: $(inputs.msout_name=="."?inputs.msin:inputs.msout_name)
-
+ glob: '$(inputs.msout_name=="."?inputs.msin:inputs.msout_name)'
- id: h5parm
doc: Filename of output H5Parm (to be read by e.g. losoto)
type: File
- format: lofar:#H5Parm
outputBinding:
glob: $(inputs.output_name_h5parm)
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+ format: 'lofar:#H5Parm'
+arguments:
+ - 'steps=[ddecal]'
+ - ddecal.mode=rotation+diagonal
+ - ddecal.uvlambdamin=300
+ - ddecal.maxiter=50
+ - ddecal.nchan=1
+ - ddecal.solint=1
+ - ddecal.propagateconvergedonly=True
+ - ddecal.flagdivergedonly=True
+ - ddecal.tolerance=1.e-3
+ - ddecal.usemodelcolumn=True
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+requirements:
+ - class: InlineJavascriptRequirement
$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+ - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/lofar-cwl/steps/gaincal.cwl b/lofar-cwl/steps/gaincal.cwl
index e017373e..863583df 100644
--- a/lofar-cwl/steps/gaincal.cwl
+++ b/lofar-cwl/steps/gaincal.cwl
@@ -1,114 +1,120 @@
-#!/usr/bin/env cwl-runner
-
class: CommandLineTool
cwlVersion: v1.0
+$namespaces:
+ lofar: 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
+ sbg: 'https://www.sevenbridges.com/'
id: gaincal
-baseCommand: [DPPP]
-
-requirements:
- InlineJavascriptRequirement: {}
-
-hints:
- DockerRequirement:
- dockerPull: lofareosc/lofar-pipeline-ci:latest
-
-arguments:
- - steps=[gaincal]
-
-
+baseCommand:
+ - DPPP
inputs:
- id: msin
type: Directory?
- doc: Input Measurement Set
inputBinding:
- prefix: msin=
- separate: false
+ position: 0
+ prefix: msin=
+ separate: false
+ doc: Input Measurement Set
- id: caltype
- doc: |
- The type of calibration that needs to be performed.
type:
- type: enum
- symbols:
- - diagonal
- - fulljones
- - phaseonly
- - scalarphase
- - amplitude
- - scalaramplitude
- - tec
- - tecandphase
+ type: enum
+ symbols:
+ - diagonal
+ - fulljones
+ - phaseonly
+ - scalarphase
+ - amplitude
+ - scalaramplitude
+ - tec
+ - tecandphase
+ name: caltype
inputBinding:
- prefix: gaincal.caltype=
- separate: false
+ position: 0
+ prefix: gaincal.caltype=
+ separate: false
+ doc: |
+ The type of calibration that needs to be performed.
- id: sourcedb
type: File
inputBinding:
- prefix: gaincal.sourcedb=
- separate: false
- - id: usebeammodel
+ position: 0
+ prefix: gaincal.sourcedb=
+ separate: false
+ - default: true
+ id: usebeammodel
type: boolean
- default: true
inputBinding:
- prefix: gaincal.usebeammodel=true
- separate: false
- - id: solint
+ position: 0
+ prefix: gaincal.usebeammodel=true
+ separate: false
+ - default: 1
+ id: solint
type: int
- default: 1
- doc: |
- Number of time slots on which a solution is assumed to be constant (same as CellSize.Time in BBS).
- 0 means all time slots. Note that for larger settings of solint, and specially for solint = 0,
- the memory usage of gaincal will be large (all visibilities for a solint should fit in memory).
- - id: msin_datacolumn
+ doc: >
+ Number of time slots on which a solution is assumed to be constant (same
+ as CellSize.Time in BBS).
+
+ 0 means all time slots. Note that for larger settings of solint, and
+ specially for solint = 0,
+
+ the memory usage of gaincal will be large (all visibilities for a solint
+ should fit in memory).
+ - default: DATA
+ id: msin_datacolumn
type: string?
- default: DATA
- doc: Input data Column
inputBinding:
- prefix: msin.datacolumn=
- separate: false
- - id: onebeamperpatch
- type: boolean
+ position: 0
+ prefix: msin.datacolumn=
+ separate: false
doc: Input data Column
- default: true
+ - default: true
+ id: onebeamperpatch
+ type: boolean
inputBinding:
- prefix: gaincal.onebeamperpatch=True
- separate: false
- - id: msin_modelcolum
+ position: 0
+ prefix: gaincal.onebeamperpatch=True
+ separate: false
+ doc: Input data Column
+ - default: MODEL_DATA
+ id: msin_modelcolum
type: string
- default: MODEL_DATA
- doc: Model data Column
inputBinding:
- prefix: msin.modelcolumn=
- separate: false
-
- - id: output_name_h5parm
+ position: 0
+ prefix: msin.modelcolumn=
+ separate: false
+ doc: Model data Column
+ - default: instrument.h5
+ id: output_name_h5parm
type: string
- default: instrument.h5
inputBinding:
- prefix: gaincal.parmdb=
- separate: false
- - id: msout_name
+ position: 0
+ prefix: gaincal.parmdb=
+ separate: false
+ - default: out.MS
+ id: msout_name
type: string
- doc: Output Measurement Set
- default: out.MS
inputBinding:
- prefix: msout=
- separate: false
-
+ position: 0
+ prefix: msout=
+ separate: false
+ doc: Output Measurement Set
outputs:
- id: msout
doc: Output Measurement Set
type: Directory
outputBinding:
- glob: $(inputs.msout_name=="."?inputs.msin:inputs.msout_name)
-
+ glob: '$(inputs.msout_name=="."?inputs.msin:inputs.msout_name)'
- id: h5parm
doc: Filename of output H5Parm (to be read by e.g. losoto)
type: File
- format: lofar:#H5Parm
outputBinding:
glob: $(inputs.output_name_h5parm)
-
-$namespaces:
- lofar: https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+ format: 'lofar:#H5Parm'
+arguments:
+ - 'steps=[gaincal]'
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+requirements:
+ - class: InlineJavascriptRequirement
$schema:
- - https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl
+ - 'https://git.astron.nl/eosc/ontologies/raw/master/schema/lofar.owl'
diff --git a/lofar-cwl/steps/interpolate_visibilities.cwl b/lofar-cwl/steps/interpolate_visibilities.cwl
index 7d7c97f9..f5928885 100644
--- a/lofar-cwl/steps/interpolate_visibilities.cwl
+++ b/lofar-cwl/steps/interpolate_visibilities.cwl
@@ -3,50 +3,52 @@ cwlVersion: v1.0
$namespaces:
sbg: 'https://www.sevenbridges.com/'
id: interpolate_visibilities
-
baseCommand:
- DPPP
-arguments:
- - msout.datacolumn=INTERP_DATA
- - msout=.
- - msin.datacolumn=DATA
- - steps=[interpolate]
inputs:
- id: msin
type: Directory
inputBinding:
- prefix: 'msin='
+ position: 0
+ prefix: msin=
separate: false
- - id: storage_manager
+ - default: Dysco
+ id: storage_manager
type: string
- default: Dysco
inputBinding:
+ position: 0
+ prefix: msout.storagemanager=
separate: false
- prefix: 'msout.storagemanager='
- - id: storage_manager_databitrate
+ - default: 0
+ id: storage_manager_databitrate
type: int
- default: 0
inputBinding:
- prefix: 'msout.storagemanager.databitrate='
+ position: 0
+ prefix: msout.storagemanager.databitrate=
separate: false
- id: window_size
type: int
inputBinding:
- prefix: 'interpolate.windowsize='
+ position: 0
+ prefix: interpolate.windowsize=
separate: false
-
outputs:
- id: msout
type: Directory
outputBinding:
glob: $(inputs.msin.basename)
label: interpolate_visibilities
-hints:
- - class: DockerRequirement
- dockerPull: lofareosc/lofar-pipeline-ci:latest
-
+arguments:
+ - msout.datacolumn=INTERP_DATA
+ - msout=.
+ - msin.datacolumn=DATA
+ - 'steps=[interpolate]'
requirements:
- class: InitialWorkDirRequirement
listing:
- entry: $(inputs.msin)
writable: true
+ - class: InlineJavascriptRequirement
+hints:
+ - class: DockerRequirement
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
diff --git a/lofar-cwl/steps/wsclean.cwl b/lofar-cwl/steps/wsclean.cwl
index b1ab1734..08d1e3d6 100644
--- a/lofar-cwl/steps/wsclean.cwl
+++ b/lofar-cwl/steps/wsclean.cwl
@@ -1,42 +1,47 @@
class: CommandLineTool
cwlVersion: v1.0
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
id: wsclean
baseCommand:
- wsclean
inputs:
- - id: image_size
- type: int[]
- default: [1000, 1000]
+ - default:
+ - 1000
+ - 1000
+ id: image_size
+ type: 'int[]'
inputBinding:
position: 1
- prefix: -size
- - id: image_scale
+ prefix: '-size'
+ - default: 0.01
+ id: image_scale
type: float
- default: 0.01
inputBinding:
position: 1
- prefix: -scale
+ prefix: '-scale'
valueFrom: $(self)deg
- - id: image_name
- default: image
+ - default: image
+ id: image_name
type: string
inputBinding:
position: 1
- prefix: -name
+ prefix: '-name'
- id: msin
type: Directory
inputBinding:
position: 2
outputs:
- dirty_image:
+ - id: dirty_image
type: File
outputBinding:
glob: $(inputs.image_name)-dirty.fits
- image:
+ - id: image
type: File
outputBinding:
glob: $(inputs.image_name)-image.fits
label: WSClean
requirements:
- class: DockerRequirement
- dockerPull: lofareosc/lofar-pipeline-ci:latest
+ dockerPull: 'lofareosc/prefactor:HBAcalibrator'
+ - class: InlineJavascriptRequirement
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index ea2b37e4..5979708a 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -186,10 +186,10 @@ inputs:
'sbg:x': -1000
'sbg:y': 1800
outputs:
- - id: msout
+ - id: logs
outputSource:
- - prefactor/msout
- type: 'Directory[]'
+ - save_logfiles/log_dir
+ type: Directory
'sbg:x': 1500
'sbg:y': -100
- id: results
@@ -335,7 +335,6 @@ steps:
source:
- save_solutions/log_dir
- save_inspection/log_dir
- - save_logfiles/log_dir
- prefactor/msout
- id: sub_directory_name
default: results
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
new file mode 100644
index 00000000..57b3cea6
--- /dev/null
+++ b/workflows/HBA_target.cwl
@@ -0,0 +1,393 @@
+class: Workflow
+cwlVersion: v1.0
+id: HBA_target
+label: HBA_target
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msin
+ type: 'Directory[]'
+ 'sbg:x': -1000
+ 'sbg:y': -1700
+ - id: cal_solutions
+ type: 'File'
+ 'sbg:x': -1000
+ 'sbg:y': -1600
+ - id: refant
+ type: string?
+ default: CS001HBA0
+ 'sbg:x': -1000
+ 'sbg:y': -1500
+ - id: flag_baselines
+ type: 'string[]?'
+ default: []
+ 'sbg:x': -1000
+ 'sbg:y': -1400
+ - id: process_baselines_target
+ type: string?
+ default: '[CR]S*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1300
+ - id: filter_baselines
+ type: string?
+ default: '[CR]S*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1200
+ - id: do_smooth
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -1100
+ - id: rfistrategy
+ type: string?
+ default: HBAdefault.rfis
+ 'sbg:x': -1000
+ 'sbg:y': -1000
+ - id: min_unflagged_fraction
+ type: float?
+ default: 0.5
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: compression_bitrate
+ type: int?
+ default: 16
+ 'sbg:x': -1000
+ 'sbg:y': -800
+ - id: raw_data
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -700
+ - id: propagatesolutions
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': -600
+ - id: demix_sources
+ type: 'string[]?'
+ default:
+ - CasA
+ - CygA
+ 'sbg:x': -1000
+ 'sbg:y': -500
+ - id: demix_target
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': -400
+ - id: demix_freqstep
+ type: int?
+ default: 16
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: demix_timestep
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: demix
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: apply_steps
+ type: string?
+ default: 'applyclock,applybeam,applyRM'
+ 'sbg:x': -1000
+ 'sbg:y': 0
+ - id: clipATeam
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: gsmcal_step
+ type: string?
+ default: 'phase'
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: updateweights
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': 300
+ - id: max_dppp_threads
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': 400
+ - id: memoryperc
+ type: int?
+ default: 20
+ 'sbg:x': -1000
+ 'sbg:y': 500
+ - id: min_length
+ type: int?
+ default: 5
+ 'sbg:x': -1000
+ 'sbg:y': 600
+ - id: overhead
+ type: float?
+ default: 0.7
+ 'sbg:x': -1000
+ 'sbg:y': 700
+ - id: min_separation
+ type: int?
+ default: 30
+ 'sbg:x': -1000
+ 'sbg:y': 800
+ - id: A-Team_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 900
+ - id: target_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 1000
+ - id: use_target
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': 1100
+ - id: skymodel_source
+ type: string?
+ default: 'TGSS'
+ 'sbg:x': -1000
+ 'sbg:y': 1200
+ - id: avg_timeresolution
+ type: float?
+ default: 4.
+ 'sbg:x': -1000
+ 'sbg:y': 1300
+ - id: avg_freqresolution
+ type: string?
+ default: 48.82kHz
+ 'sbg:x': -1000
+ 'sbg:y': 1400
+ - id: avg_timeresolution_concat
+ type: float?
+ default: 8.
+ 'sbg:x': -1000
+ 'sbg:y': 1500
+ - id: avg_freqresolution_concat
+ type: string?
+ default: 97.64kHz
+ 'sbg:x': -1000
+ 'sbg:y': 1600
+ - id: num_SBs_per_group
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': 1700
+ - id: reference_stationSB
+ type: int?
+ default: None
+ 'sbg:x': -1000
+ 'sbg:y': 1800
+ - id: ionex_server
+ type: string?
+ default: 'ftp://ftp.aiub.unibe.ch/CODE/'
+ 'sbg:x': -1000
+ 'sbg:y': 1900
+ - id: ionex_prefix
+ type: string?
+ default: 'CODG'
+ 'sbg:x': -1000
+ 'sbg:y': 2000
+ - id: proxy_server
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2100
+ - id: proxy_port
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2200
+ - id: proxy_type
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2300
+ - id: proxy_user
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2400
+ - id: proxy_pass
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2500
+outputs:
+ - id: logs
+ outputSource:
+ - save_logfiles/log_dir
+ type: Directory
+ 'sbg:x': 1500
+ 'sbg:y': -100
+ - id: results
+ outputSource:
+ - save_results/log_dir
+ type: Directory
+ 'sbg:x': 1500
+ 'sbg:y': 0
+steps:
+ - id: prefactor
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: cal_solutions
+ source: cal_solutions
+ - id: refant
+ source: refant
+ - id: flag_baselines
+ source:
+ - flag_baselines
+ - id: process_baselines_target
+ source: process_baselines_target
+ - id: filter_baselines
+ source: filter_baselines
+ - id: do_smooth
+ source: do_smooth
+ - id: rfistrategy
+ source: rfistrategy
+ - id: min_unflagged_fraction
+ source: min_unflagged_fraction
+ - id: compression_bitrate
+ source: compression_bitrate
+ - id: raw_data
+ source: raw_data
+ - id: propagatesolutions
+ source: propagatesolutions
+ - id: demix_sources
+ source:
+ - demix_sources
+ - id: demix_target
+ source: demix_target
+ - id: demix_freqstep
+ source: demix_freqstep
+ - id: demix_timestep
+ source: demix_timestep
+ - id: demix
+ source: demix
+ - id: clipATeam
+ source: clipATeam
+ - id: updateweights
+ source: updateweights
+ - id: max_dppp_threads
+ source: max_dppp_threads
+ - id: memoryperc
+ source: memoryperc
+ - id: min_length
+ source: min_length
+ - id: overhead
+ source: overhead
+ - id: min_separation
+ source: min_separation
+ - id: calibrator_path_skymodel
+ source: calibrator_path_skymodel
+ - id: A-Team_skymodel
+ source: A-Team_skymodel
+ - id: target_skymodel
+ source: target_skymodel
+ - id: use_target
+ source: use_target
+ - id: skymodel_source
+ source: skymodel_source
+ - id: avg_timeresolution
+ source: avg_timeresolution
+ - id: avg_freqresolution
+ source: avg_freqresolution
+ - id: avg_timeresolution_concat
+ source: avg_timeresolution_concat
+ - id: avg_freqresolution_concat
+ source: avg_freqresolution_concat
+ - id: num_SBs_per_group
+ source: num_SBs_per_group
+ - id: reference_stationSB
+ source: reference_stationSB
+ - id: ionex_server
+ source: ionex_server
+ - id: ionex_prefix
+ source: ionex_prefix
+ - id: proxy_server
+ source: proxy_server
+ - id: proxy_port
+ source: proxy_port
+ - id: proxy_type
+ source: proxy_type
+ - id: proxy_user
+ source: proxy_user
+ - id: proxy_pass
+ source: proxy_pass
+ out:
+ - id: logfiles
+ - id: msout
+ - id: parset
+ - id: solutions
+ - id: inspection
+ run: ./prefactor_target.cwl
+ label: prefactor_target
+ 'sbg:x': 0
+ 'sbg:y': 0
+ - id: save_logfiles
+ in:
+ - id: log_files
+ linkMerge: merge_flattened
+ source:
+ - prefactor/logfiles
+ - id: sub_directory_name
+ default: logs
+ out:
+ - id: log_dir
+ run: ./../steps/collectlog.cwl
+ label: save_logfiles
+ 'sbg:x': 1000
+ 'sbg:y': -200
+ - id: save_inspection
+ in:
+ - id: log_files
+ linkMerge: merge_flattened
+ source:
+ - prefactor/inspection
+ - id: sub_directory_name
+ default: inspection
+ out:
+ - id: log_dir
+ run: ./../steps/collectlog.cwl
+ label: save_inspection
+ 'sbg:x': 1000
+ 'sbg:y': 0
+ - id: save_solutions
+ in:
+ - id: log_files
+ source:
+ - prefactor/solutions
+ - id: sub_directory_name
+ default: cal_values
+ out:
+ - id: log_dir
+ run: ./../steps/collectlog.cwl
+ label: save_solutions
+ 'sbg:x': 1000
+ 'sbg:y': 200
+ - id: save_results
+ in:
+ - id: log_files
+ linkMerge: merge_flattened
+ source:
+ - save_solutions/log_dir
+ - save_inspection/log_dir
+ - prefactor/msout
+ - id: sub_directory_name
+ default: results
+ out:
+ - id: log_dir
+ run: ./../steps/collectlog.cwl
+ label: save_results
+ 'sbg:x': 1200
+ 'sbg:y': 0
+requirements:
+ - class: SubworkflowFeatureRequirement
+ - class: MultipleInputFeatureRequirement
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index 7986942f..972705e8 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -5,17 +5,318 @@ label: prefactor_target
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
+ - id: msin
+ type: 'Directory[]'
+ 'sbg:x': -1000
+ 'sbg:y': -1700
- id: cal_solutions
- type: File
- 'sbg:x': -327
- 'sbg:y': -42
+ type: 'File'
+ 'sbg:x': -1000
+ 'sbg:y': -1600
+ - id: refant
+ type: string?
+ default: CS001HBA0
+ 'sbg:x': -1000
+ 'sbg:y': -1500
+ - id: flag_baselines
+ type: 'string[]?'
+ default: []
+ 'sbg:x': -1000
+ 'sbg:y': -1400
+ - id: process_baselines_target
+ type: string?
+ default: '[CR]S*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1300
+ - id: filter_baselines
+ type: string?
+ default: '[CR]S*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1200
+ - id: do_smooth
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -1100
+ - id: rfistrategy
+ type: string?
+ default: HBAdefault.rfis
+ 'sbg:x': -1000
+ 'sbg:y': -1000
+ - id: min_unflagged_fraction
+ type: float?
+ default: 0.5
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: compression_bitrate
+ type: int?
+ default: 16
+ 'sbg:x': -1000
+ 'sbg:y': -800
+ - id: raw_data
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -700
+ - id: propagatesolutions
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': -600
+ - id: demix_sources
+ type: 'string[]?'
+ default:
+ - CasA
+ - CygA
+ 'sbg:x': -1000
+ 'sbg:y': -500
+ - id: demix_target
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': -400
+ - id: demix_freqstep
+ type: int?
+ default: 16
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: demix_timestep
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: demix
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: apply_steps
+ type: string?
+ default: 'applyclock,applybeam,applyRM'
+ 'sbg:x': -1000
+ 'sbg:y': 0
+ - id: clipATeam
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: gsmcal_step
+ type: string?
+ default: 'phase'
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: updateweights
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': 300
+ - id: max_dppp_threads
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': 400
+ - id: memoryperc
+ type: int?
+ default: 20
+ 'sbg:x': -1000
+ 'sbg:y': 500
+ - id: min_length
+ type: int?
+ default: 5
+ 'sbg:x': -1000
+ 'sbg:y': 600
+ - id: overhead
+ type: float?
+ default: 0.7
+ 'sbg:x': -1000
+ 'sbg:y': 700
+ - id: min_separation
+ type: int?
+ default: 30
+ 'sbg:x': -1000
+ 'sbg:y': 800
+ - id: A-Team_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 900
+ - id: target_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 1000
+ - id: use_target
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': 1100
+ - id: skymodel_source
+ type: string?
+ default: 'TGSS'
+ 'sbg:x': -1000
+ 'sbg:y': 1200
+ - id: avg_timeresolution
+ type: float?
+ default: 4.
+ 'sbg:x': -1000
+ 'sbg:y': 1300
+ - id: avg_freqresolution
+ type: string?
+ default: 48.82kHz
+ 'sbg:x': -1000
+ 'sbg:y': 1400
+ - id: avg_timeresolution_concat
+ type: float?
+ default: 8.
+ 'sbg:x': -1000
+ 'sbg:y': 1500
+ - id: avg_freqresolution_concat
+ type: string?
+ default: 97.64kHz
+ 'sbg:x': -1000
+ 'sbg:y': 1600
+ - id: num_SBs_per_group
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': 1700
+ - id: reference_stationSB
+ type: int?
+ default: None
+ 'sbg:x': -1000
+ 'sbg:y': 1800
+ - id: ionex_server
+ type: string?
+ default: 'ftp://ftp.aiub.unibe.ch/CODE/'
+ 'sbg:x': -1000
+ 'sbg:y': 1900
+ - id: ionex_prefix
+ type: string?
+ default: 'CODG'
+ 'sbg:x': -1000
+ 'sbg:y': 2000
+ - id: proxy_server
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2100
+ - id: proxy_port
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2200
+ - id: proxy_type
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2300
+ - id: proxy_user
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2400
+ - id: proxy_pass
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 2500
outputs:
+ - id: inspection
+ linkMerge: merge_flattened
+ outputSource:
+ - prep/check_Ateam_separation.png
+ - prep/RMextract.png
+ type: File[]
+ 'sbg:x': 2000
+ 'sbg:y': -600
- id: solutions
outputSource:
- - cal_solutions
+ - prep/outh5parm
type: File
- 'sbg:x': -58
- 'sbg:y': -43
- source: input.cal_solutions
-steps: []
-requirements: []
+ 'sbg:x': 2000
+ 'sbg:y': -500
+ - id: msout
+ outputSource:
+ - prep/msout
+ type: 'Directory[]'
+ 'sbg:x': 2000
+ 'sbg:y': 0
+ - id: logfiles
+ outputSource:
+ - prep/logfiles
+ type: 'File[]'
+ linkMerge: merge_flattened
+ 'sbg:x': 2000
+ 'sbg:y': 600
+steps:
+ - id: prep
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: cal_solutions
+ source: cal_solutions
+ - id: filter_baselines
+ source: filter_baselines
+ - id: raw_data
+ source: raw_data
+ - id: propagatesolutions
+ source: propagatesolutions
+ - id: updateweights
+ source: updateweights
+ - id: demix
+ source: demix
+ - id: apply_steps
+ source: apply_steps
+ - id: clipAteam
+ source: clipAteam
+ - id: max_dppp_threads
+ source: max_dppp_threads
+ - id: memoryperc
+ source: memoryperc
+ - id: flag_baselines
+ source:
+ - flag_baselines
+ - id: avg_timeresolution
+ source: avg_timeresolution
+ - id: avg_freqresolution
+ source: avg_freqresolution
+ - id: process_baselines_target
+ source: process_baselines_target
+ - id: demix_timestep
+ source: demix_timestep
+ - id: demix_freqstep
+ source: demix_freqstep
+ - id: demix_target
+ source: demix_target
+ - id: demix_sources
+ source:
+ - demix_sources
+ - id: min_separation
+ source: min_separation
+ - id: A-Team_skymodel
+ source: A-Team_skymodel
+ - id: ionex_server
+ source: ionex_server
+ - id: ionex_prefix
+ source: ionex_prefix
+ - id: proxy_server
+ source: proxy_server
+ - id: proxy_port
+ source: proxy_port
+ - id: proxy_type
+ source: proxy_type
+ - id: proxy_user
+ source: proxy_user
+ - id: proxy_pass
+ source: proxy_pass
+ out:
+ - id: outh5parm
+ - id: logfiles
+ - id: check_Ateam_separation.png
+ - id: msout
+ - id: target_name
+ run: ./prefactor_target/prep.cwl
+ label: prep
+ 'sbg:x': 0
+ 'sbg:y': 0
+requirements:
+ - class: SubworkflowFeatureRequirement
diff --git a/workflows/prefactor_target/concat.cwl b/workflows/prefactor_target/concat.cwl
new file mode 100644
index 00000000..3faf8dbb
--- /dev/null
+++ b/workflows/prefactor_target/concat.cwl
@@ -0,0 +1,381 @@
+class: Workflow
+cwlVersion: v1.0
+id: prep
+label: prep
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msin
+ type: 'Directory[]'
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: filter_baselines
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': -800
+ - id: raw_data
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -700
+ - id: propagatesolutions
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -600
+ - id: flagunconverged
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -500
+ - id: demix
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -400
+ - id: max_dppp_threads
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: memoryperc
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: flag_baselines
+ type: 'string[]'
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: avg_timeresolution
+ type: float
+ 'sbg:x': -1000
+ 'sbg:y': 0
+ - id: avg_freqresolution
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: process_baselines_cal
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: demix_timestep
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': 300
+ - id: demix_freqstep
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': 400
+ - id: demix_target
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 500
+ - id: demix_sources
+ type: 'string[]'
+ 'sbg:x': -1000
+ 'sbg:y': 600
+ - id: min_length
+ type: int?
+ 'sbg:x': -1000
+ 'sbg:y': 700
+ - id: overhead
+ type: float?
+ 'sbg:x': -1000
+ 'sbg:y': 800
+ - id: min_separation
+ type: int?
+ 'sbg:x': -1000
+ 'sbg:y': 900
+ - id: do_smooth
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': 1000
+ - id: max_separation_arcmin
+ type: float?
+ 'sbg:x': -1000
+ 'sbg:y': 1100
+ - id: calibrator_path_skymodel
+ type: Directory?
+ 'sbg:x': -1000
+ 'sbg:y': 1200
+ - id: A-Team_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 1300
+ - id: elevation
+ type: string
+ default: 0deg..20deg
+ 'sbg:x': -1000
+ 'sbg:y': 1400
+ - id: amplmin
+ type: float
+ default: 1.e-30
+ 'sbg:x': -1000
+ 'sbg:y': 1500
+outputs:
+ - id: outh5parm
+ outputSource:
+ - h5parm_collector/outh5parm
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -500
+ - id: check_Ateam_separation.png
+ outputSource:
+ - check_ateam_separation/output_imag
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -300
+ - id: msout
+ outputSource:
+ - predict_calibrate/msout
+ type: 'Directory[]'
+ 'sbg:x': 1000
+ 'sbg:y': 0
+ - id: parset
+ outputSource:
+ - ndppp_prep_cal/parset
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 200
+ - id: calibrator_name
+ outputSource:
+ - find_skymodel_cal/model_name
+ type: string
+ 'sbg:x': 1000
+ 'sbg:y': 400
+ - id: outh5parm_logfile
+ outputSource:
+ - h5parm_collector/log
+ type: File[]
+ 'sbg:x': 1000
+ 'sbg:y': 650
+ - id: logfiles
+ linkMerge: merge_flattened
+ outputSource:
+ - concat_logfiles_calib/output
+ - concat_logfiles_predict/output
+ - concat_logfiles_blsmooth/output
+ - make_sourcedb/log
+ - find_skymodel_cal/logfile
+ - concat_logfiles_prep_cal/output
+ - make_sourcedb_ateam/log
+ - check_ateam_separation/logfile
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 800
+steps:
+ - id: ndppp_prep_cal
+ in:
+ - id: baselines_to_flag
+ default: []
+ source:
+ - flag_baselines
+ - id: elevation_to_flag
+ source: elevation
+ - id: min_amplitude_to_flag
+ source: amplmin
+ - id: memoryperc
+ default: 20
+ source: memoryperc
+ - id: raw_data
+ default: false
+ source: raw_data
+ - id: demix
+ default: false
+ source: demix
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - msin
+ - id: msin_baseline
+ default: '*'
+ source: filter_baselines
+ - id: skymodel
+ source: make_sourcedb_ateam/sourcedb
+ - id: timeresolution
+ default: 1
+ source: avg_timeresolution
+ - id: freqresolution
+ default: 12.21kHz
+ source: avg_freqresolution
+ - id: demix_timestep
+ default: 1
+ source: demix_timestep
+ - id: demix_freqstep
+ default: 1
+ source: demix_freqstep
+ - id: demix_baseline
+ default: '*'
+ source: process_baselines_cal
+ - id: target_source
+ source: demix_target
+ - id: ntimechunk
+ default: 10
+ source: max_dppp_threads
+ - id: subtract_sources
+ source:
+ - demix_sources
+ out:
+ - id: msout
+ - id: logfile
+ - id: parset
+ run: ../../subworkflow/ndppp_prep_cal.cwl
+ label: ndppp_prep_cal
+ scatter:
+ - msin
+ 'sbg:x': 0
+ 'sbg:y': 0
+ - id: predict_calibrate
+ in:
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - ndppp_prep_cal/msout
+ - id: do_smooth
+ source: do_smooth
+ - id: sourcedb
+ source: make_sourcedb/sourcedb
+ - id: flagunconverged
+ source: flagunconverged
+ - id: propagatesolutions
+ source: propagatesolutions
+ out:
+ - id: msout
+ - id: BLsmooth.log
+ - id: predict_cal.log
+ - id: calib_cal.log
+ - id: outh5parm
+ run: ../../subworkflow/predict_calibrate.cwl
+ label: predict_calibrate
+ scatter:
+ - msin
+ 'sbg:x': 500
+ 'sbg:y': 0
+ - id: check_ateam_separation
+ in:
+ - id: ms
+ source:
+ - msin
+ - id: min_separation
+ source: min_separation
+ out:
+ - id: output_imag
+ - id: logfile
+ run: ../../lofar-cwl/steps/check_ateam_separation.cwl
+ label: check_Ateam_separation
+ 'sbg:x': 0
+ 'sbg:y': -300
+ - id: make_sourcedb_ateam
+ in:
+ - id: sky_model
+ source: A-Team_skymodel
+ - id: logname
+ default: make_sourcedb_ateam.log
+ out:
+ - id: sourcedb
+ - id: log
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
+ label: make_sourcedb_ateam
+ 'sbg:x': -500
+ 'sbg:y': 300
+ - id: find_skymodel_cal
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: skymodels
+ source: calibrator_path_skymodel
+ - id: max_separation_arcmin
+ source: max_separation_arcmin
+ out:
+ - id: output_models
+ - id: model_name
+ - id: logfile
+ run: ../../steps/find_skymodel_cal.cwl
+ label: find_skymodel_cal
+ 'sbg:x': -500
+ 'sbg:y': 500
+ - id: make_sourcedb
+ in:
+ - id: sky_model
+ source: find_skymodel_cal/output_models
+ - id: output_file_name
+ default: calibrator.sourcedb
+ out:
+ - id: sourcedb
+ - id: log
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
+ label: make_sourcedb
+ 'sbg:x': 0
+ 'sbg:y': 500
+ - id: concat_logfiles_prep_cal
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source: ndppp_prep_cal/logfile
+ - id: file_prefix
+ default: ndppp_prep_cal
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_prep_cal
+ 'sbg:x': 250
+ 'sbg:y': 300
+ - id: concat_logfiles_blsmooth
+ in:
+ - id: file_list
+ source:
+ - predict_calibrate/BLsmooth.log
+ - id: file_prefix
+ default: blsmooth_prep
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_blsmooth
+ 'sbg:x': 750
+ 'sbg:y': 300
+ - id: concat_logfiles_predict
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - predict_calibrate/predict_cal.log
+ - id: file_prefix
+ default: predict_cal
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_predict
+ 'sbg:x': 750
+ 'sbg:y': 500
+ - id: concat_logfiles_calib
+ in:
+ - id: file_list
+ source:
+ - predict_calibrate/calib_cal.log
+ - id: file_prefix
+ default: calib_cal_prep
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_calib
+ 'sbg:x': 750
+ 'sbg:y': 800
+ - id: h5parm_collector
+ in:
+ - id: h5parmFiles
+ source:
+ - predict_calibrate/outh5parm
+ - id: squeeze
+ default: true
+ - id: verbose
+ default: true
+ - id: clobber
+ default: true
+ out:
+ - id: outh5parm
+ - id: log
+ run: ../../lofar-cwl/steps/H5ParmCollector.cwl
+ label: H5parm_collector
+ 'sbg:x': 750
+ 'sbg:y': -300
+requirements:
+ - class: SubworkflowFeatureRequirement
+ - class: ScatterFeatureRequirement
diff --git a/workflows/prefactor_target/gsmcal.cwl b/workflows/prefactor_target/gsmcal.cwl
new file mode 100644
index 00000000..3faf8dbb
--- /dev/null
+++ b/workflows/prefactor_target/gsmcal.cwl
@@ -0,0 +1,381 @@
+class: Workflow
+cwlVersion: v1.0
+id: prep
+label: prep
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msin
+ type: 'Directory[]'
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: filter_baselines
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': -800
+ - id: raw_data
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -700
+ - id: propagatesolutions
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -600
+ - id: flagunconverged
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -500
+ - id: demix
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -400
+ - id: max_dppp_threads
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: memoryperc
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: flag_baselines
+ type: 'string[]'
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: avg_timeresolution
+ type: float
+ 'sbg:x': -1000
+ 'sbg:y': 0
+ - id: avg_freqresolution
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: process_baselines_cal
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: demix_timestep
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': 300
+ - id: demix_freqstep
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': 400
+ - id: demix_target
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 500
+ - id: demix_sources
+ type: 'string[]'
+ 'sbg:x': -1000
+ 'sbg:y': 600
+ - id: min_length
+ type: int?
+ 'sbg:x': -1000
+ 'sbg:y': 700
+ - id: overhead
+ type: float?
+ 'sbg:x': -1000
+ 'sbg:y': 800
+ - id: min_separation
+ type: int?
+ 'sbg:x': -1000
+ 'sbg:y': 900
+ - id: do_smooth
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': 1000
+ - id: max_separation_arcmin
+ type: float?
+ 'sbg:x': -1000
+ 'sbg:y': 1100
+ - id: calibrator_path_skymodel
+ type: Directory?
+ 'sbg:x': -1000
+ 'sbg:y': 1200
+ - id: A-Team_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 1300
+ - id: elevation
+ type: string
+ default: 0deg..20deg
+ 'sbg:x': -1000
+ 'sbg:y': 1400
+ - id: amplmin
+ type: float
+ default: 1.e-30
+ 'sbg:x': -1000
+ 'sbg:y': 1500
+outputs:
+ - id: outh5parm
+ outputSource:
+ - h5parm_collector/outh5parm
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -500
+ - id: check_Ateam_separation.png
+ outputSource:
+ - check_ateam_separation/output_imag
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -300
+ - id: msout
+ outputSource:
+ - predict_calibrate/msout
+ type: 'Directory[]'
+ 'sbg:x': 1000
+ 'sbg:y': 0
+ - id: parset
+ outputSource:
+ - ndppp_prep_cal/parset
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 200
+ - id: calibrator_name
+ outputSource:
+ - find_skymodel_cal/model_name
+ type: string
+ 'sbg:x': 1000
+ 'sbg:y': 400
+ - id: outh5parm_logfile
+ outputSource:
+ - h5parm_collector/log
+ type: File[]
+ 'sbg:x': 1000
+ 'sbg:y': 650
+ - id: logfiles
+ linkMerge: merge_flattened
+ outputSource:
+ - concat_logfiles_calib/output
+ - concat_logfiles_predict/output
+ - concat_logfiles_blsmooth/output
+ - make_sourcedb/log
+ - find_skymodel_cal/logfile
+ - concat_logfiles_prep_cal/output
+ - make_sourcedb_ateam/log
+ - check_ateam_separation/logfile
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 800
+steps:
+ - id: ndppp_prep_cal
+ in:
+ - id: baselines_to_flag
+ default: []
+ source:
+ - flag_baselines
+ - id: elevation_to_flag
+ source: elevation
+ - id: min_amplitude_to_flag
+ source: amplmin
+ - id: memoryperc
+ default: 20
+ source: memoryperc
+ - id: raw_data
+ default: false
+ source: raw_data
+ - id: demix
+ default: false
+ source: demix
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - msin
+ - id: msin_baseline
+ default: '*'
+ source: filter_baselines
+ - id: skymodel
+ source: make_sourcedb_ateam/sourcedb
+ - id: timeresolution
+ default: 1
+ source: avg_timeresolution
+ - id: freqresolution
+ default: 12.21kHz
+ source: avg_freqresolution
+ - id: demix_timestep
+ default: 1
+ source: demix_timestep
+ - id: demix_freqstep
+ default: 1
+ source: demix_freqstep
+ - id: demix_baseline
+ default: '*'
+ source: process_baselines_cal
+ - id: target_source
+ source: demix_target
+ - id: ntimechunk
+ default: 10
+ source: max_dppp_threads
+ - id: subtract_sources
+ source:
+ - demix_sources
+ out:
+ - id: msout
+ - id: logfile
+ - id: parset
+ run: ../../subworkflow/ndppp_prep_cal.cwl
+ label: ndppp_prep_cal
+ scatter:
+ - msin
+ 'sbg:x': 0
+ 'sbg:y': 0
+ - id: predict_calibrate
+ in:
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - ndppp_prep_cal/msout
+ - id: do_smooth
+ source: do_smooth
+ - id: sourcedb
+ source: make_sourcedb/sourcedb
+ - id: flagunconverged
+ source: flagunconverged
+ - id: propagatesolutions
+ source: propagatesolutions
+ out:
+ - id: msout
+ - id: BLsmooth.log
+ - id: predict_cal.log
+ - id: calib_cal.log
+ - id: outh5parm
+ run: ../../subworkflow/predict_calibrate.cwl
+ label: predict_calibrate
+ scatter:
+ - msin
+ 'sbg:x': 500
+ 'sbg:y': 0
+ - id: check_ateam_separation
+ in:
+ - id: ms
+ source:
+ - msin
+ - id: min_separation
+ source: min_separation
+ out:
+ - id: output_imag
+ - id: logfile
+ run: ../../lofar-cwl/steps/check_ateam_separation.cwl
+ label: check_Ateam_separation
+ 'sbg:x': 0
+ 'sbg:y': -300
+ - id: make_sourcedb_ateam
+ in:
+ - id: sky_model
+ source: A-Team_skymodel
+ - id: logname
+ default: make_sourcedb_ateam.log
+ out:
+ - id: sourcedb
+ - id: log
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
+ label: make_sourcedb_ateam
+ 'sbg:x': -500
+ 'sbg:y': 300
+ - id: find_skymodel_cal
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: skymodels
+ source: calibrator_path_skymodel
+ - id: max_separation_arcmin
+ source: max_separation_arcmin
+ out:
+ - id: output_models
+ - id: model_name
+ - id: logfile
+ run: ../../steps/find_skymodel_cal.cwl
+ label: find_skymodel_cal
+ 'sbg:x': -500
+ 'sbg:y': 500
+ - id: make_sourcedb
+ in:
+ - id: sky_model
+ source: find_skymodel_cal/output_models
+ - id: output_file_name
+ default: calibrator.sourcedb
+ out:
+ - id: sourcedb
+ - id: log
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
+ label: make_sourcedb
+ 'sbg:x': 0
+ 'sbg:y': 500
+ - id: concat_logfiles_prep_cal
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source: ndppp_prep_cal/logfile
+ - id: file_prefix
+ default: ndppp_prep_cal
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_prep_cal
+ 'sbg:x': 250
+ 'sbg:y': 300
+ - id: concat_logfiles_blsmooth
+ in:
+ - id: file_list
+ source:
+ - predict_calibrate/BLsmooth.log
+ - id: file_prefix
+ default: blsmooth_prep
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_blsmooth
+ 'sbg:x': 750
+ 'sbg:y': 300
+ - id: concat_logfiles_predict
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - predict_calibrate/predict_cal.log
+ - id: file_prefix
+ default: predict_cal
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_predict
+ 'sbg:x': 750
+ 'sbg:y': 500
+ - id: concat_logfiles_calib
+ in:
+ - id: file_list
+ source:
+ - predict_calibrate/calib_cal.log
+ - id: file_prefix
+ default: calib_cal_prep
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_calib
+ 'sbg:x': 750
+ 'sbg:y': 800
+ - id: h5parm_collector
+ in:
+ - id: h5parmFiles
+ source:
+ - predict_calibrate/outh5parm
+ - id: squeeze
+ default: true
+ - id: verbose
+ default: true
+ - id: clobber
+ default: true
+ out:
+ - id: outh5parm
+ - id: log
+ run: ../../lofar-cwl/steps/H5ParmCollector.cwl
+ label: H5parm_collector
+ 'sbg:x': 750
+ 'sbg:y': -300
+requirements:
+ - class: SubworkflowFeatureRequirement
+ - class: ScatterFeatureRequirement
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
new file mode 100644
index 00000000..06ef5cc2
--- /dev/null
+++ b/workflows/prefactor_target/prep.cwl
@@ -0,0 +1,307 @@
+class: Workflow
+cwlVersion: v1.0
+id: prep
+label: prep
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msin
+ type: 'Directory[]'
+ 'sbg:x': -1000
+ 'sbg:y': -1700
+ - id: cal_solutions
+ type: 'File'
+ 'sbg:x': -1000
+ 'sbg:y': -1600
+ - id: flag_baselines
+ type: 'string[]?'
+ default: []
+ 'sbg:x': -1000
+ 'sbg:y': -1500
+ - id: process_baselines_target
+ type: string?
+ default: '[CR]S*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1400
+ - id: filter_baselines
+ type: string?
+ default: '[CR]S*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1300
+ - id: rfistrategy
+ type: string?
+ default: HBAdefault.rfis
+ 'sbg:x': -1000
+ 'sbg:y': -1200
+ - id: raw_data
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -1100
+ - id: propagatesolutions
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': -1000
+ - id: demix_sources
+ type: 'string[]?'
+ default:
+ - CasA
+ - CygA
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: demix_target
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': -800
+ - id: demix_freqstep
+ type: int?
+ default: 16
+ 'sbg:x': -1000
+ 'sbg:y': -700
+ - id: demix_timestep
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': -600
+ - id: demix
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -500
+ - id: apply_steps
+ type: string?
+ default: 'applyclock,applybeam,applyRM'
+ 'sbg:x': -1000
+ 'sbg:y': -400
+ - id: clipATeam
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: updateweights
+ type: boolean?
+ default: true
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: max_dppp_threads
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: memoryperc
+ type: int?
+ default: 20
+ 'sbg:x': -1000
+ 'sbg:y': 0
+ - id: min_separation
+ type: int?
+ default: 30
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: A-Team_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: avg_timeresolution
+ type: float?
+ default: 4.
+ 'sbg:x': -1000
+ 'sbg:y': 300
+ - id: avg_freqresolution
+ type: string?
+ default: 48.82kHz
+ 'sbg:x': -1000
+ 'sbg:y': 400
+ - id: ionex_server
+ type: string?
+ default: 'ftp://ftp.aiub.unibe.ch/CODE/'
+ 'sbg:x': -1000
+ 'sbg:y': 500
+ - id: ionex_prefix
+ type: string?
+ default: 'CODG'
+ 'sbg:x': -1000
+ 'sbg:y': 600
+ - id: proxy_server
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 700
+ - id: proxy_port
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 800
+ - id: proxy_type
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 900
+ - id: proxy_user
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 1000
+ - id: proxy_pass
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 1100
+outputs:
+ - id: outh5parm
+ outputSource:
+ - h5parm_collector/outh5parm
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -500
+ - id: check_Ateam_separation.png
+ outputSource:
+ - check_ateam_separation/output_imag
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -300
+ - id: msout
+ outputSource:
+ - predict_calibrate/msout
+ type: 'Directory[]'
+ 'sbg:x': 1000
+ 'sbg:y': 0
+ - id: parset
+ outputSource:
+ - ndppp_prep_cal/parset
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 200
+ - id: calibrator_name
+ outputSource:
+ - find_skymodel_cal/model_name
+ type: string
+ 'sbg:x': 1000
+ 'sbg:y': 400
+ - id: outh5parm_logfile
+ outputSource:
+ - h5parm_collector/log
+ type: File[]
+ 'sbg:x': 1000
+ 'sbg:y': 650
+ - id: logfiles
+ linkMerge: merge_flattened
+ outputSource:
+ - concat_logfiles_calib/output
+ - concat_logfiles_predict/output
+ - concat_logfiles_blsmooth/output
+ - make_sourcedb/log
+ - find_skymodel_cal/logfile
+ - concat_logfiles_prep_cal/output
+ - make_sourcedb_ateam/log
+ - check_ateam_separation/logfile
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 800
+steps:
+ - id: ndppp_prep_target
+ in:
+ - id: baselines_to_flag
+ default: []
+ source:
+ - flag_baselines
+ - id: elevation_to_flag
+ source: elevation
+ - id: min_amplitude_to_flag
+ source: amplmin
+ - id: memoryperc
+ default: 20
+ source: memoryperc
+ - id: raw_data
+ default: false
+ source: raw_data
+ - id: demix
+ default: false
+ source: demix
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - msin
+ - id: msin_baseline
+ default: '*'
+ source: filter_baselines
+ - id: skymodel
+ source: make_sourcedb_ateam/sourcedb
+ - id: timeresolution
+ default: 1
+ source: avg_timeresolution
+ - id: freqresolution
+ default: 12.21kHz
+ source: avg_freqresolution
+ - id: demix_timestep
+ default: 1
+ source: demix_timestep
+ - id: demix_freqstep
+ default: 1
+ source: demix_freqstep
+ - id: demix_baseline
+ default: '*'
+ source: process_baselines_cal
+ - id: target_source
+ source: demix_target
+ - id: ntimechunk
+ default: 10
+ source: max_dppp_threads
+ - id: subtract_sources
+ source:
+ - demix_sources
+ out:
+ - id: msout
+ - id: logfile
+ - id: parset
+ run: ../../subworkflow/ndppp_prep_cal.cwl
+ label: ndppp_prep_target
+ scatter:
+ - msin
+ 'sbg:x': 0
+ 'sbg:y': 0
+ - id: check_ateam_separation
+ in:
+ - id: ms
+ source:
+ - msin
+ - id: min_separation
+ source: min_separation
+ out:
+ - id: output_imag
+ - id: logfile
+ run: ../../lofar-cwl/steps/check_ateam_separation.cwl
+ label: check_Ateam_separation
+ 'sbg:x': 0
+ 'sbg:y': -300
+ - id: make_sourcedb_ateam
+ in:
+ - id: sky_model
+ source: A-Team_skymodel
+ - id: logname
+ default: make_sourcedb_ateam.log
+ out:
+ - id: sourcedb
+ - id: log
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
+ label: make_sourcedb_ateam
+ 'sbg:x': -500
+ 'sbg:y': 300
+ - id: concat_logfiles_prep_target
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source: ndppp_prep_cal/logfile
+ - id: file_prefix
+ default: ndppp_prep_cal
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_prep_target
+ 'sbg:x': 250
+ 'sbg:y': 300
+requirements:
+ - class: SubworkflowFeatureRequirement
+ - class: ScatterFeatureRequirement
diff --git a/workflows/prefactor_target/prep_gsmcal.cwl b/workflows/prefactor_target/prep_gsmcal.cwl
new file mode 100644
index 00000000..3faf8dbb
--- /dev/null
+++ b/workflows/prefactor_target/prep_gsmcal.cwl
@@ -0,0 +1,381 @@
+class: Workflow
+cwlVersion: v1.0
+id: prep
+label: prep
+$namespaces:
+ sbg: 'https://www.sevenbridges.com/'
+inputs:
+ - id: msin
+ type: 'Directory[]'
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: filter_baselines
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': -800
+ - id: raw_data
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -700
+ - id: propagatesolutions
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -600
+ - id: flagunconverged
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -500
+ - id: demix
+ type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -400
+ - id: max_dppp_threads
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: memoryperc
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: flag_baselines
+ type: 'string[]'
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: avg_timeresolution
+ type: float
+ 'sbg:x': -1000
+ 'sbg:y': 0
+ - id: avg_freqresolution
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: process_baselines_cal
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: demix_timestep
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': 300
+ - id: demix_freqstep
+ type: int
+ 'sbg:x': -1000
+ 'sbg:y': 400
+ - id: demix_target
+ type: string
+ 'sbg:x': -1000
+ 'sbg:y': 500
+ - id: demix_sources
+ type: 'string[]'
+ 'sbg:x': -1000
+ 'sbg:y': 600
+ - id: min_length
+ type: int?
+ 'sbg:x': -1000
+ 'sbg:y': 700
+ - id: overhead
+ type: float?
+ 'sbg:x': -1000
+ 'sbg:y': 800
+ - id: min_separation
+ type: int?
+ 'sbg:x': -1000
+ 'sbg:y': 900
+ - id: do_smooth
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': 1000
+ - id: max_separation_arcmin
+ type: float?
+ 'sbg:x': -1000
+ 'sbg:y': 1100
+ - id: calibrator_path_skymodel
+ type: Directory?
+ 'sbg:x': -1000
+ 'sbg:y': 1200
+ - id: A-Team_skymodel
+ type: File?
+ 'sbg:x': -1000
+ 'sbg:y': 1300
+ - id: elevation
+ type: string
+ default: 0deg..20deg
+ 'sbg:x': -1000
+ 'sbg:y': 1400
+ - id: amplmin
+ type: float
+ default: 1.e-30
+ 'sbg:x': -1000
+ 'sbg:y': 1500
+outputs:
+ - id: outh5parm
+ outputSource:
+ - h5parm_collector/outh5parm
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -500
+ - id: check_Ateam_separation.png
+ outputSource:
+ - check_ateam_separation/output_imag
+ type: File
+ 'sbg:x': 1000
+ 'sbg:y': -300
+ - id: msout
+ outputSource:
+ - predict_calibrate/msout
+ type: 'Directory[]'
+ 'sbg:x': 1000
+ 'sbg:y': 0
+ - id: parset
+ outputSource:
+ - ndppp_prep_cal/parset
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 200
+ - id: calibrator_name
+ outputSource:
+ - find_skymodel_cal/model_name
+ type: string
+ 'sbg:x': 1000
+ 'sbg:y': 400
+ - id: outh5parm_logfile
+ outputSource:
+ - h5parm_collector/log
+ type: File[]
+ 'sbg:x': 1000
+ 'sbg:y': 650
+ - id: logfiles
+ linkMerge: merge_flattened
+ outputSource:
+ - concat_logfiles_calib/output
+ - concat_logfiles_predict/output
+ - concat_logfiles_blsmooth/output
+ - make_sourcedb/log
+ - find_skymodel_cal/logfile
+ - concat_logfiles_prep_cal/output
+ - make_sourcedb_ateam/log
+ - check_ateam_separation/logfile
+ type: 'File[]'
+ 'sbg:x': 1000
+ 'sbg:y': 800
+steps:
+ - id: ndppp_prep_cal
+ in:
+ - id: baselines_to_flag
+ default: []
+ source:
+ - flag_baselines
+ - id: elevation_to_flag
+ source: elevation
+ - id: min_amplitude_to_flag
+ source: amplmin
+ - id: memoryperc
+ default: 20
+ source: memoryperc
+ - id: raw_data
+ default: false
+ source: raw_data
+ - id: demix
+ default: false
+ source: demix
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - msin
+ - id: msin_baseline
+ default: '*'
+ source: filter_baselines
+ - id: skymodel
+ source: make_sourcedb_ateam/sourcedb
+ - id: timeresolution
+ default: 1
+ source: avg_timeresolution
+ - id: freqresolution
+ default: 12.21kHz
+ source: avg_freqresolution
+ - id: demix_timestep
+ default: 1
+ source: demix_timestep
+ - id: demix_freqstep
+ default: 1
+ source: demix_freqstep
+ - id: demix_baseline
+ default: '*'
+ source: process_baselines_cal
+ - id: target_source
+ source: demix_target
+ - id: ntimechunk
+ default: 10
+ source: max_dppp_threads
+ - id: subtract_sources
+ source:
+ - demix_sources
+ out:
+ - id: msout
+ - id: logfile
+ - id: parset
+ run: ../../subworkflow/ndppp_prep_cal.cwl
+ label: ndppp_prep_cal
+ scatter:
+ - msin
+ 'sbg:x': 0
+ 'sbg:y': 0
+ - id: predict_calibrate
+ in:
+ - id: msin
+ linkMerge: merge_flattened
+ source:
+ - ndppp_prep_cal/msout
+ - id: do_smooth
+ source: do_smooth
+ - id: sourcedb
+ source: make_sourcedb/sourcedb
+ - id: flagunconverged
+ source: flagunconverged
+ - id: propagatesolutions
+ source: propagatesolutions
+ out:
+ - id: msout
+ - id: BLsmooth.log
+ - id: predict_cal.log
+ - id: calib_cal.log
+ - id: outh5parm
+ run: ../../subworkflow/predict_calibrate.cwl
+ label: predict_calibrate
+ scatter:
+ - msin
+ 'sbg:x': 500
+ 'sbg:y': 0
+ - id: check_ateam_separation
+ in:
+ - id: ms
+ source:
+ - msin
+ - id: min_separation
+ source: min_separation
+ out:
+ - id: output_imag
+ - id: logfile
+ run: ../../lofar-cwl/steps/check_ateam_separation.cwl
+ label: check_Ateam_separation
+ 'sbg:x': 0
+ 'sbg:y': -300
+ - id: make_sourcedb_ateam
+ in:
+ - id: sky_model
+ source: A-Team_skymodel
+ - id: logname
+ default: make_sourcedb_ateam.log
+ out:
+ - id: sourcedb
+ - id: log
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
+ label: make_sourcedb_ateam
+ 'sbg:x': -500
+ 'sbg:y': 300
+ - id: find_skymodel_cal
+ in:
+ - id: msin
+ source:
+ - msin
+ - id: skymodels
+ source: calibrator_path_skymodel
+ - id: max_separation_arcmin
+ source: max_separation_arcmin
+ out:
+ - id: output_models
+ - id: model_name
+ - id: logfile
+ run: ../../steps/find_skymodel_cal.cwl
+ label: find_skymodel_cal
+ 'sbg:x': -500
+ 'sbg:y': 500
+ - id: make_sourcedb
+ in:
+ - id: sky_model
+ source: find_skymodel_cal/output_models
+ - id: output_file_name
+ default: calibrator.sourcedb
+ out:
+ - id: sourcedb
+ - id: log
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
+ label: make_sourcedb
+ 'sbg:x': 0
+ 'sbg:y': 500
+ - id: concat_logfiles_prep_cal
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source: ndppp_prep_cal/logfile
+ - id: file_prefix
+ default: ndppp_prep_cal
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_prep_cal
+ 'sbg:x': 250
+ 'sbg:y': 300
+ - id: concat_logfiles_blsmooth
+ in:
+ - id: file_list
+ source:
+ - predict_calibrate/BLsmooth.log
+ - id: file_prefix
+ default: blsmooth_prep
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_blsmooth
+ 'sbg:x': 750
+ 'sbg:y': 300
+ - id: concat_logfiles_predict
+ in:
+ - id: file_list
+ linkMerge: merge_flattened
+ source:
+ - predict_calibrate/predict_cal.log
+ - id: file_prefix
+ default: predict_cal
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_predict
+ 'sbg:x': 750
+ 'sbg:y': 500
+ - id: concat_logfiles_calib
+ in:
+ - id: file_list
+ source:
+ - predict_calibrate/calib_cal.log
+ - id: file_prefix
+ default: calib_cal_prep
+ out:
+ - id: output
+ run: ../../steps/concatenate_files.cwl
+ label: concat_logfiles_calib
+ 'sbg:x': 750
+ 'sbg:y': 800
+ - id: h5parm_collector
+ in:
+ - id: h5parmFiles
+ source:
+ - predict_calibrate/outh5parm
+ - id: squeeze
+ default: true
+ - id: verbose
+ default: true
+ - id: clobber
+ default: true
+ out:
+ - id: outh5parm
+ - id: log
+ run: ../../lofar-cwl/steps/H5ParmCollector.cwl
+ label: H5parm_collector
+ 'sbg:x': 750
+ 'sbg:y': -300
+requirements:
+ - class: SubworkflowFeatureRequirement
+ - class: ScatterFeatureRequirement
--
GitLab