From e0e46a481d8cc6c1cbcce7c7c8e8b0d2a73f7313 Mon Sep 17 00:00:00 2001
From: Alexander Drabent <alex@transitix>
Date: Thu, 16 Jan 2020 11:50:29 +0100
Subject: [PATCH] restructure workflows
Former-commit-id: f2405ede1f1107fd118862e2972404ef2c6d36ed
---
workflows/HBA_calibrator.cwl | 63 ++-
workflows/prefactor_calibrator.cwl | 373 +++++++++++-------
.../prep.cwl} | 6 +-
3 files changed, 252 insertions(+), 190 deletions(-)
rename workflows/{prefactor_calibrator_test.cwl => prefactor_calibrator/prep.cwl} (95%)
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index 101392e7..841a3e46 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -108,128 +108,123 @@ inputs:
default: 10
'sbg:x': -1000
'sbg:y': 200
- - id: raw_data
- type: boolean?
- default: false
- 'sbg:x': -1000
- 'sbg:y': 300
- id: demix
type: boolean?
default: false
'sbg:x': -1000
- 'sbg:y': 400
+ 'sbg:y': 300
- id: ion_3rd
type: boolean?
default: false
'sbg:x': -1000
- 'sbg:y': 500
+ 'sbg:y': 400
- id: clock_smooth
type: boolean?
default: true
'sbg:x': -1000
- 'sbg:y': 600
+ 'sbg:y': 500
- id: tables2export
type: string?
default: clock
'sbg:x': -1000
- 'sbg:y': 700
+ 'sbg:y': 600
- id: final_apply
type: boolean?
default: false
'sbg:x': -1000
- 'sbg:y': 800
+ 'sbg:y': 700
- id: memoryperc
type: int?
default: 20
'sbg:x': -1000
- 'sbg:y': 900
+ 'sbg:y': 800
- id: min_length
type: int?
default: 50
'sbg:x': -1000
- 'sbg:y': 1000
+ 'sbg:y': 900
- id: overhead
type: float?
default: 0.8
'sbg:x': -1000
- 'sbg:y': 1100
+ 'sbg:y': 1000
- id: min_separation
type: int?
default: 30
'sbg:x': -1000
- 'sbg:y': 1200
+ 'sbg:y': 1100
- id: max_separation_arcmin
type: float?
default: 1
'sbg:x': -1000
- 'sbg:y': 1300
+ 'sbg:y': 1200
- id: calibrator_path_skymodel
type: Directory?
default: /data/skymodels
'sbg:x': -1000
- 'sbg:y': 1400
+ 'sbg:y': 1300
- id: A-Team_sky_model
type: File?
default: /data/skymodels/Ateam_LBA_CC.skymodel
'sbg:x': -1000
- 'sbg:y': 1500
+ 'sbg:y': 1400
- id: cal_solutions
type: string?
default: cal_solutions.h5
'sbg:x': -1000
- 'sbg:y': 1600
+ 'sbg:y': 1500
- id: avg_timeresolution
type: float?
default: 4
'sbg:x': -1000
- 'sbg:y': 1700
+ 'sbg:y': 1600
- id: avg_freqresolution
type: string?
default: 48.82kHz
'sbg:x': -1000
- 'sbg:y': 1800
+ 'sbg:y': 1700
- id: bandpass_freqresolution
type: string?
default: 195.3125kHz
'sbg:x': -1000
- 'sbg:y': 1900
+ 'sbg:y': 1800
outputs:
- id: msout
outputSource:
- prefactor/msout
type: 'Directory[]'
- 'sbg:x': 289.5600891113281
- 'sbg:y': -144.01869201660156
+ 'sbg:x': 1000
+ 'sbg:y': -200
- id: make_sourcedb_ateam.log
outputSource:
- prefactor/make_sourcedb_ateam.log
type: File?
- 'sbg:x': 317.4496154785156
- 'sbg:y': 9.373631477355957
+ 'sbg:x': 1000
+ 'sbg:y': -100
- id: logfile_prep_cal
outputSource:
- prefactor/logfile_prep_cal
type: 'File[]?'
- 'sbg:x': 315.0741882324219
- 'sbg:y': 180.26284790039062
+ 'sbg:x': 1000
+ 'sbg:y': 0
- id: check_Ateam_separation.png
outputSource:
- prefactor/check_Ateam_separation.png
type: 'File[]?'
- 'sbg:x': 287.6455078125
- 'sbg:y': 344.1313171386719
+ 'sbg:x': 1000
+ 'sbg:y': 100
- id: check_Ateam_separation.log
outputSource:
- prefactor/check_Ateam_separation.log
type: 'File[]?'
- 'sbg:x': 315.5941467285156
- 'sbg:y': 494.9656982421875
+ 'sbg:x': 1000
+ 'sbg:y': 200
- id: parset
outputSource:
- prefactor/parset
type: 'File[]'
- 'sbg:x': 293.33331298828125
- 'sbg:y': -305.4627990722656
+ 'sbg:x': 1000
+ 'sbg:y': 300
steps:
- id: prefactor
in:
@@ -312,7 +307,7 @@ steps:
- id: msout
- id: logfile_prep_cal
- id: parset
- run: ./prefactor_calibrator_test.cwl
+ run: ./prefactor_calibrator.cwl
label: prefactor_calibrator
'sbg:x': 0
'sbg:y': 0
diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl
index 5eeb2026..48d87871 100644
--- a/workflows/prefactor_calibrator.cwl
+++ b/workflows/prefactor_calibrator.cwl
@@ -5,64 +5,189 @@ label: prefactor_calibrator
$namespaces:
sbg: 'https://www.sevenbridges.com/'
inputs:
- - id: raw_data
- type: boolean
- 'sbg:x': -634
- 'sbg:y': -688
- - id: demix
- type: boolean
- 'sbg:x': -639.3313598632812
- 'sbg:y': -271
- id: msin
type: 'Directory[]'
- 'sbg:x': -635
- 'sbg:y': -543
- - id: skymodels
- type: Directory
- 'sbg:x': -123
- 'sbg:y': -850
- - id: memoryperc
- type: int
- default: 20
- 'sbg:x': -634
- 'sbg:y': -411
- - id: baselines_to_flag
- type: 'string[]'
+ 'sbg:x': -1000
+ 'sbg:y': -1700
+ - id: refant
+ type: string?
+ default: CS001HBA0
+ 'sbg:x': -1000
+ 'sbg:y': -1600
+ - id: flag_baselines
+ type: 'string[]?'
default: []
- 'sbg:x': -635.8651123046875
- 'sbg:y': -148
- - id: uvw
- type: File?
- 'sbg:x': -356.9723205566406
- 'sbg:y': -804.0709838867188
- - id: strategy
- type: File?
- 'sbg:x': -352
- 'sbg:y': -713
- - id: propagate_solutions
- type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -1500
+ - id: process_baselines_cal
+ type: string?
+ default: '*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1400
+ - id: filter_baselines
+ type: string?
+ default: '*&'
+ 'sbg:x': -1000
+ 'sbg:y': -1300
+ - id: do_smooth
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -1200
+ - id: rfistrategy
+ type: string?
+ default: HBAdefault.rfis
+ 'sbg:x': -1000
+ 'sbg:y': -1100
+ - id: max2interpolate
+ type: int?
+ default: 30
+ 'sbg:x': -1000
+ 'sbg:y': -1000
+ - id: ampRange
+ type: 'int[]?'
+ default:
+ - 0
+ - 0
+ 'sbg:x': -1000
+ 'sbg:y': -900
+ - id: skip_international
+ type: boolean?
default: true
- 'sbg:x': -112
- 'sbg:y': -737
- - id: flagunconverged
- type: boolean
+ 'sbg:x': -1000
+ 'sbg:y': -800
+ - id: raw_data
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -700
+ - id: propagatesolutions
+ type: boolean?
default: true
- 'sbg:x': -92
+ 'sbg:x': -1000
+ 'sbg:y': -600
+ - id: flagunconverged
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': -500
+ - id: maxStddev
+ type: float?
+ default: -1.0
+ 'sbg:x': -1000
'sbg:y': -400
- - id: do_smooth
- type: boolean
+ - id: solutions2transfer
+ type: File?
+ default: /data/solutions/3C48.h5
+ 'sbg:x': -1000
+ 'sbg:y': -300
+ - id: antennas2transfer
+ type: File?
+ default: '[FUSPID].*'
+ 'sbg:x': -1000
+ 'sbg:y': -200
+ - id: demix_sources
+ type: 'string[]?'
+ default:
+ - CasA
+ - CygA
+ 'sbg:x': -1000
+ 'sbg:y': -100
+ - id: demix_target
+ type: string?
+ default: ''
+ 'sbg:x': -1000
+ 'sbg:y': 0
+ - id: demix_freqstep
+ type: int?
+ default: 16
+ 'sbg:x': -1000
+ 'sbg:y': 100
+ - id: demix_timestep
+ type: int?
+ default: 10
+ 'sbg:x': -1000
+ 'sbg:y': 200
+ - id: demix
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': 300
+ - id: ion_3rd
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': 400
+ - id: clock_smooth
+ type: boolean?
default: true
- 'sbg:x': -89
- 'sbg:y': -276
- - id: min_separation
- type: int
+ 'sbg:x': -1000
+ 'sbg:y': 500
+ - id: tables2export
+ type: string?
+ default: clock
+ 'sbg:x': -1000
+ 'sbg:y': 600
+ - id: final_apply
+ type: boolean?
+ default: false
+ 'sbg:x': -1000
+ 'sbg:y': 700
+ - id: memoryperc
+ type: int?
default: 20
- 'sbg:x': -638.4987182617188
- 'sbg:y': 31.238996505737305
+ 'sbg:x': -1000
+ 'sbg:y': 800
+ - id: min_length
+ type: int?
+ default: 50
+ 'sbg:x': -1000
+ 'sbg:y': 900
+ - id: overhead
+ type: float?
+ default: 0.8
+ 'sbg:x': -1000
+ 'sbg:y': 1000
+ - id: min_separation
+ type: int?
+ default: 30
+ 'sbg:x': -1000
+ 'sbg:y': 1100
+ - id: max_separation_arcmin
+ type: float?
+ default: 1
+ 'sbg:x': -1000
+ 'sbg:y': 1200
+ - id: calibrator_path_skymodel
+ type: Directory?
+ default: /data/skymodels
+ 'sbg:x': -1000
+ 'sbg:y': 1300
- id: A-Team_sky_model
- type: string
- 'sbg:x': -889.9927978515625
- 'sbg:y': -845.9927978515625
+ type: File?
+ default: /data/skymodels/Ateam_LBA_CC.skymodel
+ 'sbg:x': -1000
+ 'sbg:y': 1400
+ - id: cal_solutions
+ type: string?
+ default: cal_solutions.h5
+ 'sbg:x': -1000
+ 'sbg:y': 1500
+ - id: avg_timeresolution
+ type: float?
+ default: 4
+ 'sbg:x': -1000
+ 'sbg:y': 1600
+ - id: avg_freqresolution
+ type: string?
+ default: 48.82kHz
+ 'sbg:x': -1000
+ 'sbg:y': 1700
+ - id: bandpass_freqresolution
+ type: string?
+ default: 195.3125kHz
+ 'sbg:x': -1000
+ 'sbg:y': 1800
outputs:
- id: outh5parm
outputSource:
@@ -95,121 +220,63 @@ outputs:
'sbg:x': 495.6245422363281
'sbg:y': -1257.4857177734375
steps:
- - id: ndppp_prep_cal
+ - id: prep
in:
- - id: baselines_to_flag
- default: []
- source:
- - baselines_to_flag
- - id: elevation_to_flag
- default: 0deg..20deg
- - id: min_amplitude_to_flag
- default: 1.e-30
- - id: memoryperc
- default: 20
- source: memoryperc
+ - id: msin
+ source: msin
+ - id: flag_baselines
+ source: flag_baselines
+ - id: process_baselines_cal
+ source: process_baselines_cal
+ - id: filter_baselines
+ source: filter_baselines
+ - id: do_smooth
+ source: do_smooth
+ - id: rfistrategy
+ source: rfistrategy
- id: raw_data
source: raw_data
+ - id: demix_sources
+ source: demix_sources
+ - id: demix_target
+ source: demix_target
+ - id: demix_freqstep
+ source: demix_freqstep
+ - id: demix_timestep
+ source: demix_timestep
+ - id: demix_target
+ source: demix_target
+ - id: demix_target
+ source: demix_target
- id: demix
source: demix
- - id: msin
- linkMerge: merge_flattened
- source:
- - msin
- - id: skymodel
- source: make_sourcedb/sourcedb
- - id: timeresolution
- default: 1
- - id: freqresolution
- default: 1
+ - id: memoryperc
+ source: memoryperc
+ - id: min_length
+ source: min_length
+ - id: overhead
+ source: overhead
+ - id: min_separation
+ source: min_separation
+ - id: max_separation_arcmin
+ source: max_separation_arcmin
+ - id: calibrator_path_skymodel
+ source: calibrator_path_skymodel
+ - id: A-Team_skymodel
+ source: A-Team_skymodel
+ - id: avg_timeresolution
+ source: avg_timeresolution
+ - id: avg_freqresolution
+ source: avg_freqresolution
out:
- id: msout
- run: ../subworkflow/ndppp_prep_cal.cwl
- label: ndppp_prep_cal
+ run: prefactor_calibrator/prep.cwl
+ label: prep
scatter:
- msin
'sbg:x': -359
'sbg:y': -519
- - id: aoflagger
- in:
- - id: msin
- source:
- - ndppp_prep_cal/msout
- - id: strategy
- source: strategy
- - id: uvw
- source: uvw
- out:
- - id: output_ms
- run: ../lofar-cwl/steps/AOFlagger.cwl
- label: AOFlagger
- scatterMethod: flat_crossproduct
- 'sbg:x': -164
- 'sbg:y': -565
- - id: calib_cal
- in:
- - id: msin
- source: aoflagger/output_ms
- - id: skymodels
- source: skymodels
- - id: flagunconverged
- source: flagunconverged
- - id: propagate_solutions
- source: propagate_solutions
- - id: do_smooth
- source: do_smooth
- out:
- - id: h5parm
- - id: msout
- run: ../subworkflow/calib_cal.cwl
- label: calib_cal
- scatter:
- - msin
- 'sbg:x': 47
- 'sbg:y': -583
- - id: h5parm_collector
- in:
- - id: h5parmFiles
- source:
- - calib_cal/h5parm
- - id: squeeze
- default: true
- - id: verbose
- default: true
- - id: clobber
- default: true
- out:
- - id: outh5parm
- - id: log
- run: ../lofar-cwl/steps/H5ParmCollector.cwl
- label: H5parm_collector
- 'sbg:x': 292.1979675292969
- 'sbg:y': -543.7208251953125
- - id: check_ateam_separation
- in:
- - id: ms
- source:
- - msin
- - id: min_separation
- source: min_separation
- out:
- - id: output_imag
- - id: logfile
- run: ../lofar-cwl/steps/check_ateam_separation.cwl
- label: check_Ateam_separation
- 'sbg:x': -331.452392578125
- 'sbg:y': 23.596216201782227
- - id: make_sourcedb
- in:
- - id: sky_model
- source: A-Team_sky_model
- out:
- - id: sourcedb
- - id: log
- run: ../lofar-cwl/steps/makesourcedb.cwl
- label: make_sourcedb_ateam
- 'sbg:x': -628.7400512695312
- 'sbg:y': -847.117919921875
+
requirements:
- class: SubworkflowFeatureRequirement
- class: ScatterFeatureRequirement
diff --git a/workflows/prefactor_calibrator_test.cwl b/workflows/prefactor_calibrator/prep.cwl
similarity index 95%
rename from workflows/prefactor_calibrator_test.cwl
rename to workflows/prefactor_calibrator/prep.cwl
index aa2db155..135c8bd4 100644
--- a/workflows/prefactor_calibrator_test.cwl
+++ b/workflows/prefactor_calibrator/prep.cwl
@@ -113,7 +113,7 @@ steps:
- id: msout
- id: logfile
- id: parset
- run: ../subworkflow/ndppp_prep_cal.cwl
+ run: ../../subworkflow/ndppp_prep_cal.cwl
label: ndppp_prep_cal
scatter:
- msin
@@ -129,7 +129,7 @@ steps:
out:
- id: output_imag
- id: logfile
- run: ../lofar-cwl/steps/check_ateam_separation.cwl
+ run: ../../lofar-cwl/steps/check_ateam_separation.cwl
label: check_Ateam_separation
'sbg:x': -331.452392578125
'sbg:y': 23.596216201782227
@@ -140,7 +140,7 @@ steps:
out:
- id: sourcedb
- id: log
- run: ../lofar-cwl/steps/makesourcedb.cwl
+ run: ../../lofar-cwl/steps/makesourcedb.cwl
label: make_sourcedb_ateam
'sbg:x': -647.0108642578125
'sbg:y': -981.3153076171875
--
GitLab