From fd7749331db2e416bf8d19083edec09ba7265690 Mon Sep 17 00:00:00 2001
From: alex <alex@tls-tautenburg.de>
Date: Fri, 11 Mar 2022 13:34:33 +0000
Subject: [PATCH] automized demixing selection (RAP-414)
---
.gitlab-ci.yml | 4 +-
docs/source/calibrator.rst | 5 +-
docs/source/parset.rst | 5 +-
docs/source/target.rst | 5 +-
scripts/make_summary.py | 7 +-
steps/H5ParmCollector.cwl | 2 +-
steps/average.cwl | 2 +-
steps/check_demix.cwl | 127 ++++++++++++++++++
steps/concatenate_files.cwl | 2 +-
steps/dppp_make_parset_cal.cwl | 14 +-
steps/dppp_make_parset_target.cwl | 4 +-
steps/dpppconcat.cwl | 2 +-
steps/filter_ms_group.cwl | 2 +-
steps/filter_predict.cwl | 2 +-
steps/gaincal.cwl | 2 +-
steps/plot_unflagged.cwl | 2 +-
steps/predict.cwl | 2 +-
steps/summary.cwl | 2 +-
steps/uvplot.cwl | 2 +-
steps/wsclean.cwl | 6 +-
workflows/HBA_calibrator.cwl | 33 +++--
workflows/HBA_target.cwl | 20 +--
workflows/LBA_calibrator.cwl | 33 +++--
workflows/prefactor_calibrator.cwl | 32 +++--
workflows/prefactor_calibrator/bandpass.cwl | 4 +-
workflows/prefactor_calibrator/bp.cwl | 16 +--
workflows/prefactor_calibrator/clocktec.cwl | 2 +-
.../prefactor_calibrator/faraday_rotation.cwl | 2 +-
workflows/prefactor_calibrator/fr.cwl | 16 +--
workflows/prefactor_calibrator/ion.cwl | 10 +-
.../prefactor_calibrator/ndppp_prep_cal.cwl | 14 +-
workflows/prefactor_calibrator/pa.cwl | 16 +--
workflows/prefactor_calibrator/pol_align.cwl | 2 +-
workflows/prefactor_calibrator/prep.cwl | 64 ++++++---
workflows/prefactor_target.cwl | 21 +--
workflows/prefactor_target/concat.cwl | 2 +-
workflows/prefactor_target/finalize.cwl | 14 +-
workflows/prefactor_target/gsmcal.cwl | 10 +-
.../prefactor_target/ndppp_prep_targ.cwl | 6 +-
workflows/prefactor_target/prep.cwl | 49 ++++---
40 files changed, 390 insertions(+), 175 deletions(-)
create mode 100644 steps/check_demix.cwl
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a2af2a52..ba7555d4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,7 +121,7 @@ run_hba_calibrator:
needs: ["versioning", "download_data"]
image: $INTEGRATION_IMAGE
script:
- - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --preserve-environment OPENBLAS_NUM_THREADS --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
+ - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_calibrator/ workflows/HBA_calibrator.cwl test_jobs/HBA_calibrator.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_calibrator
after_script:
- find /tmp/run_hba_calibrator -name "*.log" -print0 | tar czf hba_calibrator_logs.tar.gz --null -T -
@@ -135,7 +135,7 @@ run_hba_target:
needs: ["versioning", "download_data"]
image: $INTEGRATION_IMAGE
script:
- - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --preserve-environment OPENBLAS_NUM_THREADS --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
+ - cwltool --no-container --preserve-environment PATH --preserve-environment PREFACTOR_DATA_ROOT --preserve-environment PYTHONPATH --outdir results --leave-tmpdir --tmpdir-prefix /tmp/run_hba_target/ workflows/HBA_target.cwl test_jobs/HBA_target.json
- test_jobs/check_workflow_results.py results /builds/eosc/prefactor3-cwl/data/results_target
after_script:
- find /tmp/run_hba_target -name "*.log" -print0 | tar czf hba_target_logs.tar.gz --null -T -
diff --git a/docs/source/calibrator.rst b/docs/source/calibrator.rst
index 34fd5720..50dfb813 100644
--- a/docs/source/calibrator.rst
+++ b/docs/source/calibrator.rst
@@ -258,10 +258,11 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
*Demixing options* (only used if demix step is added to the ``prep_cal_strategy`` variable)
- ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
-- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
- ``demix_freqstep``: number of channels to average when demixing (default: 16)
- ``demix_timestep`` : number of time slots to average when demixing (default: 10)
-- ``demix``: enable demixing (default: ``false``)
+- ``demix``: if ``true`` force demixing using all sources of ``demix_sources``, if ``false`` do not demix (default: ``null``, automatically determines sources to be demixed according to ``min_separation``)
+- ``lbfgs_historysize``: for the LBFGS solver: the history size, specified as a multiple of the parameter vector, to use to approximate the inverse Hessian (default: 10)
+- ``lbfgs_robustdof``: for the LBFGS solver: the degrees of freedom (DOF) given to the noise model (default: 200)
*Further pipeline options*
diff --git a/docs/source/parset.rst b/docs/source/parset.rst
index cf6a442e..23002c60 100644
--- a/docs/source/parset.rst
+++ b/docs/source/parset.rst
@@ -61,7 +61,6 @@ There are more parameters you may want to adjust that can be added to this input
"do_transfer": false,
"trusted_sources": "3C48,3C147,3C196,3C295,3C380",
"demix_sources": ["CasA","CygA"],
- "demix_target": "",
"demix_freqstep": 16,
"demix_timestep": 10,
"demix": false,
@@ -78,7 +77,9 @@ There are more parameters you may want to adjust that can be added to this input
"A-Team_skymodel": null,
"avg_timeresolution": 4,
"avg_freqresolution": "48.82kHz",
- "bandpass_freqresolution": "195.3125kHz"
+ "bandpass_freqresolution": "195.3125kHz",
+ "lbfgs_historysize" : 10,
+ "lbfgs_robustdof" : 200
}
If you just want to alter one of the defaults it is sufficient to override it by specifing its new value the JSON input file::
diff --git a/docs/source/target.rst b/docs/source/target.rst
index 45f7b567..f8edfc87 100644
--- a/docs/source/target.rst
+++ b/docs/source/target.rst
@@ -280,10 +280,11 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
*Demixing and clipping options*
- ``demix_sources``: choose sources to demix (provided as list), e.g., ``[CasA,CygA]``
-- ``demix_target``: if given, the target source model (its patch in the SourceDB) is taken into account when solving (default: ``""``)
- ``demix_freqstep``: number of channels to average when demixing (default: 16)
- ``demix_timestep`` : number of time slots to average when demixing (default: 10)
-- ``demix``: enable demixing (default: ``false``)
+- ``demix``: if ``true`` force demixing using all sources of ``demix_sources``, if ``false`` do not demix (default: ``null``, automatically determines sources to be demixed according to ``min_separation``)
+- ``lbfgs_historysize``: for the LBFGS solver: the history size, specified as a multiple of the parameter vector, to use to approximate the inverse Hessian (default: 10)
+- ``lbfgs_robustdof``: for the LBFGS solver: the degrees of freedom (DOF) given to the noise model (default: 200)
- ``clip_sources``: list of the skymodel patches to be used for Ateamclipping (default: ``[VirA_4_patch,CygAGG,CasA_4_patch,TauAGG]``)
*Further pipeline options*
diff --git a/scripts/make_summary.py b/scripts/make_summary.py
index 61e9572e..0fb56fae 100755
--- a/scripts/make_summary.py
+++ b/scripts/make_summary.py
@@ -91,10 +91,9 @@ def main(flagFiles = None, pipeline = 'prefactor', run_type = 'calibrator', filt
demix_list = []
for i in range(len(json_output['metrics'][pipeline]['close_sources'])):
Ateam_name = json_output['metrics'][pipeline]['close_sources'][i]['source']
- if demix:
- if Ateam_name in demix_sources:
- json_output['metrics'][pipeline]['close_sources'][i]['mitigation'] = 'demix'
- demix_list.append(Ateam_name)
+ if Ateam_name in demix_sources and demix:
+ json_output['metrics'][pipeline]['close_sources'][i]['mitigation'] = 'demix'
+ demix_list.append(Ateam_name)
elif Ateam_name in clip_sources:
json_output['metrics'][pipeline]['close_sources'][i]['mitigation'] = 'clip'
clip_list.append(Ateam_name)
diff --git a/steps/H5ParmCollector.cwl b/steps/H5ParmCollector.cwl
index 04f836df..d1d295d3 100755
--- a/steps/H5ParmCollector.cwl
+++ b/steps/H5ParmCollector.cwl
@@ -12,7 +12,7 @@ baseCommand:
inputs:
- id: h5parmFiles
type:
- - 'File[]'
+ - File[]
- File
format: lofar:#H5Parm
inputBinding:
diff --git a/steps/average.cwl b/steps/average.cwl
index 8990ba9c..7db00116 100644
--- a/steps/average.cwl
+++ b/steps/average.cwl
@@ -79,7 +79,7 @@ outputs:
outputBinding:
glob: $(inputs.msout_name)
- id: logfile
- type: 'File[]'
+ type: File[]
outputBinding:
glob: average*.log
arguments:
diff --git a/steps/check_demix.cwl b/steps/check_demix.cwl
new file mode 100644
index 00000000..ccbe3f08
--- /dev/null
+++ b/steps/check_demix.cwl
@@ -0,0 +1,127 @@
+class: CommandLineTool
+cwlVersion: v1.2
+id: check_demix
+baseCommand:
+ - python3
+ - check_demix.py
+inputs:
+ - id: skymodel
+ type: File?
+ - id: demix_sources
+ type: string[]?
+ default:
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
+ - id: clip_sources
+ type: string[]?
+ default:
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
+ - id: demix
+ type: boolean?
+ inputBinding:
+ valueFrom: '$(self ? 1 : 0)'
+ - id: Ateam_separation_file
+ type: File
+
+outputs:
+ - id: out_demix
+ type: boolean
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: '$(JSON.parse(self[0].contents).demix == 0 ? false : true)'
+ - id: out_demix_sources
+ type: string[]
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: '$(JSON.parse(self[0].contents).demix_sources)'
+ - id: out_clip_sources
+ type: string[]
+ outputBinding:
+ loadContents: true
+ glob: 'out.json'
+ outputEval: '$(JSON.parse(self[0].contents).clip_sources)'
+ - id: logfile
+ type: File
+ outputBinding:
+ glob: check_demix.log
+label: check_demix
+
+requirements:
+ - class: InlineJavascriptRequirement
+ - class: InitialWorkDirRequirement
+ listing:
+ - entryname: check_demix.py
+ entry: |
+ import json
+ import os
+ import lsmtool
+ import numpy, math
+ null = None
+
+ targets = {'CasA' : {'ra' : 6.123487680622104, 'dec' : 1.0265153995604648}, \
+ 'CygA' : {'ra' : 5.233686575770755, 'dec' : 0.7109409582180791}, \
+ 'TauA' : {'ra' : 1.4596748493730913, 'dec' : 0.38422502335921294}, \
+ 'HerA' : {'ra' : 4.4119087330382163, 'dec' : 0.087135562905816893}, \
+ 'VirA' : {'ra' : 3.276086511413598, 'dec' : 0.21626589533567378}, \
+ 'Sun' : {'ra' : 0, 'dec' : 0 }, \
+ 'Jupiter': {'ra' : 0, 'dec' : 0 }, \
+ 'Moon' : {'ra' : 0, 'dec' : 0 }}
+
+ inputs = json.loads(r"""$(inputs)""")
+
+ demix = inputs['demix']
+ demix_sources = inputs['demix_sources']
+ clip_sources = inputs['clip_sources']
+ skymodel = inputs['skymodel']
+ Ateam_separation_file = inputs['Ateam_separation_file']["path"]
+
+ if skymodel is None:
+ skymodel = os.path.expandvars("$PREFACTOR_DATA_ROOT/skymodels/Ateam_LBA_CC.skymodel")
+ else:
+ skymodel = skymodel["path"]
+ s = lsmtool.load(skymodel)
+
+ f = open(Ateam_separation_file, 'r')
+ Ateam_separation = json.load(f)
+ f.close()
+
+ if demix is None:
+ demix = 0
+ demix_list = []
+ for item in Ateam_separation:
+ ra = math.degrees(targets[item['source']]['ra'])
+ dec = math.degrees(targets[item['source']]['dec'])
+ dist_deg = s.getDistance(ra, dec, byPatch=True)
+ if any(dist_deg * 60.0 < 6.0):
+ for index in numpy.where(dist_deg * 60 < 6.0)[0]:
+ patch_name = s.getPatchNames()[index]
+ if patch_name in demix_sources:
+ demix_list.append(patch_name)
+ demix = 1
+ demix_sources = demix_list
+ if demix == 1:
+ clip_sources = list(set(clip_sources).difference(demix_sources))
+
+ print("The following sources will be demixed: ", demix_sources)
+ print("The following sources will be clipped: ", clip_sources)
+
+ cwl_output = {"demix_sources" : demix_sources,
+ "clip_sources" : clip_sources,
+ "demix" : demix}
+
+ with open('./out.json', 'w') as fp:
+ json.dump(cwl_output, fp)
+
+hints:
+ - class: DockerRequirement
+ dockerPull: lofareosc/prefactor3-cwl
+
+stdout: check_demix.log
+stderr: check_demix_err.log
diff --git a/steps/concatenate_files.cwl b/steps/concatenate_files.cwl
index fa7b040b..e6bfca3b 100644
--- a/steps/concatenate_files.cwl
+++ b/steps/concatenate_files.cwl
@@ -6,7 +6,7 @@ baseCommand:
- bulk_rename.sh
inputs:
- id: file_list
- type: 'File[]'
+ type: File[]
inputBinding:
position: 0
- id: file_prefix
diff --git a/steps/dppp_make_parset_cal.cwl b/steps/dppp_make_parset_cal.cwl
index ffa398f7..8f56cbd5 100755
--- a/steps/dppp_make_parset_cal.cwl
+++ b/steps/dppp_make_parset_cal.cwl
@@ -16,7 +16,7 @@ inputs:
type: int?
default: 20
- id: baselines_to_flag
- type: 'string[]?'
+ type: string[]?
default: []
- id: elevation_to_flag
type: string?
@@ -43,13 +43,19 @@ inputs:
type: string?
default: ''
- id: subtract_sources
- type: 'string[]?'
+ type: string[]?
default:
- CasA
- CygA
- id: ntimechunk
type: int?
default: 10
+ - id: lbfgs_historysize
+ type: int?
+ default: 10
+ - id: lbfgs_robustdof
+ type: float?
+ default: 200
outputs:
- id: output
type: File
@@ -113,3 +119,7 @@ requirements:
demix.freqstep = 1
demix.timestep = 1
demix.instrumentmodel = instrument
+ demix.uselbfgssolver = True
+ demix.lbfgs.historysize = $(inputs.lbfgs_historysize)
+ demix.lbfgs.robustdof = $(inputs.lbfgs_robustdof)
+ demix.maxiter = 20
diff --git a/steps/dppp_make_parset_target.cwl b/steps/dppp_make_parset_target.cwl
index 99a17384..9b61b28e 100755
--- a/steps/dppp_make_parset_target.cwl
+++ b/steps/dppp_make_parset_target.cwl
@@ -31,7 +31,7 @@ inputs:
type: int?
default: 20
- id: baselines_to_flag
- type: 'string[]?'
+ type: string[]?
default: []
- id: elevation_to_flag
type: string?
@@ -61,7 +61,7 @@ inputs:
type: string?
default: 'pointing'
- id: subtract_sources
- type: 'string[]?'
+ type: string[]?
default:
- CasA
- CygA
diff --git a/steps/dpppconcat.cwl b/steps/dpppconcat.cwl
index 09d87e2b..951ad967 100644
--- a/steps/dpppconcat.cwl
+++ b/steps/dpppconcat.cwl
@@ -141,7 +141,7 @@ outputs:
glob: $(inputs.jsonfilename)
outputEval: $(JSON.parse(self[0].contents).flagged_fraction_dict)
- id: logfile
- type: 'File[]'
+ type: File[]
outputBinding:
glob: concat*.log
arguments:
diff --git a/steps/filter_ms_group.cwl b/steps/filter_ms_group.cwl
index c5aff4c4..140b0885 100644
--- a/steps/filter_ms_group.cwl
+++ b/steps/filter_ms_group.cwl
@@ -14,7 +14,7 @@ inputs:
inputBinding:
position: 1
- id: measurement_sets
- type: 'Directory[]'
+ type: Directory[]
inputBinding:
position: 2
outputs:
diff --git a/steps/filter_predict.cwl b/steps/filter_predict.cwl
index 4dd0561d..08732bc4 100644
--- a/steps/filter_predict.cwl
+++ b/steps/filter_predict.cwl
@@ -36,7 +36,7 @@ inputs:
separate: false
- default: null
id: sources
- type: 'string[]?'
+ type: string[]?
inputBinding:
position: 0
prefix: predict.sources=
diff --git a/steps/gaincal.cwl b/steps/gaincal.cwl
index bf62f431..b2cb425c 100644
--- a/steps/gaincal.cwl
+++ b/steps/gaincal.cwl
@@ -142,7 +142,7 @@ inputs:
separate: false
- default: null
id: blrange
- type: 'int[]?'
+ type: int[]?
inputBinding:
position: 0
prefix: filter.blrange=
diff --git a/steps/plot_unflagged.cwl b/steps/plot_unflagged.cwl
index 5e742ad9..9f2b2371 100644
--- a/steps/plot_unflagged.cwl
+++ b/steps/plot_unflagged.cwl
@@ -5,7 +5,7 @@ baseCommand:
- python3
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
inputBinding:
position: 1
- id: unflagged_fraction
diff --git a/steps/predict.cwl b/steps/predict.cwl
index fe120d0a..727c74fb 100644
--- a/steps/predict.cwl
+++ b/steps/predict.cwl
@@ -36,7 +36,7 @@ inputs:
separate: false
- default: null
id: sources
- type: 'string[]?'
+ type: string[]?
inputBinding:
position: 0
prefix: predict.sources=
diff --git a/steps/summary.cwl b/steps/summary.cwl
index 579ab8da..8fd9609f 100644
--- a/steps/summary.cwl
+++ b/steps/summary.cwl
@@ -8,7 +8,7 @@ baseCommand:
inputs:
- id: flagFiles
type:
- - 'File[]'
+ - File[]
- File
inputBinding:
position: 1
diff --git a/steps/uvplot.cwl b/steps/uvplot.cwl
index e95d7281..50adbec7 100644
--- a/steps/uvplot.cwl
+++ b/steps/uvplot.cwl
@@ -8,7 +8,7 @@ baseCommand:
inputs:
- id: MSfiles
type:
- - 'Directory[]'
+ - Directory[]
- Directory
inputBinding:
position: 2
diff --git a/steps/wsclean.cwl b/steps/wsclean.cwl
index cceaf9b8..e44d3321 100644
--- a/steps/wsclean.cwl
+++ b/steps/wsclean.cwl
@@ -5,7 +5,7 @@ baseCommand:
- wsclean
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
inputBinding:
position: 2
shellQuote: false
@@ -15,7 +15,7 @@ inputs:
default:
- 2500
- 2500
- type: 'int[]'
+ type: int[]
inputBinding:
position: 1
shellQuote: false
@@ -206,7 +206,7 @@ outputs:
outputBinding:
glob: [$(inputs.image_name)-MFS-image.fits, $(inputs.image_name)-image.fits]
- id: logfile
- type: 'File[]'
+ type: File[]
outputBinding:
glob: 'wsclean*.log'
label: WSClean
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index a989afee..f32f2f00 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -4,12 +4,12 @@ id: HBA_calibrator
label: HBA_calibrator
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: refant
type: string?
default: 'CS00.*'
- id: flag_baselines
- type: 'string[]?'
+ type: string[]?
default: []
- id: process_baselines_cal
type: string?
@@ -32,7 +32,7 @@ inputs:
type: int?
default: 30
- id: ampRange
- type: 'float[]?'
+ type: float[]?
default:
- 0
- 0
@@ -63,13 +63,12 @@ inputs:
type: string
default: '3C48,3C147,3C196,3C295,3C380'
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default:
- - CasA
- - CygA
- - id: demix_target
- type: string?
- default: ''
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
- id: demix_freqstep
type: int?
default: 16
@@ -78,7 +77,6 @@ inputs:
default: 10
- id: demix
type: boolean?
- default: false
- id: ion_3rd
type: boolean?
default: false
@@ -119,6 +117,12 @@ inputs:
- id: bandpass_freqresolution
type: string?
default: 195.3125kHz
+ - id: lbfgs_historysize
+ type: int?
+ default: 10
+ - id: lbfgs_robustdof
+ type: float?
+ default: 200
outputs:
- id: log_files
outputSource:
@@ -181,10 +185,7 @@ steps:
- id: trusted_sources
source: trusted_sources
- id: demix_sources
- source:
- - demix_sources
- - id: demix_target
- source: demix_target
+ source: demix_sources
- id: demix_freqstep
source: demix_freqstep
- id: demix_timestep
@@ -219,6 +220,10 @@ steps:
source: avg_freqresolution
- id: bandpass_freqresolution
source: bandpass_freqresolution
+ - id: lbfgs_historysize
+ source: lbfgs_historysize
+ - id: lbfgs_robustdof
+ source: lbfgs_robustdof
out:
- id: logfiles
- id: solutions
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 6a20b74c..2cb2ff69 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -4,14 +4,14 @@ id: HBA_target
label: HBA_target
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: cal_solutions
- type: 'File'
+ type: File
- id: refant
type: string?
default: 'CS00.*'
- id: flag_baselines
- type: 'string[]?'
+ type: string[]?
default: []
- id: process_baselines_target
type: string?
@@ -39,10 +39,12 @@ inputs:
type: boolean?
default: true
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default:
- - CasA
- - CygA
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
- id: demix_freqstep
type: int?
default: 16
@@ -51,7 +53,6 @@ inputs:
default: 10
- id: demix
type: boolean?
- default: false
- id: apply_tec
type: boolean?
default: false
@@ -141,7 +142,7 @@ inputs:
type: string?
default: null
- id: clip_sources
- type: 'string[]?'
+ type: string[]?
default:
- VirA_4_patch
- CygAGG
@@ -217,8 +218,7 @@ steps:
- id: apply_beam
source: apply_beam
- id: demix_sources
- source:
- - demix_sources
+ source: demix_sources
- id: demix_freqstep
source: demix_freqstep
- id: demix_timestep
diff --git a/workflows/LBA_calibrator.cwl b/workflows/LBA_calibrator.cwl
index baf35ae2..0bfa3a33 100644
--- a/workflows/LBA_calibrator.cwl
+++ b/workflows/LBA_calibrator.cwl
@@ -4,12 +4,12 @@ id: HBA_calibrator
label: HBA_calibrator
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: refant
type: string?
default: 'CS00.*'
- id: flag_baselines
- type: 'string[]?'
+ type: string[]?
default: []
- id: process_baselines_cal
type: string?
@@ -32,7 +32,7 @@ inputs:
type: int?
default: 30
- id: ampRange
- type: 'float[]?'
+ type: float[]?
default:
- 0
- 0
@@ -63,13 +63,12 @@ inputs:
type: string
default: '3C48,3C147,3C196,3C295,3C380'
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default:
- - CasA
- - CygA
- - id: demix_target
- type: string?
- default: ''
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
- id: demix_freqstep
type: int?
default: 16
@@ -78,7 +77,6 @@ inputs:
default: 10
- id: demix
type: boolean?
- default: false
- id: ion_3rd
type: boolean?
default: true
@@ -119,6 +117,12 @@ inputs:
- id: bandpass_freqresolution
type: string?
default: 195.3125kHz
+ - id: lbfgs_historysize
+ type: int?
+ default: 10
+ - id: lbfgs_robustdof
+ type: float?
+ default: 200
outputs:
- id: log_files
outputSource:
@@ -181,10 +185,7 @@ steps:
- id: trusted_sources
source: trusted_sources
- id: demix_sources
- source:
- - demix_sources
- - id: demix_target
- source: demix_target
+ source: demix_sources
- id: demix_freqstep
source: demix_freqstep
- id: demix_timestep
@@ -219,6 +220,10 @@ steps:
source: avg_freqresolution
- id: bandpass_freqresolution
source: bandpass_freqresolution
+ - id: lbfgs_historysize
+ source: lbfgs_historysize
+ - id: lbfgs_robustdof
+ source: lbfgs_robustdof
out:
- id: logfiles
- id: solutions
diff --git a/workflows/prefactor_calibrator.cwl b/workflows/prefactor_calibrator.cwl
index 637da953..ade7b7be 100644
--- a/workflows/prefactor_calibrator.cwl
+++ b/workflows/prefactor_calibrator.cwl
@@ -4,12 +4,12 @@ id: prefactor_calibrator
label: prefactor_calibrator
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: refant
type: string?
default: 'CS00.*'
- id: flag_baselines
- type: 'string[]?'
+ type: string[]?
default: []
- id: process_baselines_cal
type: string?
@@ -31,7 +31,7 @@ inputs:
type: int?
default: 30
- id: ampRange
- type: 'float[]?'
+ type: float[]?
default:
- 0
- 0
@@ -62,13 +62,12 @@ inputs:
type: string
default: '3C48,3C147,3C196,3C295'
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default:
- - CasA
- - CygA
- - id: demix_target
- type: string?
- default: ''
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
- id: demix_freqstep
type: int?
default: 16
@@ -77,7 +76,6 @@ inputs:
default: 10
- id: demix
type: boolean?
- default: false
- id: ion_3rd
type: boolean?
default: false
@@ -118,6 +116,12 @@ inputs:
- id: bandpass_freqresolution
type: string?
default: 195.3125kHz
+ - id: lbfgs_historysize
+ type: int?
+ default: 10
+ - id: lbfgs_robustdof
+ type: float?
+ default: 200
outputs:
- id: inspection
linkMerge: merge_flattened
@@ -140,7 +144,7 @@ outputs:
- fr/logfiles
- bp/logfiles
- ion/logfiles
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: summary_file
outputSource:
@@ -192,8 +196,6 @@ steps:
source: demix_timestep
- id: demix_freqstep
source: demix_freqstep
- - id: demix_target
- source: demix_target
- id: demix_sources
source:
- demix_sources
@@ -213,6 +215,10 @@ steps:
source: calibrator_path_skymodel
- id: rfistrategy
source: rfistrategy
+ - id: lbfgs_historysize
+ source: lbfgs_historysize
+ - id: lbfgs_robustdof
+ source: lbfgs_robustdof
out:
- id: outh5parm
- id: logfiles
diff --git a/workflows/prefactor_calibrator/bandpass.cwl b/workflows/prefactor_calibrator/bandpass.cwl
index 8ffda35f..f9d6294a 100644
--- a/workflows/prefactor_calibrator/bandpass.cwl
+++ b/workflows/prefactor_calibrator/bandpass.cwl
@@ -4,7 +4,7 @@ id: losoto_bandpass
label: losoto_bandpass
inputs:
- id: ampRange
- type: 'float[]?'
+ type: float[]?
default: [0, 0]
- id: skipInternational
type: boolean?
@@ -37,7 +37,7 @@ outputs:
- bandpass/logfile
- interp/logfile
- smoothb/logfile
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
steps:
- id: duplicateAbkp
diff --git a/workflows/prefactor_calibrator/bp.cwl b/workflows/prefactor_calibrator/bp.cwl
index caa80538..59228532 100644
--- a/workflows/prefactor_calibrator/bp.cwl
+++ b/workflows/prefactor_calibrator/bp.cwl
@@ -10,7 +10,7 @@ inputs:
type: string?
default: 'POINTING'
- id: ampRange
- type: 'float[]?'
+ type: float[]?
- id: skipInternational
type: boolean?
default: true
@@ -30,13 +30,13 @@ inputs:
type: boolean?
default: true
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: h5parm
type: File
- id: solutions2transfer
type: File?
- id: inh5parm_logfile
- type: 'File[]'
+ type: File[]
- id: do_smooth
type: boolean?
default: false
@@ -68,7 +68,7 @@ outputs:
- losoto_plot_B2/output_plots
- losoto_plot_B3/output_plots
- transfer_solutions/plots
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: outsolutions
outputSource:
@@ -83,12 +83,12 @@ outputs:
- concat_logfiles_apply_pa/output
- concat_logfiles_apply_fr/output
- concat_logfiles_beam/output
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: outh5parm_logfile
outputSource:
- h5parm_collector/log
- type: 'File[]'
+ type: File[]
steps:
- id: bandpass
in:
@@ -108,7 +108,7 @@ steps:
out:
- id: output_h5parm
- id: logfiles
- run: .//bandpass.cwl
+ run: ./bandpass.cwl
label: bandpass
- id: losoto_plot_A1
in:
@@ -367,7 +367,7 @@ steps:
- id: apply_pa.log
- id: applybeam.log
- id: final_flags_out
- run: .//apply_calibrate_bp.cwl
+ run: ./apply_calibrate_bp.cwl
label: apply_calibrate_bp
scatter:
- msin
diff --git a/workflows/prefactor_calibrator/clocktec.cwl b/workflows/prefactor_calibrator/clocktec.cwl
index 8f57a186..4423e917 100644
--- a/workflows/prefactor_calibrator/clocktec.cwl
+++ b/workflows/prefactor_calibrator/clocktec.cwl
@@ -30,7 +30,7 @@ outputs:
- duplicateCbkp/log
- smooth/logfile
- losoto_flagstation/log
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: parset
outputSource:
diff --git a/workflows/prefactor_calibrator/faraday_rotation.cwl b/workflows/prefactor_calibrator/faraday_rotation.cwl
index 988f97cf..1ab8a516 100644
--- a/workflows/prefactor_calibrator/faraday_rotation.cwl
+++ b/workflows/prefactor_calibrator/faraday_rotation.cwl
@@ -20,7 +20,7 @@ outputs:
outputSource:
- losoto_duplicate/log
- losoto_faraday/log
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
steps:
- id: losoto_duplicate
diff --git a/workflows/prefactor_calibrator/fr.cwl b/workflows/prefactor_calibrator/fr.cwl
index 123f4534..7e699024 100644
--- a/workflows/prefactor_calibrator/fr.cwl
+++ b/workflows/prefactor_calibrator/fr.cwl
@@ -10,14 +10,14 @@ inputs:
type: boolean?
default: true
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: h5parm
type: File
- id: refant
type: string?
default: 'CS001HBA0'
- id: inh5parm_logfile
- type: 'File[]'
+ type: File[]
- id: do_smooth
type: boolean?
default: false
@@ -31,7 +31,7 @@ outputs:
- id: msout
outputSource:
- apply_calibrate_fr/msout
- type: 'Directory[]'
+ type: Directory[]
- id: inspection
outputSource:
- losoto_plot_P3/output_plots
@@ -39,7 +39,7 @@ outputs:
- losoto_plot_Rot3/output_plots
- losoto_plot_A3/output_plots
- losoto_plot_fr/output_plots
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: outsolutions
outputSource:
@@ -51,12 +51,12 @@ outputs:
- concat_logfiles_calib/output
- concat_logfiles_blsmooth/output
- concat_logfiles_apply/output
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: outh5parm_logfile
outputSource:
- h5parm_collector/log
- type: 'File[]'
+ type: File[]
steps:
- id: faraday_rot
in:
@@ -68,7 +68,7 @@ steps:
out:
- id: output_h5parm
- id: logfiles
- run: .//faraday_rotation.cwl
+ run: ./faraday_rotation.cwl
label: FaradayRot
- id: losoto_plot_P3
in:
@@ -251,7 +251,7 @@ steps:
- id: apply_cal.log
- id: calib_cal.log
- id: outh5parm
- run: .//apply_calibrate_fr.cwl
+ run: ./apply_calibrate_fr.cwl
label: apply_calibrate_fr
scatter:
- msin
diff --git a/workflows/prefactor_calibrator/ion.cwl b/workflows/prefactor_calibrator/ion.cwl
index 9ff08b4e..1e282334 100644
--- a/workflows/prefactor_calibrator/ion.cwl
+++ b/workflows/prefactor_calibrator/ion.cwl
@@ -7,7 +7,7 @@ inputs:
type: string?
default: '*&'
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default:
- CasA
- CygA
@@ -34,7 +34,7 @@ inputs:
type: string?
default: 'clock'
- id: inh5parm_logfile
- type: 'File[]'
+ type: File[]
- id: maxStddev
type: float?
default: -1.0
@@ -54,7 +54,7 @@ outputs:
- losoto_plot_tec3rd/output_plots
- losoto_plot_clock/output_plots
- losoto_plot_Pr/output_plots
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: outsolutions
outputSource:
@@ -64,7 +64,7 @@ outputs:
outputSource:
- concat_logfiles_ion/output
- concat_logfiles_summary/output
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
steps:
- id: clocktec
@@ -81,7 +81,7 @@ steps:
- id: output_h5parm
- id: logfiles
- id: parset
- run: .//clocktec.cwl
+ run: ./clocktec.cwl
label: clocktec
- id: losoto_plot_P3
in:
diff --git a/workflows/prefactor_calibrator/ndppp_prep_cal.cwl b/workflows/prefactor_calibrator/ndppp_prep_cal.cwl
index 6c112cc4..30788a4f 100644
--- a/workflows/prefactor_calibrator/ndppp_prep_cal.cwl
+++ b/workflows/prefactor_calibrator/ndppp_prep_cal.cwl
@@ -4,7 +4,7 @@ id: ndppp_prep_cal
label: ndppp_prep_cal
inputs:
- id: baselines_to_flag
- type: 'string[]?'
+ type: string[]?
default: []
- id: elevation_to_flag
type: string?
@@ -52,8 +52,14 @@ inputs:
type: int?
default: 16
- id: subtract_sources
- type: 'string[]?'
+ type: string[]?
default: []
+ - id: lbfgs_historysize
+ type: int?
+ default: 10
+ - id: lbfgs_robustdof
+ type: float?
+ default: 200
outputs:
- id: msout
outputSource:
@@ -102,6 +108,10 @@ steps:
- subtract_sources
- id: ntimechunk
source: ntimechunk
+ - id: lbfgs_historysize
+ source: lbfgs_historysize
+ - id: lbfgs_robustdof
+ source: lbfgs_robustdof
out:
- id: output
run: ../../steps/dppp_make_parset_cal.cwl
diff --git a/workflows/prefactor_calibrator/pa.cwl b/workflows/prefactor_calibrator/pa.cwl
index 57f279e3..b01b559f 100644
--- a/workflows/prefactor_calibrator/pa.cwl
+++ b/workflows/prefactor_calibrator/pa.cwl
@@ -10,14 +10,14 @@ inputs:
type: boolean?
default: true
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: h5parm
type: File
- id: refant
type: string?
default: 'CS001HBA0'
- id: inh5parm_logfile
- type: 'File[]'
+ type: File[]
- id: do_smooth
type: boolean?
default: false
@@ -32,7 +32,7 @@ outputs:
- id: msout
outputSource:
- apply_calibrate_pa/msout
- type: 'Directory[]'
+ type: Directory[]
- id: inspection
outputSource:
- losoto_plot_P3/output_plots
@@ -42,7 +42,7 @@ outputs:
- losoto_plot_Align/output_plots
- losoto_plot_Pr/output_plots
- losoto_plot_Pr2/output_plots
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: outsolutions
outputSource:
@@ -55,12 +55,12 @@ outputs:
- concat_logfiles_blsmooth/output
- concat_logfiles_beam/output
- concat_logfiles_apply/output
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: outh5parm_logfile
outputSource:
- h5parm_collector/log
- type: 'File[]'
+ type: File[]
steps:
- id: pol_align
in:
@@ -74,7 +74,7 @@ steps:
out:
- id: output_h5parm
- id: logfiles
- run: .//pol_align.cwl
+ run: ./pol_align.cwl
label: PolAlign
- id: losoto_plot_P3
in:
@@ -330,7 +330,7 @@ steps:
- id: calib_cal.log
- id: outh5parm
- id: applybeam.log
- run: .//apply_calibrate_pa.cwl
+ run: ./apply_calibrate_pa.cwl
label: apply_calibrate_pa
scatter:
- msin
diff --git a/workflows/prefactor_calibrator/pol_align.cwl b/workflows/prefactor_calibrator/pol_align.cwl
index b71b286e..7f0f7410 100644
--- a/workflows/prefactor_calibrator/pol_align.cwl
+++ b/workflows/prefactor_calibrator/pol_align.cwl
@@ -25,7 +25,7 @@ outputs:
- losoto_polalign/log
- losoto_residual/log
linkMerge: merge_flattened
- type: 'File[]'
+ type: File[]
steps:
- id: losoto_duplicate
diff --git a/workflows/prefactor_calibrator/prep.cwl b/workflows/prefactor_calibrator/prep.cwl
index 08b8364a..fc6fa6e0 100644
--- a/workflows/prefactor_calibrator/prep.cwl
+++ b/workflows/prefactor_calibrator/prep.cwl
@@ -4,7 +4,7 @@ id: prep
label: prep
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: filter_baselines
type: string?
default: '*&'
@@ -19,7 +19,6 @@ inputs:
default: false
- id: demix
type: boolean?
- default: false
- id: max_dppp_threads
type: int?
default: 10
@@ -27,7 +26,7 @@ inputs:
type: int?
default: 20
- id: flag_baselines
- type: 'string[]?'
+ type: string[]?
default: []
- id: avg_timeresolution
type: int?
@@ -44,11 +43,8 @@ inputs:
- id: demix_freqstep
type: int?
default: 16
- - id: demix_target
- type: string?
- default: ''
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default: []
- id: min_length
type: int?
@@ -79,6 +75,12 @@ inputs:
type:
- File?
- string?
+ - id: lbfgs_historysize
+ type: int?
+ default: 10
+ - id: lbfgs_robustdof
+ type: float?
+ default: 200
outputs:
- id: outh5parm
outputSource:
@@ -91,7 +93,7 @@ outputs:
- id: msout
outputSource:
- predict_calibrate/msout
- type: 'Directory[]'
+ type: Directory[]
- id: calibrator_name
outputSource:
- find_skymodel_cal/model_name
@@ -99,7 +101,7 @@ outputs:
- id: outh5parm_logfile
outputSource:
- h5parm_collector/log
- type: 'File[]'
+ type: File[]
- id: check_Ateam_separation.json
outputSource:
- check_ateam_separation/output_json
@@ -123,8 +125,9 @@ outputs:
- concat_logfiles_prep_cal/output
- make_sourcedb_ateam/log
- check_ateam_separation/logfile
+ - check_demix/logfile
- aoflag/logfile
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
steps:
- id: select
@@ -153,8 +156,7 @@ steps:
default: false
source: raw_data
- id: demix
- default: false
- source: demix
+ source: check_demix/out_demix
- id: msin
linkMerge: merge_flattened
source:
@@ -180,18 +182,21 @@ steps:
default: '*'
source: process_baselines_cal
- id: target_source
- source: demix_target
+ source: find_skymodel_cal/model_name
- id: ntimechunk
default: 10
source: max_dppp_threads
- id: subtract_sources
- source:
- - demix_sources
+ source: check_demix/out_demix_sources
+ - id: lbfgs_historysize
+ source: lbfgs_historysize
+ - id: lbfgs_robustdof
+ source: lbfgs_robustdof
out:
- id: msout
- id: flagged_fraction_dict
- id: logfile
- run: .//ndppp_prep_cal.cwl
+ run: ./ndppp_prep_cal.cwl
label: ndppp_prep_cal
scatter:
- msin
@@ -216,7 +221,7 @@ steps:
- id: calib_cal.log
- id: flagged_fraction_dict
- id: outh5parm
- run: .//predict_calibrate.cwl
+ run: ./predict_calibrate.cwl
label: predict_calibrate
scatter:
- msin
@@ -271,12 +276,37 @@ steps:
- id: logfile
run: ../../steps/check_ateam_separation.cwl
label: check_Ateam_separation
+ - id: check_demix
+ in:
+ - id: demix
+ source: demix
+ - id: skymodel
+ source: A-Team_skymodel
+ - id: demix_sources
+ source: demix_sources
+ - id: clip_sources
+ default: []
+ - id: Ateam_separation_file
+ source: check_ateam_separation/output_json
+ out:
+ - id: out_demix
+ - id: out_demix_sources
+ - id: logfile
+ run: ../../steps/check_demix.cwl
+ label: check_demix
- id: make_sourcedb_ateam
in:
- id: sky_model
source: A-Team_skymodel
+ - id: output_file_name
+ source: make_sourcedb/sourcedb
+ valueFrom: $(self.basename)
+ - id: input_sourcedb
+ source: make_sourcedb/sourcedb
- id: logname
default: make_sourcedb_ateam.log
+ - id: append
+ default: true
out:
- id: sourcedb
- id: log
diff --git a/workflows/prefactor_target.cwl b/workflows/prefactor_target.cwl
index 22b94148..8de0b30f 100644
--- a/workflows/prefactor_target.cwl
+++ b/workflows/prefactor_target.cwl
@@ -4,14 +4,14 @@ id: prefactor_target
label: prefactor_target
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: cal_solutions
type: File
- id: refant
type: string?
default: CS00.*
- id: flag_baselines
- type: 'string[]?'
+ type: string[]?
default: []
- id: process_baselines_target
type: string?
@@ -39,10 +39,12 @@ inputs:
type: boolean?
default: true
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default:
- - CasA
- - CygA
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
- id: demix_freqstep
type: int?
default: 16
@@ -51,7 +53,6 @@ inputs:
default: 10
- id: demix
type: boolean?
- default: false
- id: apply_tec
type: boolean?
default: false
@@ -141,7 +142,7 @@ inputs:
type: string?
default: null
- id: clip_sources
- type: 'string[]?'
+ type: string[]?
default:
- VirA_4_patch
- CygAGG
@@ -162,7 +163,7 @@ outputs:
- prep/inspection
- gsmcal/inspection
- finalize/inspection
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: solutions
outputSource:
@@ -171,13 +172,13 @@ outputs:
- id: msout
outputSource:
- finalize/msout
- type: 'Directory[]'
+ type: Directory[]
- id: logfiles
outputSource:
- prep/logfiles
- gsmcal/logfiles
- finalize/logfiles
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: summary_file
outputSource:
diff --git a/workflows/prefactor_target/concat.cwl b/workflows/prefactor_target/concat.cwl
index d9e92bab..bbae3ea8 100644
--- a/workflows/prefactor_target/concat.cwl
+++ b/workflows/prefactor_target/concat.cwl
@@ -4,7 +4,7 @@ id: concat
label: concat
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: group_id
type: string
- id: groups_specification
diff --git a/workflows/prefactor_target/finalize.cwl b/workflows/prefactor_target/finalize.cwl
index 99fe4979..6ace0840 100644
--- a/workflows/prefactor_target/finalize.cwl
+++ b/workflows/prefactor_target/finalize.cwl
@@ -10,19 +10,19 @@ inputs:
type: float?
default: 0.5
- id: removed_bands
- type: 'string[]?'
+ type: string[]?
default: []
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default: []
- id: demix
type: boolean?
default: false
- id: clip_sources
- type: 'string[]?'
+ type: string[]?
default: []
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: input_h5parm
type: File
- id: inh5parm_logfile
@@ -67,7 +67,7 @@ outputs:
- id: msout
outputSource:
- apply_gsmcal/msout
- type: 'Directory[]'
+ type: Directory[]
- id: solutions
outputSource:
- h5parm_pointingname/outh5parm
@@ -80,14 +80,14 @@ outputs:
- concat_logfiles_wsclean/output
- concat_logfiles_summary/output
- concat_logfiles_uvplot/output
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: inspection
outputSource:
- structure_function/structure_plot
- wsclean/image
- uvplot/output_image
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: summary_file
outputSource:
diff --git a/workflows/prefactor_target/gsmcal.cwl b/workflows/prefactor_target/gsmcal.cwl
index 0b2bbf47..4b3d8440 100644
--- a/workflows/prefactor_target/gsmcal.cwl
+++ b/workflows/prefactor_target/gsmcal.cwl
@@ -4,7 +4,7 @@ id: gsmcal
label: gsmcal
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: filter_baselines
type: string?
default: '[CR]S*&'
@@ -50,7 +50,7 @@ outputs:
- id: msout
outputSource:
- calibrate_target/msout
- type: 'Directory[]'
+ type: Directory[]
- id: outh5parm
outputSource:
- h5parm_collector/outh5parm
@@ -74,7 +74,7 @@ outputs:
- losoto_plot_Pd/output_plots
- losoto_plot_Pd2/output_plots
- plot_unflagged/output_imag
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: out_refant
outputSource:
@@ -91,12 +91,12 @@ outputs:
- concat_logfiles_unflagged/output
- ms_concat/logfile
- aoflag/logfile
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: removed_bands
outputSource:
- check_unflagged_fraction/filenames
- type: 'string[]'
+ type: string[]
- id: total_bandwidth
outputSource:
- sort_times_into_freqGroups/total_bandwidth
diff --git a/workflows/prefactor_target/ndppp_prep_targ.cwl b/workflows/prefactor_target/ndppp_prep_targ.cwl
index 20ef8cd8..4804805a 100644
--- a/workflows/prefactor_target/ndppp_prep_targ.cwl
+++ b/workflows/prefactor_target/ndppp_prep_targ.cwl
@@ -4,7 +4,7 @@ id: ndppp_prep_targ
label: ndppp_prep_targ
inputs:
- id: baselines_to_flag
- type: 'string[]?'
+ type: string[]?
default: []
- id: elevation_to_flag
type: string?
@@ -49,7 +49,7 @@ inputs:
type: int?
default: 16
- id: subtract_sources
- type: 'string[]?'
+ type: string[]?
default: []
- id: parmdb
type: File
@@ -75,7 +75,7 @@ inputs:
type: boolean?
default: true
- id: clip_sources
- type: 'string[]?'
+ type: string[]?
default:
- VirA_4_patch
- CygAGG
diff --git a/workflows/prefactor_target/prep.cwl b/workflows/prefactor_target/prep.cwl
index c1292c68..3ac1ab69 100644
--- a/workflows/prefactor_target/prep.cwl
+++ b/workflows/prefactor_target/prep.cwl
@@ -4,11 +4,11 @@ id: prep
label: prep
inputs:
- id: msin
- type: 'Directory[]'
+ type: Directory[]
- id: cal_solutions
type: File
- id: flag_baselines
- type: 'string[]?'
+ type: string[]?
default: []
- id: process_baselines_target
type: string?
@@ -20,17 +20,19 @@ inputs:
type: boolean?
default: false
- id: clip_sources
- type: 'string[]?'
+ type: string[]?
default:
- VirA_4_patch
- CygAGG
- CasA_4_patch
- TauAGG
- id: demix_sources
- type: 'string[]?'
+ type: string[]?
default:
- - CasA
- - CygA
+ - VirA_4_patch
+ - CygAGG
+ - CasA_4_patch
+ - TauAGG
- id: demix_freqstep
type: int?
default: 16
@@ -39,7 +41,6 @@ inputs:
default: 10
- id: demix
type: boolean?
- default: false
- id: apply_tec
type: boolean?
default: false
@@ -136,12 +137,12 @@ outputs:
- check_ateam_separation/output_imag
- losoto_plot_RM/output_plots
- plot_Ateamclipper/output_imag
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: msout
outputSource:
- dppp_prep_target/msout
- type: 'Directory[]'
+ type: Directory[]
- id: check_Ateam_separation.json
outputSource:
- check_ateam_separation/output_json
@@ -152,12 +153,13 @@ outputs:
- make_sourcedb_target/log
- make_sourcedb_ateam/log
- check_ateam_separation/logfile
+ - check_demix/logfile
- concat_logfiles_stationlist/output
- concat_logfiles_RMextract/output
- concat_logfiles_prep_targ/output
- concat_logfiles_predict_targ/output
- concat_logfiles_clipper_targ/output
- type: 'File[]'
+ type: File[]
linkMerge: merge_flattened
- id: initial_flags_join_out
outputSource:
@@ -267,6 +269,25 @@ steps:
- id: logfile
run: ../../steps/check_ateam_separation.cwl
label: check_Ateam_separation
+ - id: check_demix
+ in:
+ - id: demix
+ source: demix
+ - id: skymodel
+ source: A-Team_skymodel
+ - id: demix_sources
+ source: demix_sources
+ - id: clip_sources
+ source: clip_sources
+ - id: Ateam_separation_file
+ source: check_ateam_separation/output_json
+ out:
+ - id: out_demix
+ - id: out_demix_sources
+ - id: out_clip_sources
+ - id: logfile
+ run: ../../steps/check_demix.cwl
+ label: check_demix
- id: compare_station_list
in:
- id: msin
@@ -344,7 +365,7 @@ steps:
- id: raw_data
source: raw_data
- id: demix
- source: demix
+ source: check_demix/out_demix
- id: msin
linkMerge: merge_flattened
source:
@@ -364,8 +385,7 @@ steps:
- id: ntimechunk
source: max_dppp_threads
- id: subtract_sources
- source:
- - demix_sources
+ source: check_demix/out_demix_sources
- id: parmdb
source: createRMh5parm/h5parmout
- id: apply_tec_correction
@@ -383,8 +403,7 @@ steps:
- id: updateweights
source: updateweights
- id: clip_sources
- source:
- - clip_sources
+ source: check_demix/out_clip_sources
- id: target_source
source: targetname
- id: lbfgs_historysize
--
GitLab