From 3dcea9061574c4959f031e8500d93e70f70ed1cb Mon Sep 17 00:00:00 2001
From: alex <alex@tls-tautenburg.de>
Date: Fri, 19 Aug 2022 12:06:32 +0000
Subject: [PATCH] Use time chunking in aoflagger/LINC (RAP-29)
---
docs/source/calibrator.rst | 3 ++-
docs/source/parset.rst | 3 ++-
docs/source/target.rst | 3 ++-
steps/aoflag.cwl | 7 +++++++
workflows/HBA_calibrator.cwl | 5 +++++
workflows/HBA_target.cwl | 5 +++++
workflows/LBA_calibrator.cwl | 5 +++++
workflows/linc_calibrator.cwl | 5 +++++
workflows/linc_calibrator/prep.cwl | 5 +++++
workflows/linc_target.cwl | 5 +++++
workflows/linc_target/gsmcal.cwl | 5 +++++
11 files changed, 48 insertions(+), 3 deletions(-)
diff --git a/docs/source/calibrator.rst b/docs/source/calibrator.rst
index a97b18d7..d4573807 100644
--- a/docs/source/calibrator.rst
+++ b/docs/source/calibrator.rst
@@ -278,7 +278,8 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
- ``max_dp3_threads``: number of threads per process for DP3 (default: 10)
- ``memoryperc``: maximum of memory used for aoflagger in raw_flagging mode in percent (default: 20)
-- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (see the `AOFlagger manual`_`)
+- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (default: false, see the `AOFlagger manual`_`)
+- ``aoflag_chunksize``: this will split the set into intervals with the given maximum size, and flag each interval independently. This lowers the amount of memory required (default: 2000)
**Parameters you may want to adjust**
diff --git a/docs/source/parset.rst b/docs/source/parset.rst
index 75712bf6..595a6072 100644
--- a/docs/source/parset.rst
+++ b/docs/source/parset.rst
@@ -83,7 +83,8 @@ There are more parameters you may want to adjust that can be added to this input
"bandpass_freqresolution": "195.3125kHz",
"lbfgs_historysize" : 10,
"lbfgs_robustdof" : 200,
- "reorder_aoflag" : false
+ "aoflag_reorder" : false,
+ "aoflag_chunksize" : 2000
}
If you just want to alter one of the defaults it is sufficient to override it by specifing its new value the JSON input file::
diff --git a/docs/source/target.rst b/docs/source/target.rst
index d51637e6..eab993a0 100644
--- a/docs/source/target.rst
+++ b/docs/source/target.rst
@@ -304,7 +304,8 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
- ``max_dp3_threads``: number of threads per process for DP3 (default: 10)
- ``memoryperc``: maximum of memory used for aoflagger in raw_flagging mode in percent (default: 20)
-- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (see the `AOFlagger manual`_)
+- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (default: false, see the `AOFlagger manual`_`)
+- ``aoflag_chunksize``: this will split the set into intervals with the given maximum size, and flag each interval independently. This lowers the amount of memory required (default: 2000)
*Skymodel directory*
diff --git a/steps/aoflag.cwl b/steps/aoflag.cwl
index 12c46d3f..81645664 100644
--- a/steps/aoflag.cwl
+++ b/steps/aoflag.cwl
@@ -133,6 +133,13 @@ inputs:
prefix: -concatenate-frequency
position: 0
shellQuote: false
+ - id: chunk-size
+ type: int?
+ doc: This will split the set into intervals with the given maximum size, and flag each interval independently
+ inputBinding:
+ prefix: -chunk-size
+ position: 0
+ shellQuote: false
outputs:
- id: output_ms
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index 6ac44938..d49925fb 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -122,6 +122,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: log_files
outputSource:
@@ -221,6 +224,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: logfiles
- id: solutions
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 3c439b16..e939bc41 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -151,6 +151,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: calibrated_data
outputSource:
@@ -273,6 +276,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: logfiles
- id: msout
diff --git a/workflows/LBA_calibrator.cwl b/workflows/LBA_calibrator.cwl
index ba0bfd5f..eb2e0a4c 100644
--- a/workflows/LBA_calibrator.cwl
+++ b/workflows/LBA_calibrator.cwl
@@ -120,6 +120,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: log_files
outputSource:
@@ -219,6 +222,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: logfiles
- id: solutions
diff --git a/workflows/linc_calibrator.cwl b/workflows/linc_calibrator.cwl
index 6ca990f2..7168b370 100644
--- a/workflows/linc_calibrator.cwl
+++ b/workflows/linc_calibrator.cwl
@@ -121,6 +121,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: inspection
linkMerge: merge_flattened
@@ -216,6 +219,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: outh5parm
- id: logfiles
diff --git a/workflows/linc_calibrator/prep.cwl b/workflows/linc_calibrator/prep.cwl
index 0e1fbb09..fa07f983 100644
--- a/workflows/linc_calibrator/prep.cwl
+++ b/workflows/linc_calibrator/prep.cwl
@@ -80,6 +80,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: outh5parm
outputSource:
@@ -233,6 +236,8 @@ steps:
source: rfistrategy
- id: reorder
source: aoflag_reorder
+ - id: chunk-size
+ source: aoflag_chunksize
out:
- id: output_ms
- id: logfile
diff --git a/workflows/linc_target.cwl b/workflows/linc_target.cwl
index 46cef17b..a716f0e0 100644
--- a/workflows/linc_target.cwl
+++ b/workflows/linc_target.cwl
@@ -151,6 +151,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: inspection
outputSource:
@@ -310,6 +313,8 @@ steps:
source: rfistrategy
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: msout
- id: outh5parm
diff --git a/workflows/linc_target/gsmcal.cwl b/workflows/linc_target/gsmcal.cwl
index 5c66cd5f..1de4f030 100644
--- a/workflows/linc_target/gsmcal.cwl
+++ b/workflows/linc_target/gsmcal.cwl
@@ -43,6 +43,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: msout
outputSource:
@@ -290,6 +293,8 @@ steps:
source: rfistrategy
- id: reorder
source: aoflag_reorder
+ - id: chunk-size
+ source: aoflag_chunksize
out:
- id: output_ms
- id: logfile
--
GitLab