diff --git a/docs/source/calibrator.rst b/docs/source/calibrator.rst
index a97b18d77a39399d33c64488a2254c983ef25737..d457380726d2cc6cc701a3759993354e4c216406 100644
--- a/docs/source/calibrator.rst
+++ b/docs/source/calibrator.rst
@@ -278,7 +278,8 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
- ``max_dp3_threads``: number of threads per process for DP3 (default: 10)
- ``memoryperc``: maximum of memory used for aoflagger in raw_flagging mode in percent (default: 20)
-- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (see the `AOFlagger manual`_`)
+- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (default: false, see the `AOFlagger manual`_`)
+- ``aoflag_chunksize``: this will split the set into intervals with the given maximum size, and flag each interval independently. This lowers the amount of memory required (default: 2000)
**Parameters you may want to adjust**
diff --git a/docs/source/parset.rst b/docs/source/parset.rst
index 75712bf64f487e76cb7d1bebe1345bb4409d4d63..595a6072cd2c28d61eaf4e1943e9a039c67ec039 100644
--- a/docs/source/parset.rst
+++ b/docs/source/parset.rst
@@ -83,7 +83,8 @@ There are more parameters you may want to adjust that can be added to this input
"bandpass_freqresolution": "195.3125kHz",
"lbfgs_historysize" : 10,
"lbfgs_robustdof" : 200,
- "reorder_aoflag" : false
+ "aoflag_reorder" : false,
+ "aoflag_chunksize" : 2000
}
If you just want to alter one of the defaults it is sufficient to override it by specifing its new value the JSON input file::
diff --git a/docs/source/target.rst b/docs/source/target.rst
index d51637e63c2adef2bfd4e515d24f094e81262f11..eab993a07de0fc2a0947be121ffaf7b5e755a740 100644
--- a/docs/source/target.rst
+++ b/docs/source/target.rst
@@ -304,7 +304,8 @@ A comprehensive explanation of the baseline selection syntax can be found `here`
- ``max_dp3_threads``: number of threads per process for DP3 (default: 10)
- ``memoryperc``: maximum of memory used for aoflagger in raw_flagging mode in percent (default: 20)
-- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (see the `AOFlagger manual`_)
+- ``aoflag_reorder``: make aoflagger reorder the measurement set before running the detection. This prevents that aoflagger will use its memory reading mode, which is faster but uses more memory (default: false, see the `AOFlagger manual`_`)
+- ``aoflag_chunksize``: this will split the set into intervals with the given maximum size, and flag each interval independently. This lowers the amount of memory required (default: 2000)
*Skymodel directory*
diff --git a/steps/aoflag.cwl b/steps/aoflag.cwl
index 12c46d3fd8ea57a0800fa5aaa07c6a8e80d7f3da..816456648c701237388178e289bca94273564c3a 100644
--- a/steps/aoflag.cwl
+++ b/steps/aoflag.cwl
@@ -133,6 +133,13 @@ inputs:
prefix: -concatenate-frequency
position: 0
shellQuote: false
+ - id: chunk-size
+ type: int?
+ doc: This will split the set into intervals with the given maximum size, and flag each interval independently
+ inputBinding:
+ prefix: -chunk-size
+ position: 0
+ shellQuote: false
outputs:
- id: output_ms
diff --git a/workflows/HBA_calibrator.cwl b/workflows/HBA_calibrator.cwl
index 6ac44938f53f8d104d22726e43cb2e665be1be0f..d49925fbaf31b8a65ec66e75a6111b9c1ee7f6f7 100644
--- a/workflows/HBA_calibrator.cwl
+++ b/workflows/HBA_calibrator.cwl
@@ -122,6 +122,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: log_files
outputSource:
@@ -221,6 +224,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: logfiles
- id: solutions
diff --git a/workflows/HBA_target.cwl b/workflows/HBA_target.cwl
index 3c439b169d1b044f52ce4983a0bec5379cd9eb09..e939bc41a44019a0d0a950dd3e608f6e1cd1cb1c 100644
--- a/workflows/HBA_target.cwl
+++ b/workflows/HBA_target.cwl
@@ -151,6 +151,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: calibrated_data
outputSource:
@@ -273,6 +276,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: logfiles
- id: msout
diff --git a/workflows/LBA_calibrator.cwl b/workflows/LBA_calibrator.cwl
index ba0bfd5fec9b01bdc33881c584dc65dfd062e996..eb2e0a4c278102ffd95358c8995e95d559d7a082 100644
--- a/workflows/LBA_calibrator.cwl
+++ b/workflows/LBA_calibrator.cwl
@@ -120,6 +120,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: log_files
outputSource:
@@ -219,6 +222,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: logfiles
- id: solutions
diff --git a/workflows/linc_calibrator.cwl b/workflows/linc_calibrator.cwl
index 6ca990f207c59e88040bc853b1f45af5d923fd04..7168b37090f0b95005a207466184af40b97d1075 100644
--- a/workflows/linc_calibrator.cwl
+++ b/workflows/linc_calibrator.cwl
@@ -121,6 +121,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: inspection
linkMerge: merge_flattened
@@ -216,6 +219,8 @@ steps:
source: lbfgs_robustdof
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: outh5parm
- id: logfiles
diff --git a/workflows/linc_calibrator/prep.cwl b/workflows/linc_calibrator/prep.cwl
index 0e1fbb0942d0370e6b81d42d65c62af00351807c..fa07f983219bfb7b7594fc39ca117d4ec6b59810 100644
--- a/workflows/linc_calibrator/prep.cwl
+++ b/workflows/linc_calibrator/prep.cwl
@@ -80,6 +80,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: outh5parm
outputSource:
@@ -233,6 +236,8 @@ steps:
source: rfistrategy
- id: reorder
source: aoflag_reorder
+ - id: chunk-size
+ source: aoflag_chunksize
out:
- id: output_ms
- id: logfile
diff --git a/workflows/linc_target.cwl b/workflows/linc_target.cwl
index 46cef17ba633ee645282614a69c46de0f52e9491..a716f0e0952590fbbb1ff3eae5aa806b2e72cd96 100644
--- a/workflows/linc_target.cwl
+++ b/workflows/linc_target.cwl
@@ -151,6 +151,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: inspection
outputSource:
@@ -310,6 +313,8 @@ steps:
source: rfistrategy
- id: aoflag_reorder
source: aoflag_reorder
+ - id: aoflag_chunksize
+ source: aoflag_chunksize
out:
- id: msout
- id: outh5parm
diff --git a/workflows/linc_target/gsmcal.cwl b/workflows/linc_target/gsmcal.cwl
index 5c66cd5f3871de87ded976778097cb341b3db968..1de4f0307dfc5c50758277a15c31c6be7f9bab2f 100644
--- a/workflows/linc_target/gsmcal.cwl
+++ b/workflows/linc_target/gsmcal.cwl
@@ -43,6 +43,9 @@ inputs:
- id: aoflag_reorder
type: boolean?
default: false
+ - id: aoflag_chunksize
+ type: int?
+ default: 2000
outputs:
- id: msout
outputSource:
@@ -290,6 +293,8 @@ steps:
source: rfistrategy
- id: reorder
source: aoflag_reorder
+ - id: chunk-size
+ source: aoflag_chunksize
out:
- id: output_ms
- id: logfile