Skip to content
Snippets Groups Projects
Commit 6ab2fdf0 authored by Mick Veldhuis's avatar Mick Veldhuis
Browse files

Merge branch 'add-preprocess-step' into 'main' 

Add initial pre-processing step

See merge request !3
parents c86572e9 4494487f
No related branches found
No related tags found
1 merge request!3Add initial pre-processing step
Show whitespace changes
Inline Side-by-side
.gitignore
+ 1
− 1
View file @ 6ab2fdf0
rfistrategies/lofar-default.lua 0 → 100644
+ 76
− 0
View file @ 6ab2fdf0
--[[
This is the LOFAR strategy. It is almost equal to the
generic "minimal" AOFlagger strategy, version 2020-06-14.
Author: André Offringa
]]
function execute(input)
--
-- Generic settings
--
local base_threshold = 1.0 -- lower means more sensitive detection
-- How to flag complex values, options are: phase, amplitude, real, imaginary, complex
local representation = "amplitude"
local iteration_count = 3 -- how many iterations to perform?
local threshold_factor_step = 2.0 -- How much to increase the sensitivity each iteration?
local frequency_resize_factor = 3.0 -- Amount of "extra" smoothing in frequency direction
local transient_threshold_factor = 1.0 -- decreasing this value makes detection of transient RFI more aggressive
--
-- End of generic settings
--
local inpPolarizations = input:get_polarizations()
input:clear_mask()
-- For collecting statistics. Note that this is done after clear_mask(),
-- so that the statistics ignore any flags in the input data.
local copy_of_input = input:copy()
for ipol, polarization in ipairs(inpPolarizations) do
local data = input:convert_to_polarization(polarization)
data = data:convert_to_complex(representation)
local original_data = data:copy()
for i = 1, iteration_count - 1 do
local threshold_factor = threshold_factor_step ^ (iteration_count - i)
local sumthr_level = threshold_factor * base_threshold
aoflagger.sumthreshold(data, sumthr_level, sumthr_level * transient_threshold_factor, true, true)
-- Do timestep & channel flagging
local chdata = data:copy()
aoflagger.threshold_timestep_rms(data, 3.5)
aoflagger.threshold_channel_rms(chdata, 3.0 * threshold_factor, true)
data:join_mask(chdata)
-- High pass filtering steps
data:set_visibilities(original_data)
local resized_data = aoflagger.downsample(data, 3, frequency_resize_factor, true)
aoflagger.low_pass_filter(resized_data, 21, 31, 2.5, 5.0)
aoflagger.upsample(resized_data, data, 3, frequency_resize_factor)
local tmp = original_data - data
tmp:set_mask(data)
data = tmp
aoflagger.set_progress((ipol - 1) * iteration_count + i, #inpPolarizations * iteration_count)
end -- end of iterations
aoflagger.sumthreshold(data, base_threshold, base_threshold * transient_threshold_factor, true, true)
if input:is_complex() then
data = data:convert_to_complex("complex")
end
input:set_polarization_data(polarization, data)
aoflagger.set_progress(ipol, #inpPolarizations)
end -- end of polarization iterations
aoflagger.scale_invariant_rank_operator(input, 0.2, 0.2)
aoflagger.threshold_timestep_rms(input, 4.0)
input:flag_nans()
end
\ No newline at end of file
skymodels/Ateam.skymodel 0 → 100644
+ 79
− 0
View file @ 6ab2fdf0
# (Name, Type, Patch, Ra, Dec, I, Q, U, V, ReferenceFrequency='74e6', SpectralIndex='[]', MajorAxis, MinorAxis, Orientation) = format
# The above line defines the field order and is required.
# models for the A-Team members
, , CygA, 19:59:26, +40.44.00
CygA_4_2, POINT, CygA, 19:59:30.433, +40.43.56.221, 4.827e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.00637e+02
CygA_4_1, GAUSSIAN, CygA, 19:59:28.476, +40.44.01.804, 4.165e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.20516e-02, 5.34011e-03, 1.03230e+02
CygA_4_3, POINT, CygA, 19:59:22.168, +40.44.28.591, 3.896e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 9.21061e+01
CygA_4_4, POINT, CygA, 19:59:24.413, +40.44.18.617, 2.798e+03, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 6.74922e+01
CygA_4_5, POINT, CygA, 19:59:25.555, +40.43.52.813, 1.560e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
, , CasA, 23:23:28, +58.48.42
CasA_4_3, GAUSSIAN, CasA,23:23:21.044, +58.49.42.000, 4.193e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.32664e-02, 1.79281e-02, 2.73207e+01
CasA_4_2, GAUSSIAN, CasA,23:23:33.918, +58.50.13.374, 4.131e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.25963e-02, 1.87784e-02, 1.12516e+02
CasA_4_8, GAUSSIAN, CasA,23:23:34.032, +58.47.29.883, 2.536e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 2.77739e-02, 1.42664e-02, 6.26062e+01
CasA_4_5, GAUSSIAN, CasA,23:23:15.292, +58.50.00.221, 2.316e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 3.15448e-02, 9.02869e-03, 4.11857e+01
CasA_4_1, GAUSSIAN, CasA,23:23:39.632, +58.48.32.671, 2.245e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 2.25927e-02, 1.70874e-02, 8.48056e+01
CasA_4_7, GAUSSIAN, CasA,23:23:24.110, +58.47.43.607, 2.201e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 4.64283e-02, 1.49372e-02, 2.12611e+01
CasA_4_4, GAUSSIAN, CasA,23:23:14.939, +58.48.09.511, 2.099e+03, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 3.09146e-02, 1.42990e-02, 1.61549e+02
CasA_4_9, GAUSSIAN, CasA,23:23:10.817, +58.48.46.409, 9.790e+02, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 9.62058e-03, 7.79284e-03, 1.77482e+02
CasA_4_6, GAUSSIAN, CasA,23:23:26.327, +58.51.14.147, 2.444e+02, 0.0, 0.0, 0.0, 7.37817e+07, [-0.8], 8.75852e-03, 6.94444e-03, 8.12790e+01
, , TauA, 05:34:32, +21.59.55
TauA_4_1, GAUSSIAN, TauA, 05:34:32.269, +21.59.47.597, 1.911e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.94949e-02, 1.31559e-02, 1.65623e+02
TauA_4_11, GAUSSIAN, TauA, 05:34:28.639, +22.00.53.809, 1.344e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.82500e-02, 1.22959e-02, 2.49193e+01
TauA_4_3, GAUSSIAN, TauA, 05:34:25.671, +22.01.26.361, 1.247e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.76133e-02, 1.13962e-02, 5.60626e+01
TauA_4_9, GAUSSIAN, TauA, 05:34:23.420, +22.00.12.563, 1.096e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.82128e-02, 1.71142e-02, 3.38306e+01
TauA_4_15, GAUSSIAN, TauA, 05:34:22.884, +22.01.56.603, 9.243e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.99931e-02, 1.65627e-02, 1.76338e+02
TauA_4_20, GAUSSIAN, TauA, 05:34:29.086, +21.59.21.722, 8.792e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.23261e-02, 9.03648e-03, 1.09196e+02
TauA_4_10, GAUSSIAN, TauA, 05:34:37.530, +21.59.06.719, 8.641e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.13390e-02, 1.33726e-02, 1.27644e+02
TauA_4_2, POINT, TauA, 05:34:30.550, +21.59.58.106, 8.276e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
TauA_4_5, GAUSSIAN, TauA, 05:34:40.748, +21.59.31.921, 7.008e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.59971e-02, 1.25206e-02, 1.51745e+02
TauA_4_14, GAUSSIAN, TauA, 05:34:30.570, +21.58.33.304, 6.187e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.87859e-02, 1.14519e-02, 8.42720e+01
TauA_4_7, GAUSSIAN, TauA, 05:34:36.888, +22.01.01.174, 5.537e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.93364e-02, 1.21462e-02, 1.39794e+02
TauA_4_8, GAUSSIAN, TauA, 05:34:35.324, +22.00.10.883, 5.525e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.81832e-02, 1.35652e-02, 1.06533e+02
TauA_4_4, GAUSSIAN, TauA, 05:34:23.931, +21.58.55.252, 5.076e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.77800e-02, 1.52796e-02, 7.50003e+01
TauA_4_13, GAUSSIAN, TauA, 05:34:36.235, +21.58.03.089, 4.862e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.59649e-02, 9.18833e-03, 8.58303e+01
TauA_4_6, GAUSSIAN, TauA, 05:34:27.470, +21.58.50.002, 4.223e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.54141e-02, 7.87103e-03, 5.52923e+01
TauA_4_21, GAUSSIAN, TauA, 05:34:34.585, +22.00.45.580, 4.175e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.19587e-02, 6.70123e-03, 1.72839e+02
TauA_4_19, GAUSSIAN, TauA, 05:34:19.534, +22.01.07.470, 3.906e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.20399e-02, 1.14898e-02, 1.76500e+02
TauA_4_17, GAUSSIAN, TauA, 05:34:29.426, +21.59.51.003, 2.848e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.10631e-02, 5.13228e-03, 1.27622e+02
TauA_4_23, GAUSSIAN, TauA, 05:34:26.246, +22.02.05.085, 2.699e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.28687e-02, 1.05568e-02, 1.44302e+02
TauA_4_16, GAUSSIAN, TauA, 05:34:32.269, +22.01.21.318, 2.557e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.16784e-02, 7.66067e-03, 1.43877e+02
TauA_4_12, GAUSSIAN, TauA, 05:34:22.991, +22.00.33.000, 2.127e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.49607e-02, 8.16407e-03, 1.16642e+02
TauA_4_18, GAUSSIAN, TauA, 05:34:26.292, +22.02.39.501, 1.727e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.19757e-02, 6.32082e-03, 1.08185e+02
TauA_4_22, GAUSSIAN, TauA, 05:34:37.023, +21.58.27.987, 1.357e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.54110e-03, 5.01393e-03, 7.39302e+01
TauA_4_24, GAUSSIAN, TauA, 05:34:31.108, +22.00.32.726, 1.106e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.59017e-02, 6.94444e-03, 5.84987e+01
TauA_4_26, GAUSSIAN, TauA, 05:32:23.875, +21.50.58.488, 1.780e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.05524e-02, 3.94647e-03, 6.84955e+01
TauA_4_25, POINT, TauA, 05:34:30.647, +21.59.16.928, 9.776e-01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 2.05933e+01
, , VirA, 12:30:35, +12.19.45
VirA_4_17, GAUSSIAN, VirA, 12:30:34.383, +12.20.04.012, 3.219e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.42972e-02, 6.11535e-02, 6.50364e+01
VirA_4_2, GAUSSIAN, VirA, 12:30:47.034, +12.23.43.975, 2.591e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.54400e-03, 4.07169e-03, 1.06884e+02
VirA_4_1, POINT, VirA, 12:30:48.961, +12.23.33.770, 1.516e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.70815e+02
VirA_4_19, GAUSSIAN, VirA, 12:30:41.047, +12.17.36.983, 1.094e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 4.63137e-02, 2.79725e-02, 8.64763e+01
VirA_4_20, GAUSSIAN, VirA, 12:30:49.250, +12.24.02.102, 1.069e+02, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 9.92793e-02, 2.33402e-02, 1.08930e+01
VirA_4_7, GAUSSIAN, VirA, 12:31:00.320, +12.26.38.867, 8.851e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.83541e-02, 3.47954e-02, 1.53069e+02
VirA_4_6, GAUSSIAN, VirA, 12:30:55.490, +12.23.49.301, 7.499e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 6.64979e-02, 1.79950e-02, 1.72468e+01
VirA_4_4, GAUSSIAN, VirA, 12:30:59.933, +12.22.42.187, 7.388e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.52817e-01, 1.69367e-02, 3.28830e+01
VirA_4_21, GAUSSIAN, VirA, 12:30:42.588, +12.22.06.778, 6.153e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.81610e-02, 1.87921e-02, 2.03833e+01
VirA_4_9, GAUSSIAN, VirA, 12:30:50.661, +12.23.49.618, 5.456e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.36474e-02, 8.25082e-03, 8.58262e+01
VirA_4_3, POINT, VirA, 12:30:48.239, +12.23.18.068, 5.398e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 1.11922e+02
VirA_4_14, GAUSSIAN, VirA, 12:30:39.222, +12.23.13.204, 5.142e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.40448e-02, 1.75673e-02, 1.70476e+02
VirA_4_5, POINT, VirA, 12:30:45.851, +12.23.49.562, 4.189e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 8.15603e+01
VirA_4_8, GAUSSIAN, VirA, 12:30:52.150, +12.18.29.472, 3.936e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.38379e-02, 2.30198e-02, 1.60497e+02
VirA_4_15, GAUSSIAN, VirA, 12:31:01.387, +12.24.37.942, 3.313e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.14509e-02, 1.18403e-02, 7.44971e+01
VirA_4_12, GAUSSIAN, VirA, 12:31:03.637, +12.23.31.633, 2.835e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 3.23407e-02, 9.02810e-03, 3.03022e+00
VirA_4_11, GAUSSIAN, VirA, 12:30:59.512, +12.22.44.364, 2.701e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.66163e-02, 1.08450e-02, 1.34667e+02
VirA_4_13, GAUSSIAN, VirA, 12:31:00.272, +12.23.41.572, 2.676e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.52105e-02, 1.11420e-02, 1.04009e+02
VirA_4_10, POINT, VirA, 12:30:48.452, +12.23.32.663, 2.217e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
VirA_4_16, GAUSSIAN, VirA, 12:30:43.137, +12.23.59.978, 1.788e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.68245e-02, 9.05564e-03, 9.84731e+01
VirA_4_24, GAUSSIAN, VirA, 12:30:36.867, +12.18.49.897, 1.130e+01, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 2.28602e-02, 1.66793e-02, 1.45577e+02
VirA_4_18, POINT, VirA, 12:30:47.114, +12.23.26.692, 8.179e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 7.63889e-03, 6.94444e-03, 0.00000e+00
VirA_4_23, GAUSSIAN, VirA, 12:30:44.464, +12.19.17.889, 4.819e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 1.37460e-02, 1.36631e-02, 1.48507e+02
VirA_4_22, GAUSSIAN, VirA, 12:30:44.127, +12.23.08.175, 4.207e+00, 0.0, 0.0, 0.0, 7.38000e+07, [-0.8], 8.85467e-03, 5.44859e-03, 1.33216e+02
steps/preprocess.cwl
+ 93
− 0
View file @ 6ab2fdf0
cwlVersion: v1.2
class: CommandLineTool
baseCommand: DP3
label: Performs pre-processing on raw LOFAR data.
doc: |
This tool performs flagging, demixing, averaging, and Dysco compression on
raw LOFAR data with the Default Pre-Processing Pipeline (DP3). The input is
a raw LOFAR MeasurementSet (MS); to open raw MSs, the LOFAR Storage Manager
(LofarStMan) needs to be installed. The output is a pre-processed MS, which
can further be processed by, e.g., the LOFAR Initial Calibration (LINC) Pipeline.
arguments:
- steps=[aoflagger,demix]
- msin.autoweight=true
- msout.storagemanager=dysco
inputs:
- id: msin
type: Directory
inputBinding:
prefix: msin=
separate: false
doc: Input raw LOFAR MS
- id: rfistrategy
type:
- string?
- File?
inputBinding:
prefix: aoflagger.strategy=
separate: false
default:
class: File
path: ../rfistrategies/lofar-default.lua
doc: AOFlagger flagging strategy file
- id: demix_skymodel
type:
- string?
- File?
inputBinding:
prefix: demix.skymodel=
separate: false
default:
class: File
path: ../skymodels/Ateam.skymodel
doc: Skymodel to use for calibration
- id: demix_sources
type: string[]?
inputBinding:
prefix: demix.subtractsoures=
separate: false
default: []
doc: Sources to subtract during demixing
- id: demix_timestep
type: int
inputBinding:
prefix: demix.demixtimestep=
separate: false
doc: Number of time steps to average during demixing
- id: demix_freqstep
type: int
inputBinding:
prefix: demix.demixfreqstep=
separate: false
doc: Number of channels to average during demixing
- id: avg_timestep
type: int
inputBinding:
prefix: demix.timestep=
separate: false
doc: Number of time steps to average
- id: avg_freqstep
type: int
inputBinding:
prefix: demix.freqstep=
separate: false
doc: Number of channels to average
- id: msout_name
type: string
inputBinding:
prefix: msout=
separate: false
doc: Output MS
outputs:
- id: msout
type: Directory
outputBinding:
glob: '$(inputs.msout_name == "." ? inputs.msin.basename : inputs.msout_name)'
doc: Output MS
requirements:
InlineJavascriptRequirement: {}
\ No newline at end of file
tests/preprocess_input.json 0 → 100644
+ 12
− 0
View file @ 6ab2fdf0
{
"msin": {
"class": "Directory",
"path": "L2014581_SAP001_SB400_uv.MS"
},
"demix_sources": [],
"demix_timestep": 10,
"demix_freqstep": 64,
"avg_timestep": 1,
"avg_freqstep": 4,
"msout_name": "L2014581_SAP001_SB400_uv_PREPROCESSED.MS"
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment