Skip to content
Snippets Groups Projects
Commit 791b90f3 authored by Matthijs van der Wild's avatar Matthijs van der Wild
Browse files

Merge branch 'clipper' into 'master'

RAP-94 Clipper

See merge request RD/DP3!1229
parents 1b792f26 327851de
No related branches found
No related tags found
No related merge requests found
......@@ -498,6 +498,7 @@ add_library(
steps/BDAAverager.cc
steps/BDAExpander.cc
steps/BdaGroupPredict.cc
steps/Clipper.cc
steps/Counter.cc
steps/Demixer.cc
steps/DummyStep.cc
......
......@@ -24,6 +24,7 @@
#include "../steps/BDAAverager.h"
#include "../steps/BDAExpander.h"
#include "../steps/BdaGroupPredict.h"
#include "../steps/Clipper.h"
#include "../steps/Counter.h"
#include "../steps/DDECal.h"
#include "../steps/BdaDdeCal.h"
......@@ -167,6 +168,8 @@ std::shared_ptr<Step> MakeSingleStep(const std::string& type,
step = std::make_shared<steps::AntennaFlagger>(parset, prefix);
} else if (type == "uvwflagger" || type == "uvwflag") {
step = std::make_shared<steps::UVWFlagger>(parset, prefix, inputType);
} else if (type == "clipper") {
step = std::make_shared<steps::Clipper>(parset, prefix);
} else if (type == "columnreader") {
step = std::make_shared<steps::MsColumnReader>(parset, prefix);
} else if (type == "counter" || type == "count") {
......
description: >-
Simulates visibilities from a given skymodel and uses the result to clip bright sources from the data`.`
Because this steps performs prediction, options of the Predict step are also valid`.`
inputs:
step_name:
type: string
doc: unique name for the step `.`
default: clipper
type:
type: string
doc: Case-insensitive step type; must be 'clipper' `.`
default: clipper
amplmax:
type: float
doc: >-
Sets the clip level for the data`.`
default: 50.0 for LBA, 5.0 for HBA
timestep:
type: int
doc: >-
Perform the visibility prediction only once every `timestep` steps`.`
The next `timestep-1` will use these visibilities until the next prediction`.`
default: 5
// Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
// SPDX-License-Identifier: GPL-3.0-or-later
#include "Clipper.h"
#include "NullStep.h"
#include "MsColumnReader.h"
#include <iostream>
#include <xtensor/xcomplex.hpp>
#include "../base/FlagCounter.h"
#include <dp3/base/DP3.h>
using dp3::base::DPBuffer;
using dp3::base::DPInfo;
namespace dp3 {
namespace steps {
Clipper::Clipper(const common::ParameterSet& parset, const std::string& prefix)
: name_(prefix),
counter_(0),
time_step_(parset.getInt(prefix + "timestep", 5)),
ampl_max_(parset.getFloat(prefix + "amplmax", 0.0)) {
predict_step_ =
std::make_shared<OnePredict>(parset, prefix, std::vector<std::string>());
result_step_ = std::make_shared<ResultStep>();
predict_step_->setNextStep(result_step_);
}
void Clipper::updateInfo(const DPInfo& info_in) {
Step::updateInfo(info_in);
if (ampl_max_ == 0.0) {
std::string antennaSet(info_in.antennaSet());
if (antennaSet.substr(0, 3) == "LBA") {
ampl_max_ = 50.0;
} else {
ampl_max_ = 5.0;
}
}
predict_step_->updateInfo(info_in);
}
void Clipper::show(std::ostream& os) const {
os << "Clipper " << name_ << '\n';
os << " time step: " << time_step_ << '\n';
os << " max amplitude: " << ampl_max_ << '\n';
predict_step_->show(os);
}
void Clipper::showTimings(std::ostream& os, double duration) const {
os << " ";
base::FlagCounter::showPerc1(os, timer_.getElapsed(), duration);
os << " Clipper " << name_ << '\n';
}
bool Clipper::process(std::unique_ptr<DPBuffer> buffer) {
timer_.start();
if (counter_ % time_step_ == 0) {
std::unique_ptr<DPBuffer> substep_buffer =
std::make_unique<DPBuffer>(*buffer);
predict_step_->process(std::move(substep_buffer));
std::unique_ptr<DPBuffer> result_buffer = result_step_->take();
last_flags_ = xt::eval(xt::abs(result_buffer->GetData())) > ampl_max_;
counter_ = 0;
}
counter_++;
buffer->GetFlags() = buffer->GetFlags() || last_flags_;
timer_.stop();
getNextStep()->process(std::move(buffer));
return false;
}
void Clipper::finish() { getNextStep()->finish(); }
} // namespace steps
} // namespace dp3
// Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef DP3_STEPS_CLIPPER_H_
#define DP3_STEPS_CLIPPER_H_
#include <dp3/steps/Step.h>
#include "../common/ParameterSet.h"
#include "OnePredict.h"
#include "PreFlagger.h"
#include "ResultStep.h"
#include "../common/Timer.h"
namespace dp3 {
namespace steps {
/// @brief DP3 step for clipping bright sources from the data
/// This class is a Step class that predicts visibilities from
/// a skymodel and clips the bright sources from the input data.
class Clipper : public Step {
public:
/// Construct the object.
/// Parameters are obtained from the parset using the given prefix.
Clipper(const common::ParameterSet&, const std::string& prefix);
common::Fields getRequiredFields() const override {
return kFlagsField | kUvwField;
}
common::Fields getProvidedFields() const override { return kFlagsField; }
/// Process the data. The step forwards the data to its next step.
bool process(std::unique_ptr<base::DPBuffer>) override;
/// Finish the processing of this step and subsequent steps.
void finish() override;
/// Update the general info.
void updateInfo(const base::DPInfo&) override;
/// Show the step parameters.
void show(std::ostream&) const override;
/// Show the timings.
void showTimings(std::ostream&, double duration) const override;
private:
std::string name_;
common::NSTimer timer_;
size_t counter_;
size_t time_step_;
float ampl_max_;
std::shared_ptr<OnePredict> predict_step_;
std::shared_ptr<ResultStep> result_step_;
std::unique_ptr<dp3::base::DPBuffer> predict_buffer_;
base::DPBuffer::FlagsType last_flags_;
};
} // namespace steps
} // namespace dp3
#endif
......@@ -6,6 +6,7 @@ add_python_tests(
tApplyCal2
tBdaExpander
tBdaPredict
tClipper
tColumnReader
tDemix
tGainCal
......
# Copyright (C) 2023 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later
import pytest
import shutil
import os
import uuid
from subprocess import check_call, check_output
# Append current directory to system path in order to import testconfig
import sys
sys.path.append(".")
import testconfig as tcf
from utils import assert_taql, untar_ms, get_taql_result
"""
Tests for clipper (coarsely predict and clip bright sources).
Script can be invoked in two ways:
- as standalone from the build/steps/test/integration directory,
using `pytest source/tClipper.py` (extended with pytest options of your choice)
- using ctest, see DP3/steps/test/integration/CMakeLists.txt
"""
MSIN = "tNDPPP-generic.MS"
MSOUT = "clipper.MS"
CWD = os.getcwd()
TEST_AMPLMAX = 1.5
@pytest.fixture(autouse=True)
def source_env():
os.chdir(CWD)
tmpdir = str(uuid.uuid4())
os.mkdir(tmpdir)
os.chdir(tmpdir)
untar_ms(f"{tcf.RESOURCEDIR}/{MSIN}.tgz")
# Tests are executed here
yield
# Post-test: clean up
os.chdir(CWD)
shutil.rmtree(tmpdir)
@pytest.fixture()
def create_model_data():
check_call(
[
tcf.DP3EXE,
f"msin={MSIN}",
f"msout={MSOUT}",
"steps=[]",
]
)
def test_clipper(create_model_data):
with open("clipper.skymodel", "w") as f:
f.write(
"FORMAT = Name, Type, Patch, Ra, Dec, I, MajorAxis, MinorAxis, PositionAngle, ReferenceFrequency='134e6', SpectralIndex='[0.0]'\n"
)
f.write(", , dummy, 01:37:41.3, +15.09.35.132\n")
f.write(
"src_0, POINT, dummy, 01:37:41.3, +15.09.35.132, 1000, , , , ,"
)
taql_command = f"UPDATE {MSOUT} set FLAG=false"
get_taql_result(taql_command)
taql_command = f"select from {MSOUT} where any(FLAG)"
assert_taql(taql_command, 0)
check_call(
[
tcf.DP3EXE,
f"msin={MSOUT}",
"msout=.",
"steps=[clipper]",
"clipper.sourcedb=clipper.skymodel",
"clipper.usebeammodel=true",
f"clipper.amplmax={TEST_AMPLMAX}",
"clipper.timestep=1",
]
)
assert_taql(taql_command, 102)
check_call(
[
tcf.DP3EXE,
f"msin={MSOUT}",
"msout=.",
"msout.datacolumn=MODEL_DATA",
"steps=[predict]",
"predict.sourcedb=clipper.skymodel",
"predict.usebeammodel=true",
]
)
# Check that high amplitudes are flagged.
taql_command = f"select from {MSOUT} where any(not FLAG && (MODEL_DATA > {TEST_AMPLMAX}))"
assert_taql(taql_command, 0)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment