Skip to content
Snippets Groups Projects
Commit 77ef8199 authored by Andre Offringa's avatar Andre Offringa
Browse files

Bug 1491: fixed experiment to use fuzzy ratios instead of inconsistent checks

parent 36f04f3d
No related branches found
No related tags found
No related merge requests found
...@@ -22,10 +22,10 @@ ...@@ -22,10 +22,10 @@
#include <AOFlagger/test/testingtools/testgroup.h> #include <AOFlagger/test/testingtools/testgroup.h>
#include <AOFlagger/test/experiments/defaultstrategyspeedtest.h> //#include <AOFlagger/test/experiments/defaultstrategyspeedtest.h>
//#include <AOFlagger/test/experiments/filterresultstest.h> //#include <AOFlagger/test/experiments/filterresultstest.h>
//#include <AOFlagger/test/experiments/scaleinvariantdilationexperiment.h> //#include <AOFlagger/test/experiments/scaleinvariantdilationexperiment.h>
//#include <AOFlagger/test/experiments/rankoperatorrocexperiment.h> #include <AOFlagger/test/experiments/rankoperatorrocexperiment.h>
class ExperimentsTestGroup : public TestGroup { class ExperimentsTestGroup : public TestGroup {
public: public:
...@@ -33,8 +33,8 @@ class ExperimentsTestGroup : public TestGroup { ...@@ -33,8 +33,8 @@ class ExperimentsTestGroup : public TestGroup {
virtual void Initialize() virtual void Initialize()
{ {
//Add(new RankOperatorROCExperiment()); Add(new RankOperatorROCExperiment());
Add(new DefaultStrategySpeedTest()); //Add(new DefaultStrategySpeedTest());
//Add(new FilterResultsTest()); //Add(new FilterResultsTest());
//Add(new ScaleInvariantDilationExperiment()); //Add(new ScaleInvariantDilationExperiment());
} }
......
...@@ -70,22 +70,30 @@ class RankOperatorROCExperiment : public UnitTest { ...@@ -70,22 +70,30 @@ class RankOperatorROCExperiment : public UnitTest {
static rfiStrategy::Strategy *createThresholdStrategy(); static rfiStrategy::Strategy *createThresholdStrategy();
static void executeTest(enum TestType testType); static void executeTest(enum TestType testType);
static num_t getRatio(Image2DPtr groundTruth, Image2DPtr result) static num_t getRatio(Image2DPtr groundTruth, Mask2DCPtr resultMask, bool inverseTruth, bool invertMask)
{ {
num_t rfiTotal = result->Sum(); num_t totalTruth = groundTruth->Sum();
if(inverseTruth)
{
totalTruth = groundTruth->Width() * groundTruth->Height() - totalTruth;
}
num_t sum = 0.0; num_t sum = 0.0;
for(size_t y=0;y<groundTruth->Height();++y) for(size_t y=0;y<groundTruth->Height();++y)
{ {
for(size_t x=0;x<groundTruth->Width();++x) for(size_t x=0;x<groundTruth->Width();++x)
{ {
sum += groundTruth->Value(x, y) * result->Value(x, y); num_t truth = inverseTruth ? (1.0 - groundTruth->Value(x, y)) : groundTruth->Value(x, y);
if(resultMask->Value(x, y) != invertMask)
{
sum += truth;
} }
} }
return sum / rfiTotal; }
return sum / totalTruth;
} }
}; };
const unsigned RankOperatorROCExperiment::_repeatCount = 20; const unsigned RankOperatorROCExperiment::_repeatCount = 2;
inline rfiStrategy::Strategy *RankOperatorROCExperiment::createThresholdStrategy() inline rfiStrategy::Strategy *RankOperatorROCExperiment::createThresholdStrategy()
{ {
...@@ -161,9 +169,6 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -161,9 +169,6 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
break; break;
} }
size_t grTotalRFI = 0;
numl_t grTotalRFISum = 0.0;
numl_t numl_t
grRankTpRatio[ETA_STEPS+1], grRankFpRatio[ETA_STEPS+1], grRankTpRatio[ETA_STEPS+1], grRankFpRatio[ETA_STEPS+1],
grRankTpSum[ETA_STEPS+1], grRankFpSum[ETA_STEPS+1], grRankTpSum[ETA_STEPS+1], grRankFpSum[ETA_STEPS+1],
...@@ -187,10 +192,8 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -187,10 +192,8 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
for(unsigned repeatIndex=0 ; repeatIndex<_repeatCount ; ++repeatIndex) for(unsigned repeatIndex=0 ; repeatIndex<_repeatCount ; ++repeatIndex)
{ {
Mask2DPtr mask = Mask2D::CreateSetMaskPtr<false>(width, height); Mask2DPtr mask = Mask2D::CreateSetMaskPtr<false>(width, height);
Image2DCPtr Image2DPtr
groundTruthMask = Image2D::CreateZeroImagePtr(width, height), groundTruth = Image2D::CreateZeroImagePtr(width, height);
realTruth = Image2D::CreateZeroImagePtr(width, height),
imagTruth = Image2D::CreateZeroImagePtr(width, height);
Image2DPtr realImage, imagImage; Image2DPtr realImage, imagImage;
Image2DCPtr rfiLessImage; Image2DCPtr rfiLessImage;
TimeFrequencyData rfiLessData, data; TimeFrequencyData rfiLessData, data;
...@@ -198,7 +201,7 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -198,7 +201,7 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
{ {
case GaussianBroadband: case GaussianBroadband:
{ {
Image2DCPtr Image2DPtr
realTruth = Image2D::CreateZeroImagePtr(width, height), realTruth = Image2D::CreateZeroImagePtr(width, height),
imagTruth = Image2D::CreateZeroImagePtr(width, height); imagTruth = Image2D::CreateZeroImagePtr(width, height);
realImage = MitigationTester::CreateTestSet(2, mask, width, height), realImage = MitigationTester::CreateTestSet(2, mask, width, height),
...@@ -210,12 +213,12 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -210,12 +213,12 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
MitigationTester::AddGaussianBroadbandToTestSet(imagImage, mask); MitigationTester::AddGaussianBroadbandToTestSet(imagImage, mask);
MitigationTester::AddGaussianBroadbandToTestSet(realTruth, mask); MitigationTester::AddGaussianBroadbandToTestSet(realTruth, mask);
MitigationTester::AddGaussianBroadbandToTestSet(imagTruth, mask); MitigationTester::AddGaussianBroadbandToTestSet(imagTruth, mask);
groundTruthMask = TimeFrequencyData(SinglePolarisation, realTruth, imagTruth).GetSingleImage(); groundTruth = Image2D::CreateCopy(TimeFrequencyData(SinglePolarisation, realTruth, imagTruth).GetSingleImage());
data = TimeFrequencyData(SinglePolarisation, realImage, imagImage); data = TimeFrequencyData(SinglePolarisation, realImage, imagImage);
} break; } break;
case SinusoidalBroadband: case SinusoidalBroadband:
{ {
Image2DCPtr Image2DPtr
realTruth = Image2D::CreateZeroImagePtr(width, height), realTruth = Image2D::CreateZeroImagePtr(width, height),
imagTruth = Image2D::CreateZeroImagePtr(width, height); imagTruth = Image2D::CreateZeroImagePtr(width, height);
realImage = MitigationTester::CreateTestSet(2, mask, width, height), realImage = MitigationTester::CreateTestSet(2, mask, width, height),
...@@ -227,21 +230,14 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -227,21 +230,14 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
MitigationTester::AddSinusoidalBroadbandToTestSet(imagImage, mask); MitigationTester::AddSinusoidalBroadbandToTestSet(imagImage, mask);
MitigationTester::AddSinusoidalBroadbandToTestSet(realTruth, mask); MitigationTester::AddSinusoidalBroadbandToTestSet(realTruth, mask);
MitigationTester::AddSinusoidalBroadbandToTestSet(imagTruth, mask); MitigationTester::AddSinusoidalBroadbandToTestSet(imagTruth, mask);
groundTruthMask = TimeFrequencyData(SinglePolarisation, realTruth, imagTruth).GetSingleImage(); groundTruth = Image2D::CreateCopy(TimeFrequencyData(SinglePolarisation, realTruth, imagTruth).GetSingleImage());
data = TimeFrequencyData(SinglePolarisation, realImage, imagImage); data = TimeFrequencyData(SinglePolarisation, realImage, imagImage);
} break; } break;
} }
Image2DPtr invGroundTruthMask = Image2D::CreateUnsetImagePtr(width, height);
groundTruthMask->MultiplyValues(1.0/groundTruthMask->GetMaximum());
for(size_t y=0;y<height;++y)
{
for(size_t x=0;x<width;++x)
invGroundTruthMask->SetValue(x, y, 1.0 - groundTruthMask->Value(x, y));
}
const num_t totalRFISum = groundTruthMask->Sum();
data.Trim(0, 0, 180, height); data.Trim(0, 0, 180, height);
groundTruthMask->SetTrim(0, 0, 180, height); groundTruth->SetTrim(0, 0, 180, height);
groundTruth->MultiplyValues(1.0/groundTruth->GetMaximum());
Image2DCPtr inputImage = data.GetSingleImage(); Image2DCPtr inputImage = data.GetSingleImage();
rfiStrategy::ArtifactSet artifacts(0); rfiStrategy::ArtifactSet artifacts(0);
...@@ -264,24 +260,22 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -264,24 +260,22 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
const num_t eta = (num_t) i / (num_t) ETA_STEPS; const num_t eta = (num_t) i / (num_t) ETA_STEPS;
Mask2DPtr tempMask = Mask2D::CreateCopy(input); Mask2DPtr tempMask = Mask2D::CreateCopy(input);
ScaleInvariantDilation::DilateVertically(tempMask, eta); ScaleInvariantDilation::DilateVertically(tempMask, eta);
size_t totalPositives = tempMask->GetCount<true>(); size_t totalPositives = tempMask->GetCount<true>();
tempMask->Invert(); double tpFuzzyRatio = getRatio(groundTruth, tempMask, false, false);
num_t flaggedSum = ThresholdTools::Sum(inputImage, tempMask); double fpFuzzyRatio = getRatio(groundTruth, tempMask, true, false);
tempMask->Invert();
tempMask->Intersect(groundTruthMask); tempMask->Intersect(mask);
size_t truePositives = tempMask->GetCount<true>(); size_t truePositives = tempMask->GetCount<true>();
size_t falsePositives = totalPositives - truePositives; size_t falsePositives = totalPositives - truePositives;
tempMask->Invert();
num_t tpSum = ThresholdTools::Sum(inputImage, tempMask);
num_t noiseSum = ThresholdTools::Sum(rfiLessImage, tempMask);
double tpRatio = (double) truePositives / totalRFI; double tpRatio = (double) truePositives / totalRFI;
double fpRatio = (double) falsePositives / totalRFI; double fpRatio = (double) falsePositives / totalRFI;
grRankTpRatio[i] += tpRatio; grRankTpRatio[i] += tpRatio;
grRankFpRatio[i] += fpRatio; grRankFpRatio[i] += fpRatio;
grRankTpSum[i] += (tpSum - noiseSum)/totalRFISum; grRankTpSum[i] += tpFuzzyRatio;
grRankFpSum[i] += (flaggedSum - tpSum)/totalRFISum; grRankFpSum[i] += fpFuzzyRatio;
} }
for(size_t i=0;i<DIL_STEPS;++i) for(size_t i=0;i<DIL_STEPS;++i)
...@@ -290,28 +284,26 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -290,28 +284,26 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
Mask2DPtr tempMask = Mask2D::CreateCopy(input); Mask2DPtr tempMask = Mask2D::CreateCopy(input);
StatisticalFlagger::EnlargeFlags(tempMask, 0, dilSize); StatisticalFlagger::EnlargeFlags(tempMask, 0, dilSize);
size_t totalPositives = tempMask->GetCount<true>(); size_t totalPositives = tempMask->GetCount<true>();
tempMask->Invert(); double tpFuzzyRatio = getRatio(groundTruth, tempMask, false, false);
num_t flaggedSum = ThresholdTools::Sum(inputImage, tempMask); double fpFuzzyRatio = getRatio(groundTruth, tempMask, true, false);
tempMask->Invert();
tempMask->Intersect(groundTruthMask); tempMask->Intersect(mask);
size_t truePositives = tempMask->GetCount<true>(); size_t truePositives = tempMask->GetCount<true>();
size_t falsePositives = totalPositives - truePositives; size_t falsePositives = totalPositives - truePositives;
tempMask->Invert();
num_t tpSum = ThresholdTools::Sum(inputImage, tempMask);
num_t noiseSum = ThresholdTools::Sum(rfiLessImage, tempMask);
double tpRatio = (double) truePositives / totalRFI; double tpRatio = (double) truePositives / totalRFI;
double fpRatio = (double) falsePositives / totalRFI; double fpRatio = (double) falsePositives / totalRFI;
grDilTpRatio[i] += tpRatio; grDilTpRatio[i] += tpRatio;
grDilFpRatio[i] += fpRatio; grDilFpRatio[i] += fpRatio;
grDilTpSum[i] += (tpSum - noiseSum)/totalRFISum; grDilTpSum[i] += tpFuzzyRatio;
grDilFpSum[i] += (flaggedSum - tpSum)/totalRFISum; grDilFpSum[i] += fpFuzzyRatio;
} }
grTotalRFI += totalRFI; //grTotalRFI += totalRFI;
grTotalRFISum += totalRFISum; //grTotalRFISum += totalRFISum;
std::cout << '.' << std::flush; std::cout << '.' << std::flush;
} }
...@@ -343,7 +335,7 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType) ...@@ -343,7 +335,7 @@ void RankOperatorROCExperiment::executeTest(enum TestType testType)
<< '\n'; << '\n';
} }
TestNoisePerformance(grTotalRFI / _repeatCount, grTotalRFISum / _repeatCount, testname); //TestNoisePerformance(grTotalRFI / _repeatCount, grTotalRFISum / _repeatCount, testname);
} }
inline void RankOperatorROCExperiment::TestNoisePerformance(size_t totalRFI, double totalRFISum, const std::string &testname) inline void RankOperatorROCExperiment::TestNoisePerformance(size_t totalRFI, double totalRFISum, const std::string &testname)
......
...@@ -316,7 +316,7 @@ namespace rfiStrategy { ...@@ -316,7 +316,7 @@ namespace rfiStrategy {
maxRecommendedBufferSize = msImageSet->Reader()->GetMaxRecommendedBufferSize(threadCount) - _action.GetBaselinesInBufferCount(); maxRecommendedBufferSize = msImageSet->Reader()->GetMaxRecommendedBufferSize(threadCount) - _action.GetBaselinesInBufferCount();
} else { } else {
minRecommendedBufferSize = 1; minRecommendedBufferSize = 1;
maxRecommendedBufferSize = 1; maxRecommendedBufferSize = 2;
} }
do { do {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment