diff --git a/CEP/DP3/CMakeLists.txt b/CEP/DP3/CMakeLists.txt
deleted file mode 100644
index d66c282e71e7b17be773fe0fad36d470eae3866b..0000000000000000000000000000000000000000
--- a/CEP/DP3/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-# $Id$
-
-lofar_add_package(DPPP)
-lofar_add_package(TestDynDPPP)
-lofar_add_package(PythonDPPP)
-lofar_add_package(DPPP_AOFlag)
-lofar_add_package(SPW_Combine SPWCombine)
-
-lofar_find_package(Armadillo)
-if(${ARMADILLO_FOUND})
- if(CMAKE_CXX_FLAGS MATCHES ".*\\+\\+11.*")
- lofar_add_package(DPPP_DDECal)
-lofar_add_package(DPPP_Interpolate)
- else()
- message(WARNING "DPPP_DDECal and DPPP_Interpolate will not build if you have no C++11 support, please build in directory gnucxx11_opt")
- endif()
-else()
- message(WARNING "Armadillo was not found, NOT building DPPP_DDECal")
-endif()
diff --git a/CEP/DP3/DPPP/CMakeLists.txt b/CEP/DP3/DPPP/CMakeLists.txt
deleted file mode 100644
index dc04158ff3ccc12138512b609edb3fd8ef84f427..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# $Id$
-
-lofar_package(DPPP 1.0 DEPENDS LofarStMan Common MS ParmDB StationResponse)
-
-include(LofarFindPackage)
-lofar_find_package(Casacore COMPONENTS casa ms tables REQUIRED)
-lofar_find_package(HDF5 COMPONENTS CXX REQUIRED)
-
-add_subdirectory(include/DPPP)
-add_subdirectory(src)
-add_subdirectory(share)
-add_subdirectory(etc)
-add_subdirectory(test)
diff --git a/CEP/DP3/DPPP/etc/CMakeLists.txt b/CEP/DP3/DPPP/etc/CMakeLists.txt
deleted file mode 100644
index 75d2e7c9a9efd373eca247d08848e74c273ec6e4..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/etc/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-# $Id: CMakeLists.txt 22196 2012-10-03 05:47:41Z diepen $
-
-# Logger configuration
-install(FILES
- DPPP.log_prop
- DESTINATION etc)
diff --git a/CEP/DP3/DPPP/etc/DPPP.log_prop b/CEP/DP3/DPPP/etc/DPPP.log_prop
deleted file mode 100644
index e44074d626ddb606bee730941fa8c6e4f4de5372..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/etc/DPPP.log_prop
+++ /dev/null
@@ -1,7 +0,0 @@
-log4cplus.rootLogger=TRACE, TESTOUT
-log4cplus.logger.TRC=DEBUG
-log4cplus.appender.TESTOUT=log4cplus::ConsoleAppender
-log4cplus.appender.TESTOUT.logToStdErr=False
-log4cplus.appender.TESTOUT.layout=log4cplus::PatternLayout
-#log4cplus.appender.TESTOUT.layout.ConversionPattern=%-5p %c{3} [%b:%L] - %m%n
-log4cplus.appender.TESTOUT.layout.ConversionPattern=%-5p - %m%n
diff --git a/CEP/DP3/DPPP/include/DPPP/Apply.h b/CEP/DP3/DPPP/include/DPPP/Apply.h
deleted file mode 100644
index 5ae8db11b829d6ab4423461cc924588e1a227128..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Apply.h
+++ /dev/null
@@ -1,62 +0,0 @@
-//# Apply.h: Apply station Jones matrices to a set of visibilities.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_APPLY_H
-#define DPPP_APPLY_H
-
-// \file
-// Apply station Jones matrices to a set of visibilities.
-
-#include <DPPP/Baseline.h>
-#include <DPPP/Cursor.h>
-#include <Common/lofar_complex.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-// Apply station Jones matrices to a set of visibilities.
-//
-// \param[in] nBaseline
-// Number of baselines.
-// \param[in] nChannel
-// Number of frequency channels.
-// \param[in] baselines
-// A cursor for a 1-D buffer of baselines of shape (\p nBaseline).
-// \param[in] coeff
-// A cursor for a 2-D buffer of Jones matrix coefficients of shape
-// (No. of stations, 8). Each station index contained in \p baselines should be
-// a valid index for the first axis of \p coeff.
-// \param[in] data
-// A cursor for a 3-D buffer of visibilities of shape
-// (\p nBaseline, \p nChannel, 4).
-void apply(size_t nBaseline, size_t nChannel, const_cursor<Baseline> baselines,
- const_cursor<double> coeff, cursor<dcomplex> data);
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/ApplyBeam.h b/CEP/DP3/DPPP/include/DPPP/ApplyBeam.h
deleted file mode 100644
index b8cf9b7611e45018750c28f18a51ee13d7fcd076..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/ApplyBeam.h
+++ /dev/null
@@ -1,124 +0,0 @@
-//# ApplyBeam.h: DPPP step class to ApplyBeam visibilities from a source model
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_APPLYBEAM_H
-#define DPPP_APPLYBEAM_H
-
-// @file
-// @brief DPPP step class to apply the beam model (optionally inverted)
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/Position.h>
-#include <StationResponse/Station.h>
-#include <StationResponse/Types.h>
-#include <casacore/casa/Arrays/Cube.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to apply the beam model, optionally inverted.
- // The input MeasurementSet it operates on, must have the LOFAR subtables
- // defining the station layout and tiles/dipoles used.
-
- class ApplyBeam: public DPStep
- {
- public:
- // Modes for the beam: only array factor, only element beam, or both (default)
- enum BeamMode {DEFAULT=1, ARRAY_FACTOR=2, ELEMENT=3};
-
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- ApplyBeam (DPInput*, const ParameterSet&, const string& prefix, bool substep=false);
-
- ApplyBeam();
-
- virtual ~ApplyBeam();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process(const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo(const DPInfo&);
-
- // Show the step parameters.
- virtual void show(std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings(std::ostream&, double duration) const;
-
- bool invert() {
- return itsInvert;
- }
-
- template<typename T>
- static void applyBeam(
- const DPInfo& info, double time, T* data0, float* weight0,
- const StationResponse::vector3r_t& srcdir,
- const StationResponse::vector3r_t& refdir,
- const StationResponse::vector3r_t& tiledir,
- const vector<StationResponse::Station::Ptr>& antBeamInfo,
- vector<StationResponse::matrix22c_t>& beamValues,
- bool useChannelFreq, bool invert, int mode,
- bool doUpdateWeights=false);
-
- private:
- StationResponse::vector3r_t dir2Itrf(
- const casacore::MDirection& dir,
- casacore::MDirection::Convert& measConverter);
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuffer;
- bool itsInvert;
- bool itsUpdateWeights;
- bool itsUseChannelFreq;
- Position itsPhaseRef;
- BeamMode itsMode;
-
- uint itsDebugLevel;
-
- // The info needed to calculate the station beams.
- vector<vector<StationResponse::Station::Ptr> > itsAntBeamInfo;
- vector<casacore::MeasFrame> itsMeasFrames;
- vector<casacore::MDirection::Convert> itsMeasConverters;
- vector<vector<StationResponse::matrix22c_t> > itsBeamValues;
-
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-#endif
-
-#include "ApplyBeam.tcc"
diff --git a/CEP/DP3/DPPP/include/DPPP/ApplyBeam.tcc b/CEP/DP3/DPPP/include/DPPP/ApplyBeam.tcc
deleted file mode 100644
index 40b9869f637c24375f5d155b6f21edf85fdc0b08..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/ApplyBeam.tcc
+++ /dev/null
@@ -1,175 +0,0 @@
-//# ApplyBeam.tcc: DPPP step class to ApplyBeam visibilities from a source model
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_ApplyBeam_TCC
-#define DPPP_ApplyBeam_TCC
-
-#include <lofar_config.h>
-#include <DPPP/ApplyBeam.h>
-
-#include <Common/ParameterSet.h>
-#include <Common/Timer.h>
-#include <Common/OpenMP.h>
-#include <ParmDB/ParmDBMeta.h>
-#include <ParmDB/PatchInfo.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/FlagCounter.h>
-#include <DPPP/Position.h>
-#include <DPPP/Simulator.h>
-#include <DPPP/Simulate.h>
-#include <DPPP/ApplyCal.h>
-
-#include <StationResponse/AntennaField.h>
-
-#include <DPPP/Stokes.h>
-#include <DPPP/PointSource.h>
-#include <DPPP/GaussianSource.h>
-#include <ParmDB/SourceDB.h>
-
-#include <casacore/casa/Arrays/Array.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/tables/Tables/RefRows.h>
-
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-namespace LOFAR {
- namespace DPPP {
-
-
-// applyBeam is templated on the type of the data, could be double or float
-template<typename T>
-void ApplyBeam::applyBeam(
- const DPInfo& info, double time, T* data0, float* weight0,
- const StationResponse::vector3r_t& srcdir,
- const StationResponse::vector3r_t& refdir,
- const StationResponse::vector3r_t& tiledir,
- const vector<StationResponse::Station::Ptr>& antBeamInfo,
- vector<StationResponse::matrix22c_t>& beamValues, bool useChannelFreq,
- bool invert, int mode, bool doUpdateWeights)
- {
- // Get the beam values for each station.
- uint nCh = info.chanFreqs().size();
- uint nSt = beamValues.size() / nCh;
- uint nBl = info.nbaselines();
-
- // Store array factor in diagonal matrix (in other modes this variable
- // is not used).
- StationResponse::diag22c_t af_tmp;
-
- double reffreq=info.refFreq();
-
- // Apply the beam values of both stations to the ApplyBeamed data.
- dcomplex tmp[4];
- for (size_t ch = 0; ch < nCh; ++ch) {
- if (useChannelFreq) {
- reffreq=info.chanFreqs()[ch];
- }
-
- switch (mode) {
- case DEFAULT:
- // Fill beamValues for channel ch
- for (size_t st = 0; st < nSt; ++st) {
- beamValues[nCh * st + ch] = antBeamInfo[st]->response(time,
- info.chanFreqs()[ch], srcdir,
- reffreq, refdir, tiledir);
- if (invert) {
- ApplyCal::invert((dcomplex*)(&(beamValues[nCh * st + ch])));
- }
- }
- break;
- case ARRAY_FACTOR:
- // Fill beamValues for channel ch
- for (size_t st = 0; st < nSt; ++st) {
- af_tmp = antBeamInfo[st]->arrayFactor(time,
- info.chanFreqs()[ch], srcdir,
- reffreq, refdir, tiledir);
- beamValues[nCh * st + ch][0][1]=0.;
- beamValues[nCh * st + ch][1][0]=0.;
-
- if (invert) {
- beamValues[nCh * st + ch][0][0]=1./af_tmp[0];
- beamValues[nCh * st + ch][1][1]=1./af_tmp[1];
- } else {
- beamValues[nCh * st + ch][0][0]=af_tmp[0];
- beamValues[nCh * st + ch][1][1]=af_tmp[1];
- }
- }
- break;
- case ELEMENT:
- // Fill beamValues for channel ch
- for (size_t st = 0; st < nSt; ++st) {
- LOFAR::StationResponse::AntennaField::ConstPtr field =
- *(antBeamInfo[st]->beginFields());
-
- beamValues[nCh * st + ch] = field->elementResponse(time,
- info.chanFreqs()[ch],
- srcdir);
- if (invert) {
- ApplyCal::invert((dcomplex*)(&(beamValues[nCh * st + ch])));
- }
- }
- break;
- }
-
- // Apply beam for channel ch on all baselines
- // For mode=ARRAY_FACTOR, too much work is done here because we know
- // that r and l are diagonal
- for (size_t bl = 0; bl < nBl; ++bl) {
- T* data = data0 + bl * 4 * nCh + ch * 4;
- StationResponse::matrix22c_t *left = &(beamValues[nCh
- * info.getAnt1()[bl]]);
- StationResponse::matrix22c_t *right = &(beamValues[nCh
- * info.getAnt2()[bl]]);
- dcomplex l[] = { left[ch][0][0], left[ch][0][1],
- left[ch][1][0], left[ch][1][1] };
- // Form transposed conjugate of right.
- dcomplex r[] = { conj(right[ch][0][0]), conj(right[ch][1][0]),
- conj(right[ch][0][1]), conj(right[ch][1][1]) };
- // left*data
- tmp[0] = l[0] * dcomplex(data[0]) + l[1] * dcomplex(data[2]);
- tmp[1] = l[0] * dcomplex(data[1]) + l[1] * dcomplex(data[3]);
- tmp[2] = l[2] * dcomplex(data[0]) + l[3] * dcomplex(data[2]);
- tmp[3] = l[2] * dcomplex(data[1]) + l[3] * dcomplex(data[3]);
- // data*conj(right)
- data[0] = tmp[0] * r[0] + tmp[1] * r[2];
- data[1] = tmp[0] * r[1] + tmp[1] * r[3];
- data[2] = tmp[2] * r[0] + tmp[3] * r[2];
- data[3] = tmp[2] * r[1] + tmp[3] * r[3];
-
- if (doUpdateWeights) {
- ApplyCal::applyWeights(l, r, weight0 + bl * 4 * nCh + ch * 4);
- }
- }
- }
- }
- }
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/ApplyCal.h b/CEP/DP3/DPPP/include/DPPP/ApplyCal.h
deleted file mode 100644
index cf458c442a8b585056348c7a58e617b5b733a981..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/ApplyCal.h
+++ /dev/null
@@ -1,130 +0,0 @@
-//# ApplyCal.h: DPPP step class to ApplyCal visibilities from a source model
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_ApplyCal_H
-#define DPPP_ApplyCal_H
-
-// @file
-// @brief DPPP step class to apply multiple calibration solutions
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-
-#include <DPPP/OneApplyCal.h>
-
-#include <utility>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to apply multiple ParmDB or H5Parm
- // solutions to data.
-
- class ApplyCal: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- ApplyCal (DPInput*, const ParameterSet&, const string& prefix,
- bool substep=false, std::string predictDirection="");
-
- // Empty constructor
- ApplyCal ();
-
- virtual ~ApplyCal();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Set the next step. It squeezes in the actual OneApplyCal steps
- // between this ApplyCal step and the next step.
- virtual void setNextStep (DPStep::ShPtr nextStep);
-
- // Show the step. When ApplyCal is a step in the main chain, this does
- // nothing; the nextStep mechanism in DPRun will call show on the actual
- // OneApplyCals.
- virtual void show(std::ostream&) const;
-
- // Show the timings. When ApplyCal is a step in the main chain, this does
- // nothing; the nextStep mechanism in DPRun will call show on the actual
- // OneApplyCals.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Invert a 2x2 matrix in place
- static void invert (casacore::DComplex* v, double sigmaMMSE=0);
-
- // Apply a diagonal Jones matrix to the 2x2 visibilities matrix: A.V.B^H
- static void applyDiag (const casacore::DComplex* gainA,
- const casacore::DComplex* gainB,
- casacore::Complex* vis, float* weight, bool* flag,
- uint bl, uint chan, bool updateWeights,
- FlagCounter& flagCounter);
-
- // Apply a diagonal Jones matrix to the 2x2 visibilities matrix: A.V.B^H,
- // where the solution is equal for both polarizations
- static void applyScalar(const casacore::DComplex* gainA,
- const casacore::DComplex* gainB,
- casacore::Complex* vis, float* weight, bool* flag,
- uint bl, uint chan, bool updateWeights,
- FlagCounter& flagCounter);
-
- // Apply a full Jones matrix to the 2x2 visibilities matrix: A.V.B^H
- static void applyFull (const casacore::DComplex* gainA,
- const casacore::DComplex* gainB,
- casacore::Complex* vis, float* weight, bool* flag,
- uint bl, uint chan, bool updateWeights,
- FlagCounter& flagCounter);
-
- // Do the same as the combination of BBS + python script
- // covariance2weight.py (cookbook), except it stores weights per freq.
- // The diagonal of covariance matrix is transferred to the weights.
- // Note that the real covariance (mixing of noise terms after which they
- // are not independent anymore) is not stored.
- // The input covariance matrix C is assumed to be diagonal with elements
- // w_i (the weights), the result the diagonal of
- // (gainA kronecker gainB^H).C.(gainA kronecker gainB^H)^H
- static void applyWeights (const casacore::DComplex* gainA,
- const casacore::DComplex* gainB,
- float* weight);
-
- private:
- //# Data members.
- bool itsIsSubstep;
- string itsName;
-
- std::vector<OneApplyCal::ShPtr> itsApplyCals;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Averager.h b/CEP/DP3/DPPP/include/DPPP/Averager.h
deleted file mode 100644
index 90e40f47688c0b8cd2f39e5d8a32ba3d61e86055..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Averager.h
+++ /dev/null
@@ -1,123 +0,0 @@
-//# Averager.h: DPPP step class to average in time and/or freq
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_AVERAGER_H
-#define DPPP_AVERAGER_H
-
-// @file
-// @brief DPPP step class to average in time and/or freq
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <casacore/casa/Arrays/Cube.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class calculating the weighted average of
- // data in time and/or frequency.
- // <br>
- // Only unflagged data points are used. The average is calculated as
- // <tt>sum(data*weight) / sum(weight)</tt> and the sum of the weights
- // is the weight of the new data point. If all data point to use are
- // flagged, the resulting data point and weight are set to zero and flagged.
- //
- // It keeps track of the FullResFlags. It sets them if the corresponding
- // data point is flagged. Note that multiple FullResFlags elements map to
- // a single data point if some averaging was done before.
-
- class Averager: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- Averager (DPInput*, const ParameterSet&, const string& prefix);
-
- // Construct the object using the given parameters.
- Averager (DPInput*, const string& stepname,
- uint nchanAvg, uint ntimeAvg);
-
- virtual ~Averager();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- // Average into itsBufOut.
- void average();
-
- // Copy the fullRes flags in the input buffer to the correct
- // time index in the output buffer.
- // If a flag is set, set all flags in corresponding FullRes window.
- void copyFullResFlags (const casacore::Cube<bool>& fullResFlags,
- const casacore::Cube<bool>& flags,
- int timeIndex);
-
- // Get the value in Hertz of a string like "1000 MHz". If unit is
- // omitted it defaults to Hertz
- double getFreqHz(const string& freqstr);
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuf;
- DPBuffer itsBufTmp;
- DPBuffer itsBufOut;
- casacore::Cube<int> itsNPoints;
- casacore::Cube<casacore::Complex> itsAvgAll;
- casacore::Cube<float> itsWeightAll;
- casacore::Cube<bool> itsFullResFlags;
- double itsFreqResolution;
- double itsTimeResolution;
- uint itsNChanAvg;
- uint itsNTimeAvg;
- uint itsMinNPoint;
- float itsMinPerc;
- uint itsNTimes;
- double itsTimeInterval;
- bool itsNoAvg; //# No averaging (i.e. both 1)?
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Baseline.h b/CEP/DP3/DPPP/include/DPPP/Baseline.h
deleted file mode 100644
index 5655c1389a74d5364e00ab522a291e2144f907fd..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Baseline.h
+++ /dev/null
@@ -1,47 +0,0 @@
-//# Baseline.h: Pair of stations that together form a baseline (interferometer).
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_BASELINE_H
-#define DPPP_BASELINE_H
-
-// \file
-// Pair of stations that together form a baseline (interferometer).
-
-#include <cstddef>
-#include <utility>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-typedef std::pair<size_t, size_t> Baseline;
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/BaselineSelection.h b/CEP/DP3/DPPP/include/DPPP/BaselineSelection.h
deleted file mode 100644
index 62239cb5ac64b34527a26068979e29cbf529e972..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/BaselineSelection.h
+++ /dev/null
@@ -1,102 +0,0 @@
-//# BaselineSelection.h: Class to handle the baseline selection
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_BASELINESELECTION_H
-#define DPPP_BASELINESELECTION_H
-
-// @file
-// @brief Class to handle the baseline selection
-
-#include <DPPP/DPInfo.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Arrays/Matrix.h>
-
-namespace LOFAR {
- class ParameterSet;
- class ParameterValue;
-
- namespace DPPP {
-
- // Class containing a few static functions to parse a baseline selection
- // string.
- class BaselineSelection
- {
- public:
- // Default constructor has no selection.
- BaselineSelection();
-
- // Construct from the parset using the given prefix.
- // The keys used are:
- // <ul>
- // <li> baseline: for a baseline selection (e.g. CS*&)
- // <li> corrtype: for correlation selection (auto, cross, or empty)
- // <li> blrange: ranges of baseline lengths (in m)
- // <li> minbl: minimum baseline length (in m); only if minmax=true
- // <li> maxbl: maximum baseline length (in m); only if minmax=true
- // </ul>
- BaselineSelection (const ParameterSet&, const string& prefix,
- bool minmax=false,
- const string& defaultCorrType=string(),
- const string& defaultBaseline=string());
-
- // Is there any selection?
- bool hasSelection() const;
-
- // Show the parameters.
- // Optional extra blanks can be put before the value.
- void show (ostream& os, const string& blanks = string()) const;
-
- // Form the selection matrix telling for each baseline if it is selected.
- // If no selection is made, all values in the matrix are true.
- casacore::Matrix<bool> apply (const DPInfo& info) const;
-
- // Form the selection vector telling if a baseline in the DPInfo object
- // is selected.
- casacore::Vector<bool> applyVec (const DPInfo& info) const;
-
- private:
- // Convert the baseline selection string.
- void handleBL (casacore::Matrix<bool>& selectBL,
- const DPInfo& info) const;
-
- // Handle a vector of baseline specifications.
- casacore::Matrix<bool> handleBLVector (const ParameterValue& pvBL,
- const casacore::Vector<casacore::String>&) const;
-
- // Handle the correlation type selection.
- void handleCorrType (casacore::Matrix<bool>& selectBL) const;
-
- // Handle the baseline length selection.
- void handleLength (casacore::Matrix<bool>& selectBL,
- const DPInfo& info) const;
-
- //# Data members
- string itsStrBL;
- string itsCorrType;
- vector<double> itsRangeBL;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/CMakeLists.txt b/CEP/DP3/DPPP/include/DPPP/CMakeLists.txt
deleted file mode 100644
index d760372fb01e50e01a557235c2dc92b76cd088dc..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/CMakeLists.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-# $Id$
-
-# List of header files that will be installed.
-set(inst_HEADERS
- DPRun.h DPStep.h DPInput.h DPBuffer.h DPInfo.h ApplyCal.h
- DPLogger.h ProgressMeter.h FlagCounter.h
- UVWCalculator.h BaselineSelection.h
- MSReader.h MSWriter.h MSUpdater.h Counter.h
- Averager.h MedFlagger.h PreFlagger.h UVWFlagger.h
- StationAdder.h ScaleData.h Filter.h
- PhaseShift.h Demixer.h
- Cursor.h CursorUtilCasa.h Position.h Stokes.h SourceDBUtil.h
- Apply.h EstimateMixed.h Simulate.h Simulator.h SubtractMixed.h Baseline.h
- ModelComponent.h PointSource.h GaussianSource.h Patch.h
- ModelComponentVisitor.h
- DemixerNew.h DemixInfo.h DemixWorker.h
- ApplyBeam.h ApplyBeam.tcc
- Predict.h OneApplyCal.h
- GainCal.h StefCal.h PhaseFitter.h
- StManParsetKeys.h H5Parm.h DummyStep.h H5ParmPredict.h GridInterpolate.h
- Upsample.h Split.h
-)
-
-# Create symbolic link to include directory.
-execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
- ${CMAKE_CURRENT_SOURCE_DIR}
- ${CMAKE_BINARY_DIR}/include/${PACKAGE_NAME})
-
-# Install header files.
-install(FILES ${inst_HEADERS} DESTINATION include/${PACKAGE_NAME})
diff --git a/CEP/DP3/DPPP/include/DPPP/Counter.h b/CEP/DP3/DPPP/include/DPPP/Counter.h
deleted file mode 100644
index 8215f2383e12162f511a6bd992366e01ada4f2a6..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Counter.h
+++ /dev/null
@@ -1,82 +0,0 @@
-//# Counter.h: DPPP step class to count flags
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_COUNTER_H
-#define DPPP_COUNTER_H
-
-// @file
-// @brief DPPP step class to count flags
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/FlagCounter.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class counting the number of flags per
- // baseline and channel.
- // It can be used for test purposes to know how many flags have been
- // set by the previous steps.
-
- class Counter: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- Counter (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~Counter();
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the flag counts.
- virtual void showCounts (std::ostream&) const;
-
- private:
- //# Data members.
- string itsName;
- bool itsFlagData;
- uint itsCount;
- FlagCounter itsFlagCounter;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Cursor.h b/CEP/DP3/DPPP/include/DPPP/Cursor.h
deleted file mode 100644
index c27954b002b50a84ce93c5a3c69bad6a013d077c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Cursor.h
+++ /dev/null
@@ -1,297 +0,0 @@
-//# Cursor.h: Multi-dimensional iterators.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_CURSOR_H
-#define DPPP_CURSOR_H
-
-// \file
-// Multi-dimensional iterators.
-
-#include <Common/lofar_algorithm.h>
-#include <Common/LofarLogger.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-template <typename T>
-class cursor
-{
-public:
- cursor()
- : itsPointer(0),
- itsRank(0)
- {
- }
-
- cursor(T *pointer)
- : itsPointer(pointer),
- itsRank(1)
- {
- fill(itsStrides, itsStrides + MAX_RANK, 0);
- itsStrides[0] = 1;
- }
-
- template <typename T_STRIDE>
- cursor(T *pointer, size_t rank, const T_STRIDE *strides)
- : itsPointer(pointer),
- itsRank(rank)
- {
- DBGASSERT (rank <= MAX_RANK);
- copy(strides, strides + itsRank, itsStrides);
- fill(itsStrides + itsRank, itsStrides + MAX_RANK, 0);
- }
-
- size_t rank() const
- {
- return itsRank;
- }
-
- cursor &operator++()
- {
- itsPointer += itsStrides[0];
- return *this;
- }
-
- cursor operator++(int)
- {
- cursor tmp = *this;
- itsPointer += itsStrides[0];
- return tmp;
- }
-
- cursor &operator+=(size_t n)
- {
- itsPointer += n * itsStrides[0];
- return *this;
- }
-
- cursor &operator-=(size_t n)
- {
- itsPointer -= n * itsStrides[0];
- return *this;
- }
-
- T &operator*()
- {
- return *itsPointer;
- }
-
- const T &operator*() const
- {
- return *itsPointer;
- }
-
- T *operator->()
- {
- return itsPointer;
- }
-
- const T *operator->() const
- {
- return itsPointer;
- }
-
- T &operator[](size_t n)
- {
- return *(itsPointer + n * itsStrides[0]);
- }
-
- const T &operator[](size_t n) const
- {
- return *(itsPointer + n * itsStrides[0]);
- }
-
- void forward(size_t i)
- {
- itsPointer += itsStrides[i];
- }
-
- void forward(size_t i, size_t n)
- {
- itsPointer += n * itsStrides[i];
- }
-
- void backward(size_t i)
- {
- itsPointer -= itsStrides[i];
- }
-
- void backward(size_t i, size_t n)
- {
- itsPointer -= n * itsStrides[i];
- }
-
- T *address()
- {
- return itsPointer;
- }
-
- const T *address() const
- {
- return itsPointer;
- }
-
- size_t stride(size_t i) const
- {
- return itsStrides[i];
- }
-
-private:
- static const size_t MAX_RANK = 5;
-
- T* itsPointer;
- size_t itsRank;
- size_t itsStrides[MAX_RANK];
-};
-
-template <typename T>
-class const_cursor
-{
-public:
- const_cursor()
- : itsPointer(0),
- itsRank(0)
- {
- }
-
- const_cursor(const T *pointer)
- : itsPointer(pointer),
- itsRank(1)
- {
- fill(itsStrides, itsStrides + MAX_RANK, 0);
- itsStrides[0] = 1;
- }
-
- template <typename T_STRIDE>
- const_cursor(const T *pointer, size_t rank, const T_STRIDE *strides)
- : itsPointer(pointer),
- itsRank(rank)
- {
- copy(strides, strides + itsRank, itsStrides);
- fill(itsStrides + itsRank, itsStrides + MAX_RANK, 0);
- }
-
- const_cursor(const cursor<T> &other)
- : itsPointer(other.address()),
- itsRank(other.rank())
- {
- for(size_t i = 0; i < itsRank; ++i)
- {
- itsStrides[i] = other.stride(i);
- }
- fill(itsStrides + itsRank, itsStrides + MAX_RANK, 0);
- }
-
- size_t rank() const
- {
- return itsRank;
- }
-
- const_cursor &operator++()
- {
- itsPointer += itsStrides[0];
- return *this;
- }
-
- const_cursor operator++(int)
- {
- const_cursor tmp = *this;
- itsPointer += itsStrides[0];
- return tmp;
- }
-
- const_cursor &operator+=(size_t n)
- {
- itsPointer += n * itsStrides[0];
- return *this;
- }
-
- const_cursor &operator-=(size_t n)
- {
- itsPointer -= n * itsStrides[0];
- return *this;
- }
-
- const T &operator*() const
- {
- return *itsPointer;
- }
-
- const T *operator->() const
- {
- return itsPointer;
- }
-
- const T &operator[](size_t n) const
- {
- return *(itsPointer + n * itsStrides[0]);
- }
-
- void forward(size_t i)
- {
- itsPointer += itsStrides[i];
- }
-
- void forward(size_t i, size_t n)
- {
- itsPointer += n * itsStrides[i];
- }
-
- void backward(size_t i)
- {
- itsPointer -= itsStrides[i];
- }
-
- void backward(size_t i, size_t n)
- {
- itsPointer -= n * itsStrides[i];
- }
-
- const T *address() const
- {
- return itsPointer;
- }
-
- size_t stride(size_t i) const
- {
- return itsStrides[i];
- }
-
-private:
- static const size_t MAX_RANK = 5;
-
- const T* itsPointer;
- size_t itsRank;
- size_t itsStrides[MAX_RANK];
-};
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/CursorUtilCasa.h b/CEP/DP3/DPPP/include/DPPP/CursorUtilCasa.h
deleted file mode 100644
index 807a710432863caaeb6d8a4a9b4a8d70a2fb5339..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/CursorUtilCasa.h
+++ /dev/null
@@ -1,71 +0,0 @@
-//# CursorUtilCasa.h: Helper functions for creating cursors for CASA arrays.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_CURSORUTILCASA_H
-#define DPPP_CURSORUTILCASA_H
-
-// \file
-// Helper functions for creating cursors for CASA arrays.
-
-#include <DPPP/Cursor.h>
-#include <casacore/casa/Arrays/Array.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-template <typename T>
-cursor<T> casa_cursor(casacore::Array<T> &array)
-{
- return cursor<T>(array.data(), array.ndim(), array.steps().storage());
-}
-
-template <typename T>
-cursor<T> casa_cursor(casacore::Array<T> &array, const casacore::IPosition &offset)
-{
- return cursor<T>(&(array(offset)), array.ndim(), array.steps().storage());
-}
-
-template <typename T>
-const_cursor<T> casa_const_cursor(const casacore::Array<T> &array)
-{
- return const_cursor<T>(array.data(), array.ndim(), array.steps().storage());
-}
-
-template <typename T>
-const_cursor<T> casa_const_cursor(const casacore::Array<T> &array,
- const casacore::IPosition &offset)
-{
- return const_cursor<T>(&(array(offset)), array.ndim(),
- array.steps().storage());
-}
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DPBuffer.h b/CEP/DP3/DPPP/include/DPPP/DPBuffer.h
deleted file mode 100644
index a5d00d8cde417dad26243ec233059f0ad478b549..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DPBuffer.h
+++ /dev/null
@@ -1,215 +0,0 @@
-//# DPBuffer.h: Buffer holding the data of a timeslot/band
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DPBUFFER_H
-#define DPPP_DPBUFFER_H
-
-/// @file
-/// @brief Buffer holding the data of a timeslot/band
-
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/BasicSL/Complex.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- // @ingroup NDPPP
-
- // This class holds the data for one time slot in Array variables.
- // It makes heavy use of reference semantics to avoid data copying
- // when data are pushed from one step to another.
- // This means that a data array can be shared between DPStep objects.
- // So if a DPStep object changes data in a buffer, it has to be sure
- // it can do it. If needed, Array::unique should be called to ensure
- // the array is not shared.
- //
- // The following data can be kept in a DPBuffer object.
- // <table>
- // <tr>
- // <td>TIME</td>
- // <td>The time slot center of the current data (in MJD seconds).</td>
- // </tr>
- // <tr>
- // <td>ROWNRS</td>
- // <td>The row numbers of the current time slot. It can be empty
- // when e.g. a time slot is inserted or if data are averaged.</td>
- // </tr>
- // <tr>
- // <td>DATA</td>
- // <td>The visibility data as [ncorr,nchan,nbaseline].</td>
- // </tr>
- // <tr>
- // <td>FLAG</td>
- // <td>The data flags as [ncorr,nchan,nbaseline] (True is bad).
- // Note that the ncorr axis is redundant because NDPPP will always
- // have the same flag for all correlations. The reason all
- // correlations are there is because the MS expects them.</td>
- // </tr>
- // <tr>
- // <td>WEIGHT</td>
- // <td>The data weights as [ncorr,nchan,nbaseline].
- // Similarly to FLAG the ncorr axis is redundant because the
- // same weight is used for all correlations.</td>
- // </tr>
- // <tr>
- // <td>UVW</td>
- // <td>The UVW coordinates in meters as [3,nbaseline].</td>
- // </tr>
- // <tr>
- // <td>FULLRESFLAG</td>
- // <td>The flags before any averaging was done. In the MS they are kept
- // in column LOFAR_FULL_RES_FLAG. They are used to deal in BBS
- // in a smart way with bandwidth and time smearing.
- // The shape of the array is [nchan, ntimeavg, nbaseline], where
- // ntimeavg is the number of time slots averaged to a single one.
- // The number of channels averaged to a single one can be determined
- // by dividing nchan by the number of channels in the data (or flags).
- // </td>
- // </tr>
- // </table>
- // The FLAG data member should always be filled in, so the first DPStep
- // (MSReader) will do that. The DATA data member is filled in if any
- // DPStep needs DATA. Other data members are filled on demand.
- // The DPInput::fetch functions should be used to get data for those
- // members. They take care that the input buffer's data are used if
- // available, otherwise they get it from the DPInput object.
- // In that way as little memory as needed is used. Note that e.g. the
- // AOFlagger can use a lot of memory if a large time window is used.
- //
- // Until early 2015 NDPPP used the strategy of shallow data copies.
- // I.e., a step increased the data reference counter and did not make
- // an actual copy. Only when data were changed, a new data array was made.
- // Thus MSReader allocated a new array when it read the data.
- // However, it appeared this strategy lead to memory fragmentation and
- // to sudden jumps in memory usage on Linux systems.
- // <br>Therefore the strategy was changed to having each step preallocate
- // its buffers and making deep copies when moving data from one step to
- // the next one. It appeared that it not only improved memory usage,
- // but also improved performance, possible due to far less mallocs.
- //
- // The buffer/step guidelines are as follows:
- // 1. If a step keeps a buffer for later processing (e.g. AORFlagger),
- // it must make a copy of the buffer because the input data arrays
- // might have changed before that step processes the data.
- // 2. A shallow copy of a data member can be used if a step processes
- // the data immediately (e.g. Averager).
- // The DPInput::fetch functions come in those 2 flavours.
-
- class DPBuffer
- {
- public:
- // Construct object with empty arrays.
- DPBuffer();
-
- // The copy constructor uses reference copies.
- DPBuffer (const DPBuffer&);
-
- // Assignment uses reference copies.
- DPBuffer& operator= (const DPBuffer&);
-
- // Make a deep copy of all arrays in that to this.
- void copy (const DPBuffer& that);
-
- // Reference only the arrays that are filled in that.
- void referenceFilled (const DPBuffer& that);
-
- // Set or get the visibility data per corr,chan,baseline.
- void setData (const casacore::Cube<casacore::Complex>& data)
- { itsData.reference (data); }
- const casacore::Cube<casacore::Complex>& getData() const
- { return itsData; }
- casacore::Cube<casacore::Complex>& getData()
- { return itsData; }
-
- // Set or get the flags per corr,chan,baseline.
- void setFlags (const casacore::Cube<bool>& flags)
- { itsFlags.reference (flags); }
- const casacore::Cube<bool>& getFlags() const
- { return itsFlags; }
- casacore::Cube<bool>& getFlags()
- { return itsFlags; }
-
- // Set or get the weights per corr,chan,baseline.
- void setWeights (const casacore::Cube<float>& weights)
- { itsWeights.reference (weights); }
- const casacore::Cube<float>& getWeights() const
- { return itsWeights; }
- casacore::Cube<float>& getWeights()
- { return itsWeights; }
-
- // Set or get the flags at the full resolution per chan,timeavg,baseline.
- void setFullResFlags (const casacore::Cube<bool>& flags)
- { itsFullResFlags.reference (flags); }
- const casacore::Cube<bool>& getFullResFlags() const
- { return itsFullResFlags; }
- casacore::Cube<bool>& getFullResFlags()
- { return itsFullResFlags; }
-
- // Get or set the time.
- void setTime (double time)
- { itsTime = time; }
- double getTime() const
- { return itsTime; }
-
- // Get or set the exposure.
- void setExposure (double exposure)
- { itsExposure = exposure; }
- double getExposure() const
- { return itsExposure; }
-
- // Get or set the row numbers used by the DPInput class.
- // It can be empty (e.g. when MSReader inserted a dummy time slot).
- void setRowNrs (const casacore::Vector<uint>& rownrs)
- { itsRowNrs.reference (rownrs); }
- const casacore::Vector<uint>& getRowNrs() const
- { return itsRowNrs; }
-
- // Get or set the UVW coordinates per baseline.
- void setUVW (const casacore::Matrix<double>& uvw)
- { itsUVW.reference (uvw); }
- const casacore::Matrix<double>& getUVW() const
- { return itsUVW; }
- casacore::Matrix<double>& getUVW()
- { return itsUVW; }
-
- // Merge the flags into the pre-average flags.
- // For each flagged point, the corresponding pre-average flags are set.
- static void mergeFullResFlags (casacore::Cube<bool>& fullResFlags,
- const casacore::Cube<bool>& flags);
-
- private:
- double itsTime;
- double itsExposure;
- casacore::Vector<uint> itsRowNrs;
- casacore::Cube<casacore::Complex> itsData; //# ncorr,nchan,nbasel
- casacore::Cube<bool> itsFlags; //# ncorr,nchan,nbasel
- casacore::Matrix<double> itsUVW; //# 3,nbasel
- casacore::Cube<float> itsWeights; //# ncorr,nchan,nbasel
- casacore::Cube<bool> itsFullResFlags; //# fullres_nchan,ntimeavg,nbl
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DPInfo.h b/CEP/DP3/DPPP/include/DPPP/DPInfo.h
deleted file mode 100644
index f47e61d5d81fcd87fa703f9ae5f961c735f8d0d9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DPInfo.h
+++ /dev/null
@@ -1,309 +0,0 @@
-//# DPInfo.h: General info about DPPP data processing attributes like averaging
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DPINFO_H
-#define DPPP_DPINFO_H
-
-// @file
-// @brief General info about DPPP data processing attributes like averaging
-
-#include <Common/LofarTypes.h>
-#include <Common/lofar_vector.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MPosition.h>
-#include <casacore/measures/Measures/MeasureHolder.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Containers/Record.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- //# Forward declarations.
- class DPInput;
-
- // @ingroup NDPPP
-
- // This class contains the information about the number of correlations,
- // channels, baselines, and times.
- // It is initialized by the first step and updated by steps like
- // Averager that change the number of channels or times.
- // Steps can take information from it to know about shapes.
-
- class DPInfo
- {
- public:
- // Default constructor.
- DPInfo();
-
- // Set the initial info from the input.
- void init (uint ncorr, uint nchan,
- uint ntime, double startTime, double timeInterval,
- const string& msName, const string& antennaSet);
-
- // Set nr of channels.
- void setNChan (uint nchan)
- { itsNChan = nchan; }
-
- // Set time interval
- void setTimeInterval (double timeInterval)
- { itsTimeInterval = timeInterval; }
-
- // Set the frequency info.
- // An empty resolutions or effectiveBW is default to chanWidths.
- // If totalBW is 0, it is set to the sum of effectiveBW.
- // If refFreq is 0, it is set to the middle of chanFreqs (mean if even).
- void set (const casacore::Vector<double>& chanFreqs,
- const casacore::Vector<double>& chanWidths,
- const casacore::Vector<double>& resolutions= casacore::Vector<double>(),
- const casacore::Vector<double>& effectiveBW= casacore::Vector<double>(),
- double totalBW = 0,
- double refFreq = 0);
-
- // Set array info.
- void set (const casacore::MPosition& arrayPos,
- const casacore::MDirection& phaseCenter,
- const casacore::MDirection& delayCenter,
- const casacore::MDirection& tileBeamDir);
-
- // Set the info for the given antennae and baselines.
- void set (const casacore::Vector<casacore::String>& antNames,
- const casacore::Vector<casacore::Double>& antDiam,
- const vector<casacore::MPosition>& antPos,
- const casacore::Vector<casacore::Int>& ant1,
- const casacore::Vector<casacore::Int>& ant2);
-
- // Set the name of the data column
- void setDataColName(const casacore::String& dataColName) {
- itsDataColName=dataColName;
- }
-
- // Set the name of the weight column
- void setWeightColName(const casacore::String& weightColName) {
- itsWeightColName=weightColName;
- }
-
- // Update the info for the given average factors.
- // If chanAvg is higher than the actual nr of channels, it is reset.
- // The same is true for timeAvg.
- // It returns the possibly reset nr of channels to average.
- uint update (uint chanAvg, uint timeAvg);
-
- // Update the info from the given selection parameters.
- // Optionally unused stations are really removed from the antenna lists.
- void update (uint startChan, uint nchan,
- const vector<uint>& baselines, bool remove);
-
- // Remove unused stations from the antenna lists.
- void removeUnusedAnt();
-
- // Set the phase center.
- // If original=true, it is set to the original phase center.
- void setPhaseCenter (const casacore::MDirection& phaseCenter, bool original)
- { itsPhaseCenter=phaseCenter; itsPhaseCenterIsOriginal = original; }
-
-
- // Get the info.
- const string& msName() const
- { return itsMSName; }
- const string& antennaSet() const
- { return itsAntennaSet; }
- uint ncorr() const
- { return itsNCorr; }
- uint nchan() const
- { return itsNChan; }
- uint startchan() const
- { return itsStartChan; }
- uint origNChan() const
- { return itsOrigNChan; }
- uint nchanAvg() const
- { return itsChanAvg; }
- uint nantenna() const
- { return itsAntNames.size(); }
- uint nbaselines() const
- { return itsAnt1.size(); }
- uint ntime() const
- { return itsNTime; }
- uint ntimeAvg() const
- { return itsTimeAvg; }
- double startTime() const
- { return itsStartTime; }
- double timeInterval() const
- { return itsTimeInterval; }
- const casacore::Vector<casacore::Int>& getAnt1() const
- { return itsAnt1; }
- const casacore::Vector<casacore::Int>& getAnt2() const
- { return itsAnt2; }
- const casacore::Vector<casacore::String>& antennaNames() const
- { return itsAntNames; }
- const casacore::Vector<casacore::Double>& antennaDiam() const
- { return itsAntDiam; }
- const vector<casacore::MPosition>& antennaPos() const
- { return itsAntPos; }
- const casacore::MPosition& arrayPos() const
- { return itsArrayPos; }
- const casacore::MPosition arrayPosCopy() const
- { return copyMeasure(casacore::MeasureHolder(itsArrayPos)).asMPosition(); }
- const casacore::MDirection& phaseCenter() const
- { return itsPhaseCenter; }
- const casacore::MDirection phaseCenterCopy() const
- { return copyMeasure(casacore::MeasureHolder(itsPhaseCenter)).asMDirection(); }
- bool phaseCenterIsOriginal() const
- { return itsPhaseCenterIsOriginal; }
- const casacore::MDirection& delayCenter() const
- { return itsDelayCenter; }
- const casacore::MDirection delayCenterCopy() const
- { return copyMeasure(casacore::MeasureHolder(itsDelayCenter)).asMDirection(); }
- const casacore::MDirection& tileBeamDir() const
- { return itsTileBeamDir; }
- const casacore::MDirection tileBeamDirCopy() const
- { return copyMeasure(casacore::MeasureHolder(itsTileBeamDir)).asMDirection(); }
- const casacore::Vector<double>& chanFreqs() const
- { return itsChanFreqs; }
- const casacore::Vector<double>& chanWidths() const
- { return itsChanWidths; }
- const casacore::Vector<double>& resolutions() const
- { return itsResolutions; }
- const casacore::Vector<double>& effectiveBW() const
- { return itsEffectiveBW; }
- const casacore::String& getDataColName() const
- { return itsDataColName; }
- const casacore::String& getWeightColName() const
- { return itsWeightColName; }
- double totalBW() const
- { return itsTotalBW; }
- double refFreq() const
- { return itsRefFreq; }
-
- // Get the antenna numbers actually used in the (selected) baselines.
- // E.g. [0,2,5,6]
- const vector<int>& antennaUsed() const
- { return itsAntUsed; }
-
- // Get the indices of all antennae in the used antenna vector above.
- // -1 means that the antenna is not used.
- // E.g. [0,-1,1,-1,-1,2,3] for the example above.
- const vector<int>& antennaMap() const
- { return itsAntMap; }
-
- // Are the visibility data needed?
- bool needVisData() const
- { return itsNeedVisData; }
- // Does the last step need to write data and/or flags?
- bool needWrite() const
- { return itsWriteData || itsWriteFlags || itsWriteWeights; }
- bool writeData() const
- { return itsWriteData; }
- bool writeFlags() const
- { return itsWriteFlags; }
- bool writeWeights() const
- { return itsWriteWeights; }
- // Has the meta data been changed in a step (precluding an update)?
- bool metaChanged() const
- { return itsMetaChanged; }
-
- // Set if visibility data needs to be read.
- void setNeedVisData()
- { itsNeedVisData = true; }
- // Set if data needs to be written.
- void setWriteData()
- { itsWriteData = true; }
- void setWriteFlags()
- { itsWriteFlags = true; }
- void setWriteWeights()
- { itsWriteWeights = true; }
- // Clear all write flags.
- void clearWrites()
- { itsWriteData = itsWriteFlags = itsWriteWeights = false; }
- // Set change of meta data.
- void setMetaChanged()
- { itsMetaChanged = true; }
- void clearMetaChanged()
- { itsMetaChanged = false; }
-
- // Get the baseline table index of the autocorrelations.
- // A negative value means there are no autocorrelations for that antenna.
- const vector<int>& getAutoCorrIndex() const;
-
- // Get the lengths of the baselines (in meters).
- const vector<double>& getBaselineLengths() const;
-
- // Convert to a Record.
- // The names of the fields in the record are the data names without 'its'.
- casacore::Record toRecord() const;
-
- // Update the DPInfo object from a Record.
- // It is possible that only a few fields are defined in the record.
- void fromRecord (const casacore::Record& rec);
-
- private:
- // Set which antennae are actually used.
- void setAntUsed();
-
- // Creates a real copy of a casacore::Measure by exporting to a Record
- static casacore::MeasureHolder copyMeasure(const casacore::MeasureHolder fromMeas);
-
- //# Data members.
- bool itsNeedVisData; //# Are the visibility data needed?
- bool itsWriteData; //# Must the data be written?
- bool itsWriteFlags; //# Must the flags be written?
- bool itsWriteWeights; //# Must the weights be written?
- bool itsMetaChanged; //# Are meta data changed? (e.g., by averaging)
- string itsMSName;
- casacore::String itsDataColName;
- casacore::String itsWeightColName;
- string itsAntennaSet;
- uint itsNCorr;
- uint itsStartChan;
- uint itsOrigNChan;
- uint itsNChan;
- uint itsChanAvg;
- uint itsNTime;
- uint itsTimeAvg;
- double itsStartTime;
- double itsTimeInterval;
- casacore::MDirection itsPhaseCenter;
- bool itsPhaseCenterIsOriginal;
- casacore::MDirection itsDelayCenter;
- casacore::MDirection itsTileBeamDir;
- casacore::MPosition itsArrayPos;
- casacore::Vector<double> itsChanFreqs;
- casacore::Vector<double> itsChanWidths;
- casacore::Vector<double> itsResolutions;
- casacore::Vector<double> itsEffectiveBW;
- double itsTotalBW;
- double itsRefFreq;
- casacore::Vector<casacore::String> itsAntNames;
- casacore::Vector<casacore::Double> itsAntDiam;
- vector<casacore::MPosition> itsAntPos;
- vector<int> itsAntUsed;
- vector<int> itsAntMap;
- casacore::Vector<casacore::Int> itsAnt1; //# ant1 of all baselines
- casacore::Vector<casacore::Int> itsAnt2; //# ant2 of all baselines
- mutable vector<double> itsBLength; //# baseline lengths
- mutable vector<int> itsAutoCorrIndex; //# autocorr index per ant
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DPInput.h b/CEP/DP3/DPPP/include/DPPP/DPInput.h
deleted file mode 100644
index 4f1370ca9d62fa35a43081abf96253da958cc344..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DPInput.h
+++ /dev/null
@@ -1,140 +0,0 @@
-//# DPInput.h: Abstract base class for a DPStep generating input
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DPINPUT_H
-#define DPPP_DPINPUT_H
-
-// @file
-// @brief Abstract base class for a DPStep generating input
-
-#include <DPPP/DPStep.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/UVWCalculator.h>
-#include <DPPP/FlagCounter.h>
-#include <StationResponse/Station.h>
-#include <Common/lofar_vector.h>
-
-#include <casacore/tables/Tables/TableIter.h>
-#include <casacore/tables/Tables/RefRows.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Arrays/Slicer.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MPosition.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- // @ingroup NDPPP
-
- // This class is the abstract base class for a DPStep object that
- // handles the input. A concrete example is MSReader that reads the
- // data from a MeasurementSet. However, it is also possible to have
- // input steps generating data on the fly as done in test programs
- // like tAverager.cc.
- //
- // A particular task of the class is to fetch the input for various
- // data items like weight, uvw, etc.. This is done by testing if the
- // item's data array is in the DPBuffer. If so, it will be returned.
- // Otherwise the appropriate 'get' function will be called to read the
- // data array from the input.
- // The derived classes should implement those 'get' functions, unless
- // they are sure the data arrays are always put in the buffer.
-
- class DPInput: public DPStep
- {
- public:
- // Define the shared pointer for this type.
- typedef shared_ptr<DPInput> ShPtr;
-
- virtual ~DPInput();
-
- // Read the UVW at the given row numbers into the buffer.
- // The default implementation throws an exception.
- virtual void getUVW (const casacore::RefRows& rowNrs,
- double time,
- DPBuffer&);
-
- // Read the weights at the given row numbers into the buffer.
- // The default implementation throws an exception.
- virtual void getWeights (const casacore::RefRows& rowNrs,
- DPBuffer&);
-
- // Read the fullRes flags (LOFAR_FULL_RES_FLAG) at the given row numbers
- // into the buffer.
- // If undefined, false is returned.
- // The default implementation throws an exception.
- virtual bool getFullResFlags (const casacore::RefRows& rowNrs,
- DPBuffer&);
-
- // Read the model data at the given row numbers into the array.
- // The default implementation throws an exception.
- virtual void getModelData (const casacore::RefRows& rowNrs,
- casacore::Cube<casacore::Complex>&);
-
- // Get the MS name.
- // The default implementation returns an empty string.
- virtual casacore::String msName() const;
-
- // Fill the vector with station beam info from the input source (MS).
- // Only fill it for the given station names.
- // The default implementation throws an exception.
- virtual void fillBeamInfo (vector<StationResponse::Station::Ptr>&,
- const casacore::Vector<casacore::String>& antNames);
-
- // Fetch the FullRes flags.
- // If defined in the buffer, they are taken from there.
- // Otherwise there are read from the input.
- // If not defined in the input, they are filled using the flags in the
- // buffer assuming that no averaging has been done so far.
- // <src>If desired, they can be merged with the buffer's FLAG which means
- // that if an averaged channel is flagged, the corresponding FullRes
- // flags are set.
- // <br>It does a stop/start of the timer when actually reading the data.
- const casacore::Cube<bool>& fetchFullResFlags (const DPBuffer& bufin,
- DPBuffer& bufout,
- NSTimer& timer,
- bool merge=false);
-
- // Fetch the weights.
- // If defined in the buffer, they are taken from there.
- // Otherwise there are read from the input.
- // If they have to be read and if autoweighting is in effect, the buffer
- // must contain DATA to calculate the weights.
- // <br>It does a stop/start of the timer when actually reading the data.
- const casacore::Cube<float>& fetchWeights (const DPBuffer& bufin,
- DPBuffer& bufout,
- NSTimer& timer);
-
- // Fetch the UVW.
- // If defined in the buffer, they are taken from there.
- // Otherwise there are read from the input.
- // <br>It does a stop/start of the timer when actually reading the data.
- const casacore::Matrix<double>& fetchUVW (const DPBuffer& bufin,
- DPBuffer& bufout,
- NSTimer& timer);
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DPLogger.h b/CEP/DP3/DPPP/include/DPPP/DPLogger.h
deleted file mode 100644
index 2241ae8216f1ac95a0c822e2d9d9345d8fe41b33..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DPLogger.h
+++ /dev/null
@@ -1,77 +0,0 @@
-//# DPLogger.h: Log on cout/cerr or through the logging system
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DPLOGGER_H
-#define DPPP_DPLOGGER_H
-
-// @file
-// @brief Log on cout/cerr or through the logging system
-
-#include <Common/LofarLogger.h>
-#include <iostream>
-
-namespace LOFAR {
- namespace DPPP {
-
- // This class contains the flag to choose between cout/cerr and logging
- // system.
- class DPLogger
- {
- public:
- static bool useLogger;
- };
- }
-}
-
-// Log an informational message.
-#define DPLOG_INFO_STR(stream) \
- if (DPLogger::useLogger) { \
- LOG_INFO_STR (stream); \
- } else { \
- std::cout << stream << endl; \
- }
-
-// Log a fatal message.
-#define DPLOG_WARN_STR(stream) \
- if (DPLogger::useLogger) { \
- LOG_WARN_STR (stream); \
- } else { \
- std::cerr << stream << endl; \
- }
-
-// Log an informational message.
-#define DPLOG_INFO(msg, removeEndl) \
- std::string str(msg); \
- if (removeEndl && str.size() > 0 && str[str.size()-1] == '\n') { \
- str = str.substr(0, str.size()-1); \
- } \
- if (DPLogger::useLogger) { \
- LOG_INFO (str); \
- } else { \
- std::cout << str << endl; \
- }
-
-#define LOGCOUT(msg) \
- { std::ostringstream ostr; ostr<<msg; printf("%s\n", ostr.str().c_str()); }
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DPRun.h b/CEP/DP3/DPPP/include/DPPP/DPRun.h
deleted file mode 100644
index 53a85043bbd3058c99c176ae45eee1823000a939..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DPRun.h
+++ /dev/null
@@ -1,94 +0,0 @@
-//# DPRun.h: Class to run steps like averaging and flagging on an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DPRUN_H
-#define DPPP_DPRUN_H
-
-// @file
-// @brief Class to run steps like averaging and flagging on an MS
-
-#include <lofar_config.h>
-#include <DPPP/DPStep.h>
-#include <DPPP/MSReader.h>
-#include <Common/ParameterSet.h>
-
-#include <map>
-
-namespace LOFAR {
- namespace DPPP {
-
- // @ingroup NDPPP
-
- // This class contains a single static function that creates and executes
- // the steps defined in the parset file.
- // The parset file is documented on the LOFAR wiki.
-
- class DPRun
- {
- public:
- // Define the function to create a step from the given parameterset.
- typedef DPStep::ShPtr StepCtor (DPInput*, const ParameterSet&,
- const std::string& prefix);
-
- // Add a function creating a DPStep to the map.
- static void registerStepCtor (const std::string&, StepCtor*);
-
- // Create a step object from the given parameters.
- // It looks up the step type in theirStepMap. If not found, it will
- // try to load a shared library with that name and execute the
- // register function in it.
- static StepCtor* findStepCtor (const std::string& type);
-
- // Execute the steps defined in the parset file.
- // Possible parameters given at the command line are taken into account.
- static void execute (const std::string& parsetName,
- int argc=0, char* argv[] = 0);
-
- // Create the step objects.
- static DPStep::ShPtr makeSteps (const ParameterSet& parset,
- const std::string& prefix,
- DPInput* reader);
-
- private:
- // Create an output step, either an MSWriter or an MSUpdater
- // If no data are modified (for example if only count was done),
- // still an MSUpdater is created, but it will not write anything.
- // It reads the output name from the parset. If the prefix is "", it
- // reads msout or msout.name, otherwise it reads name from the output step
- // Create an updater step if an input MS was given; otherwise a writer.
- // Create an updater step only if needed (e.g. not if only count is done).
- // If the user specified an output MS name, a writer or updater is always created
- // If there is a writer, the reader needs to read the visibility data.
- // reader should be the original reader
- static DPStep::ShPtr makeOutputStep(MSReader* reader,
- const ParameterSet& parset, const string& prefix,
- casacore::String& currentMSName);
-
- // The map to create a step object from its type name.
- static std::map<std::string, StepCtor*> theirStepMap;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DPStep.h b/CEP/DP3/DPPP/include/DPPP/DPStep.h
deleted file mode 100644
index b0f9509a649bbec5af113afcc4381e59b7a06c8a..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DPStep.h
+++ /dev/null
@@ -1,262 +0,0 @@
-//# DPStep.h: Abstract base class for a DPPP step
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DPSTEP_H
-#define DPPP_DPSTEP_H
-
-// @file
-// @brief Class to hold code for virtual base class for Flaggers in DPPP
-
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/lofar_smartptr.h>
-#include <Common/Timer.h>
-#include <iosfwd>
-
-namespace LOFAR {
- namespace DPPP {
-
- // @ingroup NDPPP
-
- // This class defines a step in the DPPP pipeline.
- // It is an abstract class from which all steps should be derived.
- // A few functions can or must be implemented. They are called by
- // the NDPPP program in the following order.
- // <ul>
- // <li> 'updateInfo' should update its DPInfo object with the specific
- // step information. For example, in this way it is known
- // in all steps how the data are averaged and what the shape is.
- // <li> 'show' can be used to show the attributes.
- // <li> 'process' is called continuously to process the next time slot.
- // When processed, it should call 'process' of the next step.
- // When done (i.e. at the end of the input), it should return False.
- // <li> 'finish' finishes the processing which could mean that 'process'
- // of the next step has to be called several times. When done,
- // it should call 'finish' of the next step.
- // <li> 'addToMS' is called after 'finish'. It gives a step the opportunity
- // to add some data to the MS written/updated. It is, for example,
- // used by AOFlagger to write its statistics.
- // <li> 'showCounts' can be used to show possible counts of flags, etc.
- // </ul>
- // A DPStep object contains a DPInfo object telling the data settings for
- // a step (like channel info, baseline info, etc.).
-
- class DPStep
- {
- public:
- // Define the shared pointer for this type.
- typedef shared_ptr<DPStep> ShPtr;
-
- // Constructor to initialize.
- DPStep()
- : itsPrevStep(0)
- {}
-
- // Destructor.
- virtual ~DPStep();
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- // It should return False at the end.
- virtual bool process (const DPBuffer&) = 0;
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish() = 0;
-
- // Set the info of this step and its next step.
- // It calls the virtual function updateInfo to do the real work.
- // It returns the info of the last step.
- const DPInfo& setInfo (const DPInfo&);
-
- // Get access to the info.
- const DPInfo& getInfo() const
- { return itsInfo; }
-
- // Add some data to the MeasurementSet written/updated.
- // The default implementation only calls addToMS from the previous step
- virtual void addToMS (const string& msName);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const = 0;
-
- // Show the flag counts if needed.
- // The default implementation does nothing.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- // The default implementation does nothing.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Set the previous step.
- void setPrevStep (DPStep* prevStep)
- { itsPrevStep = prevStep; }
-
- // Get the previous step.
- DPStep* getPrevStep () const
- { return itsPrevStep; }
-
- // Set the next step.
- virtual void setNextStep (DPStep::ShPtr nextStep)
- { itsNextStep = nextStep;
- nextStep->setPrevStep(this);
- }
-
- // Get the next step.
- const DPStep::ShPtr& getNextStep() const
- { return itsNextStep; }
-
- protected:
- DPInfo& info()
- { return itsInfo; }
-
- // Update the general info (called by setInfo).
- // The default implementation copies the info.
- virtual void updateInfo (const DPInfo&);
-
- private:
- //# Data members.
- DPStep::ShPtr itsNextStep;
- DPStep* itsPrevStep; // Normal pointer for back links, prevent
- // two shared pointers to same object
- DPInfo itsInfo;
- };
-
-
-
- // @ingroup NDPPP
-
- // This class defines a null step in the DPPP pipeline.
- // It can be used as the last step in the pipeline, so other steps
- // do not need to test if there is a next step.
-
- class NullStep: public DPStep
- {
- public:
- virtual ~NullStep();
-
- // Process the data. It does nothing.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- // It does nothing.
- virtual void finish();
-
- // Show the step parameters.
- // It does nothing.
- virtual void show (std::ostream&) const;
- };
-
-
-
- // @ingroup NDPPP
-
- // This class defines step in the DPPP pipeline that keeps the result
- // to make it possible to get the result of another step.
- // It keeps the result and calls process of the next step.
-
- class ResultStep: public DPStep
- {
- public:
- typedef shared_ptr<ResultStep> ShPtr;
- // Create the object. By default it sets its next step to the NullStep.
- ResultStep();
-
- virtual ~ResultStep();
-
- // Keep the buffer.
- virtual bool process (const DPBuffer&);
-
- // Finish does not do anything.
- virtual void finish();
-
- // Show the step parameters.
- // It does nothing.
- virtual void show (std::ostream&) const;
-
- // Get the result.
- const DPBuffer& get() const
- { return itsBuffer; }
- DPBuffer& get()
- { return itsBuffer; }
-
- // Clear the buffer.
- void clear()
- { itsBuffer = DPBuffer(); }
-
- private:
- DPBuffer itsBuffer;
- };
-
-
-
- // @ingroup NDPPP
-
- // This class defines step in the DPPP pipeline that keeps the result
- // to make it possible to get the result of another step.
- // It keeps the result and calls process of the next step.
- // Buffers are accumulated until cleared.
-
- class MultiResultStep: public DPStep
- {
- public:
- // Define the shared pointer for this type.
- typedef shared_ptr<MultiResultStep> ShPtr;
-
- // Create the object. By default it sets its next step to the NullStep.
- MultiResultStep (uint size);
-
- virtual ~MultiResultStep();
-
- // Add the buffer to the vector of kept buffers.
- virtual bool process (const DPBuffer&);
-
- // Finish does not do anything.
- virtual void finish();
-
- // Show the step parameters.
- // It does nothing.
- virtual void show (std::ostream&) const;
-
- // Get the result.
- const vector<DPBuffer>& get() const
- { return itsBuffers; }
- vector<DPBuffer>& get()
- { return itsBuffers; }
-
- // Get the size of the result.
- size_t size() const
- { return itsSize; }
-
- // Clear the buffers.
- void clear()
- { itsSize = 0; }
-
- private:
- vector<DPBuffer> itsBuffers;
- size_t itsSize;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DemixInfo.h b/CEP/DP3/DPPP/include/DPPP/DemixInfo.h
deleted file mode 100644
index b3959c002f15997201ce8d5e2d70ae4349049c5f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DemixInfo.h
+++ /dev/null
@@ -1,202 +0,0 @@
-//# DemixInfo.h: DPPP struct to hold the common demix variables
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: Demixer.h 23223 2012-12-07 14:09:42Z schoenmakers $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DEMIXINFO_H
-#define DPPP_DEMIXINFO_H
-
-// @file
-// @brief DPPP struct to hold the common demix variables
-
-#include <DPPP/DPInfo.h>
-#include <DPPP/BaselineSelection.h>
-#include <DPPP/Baseline.h>
-#include <DPPP/Patch.h>
-#include <Common/ParameterSet.h>
-
-#include <casacore/casa/Arrays/Vector.h>
-
-namespace LOFAR {
- namespace DPPP {
- // @ingroup NDPPP
-
- // This struct holds the common demix variables.
- // It can be shared between the parallel DemixWorker objects.
-
- class DemixInfo
- {
- public:
- // Constructor to read and initialize the values.
- DemixInfo (const ParameterSet&, const string& prefix);
-
- // Update the info.
- void update (const DPInfo& infoSel, DPInfo& info);
-
- // Show parameters.
- void show (ostream&) const;
-
- // Get the DPInfo object.
- const DPInfo& getInfo() const
- { return itsInfoSel; }
-
- // Get settings.
- //# 0=test 1=include 2=deproject 3=ignore
- uint targetHandling() const {return itsTargetHandling;}
- uint verbose() const {return itsVerbose;}
- uint maxIter() const {return itsMaxIter;}
- uint minNBaseline() const {return itsMinNBaseline;}
- uint minNStation() const {return itsMinNStation;}
- uint nstation() const {return itsNStation;}
- uint nbl() const {return itsNBl;}
- uint ncorr() const {return itsNCorr;}
- uint nchanIn() const {return itsNChanIn;}
- uint nchanAvg() const {return itsNChanAvg;}
- uint nchanAvgSubtr() const {return itsNChanAvgSubtr;}
- uint nchanOut() const {return itsNChanOut;}
- uint nchanOutSubtr() const {return itsNChanOutSubtr;}
- uint ntimeAvg() const {return itsNTimeAvg;}
- uint ntimeAvgSubtr() const {return itsNTimeAvgSubtr;}
- uint ntimeOut() const {return itsNTimeOut;}
- uint ntimeOutSubtr() const {return itsNTimeOutSubtr;}
- uint ntimeChunk() const {return itsNTimeChunk;}
- uint chunkSize() const {return itsChunkSize;}
- double timeIntervalAvg() const {return itsTimeIntervalAvg;}
- double ratio1() const {return itsRatio1;}
- double ratio2() const {return itsRatio2;}
- double ateamAmplThreshold() const {return itsAteamAmplThreshold;}
- double targetAmplThreshold() const {return itsTargetAmplThreshold;}
- double defaultGain() const {return itsDefaultGain;}
- bool isAteamNearby() const {return itsIsAteamNearby;}
- bool propagateSolution() const {return itsPropagateSolution;}
- bool applyBeam() const {return itsApplyBeam;}
- bool solveBoth() const {return itsSolveBoth;}
- bool doSubtract() const {return itsDoSubtract;}
- const BaselineSelection& selBL() const {return itsSelBL;}
- const vector<int>& uvwSplitIndex() const {return itsUVWSplitIndex;}
- const string& predictModelName() const {return itsPredictModelName;}
- const string& demixModelName() const {return itsDemixModelName;}
- const string& targetModelName() const {return itsTargetModelName;}
- const vector<string>& sourceNames() const {return itsSourceNames;}
- const Position& phaseRef() const {return itsPhaseRef;}
- const vector<Baseline>& baselines() const {return itsBaselines;}
- const casacore::Vector<bool> selTarget() const {return itsSelTarget;}
- const casacore::Vector<double>& freqDemix() const {return itsFreqDemix;}
- const casacore::Vector<double>& freqSubtr() const {return itsFreqSubtr;}
- const vector<Patch::ConstPtr>& ateamList() const {return itsAteamList;}
- const vector<Patch::ConstPtr>& targetList() const {return itsTargetList;}
- const vector<Patch::ConstPtr>& ateamDemixList() const
- {return itsAteamDemixList;}
- const vector<Patch::ConstPtr>& targetDemixList() const
- {return itsTargetDemixList;}
-
- // Get the baselines.
- const casacore::Vector<casacore::Int>& getAnt1() const
- { return itsInfoSel.getAnt1(); }
- const casacore::Vector<casacore::Int>& getAnt2() const
- { return itsInfoSel.getAnt2(); }
-
- // Get the antenna names and used antennas.
- const casacore::Vector<casacore::String>& antennaNames() const
- { return itsInfoSel.antennaNames(); }
-
- // Get cosine of the angular distance between two sky positions.
- static double getCosAngDist (double ra1, double dec1,
- double ra2, double dec2)
- {
- return sin(dec1)*sin(dec2) + cos(dec1)*cos(dec2)*cos(ra1-ra2);
- }
-
- // Test if two positions in the sky are within delta radians.
- static bool testAngDist (double ra1, double dec1,
- double ra2, double dec2, double cosDelta)
- {
- return getCosAngDist (ra1, dec1, ra2, dec2) >= cosDelta;
- }
-
- private:
- // Create a list of patches (and components).
- vector<Patch::ConstPtr> makePatchList (const string& sdbName,
- const vector<string>& patchNames);
-
- // Make the target list for demixing with a detailed model for the
- // possible Ateam sources in it.
- void makeTargetDemixList();
-
- //# Data members.
- DPInfo itsInfoSel;
- BaselineSelection itsSelBL;
- BaselineSelection itsSelBLTarget;
- vector<int> itsUVWSplitIndex;
- string itsPredictModelName;
- string itsDemixModelName;
- string itsTargetModelName;
- vector<string> itsSourceNames;
- double itsRatio1;
- double itsRatio2;
- double itsAteamAmplThreshold;
- double itsTargetAmplThreshold;
- double itsCosTargetDelta;
- double itsAngdistThreshold;
- double itsAngdistRefFreq;
- double itsDefaultGain;
- bool itsIsAteamNearby;
- bool itsPropagateSolution;
- bool itsApplyBeam;
- bool itsSolveBoth; //# solve if both stat solvable
- bool itsDoSubtract;
- uint itsTargetHandling;
- uint itsVerbose; //# trace verbosity level
- uint itsMaxIter; //# max #iter in solve
- uint itsMinNBaseline; //# min #baselines for solve
- uint itsMinNStation; //# min #stations for solve
- uint itsNStation;
- uint itsNBl;
- uint itsNCorr;
- uint itsNChanIn;
- uint itsNChanAvgSubtr; //# subtract averaging
- uint itsNChanAvg; //# demix averaging
- uint itsNChanOutSubtr;
- uint itsNChanOut;
- uint itsNTimeAvgSubtr; //# subtract averaging
- uint itsNTimeAvg; //# demix averaging
- uint itsNTimeOutSubtr; //# #output times per chunk
- uint itsNTimeOut; //# #demix times per chunk
- uint itsChunkSize; //# predict time step
- uint itsNTimeChunk; //# nr chunks in parallel
- double itsTimeIntervalAvg;
- Position itsPhaseRef; //# original phaseref
- vector<Baseline> itsBaselines;
- casacore::Vector<bool> itsSelTarget; //# baselines in target estimate
- casacore::Vector<double> itsFreqDemix;
- casacore::Vector<double> itsFreqSubtr;
- vector<Patch::ConstPtr> itsAteamList;
- vector<Patch::ConstPtr> itsTargetList;
- vector<Patch::ConstPtr> itsAteamDemixList;
- vector<Patch::ConstPtr> itsTargetDemixList;
- vector<string> itsAteamRemoved;
- vector<string> itsTargetReplaced;
- };
-
- } //# end namespace
-} //# end namespace
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DemixWorker.h b/CEP/DP3/DPPP/include/DPPP/DemixWorker.h
deleted file mode 100644
index 1e03ddd9097070d6ef57f8407a33f36d9bf12d80..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DemixWorker.h
+++ /dev/null
@@ -1,343 +0,0 @@
-//# DemixWorker.h: Demixer helper class processing a time chunk
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: Demixer.h 23223 2012-12-07 14:09:42Z schoenmakers $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DEMIXWORKER_H
-#define DPPP_DEMIXWORKER_H
-
-// @file
-// @brief DPPP step class to average in time and/or freq
-
-#include <DPPP/DemixInfo.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/Patch.h>
-#include <DPPP/PhaseShift.h>
-#include <DPPP/Filter.h>
-#include <DPPP/EstimateNew.h>
-#include <StationResponse/Station.h>
-#include <ParmDB/ParmDB.h>
-
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/measures/Measures/MeasureHolder.h>
-#include <casacore/measures/Measures/MeasFrame.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MPosition.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MCDirection.h>
-
-namespace LOFAR {
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // DemixWorker::process processes a single time window (say, 2 minutes).
- // It predicts the A-team and target sources to determine which sources
- // have to be taken into account and which antennae have to be solved for.
- // Multiple DemixWorker::process can be executed in parallel by the parent
- // class DemixerNew.
- //
- // Each DemixWorker object references a DemixInfo object containing the
- // general info and parameters.
-
- class DemixWorker
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- DemixWorker (DPInput*,
- const string& prefix,
- const DemixInfo& info,
- const DPInfo& dpinfo,
- int workernr);
-
-
- // Process the data in the input buffers and store the result in the
- // output buffers.
- void process (const DPBuffer* bufin, uint nbufin,
- DPBuffer* bufout, vector<double>* solutions,
- uint chunkNr);
-
- // Get the number of solves.
- uint nSolves() const
- { return itsNrSolves; }
- // Get the number of converged solves.
- uint nConverged() const
- { return itsNrConverged; }
- // Get the total nr of iterations used.
- uint nIterations() const
- { return itsNrIter; }
- // Get the number of times no demix was needed.
- uint nNoDemix() const
- { return itsNrNoDemix; }
- uint nIncludeStrongTarget() const
- { return itsNrIncludeStrongTarget; }
- uint nIncludeCloseTarget() const
- { return itsNrIncludeCloseTarget; }
- uint nIgnoreTarget() const
- { return itsNrIgnoreTarget; }
- uint nDeprojectTarget() const
- { return itsNrDeprojectTarget; }
- // Get nr of times a source was demixed.
- const casacore::Vector<uint>& nsourcesDemixed() const
- { return itsNrSourcesDemixed; }
- // Get nr of times a station was demixed.
- const casacore::Vector<uint>& nstationsDemixed() const
- { return itsNrStationsDemixed; }
- // Get nr of times a station/source was demixed.
- const casacore::Matrix<uint>& statSourceDemixed() const
- { return itsStatSourceDemixed; }
- const casacore::Matrix<double>& amplSubtrMean() const
- { return itsAmplSubtrMean; }
- const casacore::Matrix<double>& amplSubtrM2() const
- { return itsAmplSubtrM2; }
- const casacore::Matrix<size_t>& amplSubtrNr() const
- { return itsAmplSubtrNr; }
-
- // Get the timings of the various processing steps.
- // <group>
- double getTotalTime() const
- { return itsTimer.getElapsed(); }
- double getCoarseTime() const
- { return itsTimerCoarse.getElapsed(); }
- double getPhaseShiftTime() const
- { return itsTimerPhaseShift.getElapsed(); }
- double getDemixTime() const
- { return itsTimerDemix.getElapsed(); }
- double getPredictTime() const
- { return itsTimerPredict.getElapsed(); }
- double getSolveTime() const
- { return itsTimerSolve.getElapsed(); }
- double getSubtractTime() const
- { return itsTimerSubtract.getElapsed(); }
- // </group>
-
- private:
- // Setup the demix processing steps for this piece of data.
- // It fills itsFirstSteps, etc. for the sources to be demixed.
- // It also determines how to handle the target (include,deproject,ignore).
- void setupDemix (uint chunkNr);
-
- // Find the median ampltitude for the selected baselines.
- // It uses itsTmpAmpl as temporary buffer.
- float findMedian (const casacore::Cube<float>& ampl, const bool* selbl);
-
- // Average the baseline UVWs in bufin and split them into UVW per station.
- // It returns the number of time averages.
- uint avgSplitUVW (const DPBuffer* bufin, uint nbufin,
- uint ntimeAvg, const vector<uint>& selbl);
-
- // Predict the target StokesI amplitude.
- // It applies the beam at each target patch.
- void predictTarget (const vector<Patch::ConstPtr>& patchList,
- uint ntime, double time, double timeStep);
-
- // Predict the StokesI amplitude of the Ateam patches and determine
- // which antennae and sources to use when demixing.
- // It applies the beam at each patch center.
- void predictAteam (const vector<Patch::ConstPtr>& patchList,
- uint ntime, double time, double timeStep);
-
- // Add the StokesI of itsPredictVis to ampl.
- void addStokesI (casacore::Matrix<float>& ampl);
-
- // Calculate the beam for demix resolution and apply to itsPredictVis.
- // If apply==False, nothing is done.
- void applyBeam (double time, const Position& pos, bool apply);
-
- // Calculate the beam for the given sky direction and frequencies.
- // Apply it to the data.
- // If apply==False, nothing is done.
- void applyBeam (double time, const Position& pos, bool apply,
- const casacore::Vector<double>& chanFreqs,
- dcomplex* data);
-
- // Convert a direction to ITRF.
- StationResponse::vector3r_t dir2Itrf (const casacore::MDirection&);
-
- // Calculate the StokesI amplitude from the predicted visibilities.
- // (0.5 * (XX+YY))
- void calcStokesI (casacore::Matrix<float>& ampl);
-
- // Simply average the data if no demixing needs to bedone.
- void average (const DPBuffer* bufin, uint nbufin,
- DPBuffer* bufout);
-
- // Add the decorrelation factor contribution for each time slot.
- void addFactors (const DPBuffer& newBuf,
- casacore::Array<casacore::DComplex>& factorBuf);
-
- // Calculate the decorrelation factors by averaging them.
- // Apply the P matrix to deproject the sources without a model.
- void makeFactors (const casacore::Array<casacore::DComplex>& bufIn,
- casacore::Array<casacore::DComplex>& bufOut,
- const casacore::Cube<float>& weightSums,
- uint nChanOut,
- uint nChanAvg);
-
- // Deproject the sources without a model.
- void deproject (casacore::Array<casacore::DComplex>& factors,
- vector<MultiResultStep*> avgResults,
- uint resultIndex);
-
- // Do the demixing.
- void handleDemix (DPBuffer* bufout, vector<double>* solutions,
- double time, double timeStep);
-
- // Solve gains and subtract sources.
- void demix (vector<double>* solutions, double time, double timeStep);
-
- // Add amplitude subtracted to the arrays for mean and stddev.
- void addMeanM2 (const vector<float>& sourceAmpl, uint src);
-
- // Merge the data of the selected baselines from the subtract buffer
- // into the full buffer.
- void mergeSubtractResult();
-
- //# Data members.
- int itsWorkerNr;
- const DemixInfo* itsMix;
- vector<PhaseShift*> itsOrigPhaseShifts;
- //# Phase shift and average steps for demix.
- vector<DPStep::ShPtr> itsOrigFirstSteps;
- //# Result of phase shifting and averaging the directions of interest
- //# at the demix resolution.
- vector<MultiResultStep*> itsAvgResults;
- vector<PhaseShift*> itsPhaseShifts;
- vector<DPStep::ShPtr> itsFirstSteps;
- DPStep::ShPtr itsAvgStepSubtr;
- Filter itsFilter;
- Filter* itsFilterSubtr;
- //# Result of averaging the target at the subtract resolution.
- MultiResultStep* itsAvgResultFull;
- MultiResultStep* itsAvgResultSubtr;
- //# The sources to demix (excluding target).
- vector<Patch::ConstPtr> itsDemixList;
- //# The info needed to calculate the station beams.
- vector<StationResponse::Station::Ptr> itsAntBeamInfo;
- //# Measure objects unique to this worker (thread).
- //# This is needed because they are not thread-safe.
- casacore::MPosition itsArrayPos;
- casacore::MDirection itsDelayCenter;
- casacore::MDirection itsTileBeamDir;
-
- //# Variables set by setupDemix and used by handleDemix.
- uint itsNDir;
- uint itsNModel;
- uint itsNSubtr;
- bool itsIgnoreTarget;
- bool itsIncludeTarget;
- //# Accumulator used for computing the demixing weights at the demix
- //# resolution. The shape of this buffer is #correlations x #channels
- //# x #baselines x #directions x #directions (fastest axis first).
- casacore::Array<casacore::DComplex> itsFactorBuf;
- //# Buffer of demixing weights at the demix resolution. Each Array is a
- //# cube of shape #correlations x #channels x #baselines of matrices of
- //# shape #directions x #directions.
- vector<casacore::Array<casacore::DComplex> > itsFactors;
- //# Accumulator used for computing the demixing weights. The shape of this
- //# buffer is #correlations x #channels x #baselines x #directions
- //# x #directions (fastest axis first).
- casacore::Array<casacore::DComplex> itsFactorBufSubtr;
- //# Buffer of demixing weights at the subtract resolution. Each Array is a
- //# cube of shape #correlations x #channels x #baselines of matrices of
- //# shape #directions x #directions.
- vector<casacore::Array<casacore::DComplex> > itsFactorsSubtr;
-
- //# Variables for conversion of directions to ITRF.
- casacore::MeasFrame itsMeasFrame;
- casacore::MDirection::Convert itsMeasConverter;
- vector<StationResponse::matrix22c_t> itsBeamValues; //# [nst,nch]
-
- //# Indices telling which Ateam sources to use.
- vector<uint> itsSrcSet;
- //# UVW per station per demix time slot
- casacore::Cube<double> itsStationUVW; //# UVW per station
- casacore::Matrix<double> itsAvgUVW; //# temp buffer
- casacore::Cube<dcomplex> itsPredictVis; //# temp buffer
- //# #nfreq x #bl x #time StokesI amplitude per A-source.
- vector<casacore::Cube<float> > itsAteamAmpl;
- //# #bl x #src telling if baseline has sufficient Ateam flux.
- casacore::Matrix<bool> itsAteamAmplSel;
- //# #nfreq x #bl x #time StokesI amplitude of target.
- casacore::Cube<float> itsTargetAmpl;
- //# Temporary buffer to determine medians.
- vector<float> itsTmpAmpl;
- //# Per A-source and for target the min and max amplitude.
- vector<double> itsAteamMinAmpl;
- vector<double> itsAteamMaxAmpl;
- double itsTargetMinAmpl;
- double itsTargetMaxAmpl;
- //# Per A-source the stations to use (matching the minimum amplitude).
- vector<vector<uint> > itsStationsToUse;
- casacore::Block<bool> itsSolveStation; //# solve station i?
- //# Per station and source the index in the unknowns vector.
- //# Note there are 8 unknowns (4 pol, ampl/phase) per source/station.
- vector<vector<int> > itsUnknownsIndex;
- //# The estimater (solver).
- EstimateNew itsEstimate;
- //# Variables for the predict.
- casacore::Matrix<double> itsUVW;
- vector<casacore::Cube<dcomplex> > itsModelVisDemix;
- vector<casacore::Cube<dcomplex> > itsModelVisSubtr;
- uint itsNTimeOut;
- uint itsNTimeOutSubtr;
- uint itsTimeIndex;
- vector<float> itsObservedAmpl;
- vector<float> itsSourceAmpl;
- vector<float> itsSumSourceAmpl;
- //# Statistics
- uint itsNrSolves;
- uint itsNrConverged;
- uint itsNrIter;
- uint itsNrNoDemix;
- uint itsNrIncludeStrongTarget;
- uint itsNrIncludeCloseTarget;
- uint itsNrIgnoreTarget;
- uint itsNrDeprojectTarget;
- //# Nr of times a source is demixed.
- casacore::Vector<uint> itsNrSourcesDemixed;
- //# Nr of times a station is demixed.
- casacore::Vector<uint> itsNrStationsDemixed;
- //# Nr of times a source/station is demixed.
- casacore::Matrix<uint> itsStatSourceDemixed;
- //# Average amplitude subtracted for middle channel [nbl,nsrc]
- casacore::Matrix<double> itsAmplSubtrMean;
- //# M2n to calculate stddev online in stable way (see Wikipedia)
- casacore::Matrix<double> itsAmplSubtrM2;
- //# N for mean/stddev amplitude calculations.
- casacore::Matrix<size_t> itsAmplSubtrNr;
- //# Timers.
- NSTimer itsTimer;
- NSTimer itsTimerCoarse;
- NSTimer itsTimerPhaseShift;
- NSTimer itsTimerDemix;
- NSTimer itsTimerPredict;
- NSTimer itsTimerSolve;
- NSTimer itsTimerSubtract;
- };
-
- } //# end namespace
-} //# end namespace
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Demixer.h b/CEP/DP3/DPPP/include/DPPP/Demixer.h
deleted file mode 100644
index da64ad327272cca36862229c5fb90005f3ad1458..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Demixer.h
+++ /dev/null
@@ -1,216 +0,0 @@
-//# Demixer.h: DPPP step class to subtract A-team sources
-//# Copyright (C) 2011
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DEMIXER_H
-#define DPPP_DEMIXER_H
-
-// @file
-// @brief DPPP step class to average in time and/or freq
-
-#include <DPPP/Baseline.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/Patch.h>
-#include <DPPP/PhaseShift.h>
-#include <DPPP/Filter.h>
-
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MPosition.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/measures/Measures/MeasFrame.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/measures/Measures/MCPosition.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- typedef vector<Patch::ConstPtr> PatchList;
-
- // This class is a DPStep class to subtract the strong A-team sources.
- // It is based on the demixing.py script made by Bas vd Tol and operates
- // per time chunk as follows:
- // <ul>
- // <li> The data are phase-shifted and averaged for each source.
- // <li> Demixing is done using the combined results.
- // <li> For each source a BBS solve, smooth, and predict is done.
- // <li> The predicted results are subtracted from the averaged data.
- // </ul>
-
- class Demixer: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- Demixer (DPInput*, const ParameterSet&, const string& prefix);
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- // Add the decorrelation factor contribution for each time slot.
- void addFactors (const DPBuffer& newBuf,
- casacore::Array<casacore::DComplex>& factorBuf);
-
- // Calculate the decorrelation factors by averaging them.
- // Apply the P matrix to deproject the sources without a model.
- void makeFactors (const casacore::Array<casacore::DComplex>& bufIn,
- casacore::Array<casacore::DComplex>& bufOut,
- const casacore::Cube<float>& weightSums,
- uint nChanOut,
- uint nChanAvg);
-
- // Do the demixing.
- void handleDemix();
-
- // Deproject the sources without a model.
- void deproject (casacore::Array<casacore::DComplex>& factors,
- vector<MultiResultStep*> avgResults,
- uint resultIndex);
-
- // Solve gains and subtract sources.
- void demix();
-
- // Export the solutions to a ParmDB.
- void dumpSolutions();
-
- // Merge the data of the selected baselines from the subtract buffer
- // into the full buffer.
- void mergeSubtractResult();
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBufTmp;
- string itsSkyName;
- string itsInstrumentName;
- double itsDefaultGain;
- size_t itsMaxIter;
- BaselineSelection itsSelBL;
- Filter itsFilter;
- vector<PhaseShift*> itsPhaseShifts;
- //# Phase shift and average steps for demix.
- vector<DPStep::ShPtr> itsFirstSteps;
- //# Result of phase shifting and averaging the directions of interest
- //# at the demix resolution.
- vector<MultiResultStep*> itsAvgResults;
- DPStep::ShPtr itsAvgStepSubtr;
- Filter* itsFilterSubtr;
- //# Result of averaging the target at the subtract resolution.
- MultiResultStep* itsAvgResultFull;
- MultiResultStep* itsAvgResultSubtr;
- //# Ignore target in demixing?
- bool itsIgnoreTarget;
- //# Name of the target. Empty if no model is available for the target.
- string itsTargetSource;
- vector<string> itsSubtrSources;
- vector<string> itsModelSources;
- vector<string> itsExtraSources;
- vector<string> itsAllSources;
-// vector<double> itsCutOffs;
- bool itsPropagateSolutions;
- uint itsNDir;
- uint itsNModel;
- uint itsNStation;
- uint itsNBl;
- uint itsNCorr;
- uint itsNChanIn;
- uint itsNTimeIn;
- uint itsNTimeDemix;
- uint itsNChanAvgSubtr;
- uint itsNTimeAvgSubtr;
- uint itsNChanOutSubtr;
- uint itsNTimeOutSubtr;
- uint itsNTimeChunk;
- uint itsNTimeChunkSubtr;
- uint itsNChanAvg;
- uint itsNTimeAvg;
- uint itsNChanOut;
- uint itsNTimeOut;
- double itsTimeIntervalAvg;
-
- //# Accumulator used for computing the demixing weights at the demix
- //# resolution. The shape of this buffer is #correlations x #channels
- //# x #baselines x #directions x #directions (fastest axis first).
- casacore::Array<casacore::DComplex> itsFactorBuf;
- //# Buffer of demixing weights at the demix resolution. Each Array is a
- //# cube of shape #correlations x #channels x #baselines of matrices of
- //# shape #directions x #directions.
- vector<casacore::Array<casacore::DComplex> > itsFactors;
-
- //# Accumulator used for computing the demixing weights. The shape of this
- //# buffer is #correlations x #channels x #baselines x #directions
- //# x #directions (fastest axis first).
- casacore::Array<casacore::DComplex> itsFactorBufSubtr;
- //# Buffer of demixing weights at the subtract resolution. Each Array is a
- //# cube of shape #correlations x #channels x #baselines of matrices of
- //# shape #directions x #directions.
- vector<casacore::Array<casacore::DComplex> > itsFactorsSubtr;
-
- PatchList itsPatchList;
- Position itsPhaseRef;
- vector<Baseline> itsBaselines;
- vector<int> itsUVWSplitIndex;
- casacore::Vector<double> itsFreqDemix;
- casacore::Vector<double> itsFreqSubtr;
- vector<double> itsUnknowns;
- vector<double> itsPrevSolution;
- uint itsTimeIndex;
- uint itsNConverged;
- FlagCounter itsFlagCounter;
-
- //# Timers.
- NSTimer itsTimer;
- NSTimer itsTimerPhaseShift;
- NSTimer itsTimerDemix;
- NSTimer itsTimerSolve;
- NSTimer itsTimerDump;
- };
-
- } //# end namespace
-} //# end namespace
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DemixerNew.h b/CEP/DP3/DPPP/include/DPPP/DemixerNew.h
deleted file mode 100644
index 5b3242cacd067be76a37e5c9e45c269c992c5bd6..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DemixerNew.h
+++ /dev/null
@@ -1,142 +0,0 @@
-//# DemixerNew.h: DPPP step class to subtract A-team sources in adaptive way
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: DemixerNew.h 23223 2012-12-07 14:09:42Z schoenmakers $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DEMIXERNEW_H
-#define DPPP_DEMIXERNEW_H
-
-// @file
-// @brief DPPP step class to subtract A-team sources in adaptive way
-
-#include <DPPP/DemixInfo.h>
-#include <DPPP/DemixWorker.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/Filter.h>
-#include <ParmDB/ParmDB.h>
-#include <Common/lofar_smartptr.h>
-#include <Common/lofar_map.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to subtract the strong A-team sources
- // in a smart way (algorithm developed by Reinout van Weeren).
- // It operates as follows:
- // <ol>
- // <li> Per time window (default 2 min) demixing is done separately.
- // <li> Using a simple Ateam model (the A and other strong sources)
- // and the beam model, the StokesI flux is predicted per source.
- // The antennae of baselines with flux>threshold are counted.
- // Only an antenna counted in at least min_antenna baselines is
- // solved for. If not such antennae exist, the source is ignored.
- // <li> The target is predicted as well. Note that an Ateam source within
- // the target field is removed from the Ateam model and added/replaced
- // in the target model.
- // The target model is usually created using gsm.py.
- // <li> For the core baselines the ratio target/Ateam flux is used to
- // determine if the target has to be included, ignored or deprojected.
- // </ol>
- // It is based on the demixing.py script made by Bas vd Tol and operates
- // per time chunk as follows:
- // <ul>
- // <li> The data are phase-shifted and averaged for each source.
- // <li> Demixing is done using the combined results.
- // <li> For each source a BBS solve, smooth, and predict is done.
- // <li> The predicted results are subtracted from the averaged data.
- // </ul>
-
- class DemixerNew: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- DemixerNew (DPInput*, const ParameterSet&, const string& prefix);
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- // Process the data collected in itsBuf.
- void processData();
-
- // Export the solutions to a ParmDB.
- void writeSolutions (double startTime, int ntime);
-
- // Add the mean and M2 (square of differences) of a part in a
- // numerically stable way.
- void addMeanM2 (double& mean, double& m2, size_t& nr,
- double partmean, double partm2, size_t partnr) const;
-
- // Show a statistic.
- void showStat (ostream& os, double n, double ntot,
- const string& str1, const string& str2) const;
-
- // Show a percentage with 1 decimal.
- void showPerc1 (ostream& os, float perc) const;
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DemixInfo itsDemixInfo;
- string itsInstrumentName;
- shared_ptr<BBS::ParmDB> itsParmDB;
- Filter itsFilter; //# only used for getInfo()
- vector<DemixWorker> itsWorkers;
- vector<DPBuffer> itsBufIn;
- vector<DPBuffer> itsBufOut;
- vector<vector<double> > itsSolutions; //# all solutions in a time window
- map<string,int> itsParmIdMap; //# -1 = new parm name
- uint itsNTime;
- uint itsNTimeOut;
- uint itsNChunk;
- //# Timers.
- NSTimer itsTimer;
- NSTimer itsTimerDemix;
- NSTimer itsTimerDump; //# writeSolutions
- NSTimer itsTimerNext; //# next step (parallel to writeSolutions)
- };
-
- } //# end namespace
-} //# end namespace
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/DummyStep.h b/CEP/DP3/DPPP/include/DPPP/DummyStep.h
deleted file mode 100644
index 9611cc0b86ea7153a9b4f2b43f4f360e64f18124..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/DummyStep.h
+++ /dev/null
@@ -1,82 +0,0 @@
-//# DummyStep.h: DPPP step class to DummyStep visibilities from a source model
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_DummyStep_H
-#define DPPP_DummyStep_H
-
-// @file
-// @brief DPPP step class to DummyStep visibilities from a source model
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-
-#include <utility>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is an empty DPStep subclass to use as implementation template
-
- class DummyStep: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- DummyStep (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~DummyStep();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuffer;
-
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/EstimateMixed.h b/CEP/DP3/DPPP/include/DPPP/EstimateMixed.h
deleted file mode 100644
index 53f8523ef1c51e08f39699e1267762f67f790a8c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/EstimateMixed.h
+++ /dev/null
@@ -1,105 +0,0 @@
-//# EstimateMixed.h: Estimate Jones matrices for several directions
-//# simultaneously. A separate data stream is used for each direction. The
-//# mixing coefficients quantify the influence of each direction on each of the
-//# other directions (including time and frequency smearing).
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_ESTIMATEMIXED_H
-#define DPPP_ESTIMATEMIXED_H
-
-// \file
-// Estimate Jones matrices for several directions simultaneously. A separate
-// data stream is used for each direction. The mixing coefficients quantify the
-// influence of each direction on each of the other directions (including time
-// and frequency smearing).
-
-#include <DPPP/Baseline.h>
-#include <DPPP/Cursor.h>
-#include <Common/lofar_complex.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-// Estimate Jones matrices for several directions simultaneously. A separate
-// data stream is used for each direction. The mixing coefficients quantify the
-// influence of each direction on each of the other directions (including time
-// and frequency smearing).
-//
-// \param[in] nDirection
-// Number of directions to estimate Jones matrices for.
-// \param[in] nStation
-// Number of stations.
-// \param[in] nBaseline
-// Number of baselines.
-// \param[in] nChannel
-// Number of frequency channels.
-// \param[in] data
-// Vector of length \p nDirection of cursors for 3-D buffers of observed
-// visiblity data of shape (\p nBaseline, \p nChannel, 4).
-// \param[in] model
-// Vector of length \p nDirection of cursors for 3-D buffers of simulated
-// visiblity data of shape (\p nBaseline, \p nChannel, 4).
-// \param[in] baselines
-// A cursor for a 1-D buffer of baselines of shape (\p nBaseline).
-// \param[in] flag
-// A cursor for a 3-D buffer of observed visibility flags of shape
-// (\p nBaseline, \p nChannel, 4).
-// \param[in] weight
-// A cursor for a 3-D buffer of observed visibility weights of shape
-// (\p nBaseline, \p nChannel, 4).
-// \param[in] mix
-// A cursor for a 5-D buffer of mixing weights of shape
-// (\p nBaseline, \p nChannel, 4, \p nDirection, \p nDirection).
-// \param[in] unknowns
-// A pointer to a buffer of unknowns of size nDirection * nStation * 8.
-// \param[in] errors
-// A pointer to a buffer of errors of size nDirection * nStation * 8. If this
-// pointer is null (0), errors will not be estimated.
-bool estimate(size_t nDirection, size_t nStation, size_t nBaseline,
- size_t nChannel, const_cursor<Baseline> baselines,
- vector<const_cursor<fcomplex> > data, vector<const_cursor<dcomplex> > model,
- const_cursor<bool> flag, const_cursor<float> weight,
- const_cursor<dcomplex> mix, double *unknowns, size_t maxiter=50);
-// @}
-
-// Estimate for a variable nr of stations per source.
-bool estimateSel(size_t nDirection, size_t nStation, size_t nBaseline,
- size_t nChannel, const_cursor<Baseline> baselines,
- vector<const_cursor<fcomplex> > data,
- vector<const_cursor<dcomplex> > model,
- const_cursor<bool> flag, const_cursor<float> weight,
- const_cursor<dcomplex> mix, double *unknowns,
- size_t nUnknowns,
- vector<dcomplex>& M, vector<dcomplex>& dM,
- vector<double>& dR, vector<double>& dI);
-
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/EstimateNew.h b/CEP/DP3/DPPP/include/DPPP/EstimateNew.h
deleted file mode 100644
index cd5fb1c967d89817d297ecc1f2e2ee57f2382078..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/EstimateNew.h
+++ /dev/null
@@ -1,149 +0,0 @@
-//# EstimateNew.h: Estimate Jones matrices for several directions and stations
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_ESTIMATENEW_H
-#define DPPP_ESTIMATENEW_H
-
-// \file
-// Estimate Jones matrices for several directions simultaneously. A separate
-// data stream is used for each direction. The mixing coefficients quantify the
-// influence of each direction on each of the other directions (including time
-// and frequency smearing).
-
-#include <DPPP/Baseline.h>
-#include <DPPP/Cursor.h>
-#include <Common/lofar_complex.h>
-#include <Common/lofar_vector.h>
-
-//# Use Block<bool> instead of vector<bool> (because testing bits is slower).
-#include <casacore/casa/Containers/Block.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- // \addtogroup NDPPP
- // @{
-
- // Estimate Jones matrices for several directions simultaneously. A separate
- // data stream is used for each direction. The mixing coefficients quantify the
- // influence of each direction on each of the other directions (including time
- // and frequency smearing).
- class EstimateNew
- {
- public:
-
- EstimateNew();
-
- // Update the object and size its internal buffers.
- void update (size_t maxndir, size_t nBaseline, size_t nStation,
- size_t nChannel, size_t maxIter, bool propagateSolution);
-
- // \param[in] data
- // Vector of length \p nDirection of cursors for 3-D buffers of observed
- // visiblity data of shape (\p nBaseline, \p nChannel, 4).
- // \param[in] model
- // Vector of length \p nDirection of cursors for 3-D buffers of simulated
- // visiblity data of shape (\p nBaseline, \p nChannel, 4).
- // \param[in] baselines
- // A cursor for a 1-D buffer of baselines of shape (\p nBaseline).
- // \param[in] flag
- // A cursor for a 3-D buffer of observed visibility flags of shape
- // (\p nBaseline, \p nChannel, 4).
- // \param[in] weight
- // A cursor for a 3-D buffer of observed visibility weights of shape
- // (\p nBaseline, \p nChannel, 4).
- // \param[in] mix
- // A cursor for a 5-D buffer of mixing weights of shape
- // (\p nBaseline, \p nChannel, 4, \p nDirection, \p nDirection).
- // \param[in] solveBoth
- // True = only use baseline if both stations are solvable
- //
- // <br>Note that the cursors are passed by value, so a copy is made.
- // In this way no reset of the cursor is needed.
- bool estimate (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet,
- const_cursor<Baseline> baselines,
- vector<const_cursor<fcomplex> > data,
- vector<const_cursor<dcomplex> > model,
- const_cursor<bool> flag,
- const_cursor<float> weight,
- const_cursor<dcomplex> mix,
- double defaultGain,
- bool solveBoth,
- uint verbose);
-
- // Get the last solution.
- // It contains zeroes for the direction-stations not solved for.
- const vector<double>& getSolution() const
- { return itsSolution; }
-
- // Get the nr of iterations used.
- size_t nIterations() const
- { return itsNrIter; }
-
- private:
- // Initialize the solution. Nr must be a multiple of 8.
- // The diagonal is set to (diag,0) or (1e-8,0), off-diagonal to (0,0).
- void initSolution (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet,
- double defaultGain);
-
- // Clear the solution for unsolvable stations
- // (essentially changing 1e-8 to 0).
- void clearNonSolvable (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet);
-
- // Update itsSolution from itsUnknowns for the unknowns to be used.
- void fillSolution (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet);
-
- // Fill itsDerivIndex for the unknowns of the given baseline
- // to be able to pass the equations to LSQFit::makeNorm.
- // It returns the number of unknowns.
- uint fillDerivIndex (size_t ndir,
- const vector<vector<int> >& unknownsIndex,
- const Baseline& baseline);
-
- //# Data members
- size_t itsNrBaselines;
- size_t itsNrStations;
- size_t itsNrChannels;
- size_t itsMaxIter;
- size_t itsNrIter;
- size_t itsNrDir;
- bool itsPropagateSolution;
- casacore::Block<bool> itsSolveStation; //# solve station i?
- vector<casacore::uInt> itsDerivIndex; //# index for LSQFit::makeIndex
- vector<double> itsUnknowns;
- vector<double> itsSolution;
- vector<dcomplex> itsM;
- vector<dcomplex> itsdM;
- vector<double> itsdR;
- vector<double> itsdI;
- };
-
- // @}
-
- } //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Filter.h b/CEP/DP3/DPPP/include/DPPP/Filter.h
deleted file mode 100644
index 48b0d525413749003524d3a0eb87fbfccefad48d..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Filter.h
+++ /dev/null
@@ -1,206 +0,0 @@
-//# Filter.h: DPPP step to filter out baselines and channels
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_FILTER_H
-#define DPPP_FILTER_H
-
-// @file
-// @brief DPPP step to filter out baselines and channels
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/BaselineSelection.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPInput step reading the data from a MeasurementSet.
- // At the beginning it finds out the shape of the data; i.e., the
- // number of correlations, channels, baselines, and time slots.
- // It requires the data to be regularly shaped.
- //
- // The object is constructed from the 'msin' keywords in the parset file.
- // Currently the following can be given:
- // <ul>
- // <li> msin: name of the MS
- // <li> msin.autoweight: calculate weights from autocorrelations? [no]
- // <li> msin.startchan: first channel to use [0]
- // <li> msin.nchan: number of channels to use [all]
- // <li> msin.useflag: use the existing flags? [yes]
- // <li> msin.datacolumn: the data column to use [DATA]
- // <li> msin.starttime: first time to use [first time in MS]
- // <li> msin.endtime: last time to use [last time in MS]
- // </ul>
- //
- // If a time slot is missing, it is inserted with flagged data set to zero.
- // Missing time slots can also be detected at the beginning or end of the
- // MS by giving the correct starttime and endtime.
- // The correct UVW coordinates are calculated for inserted time slots.
- //
- // The process function only reads the data and flags to avoid that
- // too much data is kept in memory.
- // Other columns (like WEIGHT, UVW) can be read when needed by using the
- // appropriate DPInput::fetch function.
- //
- // The data columns are handled in the following way:
- // <table>
- // <tr>
- // <td>TIME</td>
- // <td>The time slot center of the current data (in MJD seconds).
- // It is assumed that all data have the same interval, which is
- // used to find missing time slots.
- // </td>
- // </tr>
- // <tr>
- // <td>DATA</td>
- // <td>The visibility data as [ncorr,nchan,nbaseline]. Only the
- // part given by startchan and nchan is read. If a time slot is
- // inserted, all its data are zero.
- // </td>
- // </tr>
- // <tr>
- // <td>FLAG</td>
- // <td>The data flags as [ncorr,nchan,nbaseline] (True is bad).
- // They are read from the FLAG column. If a FLAG_ROW is set, all
- // flags for that baseline will be set. Also the flag of data
- // containing NaN or infinite numbers will be set.
- // All flags of an inserted time slot are set.
- // </td>
- // </tr>
- // <tr>
- // <td>WEIGHT</td>
- // <td>The data weights as [ncorr,nchan,nbaseline]. Column
- // WEIGHT_SPECTRUM is used if present and containing valid data,
- // otherwise column WEIGHT is used. The weights of an inserted
- // time slot are set to 0.
- // If autoweight is on, the autocorrelations are used to
- // calculate proper weights.
- // </td>
- // </tr>
- // <tr>
- // <td>UVW</td>
- // <td>The UVW coordinates in meters as [3,nbaseline].
- // They are calculated for a missing time slot.
- // </td>
- // </tr>
- // <tr>
- // <td>FULLRESFLAG</td>
- // <td>For each baseline the LOFAR_FULL_RES_FLAG column is stored as
- // a uChar array with shape [orignchan/8, ntimeavg]. The bits
- // represent the flags. They are converted to a Bool array with shape
- // [orignchan, ntimeavg, nbaseline].
- // If column LOFAR_FULL_RES_FLAG is not present, the flags are used
- // and it is assumed that no averaging was done yet (thus ntimeavg=1
- // and orignchan=nchan).
- // </td>
- // </tr>
- // </table>
-
- class Filter: public DPStep
- {
- public:
- // Default constructor.
- Filter();
-
- // Construct the object for the given MS.
- // Parameters are obtained from the parset using the given prefix.
- Filter (DPInput* input, const ParameterSet&, const string& prefix);
-
- // Construct the object for the given MS and baseline selection.
- Filter (DPInput* input, const BaselineSelection&);
-
- virtual ~Filter();
-
- // Process the next data chunk.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // If needed, remove the deleted stations from the subtables
- // and renumber the remaining stations.
- virtual void addToMS (const string& msName);
-
- // Does the filter step has an actual selection?
- bool hasSelection() const
- { return itsDoSelect; }
-
- // Get the indices of the selected baselines.
- const vector<uint>& getIndicesBL() const
- { return itsSelBL; }
-
- // Get the buffer.
- const DPBuffer& getBuffer() const
- { return itsBuf; }
-
- private:
- // Create the mapping from old to new id (e.g. ANTENNA_ID).
- // The removed ids get a mapping -1.
- casacore::Vector<casacore::Int>
- createIdMap (casacore::uInt nrId,
- const casacore::Vector<casacore::uInt>& removedIds) const;
-
- // Remove rows with deleted stations from a subtable.
- // Renumber the ANTENNA_ID of the remaining rows.
- // It fills nrId with the original number of rows in the subtable
- // and returns the vector of removed row numbers.
- casacore::Vector<casacore::uInt>
- renumberSubTable (const casacore::Table& ms, const casacore::String& name,
- const casacore::String& colName,
- const casacore::Vector<casacore::uInt>& removedAnt,
- const casacore::Vector<casacore::Int>& antMap,
- casacore::uInt& nrId) const;
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuf;
- DPBuffer itsBufTmp;
- casacore::String itsStartChanStr; //# startchan expression
- casacore::String itsNrChanStr; //# nchan expression
- bool itsRemoveAnt; //# Remove from ANTENNA table?
- BaselineSelection itsBaselines;
- uint itsStartChan;
- vector<uint> itsSelBL; //# Index of baselines to select
- bool itsDoSelect; //# Any selection?
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/FlagCounter.h b/CEP/DP3/DPPP/include/DPPP/FlagCounter.h
deleted file mode 100644
index 538b17c4a313c2c741296109f0ea1d3c21d48b85..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/FlagCounter.h
+++ /dev/null
@@ -1,123 +0,0 @@
-//# FlagCounter.h: Class to keep counts of nr of flagged points
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_FLAGCOUNTER_H
-#define DPPP_FLAGCOUNTER_H
-
-// @file
-// @brief Class to keep counts of nr of flagged points
-
-#include <Common/lofar_vector.h>
-#include <Common/lofar_string.h>
-#include <Common/LofarTypes.h>
-#include <casacore/casa/Arrays/Vector.h>
-
-namespace LOFAR {
- class ParameterSet;
-
- namespace DPPP {
- //# Forward Declarations.
- class DPInfo;
-
- // @ingroup NDPPP
-
- // This class contains counts the number of flags.
- // The flags can be counted per baseline, channel, and correlation.
- // Once the counting is completed, they can be printed using the 'show'
- // functions. When printing, the baselines counts are shown per antenna.
- //
- // Optionally the flagging percentages can be saved in a table.
- // The name of the table is the MS name suffixed by the step name and '.flagxx'.
-
- class FlagCounter
- {
- public:
- // The default constructor creates an empty object. It does not save.
- FlagCounter();
-
- // This constructor creates an empty object.
- // It reads info from the parset to see if percentages have to be saved.
- FlagCounter (const string& msName, const ParameterSet&,
- const string& prefix);
-
- // Size all counters and initialize them to zero using the sizes
- // from the DPInfo object.
- void init (const DPInfo& info);
-
- // Increment the count per baseline.
- void incrBaseline (uint bl)
- { itsBLCounts[bl]++; }
-
- // Increment the count per channel.
- void incrChannel (uint chan)
- { itsChanCounts[chan]++; }
-
- // Increment the count per correlation.
- void incrCorrelation (uint corr)
- { itsCorrCounts[corr]++; }
-
- // Add the contents of that to this.
- void add (const FlagCounter& that);
-
- // Get the counts.
- const vector<int64>& baselineCounts() const
- { return itsBLCounts; }
- const vector<int64>& channelCounts() const
- { return itsChanCounts; }
- const vector<int64>& correlationCounts() const
- { return itsCorrCounts; }
-
- // Print the counts and optionally save percentages in a table.
- void showBaseline (ostream& os, int64 ntimes) const;
- void showChannel (ostream& os, int64 ntimes) const;
- void showCorrelation (ostream& os, int64 ntimes) const;
-
- // Show percentage with 1 decimal.
- static void showPerc1 (std::ostream&, double value, double total);
-
- // Show percentage with 3 decimals.
- static void showPerc3 (std::ostream&, double value, double total);
-
- private:
- // Save the percentages per station in a table.
- void saveStation (int64 npoints, const casacore::Vector<int64>& nused,
- const casacore::Vector<int64>& count) const;
-
- // Save the percentages per channel.
- void saveChannel (int64 npoints,
- const casacore::Vector<int64>& count) const;
-
- //# Data members.
- const DPInfo* itsInfo;
- string itsSaveName;
- double itsWarnPerc;
- bool itsShowFF;
- vector<int64> itsBLCounts;
- vector<int64> itsChanCounts;
- vector<int64> itsCorrCounts;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/GainCal.h b/CEP/DP3/DPPP/include/DPPP/GainCal.h
deleted file mode 100644
index 559dc0a30e0eb91ac3478fb712895cda7d693684..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/GainCal.h
+++ /dev/null
@@ -1,220 +0,0 @@
-//# GainCal.h: DPPP step class to calibrate (direction independent) gains
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.h 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_GAINCAL_H
-#define DPPP_GAINCAL_H
-
-// @file
-// @brief DPPP step class to apply a calibration correction to the data
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/PhaseFitter.h>
-#include <DPPP/BaselineSelection.h>
-#include <DPPP/StefCal.h>
-#include <DPPP/Patch.h>
-#include <DPPP/UVWFlagger.h>
-#include <DPPP/Predict.h>
-#include <ParmDB/ParmFacade.h>
-#include <ParmDB/ParmSet.h>
-#include <DPPP/SourceDBUtil.h>
-#include <DPPP/ApplyBeam.h>
-#include <StationResponse/Station.h>
-#include <StationResponse/Types.h>
-#include <ParmDB/Parm.h>
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Quanta/MVEpoch.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-
-// Convince HDF5 to use new API, even when system is configured to use 1.6 API
-#define H5Acreate_vers 2
-#define H5Tarray_create_vers 2
-#define H5Dcreate_vers 2
-#define H5Gcreate_vers 2
-#include <H5Cpp.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to calibrate (direction independent) gains.
-
- typedef std::vector<Patch::ConstPtr> PatchList;
- typedef std::pair<size_t, size_t >Baseline;
-
- class GainCal: public DPStep
- {
- public:
-
- enum CalType {COMPLEXGAIN, SCALARCOMPLEXGAIN, FULLJONES, PHASEONLY, SCALARPHASE, AMPLITUDEONLY,
- SCALARAMPLITUDE, TECANDPHASE, TEC, TECSCREEN, ROTATIONANDDIAGONAL, ROTATION};
-
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- GainCal (DPInput*, const ParameterSet&, const std::string& prefix);
-
- virtual ~GainCal();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Convert string to a CalType
- static CalType stringToCalType(const std::string& mode);
-
- // Convert CalType to a string
- static std::string calTypeToString(CalType caltype);
-
- // Make a soltab with the given type
- static std::vector<H5Parm::SolTab> makeSolTab(H5Parm& h5parm, CalType caltype,
- std::vector<H5Parm::AxisInfo>& axes);
-
- private:
- // Perform stefcal (polarized or unpolarized)
- void stefcal();
-
- // Check for scalar mode
- static bool scalarMode(CalType caltype);
-
- // Check for diagonal mode
- static bool diagonalMode(CalType caltype);
-
- // Apply the solution
- void applySolution(DPBuffer& buf, const casacore::Cube<casacore::DComplex>& invsol);
-
- // Invert solution (for applying it)
- casacore::Cube<casacore::DComplex> invertSol(const casacore::Cube<casacore::DComplex>& sol);
-
- // Fills the matrices itsVis and itsMVis
- void fillMatrices (casacore::Complex* model, casacore::Complex* data,
- float* weight, const casacore::Bool* flag);
-
- // Initialize the parmdb
- void initParmDB();
-
- // Get parmdbname from itsMode
- std::string parmName();
-
- // Determine which stations are used
- void setAntennaUsed();
-
- // Write out the solutions of the current parameter chunk (timeslotsperparmupdate)
- // Variant for writing ParmDB
- void writeSolutionsParmDB(double startTime);
-
- // Write out the solutions of the current parameter chunk (timeslotsperparmupdate)
- // Variant for writing H5Parm
- void writeSolutionsH5Parm(double startTime);
-
- //# Data members.
- DPInput* itsInput;
- std::string itsName;
- std::vector<DPBuffer> itsBuf;
- bool itsUseModelColumn;
- casacore::Cube<casacore::Complex> itsModelData;
- std::string itsParmDBName;
- bool itsUseH5Parm;
- boost::shared_ptr<BBS::ParmDB> itsParmDB;
- std::string itsParsetString; // Parset, for logging in H5Parm
-
- CalType itsMode;
-
- uint itsDebugLevel;
- bool itsDetectStalling;
-
- bool itsApplySolution;
-
- std::vector<casacore::Cube<casacore::DComplex> > itsSols; // for every timeslot, nCr x nSt x nFreqCells
- std::vector<casacore::Matrix<double> > itsTECSols; // for every timeslot, 2 x nSt (alpha and beta)
- std::vector<double> itsFreqData; // Mean frequency for every freqcell
-
- std::vector<casacore::CountedPtr<PhaseFitter> > itsPhaseFitters; // Length nSt
-
- std::vector<StefCal> iS;
-
- UVWFlagger itsUVWFlagStep;
- ResultStep::ShPtr itsDataResultStep; // Result step for data after UV-flagging
-
- Predict itsPredictStep;
- ApplyBeam itsApplyBeamStep; // Beam step for applying beam to modelcol
- ResultStep::ShPtr itsResultStep; // For catching results from Predict or Beam
- bool itsApplyBeamToModelColumn;
-
- BaselineSelection itsBaselineSelection; // Filter
- casacore::Vector<bool> itsSelectedBL; // Vector (length nBl) telling
- // which baselines are selected
- casacore::Vector<bool> itsAntennaUsed; // Vector (length nSt) telling
- // which stations are solved for
-
- std::map<std::string,int> itsParmIdMap; //# -1 = new parm name
-
- uint itsMaxIter;
- double itsTolerance;
- bool itsPropagateSolutions;
- uint itsSolInt; // Time cell size
- uint itsNChan; // Frequency cell size
- uint itsNFreqCells;
-
- uint itsTimeSlotsPerParmUpdate;
- uint itsConverged;
- uint itsNonconverged;
- uint itsFailed;
- uint itsStalled;
- std::vector<uint> itsNIter; // Total iterations made (for converged, stalled, nonconverged, failed)
- uint itsStepInParmUpdate; // Timestep within parameter update
- double itsChunkStartTime; // First time value of chunk to be stored
- uint itsStepInSolInt; // Timestep within solint
-
- casacore::Array<casacore::DComplex> itsAllSolutions; // Array that holds all solutions for all iterations
-
- FlagCounter itsFlagCounter;
-
- NSTimer itsTimer;
- NSTimer itsTimerPredict;
- NSTimer itsTimerSolve;
- NSTimer itsTimerPhaseFit;
- NSTimer itsTimerWrite;
- NSTimer itsTimerFill;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/GaussianSource.h b/CEP/DP3/DPPP/include/DPPP/GaussianSource.h
deleted file mode 100644
index 3cc56ebd272d51e6b090215edc7dcea284d6b191..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/GaussianSource.h
+++ /dev/null
@@ -1,93 +0,0 @@
-//# GaussianSource.h: Gaussian source model component.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_GAUSSIANSOURCE_H
-#define DPPP_GAUSSIANSOURCE_H
-
-// \file
-// Gaussian source model component.
-
-#include <DPPP/PointSource.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-class GaussianSource: public PointSource
-{
-public:
- typedef shared_ptr<GaussianSource> Ptr;
- typedef shared_ptr<const GaussianSource> ConstPtr;
-
- GaussianSource(const Position &position);
- GaussianSource(const Position &position, const Stokes &stokes);
-
- // Set position angle in radians. The position angle is the smallest angle
- // between the major axis and North, measured positively North over East.
- void setPositionAngle(double angle);
- double positionAngle() const;
-
- // Set the major axis length (FWHM in radians).
- void setMajorAxis(double fwhm);
- double majorAxis() const;
-
- // Set the minor axis length (FWHM in radians).
- void setMinorAxis(double fwhm);
- double minorAxis() const;
-
- virtual void accept(ModelComponentVisitor &visitor) const;
-
-private:
- double itsPositionAngle;
- double itsMajorAxis;
- double itsMinorAxis;
-};
-
-// @}
-
-// -------------------------------------------------------------------------- //
-// - Implementation: GaussianSource - //
-// -------------------------------------------------------------------------- //
-
-inline double GaussianSource::positionAngle() const
-{
- return itsPositionAngle;
-}
-
-inline double GaussianSource::majorAxis() const
-{
- return itsMajorAxis;
-}
-
-inline double GaussianSource::minorAxis() const
-{
- return itsMinorAxis;
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/GridInterpolate.h b/CEP/DP3/DPPP/include/DPPP/GridInterpolate.h
deleted file mode 100644
index cfd05379232468c37b20e9281adee20df4f75d69..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/GridInterpolate.h
+++ /dev/null
@@ -1,67 +0,0 @@
-//# GridInterpolate.h: Interpolate data from regular 2d grid to another
-//# Copyright (C) 2018
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GridInterpolate.h 37169 2017-04-19 12:41:21Z dijkema $
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_GRIDINTERPOLATE_H
-#define DPPP_GRIDINTERPOLATE_H
-
-// @file
-// @brief Interpolate data from regular 2d grid to another
-
-#include <vector>
-#include <cassert>
-#include <stdexcept>
-
-namespace LOFAR {
- //! Get the nearest-neighbor indices
- ///*! \param ax_src[in] Vector with points where the data is defined.
- // Should be increasing.
- // * \param ax_tgt[in] Vector with the points at which the values are
- // needed. Should be increasing.
- // * \param[out] indices Vector (same length as ax_tgt) with for each number
- // in ax_src, the index of the nearest point in ax_src.
- // * \param[in] nearest Get the nearest point. If false, gets the largest
- // point that is smaller.
- // */
- void getAxisIndices(const std::vector<double>& ax_src,
- const std::vector<double>& ax_tgt,
- std::vector<size_t>& indices,
- bool nearest = true);
-
- //! Regrid 2d-gridded data onto another 2d grid
- /*! \param[in] x_src x-axis on which the data is defined
- * \param[in] y_src y-axis on which the data is defined
- * \param[in] x_tgt x-axis on which the data will be evaluated
- * \param[in] y_tgt y-axis on which the data will be evaluated
- * \param[in] vals_src original data, y-axis varies fastest
- * \param[out] vals_tgt regridded data, y-axis varies fastest
- */
- void gridNearestNeighbor(const std::vector<double>& x_src,
- const std::vector<double>& y_src,
- const std::vector<double>& x_tgt,
- const std::vector<double>& y_tgt,
- const double* vals_src,
- double* vals_tgt,
- bool nearest = true);
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/H5Parm.h b/CEP/DP3/DPPP/include/DPPP/H5Parm.h
deleted file mode 100644
index d2a3a3930886394b5538e0d475d65bc64c07259f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/H5Parm.h
+++ /dev/null
@@ -1,254 +0,0 @@
-#ifndef DPPP_H5PARM_H
-#define DPPP_H5PARM_H
-
-#include <string>
-#include <vector>
-#include <complex>
-#include <map>
-
-#include <H5Cpp.h>
-
-#include <utility>
-
-namespace LOFAR {
- class H5Parm : private H5::H5File
- {
- typedef struct antenna_t {
- char name[16];
- float position[3];
- } antenna_t;
-
- typedef struct source_t {
- char name[128];
- float dir[2];
- } source_t;
-
- typedef struct polarization_t {
- char name[2];
- } polarization_t;
-
- public:
- // A name and the length of an exis, e.g. ('freq', 800) for 800 frequencies
- struct AxisInfo {
- public: AxisInfo(const std::string& name, uint size) :
- name(name), size(size) {};
-
- std::string name;
- uint size;
- };
-
- // SolTab is a solution table as defined in the H5Parm standard. It
- // contains one solution, e.g. all TEC values, with different axes
- // for that solution (e.g. time, freq, pol).
- class SolTab : private H5::Group {
- public:
- // Default constructor
- SolTab() {};
-
- // Create a new soltab, add it to its parent
- SolTab(H5::Group group,
- const std::string& type,
- const std::vector<AxisInfo> axes // Axes, fastest varying last
- );
-
- // Create a soltab from a H5::Group (for reading existing files)
- SolTab(H5::Group& group);
-
- // The destructor could check for valid subtables
- virtual ~SolTab();
-
- SolTab operator=(H5::Group group);
-
- std::vector<AxisInfo>& getAxes() {return _axes;}
-
- AxisInfo getAxis(uint i) const;
-
- // Get an axis, throw an exception if it does not exist
- AxisInfo getAxis(const std::string& axisName) const;
-
- size_t nAxes() { return _axes.size(); }
-
- bool hasAxis(const std::string& axisName);
-
- // Get the index of an axis
- size_t getAxisIndex(const std::string& axisname);
-
- void setAntennas(const std::vector<std::string>& solAntennas);
-
- void setSources(const std::vector<std::string>& solSources);
-
- void setPolarizations(const std::vector<std::string>& polarizations);
-
- void setFreqs(const std::vector<double>& freqs);
-
- // Get the values of a real-valued axis (e.g. "time" or "freq")
- std::vector<double> getRealAxis(const std::string& axisname) const;
-
- // Get the values of a string-valued axis (e.g. "dir" or "pol")
- std::vector<std::string> getStringAxis(const std::string& axisname);
-
- // Get the index of freq, using nearest neighbor
- // This assumes that the frequencies are in increasing order.
- hsize_t getFreqIndex(double freq) const;
-
- // Get the index of a time. Matches with 0.5*timeInterval
- // This assumes that all times are regularly spaced
- hsize_t getTimeIndex(double time) const;
-
- hsize_t getDirIndex(const std::string& dirName);
-
- // Gets the interval (in s.) between a time slot (default first) and
- // the next. Throws error if there is only one time slot.
- double getTimeInterval(size_t start=0) const {
- return getInterval("time", start);
- }
-
- // Gets the interval (in s.) between a channel (default first) and
- // the next. Throws error if there is only one frequency.
- double getFreqInterval(size_t start=0) const {
- return getInterval("freq", start);
- }
-
- void setTimes(const std::vector<double>& times);
-
- // Set metadata about an axis (like freq or time))
- void setAxisMeta(const std::string& metaName,
- const std::vector<double>& metaVals);
-
- // Set metadata about an axis (like polarization, direction)
- void setAxisMeta(const std::string& metaName,
- size_t strLen,
- const std::vector<std::string>& metaVals);
-
- // Adds a real solution.
- // If weights are emtpy, write ones everywhere
- void setValues(const std::vector<double>& vals,
- const std::vector<double>& weights,
- const std::string& history="");
-
- // Add a complex solution, taking either amplitude or phase
- void setComplexValues(const std::vector<std::complex<double> >& vals,
- const std::vector<double>& weights,
- bool toAmplitudes, const std::string& history="");
-
-
-
- // Get the name of this SolTab
- std::string getName() const;
-
- std::string getType() const {return _type;}
-
- // Get the values of this SolTab for a given antenna.
- std::vector<double> getValues(
- const std::string& antName,
- uint starttimeslot, uint ntime, uint timestep=1,
- uint startfreq=0, uint nfreq=1, uint freqstep=1,
- uint pol=0, uint dir=0) {
- return getValuesOrWeights("val", antName,
- starttimeslot, ntime, timestep,
- startfreq, nfreq, freqstep,
- pol, dir);
- }
-
- // Get the weights of this SolTab for a given antenna.
- std::vector<double> getWeights(
- const std::string& antName,
- uint starttimeslot, uint ntime, uint timestep=1,
- uint startfreq=0, uint nfreq=1, uint freqstep=1,
- uint pol=0, uint dir=0) {
- return getValuesOrWeights("weight", antName,
- starttimeslot, ntime, timestep,
- startfreq, nfreq, freqstep,
- pol, dir);
- }
-
- std::vector<double> getValuesOrWeights(
- const std::string& valOrWeight,
- const std::string& antName,
- const std::vector<double>& times,
- const std::vector<double>& freqs,
- uint pol, uint dir);
- private:
- // Get the values or weights of this SolTab for a given antenna.
- std::vector<double> getValuesOrWeights(
- const std::string& valOrWeight,
- const std::string& antName,
- uint starttimeslot, uint ntime, uint timestep,
- uint startfreq, uint nfreq, uint freqstep,
- uint pol, uint dir);
-
- void readAxes();
-
- void fillCache(std::map<std::string, hsize_t>& cache,
- const std::string& tableName);
-
- // Get the interval of the axis axisName
- double getInterval(const std::string& axisName, size_t start=0) const;
- hsize_t getAntIndex(const std::string& antName);
- hsize_t getNamedIndex(std::map<std::string, hsize_t>& cache,
- const std::string& tableName,
- const std::string& elementName);
-
- std::string _type;
- std::vector<AxisInfo> _axes;
- std::map<std::string, hsize_t> _antMap;
- std::map<std::string, hsize_t> _dirMap;
- };
-
- // Open existing H5Parm or create a new one
- // Default name is given by solSetName, if that does not exist continue
- // searching for sol000, sol001, etc.
- // Only one solset of an H5Parm can be opened at once; this object only
- // provides info about one SolSet (even though the file can contain more).
- H5Parm(const std::string& filename, bool forceNew=false,
- bool forceNewSolSet=false, const std::string& solSetName="");
-
-
- H5Parm();
-
- virtual ~H5Parm();
-
- // Add metadata (directions on the sky in J2000) about named sources
- void addSources (const std::vector<std::string>& names,
- const std::vector<std::pair<double, double> >& dirs);
-
- // Add metadata (positions on earth in ITRF) about antennas
- void addAntennas(const std::vector<std::string>& names,
- const std::vector<std::vector<double> >& positions);
-
- // Add metadata about polarizations
- void addPolarizations(const std::vector<std::string>& polarizations);
-
- // Add a version stamp in the attributes of the group
- static void addVersionStamp(H5::Group &node);
-
- // Create and return a new soltab. Type is the type as used in BBS
- SolTab& createSolTab(const std::string& name,
- const std::string& type,
- const std::vector<AxisInfo> axes);
-
- SolTab& getSolTab(const std::string& name);
-
- // Get the name of the one SolSet used in this H5Parm
- std::string getSolSetName() const ;
-
- // Get the number of SolTabs in the active solset of this h5parm
- size_t nSolTabs() { return _solTabs.size(); }
-
- // Is the given soltab resent in the active solset of this h5parm
- bool hasSolTab(const std::string& solTabName) const;
- private:
-
- static double takeAbs(std::complex<double> c) {
- return std::abs(c);
- }
- static double takeArg(std::complex<double> c) {
- return std::arg(c);
- }
-
- std::map<std::string, SolTab> _solTabs;
- H5::Group _solSet;
- };
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/H5ParmPredict.h b/CEP/DP3/DPPP/include/DPPP/H5ParmPredict.h
deleted file mode 100644
index ad1a464583613167d05c7aac11033c1b7213d17d..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/H5ParmPredict.h
+++ /dev/null
@@ -1,94 +0,0 @@
-//# H5ParmPredict.h: DPPP step class to H5ParmPredict visibilities from a source model
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_H5ParmPredict_H
-#define DPPP_H5ParmPredict_H
-
-// @file
-// @brief DPPP step class to H5ParmPredict visibilities from a source model
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/Predict.h>
-
-#include <DPPP/H5Parm.h>
-
-#include <utility>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to H5ParmPredict visibilities with optionally beam
-
- typedef std::pair<size_t, size_t> Baseline;
- typedef std::pair<ModelComponent::ConstPtr, Patch::ConstPtr> Source;
-
- class H5ParmPredict: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- H5ParmPredict (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~H5ParmPredict();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuffer;
-
- std::vector<Predict::ShPtr> itsPredictSteps;
- ResultStep* itsResultStep;
-
- std::string itsH5ParmName;
- std::vector<std::string> itsDirections;
-
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/MSReader.h b/CEP/DP3/DPPP/include/DPPP/MSReader.h
deleted file mode 100644
index ffb5ca1fbfaa4c4b14fcc821ebfb06db3884aaaf..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/MSReader.h
+++ /dev/null
@@ -1,332 +0,0 @@
-//# MSReader.h: DPPP step reading from an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_MSREADER_H
-#define DPPP_MSREADER_H
-
-// @file
-// @brief DPPP step reading from an MS
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/UVWCalculator.h>
-#include <DPPP/FlagCounter.h>
-#include <casacore/tables/Tables/TableIter.h>
-#include <casacore/tables/Tables/RefRows.h>
-#include <casacore/casa/Arrays/Slicer.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPInput step reading the data from a MeasurementSet.
- // At the beginning it finds out the shape of the data; i.e., the
- // number of correlations, channels, baselines, and time slots.
- // It requires the data to be regularly shaped.
- //
- // The object is constructed from the 'msin' keywords in the parset file.
- // Currently the following can be given:
- // <ul>
- // <li> msin: name of the MS
- // <li> msin.autoweight: calculate weights from autocorrelations? [no]
- // <li> msin.startchan: first channel to use [0]
- // <li> msin.nchan: number of channels to use [all]
- // <li> msin.useflag: use the existing flags? [yes]
- // <li> msin.datacolumn: the data column to use [DATA]
- // <li> msin.weightcolumn: the weights column to use [WEIGHT_SPECTRUM or
- // WEIGHT]
- // <li> msin.starttime: first time to use [first time in MS]
- // <li> msin.endtime: last time to use [last time in MS]
- // </ul>
- //
- // If a time slot is missing, it is inserted with flagged data set to zero.
- // Missing time slots can also be detected at the beginning or end of the
- // MS by giving the correct starttime and endtime.
- // The correct UVW coordinates are calculated for inserted time slots.
- //
- // The process function only reads the data and flags to avoid that
- // too much data is kept in memory.
- // Other columns (like WEIGHT, UVW) can be read when needed by using the
- // appropriate DPInput::fetch function.
- //
- // The data columns are handled in the following way:
- // <table>
- // <tr>
- // <td>TIME</td>
- // <td>The time slot center of the current data (in MJD seconds).
- // It is assumed that all data have the same interval, which is
- // used to find missing time slots.
- // </td>
- // </tr>
- // <tr>
- // <td>DATA</td>
- // <td>The visibility data as [ncorr,nchan,nbaseline]. Only the
- // part given by startchan and nchan is read. If a time slot is
- // inserted, all its data are zero.
- // </td>
- // </tr>
- // <tr>
- // <td>FLAG</td>
- // <td>The data flags as [ncorr,nchan,nbaseline] (True is bad).
- // They are read from the FLAG column. If a FLAG_ROW is set, all
- // flags for that baseline will be set. Also the flag of data
- // containing NaN or infinite numbers will be set.
- // All flags of an inserted time slot are set.
- // </td>
- // </tr>
- // <tr>
- // <td>WEIGHT</td>
- // <td>The data weights as [ncorr,nchan,nbaseline]. Column
- // WEIGHT_SPECTRUM is used if present and containing valid data,
- // otherwise column WEIGHT is used. The weights of an inserted
- // time slot are set to 0.
- // If autoweight is on, the autocorrelations are used to
- // calculate proper weights.
- // </td>
- // </tr>
- // <tr>
- // <td>UVW</td>
- // <td>The UVW coordinates in meters as [3,nbaseline].
- // They are calculated for a missing time slot.
- // </td>
- // </tr>
- // <tr>
- // <td>FULLRESFLAG</td>
- // <td>For each baseline the LOFAR_FULL_RES_FLAG column is stored as
- // a uChar array with shape [orignchan/8, ntimeavg]. The bits
- // represent the flags. They are converted to a Bool array with shape
- // [orignchan, ntimeavg, nbaseline].
- // If column LOFAR_FULL_RES_FLAG is not present, the flags are used
- // and it is assumed that no averaging was done yet (thus ntimeavg=1
- // and orignchan=nchan).
- // </td>
- // </tr>
- // </table>
-
- class MSReader: public DPInput
- {
- public:
- // Default constructor.
- MSReader();
-
- // Construct the object for the given MS.
- // Parameters are obtained from the parset using the given prefix.
- // The missingData argument is for MultiMSReader.
- MSReader (const std::string& msName,
- const ParameterSet&, const string& prefix,
- bool missingData = false);
-
- virtual ~MSReader();
-
- // Process the next data chunk.
- // It returns false when at the end.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Add some data to the MeasurementSet written/updated.
- // Do nothing.
- virtual void addToMS (const string&) {};
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // If needed, show the flag counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Read the UVW at the given row numbers into the buffer.
- virtual void getUVW (const casacore::RefRows& rowNrs,
- double time,
- DPBuffer&);
-
- // Read the weights at the given row numbers into the buffer.
- // Note: the buffer must contain DATA if autoweighting is in effect.
- virtual void getWeights (const casacore::RefRows& rowNrs,
- DPBuffer&);
-
- // Read the fullRes flags (LOFAR_FULL_RES_FLAG) at the given row numbers
- // into the buffer.
- // If there is no such column, the flags are set to false and false is
- // returned.
- virtual bool getFullResFlags (const casacore::RefRows& rowNrs,
- DPBuffer&);
-
- // Read the model data at the given row numbers into the array.
- virtual void getModelData (const casacore::RefRows& rowNrs,
- casacore::Cube<casacore::Complex>&);
-
- // Fill the vector with station beam info from the input MS.
- // Only fill it for the given station names.
- virtual void fillBeamInfo (vector<StationResponse::Station::Ptr>&,
- const casacore::Vector<casacore::String>& antNames);
-
- // Tell if the visibility data are to be read.
- virtual void setReadVisData (bool readVisData);
-
- // Get the main MS table.
- casacore::Table& table()
- { return itsMS; }
-
- // Get the name of the data column to be used.
- const casacore::String& dataColumnName() const
- { return itsDataColName; }
-
- const casacore::String& weightColumnName() const
- { return itsWeightColName; }
-
- const casacore::String& modelColumnName() const
- { return itsModelColName; }
-
- // Get the slicer in the FLAG and DATA column.
- const casacore::Slicer& colSlicer() const
- { return itsColSlicer; }
-
- // Get the rownrs for meta info of missing time slots.
- // It uses the rows of the first time slot.
- const casacore::Vector<uint>& getBaseRowNrs() const
- { return itsBaseRowNrs; }
-
- // Get the name of the MS.
- virtual casacore::String msName() const;
-
- // Get the time information.
- double firstTime() const
- { return itsFirstTime; }
- double lastTime() const
- { return itsLastTime; }
-
- // Get the selected spectral window.
- uint spectralWindow() const
- { return itsSpw; }
-
- // Get the baseline selection.
- const string& baselineSelection() const
- { return itsSelBL; }
-
- // Is the data column missing?
- bool missingData() const
- { return itsMissingData; }
-
- // Get the start channel.
- uint startChan() const
- { return itsStartChan; }
-
- // Get the nr of averaged full resolution channels.
- uint nchanAvgFullRes() const
- { return itsFullResNChanAvg; }
- // Get the nr of averaged full resolution time slots.
- uint ntimeAvgFullRes() const
- { return itsFullResNTimeAvg; }
-
- // Tell if the input MS has LOFAR_FULL_RES_FLAG.
- bool hasFullResFlags() const
- { return itsHasFullResFlags; }
-
- // Get access to the buffer.
- const DPBuffer& getBuffer() const
- { return itsBuffer; }
-
- // Flags inf and NaN
- static void flagInfNaN(const casacore::Cube<casacore::Complex>& dataCube,
- casacore::Cube<bool>& flagsCube, FlagCounter& flagCounter);
-
- private:
- // Prepare the access to the MS.
- // Return the first and last time and the interval.
- void prepare (double& firstTime, double& lastTime,
- double& interval);
-
- // Do the rest of the preparation.
- void prepare2();
-
- // Skip the first times in the MS in case a start time was given.
- // If needed, it sets itsFirstTime properly.
- void skipFirstTimes();
-
- // Calculate the UVWs for a missing time slot.
- void calcUVW (double time, DPBuffer&);
-
- // Calculate the weights from the autocorrelations.
- void autoWeight (casacore::Cube<float>& weights, const DPBuffer& buf);
-
- protected:
- //# Data members.
- casacore::String itsMSName;
- casacore::Table itsMS;
- casacore::Table itsSelMS; //# possible selection of spw, baseline
- casacore::TableIterator itsIter;
- casacore::String itsDataColName;
- casacore::String itsWeightColName;
- casacore::String itsModelColName;
- casacore::String itsStartChanStr; //# startchan expression
- casacore::String itsNrChanStr; //# nchan expression
- string itsSelBL; //# Baseline selection string
- bool itsReadVisData; //# read visibility data?
- bool itsNeedSort; //# sort needed on time,baseline?
- bool itsAutoWeight; //# calculate weights from autocorr?
- bool itsAutoWeightForce; //# always calculate weights?
- bool itsHasWeightSpectrum;
- bool itsUseFlags;
- bool itsUseAllChan; //# all channels (i.e. no slicer)?
- bool itsMissingData; //# allow missing data column?
- int itsSpw; //# spw (band) to use (<0 no select)
- uint itsNrBl;
- uint itsNrCorr;
- uint itsNrChan;
- uint itsStartChan;
- double itsTimeTolerance; //# tolerance for time comparison
- double itsTimeInterval;
- double itsStartTime;
- double itsFirstTime;
- double itsLastTime;
- double itsNextTime;
- double itsLastMSTime;
- uint itsNrRead; //# nr of time slots read from MS
- uint itsNrInserted; //# nr of inserted time slots
- casacore::Slicer itsColSlicer; //# slice in corr,chan column
- casacore::Slicer itsArrSlicer; //# slice in corr,chan,bl array
- bool itsHasFullResFlags;
- uint itsFullResNChanAvg;
- uint itsFullResNTimeAvg;
- DPBuffer itsBuffer;
- UVWCalculator itsUVWCalc;
- casacore::Vector<uint> itsBaseRowNrs; //# rownrs for meta of missing times
- FlagCounter itsFlagCounter;
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/MSUpdater.h b/CEP/DP3/DPPP/include/DPPP/MSUpdater.h
deleted file mode 100644
index cd6237b29895f25e17bbe12959e958242385ee95..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/MSUpdater.h
+++ /dev/null
@@ -1,134 +0,0 @@
-//# MSUpdater.h: DPPP step writing to an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_MSUPDATER_H
-#define DPPP_MSUPDATER_H
-
-// @file
-// @brief DPPP step writing to an MS
-
-#include <DPPP/DPStep.h>
-#include <DPPP/StManParsetKeys.h>
-
-#include <Common/LofarTypes.h>
-#include <casacore/tables/Tables/ColumnDesc.h>
-#include <casacore/tables/Tables/RefRows.h>
-#include <casacore/tables/Tables/Table.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- //# Forward Declarations.
- class MSReader;
-
- // @ingroup NDPPP
-
- // This class updates the flags in an existing MeasurementSet.
- // Hardly anything is done in this class.
- // It uses function putFlags in MSReader to do the actual write.
- //
- // Like MSWriter it adds an entry to the HISTORY table of the MS
- // containing the parset values and DPPP version.
-
- class MSUpdater: public DPStep
- {
- public:
- MSUpdater (MSReader* reader, casacore::String msName,
- const ParameterSet& parset, const std::string& prefix,
- bool writeHistory=true);
-
- virtual ~MSUpdater();
-
- // Process the next data chunk.
- // It returns false when at the end.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Add some data to the MeasurementSet written/updated.
- // Calls addToMS from the previous step, with the current output msname.
- virtual void addToMS (const string&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Tests if an update of the buffer described in info to the MS msName
- // is possible. When throwError is true, it will throw an error with a
- // descriptive string before returning false
- static bool updateAllowed (const DPInfo& info, casacore::String msName,
- bool throwError=true);
-
- private:
- // Write the flags at the given row numbers.
- void putFlags (const casacore::RefRows& rowNrs,
- const casacore::Cube<bool>& flags);
-
- // Write the weights at the given row numbers
- void putWeights (const casacore::RefRows& rowNrs,
- const casacore::Cube<float>& weights);
-
- // Write the data at the given row numbers.
- void putData (const casacore::RefRows& rowNrs,
- const casacore::Cube<casacore::Complex>& data);
-
- // If not existing yet, add the column specified by colname.
- // Column will containt arrays of type datatype.
- // If the column has been added, this function returns true
- bool addColumn(const string& colname, const casacore::DataType dataType,
- const casacore::ColumnDesc& cd);
-
- //# Data members
- MSReader* itsReader;
- string itsName;
- casacore::String itsMSName;
- casacore::Table itsMS;
- const ParameterSet& itsParset;
- DPBuffer itsBuffer;
- casacore::String itsDataColName;
- casacore::String itsWeightColName;
- uint itsNrTimesFlush; //# flush every N time slots (0=no flush)
- bool itsWriteData;
- bool itsWriteWeights;
- bool itsWriteFlags;
- uint itsNrDone; //# nr of time slots written
- bool itsDataColAdded; //# has data column been added?
- bool itsWeightColAdded; //# has weight column been added?
- bool itsWriteHistory; //# Should history be written?
- NSTimer itsTimer;
- uint itsTileSize;
- StManParsetKeys itsStManKeys;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/MSWriter.h b/CEP/DP3/DPPP/include/DPPP/MSWriter.h
deleted file mode 100644
index 08500b34fe78e484c435d1e85a3e2edb3a35f32f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/MSWriter.h
+++ /dev/null
@@ -1,200 +0,0 @@
-//# MSWriter.h: DPPP step writing to an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_MSWRITER_H
-#define DPPP_MSWRITER_H
-
-// @file
-// @brief DPPP step writing to an MS
-
-#include <DPPP/DPStep.h>
-#include <DPPP/MSReader.h>
-#include <DPPP/StManParsetKeys.h>
-
-#include <casacore/tables/Tables/Table.h>
-#include <casacore/tables/Tables/ColumnDesc.h>
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/tables/Tables/ArrayColumn.h>
-#include <casacore/tables/DataMan/TiledColumnStMan.h>
-
-namespace LOFAR {
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep creating a new MeasurementSet and writing
- // all data in it.
- // Most meta information (subtables and meta columns in main table) is
- // copied from the input MeasurementSet given by the MSReader object.
- // <br>
- // In principle the new MS uses the same storage managers as used in the
- // input MS, but in case of an MS stored with LofarStMan it will use the
- // optimal storage managers (ISM for slowly varying meta data, TSM for
- // bulk data, SSM for others).
- //
- // The SPECTRAL_WINDOW table will be changed to reflect the channels
- // being used or averaged.
- // The OBSERVATION table will be updated for the correct start and end time.
- // The HISTORY table gets an entry containing the parset values and the
- // DPPP version.
-
- class MSWriter: public DPStep
- {
- public:
- explicit MSWriter (MSReader* reader, const std::string& outName,
- const ParameterSet&, const string& prefix);
-
- virtual ~MSWriter();
-
- // Process the next data chunk.
- // It returns false when at the end.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Add some data to the MeasurementSet written/updated.
- // Calls addToMS from the previous step, with the current output msname.
- virtual void addToMS (const string&);
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Write the parset info into the HISTORY table of the MS.
- static void writeHistory (casacore::Table& ms,
- const ParameterSet& parset);
-
- private:
- // Create an array column description and add to table with given
- // stoage manager (if given).
- void makeArrayColumn (casacore::ColumnDesc desc, const casacore::IPosition& shape,
- casacore::DataManager* dm, casacore::Table& table, bool makeDirectColumn = false);
-
- // Create the MS by cloning all subtables from the input MS.
- // All output columns in the main table are using normal storage managers.
- // The SPECTRAL_WINDOW table is adapted as needed.
- void createMS (const std::string& outName, const DPInfo& info,
- uint tileSize, uint tileNChan);
-
- // Update the SPECTRAL_WINDOW table for averaged channels.
- void updateSpw (const string& outName, const DPInfo& info);
-
- // Update the OBSERVATION table with the correct start and end time.
- void updateObs (const string& outName);
-
- // Update the FIELD table with the new phase center.
- void updateField (const string& outName, const DPInfo& info);
-
- // Write the data, flags, etc.
- void writeData (casacore::Table& out, const DPBuffer& buf);
-
- // Write the full resolution flags (flags before any averaging).
- void writeFullResFlags (casacore::Table& out, const DPBuffer& buf);
-
- // Write all meta data columns for a time slot (ANTENNA1, etc.)
- void writeMeta (casacore::Table& out, const DPBuffer& buf);
-
- // Copy meta data columns for a time slot (ANTENNA1, etc.)
- // It also copies all time info if possible.
- void copyMeta (const casacore::Table& in, casacore::Table& out,
- bool copyTimeInfo);
-
- // Copy the contents of a scalar column.
- template<typename T> void fillSca (const T& value,
- casacore::Table& out,
- const casacore::String& columnName)
- {
- casacore::ScalarColumn<T> outCol(out, columnName);
- outCol.fillColumn (value);
- }
-
- // Copy the contents of an array column.
- template<typename T> void fillArr (const casacore::Array<T>& value,
- casacore::Table& out,
- const casacore::String& columnName)
- {
- casacore::ArrayColumn<T> outCol(out, columnName);
- outCol.fillColumn (value);
- }
-
- // Copy the contents of a scalar column.
- template<typename T> void copySca (const casacore::Table& in,
- casacore::Table& out,
- const casacore::String& columnName)
- {
- casacore::ROScalarColumn<T> inCol(in, columnName);
- casacore::ScalarColumn<T> outCol(out, columnName);
- outCol.putColumn (inCol.getColumn());
- }
-
- // Copy the contents of an array column.
- template<typename T> void copyArr (const casacore::Table& in,
- casacore::Table& out,
- const casacore::String& columnName)
- {
- casacore::ROArrayColumn<T> inCol(in, columnName);
- casacore::ArrayColumn<T> outCol(out, columnName);
- outCol.putColumn (inCol.getColumn());
- }
-
- //# Data items.
- MSReader* itsReader;
- string itsName;
- string itsOutName;
- DPBuffer itsBuffer;
- casacore::Table itsMS;
- ParameterSet itsParset; //# parset for writing history
- casacore::String itsDataColName;
- casacore::String itsWeightColName;
- double itsInterval;
- bool itsOverwrite; //# Overwrite an existing output MS?
- bool itsCopyCorrData;
- bool itsCopyModelData;
- bool itsWriteFullResFlags;
- uint itsTileSize;
- uint itsTileNChan;
- uint itsNrCorr;
- uint itsNrChan;
- uint itsNrBl;
- uint itsNrTimes;
- uint itsNChanAvg; //# nr of channels in input averaged to 1
- uint itsNTimeAvg; //# nr of times in input averaged to 1
- uint itsNrTimesFlush;//# flush every N time slots (0=no flush)
- uint itsNrDone; //# nr of time slots written
- std::string itsVdsDir; //# directory where to put VDS file
- std::string itsClusterDesc; //# name of clusterdesc file
- NSTimer itsTimer;
- StManParsetKeys itsStManKeys;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/MedFlagger.h b/CEP/DP3/DPPP/include/DPPP/MedFlagger.h
deleted file mode 100644
index 9b02b1c0f708c032c9eed59e06092a8f7d4f3b66..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/MedFlagger.h
+++ /dev/null
@@ -1,155 +0,0 @@
-//# MedFlagger.h: DPPP step class to flag data based on median filtering
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_MEDFLAGGER_H
-#define DPPP_MEDFLAGGER_H
-
-// @file
-// @brief DPPP step class to flag data based on median filtering
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/FlagCounter.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR {
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class flagging data points based on the median
- // of the absolute difference of the data and the median of the data.
- // Both medians are taken in a time/frequency window around the data point.
- // Only unflagged data points in the window are taken into account.
- // The size of the window is given in the parset file.
- //
- // The window around data points at the edges is formed by mirroring the
- // data at the edge. For example, for channel 0 and a window size of 7
- // the data are mirrored, thus channels 3,2,1,0,1,2,3 will be used.
- // For channel 1 the channels 2,1,0,1,2,3,4 will be used.
- // The test program tMirror.cc can be used to check the correctness of
- // the alogorithm to determine the channels to use.
- //
- // Taking the median is an O(N) operation, thus doing it for all data
- // points is an O(N^2) operation. The test program tMedian.cc can be
- // used to test the performance of the algorithms to determine the median.
- // It shows that casacore's kthLargest outperforms STL's nth_element.
- // <br>
- // Shuffling the data around to be able to determine the medians is also
- // an expensive operation, but takes less time than the medians themselves.
- //
- // When a correlation is flagged, all correlations for that data point
- // are flagged. It is possible to specify which correlations have to be
- // taken into account when flagging. Using, say, only XX may boost
- // performance with a factor 4, but miss points to be flagged.
- // It is also possible to specify the order in which the correlations
- // have to be tested.
- //
- // It is possible to flag specific baselines only using a selection on
- // baseline length.
- // <br>Furthermore it is possible to only flag the autocorrelations and
- // apply the resulting flags to the crosscorrelations, possibly selected
- // on baseline length.
-
- class MedFlagger: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- MedFlagger (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~MedFlagger();
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- // It is used to adjust the parms if needed.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the flagger counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Flag for the entry at the given index.
- // Use the given time entries for the medians.
- // Process the result in the next step.
- void flag (uint index, const vector<uint>& timeEntries);
-
- // Compute the median factors for given baseline, channel, and
- // correlation.
- void computeFactors (const vector<uint>& timeEntries,
- uint bl, int chan, int corr,
- int nchan, int ncorr,
- float& Z1, float& Z2,
- float* tempBuf,
- NSTimer& moveTimer, NSTimer& medianTimer);
-
- // Get the values of the expressions for each baseline.
- void getExprValues (int maxNChan, int maxNTime);
-
- protected:
- //# Data members.
- DPInput* itsInput;
- string itsName;
- string itsThresholdStr;
- string itsFreqWindowStr;
- string itsTimeWindowStr;
- vector<float> itsThresholdArr; //# threshold per baseline
- vector<uint> itsFreqWindowArr; //# freq window size per baseline
- vector<uint> itsTimeWindowArr; //# time window size per baseline
- float itsThreshold;
- uint itsFreqWindow;
- uint itsTimeWindow;
- uint itsNTimes;
- uint itsNTimesDone;
- vector<uint> itsFlagCorr;
- bool itsApplyAutoCorr;
- vector<int> itsAutoCorrIndex; //# baseline index of autocorrelations
- uint itsNrAutoCorr;
- double itsMinBLength; //# minimum baseline length
- double itsMaxBLength; //# maximum baseline length
- vector<double> itsBLength; //# length of each baseline
- vector<DPBuffer> itsBuf;
- vector<casacore::Cube<float> > itsAmpl; //# amplitudes of the data
- FlagCounter itsFlagCounter;
- NSTimer itsTimer;
- NSTimer itsComputeTimer; //# move/median timer
- double itsMoveTime; //# data move timer (sum all threads)
- double itsMedianTime; //# median timer (sum of all threads)
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/ModelComponent.h b/CEP/DP3/DPPP/include/DPPP/ModelComponent.h
deleted file mode 100644
index 34c90cbe96ce9d1f4f9c00737616c32ef4c6716e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/ModelComponent.h
+++ /dev/null
@@ -1,58 +0,0 @@
-//# ModelComponent.h: Base class for model components.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_MODELCOMPONENT_H
-#define DPPP_MODELCOMPONENT_H
-
-// \file
-// Base class for model components.
-
-#include <Common/lofar_smartptr.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-class ModelComponentVisitor;
-class Position;
-
-// \addtogroup NDPPP
-// @{
-
-class ModelComponent
-{
-public:
- typedef shared_ptr<ModelComponent> Ptr;
- typedef shared_ptr<const ModelComponent> ConstPtr;
-
- virtual ~ModelComponent();
- virtual const Position &position() const = 0;
- virtual void accept(ModelComponentVisitor&) const = 0;
-};
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/ModelComponentVisitor.h b/CEP/DP3/DPPP/include/DPPP/ModelComponentVisitor.h
deleted file mode 100644
index c45857cd0f9489ae2d3ccae80e23a2ed4f25538b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/ModelComponentVisitor.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//# ModelComponentVisitor.h: Base class for visitors that visit model component
-//# hierarchies.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_MODELCOMPONENTVISITOR_H
-#define DPPP_MODELCOMPONENTVISITOR_H
-
-// \file
-// Base class for visitors that visit model component hierarchies.
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-class PointSource;
-class GaussianSource;
-
-// \addtogroup NDPPP
-// @{
-
-class ModelComponentVisitor
-{
-public:
- virtual ~ModelComponentVisitor();
-
- virtual void visit(const PointSource&) = 0;
- virtual void visit(const GaussianSource&) = 0;
-};
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/MultiMSReader.h b/CEP/DP3/DPPP/include/DPPP/MultiMSReader.h
deleted file mode 100644
index e7ef49617901c245513f9f3486df8932fd9a1d66..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/MultiMSReader.h
+++ /dev/null
@@ -1,201 +0,0 @@
-//# MultiMSReader.h: DPPP step reading from multiple MSs
-//# Copyright (C) 2011
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: MultiMSReader.h 17800 2011-04-19 12:37:59Z diepen $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_MULTIMSREADER_H
-#define DPPP_MULTIMSREADER_H
-
-// @file
-// @brief DPPP step reading from multiple MSs
-
-#include <DPPP/MSReader.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/UVWCalculator.h>
-#include <DPPP/FlagCounter.h>
-#include <casacore/tables/Tables/TableIter.h>
-#include <casacore/tables/Tables/RefRows.h>
-#include <casacore/casa/Arrays/Slicer.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPInput step reading the data from a MeasurementSet.
- // At the beginning it finds out the shape of the data; i.e., the
- // number of correlations, channels, baselines, and time slots.
- // It requires the data to be regularly shaped.
- //
- // The object is constructed from the 'msin' keywords in the parset file.
- // Currently the following can be given:
- // <ul>
- // <li> msin: name of the MS
- // <li> msin.autoweight: calculate weights from autocorrelations? [no]
- // <li> msin.startchan: first channel to use [0]
- // <li> msin.nchan: number of channels to use [all]
- // <li> msin.useflag: use the existing flags? [yes]
- // <li> msin.datacolumn: the data column to use [DATA]
- // <li> msin.starttime: first time to use [first time in MS]
- // <li> msin.endtime: last time to use [last time in MS]
- // </ul>
- //
- // If a time slot is missing, it is inserted with flagged data set to zero.
- // Missing time slots can also be detected at the beginning or end of the
- // MS by giving the correct starttime and endtime.
- // The correct UVW coordinates are calculated for inserted time slots.
- //
- // The process function only reads the data and flags to avoid that
- // too much data is kept in memory.
- // Other columns (like WEIGHT, UVW) can be read when needed by using the
- // appropriate DPInput::fetch function.
- //
- // The data columns are handled in the following way:
- // <table>
- // <tr>
- // <td>TIME</td>
- // <td>The time slot center of the current data (in MJD seconds).
- // It is assumed that all data have the same interval, which is
- // used to find missing time slots.
- // </td>
- // </tr>
- // <tr>
- // <td>DATA</td>
- // <td>The visibility data as [ncorr,nchan,nbaseline]. Only the
- // part given by startchan and nchan is read. If a time slot is
- // inserted, all its data are zero.
- // </td>
- // </tr>
- // <tr>
- // <td>FLAG</td>
- // <td>The data flags as [ncorr,nchan,nbaseline] (True is bad).
- // They are read from the FLAG column. If a FLAG_ROW is set, all
- // flags for that baseline will be set. Also the flag of data
- // containing NaN or infinite numbers will be set.
- // All flags of an inserted time slot are set.
- // </td>
- // </tr>
- // <tr>
- // <td>WEIGHT</td>
- // <td>The data weights as [ncorr,nchan,nbaseline]. Column
- // WEIGHT_SPECTRUM is used if present and containing valid data,
- // otherwise column WEIGHT is used. The weights of an inserted
- // time slot are set to 0.
- // If autoweight is on, the autocorrelations are used to
- // calculate proper weights.
- // </td>
- // </tr>
- // <tr>
- // <td>UVW</td>
- // <td>The UVW coordinates in meters as [3,nbaseline].
- // They are calculated for a missing time slot.
- // </td>
- // </tr>
- // <tr>
- // <td>FULLRESFLAG</td>
- // <td>For each baseline the LOFAR_FULL_RES_FLAG column is stored as
- // a uChar array with shape [orignchan/8, ntimeavg]. The bits
- // represent the flags. They are converted to a Bool array with shape
- // [orignchan, ntimeavg, nbaseline].
- // If column LOFAR_FULL_RES_FLAG is not present, the flags are used
- // and it is assumed that no averaging was done yet (thus ntimeavg=1
- // and orignchan=nchan).
- // </td>
- // </tr>
- // </table>
-
- class MultiMSReader: public MSReader
- {
- public:
- // Construct the object for the given MS.
- // Parameters are obtained from the parset using the given prefix.
- MultiMSReader (const vector<string>& msNames,
- const ParameterSet&, const string& prefix);
-
- virtual ~MultiMSReader();
-
- // Process the next data chunk.
- // It returns false when at the end.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info (by initializing it).
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // If needed, show the flag counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Read the UVW at the given row numbers.
- virtual void getUVW (const casacore::RefRows& rowNrs,
- double time,
- DPBuffer& buf);
-
- // Read the weights at the given row numbers.
- virtual void getWeights (const casacore::RefRows& rowNrs,
- DPBuffer& buf);
-
- // Read the FullRes flags (LOFAR_FULL_RES_FLAG) at the given row numbers.
- // It returns a 3-dim array [norigchan, ntimeavg, nbaseline].
- // If undefined, false is returned.
- virtual bool getFullResFlags (const casacore::RefRows& rowNrs,
- DPBuffer& buf);
-
- // Tell if the visibility data are to be read.
- virtual void setReadVisData (bool readVisData);
-
- private:
- // Handle the info for all bands.
- void handleBands();
-
- // Sort the bands (MSs) inorder of frequency.
- void sortBands();
-
- // Fill the band info where some MSs are missing.
- void fillBands();
-
- //# Data members.
- bool itsOrderMS; //# sort multi MS in order of freq?
- int itsFirst; //# first valid MSReader (<0 = none)
- int itsNMissing; //# nr of missing MSs
- vector<string> itsMSNames;
- vector<MSReader*> itsReaders; //# same as itsSteps
- vector<DPStep::ShPtr> itsSteps; //# used for automatic destruction
- vector<DPBuffer> itsBuffers;
- uint itsFillNChan; //# nr of chans for missing MSs
- FlagCounter itsFlagCounter;
- bool itsRegularChannels; // Are resulting channels regularly spaced
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/OneApplyCal.h b/CEP/DP3/DPPP/include/DPPP/OneApplyCal.h
deleted file mode 100644
index 2170a7f8630fa818ce556b0978f38f2be7e461e5..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/OneApplyCal.h
+++ /dev/null
@@ -1,142 +0,0 @@
-//# OneApplyCal.h: DPPP step class to apply a calibration correction to the data
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: OneApplyCal.h 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_ONEAPPLYCAL_H
-#define DPPP_ONEAPPLYCAL_H
-
-// @file
-// @brief DPPP step class to apply a calibration correction to the data
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/H5Parm.h>
-#include <ParmDB/ParmFacade.h>
-#include <ParmDB/ParmSet.h>
-#include <ParmDB/Parm.h>
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <DPPP/FlagCounter.h>
-
-namespace LOFAR {
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class applying calibration parameters to the data.
- // It only applies one correction.
-
- class OneApplyCal: public DPStep
- {
- public:
- // Define the shared pointer for this type.
- typedef shared_ptr<OneApplyCal> ShPtr;
-
- enum CorrectType {GAIN, FULLJONES, TEC, CLOCK, ROTATIONANGLE, SCALARPHASE, PHASE,
- ROTATIONMEASURE, SCALARAMPLITUDE, AMPLITUDE};
-
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- OneApplyCal (DPInput*, const ParameterSet&, const std::string& prefix,
- const std::string& defaultPrefix,
- bool substep=false, std::string predictDirection="");
-
- virtual ~OneApplyCal();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- bool invert() {
- return itsInvert;
- }
-
- private:
- // Read parameters from the associated parmdb and store them in itsParms
- void updateParms (const double bufStartTime);
-
- // If needed, show the flag counts.
- virtual void showCounts (std::ostream&) const;
-
- void initDataArrays();
-
- // Check the number of polarizations in the parmdb or h5parm
- uint nPol(const std::string& parmName);
-
- // Replace values by NaN on places where weight is zero
- static void applyFlags(std::vector<double>& values,
- const std::vector<double>& weights);
-
- static std::string correctTypeToString(CorrectType);
- static CorrectType stringToCorrectType(const string&);
-
- //# Data members.
- DPInput* itsInput;
- DPBuffer itsBuffer;
- string itsName;
- string itsParmDBName;
- bool itsUseH5Parm;
- string itsSolSetName;
- boost::shared_ptr<BBS::ParmFacade> itsParmDB;
- H5Parm itsH5Parm;
- string itsSolTabName;
- H5Parm::SolTab itsSolTab;
- H5Parm::SolTab itsSolTab2; // in the case of full Jones, amp and phase table need to be open
- CorrectType itsCorrectType;
- bool itsInvert;
- uint itsTimeSlotsPerParmUpdate;
- double itsSigmaMMSE;
- bool itsUpdateWeights;
-
- uint itsCount; // number of steps
-
- // Expressions to search for in itsParmDB
- vector<casacore::String> itsParmExprs;
-
- // parameters, numparms, antennas, time x frequency
- casacore::Cube<casacore::DComplex> itsParms;
- uint itsTimeStep; // time step within current chunk
- uint itsNCorr;
- double itsTimeInterval;
- double itsLastTime; // last time of current chunk
- FlagCounter itsFlagCounter;
- bool itsUseAP; //# use ampl/phase or real/imag
- hsize_t itsDirection;
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Patch.h b/CEP/DP3/DPPP/include/DPPP/Patch.h
deleted file mode 100644
index 9aec497bf7e92c5466d23d4f920bb80a965c4151..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Patch.h
+++ /dev/null
@@ -1,141 +0,0 @@
-//# Patch.h: A set of sources for which direction dependent effects are assumed
-//# to be equal.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_PATCH_H
-#define DPPP_PATCH_H
-
-// \file
-// A set of sources for which direction dependent effects are assumed to be
-// equal.
-
-#include <DPPP/ModelComponent.h>
-#include <DPPP/Position.h>
-#include <Common/lofar_vector.h>
-#include <Common/lofar_string.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-class Patch
-{
-public:
- typedef shared_ptr<Patch> Ptr;
- typedef shared_ptr<const Patch> ConstPtr;
- typedef vector<ModelComponent::ConstPtr>::const_iterator const_iterator;
-
- template <typename T>
- Patch(const string &name, T first, T last);
-
- const string &name() const;
- const Position &position() const;
- double brightness() const;
- void setPosition (const Position&);
- void setBrightness (double);
-
- size_t nComponents() const;
- ModelComponent::ConstPtr component(size_t i) const;
-
- const_iterator begin() const;
- const_iterator end() const;
-
- // Compute the position as the average of the positions of the components.
- void computePosition();
-
-private:
-
- string itsName;
- Position itsPosition;
- double itsBrightness;
- vector<ModelComponent::ConstPtr> itsComponents;
-};
-
-
-// @}
-
-// -------------------------------------------------------------------------- //
-// - Implementation: Patch - //
-// -------------------------------------------------------------------------- //
-
-template <typename T>
-Patch::Patch(const string &name, T first, T last)
- : itsName(name),
- itsComponents(first, last)
-{
- computePosition();
-}
-
-inline const string &Patch::name() const
-{
- return itsName;
-}
-
-inline const Position &Patch::position() const
-{
- return itsPosition;
-}
-
-inline double Patch::brightness() const
-{
- return itsBrightness;
-}
-
-inline void Patch::setPosition (const Position& pos)
-{
- itsPosition = pos;
-}
-
-inline void Patch::setBrightness (double brightness)
-{
- itsBrightness = brightness;
-}
-
-inline size_t Patch::nComponents() const
-{
- return itsComponents.size();
-}
-
-inline ModelComponent::ConstPtr Patch::component(size_t i) const
-{
- return itsComponents[i];
-}
-
-inline Patch::const_iterator Patch::begin() const
-{
- return itsComponents.begin();
-}
-
-inline Patch::const_iterator Patch::end() const
-{
- return itsComponents.end();
-}
-
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/PhaseFitter.h b/CEP/DP3/DPPP/include/DPPP/PhaseFitter.h
deleted file mode 100644
index 93caa23c16b3e948654de3aed0f5f952027b16c3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/PhaseFitter.h
+++ /dev/null
@@ -1,316 +0,0 @@
-//# phasefitter.h: Class to perform phase fitting (TEC), allowing phase wraps
-//# Copyright (C) 2016
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: phasefitter.cc 21598 2012-07-16 08:07:34Z offringa $
-//#
-//# @author André Offringa
-
-/**
- * @file phasefitter.h Implements TEC model phase filter @ref PhaseFitter.
- * @author André Offringa
- * @date 2016-04-06
- */
-
-#ifndef PHASE_FITTER_H
-#define PHASE_FITTER_H
-
-#include <vector>
-#include <cmath>
-#include <cstring>
-
-/**
- * Phase fitter that can force phase solutions over frequency onto a TEC model.
- * To use:
- * - Construct and set the frequencies (with @ref FrequencyData() ) and if already possible
- * the weights (@ref WeightData()).
- * - Perform a calibration iteration.
- * - Set the phase values (@ref PhaseData()) and, if not yet done, the weights (@ref WeightData())
- * to the current phase solutions / weights of a single antenna.
- * - Call @ref FitDataToTEC1Model() or @ref FitDataToTEC2Model().
- * - Read the new phase values.
- * - When fitting multiple polarizations, the above steps should be done twice for each
- * diagonal value of the Jones matrix. Also repeat for all antennae.
- * - Continue the iteration with the new phase values.
- *
- * Methods with name containing TEC1 refer to a single-parameter TEC model (no delay), while
- * methods with TEC2 refer to dual-parameter TEC model (TEC + delay).
- */
-class PhaseFitter
-{
- public:
- PhaseFitter() :
- _phases(),
- _frequencies(),
- _weights(),
- _fittingAccuracy(1e-6)
- { }
-
- /**
- * Construct a phase fitter for the given number of channels.
- * Weights are initialized to unity. The fitting accuracy is
- * initialized to 1e-6.
- * @param channelCount number of channels.
- */
- PhaseFitter(size_t channelCount) :
- _phases(channelCount, 0.0),
- _frequencies(channelCount, 0.0),
- _weights(channelCount, 1.0),
- _fittingAccuracy(1e-6)
- { }
-
- /**
- * Change the number of channels to be fitted. All phase, weight and frequency data is discarded.
- * @param channelCount New number of channels
- */
- void SetChannelCount(size_t channelCount)
- {
- _phases.assign(channelCount, 0.0);
- _frequencies.assign(channelCount, 0.0);
- _weights.assign(channelCount, 1.0);
- }
-
- /**
- * Fits the given phase values to a TEC model and returns the parameters. This function is
- * robust even when phase wrapping occurs.
- * After this call, the @ref PhaseData() satisfy the TEC model.
- * The TEC model has two parameters and are fitted as described in
- * @ref FitTEC2ModelParameters().
- * @param alpha Found value for the alpha parameter.
- * @param beta Found value for the beta parameter.
- * @returns Cost of the found solution.
- */
- double FitDataToTEC2Model(double& alpha, double& beta);
-
- /**
- * Like @ref FitDataToTEC2Model(double&,double&), but without returning the parameters.
- */
- void FitDataToTEC2Model()
- {
- double a,b;
- FitDataToTEC2Model(a, b);
- }
-
- /**
- * Fits the given phase values to a TEC model using prior estimates of the
- * model parameters. This method is similar to @ref FitDataToTEC2Model(),
- * except that it will use the provided initial values of alpha and
- * beta to speed up the solution. When the provided initial values
- * are not accurate, the fit might not be accurate.
- *
- * @todo No fast method has been implemented -- instead it will perform a full parameter search.
- * @param alpha Estimate of alpha parameter on input, found value on output.
- * @param beta Estimate of beta parameter on input, found value on output.
- */
- double FitDataToTEC2ModelWithInitialValues(double& alpha, double& beta)
- {
- return FitDataToTEC2Model(alpha, beta);
- }
-
- /**
- * Fit the data and get the best fitting parameters. The model
- * used is f(nu) = alpha/nu + beta, with possible 2pi wrappings in the
- * data.
- * The phase data is not changed.
- * The alpha parameter is linearly related to the TEC. The beta parameter
- * is a constant phase offset, given in radians.
- * The fitting algorithm uses a combination of brute force searching and
- * binary-like searching (ternary search).
- * @param alpha Will be set to the fitted value for the alpha parameter
- * (value on input is not used).
- * @param beta Will be set to the fitted value for the beta parameter
- * (value on input is not used).
- */
- void FitTEC2ModelParameters(double& alpha, double& beta) const;
-
- /**
- * Get a pointer to the array of phase values. This array should be filled with the
- * phases of solutions before calling one of the fit methods. @ref FitDataToTEC1Model()
- * and ~TEC2~ sets this array to the fitted phases.
- * Normally, these values should be set to std::arg(z) or atan2(z.imag(), z.real()), where
- * z is a complex solution for one polarizations. All phases should correspond to the same
- * polarizations, i.e., different polarizations (xx/yy/ll/rr, etc.) should be independently
- * fitted.
- * @returns Array of @ref Size() doubles with the phases.
- */
- double* PhaseData() { return _phases.data(); }
-
- /**
- * Get a constant pointer to the array of values.
- * @returns Constant array of @ref Size() doubles with the phases.
- */
- const double* PhaseData() const { return _phases.data(); }
-
- /**
- * Get a pointer to the array of frequency values. This array should be set to the
- * frequency values of the channels, such that FrequencyData()[ch] is the frequency
- * corresponding to the phase value PhaseData()[ch]. The fitter will not change this
- * array.
- * @returns Array of @ref Size() doubles with the frequencies in Hz.
- */
- double* FrequencyData() { return _frequencies.data(); }
-
- /**
- * Constant frequency data.
- * @returns Constant array of @ref Size() doubles with the frequencies in Hz.
- */
- const double* FrequencyData() const { return _frequencies.data(); }
-
- /**
- * This array should be filled with the weights
- * of the channel phase solutions. If the solver supports weights during solving, this value should
- * be set to the sum of weights of all visibilities that are used for the solution of
- * this channel. If the solver does not support weights, it should be set to the number
- * of unflagged visibilities used by the solver to generate the corresponding phase. Another
- * way of saying this, is that the weights should be set to the esimated inverse variance of the phase
- * solutions.
- *
- * The use of weights will make sure that noisy channels do not bias the result. Weighting is
- * for example helpful to avoid that the few remaining samples in a badly RFI contaminated
- * channels cause the fit to be inaccurate.
- *
- * While the weights could be different for each antenna solution, generally the weight of a channel
- * is constant over the antennas. The latter implies that the weights can be set once before
- * the solution starts, and only the @ref PhaseData() need to be changed within solution
- * iterations.
- *
- * The weights are initially set to one.
- *
- * @returns Array of @ref Size() doubles with the weights.
- */
- double* WeightData() {
- return _weights.data();
- }
-
- /**
- * Constant array of weights, as described above.
- * @returns Constant array of @ref Size() doubles with weights.
- */
- const double* WeightData() const { return _weights.data(); }
-
- /**
- * Number of channels used for the fitter.
- * @returns Number of channels.
- */
- size_t Size() const { return _phases.size(); }
-
- /**
- * Get the fitting accuracy. The fitter will stop once this accuracy is approximately reached.
- * The default value is 1e-6.
- * @returns Fitting accuracy.
- */
- double FittingAccuracy() const { return _fittingAccuracy; }
-
- /**
- * Change the fitting accuracy. See @ref FittingAccuracy().
- * @param newAccuracy New accuracy.
- */
- void SetFittingAccuracy(double newAccuracy) { _fittingAccuracy = newAccuracy; }
-
- /**
- * Evaluate the cost function for given TEC model parameters. The higher the
- * cost, the worser the data fit the given parameters.
- * @param alpha TEC parameter
- * @param beta Phase offset parameter
- * @returns sum of | nu_i / alpha + beta - theta_i |, and each sum term is
- * phase unwrapped.
- */
- double TEC2ModelCost(double alpha, double beta) const;
-
- /**
- * Like @ref TEC2ModelFunc(), but 2-pi wrapped.
- * @param nu Frequency in Hz
- * @param alpha TEC parameter (in undefined units)
- * @param beta Phase offset parameter (in radians)
- * @returns | nu_i / alpha + beta | % 2pi
- */
- static double TEC2ModelFunc(double nu, double alpha, double beta)
- {
- return alpha / nu + beta;
- }
-
- double FitDataToTEC1Model(double& alpha);
-
- /**
- * Like @ref FitDataToTEC1Model(double&,double&), but without returning the parameters.
- */
- void FitDataToTEC1Model()
- {
- double a;
- FitDataToTEC1Model(a);
- }
-
- /**
- * Like @ref TEC2ModelFunc(), but 2-pi wrapped.
- * @param nu Frequency in Hz
- * @param alpha TEC parameter (in undefined units)
- * @param beta Phase offset parameter (in radians)
- * @returns | nu_i / alpha + beta | % 2pi
- */
- static double TEC2ModelFuncWrapped(double nu, double alpha, double beta)
- {
- return fmod(alpha / nu + beta, 2.0*M_PI);
- }
-
- double FitDataToTEC1ModelWithInitialValues(double& alpha)
- {
- return FitDataToTEC1Model(alpha);
- }
-
- void FitTEC1ModelParameters(double& alpha) const;
-
- /**
- * Evaluate the cost function for given TEC model parameter. The higher the
- * cost, the worser the data fit the given parameters.
- * @param alpha TEC parameter
- * @returns sum of | alpha / nu_i - theta_i |, and each sum term is
- * phase unwrapped.
- */
- double TEC1ModelCost(double alpha) const;
-
- /**
- * Like @ref TEC1ModelFunc(), but 2-pi wrapped.
- * @param nu Frequency in Hz
- * @param alpha TEC parameter (in undefined units)
- * @param beta Phase offset parameter (in radians)
- * @returns | alpha / nu_i + beta | % 2pi
- */
- static double TEC1ModelFuncWrapped(double nu, double alpha)
- {
- return fmod(alpha / nu, 2.0*M_PI);
- }
-
- static double AlphaToTEC(double alpha)
- {
- return alpha / -8.44797245e9;
- }
- private:
- std::vector<double> _phases, _frequencies, _weights;
- double _fittingAccuracy;
-
- double fitTEC2ModelBeta(double alpha, double betaEstimate) const;
- void bruteForceSearchTEC2Model(double& lowerAlpha, double& upperAlpha, double& beta) const;
- double ternarySearchTEC2ModelAlpha(double startAlpha, double endAlpha, double& beta) const;
- void fillDataWithTEC2Model(double alpha, double beta);
- void fillDataWithTEC1Model(double alpha);
-
- void bruteForceSearchTEC1Model(double& lowerAlpha, double& upperAlpha) const;
- double ternarySearchTEC1ModelAlpha(double startAlpha, double endAlpha) const;
-};
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/PhaseShift.h b/CEP/DP3/DPPP/include/DPPP/PhaseShift.h
deleted file mode 100644
index 94c3f9e9bec0067a0269e460d3282b71c8eb5168..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/PhaseShift.h
+++ /dev/null
@@ -1,114 +0,0 @@
-//# PhaseShift.h: DPPP step class to shift the data to another phase center
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_PHASESHIFT_H
-#define DPPP_PHASESHIFT_H
-
-// @file
-// @brief DPPP step class to shift the data to another phase center
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <casacore/casa/Arrays/Matrix.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to shift the data and UVW coordinates
- // to another phase center. If no phase center is given, a shift is
- // done back to the original phase center.
- //
- // The code is based on the script phaseshift.py by Bas vd Tol.
-
- class PhaseShift: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- // This is the standard constructor where the phasecenter must be given.
- PhaseShift (DPInput*, const ParameterSet&, const string& prefix);
-
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- // This is a constructor for Demixer where the phasecenter has the
- // given default value.
- PhaseShift (DPInput*, const ParameterSet&, const string& prefix,
- const vector<string>& defVal);
-
- virtual ~PhaseShift();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Fill the transformation matrix for given ra/dec.
- static void fillTransMatrix (casacore::Matrix<double>& mat,
- double ra, double dec);
-
- // Get the phasors resulting from the last process step.
- // This is used in the Demixer.
- const casacore::Matrix<casacore::DComplex>& getPhasors() const
- { return itsPhasors; }
-
- // Get the phase center.
- const vector<string>& getPhaseCenter() const
- { return itsCenter; }
-
- private:
- // Interpret the phase center specification.
- // Currently only J2000 RA and DEC can be given.
- casacore::MDirection handleCenter();
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuf;
- vector<string> itsCenter;
- vector<double> itsFreqC; //# freq/C
- casacore::Matrix<double> itsMat1; //# TT in phasehift.py
- double itsXYZ[3]; //# numpy.dot((w-w1).T, T)
- casacore::Matrix<casacore::DComplex> itsPhasors; //# phase factor per chan,bl
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/PointSource.h b/CEP/DP3/DPPP/include/DPPP/PointSource.h
deleted file mode 100644
index 2d3935b91f163bbb4844090e6f5c11cbd6d0ad71..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/PointSource.h
+++ /dev/null
@@ -1,105 +0,0 @@
-//# PointSource.h: Point source model component with optional spectral index and
-//# rotation measure.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_POINTSOURCE_H
-#define DPPP_POINTSOURCE_H
-
-// \file
-// Point source model component with optional spectral index and rotation
-// measure.
-
-#include <DPPP/ModelComponent.h>
-#include <DPPP/Position.h>
-#include <DPPP/Stokes.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-class PointSource: public ModelComponent
-{
-public:
- typedef shared_ptr<PointSource> Ptr;
- typedef shared_ptr<const PointSource> ConstPtr;
-
- PointSource(const Position &position);
- PointSource(const Position &position, const Stokes &stokes);
-
- virtual const Position &position() const;
- void setPosition(const Position &position);
-
- void setStokes(const Stokes &stokes);
-
- template <typename T>
- void setSpectralTerms(double refFreq, bool isLogarithmic, T first, T last);
-
- void setRotationMeasure(double fraction, double angle, double rm);
-
- Stokes stokes(double freq) const;
-
- virtual void accept(ModelComponentVisitor &visitor) const;
-
-private:
- bool hasSpectralTerms() const;
- bool hasRotationMeasure() const;
-
- Position itsPosition;
- Stokes itsStokes;
- double itsRefFreq;
- vector<double> itsSpectralTerms;
- double itsPolarizedFraction;
- double itsPolarizationAngle;
- double itsRotationMeasure;
- bool itsHasRotationMeasure;
- bool itsHasLogarithmicSI;
-};
-
-// @}
-
-// -------------------------------------------------------------------------- //
-// - Implementation: PointSource - //
-// -------------------------------------------------------------------------- //
-
-template <typename T>
-void PointSource::setSpectralTerms(double refFreq, bool isLogarithmic, T first, T last)
-{
- itsRefFreq = refFreq;
- itsHasLogarithmicSI = isLogarithmic;
- itsSpectralTerms.clear();
- itsSpectralTerms.insert(itsSpectralTerms.begin(), first, last);
-}
-
-inline const Position &PointSource::position() const
-{
- return itsPosition;
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Position.h b/CEP/DP3/DPPP/include/DPPP/Position.h
deleted file mode 100644
index e89381d9949c1fba81a793b988f6dd9556df9e0c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Position.h
+++ /dev/null
@@ -1,71 +0,0 @@
-//# Position.h: A position on the celestial sphere.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_POSITION_H
-#define DPPP_POSITION_H
-
-// \file
-// A position on the celestial sphere.
-
-#include <cstring>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-class Position
-{
-public:
- Position();
- Position(double alpha, double delta);
-
- const double &operator[](size_t i) const;
- double &operator[](size_t i);
-
-private:
- double itsPosition[2];
-};
-
-// @}
-
-// -------------------------------------------------------------------------- //
-// - Implementation: Position - //
-// -------------------------------------------------------------------------- //
-
-inline const double &Position::operator[](size_t i) const
-{
- return itsPosition[i];
-}
-
-inline double &Position::operator[](size_t i)
-{
- return itsPosition[i];
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/PreFlagger.h b/CEP/DP3/DPPP/include/DPPP/PreFlagger.h
deleted file mode 100644
index 9640841c933a452c1a47ea57ff129b3a0c4747b2..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/PreFlagger.h
+++ /dev/null
@@ -1,296 +0,0 @@
-//# PreFlagger.h: DPPP step class to flag data on channel, baseline, or time
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_PREFLAGGER_H
-#define DPPP_PREFLAGGER_H
-
-// @file
-// @brief DPPP step class to flag data on channel, baseline, or time
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/BaselineSelection.h>
-#include <Common/lofar_vector.h>
-#include <casacore/measures/Measures/MDirection.h>
-
-namespace LOFAR {
- class ParameterSet;
- class ParameterValue;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class flagging data points based on data
- // selections given in the parset file.
- // The following selections can be given:
- // <ul>
- // <li> minimum and/or maximum UV distance (projected baseline length)
- // <li> minimum and/or maximum baseline length (intrinsic)
- // <li> autocorrelations or crosscorrelations
- // <li> baselines using names for antenna 1 and 2
- // <li> antennae using antenna names
- // <li> minimum and/or maximum amplitude/phase/real/imag per correlation
- // <li> channel numbers
- // <li> frequency ranges
- // <li> sequence nr or time ranges
- // <li> LST
- // <li> azimuth/elevation
- // </ul>
- // The antenna names can contain shell-style wildcards (* ? [] {}).
- //
- // All selections are ANDed, thus only the data points matching all
- // selections are flagged. It is however, possible to specify a logical
- // expression of selections by means of the internal PSet class.
- // A PSet objects contains a set of ANDed selections. The PSets can
- // be logically combined by the user using the normal logical operators.
-
- class PreFlagger: public DPStep
- {
- // Make this Test class a friend, so it can access private code.
- friend class TestPSet;
-
- public:
- enum Mode {SetFlag, ClearFlag, SetComp, ClearComp};
-
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- PreFlagger (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~PreFlagger();
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the average info.
- // It is used to adjust the parms if needed.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the flag counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- // This internal class represents a single set of ANDed selections.
- // PSets can be logically combined by the PreFlagger class.
- class PSet
- {
- // Make this Test class a friend, so it can access private code.
- friend class TestPSet;
-
- public:
- // Define the operators in pset expressions.
- // They have to have negative values in order of precedence.
- enum Oper {
- OpParen = -1,
- OpOr = -2,
- OpAnd = -3,
- OpNot = -4
- };
-
- // Define the shared pointer for this type.
- typedef shared_ptr<PSet> ShPtr;
-
- // Default constructor (for test purposes).
- PSet()
- {}
-
- // Construct from the parset parameters.
- PSet (DPInput*, const ParameterSet& parset, const string& prefix);
-
- // Set and return the flags.
- casacore::Cube<bool>* process (const DPBuffer&, DPBuffer&, uint timeSlot,
- const casacore::Block<bool>& matchBL,
- NSTimer& timer);
-
- // Update the general info.
- // It is used to adjust the parms if needed.
- void updateInfo (const DPInfo&);
-
- // Show the pset parameters.
- void show (std::ostream&, bool showName) const;
-
- private:
- // Test if the time matches the time ranges.
- bool matchTime (double time, uint timeSlot) const;
-
- // Test if the value matches one of the ranges in the vector.
- bool matchRange (double v, const vector<double>& ranges) const;
-
- // Clear itsMatchBL for mismatching baselines.
- // If returns false if no matches were found.
- bool flagBL();
-
- // Clear itsMatchBL for baselines with mismatching UV distances.
- // If returns false if no matches were found.
- bool flagUV (const casacore::Matrix<double>& uvw);
-
- // Clear itsMatchBL for baselines with mismatching AzEl.
- // If returns false if no matches were found.
- bool flagAzEl (double time);
-
- // Test if azimuth or elevation of given antenna mismatches.
- // If so, clear itsMatchBL for all baselines containing the antenna.
- void testAzEl (casacore::MDirection::Convert& converter,
- uint blnr, int ant,
- const int* ant1, const int* ant2);
-
- // Set the flags based on amplitude threshold per correlation.
- void flagAmpl (const casacore::Cube<float>& amplitudes);
-
- // Set the flags based on phase threshold per correlation.
- void flagPhase (const casacore::Cube<casacore::Complex>& data);
-
- // Set the flags based on real/imaginary threshold per correlation.
- void flagReal (const casacore::Cube<casacore::Complex>& data);
- void flagImag (const casacore::Cube<casacore::Complex>& data);
-
- // Flag the channels given in itsChannels.
- void flagChannels();
-
- // Convert a string of (date)time ranges to double. Each range
- // must be given with .. or +-.
- // <tt>asTime=true</tt> means that the strings should contain times,
- // otherwise date/times.
- vector<double> fillTimes (const vector<string>& str, bool asTime,
- bool canEndBeforeStart);
-
- // Read the string as time or date/time and convert to seconds.
- // usepm indicates if the value is a plusminus value. If so, the
- // value must be a positive time.
- double getSeconds (const string& str, bool asTime, bool usepm);
-
- // Fill the baseline matrix; set true for baselines to flag.
- void fillBLMatrix();
-
- // Fill itsChannels if channel/freq selection is done.
- void fillChannels (const DPInfo&);
-
- // Return a vector with a value per correlation.
- // If no parm value given use the default.
- // If the parm value is a vector, use the given values. Use the
- // default for non-given correlations.
- // If the parm value is a single value, use it for all correlations.
- // <br>doFlag is set if values are given.
- vector<float> fillValuePerCorr (const ParameterValue& value,
- float defVal, bool& doFlag);
-
- // Handle the frequency ranges given and determine which channels
- // have to be flagged.
- casacore::Vector<bool> handleFreqRanges
- (const casacore::Vector<double>& chanFreqs);
-
- // Get the value and possible unit.
- // If no unit is given, the argument is left untouched.
- void getValue (const string& str, double& value, casacore::String& unit);
-
- // Get the frequency in Hz using the value and unit.
- double getFreqHz (double value, const casacore::String& unit);
-
- // Convert a PSet expression to Reversed Polish Notation in itsRpn.
- // It returns the names of all PSets.
- vector<string> exprToRpn (const string& expr);
-
- //# Data members of PreFlagger::PSet.
- DPInput* itsInput;
- const DPInfo* itsInfo;
- string itsName;
- string itsStrExpr;
- bool itsFlagOnTimeOnly; //# true = only flag on time info
- bool itsFlagOnTime; //# true = do time based flagging
- bool itsFlagOnUV; //# true = do uv distance based flagging
- bool itsFlagOnBL; //# true = do ant/bl based flagging
- bool itsFlagOnAmpl; //# true = do amplitude based flagging
- bool itsFlagOnPhase;//# true = do phase based flagging
- bool itsFlagOnReal; //# true = do real based flagging
- bool itsFlagOnImag; //# true = do imag based flagging
- bool itsFlagOnAzEl; //# true = do Az/El based flagging
- BaselineSelection itsSelBL;
- double itsMinUV; //# minimum UV distance; <0 means ignore
- double itsMaxUV; //# maximum UV distance; <0 means ignore
- casacore::Matrix<bool> itsFlagBL; //# true = flag baseline [i,j]
- vector<double> itsAzimuth; //# azimuth ranges to be flagged
- vector<double> itsElevation;//# elevation ranges to be flagged
- vector<double> itsTimes; //# time of day ranges to be flagged
- vector<double> itsLST; //# sidereal time ranges to be flagged
- vector<double> itsATimes; //# absolute time ranges to be flagged
- vector<double> itsRTimes; //# relative time ranges to be flagged
- vector<uint> itsTimeSlot; //# time slots to be flagged
- vector<float> itsAmplMin; //# minimum amplitude for each corr
- vector<float> itsAmplMax; //# maximum amplitude for each corr
- vector<float> itsPhaseMin; //# minimum phase for each corr
- vector<float> itsPhaseMax; //# maximum phase for each corr
- vector<float> itsRealMin; //# minimum real for each corr
- vector<float> itsRealMax; //# maximum real for each corr
- vector<float> itsImagMin; //# minimum imaginary for each corr
- vector<float> itsImagMax; //# maximum imaginary for each corr
- vector<uint> itsChannels; //# channels to be flagged.
- vector<string> itsStrChan; //# channel ranges to be flagged.
- vector<string> itsStrFreq; //# frequency ranges to be flagged
- vector<string> itsStrTime; //# time ranges to be flagged
- vector<string> itsStrLST; //# LST ranges to be flagged
- vector<string> itsStrATime; //# absolute time ranges to be flagged
- vector<string> itsStrRTime; //# relative time ranges to be flagged
- vector<string> itsStrAzim; //# azimuth ranges to be flagged
- vector<string> itsStrElev; //# elevation ranges to be flagged
- vector<int> itsRpn; //# PSet expression in RPN form
- vector<PSet::ShPtr> itsPSets; //# PSets used in itsRpn
- casacore::Matrix<bool> itsChanFlags; //# flags for channels to be flagged
- casacore::Cube<bool> itsFlags;
- casacore::Block<bool> itsMatchBL; //# true = baseline in buffer matches
- };
-
- // Set the flags in outPtr where inPtr matches mode.
- void setFlags (const bool* inPtr, bool* outPtr,
- uint nrcorr, uint nrchan, uint nrbl, bool mode);
-
- // Clear the flags in outPtr where inPtr matches mode.
- // If the corresponding data point of a flag is invalid
- // (non-finite or zero), it is always flagged.
- void clearFlags (const bool* inPtr, bool* outPtr,
- uint nrcorr, uint nrchan, uint nrbl, bool mode,
- const DPBuffer& buf);
-
- //# Data members of PreFlagger.
- string itsName;
- DPInput* itsInput;
- DPBuffer itsBuffer;
- Mode itsMode;
- NSTimer itsTimer;
- PSet itsPSet;
- uint itsCount;
- FlagCounter itsFlagCounter;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Predict.h b/CEP/DP3/DPPP/include/DPPP/Predict.h
deleted file mode 100644
index d4899a387407543fda225526b0b9e1971764b1ca..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Predict.h
+++ /dev/null
@@ -1,162 +0,0 @@
-//# Predict.h: DPPP step class to predict visibilities from a source model
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_PREDICT_H
-#define DPPP_PREDICT_H
-
-// @file
-// @brief DPPP step class to predict visibilities from a source model
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/Patch.h>
-#include <DPPP/SourceDBUtil.h>
-#include <DPPP/ApplyBeam.h>
-#include <DPPP/ModelComponent.h>
-#include <StationResponse/Station.h>
-#include <StationResponse/Types.h>
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Quanta/MVEpoch.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <utility>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to predict visibilities with optionally beam
-
- typedef std::pair<size_t, size_t> Baseline;
- typedef std::pair<ModelComponent::ConstPtr, Patch::ConstPtr> Source;
-
- class Predict: public DPStep
- {
- public:
- typedef shared_ptr<Predict> ShPtr;
-
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- Predict (DPInput*, const ParameterSet&, const string& prefix);
-
- // Constructor with explicit sourcelist
- Predict (DPInput*, const ParameterSet&, const string& prefix,
- const vector<string>& sourcePatterns);
-
- // The actual constructor
- void init (DPInput*, const ParameterSet&, const string& prefix,
- const vector<string>& sourcePatterns);
-
- // Set the applycal substep
- void setApplyCal(DPInput*, const ParameterSet&, const string& prefix);
-
- // Set the operation type
- void setOperation(const std::string& type);
-
- Predict();
-
- virtual ~Predict();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Prepare the sources
- void setSources(const vector<string>& sourcePatterns);
-
- // Return the direction of the first patch
- std::pair<double, double> getFirstDirection() const;
-
- private:
- StationResponse::vector3r_t dir2Itrf (const casacore::MDirection& dir,
- casacore::MDirection::Convert& measConverter);
- void addBeamToData (Patch::ConstPtr patch, double time,
- const StationResponse::vector3r_t& refdir,
- const StationResponse::vector3r_t& tiledir,
- uint thread, uint nSamples, dcomplex* data0);
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuffer;
- string itsSourceDBName;
- string itsOperation;
- bool itsApplyBeam;
- bool itsStokesIOnly;
- bool itsUseChannelFreq;
- bool itsOneBeamPerPatch;
- Position itsPhaseRef;
-
- bool itsDoApplyCal;
- ApplyCal itsApplyCalStep;
- DPBuffer itsTempBuffer;
- ResultStep* itsResultStep; // For catching results from ApplyCal
-
- uint itsDebugLevel;
-
- vector<Baseline> itsBaselines;
-
- // Vector containing info on converting baseline uvw to station uvw
- vector<int> itsUVWSplitIndex;
-
- // UVW coordinates per station (3 coordinates per station)
- casacore::Matrix<double> itsUVW;
-
- // The info needed to calculate the station beams.
- vector<vector<StationResponse::Station::Ptr> > itsAntBeamInfo;
- vector<casacore::MeasFrame> itsMeasFrames;
- vector<casacore::MDirection::Convert> itsMeasConverters;
- vector<vector<StationResponse::matrix22c_t> > itsBeamValues;
- ApplyBeam::BeamMode itsBeamMode;
-
- std::string itsDirectionsStr; // Definition of patches, to pass to applycal
- vector<Patch::ConstPtr> itsPatchList;
- vector<Source> itsSourceList;
-
- vector<casacore::Cube<dcomplex> > itsModelVis; // one for every thread
- vector<casacore::Cube<dcomplex> > itsModelVisPatch;
-
- NSTimer itsTimer;
- NSTimer itsTimerPredict;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/ProgressMeter.h b/CEP/DP3/DPPP/include/DPPP/ProgressMeter.h
deleted file mode 100644
index 8c3912a1a83d1700ce19cea947cf84707a9df571..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/ProgressMeter.h
+++ /dev/null
@@ -1,128 +0,0 @@
-//# ProgressMeter.h: Visual indication of a tasks progress.
-//# Copyright (C) 1997,2000
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef LOFAR_COMMON_PROGRESSMETER_H
-#define LOFAR_COMMON_PROGRESSMETER_H
-
-//# Includes
-#include <Common/lofar_string.h>
-
-namespace LOFAR {
-
-// @ingroup NDPPP
-
-// This class shows the progress of a task.
-// It is copied from casacore.
-//
-// It shows the progress on stdout using a line with dots and percentages.
-//
-// It is possible to attach the progressmeter to function in, say, a GUI
-// that can show the progress in a more visual way.
-//
-// The progress meter will usually be removed from the screen once the maximum
-// value is set, so you should not reuse the ProgressMeter after that has
-// happened. It is harmless, but it will not result in any visual feedback for
-// the user.
-//
-// While the "min" is usually less than "max", if in fact it is greater than
-// "max" the progress meter will count down correctly.
-//
-// For example:
-// <srcblock>
-// void calculate(uint n) {
-// int skip = n / 200;
-// ProgressMeter meter(0, n, "Title", "Subtitle", "", "", true, skip);
-// for (uint i=0; i<n; i++) {
-// ... calculate ...
-// meter.update(i);
-// }
-// }
-// </srcblock>
-
-class ProgressMeter
-{
-public:
- // Makes a null progress meter, i.e. no updates to the screen are
- // generated.
- ProgressMeter();
-
- // Create a progress meter with the given min and max values and labels.
- // if <tt>estimateTime=true</tt>, an estimate of the
- // time remaining will be made for the user. This estimate assumes that
- // the remaining portion will compute at the same rate as the portion
- // completed so far, so the time should not be estimated for processes
- // which are non-linear.
- //
- // Any labels which are set to the empty string will have sensible defaults
- // supplied. In particular, <tt>minlabel</tt> and <tt>maxlabel</tt>
- // will be set to the display the minimum and maximum values.
- //
- // Normally the progress bar will be updated with every call to
- // <tt>update()</tt>. If however you will be sending many events
- // then you might want to update the GUI every <tt>updateEvery</tt>'th
- // event for efficiency. Generally there's no point updating more than
- // a couple of hundred times since the eye can't distinguish differences
- // in the progress bar position at that level. If updateEvery is <=0, it
- // is set to 1 for you.
- ProgressMeter(double min, double max,
- const string& title, const string& subtitle,
- const string& minlabel, const string& maxlabel,
- bool estimateTime=true, int updateEvery=1);
-
- // The destruction of the meter will cause an update to be sent with the
- // maximum value. This will usually cause the GUI window to be removed
- // from the screen. Thus the progress meter should generally live as long
- // as the calculation it is tracking.
- ~ProgressMeter();
-
- void update(double value, bool force=false);
-
- // Get the min and max values of the progress meter.
- // <group>
- double min() const
- { return min_p; }
- double max() const
- { return max_p; }
- // </group>
-
- friend class ObjectController;
-private:
- int id_p;
- double min_p, max_p;
- int update_every_p, update_count_p;
-
- // These are set by ObjectController for executables that have the tasking
- // system in them, otherwise they are null and this class just does no-ops.
- static int (*creation_function_p)(double, double,
- const string&, const string&,
- const string&, const string&,
- bool);
- static void (*update_function_p)(int, double);
-
- // Undefined and inaccessible
- ProgressMeter(const ProgressMeter&);
- ProgressMeter& operator=(const ProgressMeter&);
-};
-
-
-} //# NAMESPACE LOFAR END
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/ScaleData.h b/CEP/DP3/DPPP/include/DPPP/ScaleData.h
deleted file mode 100644
index 0e5504ba58709be29b81ac2045db5e4628bdc261..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/ScaleData.h
+++ /dev/null
@@ -1,100 +0,0 @@
-//# ScaleData.h: DPPP step class for freq-dependent scaling of the data
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: ScaleData.h 23223 2012-12-07 14:09:42Z schoenmakers $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_SCALEDATA_H
-#define DPPP_SCALEDATA_H
-
-// @file
-// @brief DPPP step class for freq-dependent scaling of the data
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <casacore/casa/Arrays/Cube.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class scaling the data using a polynomial
- // in frequency (in MHz) for LBA and HBA. The coefficients can be given
- // as ParSet parameters having a default determined by Adam Deller.
- //
- // The polynomial coefficients can depend on station by giving them
- // per station name regular expression. The default coefficients are
- // used for the station not matching any regular expression.
- //
- // The data are multiplied with a factor sqrt(scale[ant1] * scale[ant2]).
- // An extra scale factor can be applied to compensate for the different
- // number of dipoles or tiles or for missing ones. By default that
- // extra scale factor is only applied to stations using the default
- // coefficients, because it is assumed that coefficients are scaled well
- // when specifying them explicitly for stations.
-
- class ScaleData: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- ScaleData (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~ScaleData();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- // Fill the scale factors for stations having different nr of tiles.
- void fillSizeScaleFactors (uint nNominal, vector<double>& fact);
-
- //# Data members.
- string itsName;
- bool itsScaleSizeGiven;
- bool itsScaleSize;
- vector<string> itsStationExp; // station regex strings
- vector<string> itsCoeffStr; // coeff per station regex
- vector<vector<double> > itsStationFactors; // scale factor per station,freq
- casacore::Cube<double> itsFactors; // scale factor per baseline,freq,pol
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Simulate.h b/CEP/DP3/DPPP/include/DPPP/Simulate.h
deleted file mode 100644
index 5f40d29054b3a57295bf3e7e3938eae37a5afc6b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Simulate.h
+++ /dev/null
@@ -1,134 +0,0 @@
-//# Simulate.h: Simulate visibilities for a patch of sources.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_SIMULATE_H
-#define DPPP_SIMULATE_H
-
-// \file
-// Simulate visibilities for a patch of sources.
-
-#include <DPPP/Baseline.h>
-#include <DPPP/Cursor.h>
-#include <DPPP/Patch.h>
-#include <DPPP/Position.h>
-#include <Common/lofar_complex.h>
-
-//# Forward Declaration.
-namespace casacore
-{
- template<typename T> class Vector;
- template<typename T> class Matrix;
-}
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
- // \addtogroup NDPPP
- // @{
-
- // Setup the splitting of the baseline UVWs into station UVWs. It returns
- // the indices of the baselines needed to split the baseline UVWs into
- // station UVWs. They are in such an order that the UVW of a station is known
- // before used in another baseline to derive the UVW of the other station.
- // It can handle cases where baselines occur in disjoint station groups
- // like 0-1, 0-2, 1-2 and 3-4, 4-5, 5-6.
- // Note that the first station of a group gets UVW=0. All other station UVWs
- // are relative to it using the baseline UVWs.
- // Also note that nr of groups can be derived from the size of the returned
- // vector (because it contains no entry for the first antenna in a group).
- vector<int> nsetupSplitUVW (uint nant, const casacore::Vector<int>& ant1,
- const casacore::Vector<int>& ant2);
-
- // Do the actual splitting of baseline UVWs into station UVWs using
- // the index vector generated by setupSplitUVW.
- void nsplitUVW (const vector<int>& blindex,
- const vector<Baseline>& baselines,
- const casacore::Matrix<double>& uvwbl,
- casacore::Matrix<double>& uvwant);
-
-// Split baseline UVW coordinates into station UVW coordinates (by assuming the
-// station with index 0 has UVW coordinates of (0, 0, 0)).
-//
-// \param[in] nStation
-// The number of stations.
-// \param[in] nBaseline
-// The number of baselines.
-// \param[in] baselines
-// A cursor for a 1-D buffer of baselines of shape (\p nBaseline).
-// \param[in] uvw
-// A cursor for a 2-D buffer of UVW coordinates of shape (\p nBaseline, 3).
-// \param[in] split
-// A cursor for a 2-D buffer of station UVW coordinates of shape
-// (\p nStation, 3).
-void splitUVW(size_t nStation, size_t nBaseline,
- const_cursor<Baseline> baselines, const_cursor<double> uvw,
- cursor<double> split);
-
-// Transform UVW coordinates from phase reference position \p from to phase
-// reference position \p to. The transformation is performed in place.
-//
-// \param[in] from
-// Current phase reference position for the UVW coordinates.
-// \param[in] to
-// New phase reference position for the UVW coordinates.
-// \param[in] nUVW
-// The number of UVW coordinates to transform.
-// \param[in] uvw
-// A 2-D buffer of UVW coordinates of shape (\p UVW, 3).
-void rotateUVW(const Position &from, const Position &to, size_t nUVW,
- double *uvw);
-
-// Simulate visibilities for a patch of sources. The computed visibilities are
-// added to \p vis.
-//
-// \param[in] reference
-// Phase reference position.
-// \param[in] patch
-// Patch of sources to simulate visibilities for.
-// \param[in] nStation
-// The number of stations.
-// \param[in] nBaseline
-// The number of baselines.
-// \param[in] nChannel
-// The number of frequency channels.
-// \param[in] baselines
-// A cursor for a 1-D buffer of baselines of shape (\p nBaseline).
-// \param[in] freq
-// A cursor for a 1-D buffer of channel frequencies of shape (\p nChannel).
-// \param[in] uvw
-// A cursor for a 2-D buffer of station UVW coordinates of shape
-// (\p nStation, 3).
-// \param[in] buffer
-// A cursor for a 3-D buffer of shape (\p nBaseline, \p nChannel, 4) into which
-// the simulated visibilities will be written.
-void simulate(const Position &reference, const Patch::ConstPtr &patch,
- size_t nStation, size_t nBaseline, size_t nChannel,
- const_cursor<Baseline> baselines, const_cursor<double> freq,
- const_cursor<double> uvw, cursor<dcomplex> buffer);
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Simulator.h b/CEP/DP3/DPPP/include/DPPP/Simulator.h
deleted file mode 100644
index 19c2bacc4b79eaedcca9b0c1fef9ad8d618d1644..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Simulator.h
+++ /dev/null
@@ -1,112 +0,0 @@
-//# Simulator.h: Compute visibilities for different model components types
-//# (implementation of ModelComponentVisitor).
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_SIMULATOR_H
-#define DPPP_SIMULATOR_H
-
-// \file
-// Compute visibilities for different model components types (implementation of
-// ModelComponentVisitor).
-
-#include <DPPP/Baseline.h>
-#include <DPPP/ModelComponent.h>
-#include <DPPP/ModelComponentVisitor.h>
-#include <DPPP/Position.h>
-
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Arrays/Matrix.h>
-#include <casacore/casa/Arrays/Cube.h>
-
-#include <Common/lofar_complex.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-class Simulator: public ModelComponentVisitor
-{
-public:
- Simulator(const Position &reference, size_t nStation, size_t nBaseline,
- size_t nChannel, const casacore::Vector<Baseline>& baselines,
- const casacore::Vector<double>& freq, const casacore::Matrix<double>& uvw,
- casacore::Cube<dcomplex>& buffer, bool stokesIOnly=false);
-
- template <typename T>
- class Matrix {
- public:
- Matrix():
- itsNRows(0) {
- } ;
-
- Matrix(size_t nrows, size_t ncols)
- {
- resize(nrows, ncols);
- }
-
- void resize(size_t nrows, size_t ncols) {
- itsNRows = nrows;
- itsData.resize(nrows*ncols);
- }
-
- T& operator()(size_t row, size_t col) {
- return itsData[col*itsNRows+row];
- }
-
- T* data() {
- return &itsData[0];
- }
-
- private:
- std::vector<T> itsData;
- size_t itsNRows;
- };
-
- void simulate(const ModelComponent::ConstPtr &component);
-
-private:
- virtual void visit(const PointSource &component);
- virtual void visit(const GaussianSource &component);
-
-private:
- Position itsReference;
- size_t itsNStation, itsNBaseline, itsNChannel;
- bool itsStokesIOnly;
- const casacore::Vector<Baseline> itsBaselines;
- const casacore::Vector<double> itsFreq;
- const casacore::Matrix<double> itsUVW;
- casacore::Cube<dcomplex> itsBuffer;
- Matrix<dcomplex> itsShiftBuffer;
- Matrix<dcomplex> itsSpectrumBuffer;
-};
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/SourceDBUtil.h b/CEP/DP3/DPPP/include/DPPP/SourceDBUtil.h
deleted file mode 100644
index 52fdd76ad48deb0195c074df7f1ccd0bfbc2b499..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/SourceDBUtil.h
+++ /dev/null
@@ -1,71 +0,0 @@
-//# SourceDBUtil.h: Helper functions to extract patch and source information
-//# from a SourceDB.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_SOURCEDBUTIL_H
-#define DPPP_SOURCEDBUTIL_H
-
-// \file
-// Helper functions to extract patch and source information from a SourceDB.
-
-#include <DPPP/Patch.h>
-#include <Common/lofar_string.h>
-#include <Common/LofarTypes.h>
-
-namespace LOFAR
-{
-namespace BBS
-{
-class SourceDB;
-}
-
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
- vector<Patch::ConstPtr> makePatches(BBS::SourceDB &sourceDB,
- const vector<string> &patchNames,
- uint nModel);
-
- // Create a source list (with patch name) from a patchlist
- // Needed for efficient multithreading
- std::vector<std::pair<ModelComponent::ConstPtr,Patch::ConstPtr> >
- makeSourceList (const std::vector<Patch::ConstPtr>& patchList);
-
- // From a given PatchList, create a new one with one patch per component
- vector<Patch::ConstPtr> makeOnePatchPerComponent(
- const vector<Patch::ConstPtr>&);
-
- vector<string> makePatchList(BBS::SourceDB &sourceDB,
- vector<string> patterns);
-
- bool checkPolarized(BBS::SourceDB &sourceDB,
- const vector<string> &patchNames,
- uint nModel);
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Split.h b/CEP/DP3/DPPP/include/DPPP/Split.h
deleted file mode 100644
index f35b956e1ffb30cc6c6cf81cbc38728b5545e77c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Split.h
+++ /dev/null
@@ -1,81 +0,0 @@
-//# Split.h: DPPP step class to Split visibilities from a source model
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_Split_H
-#define DPPP_Split_H
-
-// @file
-// @brief DPPP step class to Split visibilities from a source model
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-
-#include <utility>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is an empty DPStep subclass to use as implementation template
-
- class Split: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- Split (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~Split();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- //# Data members.
- string itsName;
-
- std::vector<std::string> itsReplaceParms; // The names of the parameters that differ along the substeps
- std::vector<DPStep::ShPtr> itsSubsteps;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/StManParsetKeys.h b/CEP/DP3/DPPP/include/DPPP/StManParsetKeys.h
deleted file mode 100644
index 158b0e022ca712f9b62288dede17372f375ae792..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/StManParsetKeys.h
+++ /dev/null
@@ -1,63 +0,0 @@
-#ifndef DPPP_STMANPARSETKEYS_H
-#define DPPP_STMANPARSETKEYS_H
-
-#include <Common/ParameterSet.h>
-
-#include <string>
-
-#include <casacore/casa/Containers/Record.h>
-
-namespace LOFAR {
- namespace DPPP {
- struct StManParsetKeys
- {
- casacore::String stManName;
- uint dyscoDataBitRate; //# Bits per data float, or 0 if data column is not compressed
- uint dyscoWeightBitRate; //# Bits per weight float, or 0 if weight column is not compressed
- std::string dyscoDistribution; //# Distribution assumed for compression; e.g. "Uniform" or "TruncatedGaussian"
- double dyscoDistTruncation; //# For truncated distributions, the truncation point (e.g. 3 for 3 sigma in TruncGaus).
- std::string dyscoNormalization; //# Kind of normalization; "AF", "RF" or "Row".
-
- void Set(const ParameterSet& parset, const std::string& prefix)
- {
- stManName = toLower(parset.getString(prefix+"storagemanager",
- parset.getString(prefix+"storagemanager.name",
- string())));
- if(stManName == "dysco") {
- dyscoDataBitRate = parset.getInt(
- prefix+"storagemanager.databitrate", 10);
- dyscoWeightBitRate = parset.getInt(
- prefix+"storagemanager.weightbitrate", 12);
- dyscoDistribution = parset.getString(
- prefix+"storagemanager.distribution",
- "TruncatedGaussian");
- dyscoDistTruncation = parset.getDouble(
- prefix+"storagemanager.disttruncation", 2.5);
- dyscoNormalization = parset.getString(
- prefix+"storagemanager.normalization", "AF");
- }
- }
-
- casacore::Record GetDyscoSpec() const
- {
- casacore::Record dyscoSpec;
- dyscoSpec.define ("distribution", dyscoDistribution);
- dyscoSpec.define ("normalization", dyscoNormalization);
- dyscoSpec.define ("distributionTruncation", dyscoDistTruncation);
- // DPPP uses bitrate of 0 to disable compression of the data/weight column.
- // However, Dysco does not allow the data or weight bitrates to be set to 0,
- // so we set the values to something different. The values are not actually used.
- uint dataBitRate = dyscoDataBitRate;
- if(dataBitRate == 0)
- dataBitRate = 16;
- dyscoSpec.define ("dataBitCount", dataBitRate);
- uint weightBitRate = dyscoWeightBitRate;
- if(weightBitRate == 0)
- weightBitRate = 16;
- dyscoSpec.define ("weightBitCount", weightBitRate);
- return dyscoSpec;
- }
- };
- }
-}
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/StationAdder.h b/CEP/DP3/DPPP/include/DPPP/StationAdder.h
deleted file mode 100644
index dd0dbf499b285187ca69e989988e2e72d10110e0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/StationAdder.h
+++ /dev/null
@@ -1,124 +0,0 @@
-//# StationAdder.h: DPPP step class to add stations as a superstation
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_STATIONADDER_H
-#define DPPP_STATIONADDER_H
-
-// @file
-// @brief DPPP step class to add stations as a superstation
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/UVWCalculator.h>
-#include <Common/ParameterRecord.h>
-#include <casacore/measures/Measures/MPosition.h>
-
-namespace LOFAR {
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class summing stations to a superstation.
- //
- // It is possible to define one or more groups of stations to be summed.
- // Each group has a name which is the name of the new station.
- // The complex values of baselines are added for which one station occurs
- // in only one group. A baseline is not added if no or both stations are
- // member of a summing group.
- // <br>The summation is done in a weighted way, where the weight of a
- // new station is the sum of the original weights. Optionally weights 1
- // can be used instead of the original weights.
- //
- // Only unflagged data points are used. If too few data points are
- // unflagged, the output data point is flagged.
- //
- // Questions:
- // 1. check if phases do not differ too much? Flag if too much?
- // 2. must all stations exist or possible that some don't?
-
- class StationAdder: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- StationAdder (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~StationAdder();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Add new meta info to the MS.
- virtual void addToMS (const string& msName);
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Return the indices of the stations in antennaNames matching
- // the pattern list.
- // The patterns are processed from left to right. A pattern can start
- // with ! or ^ meaning that the the matches are discarded. In this
- // way first a broad pattern can be given, which can be narrowed down.
- // A warning is given if a pattern does not match any station name.
- static vector<int> getMatchingStations
- (const casacore::Vector<casacore::String>& antennaNames,
- const vector<string>& patterns);
-
- private:
- // Update the beam info subtables.
- void updateBeamInfo (const string& msName, uint origNant,
- casacore::Table& antTab);
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuf;
- DPBuffer itsBufTmp;
- ParameterRecord itsStatRec; // stations definitions
- vector<casacore::Vector<int> > itsParts; // the stations in each superstation
- vector<vector<int> > itsBufRows; // old baseline rows in each new baseline
- uint itsMinNPoint ; // flag data if too few unflagged data
- bool itsMakeAutoCorr; // also form new auto-correlations?
- bool itsSumAutoCorr; // sum auto- or cross-correlations?
- bool itsDoAverage; // average or sum?
- bool itsUseWeight; // false = use weight 1 per station
- UVWCalculator itsUVWCalc;
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/StefCal.h b/CEP/DP3/DPPP/include/DPPP/StefCal.h
deleted file mode 100644
index a509d3fd264d63f4bf24ba041e1be5a8e1570366..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/StefCal.h
+++ /dev/null
@@ -1,143 +0,0 @@
-//# StefCal.h: Perform StefCal algorithm for gain calibration
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: StefCal.h 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_STEFCAL_H
-#define DPPP_STEFCAL_H
-
-// @file
-// @brief DPPP step class to apply a calibration correction to the data
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-
-namespace LOFAR {
-
- namespace DPPP {
- // @ingroup NDPPP
-
- class StefCal
- {
- public:
- enum Status {CONVERGED=1, NOTCONVERGED=2, STALLED=3, FAILED=4};
-
- enum StefCalMode {DEFAULT, PHASEONLY, AMPLITUDEONLY, FULLJONES};
-
- // mode can be "diagonal", "fulljones", "phaseonly", "scalarphase"
- StefCal(uint solInt, uint nChan, StefCalMode mode, bool scalar,
- double tolerance, uint maxAntennas, bool detectStalling,
- uint debugLevel);
-
- // Sets visibility matrices to zero
- void resetVis();
-
- // Initializes a new run of stefcal, resizes all internal vectors
- // If initSolutions is false, you are responsible for setting them
- // before running the solver. You could set the solutions to those
- // of the previous time step.
- void init(bool initSolutions);
-
- // Perform sone iteration of stefcal. Returns CONVERGED, NOTCONVERGED
- // or STALLED
- Status doStep(uint iter);
-
- // Returns the solution. The return matrix has a length of maxAntennas,
- // which is zero for antennas for which no solution was computed.
- // The mapping is stored in the antenna map
- casacore::Matrix<casacore::DComplex> getSolution(bool setNaNs);
-
- double getWeight() {
- return _totalWeight;
- }
-
- // Increments the weight (only relevant for TEC-fitting)
- void incrementWeight(float weight);
-
- // Returns a reference to the visibility matrix
- casacore::Array<casacore::DComplex>& getVis() {
- return _vis;
- }
-
- // Returns a reference to the model visibility matrix
- casacore::Array<casacore::DComplex>& getMVis() {
- return _mvis;
- }
-
- casacore::Vector<bool>& getStationFlagged() {
- return _stationFlagged;
- }
-
- // Number of correlations in the solution (1,2 or 4)
- uint numCorrelations() {
- return _savedNCr;
- }
-
- // Number of correlations (1 or 4)
- uint nCr() {
- return _nCr;
- }
-
- // Clear antFlagged
- void clearStationFlagged();
-
- private:
- // Perform relaxation
- Status relax(uint iter);
-
- void doStep_polarized();
- void doStep_unpolarized();
-
- double getAverageUnflaggedSolution();
-
- uint _savedNCr;
- casacore::Vector<bool> _stationFlagged ; // Contains true for totally flagged stations
- casacore::Array<casacore::DComplex> _vis; // Visibility matrix
- casacore::Array<casacore::DComplex> _mvis; // Model visibility matrix
- casacore::Matrix<casacore::DComplex> _g; // Solution, indexed by station, correlation
- casacore::Matrix<casacore::DComplex> _gx; // Previous solution
- casacore::Matrix<casacore::DComplex> _gxx; // Solution before previous solution
- casacore::Matrix<casacore::DComplex> _gold; // Previous solution
- casacore::Matrix<casacore::DComplex> _h; // Hermitian transpose of previous solution
- casacore::Matrix<casacore::DComplex> _z; // Internal stefcal vector
-
- uint _nSt; // number of stations in the current solution
- uint _nUn; // number of unknowns
- uint _nCr; // number of correlations (1 or 4)
- uint _nSp; // number that is two for scalarphase, one else
- uint _badIters; // number of bad iterations, for stalling detection
- uint _veryBadIters; // number of iterations where solution got worse
- uint _solInt; // solution interval
- uint _nChan; // number of channels
- StefCalMode _mode; // diagonal, scalarphase, fulljones or phaseonly
- bool _scalar; // false if each polarization has a separate solution
- double _tolerance;
- double _totalWeight;
- bool _detectStalling;
- uint _debugLevel;
-
- double _dg, _dgx; // previous convergence
- std::vector<double> _dgs; // convergence history
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Stokes.h b/CEP/DP3/DPPP/include/DPPP/Stokes.h
deleted file mode 100644
index e720f2cf424ab114a357c5098a15fe43cf0345a3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Stokes.h
+++ /dev/null
@@ -1,50 +0,0 @@
-//# Stokes.h: Complex Stokes vector.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_STOKES_H
-#define DPPP_STOKES_H
-
-// \file
-// Complex Stokes vector.
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-
-class Stokes
-{
-public:
- Stokes();
-
- double I, Q, U, V;
-};
-
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/SubtractMixed.h b/CEP/DP3/DPPP/include/DPPP/SubtractMixed.h
deleted file mode 100644
index 3f2af79e88fd23adef0fe9f0a39e33140484e5b7..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/SubtractMixed.h
+++ /dev/null
@@ -1,66 +0,0 @@
-//# SubtractMixed.h: Subtract visibilities from buffer after weighting by mixing
-//# coefficients.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_SUBTRACTMIXED_H
-#define DPPP_SUBTRACTMIXED_H
-
-// \file
-// Subtract visibilities from a buffer after weighting by mixing coefficients.
-
-#include <DPPP/Baseline.h>
-#include <DPPP/Cursor.h>
-#include <Common/lofar_complex.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-// Subtract visibilities from a buffer after weighting by mixing coefficients.
-//
-// \param[in] nBaseline
-// Number of baselines.
-// \param[in] nChannel
-// Number of frequency channels.
-// \param[in] baselines
-// A cursor for a 1-D buffer of baselines of shape (\p nBaseline).
-// \param[in] data
-// A cursor for a 3-D buffer of observed visibilities of shape
-// (\p nBaseline, \p nChannel, 4).
-// \param[in] model
-// A cursor for a 3-D buffer of simulated visibilities of shape
-// (\p nBaseline, \p nChannel, 4).
-// \param[in] weight
-// A cursor for a 3-D buffer of mixing weight of shape
-// (\p nBaseline, \p nChannel, 4).
-void subtract(size_t nBaseline, size_t nChannel,
- const_cursor<Baseline> baselines, cursor<fcomplex> data,
- const_cursor<dcomplex> model, const_cursor<dcomplex> weight);
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/SubtractNew.h b/CEP/DP3/DPPP/include/DPPP/SubtractNew.h
deleted file mode 100644
index 000f22d011ab70d8578ed8b0b7d199ded562028e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/SubtractNew.h
+++ /dev/null
@@ -1,68 +0,0 @@
-//# SubtractNew.h: Subtract visibilities from buffer after weighting by mixing
-//# coefficients.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef DPPP_SUBTRACTNEW_H
-#define DPPP_SUBTRACTNEW_H
-
-// \file
-// Subtract visibilities from a buffer after weighting by mixing coefficients.
-
-#include <DPPP/Baseline.h>
-#include <DPPP/Cursor.h>
-#include <Common/lofar_complex.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-// \addtogroup NDPPP
-// @{
-// Subtract visibilities from a buffer after weighting by mixing coefficients.
-//
-// \param[in] nBaseline
-// Number of baselines.
-// \param[in] nChannel
-// Number of frequency channels.
-// \param[in] baselines
-// A cursor for a 1-D buffer of baselines of shape (\p nBaseline).
-// \param[in] data
-// A cursor for a 3-D buffer of observed visibilities of shape
-// (\p nBaseline, \p nChannel, 4).
-// \param[in] model
-// A cursor for a 3-D buffer of simulated visibilities of shape
-// (\p nBaseline, \p nChannel, 4).
-// \param[in] weight
-// A cursor for a 3-D buffer of mixing weight of shape
-// (\p nBaseline, \p nChannel, 4).
-void subtract(size_t nBaseline, size_t nChannel,
- const_cursor<Baseline> baselines, cursor<fcomplex> data,
- const_cursor<dcomplex> model, const_cursor<dcomplex> weight,
- vector<float>& sumAmpl);
-// @}
-
-} //# namespace DPPP
-} //# namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/UVWCalculator.h b/CEP/DP3/DPPP/include/DPPP/UVWCalculator.h
deleted file mode 120000
index a38d43b2609036f867340ecec65b0aa5e596fbb1..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/UVWCalculator.h
+++ /dev/null
@@ -1 +0,0 @@
-../../src/UVWCalculator/UVWCalculator.h
\ No newline at end of file
diff --git a/CEP/DP3/DPPP/include/DPPP/UVWFlagger.h b/CEP/DP3/DPPP/include/DPPP/UVWFlagger.h
deleted file mode 100644
index 3939d5ebd656f38b5ca7df9c5a8564fbb635c5d5..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/UVWFlagger.h
+++ /dev/null
@@ -1,136 +0,0 @@
-//# UVWFlagger.h: DPPP step class to flag data on UVW coordinates
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_UVWFLAGGER_H
-#define DPPP_UVWFLAGGER_H
-
-// @file
-// @brief DPPP step class to flag data on UVW coordinates
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/UVWCalculator.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR {
- class ParameterSet;
- class ParameterValue;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class flagging data points based on data
- // selections given in the parset file.
- // The following selections can be given:
- // <ul>
- // <li> minimum and/or maximum UV distance
- // <li> minimum or maximum value for U, V, and/or W
- // <li> both can be used with a different phase center which can be
- // be given as a position or as a moving source like SUN or JUPITER.
- // </ul>
- // The UVW values can be given in meters or in wavelengths.
-
- class UVWFlagger: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- // The antenna names are used to find antenna numbers.
- // The channel frequencies as they are in the input step must be given
- // starting at the start-channel.
- UVWFlagger (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~UVWFlagger();
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- // It is used to adjust the parms if needed.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the flag counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- // Test if uvw matches a range in meters.
- bool testUVWm (double uvw, const vector<double>& ranges);
-
- // Set flags for channels where uvw (in m) matches a range in wavelengths.
- void testUVWl (double uvw, const vector<double>& ranges,
- bool* flagPtr, uint nrcorr);
-
- // Return a vector with UVW ranges.
- // It looks for the named parameter suffixed with 'range', 'min', and
- // 'max'. The returned vector contains 2 subsequent values for each range
- // (min and max are also turned into a range).
- // Optionally the values are squared to avoid having to take a sqrt
- // of the data's UVW coordinates.
- vector<double> fillUVW (const ParameterSet& parset,
- const string& prefix,
- const string& name,
- bool square);
-
- // Handle the specification of a phase center.
- // It setups the UVWCalculator.
- void handleCenter();
-
- //# Data members.
- DPInput* itsInput;
- string itsName;
- DPBuffer itsBuffer;
- uint itsNTimes;
-
- // If nothing is filled in, this step just passes through data
- bool itsIsDegenerate;
-
- casacore::Vector<double> itsRecWavel; //# reciprokes of wavelengths
- vector<double> itsRangeUVm; //# UV ranges (in m) to be flagged
- vector<double> itsRangeUm; //# U ranges (in m) to be flagged
- vector<double> itsRangeVm; //# V ranges (in m) to be flagged
- vector<double> itsRangeWm; //# W ranges (in m) to be flagged
- vector<double> itsRangeUVl; //# UV ranges (in wavel) to be flagged
- vector<double> itsRangeUl; //# U ranges (in wavel) to be flagged
- vector<double> itsRangeVl; //# V ranges (in wavel) to be flagged
- vector<double> itsRangeWl; //# W ranges (in wavel) to be flagged
- UVWCalculator itsUVWCalc;
- vector<string> itsCenter;
- NSTimer itsTimer;
- NSTimer itsUVWTimer;
- FlagCounter itsFlagCounter;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/include/DPPP/Upsample.h b/CEP/DP3/DPPP/include/DPPP/Upsample.h
deleted file mode 100644
index d2baa9014edca30953211039e2e6d2972b9c58c3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/include/DPPP/Upsample.h
+++ /dev/null
@@ -1,83 +0,0 @@
-//# Upsample.h: DPPP step class to upsample visibilities
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id:
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_DummyStep_H
-#define DPPP_DummyStep_H
-
-// @file
-// @brief DPPP step class to Upsample visibilities
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-
-#include <utility>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is an empty DPStep subclass to use as implementation template
-
- class Upsample: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- Upsample (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~Upsample();
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- private:
- //# Data members.
- string itsName;
- double itsOldTimeInterval;
- uint itsTimeStep;
-
- std::vector<DPBuffer> itsPrevBuffers;
- std::vector<DPBuffer> itsBuffers;
- uint itsFirstToFlush;
-
- NSTimer itsTimer;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/share/CMakeLists.txt b/CEP/DP3/DPPP/share/CMakeLists.txt
deleted file mode 100644
index 75e8809b010017d9ebc42db2874d8a08be3703c2..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/share/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# $Id$
-
-# Data files
-install(FILES
- LBAdefault
- HBAdefault
- DESTINATION share/rfistrategies)
diff --git a/CEP/DP3/DPPP/share/HBAdefault b/CEP/DP3/DPPP/share/HBAdefault
deleted file mode 100644
index b6844bb00131967a696e281415748f8a87e8b521..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/share/HBAdefault
+++ /dev/null
@@ -1,90 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- This is a strategy configuration file for the AOFlagger RFI
-detector by André Offringa (offringa@gmail.com).
-Created by AOFlagger 2.7.2 (2015-09-27)
--->
-<rfi-strategy format-version="3.7" reader-version-required="3.4">
- <action type="Strategy">
- <children>
- <action type="SetFlaggingAction">
- <new-flagging>0</new-flagging>
- </action>
- <action type="ForEachPolarisationBlock">
- <on-xx>0</on-xx>
- <on-xy>1</on-xy>
- <on-yx>1</on-yx>
- <on-yy>0</on-yy>
- <on-stokes-i>0</on-stokes-i>
- <on-stokes-q>0</on-stokes-q>
- <on-stokes-u>0</on-stokes-u>
- <on-stokes-v>0</on-stokes-v>
- <children>
- <action type="ForEachComplexComponentAction">
- <on-amplitude>1</on-amplitude>
- <on-phase>0</on-phase>
- <on-real>0</on-real>
- <on-imaginary>0</on-imaginary>
- <restore-from-amplitude>0</restore-from-amplitude>
- <children>
- <action type="IterationBlock">
- <iteration-count>2</iteration-count>
- <sensitivity-start>4</sensitivity-start>
- <children>
- <action type="SumThresholdAction">
- <base-sensitivity>1</base-sensitivity>
- <time-direction-flagging>1</time-direction-flagging>
- <frequency-direction-flagging>1</frequency-direction-flagging>
- </action>
- <action type="CombineFlagResults">
- <children>
- <action type="FrequencySelectionAction">
- <threshold>3</threshold>
- </action>
- <action type="TimeSelectionAction">
- <threshold>3.5</threshold>
- </action>
- </children>
- </action>
- <action type="SetImageAction">
- <new-image>1</new-image>
- </action>
- <action type="ChangeResolutionAction">
- <time-decrease-factor>3</time-decrease-factor>
- <frequency-decrease-factor>3</frequency-decrease-factor>
- <restore-revised>1</restore-revised>
- <restore-masks>0</restore-masks>
- <children>
- <action type="HighPassFilterAction">
- <horizontal-kernel-sigma-sq>2.5</horizontal-kernel-sigma-sq>
- <vertical-kernel-sigma-sq>5</vertical-kernel-sigma-sq>
- <window-width>21</window-width>
- <window-height>31</window-height>
- <mode>1</mode>
- </action>
- </children>
- </action>
- </children>
- </action>
- <action type="SumThresholdAction">
- <base-sensitivity>1</base-sensitivity>
- <time-direction-flagging>1</time-direction-flagging>
- <frequency-direction-flagging>1</frequency-direction-flagging>
- </action>
- </children>
- </action>
- </children>
- </action>
- <action type="SetFlaggingAction">
- <new-flagging>4</new-flagging>
- </action>
- <action type="StatisticalFlagAction">
- <enlarge-frequency-size>0</enlarge-frequency-size>
- <enlarge-time-size>0</enlarge-time-size>
- <max-contaminated-frequencies-ratio>0.5</max-contaminated-frequencies-ratio>
- <max-contaminated-times-ratio>0.5</max-contaminated-times-ratio>
- <minimum-good-frequency-ratio>0.2</minimum-good-frequency-ratio>
- <minimum-good-time-ratio>0.2</minimum-good-time-ratio>
- </action>
- </children>
- </action>
-</rfi-strategy>
\ No newline at end of file
diff --git a/CEP/DP3/DPPP/share/LBAdefault b/CEP/DP3/DPPP/share/LBAdefault
deleted file mode 100644
index a0f1f3c18893fc3c78194d7f85d37622a778f87a..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/share/LBAdefault
+++ /dev/null
@@ -1,93 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- This is a strategy configuration file for the AOFlagger RFI
-detector by André Offringa (offringa@gmail.com).
-Created by AOFlagger 2.7.2 (2015-09-27)
--->
-<rfi-strategy format-version="3.7" reader-version-required="3.4">
- <action type="Strategy">
- <children>
- <action type="SetFlaggingAction">
- <new-flagging>0</new-flagging>
- </action>
- <action type="ForEachPolarisationBlock">
- <on-xx>1</on-xx>
- <on-xy>1</on-xy>
- <on-yx>1</on-yx>
- <on-yy>1</on-yy>
- <on-stokes-i>0</on-stokes-i>
- <on-stokes-q>0</on-stokes-q>
- <on-stokes-u>0</on-stokes-u>
- <on-stokes-v>0</on-stokes-v>
- <children>
- <action type="ForEachComplexComponentAction">
- <on-amplitude>1</on-amplitude>
- <on-phase>0</on-phase>
- <on-real>0</on-real>
- <on-imaginary>0</on-imaginary>
- <restore-from-amplitude>0</restore-from-amplitude>
- <children>
- <action type="IterationBlock">
- <iteration-count>2</iteration-count>
- <sensitivity-start>4</sensitivity-start>
- <children>
- <action type="SumThresholdAction">
- <base-sensitivity>1</base-sensitivity>
- <time-direction-flagging>1</time-direction-flagging>
- <frequency-direction-flagging>1</frequency-direction-flagging>
- </action>
- <action type="CombineFlagResults">
- <children>
- <action type="FrequencySelectionAction">
- <threshold>3</threshold>
- </action>
- <action type="TimeSelectionAction">
- <threshold>3.5</threshold>
- </action>
- </children>
- </action>
- <action type="SetImageAction">
- <new-image>1</new-image>
- </action>
- <action type="ChangeResolutionAction">
- <time-decrease-factor>3</time-decrease-factor>
- <frequency-decrease-factor>3</frequency-decrease-factor>
- <restore-revised>1</restore-revised>
- <restore-masks>0</restore-masks>
- <children>
- <action type="HighPassFilterAction">
- <horizontal-kernel-sigma-sq>2.5</horizontal-kernel-sigma-sq>
- <vertical-kernel-sigma-sq>5</vertical-kernel-sigma-sq>
- <window-width>21</window-width>
- <window-height>31</window-height>
- <mode>1</mode>
- </action>
- </children>
- </action>
- </children>
- </action>
- <action type="SumThresholdAction">
- <base-sensitivity>1</base-sensitivity>
- <time-direction-flagging>1</time-direction-flagging>
- <frequency-direction-flagging>1</frequency-direction-flagging>
- </action>
- </children>
- </action>
- </children>
- </action>
- <action type="SetFlaggingAction">
- <new-flagging>4</new-flagging>
- </action>
- <action type="StatisticalFlagAction">
- <enlarge-frequency-size>0</enlarge-frequency-size>
- <enlarge-time-size>0</enlarge-time-size>
- <max-contaminated-frequencies-ratio>0.5</max-contaminated-frequencies-ratio>
- <max-contaminated-times-ratio>0.5</max-contaminated-times-ratio>
- <minimum-good-frequency-ratio>0.2</minimum-good-frequency-ratio>
- <minimum-good-time-ratio>0.2</minimum-good-time-ratio>
- </action>
- <action type="TimeSelectionAction">
- <threshold>3.5</threshold>
- </action>
- </children>
- </action>
-</rfi-strategy>
\ No newline at end of file
diff --git a/CEP/DP3/DPPP/src/Apply.cc b/CEP/DP3/DPPP/src/Apply.cc
deleted file mode 100644
index cd5dadbfa8a2ccb2f0fbc48b9feb6f1d6d812d09..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Apply.cc
+++ /dev/null
@@ -1,97 +0,0 @@
-//# Apply.cc: Apply station Jones matrices to a set of visibilities.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/Apply.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-void apply(size_t nBaseline, size_t nChannel, const_cursor<Baseline> baselines,
- const_cursor<double> coeff, cursor<dcomplex> data)
-{
- for(size_t bl = 0; bl < nBaseline; ++bl)
- {
- const size_t p = baselines->first;
- const size_t q = baselines->second;
-
- if(p != q)
- {
- // Jones matrix for station P.
- coeff.forward(1, p);
- const dcomplex Jp_00(coeff[0], coeff[1]);
- const dcomplex Jp_01(coeff[2], coeff[3]);
- const dcomplex Jp_10(coeff[4], coeff[5]);
- const dcomplex Jp_11(coeff[6], coeff[7]);
- coeff.backward(1, p);
-
- // Jones matrix for station Q, conjugated.
- coeff.forward(1, q);
- const dcomplex Jq_00(coeff[0], -coeff[1]);
- const dcomplex Jq_01(coeff[2], -coeff[3]);
- const dcomplex Jq_10(coeff[4], -coeff[5]);
- const dcomplex Jq_11(coeff[6], -coeff[7]);
- coeff.backward(1, q);
-
- // Compute (Jp x conj(Jq)) * vec(data), where 'x' denotes the
- // Kronecker product.
- for(size_t ch = 0; ch < nChannel; ++ch)
- {
- // Fetch visibilities.
- const dcomplex xx = data[0];
- const dcomplex xy = data[1];
- const dcomplex yx = data[2];
- const dcomplex yy = data[3];
-
- // Precompute terms involving conj(Jq) and data. Each term
- // appears twice in the computation of (Jp x conj(Jq))
- // * vec(data).
- const dcomplex Jq_00xx_01xy = Jq_00 * xx + Jq_01 * xy;
- const dcomplex Jq_00yx_01yy = Jq_00 * yx + Jq_01 * yy;
- const dcomplex Jq_10xx_11xy = Jq_10 * xx + Jq_11 * xy;
- const dcomplex Jq_10yx_11yy = Jq_10 * yx + Jq_11 * yy;
-
- // Compute (Jp x conj(Jq)) * vec(data) from the precomputed
- // terms.
- data[0] = Jp_00 * Jq_00xx_01xy + Jp_01 * Jq_00yx_01yy;
- data[1] = Jp_00 * Jq_10xx_11xy + Jp_01 * Jq_10yx_11yy;
- data[2] = Jp_10 * Jq_00xx_01xy + Jp_11 * Jq_00yx_01yy;
- data[3] = Jp_10 * Jq_10xx_11xy + Jp_11 * Jq_10yx_11yy;
-
- // Move to the next channel.
- data.forward(1);
- } // Channels.
-
- // Reset cursor to the beginning of the current baseline.
- data.backward(1, nChannel);
- }
-
- // Move to the next baseline.
- data.forward(2);
- ++baselines;
- } // Baselines.
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/ApplyBeam.cc b/CEP/DP3/DPPP/src/ApplyBeam.cc
deleted file mode 100644
index 42d07008d99c5434719e6fd969daf1eb543ab261..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/ApplyBeam.cc
+++ /dev/null
@@ -1,208 +0,0 @@
-//# ApplyBeam.cc: DPPP step class to ApplyBeam visibilities
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/ApplyBeam.h>
-
-#include <iostream>
-//#include <iomanip>
-#include <Common/ParameterSet.h>
-#include <Common/Timer.h>
-#include <Common/StringUtil.h>
-#include <Common/OpenMP.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/FlagCounter.h>
-#include <DPPP/Position.h>
-
-#include <casacore/casa/Arrays/Array.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-using namespace casacore;
-using namespace LOFAR::BBS;
-
-namespace LOFAR {
- namespace DPPP {
-
- ApplyBeam::ApplyBeam(DPInput* input, const ParameterSet& parset,
- const string& prefix, bool substep)
- :
- itsInput(input),
- itsName(prefix),
- itsUpdateWeights(parset.getBool(prefix + "updateweights", false)),
- itsUseChannelFreq(parset.getBool(prefix + "usechannelfreq", true)),
- itsDebugLevel(parset.getInt(prefix + "debuglevel", 0))
- {
- // only read 'invert' parset key if it is a separate step
- // if applybeam is called from gaincal/predict, the invert key should always be false
- if (substep) {
- itsInvert=false;
- } else {
- itsInvert=parset.getBool(prefix + "invert", true);
- }
- string mode=toLower(parset.getString(prefix + "beammode","default"));
- ASSERT (mode=="default" || mode=="array_factor" || mode=="element");
- if (mode=="default") {
- itsMode=DEFAULT;
- } else if (mode=="array_factor") {
- itsMode=ARRAY_FACTOR;
- } else if (mode=="element") {
- itsMode=ELEMENT;
- } else {
- THROW(Exception, "Beammode should be DEFAULT, ARRAY_FACTOR or ELEMENT");
- }
- }
-
- ApplyBeam::ApplyBeam()
- {
- }
-
- ApplyBeam::~ApplyBeam()
- {
- }
-
- void ApplyBeam::updateInfo(const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- if (itsUpdateWeights) {
- info().setWriteWeights();
- }
-
- MDirection dirJ2000(
- MDirection::Convert(infoIn.phaseCenter(), MDirection::J2000)());
- Quantum<Vector<Double> > angles = dirJ2000.getAngle();
- itsPhaseRef = Position(angles.getBaseValue()[0],
- angles.getBaseValue()[1]);
-
- const size_t nSt = info().nantenna();
- const size_t nCh = info().nchan();
-
- itsBeamValues.resize(OpenMP::maxThreads());
-
- // Create the Measure ITRF conversion info given the array position.
- // The time and direction are filled in later.
- itsMeasConverters.resize(OpenMP::maxThreads());
- itsMeasFrames.resize(OpenMP::maxThreads());
- itsAntBeamInfo.resize(OpenMP::maxThreads());
-
- for (uint thread = 0; thread < OpenMP::maxThreads(); ++thread) {
- itsBeamValues[thread].resize(nSt * nCh);
- itsMeasFrames[thread].set(info().arrayPosCopy());
- itsMeasFrames[thread].set(
- MEpoch(MVEpoch(info().startTime() / 86400), MEpoch::UTC));
- itsMeasConverters[thread].set(
- MDirection::J2000,
- MDirection::Ref(MDirection::ITRF, itsMeasFrames[thread]));
- itsInput->fillBeamInfo(itsAntBeamInfo[thread], info().antennaNames());
- }
- }
-
- void ApplyBeam::show(std::ostream& os) const
- {
- os << "ApplyBeam " << itsName << endl;
- os << " mode: ";
- if (itsMode==DEFAULT)
- os<<"default";
- else if (itsMode==ARRAY_FACTOR)
- os<<"array_factor";
- else os<<"element";
- os << endl;
- os << " use channelfreq: " << boolalpha << itsUseChannelFreq << endl;
- os << " invert: " << boolalpha << itsInvert << endl;
- os << " update weights: " << boolalpha << itsUpdateWeights << endl;
- }
-
- void ApplyBeam::showTimings(std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1(os, itsTimer.getElapsed(), duration);
- os << " ApplyBeam " << itsName << endl;
- }
-
- bool ApplyBeam::process(const DPBuffer& bufin)
- {
- itsTimer.start();
- itsBuffer.copy (bufin);
- Complex* data=itsBuffer.getData().data();
-
- if (itsUpdateWeights) {
- itsInput->fetchWeights (bufin, itsBuffer, itsTimer);
- }
- float* weight = itsBuffer.getWeights().data();
-
- double time = itsBuffer.getTime();
-
- //Set up directions for beam evaluation
- StationResponse::vector3r_t refdir, tiledir;
-
- for (uint thread = 0; thread < OpenMP::maxThreads(); ++thread) {
- itsMeasFrames[thread].resetEpoch(
- MEpoch(MVEpoch(time / 86400), MEpoch::UTC));
- //Do a conversion on all threads, because converters are not
- //thread safe and apparently need to be used at least once
- refdir = dir2Itrf(info().delayCenter(), itsMeasConverters[thread]);
- tiledir = dir2Itrf(info().tileBeamDir(), itsMeasConverters[thread]);
- }
-
- uint thread = OpenMP::threadNum();
-
- StationResponse::vector3r_t srcdir = refdir;
- applyBeam(info(), time, data, weight, srcdir, refdir, tiledir,
- itsAntBeamInfo[thread], itsBeamValues[thread],
- itsUseChannelFreq, itsInvert, itsMode, itsUpdateWeights);
-
- itsTimer.stop();
- getNextStep()->process(itsBuffer);
- return false;
- }
-
- StationResponse::vector3r_t ApplyBeam::dir2Itrf(
- const MDirection& dir, MDirection::Convert& measConverter)
- {
- const MDirection& itrfDir = measConverter(dir);
- const Vector<Double>& itrf = itrfDir.getValue().getValue();
- StationResponse::vector3r_t vec;
- vec[0] = itrf[0];
- vec[1] = itrf[1];
- vec[2] = itrf[2];
- return vec;
- }
-
- void ApplyBeam::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/ApplyCal.cc b/CEP/DP3/DPPP/src/ApplyCal.cc
deleted file mode 100644
index 2a41912f49a226abca37b94437f2864cbc717478..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/ApplyCal.cc
+++ /dev/null
@@ -1,299 +0,0 @@
-//# GainCal.cc: DPPP step class to ApplyCal visibilities
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/ApplyCal.h>
-
-#include <iostream>
-#include <Common/ParameterSet.h>
-#include <Common/ParameterValue.h>
-#include <Common/Timer.h>
-
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- ApplyCal::ApplyCal (DPInput* input,
- const ParameterSet& parset,
- const string& prefix,
- bool substep,
- string predictDirection
- )
- : itsIsSubstep(substep)
- {
- vector<string> subStepNames;
- ParameterValue namesPar (parset.getString(prefix + "steps", ""));
-
- if (namesPar.isVector()) {
- subStepNames = namesPar.getStringVector();
- } else {
- subStepNames.push_back(namesPar.getString());
- }
-
- vector<string>::const_iterator subStepNameIter;
- for (subStepNameIter = subStepNames.begin();
- subStepNameIter != subStepNames.end();
- ++subStepNameIter) {
- string subStepName = (*subStepNameIter);
- string subStepPrefix;
- if (subStepName.empty()) {
- // No substeps given, use parameters of this step
- subStepPrefix = prefix;
- } else {
- // Substeps given, use named parameters like applycal.applySol.parmdb
- subStepPrefix = prefix + subStepName + ".";
- }
- itsApplyCals.push_back(OneApplyCal::ShPtr(new OneApplyCal(
- input, parset, subStepPrefix, prefix, substep,
- predictDirection)));
- }
-
- uint numSteps = itsApplyCals.size();
- for (uint step=0; step<numSteps-1; ++step) {
- itsApplyCals[step]->setNextStep(itsApplyCals[step+1]);
- }
- }
-
- ApplyCal::ApplyCal()
- {}
-
- ApplyCal::~ApplyCal()
- {}
-
- void ApplyCal::setNextStep (DPStep::ShPtr nextStep)
- {
- DPStep::setNextStep(itsApplyCals[0]);
- itsApplyCals[itsApplyCals.size()-1]->setNextStep(nextStep);
- }
-
- void ApplyCal::show(std::ostream& os) const
- {
- // If not a substep, show will be called by DPRun,
- // through the nextStep() mechanism
- if (itsIsSubstep) {
- vector<OneApplyCal::ShPtr>::const_iterator applycalIter;
-
- for (applycalIter = itsApplyCals.begin();
- applycalIter != itsApplyCals.end();
- applycalIter++) {
- (*applycalIter)->show(os);
- }
- }
- }
-
- void ApplyCal::showTimings (std::ostream& os, double duration) const
- {
- if (itsIsSubstep) {
- vector<OneApplyCal::ShPtr>::const_iterator iter;
- for (iter = itsApplyCals.begin();
- iter != itsApplyCals.end();
- iter++) {
- (*iter)->showTimings(os, duration);
- }
- }
- }
-
- bool ApplyCal::process (const DPBuffer& bufin)
- {
- getNextStep()->process(bufin);
- return true;
- }
-
-
- void ApplyCal::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void ApplyCal::applyDiag (const DComplex* gainA, const DComplex* gainB,
- Complex* vis, float* weight, bool* flag,
- uint bl, uint chan, bool updateWeights,
- FlagCounter& flagCounter) {
- // If parameter is NaN or inf, do not apply anything and flag the data
- if (! (isFinite(gainA[0].real()) && isFinite(gainA[0].imag()) &&
- isFinite(gainB[0].real()) && isFinite(gainB[0].imag()) &&
- isFinite(gainA[1].real()) && isFinite(gainA[1].imag()) &&
- isFinite(gainB[1].real()) && isFinite(gainB[1].imag())) ) {
- // Only update flagcounter for first correlation
- if (!flag[0]) {
- flagCounter.incrChannel(chan);
- flagCounter.incrBaseline(bl);
- }
- for (uint corr=0; corr<4; ++corr) {
- flag[corr]=true;
- }
- return;
- }
-
- vis[0] *= gainA[0] * conj(gainB[0]);
- vis[1] *= gainA[0] * conj(gainB[1]);
- vis[2] *= gainA[1] * conj(gainB[0]);
- vis[3] *= gainA[1] * conj(gainB[1]);
-
- if (updateWeights) {
- weight[0] /= norm(gainA[0]) * norm(gainB[0]);
- weight[1] /= norm(gainA[0]) * norm(gainB[1]);
- weight[2] /= norm(gainA[1]) * norm(gainB[0]);
- weight[3] /= norm(gainA[1]) * norm(gainB[1]);
- }
- }
-
- void ApplyCal::applyScalar(const DComplex* gainA, const DComplex* gainB,
- Complex* vis, float* weight, bool* flag,
- uint bl, uint chan, bool updateWeights,
- FlagCounter& flagCounter) {
- // If parameter is NaN or inf, do not apply anything and flag the data
- if (! (isFinite(gainA[0].real()) && isFinite(gainA[0].imag()) &&
- isFinite(gainB[0].real()) && isFinite(gainB[0].imag())) ) {
- // Only update flagcounter for first correlation
- if (!flag[0]) {
- flagCounter.incrChannel(chan);
- flagCounter.incrBaseline(bl);
- }
- for (uint corr=0; corr<4; ++corr) {
- flag[corr]=true;
- }
- return;
- }
-
- vis[0] *= gainA[0] * conj(gainB[0]);
- vis[1] *= gainA[0] * conj(gainB[0]);
- vis[2] *= gainA[0] * conj(gainB[0]);
- vis[3] *= gainA[0] * conj(gainB[0]);
-
- if (updateWeights) {
- weight[0] /= norm(gainA[0]) * norm(gainB[0]);
- weight[1] /= norm(gainA[0]) * norm(gainB[0]);
- weight[2] /= norm(gainA[0]) * norm(gainB[0]);
- weight[3] /= norm(gainA[0]) * norm(gainB[0]);
- }
- }
-
- // Inverts complex 2x2 input matrix
- void ApplyCal::invert (DComplex* v, double sigmaMMSE)
- {
- // Add the variance of the nuisance term to the elements on the diagonal.
- const double variance = sigmaMMSE * sigmaMMSE;
- DComplex v0 = v[0] + variance;
- DComplex v3 = v[3] + variance;
- // Compute inverse in the usual way.
- DComplex invDet(1.0 / (v0 * v3 - v[1] * v[2]));
- v[0] = v3 * invDet;
- v[2] = v[2] * -invDet;
- v[1] = v[1] * -invDet;
- v[3] = v0 * invDet;
- }
-
- void ApplyCal::applyFull (const DComplex* gainA, const DComplex* gainB,
- Complex* vis, float* weight, bool* flag,
- uint bl, uint chan, bool doUpdateWeights,
- FlagCounter& flagCounter) {
- DComplex gainAxvis[4];
-
- // If parameter is NaN or inf, do not apply anything and flag the data
- bool anyinfnan = false;
- for (uint corr=0; corr<4; ++corr) {
- if (! (isFinite(gainA[corr].real()) && isFinite(gainA[corr].imag()) &&
- isFinite(gainB[corr].real()) && isFinite(gainB[corr].imag())) ) {
- anyinfnan = true;
- break;
- }
- }
- if (anyinfnan) {
- // Only update flag counter for first correlation
- if (!flag[0]) {
- flagCounter.incrChannel(chan);
- flagCounter.incrBaseline(bl);
- }
- for (uint corr=0; corr<4; ++corr) {
- flag[corr]=true;
- }
- return;
- }
-
- // gainAxvis = gainA * vis
- for (uint row=0;row<2;++row) {
- for (uint col=0;col<2;++col) {
- gainAxvis[2*row+col]=gainA[2*row+0] * DComplex(vis[2*0+col]) +
- gainA[2*row+1] * DComplex(vis[2*1+col]);
- }
- }
-
- // vis = gainAxvis * gainB^H
- for (uint row=0;row<2;++row) {
- for (uint col=0;col<2;++col) {
- vis[2*row+col]=gainAxvis[2*row+0] * conj(gainB[2*col+0])+
- gainAxvis[2*row+1] * conj(gainB[2*col+1]);
- }
- }
-
- if (doUpdateWeights) {
- applyWeights(gainA, gainB, weight);
- }
- }
-
- void ApplyCal::applyWeights(const DComplex* gainA,
- const DComplex* gainB,
- float* weight) {
- float cov[4], normGainA[4], normGainB[4];
- for (uint i=0;i<4;++i) {
- cov[i]=1./weight[i];
- normGainA[i]=norm(gainA[i]);
- normGainB[i]=norm(gainB[i]);
- }
-
- weight[0]=cov[0]*(normGainA[0]*normGainB[0])
- +cov[1]*(normGainA[0]*normGainB[1])
- +cov[2]*(normGainA[1]*normGainB[0])
- +cov[3]*(normGainA[1]*normGainB[1]);
- weight[0]=1./weight[0];
-
- weight[1]=cov[0]*(normGainA[0]*normGainB[2])
- +cov[1]*(normGainA[0]*normGainB[3])
- +cov[2]*(normGainA[1]*normGainB[2])
- +cov[3]*(normGainA[1]*normGainB[3]);
- weight[1]=1./weight[1];
-
- weight[2]=cov[0]*(normGainA[2]*normGainB[0])
- +cov[1]*(normGainA[2]*normGainB[1])
- +cov[2]*(normGainA[3]*normGainB[0])
- +cov[3]*(normGainA[3]*normGainB[1]);
- weight[2]=1./weight[2];
-
- weight[3]=cov[0]*(normGainA[2]*normGainB[2])
- +cov[1]*(normGainA[2]*normGainB[3])
- +cov[2]*(normGainA[3]*normGainB[2])
- +cov[3]*(normGainA[3]*normGainB[3]);
- weight[3]=1./weight[3];
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/Averager.cc b/CEP/DP3/DPPP/src/Averager.cc
deleted file mode 100644
index 92b43124e3995e0e17799977c702a773f6589778..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Averager.cc
+++ /dev/null
@@ -1,408 +0,0 @@
-//# Averager.cc: DPPP step class to average in time and/or freq
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Averager.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <Common/StringUtil.h>
-
-#include <iostream>
-#include <iomanip>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- Averager::Averager (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsMinNPoint (parset.getUint (prefix+"minpoints", 1)),
- itsMinPerc (parset.getFloat (prefix+"minperc", 0.) / 100.),
- itsNTimes (0),
- itsTimeInterval (0),
- itsNoAvg (true)
- {
- string freqResolutionStr = parset.getString(prefix+"freqresolution","0");
- itsFreqResolution = getFreqHz(freqResolutionStr);
-
- if (itsFreqResolution > 0) {
- itsNChanAvg = 0; // Will be set later in updateinfo
- } else {
- itsNChanAvg = parset.getUint (prefix+"freqstep", 1);
- }
-
- itsTimeResolution = parset.getFloat(prefix+"timeresolution", 0.);
- if (itsTimeResolution > 0) {
- itsNTimeAvg = 0; // Will be set later in updateInfo
- } else {
- itsNTimeAvg = parset.getUint(prefix+"timestep", 1);
- }
- }
-
- Averager::Averager (DPInput* input, const string& stepName,
- uint nchanAvg, uint ntimeAvg)
- : itsInput (input),
- itsName (stepName),
- itsFreqResolution (0),
- itsTimeResolution (0),
- itsNChanAvg (nchanAvg),
- itsNTimeAvg (ntimeAvg),
- itsMinNPoint (1),
- itsMinPerc (0),
- itsNTimes (0),
- itsTimeInterval (0)
- {
- if (itsNChanAvg <= 0) itsNChanAvg = 1;
- if (itsNTimeAvg <= 0) itsNTimeAvg = 1;
- itsNoAvg = (itsNChanAvg == 1 && itsNTimeAvg == 1);
- }
-
- Averager::~Averager()
- {}
-
- void Averager::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- info().setWriteFlags();
- info().setMetaChanged();
-
- if (itsNChanAvg <= 0) {
- if (itsFreqResolution > 0) {
- double chanwidth = infoIn.chanWidths()[0];
- itsNChanAvg = std::max(1, (int)(itsFreqResolution / chanwidth + 0.5));
- } else {
- itsNChanAvg = 1;
- }
- }
-
- itsTimeInterval = infoIn.timeInterval();
- if (itsNTimeAvg <= 0) {
- if (itsTimeResolution > 0) {
- itsNTimeAvg = std::max(1, (int)(itsTimeResolution / itsTimeInterval + 0.5));
- } else {
- itsNTimeAvg = 1;
- }
- }
-
- itsNoAvg = (itsNChanAvg == 1 && itsNTimeAvg == 1);
-
- // Adapt averaging to available nr of channels and times.
- itsNTimeAvg = std::min (itsNTimeAvg, infoIn.ntime());
- itsNChanAvg = info().update (itsNChanAvg, itsNTimeAvg);
- }
-
- void Averager::show (std::ostream& os) const
- {
- os << "Averager " << itsName << std::endl;
- os << " freqstep: " << itsNChanAvg;
- if (itsFreqResolution>0) {
- os << " (set by freqresolution: " << itsFreqResolution << " Hz)" << std::endl;
- }
- os << " timestep: " << itsNTimeAvg;
- if (itsTimeResolution>0) {
- os << " (set by timeresolution: " << itsTimeResolution << ")";
- }
- os << std::endl;
- os << " minpoints: " << itsMinNPoint << std::endl;
- os << " minperc: " << 100*itsMinPerc << std::endl;
- }
-
- void Averager::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " Averager " << itsName << endl;
- }
-
- bool Averager::process (const DPBuffer& buf)
- {
- // Nothing needs to be done if no averaging.
- if (itsNoAvg) {
- getNextStep()->process (buf);
- return true;
- }
- itsTimer.start();
- // Sum the data in time applying the weights.
- // The summing in channel and the averaging is done in function average.
- if (itsNTimes == 0) {
- // The first time we assign because that is faster than first clearing
- // and adding thereafter.
- itsBuf.getData().assign (buf.getData());
- itsBuf.getFlags().assign (buf.getFlags());
- itsBuf.getUVW().assign (itsInput->fetchUVW(buf, itsBuf, itsTimer));
- itsBuf.getWeights().assign (itsInput->fetchWeights(buf, itsBuf, itsTimer));
- IPosition shapeIn = buf.getData().shape();
- itsNPoints.resize (shapeIn);
- itsAvgAll.reference (buf.getData() * itsBuf.getWeights());
- itsWeightAll.resize (shapeIn);
- itsWeightAll = itsBuf.getWeights();
- // Take care of the fullRes flags.
- // We have to shape the output array and copy to a part of it.
- const Cube<bool>& fullResFlags =
- itsInput->fetchFullResFlags (buf, itsBufTmp, itsTimer);
- IPosition ofShape = fullResFlags.shape();
- ofShape[1] *= itsNTimeAvg; // more time entries, same chan and bl
- itsBuf.getFullResFlags().resize (ofShape);
- itsBuf.getFullResFlags() = true; // initialize for times missing at end
- copyFullResFlags (fullResFlags, buf.getFlags(), 0);
- // Set middle of new interval.
- double time = buf.getTime() + 0.5*(itsNTimeAvg-1)*itsTimeInterval;
- itsBuf.setTime (time);
- itsBuf.setExposure (itsNTimeAvg*itsTimeInterval);
- // Only set.
- itsNPoints = 1;
- // Set flagged points to zero.
- Array<bool>::const_contiter infIter = buf.getFlags().cbegin();
- Array<Complex>::contiter dataIter = itsBuf.getData().cbegin();
- Array<float>::contiter wghtIter = itsBuf.getWeights().cbegin();
- Array<int>::contiter outnIter = itsNPoints.cbegin();
- Array<int>::contiter outnIterEnd = itsNPoints.cend();
- while (outnIter != outnIterEnd) {
- if (*infIter) {
- // Flagged data point
- *outnIter = 0;
- *dataIter = Complex();
- *wghtIter = 0;
- } else {
- // Weigh the data point
- *dataIter *= *wghtIter;
- }
- ++infIter;
- ++dataIter;
- ++wghtIter;
- ++outnIter;
- }
- } else {
- // Not the first time.
- // For now we assume that all timeslots have the same nr of baselines,
- // so check if the buffer sizes are the same.
- ASSERT (itsBuf.getData().shape() == buf.getData().shape());
- itsBufTmp.referenceFilled (buf);
- itsBuf.getUVW() += itsInput->fetchUVW (buf, itsBufTmp, itsTimer);
- copyFullResFlags (itsInput->fetchFullResFlags (buf, itsBufTmp, itsTimer),
- buf.getFlags(), itsNTimes);
- const Cube<float>& weights =
- itsInput->fetchWeights (buf, itsBufTmp, itsTimer);
- // Ignore flagged points.
- Array<Complex>::const_contiter indIter = buf.getData().cbegin();
- Array<float>::const_contiter inwIter = weights.cbegin();
- Array<bool>::const_contiter infIter = buf.getFlags().cbegin();
- Array<Complex>::contiter outdIter = itsBuf.getData().cbegin();
- Array<Complex>::contiter alldIter = itsAvgAll.cbegin();
- Array<float>::contiter outwIter = itsBuf.getWeights().cbegin();
- Array<float>::contiter allwIter = itsWeightAll.cbegin();
- Array<int>::contiter outnIter = itsNPoints.cbegin();
- Array<int>::contiter outnIterEnd = itsNPoints.cend();
- while (outnIter != outnIterEnd) {
- *alldIter += *indIter * *inwIter;
- *allwIter += *inwIter;
- if (!*infIter) {
- *outdIter += *indIter * *inwIter;
- *outwIter += *inwIter;
- (*outnIter)++;
- }
- ++indIter;
- ++inwIter;
- ++infIter;
- ++outdIter;
- ++alldIter;
- ++outwIter;
- ++allwIter;
- ++outnIter;
- }
- }
- // Do the averaging if enough time steps have been processed.
- itsNTimes += 1;
- if (itsNTimes >= itsNTimeAvg) {
- average();
- itsTimer.stop();
- getNextStep()->process (itsBufOut);
- itsNTimes = 0;
- } else {
- itsTimer.stop();
- }
- return true;
- }
-
- void Averager::finish()
- {
- // Average remaining entries.
- if (itsNTimes > 0) {
- itsTimer.start();
- average();
- itsTimer.stop();
- getNextStep()->process (itsBufOut);
- itsNTimes = 0;
- }
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void Averager::average()
- {
- IPosition shp = itsBuf.getData().shape();
- uint nchanin = shp[1];
- uint npin = shp[0] * nchanin;
- shp[1] = (shp[1] + itsNChanAvg - 1) / itsNChanAvg;
- itsBufOut.getData().resize (shp);
- itsBufOut.getWeights().resize (shp);
- itsBufOut.getFlags().resize (shp);
- uint ncorr = shp[0];
- uint nchan = shp[1];
- int nbl = shp[2];
- uint npout = ncorr * nchan;
- ///#pragma omp parallel for
- // GCC-4.3 only supports OpenMP 2.5 needing signed iteration variables.
- for (int k=0; k<nbl; ++k) {
- const Complex* indata = itsBuf.getData().data() + k*npin;
- const Complex* inalld = itsAvgAll.data() + k*npin;
- const float* inwght = itsBuf.getWeights().data() + k*npin;
- const float* inallw = itsWeightAll.data() + k*npin;
- const int* innp = itsNPoints.data() + k*npin;
- Complex* outdata = itsBufOut.getData().data() + k*npout;
- float* outwght = itsBufOut.getWeights().data() + k*npout;
- bool* outflags = itsBufOut.getFlags().data() + k*npout;
- for (uint i=0; i<ncorr; ++i) {
- uint inxi = i;
- uint inxo = i;
- for (uint ch=0; ch<nchan; ++ch) {
- uint nch = std::min(itsNChanAvg, nchanin - ch*itsNChanAvg);
- uint navgAll = nch * itsNTimes;
- Complex sumd;
- Complex sumad;
- float sumw = 0;
- float sumaw = 0;
- uint np = 0;
- for (uint j=0; j<nch; ++j) {
- sumd += indata[inxi]; // Note: weight is accounted for in process
- sumad += inalld[inxi];
- sumw += inwght[inxi];
- sumaw += inallw[inxi];
- np += innp[inxi];
- inxi += ncorr;
- }
- // Flag the point if insufficient unflagged data.
- if (sumw == 0 || np < itsMinNPoint || np < navgAll*itsMinPerc) {
- outdata[inxo] = (sumaw==0 ? Complex() : sumad/sumaw);
- outflags[inxo] = true;
- outwght[inxo] = sumaw;
- } else {
- outdata[inxo] = sumd / sumw;
- outflags[inxo] = false;
- outwght[inxo] = sumw;
- }
- inxo += ncorr;
- }
- }
- }
- // Set the remaining values in the output buffer.
- itsBufOut.setTime (itsBuf.getTime());
- itsBufOut.setExposure (itsBuf.getExposure());
- itsBufOut.setFullResFlags (itsBuf.getFullResFlags());
- // The result UVWs are the average of the input.
- // If ever needed, UVWCalculator can be used to calculate the UVWs.
- itsBufOut.setUVW (itsBuf.getUVW() / double(itsNTimes));
- }
-
- void Averager::copyFullResFlags (const Cube<bool>& fullResFlags,
- const Cube<bool>& flags,
- int timeIndex)
- {
- // Copy the fullRes flags to the given index.
- // Furthermore the appropriate FullRes flags are set for a
- // flagged data point. It can be the case that an input data point
- // has been averaged before, thus has fewer channels than FullResFlags.
- // nchan and nbl are the same for in and out.
- // ntimout is a multiple of ntimavg.
- IPosition shapeIn = fullResFlags.shape();
- IPosition shapeOut = itsBuf.getFullResFlags().shape();
- IPosition shapeFlg = flags.shape();
- uint nchan = shapeIn[0]; // original nr of channels
- uint ntimavg = shapeIn[1]; // nr of averaged times in input data
- uint nchanavg = nchan / shapeFlg[1]; // nr of avg chan in input data
- uint ntimout = shapeOut[1]; // nr of averaged times in output data
- int nbl = shapeIn[2]; // nr of baselines
- uint ncorr = shapeFlg[0]; // nr of correlations (in FLAG)
- // in has to be copied to the correct time index in out.
- bool* outBase = itsBuf.getFullResFlags().data() + nchan*ntimavg*timeIndex;
- ///#pragma omp parallel for
- // GCC-4.3 only supports OpenMP 2.5 needing signed iteration variables.
- for (int k=0; k<nbl; ++k) {
- const bool* inPtr = fullResFlags.data() + k*nchan*ntimavg;
- const bool* flagPtr = flags.data() + k*ncorr*shapeFlg[1];
- bool* outPtr = outBase + k*nchan*ntimout;
- memcpy (outPtr, inPtr, nchan*ntimavg*sizeof(bool));
- // Applying the flags only needs to be done if the input data
- // was already averaged before.
- if (ntimavg > 1 || nchanavg > 1) {
- for (int j=0; j<shapeFlg[1]; ++j) {
- // If a data point is flagged, the flags in the corresponding
- // FullRes window have to be set.
- // Only look at the flags of the first correlation.
- if (*flagPtr) {
- bool* avgPtr = outPtr + j*nchanavg;
- for (uint i=0; i<ntimavg; ++i) {
- std::fill (avgPtr, avgPtr+nchanavg, true);
- avgPtr += nchan;
- }
- }
- flagPtr += ncorr;
- }
- }
- }
- }
-
- double Averager::getFreqHz(const string& freqstr) {
- String unit;
- // See if a unit is given at the end.
- String v(freqstr);
- // Remove possible trailing blanks.
- rtrim(v);
- Regex regex("[a-zA-Z]+$");
- string::size_type pos = v.index (regex);
- if (pos != String::npos) {
- unit = v.from (pos);
- v = v.before (pos);
- }
- // Set value and unit.
-
- double value = strToDouble(v);
- if (unit.empty()) {
- return value;
- } else {
- Quantity q(value, unit);
- return q.getValue("Hz", true);
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/BaselineSelection.cc b/CEP/DP3/DPPP/src/BaselineSelection.cc
deleted file mode 100644
index 085a6cf0e85b607f05b688c359d101fe641455b5..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/BaselineSelection.cc
+++ /dev/null
@@ -1,272 +0,0 @@
-//# BaselineSelection.cc: Class to handle the baseline selection
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/BaselineSelection.h>
-#include <DPPP/DPLogger.h>
-#include <MS/BaselineSelect.h>
-#include <Common/ParameterSet.h>
-#include <Common/ParameterValue.h>
-#include <Common/LofarLogger.h>
-#include <Common/StringUtil.h>
-#include <Common/StreamUtil.h>
-
-using namespace casacore;
-using namespace std;
-
-namespace LOFAR {
- namespace DPPP {
-
- BaselineSelection::BaselineSelection()
- {}
-
- BaselineSelection::BaselineSelection (const ParameterSet& parset,
- const string& prefix,
- bool minmax,
- const string& defaultCorrType,
- const string& defaultBaseline)
- : itsStrBL (parset.getString (prefix + "baseline", defaultBaseline)),
- itsCorrType (parset.getString (prefix + "corrtype", defaultCorrType)),
- itsRangeBL (parset.getDoubleVector (prefix + "blrange",
- vector<double>()))
- {
- if (minmax) {
- double minbl = parset.getDouble (prefix + "blmin", -1);
- double maxbl = parset.getDouble (prefix + "blmax", -1);
- if (minbl > 0) {
- itsRangeBL.push_back (0.);
- itsRangeBL.push_back (minbl);
- }
- if (maxbl > 0) {
- itsRangeBL.push_back (maxbl);
- itsRangeBL.push_back (1e30);
- }
- }
- ASSERTSTR (itsRangeBL.size()%2 == 0,
- "NDPPP error: uneven number of lengths in baseline range");
- }
-
- bool BaselineSelection::hasSelection() const
- {
- return !((itsStrBL.empty() || itsStrBL == "[]") &&
- itsCorrType.empty() && itsRangeBL.empty());
- }
-
- void BaselineSelection::show (ostream& os, const string& blanks) const
- {
- os << " Baseline selection:" << std::endl;
- os << " baseline: " << blanks << itsStrBL << std::endl;
- os << " corrtype: " << blanks << itsCorrType << std::endl;
- os << " blrange: " << blanks << itsRangeBL << std::endl;
- }
-
- Matrix<bool> BaselineSelection::apply (const DPInfo& info) const
- {
- // Size and initialize the selection matrix.
- int nant = info.antennaNames().size();
- Matrix<bool> selectBL(nant, nant, true);
- // Apply the various parts if given.
- if (! itsStrBL.empty() && itsStrBL != "[]") {
- handleBL (selectBL, info);
- }
- if (! itsCorrType.empty()) {
- handleCorrType (selectBL);
- }
- if (! itsRangeBL.empty()) {
- handleLength (selectBL, info);
- }
- return selectBL;
- }
-
- Vector<bool> BaselineSelection::applyVec (const DPInfo& info) const
- {
- Matrix<bool> sel = apply(info);
- Vector<bool> vec;
- vec.resize (info.nbaselines());
- for (uint i=0; i<info.nbaselines(); ++i) {
- vec[i] = sel(info.getAnt1()[i], info.getAnt2()[i]);
- }
- return vec;
- }
-
- void BaselineSelection::handleBL (Matrix<bool>& selectBL,
- const DPInfo& info) const
- {
- // Handle the value(s) in the baseline selection string.
- ParameterValue pvBL(itsStrBL);
- // The value can be a vector or an MSSelection string.
- // Alas the ParameterValue vector test cannot be used, because
- // the first character of a MSSelection string can also be [.
- // So if the first is [ and a ] is found before the end and before
- // another [, it must be a MSSelection string.
- bool mssel = true;
- if (itsStrBL[0] == '[') {
- String::size_type rb = itsStrBL.find (']');
- ASSERTSTR (rb != string::npos,
- "Baseline selection " + itsStrBL +
- " has no ending ]");
- if (rb == itsStrBL.size()-1) {
- mssel = false;
- } else {
- String::size_type lb = itsStrBL.find ('[', 1);
- mssel = (lb == string::npos || lb > rb);
- }
- }
- if (!mssel) {
- // Specified as a vector of antenna name patterns.
- selectBL = selectBL && handleBLVector (pvBL, info.antennaNames());
- } else {
- // Specified in casacore's MSSelection format.
- string msName = info.msName();
- ASSERT (! msName.empty());
- std::ostringstream os;
- Matrix<bool> sel(BaselineSelect::convert (msName, itsStrBL, os));
- // Show possible messages about unknown stations.
- if (! os.str().empty()) {
- vector<string> messages = StringUtil::split (os.str(), '\n');
- for (size_t i=0; i<messages.size(); ++i) {
- DPLOG_WARN_STR (messages[i]);
- }
- }
- // The resulting matrix can be smaller because new stations might have
- // been added that are not present in the MS's ANTENNA table.
- if (sel.nrow() == selectBL.nrow()) {
- selectBL = selectBL && sel;
- } else {
- // Only and the subset.
- Matrix<bool> selBL = selectBL(IPosition(2,0),
- IPosition(2,sel.nrow()-1));
- selBL = selBL && sel;
- }
- }
- }
-
- Matrix<bool> BaselineSelection::handleBLVector (const ParameterValue& pvBL,
- const Vector<String>& antNames) const
- {
- Matrix<Bool> sel(antNames.size(), antNames.size());
- sel = false;
- vector<ParameterValue> pairs = pvBL.getVector();
- // Each ParameterValue can be a single value (antenna) or a pair of
- // values (a baseline).
- // Note that [ant1,ant2] is somewhat ambiguous; it means two antennae,
- // but one might think it means a baseline [[ant1,ant2]].
- if (pairs.size() == 2 &&
- !(pairs[0].isVector() || pairs[1].isVector())) {
- LOG_WARN_STR ("PreFlagger baseline " << pvBL.get()
- << " means two antennae, but is somewhat ambigious; "
- << "it's more clear to use [[ant1],[ant2]]");
- }
- for (uint i=0; i<pairs.size(); ++i) {
- vector<string> bl = pairs[i].getStringVector();
- if (bl.size() == 1) {
- // Turn the given antenna name pattern into a regex.
- Regex regex(Regex::fromPattern (bl[0]));
- int nmatch = 0;
- // Loop through all antenna names and set matrix for matching ones.
- for (uint i2=0; i2<antNames.size(); ++i2) {
- if (antNames[i2].matches (regex)) {
- nmatch++;
- // Antenna matches, so set all corresponding flags.
- for (uint j=0; j<antNames.size(); ++j) {
- sel(i2,j) = true;
- sel(j,i2) = true;
- }
- }
- }
- if (nmatch == 0) {
- DPLOG_WARN_STR ("PreFlagger: no matches for antenna name pattern ["
- << bl[0] << "]");
- }
- } else {
- ASSERTSTR (bl.size() == 2, "PreFlagger baseline " << bl <<
- " should contain 1 or 2 antenna name patterns");
- // Turn the given antenna name pattern into a regex.
- Regex regex1(Regex::fromPattern (bl[0]));
- Regex regex2(Regex::fromPattern (bl[1]));
- int nmatch = 0;
- // Loop through all antenna names and set matrix for matching ones.
- for (uint i2=0; i2<antNames.size(); ++i2) {
- if (antNames[i2].matches (regex2)) {
- // Antenna2 matches, now try Antenna1.
- for (uint i1=0; i1<antNames.size(); ++i1) {
- if (antNames[i1].matches (regex1)) {
- nmatch++;
- sel(i1,i2) = true;
- sel(i2,i1) = true;
- }
- }
- }
- }
- if (nmatch == 0) {
- DPLOG_WARN_STR ("PreFlagger: no matches for baseline name pattern ["
- << bl[0] << ',' << bl[1] << "]");
- }
- }
- }
- return sel;
- }
-
- void BaselineSelection::handleCorrType (Matrix<bool>& selectBL) const
- {
- // Process corrtype if given.
- string corrType = toLower(itsCorrType);
- ASSERTSTR (corrType == "auto" || corrType == "cross",
- "NDPPP corrType " << corrType
- << " is invalid; must be auto, cross, or empty string");
- if (corrType == "auto") {
- Vector<bool> diag = selectBL.diagonal().copy();
- selectBL = false;
- selectBL.diagonal() = diag;
- } else {
- selectBL.diagonal() = false;
- }
- }
-
- void BaselineSelection::handleLength (Matrix<bool>& selectBL,
- const DPInfo& info) const
- {
- // Get baseline lengths.
- const vector<double>& blength = info.getBaselineLengths();
- const Vector<Int>& ant1 = info.getAnt1();
- const Vector<Int>& ant2 = info.getAnt2();
- for (uint i=0; i<ant1.size(); ++i) {
- // Clear selection if no range matches.
- bool match = false;
- for (uint j=0; j<itsRangeBL.size(); j+=2) {
- if (blength[i] >= itsRangeBL[j] && blength[i] <= itsRangeBL[j+1]) {
- match = true;
- break;
- }
- }
- if (!match) {
- int a1 = ant1[i];
- int a2 = ant2[i];
- selectBL(a1,a2) = false;
- selectBL(a2,a1) = false;
- }
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/CMakeLists.txt b/CEP/DP3/DPPP/src/CMakeLists.txt
deleted file mode 100644
index cb4eb74985e8350cae09c0645126edb1334f73fc..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/CMakeLists.txt
+++ /dev/null
@@ -1,40 +0,0 @@
-# $Id$
-
-include(LofarPackageVersion)
-include(PythonInstall)
-
-lofar_add_library(dppp
- Package__Version.cc
- DPRun.cc DPStep.cc DPInput.cc DPBuffer.cc DPInfo.cc
- DPLogger.cc ProgressMeter.cc FlagCounter.cc
- UVWCalculator/UVWCalculator.cc BaselineSelection.cc ApplyCal.cc
- MSReader.cc MultiMSReader.cc MSWriter.cc MSUpdater.cc Counter.cc
- Averager.cc MedFlagger.cc PreFlagger.cc UVWFlagger.cc
- StationAdder.cc ScaleData.cc Filter.cc PhaseShift.cc
- Demixer.cc
- Position.cc Stokes.cc SourceDBUtil.cc
- Apply.cc EstimateMixed.cc EstimateNew.cc Simulate.cc Simulator.cc
- SubtractMixed.cc SubtractNew.cc
- ModelComponent.cc PointSource.cc GaussianSource.cc Patch.cc
- ModelComponentVisitor.cc GainCal.cc StefCal.cc
- DemixerNew.cc DemixInfo.cc DemixWorker.cc
- Predict.cc OneApplyCal.cc
- ApplyBeam.cc
- PhaseFitter.cc H5Parm.cc SolTab.cc DummyStep.cc H5ParmPredict.cc GridInterpolate.cc
- Upsample.cc Split.cc
-)
-
-lofar_add_bin_program(NDPPP NDPPP.cc)
-lofar_add_bin_program(versiondppp versiondppp.cc)
-
-install(CODE "EXECUTE_PROCESS(
- COMMAND \"${CMAKE_COMMAND}\" -E create_symlink NDPPP DPPP WORKING_DIRECTORY ${CMAKE_INSTALL_PREFIX}/bin
- )"
-)
-
-lofar_add_bin_scripts(taqlflagger)
-
-# Python modules.
-python_install(
- __init__.py
- DESTINATION lofar/dppp)
diff --git a/CEP/DP3/DPPP/src/Counter.cc b/CEP/DP3/DPPP/src/Counter.cc
deleted file mode 100644
index 7f25db6c9e957082a807155e6e98a4f18f3c6629..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Counter.cc
+++ /dev/null
@@ -1,101 +0,0 @@
-//# Counter.cc: DPPP step class to count flags
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Counter.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <iostream>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- Counter::Counter (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsName (prefix),
- itsCount (0),
- itsFlagCounter (input->msName(), parset, prefix)
- {
- itsFlagData = parset.getBool (prefix+"flagdata", false);
- }
-
- Counter::~Counter()
- {}
-
- void Counter::show (std::ostream& os) const
- {
- os << "Counter " << itsName << std::endl;
- }
-
- void Counter::showCounts (std::ostream& os) const
- {
- os << endl << "Cumulative flag counts in Counter " << itsName;
- os << endl << "=================================" << endl;
- itsFlagCounter.showBaseline (os, itsCount);
- itsFlagCounter.showChannel (os, itsCount);
- }
-
- void Counter::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- // Visibility data must be read if needed, so NaNs are flagged.
- if (itsFlagData) {
- info().setNeedVisData();
- }
- // Initialize the flag counters.
- itsFlagCounter.init (infoIn);
- }
-
- bool Counter::process (const DPBuffer& buf)
- {
- const IPosition& shape = buf.getFlags().shape();
- uint nrcorr = shape[0];
- uint nrchan = shape[1];
- uint nrbl = shape[2];
- const bool* flagPtr = buf.getFlags().data();
- for (uint i=0; i<nrbl; ++i) {
- for (uint j=0; j<nrchan; ++j) {
- if (*flagPtr) {
- itsFlagCounter.incrBaseline(i);
- itsFlagCounter.incrChannel(j);
- }
- flagPtr += nrcorr; // only count 1st corr
- }
- }
- // Let the next step do its processing.
- getNextStep()->process (buf);
- itsCount++;
- return true;
- }
-
- void Counter::finish()
- {
- // Let the next step finish its processing.
- getNextStep()->finish();
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/DPBuffer.cc b/CEP/DP3/DPPP/src/DPBuffer.cc
deleted file mode 100644
index 27f2c9a2e787e085cf8685bb0c6bd67e38b39636..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DPBuffer.cc
+++ /dev/null
@@ -1,143 +0,0 @@
-//# DPBuffer.cc: Buffer holding the data of a timeslot/band
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <DPPP/DPBuffer.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- DPBuffer::DPBuffer()
- : itsTime (0),
- itsExposure (0)
- {}
-
- DPBuffer::DPBuffer (const DPBuffer& that)
- {
- operator= (that);
- }
-
- DPBuffer& DPBuffer::operator= (const DPBuffer& that)
- {
- if (this != &that) {
- itsTime = that.itsTime;
- itsExposure = that.itsExposure;
- itsRowNrs.reference (that.itsRowNrs);
- itsData.reference (that.itsData);
- itsFlags.reference (that.itsFlags);
- itsWeights.reference (that.itsWeights);
- itsUVW.reference (that.itsUVW);
- itsFullResFlags.reference (that.itsFullResFlags);
- }
- return *this;
- }
-
- void DPBuffer::copy (const DPBuffer& that)
- {
- if (this != &that) {
- itsTime = that.itsTime;
- itsExposure = that.itsExposure;
- itsRowNrs.assign (that.itsRowNrs);
- if (! that.itsData.empty()) {
- itsData.assign (that.itsData);
- }
- if (! that.itsFlags.empty()) {
- itsFlags.assign (that.itsFlags);
- }
- if (! that.itsWeights.empty()) {
- itsWeights.assign (that.itsWeights);
- }
- if (! that.itsUVW.empty()) {
- itsUVW.assign (that.itsUVW);
- }
- if (! that.itsFullResFlags.empty()) {
- itsFullResFlags.assign (that.itsFullResFlags);
- }
- }
- }
-
- void DPBuffer::referenceFilled (const DPBuffer& that)
- {
- if (this != &that) {
- itsTime = that.itsTime;
- itsExposure = that.itsExposure;
- itsRowNrs.reference (that.itsRowNrs);
- if (! that.itsData.empty()) {
- itsData.reference (that.itsData);
- }
- if (! that.itsFlags.empty()) {
- itsFlags.reference (that.itsFlags);
- }
- if (! that.itsWeights.empty()) {
- itsWeights.reference (that.itsWeights);
- }
- if (! that.itsUVW.empty()) {
- itsUVW.reference (that.itsUVW);
- }
- if (! that.itsFullResFlags.empty()) {
- itsFullResFlags.reference (that.itsFullResFlags);
- }
- }
- }
-
- void DPBuffer::mergeFullResFlags (Cube<bool>& fullResFlags,
- const Cube<bool>& flags)
- {
- // Flag shape is [ncorr, newnchan, nbl].
- // FullRes shape is [orignchan, navgtime, nbl]
- // where orignchan = navgchan * newnchan.
- const IPosition& fullResShape = fullResFlags.shape();
- const IPosition& flagShape = flags.shape();
- int orignchan = fullResShape[0];
- int newnchan = flagShape[1];
- int navgchan = orignchan / newnchan;
- int navgtime = fullResShape[1];
- int nbl = fullResShape[2];
- int ncorr = flagShape[0];
- bool* fullResPtr = fullResFlags.data();
- const bool* flagPtr = flags.data();
- // Loop over all baselines and new channels.
- // Only use the first correlation in the loop.
- for (int j=0; j<nbl; ++j) {
- for (int i=0; i<newnchan; ++i) {
- // If ta data point is flagged, the flags in the corresponding
- // FullRes window have to be set.
- // This is needed in case a data point is further averaged.
- if (*flagPtr) {
- for (int i=0; i<navgtime; ++i) {
- std::fill (fullResPtr, fullResPtr+navgchan, true);
- fullResPtr += orignchan;
- }
- fullResPtr -= navgtime*orignchan;
- }
- flagPtr += ncorr;
- fullResPtr += navgchan;
- }
- // Set pointer to next baseline.
- fullResPtr += (navgtime-1)*orignchan;
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/DPInfo.cc b/CEP/DP3/DPPP/src/DPInfo.cc
deleted file mode 100644
index 03578710ea5554aac797011d9b9537eb18abddd5..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DPInfo.cc
+++ /dev/null
@@ -1,434 +0,0 @@
-//# DPInfo.cc: General info about DPPP data processing attributes like averaging
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPInput.h>
-#include <Common/LofarLogger.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MCPosition.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <casacore/casa/BasicSL/STLIO.h>
-
-using namespace casacore;
-using namespace std;
-
-namespace LOFAR {
- namespace DPPP {
-
- DPInfo::DPInfo()
- : itsNeedVisData (false),
- itsWriteData (false),
- itsWriteFlags (false),
- itsWriteWeights (false),
- itsMetaChanged (false),
- itsNCorr (0),
- itsStartChan (0),
- itsNChan (0),
- itsChanAvg (1),
- itsNTime (0),
- itsTimeAvg (1),
- itsStartTime (0),
- itsTimeInterval (0),
- itsPhaseCenterIsOriginal (true)
- {}
-
- void DPInfo::init (uint ncorr, uint nchan,
- uint ntime, double startTime, double timeInterval,
- const string& msName, const string& antennaSet)
- {
- itsNCorr = ncorr;
- itsNChan = nchan;
- itsOrigNChan = nchan;
- itsNTime = ntime;
- itsStartTime = startTime;
- itsTimeInterval = timeInterval;
- itsMSName = msName;
- itsAntennaSet = antennaSet;
- }
-
- void DPInfo::set (const Vector<double>& chanFreqs,
- const Vector<double>& chanWidths,
- const Vector<double>& resolutions,
- const Vector<double>& effectiveBW,
- double totalBW, double refFreq)
- {
- itsChanFreqs.reference (chanFreqs);
- itsChanWidths.reference (chanWidths);
- if (resolutions.size() == 0) {
- itsResolutions.reference (chanWidths);
- } else {
- itsResolutions.reference (resolutions);
- }
- if (effectiveBW.size() == 0) {
- itsEffectiveBW.reference (chanWidths);
- } else {
- itsEffectiveBW.reference (effectiveBW);
- }
- if (totalBW == 0) {
- itsTotalBW = sum(itsEffectiveBW);
- } else {
- itsTotalBW = totalBW;
- }
- if (refFreq == 0) {
- int n = itsChanFreqs.size();
- // Takes mean of middle elements if n is even; takes middle if odd.
- itsRefFreq = 0.5 * (itsChanFreqs[(n-1)/2] + itsChanFreqs[n/2]);
- } else {
- itsRefFreq = refFreq;
- }
- }
-
- void DPInfo::set (const MPosition& arrayPos,
- const MDirection& phaseCenter,
- const MDirection& delayCenter,
- const MDirection& tileBeamDir)
- {
- itsArrayPos = arrayPos;
- itsPhaseCenter = phaseCenter;
- itsDelayCenter = delayCenter;
- itsTileBeamDir = tileBeamDir;
- }
-
- void DPInfo::set (const Vector<String>& antNames,
- const Vector<Double>& antDiam,
- const vector<MPosition>& antPos,
- const Vector<Int>& ant1,
- const Vector<Int>& ant2)
- {
- ASSERT (antNames.size() == antDiam.size() &&
- antNames.size() == antPos.size());
- ASSERT (ant1.size() == ant2.size());
- itsAntNames.reference (antNames);
- itsAntDiam.reference (antDiam);
- itsAntPos = antPos;
- itsAnt1.reference (ant1);
- itsAnt2.reference (ant2);
- // Set which antennae are used.
- setAntUsed();
- }
-
- void DPInfo::setAntUsed()
- {
- itsAntUsed.clear();
- itsAntMap.resize (itsAntNames.size());
- std::fill (itsAntMap.begin(), itsAntMap.end(), -1);
- for (uint i=0; i<itsAnt1.size(); ++i) {
- ASSERT (itsAnt1[i] >= 0 && itsAnt1[i] < int(itsAntMap.size()) &&
- itsAnt2[i] >= 0 && itsAnt2[i] < int(itsAntMap.size()));
- itsAntMap[itsAnt1[i]] = 0;
- itsAntMap[itsAnt2[i]] = 0;
- }
- itsAntUsed.reserve (itsAntNames.size());
- for (uint i=0; i<itsAntMap.size(); ++i) {
- if (itsAntMap[i] == 0) {
- itsAntMap[i] = itsAntUsed.size();
- itsAntUsed.push_back (i);
- }
- }
- }
-
- MeasureHolder DPInfo::copyMeasure(const MeasureHolder fromMeas)
- {
- Record rec;
- String msg;
- ASSERT (fromMeas.toRecord (msg, rec));
- MeasureHolder mh2;
- ASSERT (mh2.fromRecord (msg, rec));
- return mh2;
- }
-
- uint DPInfo::update (uint chanAvg, uint timeAvg)
- {
- if (chanAvg > itsNChan) {
- chanAvg = itsNChan;
- }
- if (timeAvg > itsNTime) {
- timeAvg = itsNTime;
- }
- ASSERTSTR (itsNChan % chanAvg == 0,
- "When averaging, nr of channels must divide integrally; "
- "itsNChan=" << itsNChan << " chanAvg=" << chanAvg);
- itsChanAvg *= chanAvg;
- itsNChan = (itsNChan + chanAvg - 1) / chanAvg;
- itsTimeAvg *= timeAvg;
- itsNTime = (itsNTime + timeAvg - 1) / timeAvg;
- itsTimeInterval *= timeAvg;
- Vector<double> freqs(itsNChan);
- Vector<double> widths(itsNChan, 0.);
- Vector<double> resols(itsNChan, 0.);
- Vector<double> effBWs(itsNChan, 0.);
- double totBW = 0;
- for (uint i=0; i<itsNChan; ++i) {
- freqs[i] = 0.5 * (itsChanFreqs[i*chanAvg] +
- itsChanFreqs[(i+1)*chanAvg - 1]);
- for (uint j=0; j<chanAvg; ++j) {
- widths[i] += itsChanWidths[i*chanAvg+j];
- resols[i] += itsResolutions[i*chanAvg+j];
- effBWs[i] += itsEffectiveBW[i*chanAvg+j];
- }
- totBW += effBWs[i];
- }
- itsChanFreqs.reference (freqs);
- itsChanWidths.reference (widths);
- itsResolutions.reference (resols);
- itsEffectiveBW.reference (effBWs);
- itsTotalBW = totBW;
- return chanAvg;
- }
-
- void DPInfo::update (uint startChan, uint nchan,
- const vector<uint>& baselines, bool removeAnt)
- {
- Slice slice(startChan, nchan);
- itsStartChan=startChan;
- itsChanFreqs.reference (itsChanFreqs (slice).copy());
- itsChanWidths.reference (itsChanWidths(slice).copy());
- itsResolutions.reference (itsResolutions(slice).copy());
- itsEffectiveBW.reference (itsEffectiveBW(slice).copy());
- itsNChan = nchan;
- // Keep only selected baselines.
- if (! baselines.empty()) {
- Vector<Int> ant1 (baselines.size());
- Vector<Int> ant2 (baselines.size());
- for (uint i=0; i<baselines.size(); ++i) {
- ant1[i] = itsAnt1[baselines[i]];
- ant2[i] = itsAnt2[baselines[i]];
- }
- itsAnt1.reference (ant1);
- itsAnt2.reference (ant2);
- // Clear; they'll be recalculated if needed.
- itsBLength.resize (0);
- itsAutoCorrIndex.resize (0);
- }
- setAntUsed();
- // If needed, remove the stations and renumber the baselines.
- if (removeAnt) {
- removeUnusedAnt();
- }
- }
-
- void DPInfo::removeUnusedAnt()
- {
- if (itsAntUsed.size() < itsAntMap.size()) {
- // First remove stations.
- Vector<String> antNames (itsAntUsed.size());
- Vector<Double> antDiam (itsAntUsed.size());
- vector<MPosition> antPos;
- antPos.reserve (itsAntUsed.size());
- for (uint i=0; i<itsAntUsed.size(); ++i) {
- antNames[i] = itsAntNames[itsAntUsed[i]];
- antDiam[i] = itsAntDiam[itsAntUsed[i]];
- antPos.push_back (itsAntPos[itsAntUsed[i]]);
- }
- // Use the new vectors.
- itsAntNames.reference (antNames);
- itsAntDiam.reference (antDiam);
- itsAntPos.swap (antPos);
- // Renumber the baselines.
- for (uint i=0; i<itsAnt1.size(); ++i) {
- itsAnt1[i] = itsAntMap[itsAnt1[i]];
- itsAnt2[i] = itsAntMap[itsAnt2[i]];
- }
- // Now fill the itsAntUsed and itsAntMap vectors again.
- setAntUsed();
- // Clear; they'll be recalculated if needed.
- itsBLength.resize (0);
- itsAutoCorrIndex.resize (0);
- }
- }
-
- const vector<double>& DPInfo::getBaselineLengths() const
- {
- // Calculate the baseline lengths if not done yet.
- if (itsBLength.empty()) {
- // First get the antenna positions.
- const vector<MPosition>& antPos = antennaPos();
- vector<Vector<double> > antVec;
- antVec.reserve (antPos.size());
- for (vector<MPosition>::const_iterator iter = antPos.begin();
- iter != antPos.end(); ++iter) {
- // Convert to ITRF and keep as x,y,z in m.
- antVec.push_back
- (MPosition::Convert(*iter, MPosition::ITRF)().getValue().getValue());
- }
- // Fill in the length of each baseline.
- vector<double> blength;
- itsBLength.reserve (itsAnt1.size());
- for (uint i=0; i<itsAnt1.size(); ++i) {
- Array<double> diff(antVec[itsAnt2[i]] - antVec[itsAnt1[i]]);
- itsBLength.push_back (sqrt(sum(diff*diff)));
- }
- }
- return itsBLength;
- }
-
- const vector<int>& DPInfo::getAutoCorrIndex() const
- {
- if (itsAutoCorrIndex.empty()) {
- int nant = 1 + std::max(max(itsAnt1), max(itsAnt2));
- itsAutoCorrIndex.resize (nant);
- std::fill (itsAutoCorrIndex.begin(), itsAutoCorrIndex.end(), -1);
- // Keep the baseline table index for the autocorrelations.
- for (uint i=0; i<itsAnt1.size(); ++i) {
- if (itsAnt1[i] == itsAnt2[i]) {
- itsAutoCorrIndex[itsAnt1[i]] = i;
- }
- }
- }
- return itsAutoCorrIndex;
- }
-
- Record DPInfo::toRecord() const
- {
- Record rec;
- rec.define ("NeedVisData", itsNeedVisData);
- rec.define ("WriteData", itsWriteData);
- rec.define ("WriteFlags", itsWriteFlags);
- rec.define ("WriteWeights", itsWriteWeights);
- rec.define ("MetaChanged", itsMetaChanged);
- rec.define ("MSName", itsMSName);
- rec.define ("AntennaSet", itsAntennaSet);
- rec.define ("NCorr", itsNCorr);
- rec.define ("StartChan", itsStartChan);
- rec.define ("OrigNChan", itsOrigNChan);
- rec.define ("NChan", itsNChan);
- rec.define ("ChanAvg", itsChanAvg);
- rec.define ("NTime", itsNTime);
- rec.define ("TimeAvg", itsTimeAvg);
- rec.define ("StartTime", itsStartTime);
- rec.define ("TimeInterval", itsTimeInterval);
- rec.define ("ChanFreqs", itsChanFreqs);
- rec.define ("ChanWidths", itsChanWidths);
- rec.define ("Resolutions", itsResolutions);
- rec.define ("EffectiveBW", itsEffectiveBW);
- rec.define ("TotalBW", itsTotalBW);
- rec.define ("RefFreq", itsRefFreq);
- rec.define ("AntNames", itsAntNames);
- rec.define ("AntDiam", itsAntDiam);
- rec.define ("AntUsed", Vector<int>(itsAntUsed));
- rec.define ("AntMap", Vector<int>(itsAntMap));
- rec.define ("Ant1", itsAnt1);
- rec.define ("Ant2", itsAnt2);
- rec.define ("BLength", Vector<double>(itsBLength));
- rec.define ("AutoCorrIndex", Vector<int>(itsAutoCorrIndex));
- return rec;
- }
-
- void DPInfo::fromRecord (const Record& rec)
- {
- if (rec.isDefined ("NeedVisData")) {
- rec.get ("NeedVisData", itsNeedVisData);
- }
- if (rec.isDefined ("WriteData")) {
- rec.get ("WriteData", itsWriteData);
- }
- if (rec.isDefined ("WriteFlags")) {
- rec.get ("WriteFlags", itsWriteFlags);
- }
- if (rec.isDefined ("WriteWeights")) {
- rec.get ("WriteWeights", itsWriteWeights);
- }
- if (rec.isDefined ("MetaChanged")) {
- rec.get ("MetaChanged", itsMetaChanged);
- }
- if (rec.isDefined ("MSName")) {
- itsMSName = rec.asString ("MSName");
- }
- if (rec.isDefined ("AntennaSet")) {
- itsAntennaSet = rec.asString ("AntennaSet");
- }
- if (rec.isDefined ("NCorr")) {
- rec.get ("NCorr", itsNCorr);
- }
- if (rec.isDefined ("StartChan")) {
- rec.get ("StartChan", itsStartChan);
- }
- if (rec.isDefined ("OrigNChan")) {
- rec.get ("OrigNChan", itsOrigNChan);
- }
- if (rec.isDefined ("NChan")) {
- rec.get ("NChan", itsNChan);
- }
- if (rec.isDefined ("ChanAvg")) {
- rec.get ("ChanAvg", itsChanAvg);
- }
- if (rec.isDefined ("NTime")) {
- rec.get ("NTime", itsNTime);
- }
- if (rec.isDefined ("TimeAvg")) {
- rec.get ("TimeAvg", itsTimeAvg);
- }
- if (rec.isDefined ("StartTime")) {
- rec.get ("StartTime", itsStartTime);
- }
- if (rec.isDefined ("TimeInterval")) {
- rec.get ("TimeInterval", itsTimeInterval);
- }
- if (rec.isDefined ("ChanFreqs")) {
- rec.get ("ChanFreqs", itsChanFreqs);
- }
- if (rec.isDefined ("ChanWidths")) {
- rec.get ("ChanWidths", itsChanWidths);
- }
- if (rec.isDefined ("Resolutions")) {
- rec.get ("Resolutions", itsResolutions);
- }
- if (rec.isDefined ("EffectiveBW")) {
- rec.get ("EffectiveBW", itsEffectiveBW);
- }
- if (rec.isDefined ("TotalBW")) {
- rec.get ("TotalBW", itsTotalBW);
- }
- if (rec.isDefined ("RefFreq")) {
- rec.get ("RefFreq", itsRefFreq);
- }
- if (rec.isDefined ("AntNames")) {
- rec.get ("AntNames", itsAntNames);
- }
- if (rec.isDefined ("AntDiam")) {
- rec.get ("AntDiam", itsAntDiam);
- }
- ///if (rec.isDefined ("AntUsed")) {
- ///itsAntUsed = rec.toArrayInt("AntUsed").tovector();
- ///}
- ///if (rec.isDefined ("AntMap")) {
- /// itsAntMap = rec.toArrayInt("AntMap").tovector();
- ///}
- if (rec.isDefined ("Ant1")) {
- rec.get ("Ant1", itsAnt1);
- }
- if (rec.isDefined ("Ant2")) {
- rec.get ("Ant2", itsAnt2);
- }
- ///if (rec.isDefined ("BLength")) {
- /// itsBLength = rec.toArrayDouble("BLength").tovector();
- ///}
- ///if (rec.isDefined ("AutoCorrIndex")) {
- /// itsAutoCorrIndex = rec.toArrayInt("AutoCorrIndex").tovector();
- ///}
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/DPInput.cc b/CEP/DP3/DPPP/src/DPInput.cc
deleted file mode 100644
index 93a19b23e55926867596b5072c80045421dc8215..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DPInput.cc
+++ /dev/null
@@ -1,127 +0,0 @@
-//# DPInput.cc: Abstract base class for a DPStep generating input
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DPInput.h>
-#include <Common/Exception.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MPosition.h>
-#include <casacore/measures/Measures/MCPosition.h>
-#include <casacore/casa/Utilities/Copy.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- DPInput::~DPInput()
- {}
-
- casacore::String DPInput::msName() const
- {
- return String();
- }
-
- const Cube<bool>& DPInput::fetchFullResFlags (const DPBuffer& bufin,
- DPBuffer& bufout,
- NSTimer& timer,
- bool merge)
- {
- // If already defined in the buffer, return those fullRes flags.
- if (! bufin.getFullResFlags().empty()) {
- return bufin.getFullResFlags();
- }
- // No fullRes flags in buffer, so get them from the input.
- timer.stop();
- bool fnd = getFullResFlags (bufin.getRowNrs(), bufout);
- timer.start();
- Cube<bool>& fullResFlags = bufout.getFullResFlags();
- if (!fnd) {
- // No fullRes flags in input; form them from the flags in the buffer.
- // Only use the XX flags; no averaging done, thus navgtime=1.
- // (If any averaging was done, the flags would be in the buffer).
- IPosition shp(bufin.getFlags().shape());
- ASSERT (fullResFlags.shape()[0] == shp[1] &&
- fullResFlags.shape()[1] == 1 &&
- fullResFlags.shape()[2] == shp[2]);
- objcopy (fullResFlags.data(), bufin.getFlags().data(),
- fullResFlags.size(), 1, shp[0]); // only copy XX.
- return fullResFlags;
- }
- // There are fullRes flags.
- // If needed, merge them with the buffer's flags.
- if (merge) {
- DPBuffer::mergeFullResFlags (fullResFlags, bufin.getFlags());
- }
- return fullResFlags;
- }
-
- const Cube<float>& DPInput::fetchWeights (const DPBuffer& bufin,
- DPBuffer& bufout,
- NSTimer& timer)
- {
- // If already defined in the buffer, return those weights.
- if (! bufin.getWeights().empty()) {
- return bufin.getWeights();
- }
- // No weights in buffer, so get them from the input.
- // It might need the data and flags in the buffer.
- timer.stop();
- getWeights (bufin.getRowNrs(), bufout);
- timer.start();
- return bufout.getWeights();
- }
-
- const Matrix<double>& DPInput::fetchUVW (const DPBuffer& bufin,
- DPBuffer& bufout,
- NSTimer& timer)
- {
- // If already defined in the buffer, return those UVW.
- if (! bufin.getUVW().empty()) {
- return bufin.getUVW();
- }
- // No UVW in buffer, so get them from the input.
- timer.stop();
- getUVW (bufin.getRowNrs(), bufin.getTime(), bufout);
- timer.start();
- return bufout.getUVW();
- }
-
- void DPInput::getUVW (const RefRows&, double, DPBuffer&)
- { throw Exception ("DPInput::getUVW not implemented"); }
-
- void DPInput::getWeights (const RefRows&, DPBuffer&)
- { throw Exception ("DPInput::getWeights not implemented"); }
-
- bool DPInput::getFullResFlags (const RefRows&, DPBuffer&)
- { throw Exception ("DPInput::getFullResFlags not implemented"); }
-
- void DPInput::getModelData (const RefRows&, Cube<Complex>&)
- { throw Exception ("DPInput::getModelData not implemented"); }
-
- void DPInput::fillBeamInfo (vector<StationResponse::Station::Ptr>&,
- const Vector<String>&)
- { throw Exception ("DPInput::fillBeamInfo not implemented"); }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/DPLogger.cc b/CEP/DP3/DPPP/src/DPLogger.cc
deleted file mode 100644
index 6efa9ea631718aa10c565ca3f16370ef3ed9a6d2..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DPLogger.cc
+++ /dev/null
@@ -1,31 +0,0 @@
-//# DPLogger.cc: Log on cout/cerr or through the logging system
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DPLogger.h>
-
-namespace LOFAR {
- namespace DPPP {
- bool DPLogger::useLogger = false;
- }
-}
diff --git a/CEP/DP3/DPPP/src/DPRun.cc b/CEP/DP3/DPPP/src/DPRun.cc
deleted file mode 100644
index 809e2b8220f56388d316d5332b42071dd167e73b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DPRun.cc
+++ /dev/null
@@ -1,427 +0,0 @@
-//# DPRun.cc: Class to run steps like averaging and flagging on an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DPRun.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/MSReader.h>
-#include <DPPP/MultiMSReader.h>
-#include <DPPP/MSWriter.h>
-#include <DPPP/MSUpdater.h>
-#include <DPPP/ApplyBeam.h>
-#include <DPPP/Averager.h>
-#include <DPPP/MedFlagger.h>
-#include <DPPP/PreFlagger.h>
-#include <DPPP/UVWFlagger.h>
-#include <DPPP/PhaseShift.h>
-#include <DPPP/Demixer.h>
-#include <DPPP/DemixerNew.h>
-#include <DPPP/StationAdder.h>
-#include <DPPP/ScaleData.h>
-#include <DPPP/ApplyCal.h>
-#include <DPPP/Predict.h>
-#include <DPPP/H5ParmPredict.h>
-#include <DPPP/GainCal.h>
-#include <DPPP/Split.h>
-#include <DPPP/Upsample.h>
-#include <DPPP/Filter.h>
-#include <DPPP/Counter.h>
-#include <DPPP/ProgressMeter.h>
-#include <DPPP/DPLogger.h>
-#include <Common/Timer.h>
-#include <Common/StreamUtil.h>
-#include <Common/OpenMP.h>
-
-#include <casacore/casa/OS/Path.h>
-#include <casacore/casa/OS/DirectoryIterator.h>
-#include <casacore/casa/OS/Timer.h>
-#include <casacore/casa/OS/DynLib.h>
-#include <casacore/casa/Utilities/Regex.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- // Initialize the statics.
- std::map<std::string, DPRun::StepCtor*> DPRun::theirStepMap;
-
- void DPRun::registerStepCtor (const std::string& type, StepCtor* func)
- {
- theirStepMap[type] = func;
- }
-
- DPRun::StepCtor* DPRun::findStepCtor (const std::string& type)
- {
- std::map<std::string,StepCtor*>::const_iterator iter =
- theirStepMap.find (type);
- if (iter != theirStepMap.end()) {
- return iter->second;
- }
- // Try to load the step from a dynamic library with that name
- // (in lowercase).
- // A dot can be used to have a specific library name (so multiple
- // steps can use the same shared library).
- std::string libname(toLower(type));
- string::size_type pos = libname.find_first_of (".");
- if (pos != string::npos) {
- libname = libname.substr (0, pos);
- }
- // Try to load and initialize the dynamic library.
- casacore::DynLib dl(libname, string("libdppp_"), "register_"+libname, false);
- if (dl.getHandle()) {
- // See if registered now.
- iter = theirStepMap.find (type);
- if (iter != theirStepMap.end()) {
- return iter->second;
- }
- }
- THROW(Exception, "Step type " + type +
- " is unknown and no shared library lib" + libname + " or libdppp_" +
- libname + " found in (DY)LD_LIBRARY_PATH");
- }
-
-
- void DPRun::execute (const string& parsetName, int argc, char* argv[])
- {
- casacore::Timer timer;
- NSTimer nstimer;
- nstimer.start();
- ParameterSet parset;
- if (! parsetName.empty()) {
- parset.adoptFile (parsetName);
- }
- // Adopt possible parameters given at the command line.
- parset.adoptArgv (argc, argv); //# works fine if argc==0 and argv==0
- DPLogger::useLogger = parset.getBool ("uselogger", false);
- bool showProgress = parset.getBool ("showprogress", true);
- bool showTimings = parset.getBool ("showtimings", true);
- // checkparset is an integer parameter now, but accept a bool as well
- // for backward compatibility.
- int checkparset = 0;
- try {
- checkparset = parset.getInt ("checkparset", 0);
- } catch (...) {
- DPLOG_WARN_STR ("Parameter checkparset should be an integer value");
- checkparset = parset.getBool ("checkparset") ? 1:0;
- }
-
- bool showcounts = parset.getBool ("showcounts", true);
-
- uint numThreads = parset.getInt("numthreads", OpenMP::maxThreads());
- OpenMP::setNumThreads(numThreads);
-
- // Create the steps, link them toggether
- DPStep::ShPtr firstStep = makeSteps (parset, "", 0);
-
- // Let all steps fill their DPInfo object using the info from the previous step.
- DPInfo lastInfo = firstStep->setInfo (DPInfo());
-
- // Show the steps.
- DPStep::ShPtr step = firstStep;
- DPStep::ShPtr lastStep;
- while (step) {
- ostringstream os;
- step->show (os);
- DPLOG_INFO (os.str(), true);
- lastStep = step;
- step = step->getNextStep();
- }
- if (checkparset >= 0) {
- // Show unused parameters (might be misspelled).
- vector<string> unused = parset.unusedKeys();
- if (! unused.empty()) {
- DPLOG_WARN_STR
- (endl
- << "*** WARNING: the following parset keywords were not used ***"
- << endl
- << " maybe they are misspelled"
- << endl
- << " " << unused << endl);
- ASSERTSTR (checkparset==0, "Unused parset keywords found");
- }
- }
- // Process until the end.
- uint ntodo = firstStep->getInfo().ntime();
- DPLOG_INFO_STR ("Processing " << ntodo << " time slots ...");
- {
- ProgressMeter* progress = 0;
- if (showProgress) {
- progress = new ProgressMeter(0.0, ntodo, "NDPPP",
- "Time slots processed",
- "", "", true, 1);
- }
- double ndone = 0;
- if (showProgress && ntodo > 0) {
- progress->update (ndone, true);
- }
- DPBuffer buf;
- while (firstStep->process (buf)) {
- ++ndone;
- if (showProgress && ntodo > 0) {
- progress->update (ndone, true);
- }
- }
- delete progress;
- }
- // Finish the processing.
- DPLOG_INFO_STR ("Finishing processing ...");
- firstStep->finish();
- // Give all steps the option to add something to the MS written.
- // It starts with the last step to get the name of the output MS,
- // but each step must first call its previous step before
- // it adds something itself.
- lastStep->addToMS("");
-
- // Show the counts where needed.
- if (showcounts) {
- step = firstStep;
- while (step) {
- ostringstream os;
- step->showCounts (os);
- DPLOG_INFO (os.str(), true);
- step = step->getNextStep();
- }
- }
- // Show the overall timer.
- nstimer.stop();
- double duration = nstimer.getElapsed();
- ostringstream ostr;
- ostr << endl;
- // Output special line for pipeline use.
- if (DPLogger::useLogger) {
- ostr << "Start timer output" << endl;
- }
- timer.show (ostr, "Total NDPPP time");
- DPLOG_INFO (ostr.str(), true);
- if (showTimings) {
- // Show the timings per step.
- step = firstStep;
- while (step) {
- ostringstream os;
- step->showTimings (os, duration);
- if (! os.str().empty()) {
- DPLOG_INFO (os.str(), true);
- }
- step = step->getNextStep();
- }
- }
- if (DPLogger::useLogger) {
- ostr << "End timer output" << endl;
- }
- // The destructors are called automatically at this point.
- }
-
- DPStep::ShPtr DPRun::makeSteps (const ParameterSet& parset,
- const string& prefix,
- DPInput* reader)
- {
- DPStep::ShPtr firstStep;
- DPStep::ShPtr lastStep;
- if (!reader) {
- // Get input and output MS name.
- // Those parameters were always called msin and msout.
- // However, SAS/MAC cannot handle a parameter and a group with the same
- // name, hence one can also use msin.name and msout.name.
- vector<string> inNames = parset.getStringVector ("msin.name",
- vector<string>());
- if (inNames.empty()) {
- inNames = parset.getStringVector ("msin");
- }
- ASSERTSTR (inNames.size() > 0, "No input MeasurementSets given");
- // Find all file names matching a possibly wildcarded input name.
- // This is only possible if a single name is given.
- if (inNames.size() == 1) {
- if (inNames[0].find_first_of ("*?{['") != string::npos) {
- vector<string> names;
- names.reserve (80);
- casacore::Path path(inNames[0]);
- casacore::String dirName(path.dirName());
- casacore::Directory dir(dirName);
- // Use the basename as the file name pattern.
- casacore::DirectoryIterator dirIter (dir,
- casacore::Regex::fromPattern(path.baseName()));
- while (!dirIter.pastEnd()) {
- names.push_back (dirName + '/' + dirIter.name());
- dirIter++;
- }
- ASSERTSTR (!names.empty(), "No datasets found matching msin "
- << inNames[0]);
- inNames = names;
- }
- }
-
- // Get the steps.
- // Currently the input MS must be given.
- // In the future it might be possible to have a simulation step instead.
- // Create MSReader step if input ms given.
- if (inNames.size() == 1) {
- reader = new MSReader (inNames[0], parset, "msin.");
- } else {
- reader = new MultiMSReader (inNames, parset, "msin.");
- }
- firstStep = DPStep::ShPtr (reader);
- }
-
- casacore::Path pathIn (reader->msName());
- casacore::String currentMSName (pathIn.absoluteName());
-
- // Create the other steps.
- vector<string> steps = parset.getStringVector (prefix + "steps");
- lastStep = firstStep;
- DPStep::ShPtr step;
- for (vector<string>::const_iterator iter = steps.begin();
- iter != steps.end(); ++iter) {
- string prefix(*iter + '.');
- // The alphabetic part of the name is the default step type.
- // This allows names like average1, out3.
- string defaulttype = (*iter);
- while (defaulttype.size()>0 && std::isdigit(*defaulttype.rbegin())) {
- defaulttype.resize(defaulttype.size()-1);
- }
-
- string type = toLower(parset.getString (prefix+"type", defaulttype));
- // Define correct name for AOFlagger synonyms.
- if (type == "aoflagger" || type == "rficonsole") {
- type = "aoflag";
- }
- if (type == "averager" || type == "average" || type == "squash") {
- step = DPStep::ShPtr(new Averager (reader, parset, prefix));
- } else if (type == "madflagger" || type == "madflag") {
- step = DPStep::ShPtr(new MedFlagger (reader, parset, prefix));
- } else if (type == "preflagger" || type == "preflag") {
- step = DPStep::ShPtr(new PreFlagger (reader, parset, prefix));
- } else if (type == "uvwflagger" || type == "uvwflag") {
- step = DPStep::ShPtr(new UVWFlagger (reader, parset, prefix));
- } else if (type == "counter" || type == "count") {
- step = DPStep::ShPtr(new Counter (reader, parset, prefix));
- } else if (type == "phaseshifter" || type == "phaseshift") {
- step = DPStep::ShPtr(new PhaseShift (reader, parset, prefix));
- } else if (type == "demixer" || type == "demix") {
- step = DPStep::ShPtr(new Demixer (reader, parset, prefix));
- } else if (type == "smartdemixer" || type == "smartdemix") {
- step = DPStep::ShPtr(new DemixerNew (reader, parset, prefix));
- } else if (type == "stationadder" || type == "stationadd") {
- step = DPStep::ShPtr(new StationAdder (reader, parset, prefix));
- } else if (type == "scaledata") {
- step = DPStep::ShPtr(new ScaleData (reader, parset, prefix));
- } else if (type == "filter") {
- step = DPStep::ShPtr(new Filter (reader, parset, prefix));
- } else if (type == "applycal" || type == "correct") {
- step = DPStep::ShPtr(new ApplyCal (reader, parset, prefix));
- } else if (type == "predict") {
- step = DPStep::ShPtr(new Predict (reader, parset, prefix));
- } else if (type == "h5parmpredict") {
- step = DPStep::ShPtr(new H5ParmPredict (reader, parset, prefix));
- } else if (type == "applybeam") {
- step = DPStep::ShPtr(new ApplyBeam (reader, parset, prefix));
- } else if (type == "gaincal" || type == "calibrate") {
- step = DPStep::ShPtr(new GainCal (reader, parset, prefix));
- } else if (type == "upsample") {
- step = DPStep::ShPtr(new Upsample (reader, parset, prefix));
- } else if (type == "split" || type == "explode") {
- step = DPStep::ShPtr(new Split (reader, parset, prefix));
- } else if (type == "out" || type=="output" || type=="msout") {
- step = makeOutputStep(dynamic_cast<MSReader*>(reader), parset, prefix, currentMSName);
- } else {
- // Maybe the step is defined in a dynamic library.
- step = findStepCtor(type) (reader, parset, prefix);
- }
- if (lastStep) {
- lastStep->setNextStep (step);
- }
- lastStep = step;
- // Define as first step if not defined yet.
- if (!firstStep) {
- firstStep = step;
- }
- }
- // Add an output step if not explicitly added in steps (unless last step is a 'split' step)
- if (steps.size()==0 || (
- steps[steps.size()-1] != "out" &&
- steps[steps.size()-1] != "output" &&
- steps[steps.size()-1] != "msout" &&
- steps[steps.size()-1] != "split")) {
- step = makeOutputStep(dynamic_cast<MSReader*>(reader), parset, "msout.", currentMSName);
- lastStep->setNextStep (step);
- lastStep = step;
- }
-
- // Add a null step, so the last step can use getNextStep->process().
- DPStep::ShPtr nullStep(new NullStep());
- if (lastStep) {
- lastStep->setNextStep (nullStep);
- } else {
- firstStep = nullStep;
- }
- return firstStep;
- }
-
- DPStep::ShPtr DPRun::makeOutputStep (MSReader* reader,
- const ParameterSet& parset,
- const string& prefix,
- casacore::String& currentMSName)
- {
- DPStep::ShPtr step;
- casacore::String outName;
- bool doUpdate = false;
- if (prefix == "msout.") {
- // The last output step.
- outName = parset.getString ("msout.name", "");
- if (outName.empty()) {
- outName = parset.getString ("msout");
- }
- } else {
- // An intermediate output step.
- outName = parset.getString(prefix + "name");
- }
-
- // A name equal to . or the last name means an update of the last MS.
- if (outName.empty() || outName == ".") {
- outName = currentMSName;
- doUpdate = true;
- } else {
- casacore::Path pathOut(outName);
- if (currentMSName == pathOut.absoluteName()) {
- outName = currentMSName;
- doUpdate = true;
- }
- }
- if (doUpdate) {
- // Create MSUpdater.
- // Take care the history is not written twice.
- // Note that if there is nothing to write, the updater won't do anything.
- step = DPStep::ShPtr(new MSUpdater(dynamic_cast<MSReader*>(reader),
- outName, parset, prefix,
- outName!=currentMSName));
- } else {
- step = DPStep::ShPtr(new MSWriter (reader, outName, parset, prefix));
- reader->setReadVisData (true);
- }
- casacore::Path pathOut(outName);
- currentMSName = pathOut.absoluteName();
- return step;
- }
-
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/DPStep.cc b/CEP/DP3/DPPP/src/DPStep.cc
deleted file mode 100644
index 3bd88613dda200b2a50eecbed6bbba4588220790..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DPStep.cc
+++ /dev/null
@@ -1,124 +0,0 @@
-//# DPStep.cc: Abstract base class for a DPPP step
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DPStep.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- DPStep::~DPStep()
- {}
-
- const DPInfo& DPStep::setInfo (const DPInfo& info)
- {
- // Update the info of this step using the given info.
- updateInfo (info);
- // If there is a next step, set its info using the info of this step.
- if (getNextStep()) {
- return getNextStep()->setInfo (getInfo());
- }
- return getInfo();
- }
-
- void DPStep::updateInfo (const DPInfo& infoIn)
- { info() = infoIn; }
-
- void DPStep::addToMS (const string& msName)
- {
- if (itsPrevStep) itsPrevStep->addToMS(msName);
- }
-
- void DPStep::showCounts (std::ostream&) const
- {}
-
- void DPStep::showTimings (std::ostream&, double) const
- {}
-
-
- NullStep::~NullStep()
- {}
-
- bool NullStep::process (const DPBuffer&)
- { return true; }
-
- void NullStep::finish()
- {}
-
- void NullStep::show (std::ostream&) const
- {}
-
-
- ResultStep::ResultStep()
- {
- setNextStep (DPStep::ShPtr (new NullStep()));
- }
-
- ResultStep::~ResultStep()
- {}
-
- bool ResultStep::process (const DPBuffer& buf)
- {
- itsBuffer = buf;
- getNextStep()->process (buf);
- return true;
- }
-
- void ResultStep::finish()
- {
- getNextStep()->finish();
- }
-
- void ResultStep::show (std::ostream&) const
- {}
-
-
- MultiResultStep::MultiResultStep (uint size)
- : itsSize (0)
- {
- setNextStep (DPStep::ShPtr (new NullStep()));
- itsBuffers.resize (size);
- }
-
- MultiResultStep::~MultiResultStep()
- {}
-
- bool MultiResultStep::process (const DPBuffer& buf)
- {
- ASSERT (itsSize < itsBuffers.size());
- itsBuffers[itsSize].copy (buf);
- itsSize++;
- getNextStep()->process (buf);
- return true;
- }
-
- void MultiResultStep::finish()
- {
- getNextStep()->finish();
- }
-
- void MultiResultStep::show (std::ostream&) const
- {}
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/DemixInfo.cc b/CEP/DP3/DPPP/src/DemixInfo.cc
deleted file mode 100644
index 12c3cb6e4663ca3466c46d5bddecb42fd4ea5768..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DemixInfo.cc
+++ /dev/null
@@ -1,458 +0,0 @@
-//# DemixInfo.cc: Struct to hold the common demix variables
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: Demixer.h 23223 2012-12-07 14:09:42Z schoenmakers $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DemixInfo.h>
-#include <DPPP/PointSource.h>
-#include <DPPP/GaussianSource.h>
-#include <DPPP/Stokes.h>
-#include <DPPP/Simulate.h>
-#include <ParmDB/SourceDB.h>
-#include <Common/ParameterSet.h>
-#include <Common/StreamUtil.h>
-#include <Common/OpenMP.h>
-
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- DemixInfo::DemixInfo (const ParameterSet& parset, const string& prefix)
- : itsSelBL (parset, prefix, false, "cross"),
- itsSelBLTarget (parset, prefix+"target.", false, "cross", "CS*&"),
- itsPredictModelName (parset.getString(prefix+"estimate.skymodel", "")),
- itsDemixModelName (parset.getString(prefix+"ateam.skymodel")),
- itsTargetModelName (parset.getString(prefix+"target.skymodel")),
- itsSourceNames (parset.getStringVector (prefix+"sources")),
- itsRatio1 (parset.getDouble (prefix+"ratio1", 5.)),
- itsRatio2 (parset.getDouble (prefix+"ratio2", 0.25)),
- // Default thresholds depend on freq, so filled by function update.
- itsAteamAmplThreshold (parset.getDouble (prefix+"ateam.threshold",
- 0.)),
- itsTargetAmplThreshold (parset.getDouble (prefix+"target.threshold",
- 0.)),
- itsAngdistThreshold (parset.getDouble (prefix+"distance.threshold", 60)),
- itsAngdistRefFreq (parset.getDouble (prefix+"distance.reffreq", 60e6)),
- itsDefaultGain (parset.getDouble (prefix+"defaultgain", 1e-3)),
- itsPropagateSolution(parset.getBool (prefix+"propagatesolutions",
- false)),
- itsApplyBeam (parset.getBool (prefix+"applybeam", true)),
- itsSolveBoth (parset.getBool (prefix+"solveboth", false)),
- itsDoSubtract (parset.getBool (prefix+"subtract", true)),
- itsTargetHandling (parset.getUint (prefix+"targethandling", 0)),
- itsVerbose (parset.getUint (prefix+"verbose", 0)),
- itsMaxIter (parset.getUint (prefix+"maxiter", 50)),
- itsMinNBaseline (parset.getUint (prefix+"minnbaseline", 6)),
- itsMinNStation (parset.getUint (prefix+"minnstation", 5)),
- itsNStation (0),
- itsNBl (0),
- itsNCorr (0),
- itsNChanIn (0),
- itsNChanAvgSubtr (parset.getUint (prefix+"freqstep", 1)),
- itsNChanAvg (parset.getUint (prefix+"demixfreqstep",
- itsNChanAvgSubtr)),
- itsNChanOutSubtr (0),
- itsNChanOut (0),
- itsNTimeAvgSubtr (parset.getUint (prefix+"timestep", 1)),
- itsNTimeAvg (parset.getUint (prefix+"demixtimestep",
- itsNTimeAvgSubtr)),
- itsChunkSize (parset.getUint (prefix+"chunksize",
- itsNTimeAvg)),
- itsNTimeChunk (parset.getUint (prefix+"ntimechunk", 0)),
- itsTimeIntervalAvg (0)
- {
- if (itsNTimeChunk == 0) {
- itsNTimeChunk = OpenMP::maxThreads();
- }
- // Get delta in arcsec and take cosine of it (convert to radians first).
- double delta = parset.getDouble (prefix+"target.delta", 60.);
- itsCosTargetDelta = cos (delta / 3600. * casacore::C::pi / 180.);
- ASSERTSTR (!(itsTargetModelName.empty() || itsDemixModelName.empty()),
- "An empty name is given for a sky model");
- // If the estimate source model is given, read it.
- if (! itsPredictModelName.empty()) {
- itsAteamList = makePatchList (itsPredictModelName, itsSourceNames);
- // Use all predict patch names if no source names given.
- // In this way we're sure both A-team lists have the same sources
- // in the same order.
- if (itsSourceNames.empty()) {
- itsSourceNames.reserve (itsAteamList.size());
- for (size_t i=0; i<itsAteamList.size(); ++i) {
- itsSourceNames.push_back (itsAteamList[i]->name());
- }
- }
- }
- itsAteamDemixList = makePatchList (itsDemixModelName, itsSourceNames);
- if (itsTargetHandling != 3) {
- itsTargetList = makePatchList (itsTargetModelName, vector<string>());
- }
- // If no estimate model is given, use the demix model.
- if (itsAteamList.empty()) {
- itsAteamList = itsAteamDemixList;
- }
- if (itsSourceNames.empty()) {
- itsSourceNames.reserve (itsAteamList.size());
- for (size_t i=0; i<itsAteamList.size(); ++i) {
- itsSourceNames.push_back (itsAteamList[i]->name());
- }
- }
- // Note that the A-team models are in the same order of name.
- // Check they have matching positions.
- ASSERT (itsAteamList.size() == itsAteamDemixList.size());
- for (size_t i=0; i<itsAteamList.size(); ++i) {
- ASSERT (itsAteamList[i]->name() == itsAteamDemixList[i]->name());
- ASSERTSTR (testAngDist (itsAteamDemixList[i]->position()[0],
- itsAteamDemixList[i]->position()[1],
- itsAteamList[i]->position()[0],
- itsAteamList[i]->position()[1],
- itsCosTargetDelta),
- "Position mismatch of source " << itsAteamList[i]->name()
- << " in A-team SourceDBs (["
- << itsAteamDemixList[i]->position()[0] << ", "
- << itsAteamDemixList[i]->position()[1] << "] and ["
- << itsAteamList[i]->position()[0] << ", "
- << itsAteamList[i]->position()[1] << "])");
- }
- makeTargetDemixList();
- }
-
- void DemixInfo::makeTargetDemixList()
- {
- // Get all A-team models for demixing.
- // Note that in the constructor only some sources were read.
- // First open the SourceDB.
- BBS::SourceDB sdb(BBS::ParmDBMeta(string(), itsDemixModelName));
- sdb.lock();
- vector<Patch::ConstPtr> patchList = makePatchList (itsDemixModelName,
- vector<string>());
- // The demix target list is the same as the predict list, but A-team
- // sources must be replaced with their demix model.
- // Also these sources must be removed from the A-team model.
- vector<Patch::ConstPtr> targetDemixList;
- uint ncomponent = 0;
- itsTargetDemixList.reserve (itsTargetList.size());
- for (size_t i=0; i<itsTargetList.size(); ++i) {
- itsTargetDemixList.push_back (itsTargetList[i]);
- ncomponent += itsTargetList[i]->nComponents();
- // Look if an A-team source matches this target source.
- for (size_t j=0; j<patchList.size(); ++j) {
- if (testAngDist (itsTargetList[i]->position()[0],
- itsTargetList[i]->position()[1],
- patchList[j]->position()[0],
- patchList[j]->position()[1],
- itsCosTargetDelta)) {
- // Match, so use the detailed A-team model.
- itsTargetDemixList[i] = patchList[j];
- ncomponent += (patchList[j]->nComponents() -
- itsTargetList[i]->nComponents());
- itsTargetReplaced.push_back (patchList[j]->name());
- // A-source is in target, so remove from A-team models (if in there).
- for (size_t k=0; k<itsAteamList.size(); ++k) {
- if (testAngDist (itsTargetDemixList[i]->position()[0],
- itsTargetDemixList[i]->position()[1],
- itsAteamList[k]->position()[0],
- itsAteamList[k]->position()[1],
- itsCosTargetDelta)) {
- itsAteamRemoved.push_back (itsAteamList[k]->name());
- itsAteamList.erase (itsAteamList.begin() + k);
- itsAteamDemixList.erase (itsAteamDemixList.begin() + k);
- break;
- }
- }
- break;
- }
- }
- }
- }
-
- void DemixInfo::update (const DPInfo& infoSel, DPInfo& info)
- {
- // Remove unused antennae and renumber remaining ones.
- itsInfoSel = infoSel;
- itsInfoSel.removeUnusedAnt();
- // Get size info.
- itsNChanIn = infoSel.nchan();
- itsNCorr = infoSel.ncorr();
- ASSERTSTR (itsNCorr==4, "Demixing requires data with 4 polarizations");
- // NB. The number of baselines and stations refer to the number of
- // selected baselines and the number of unique stations participating
- // in the selected baselines.
- itsNBl = infoSel.nbaselines();
- itsNStation = infoSel.antennaUsed().size();
-
- // The default thresholds depend on frequency.
- if (itsAteamAmplThreshold <= 0) {
- if (info.refFreq() < 100e6) {
- itsAteamAmplThreshold = 50;
- } else {
- itsAteamAmplThreshold = 5;
- }
- }
- if (itsTargetAmplThreshold <= 0) {
- if (info.refFreq() < 100e6) {
- itsTargetAmplThreshold = 200;
- } else {
- itsTargetAmplThreshold = 100;
- }
- }
-
- // Setup the baseline index vector used to split the UVWs.
- itsUVWSplitIndex = nsetupSplitUVW (itsInfoSel.nantenna(),
- itsInfoSel.getAnt1(),
- itsInfoSel.getAnt2());
- if (itsVerbose > 1) {
- cout << "splitindex="<<itsUVWSplitIndex<<endl;
- }
-
- // Determine which baselines to use when estimating A-team and target.
- itsSelTarget = itsSelBLTarget.applyVec (infoSel);
-
- // Form the baselines.
- // the numbering due to unused stations.
- /// Why is that needed for predict/solve?
- for (uint i=0; i<itsNBl; ++i) {
- itsBaselines.push_back (Baseline(itsInfoSel.getAnt1()[i],
- itsInfoSel.getAnt2()[i]));
- }
-
- // Adapt averaging to available nr of channels and times.
- // Use a copy of the DPInfo, otherwise it is updated multiple times.
- DPInfo infoDemix(infoSel);
- itsNTimeAvg = std::min (itsNTimeAvg, infoSel.ntime());
- itsNChanAvg = infoDemix.update (itsNChanAvg, itsNTimeAvg);
- itsNChanOut = infoDemix.nchan();
- itsTimeIntervalAvg = infoDemix.timeInterval();
- /// itsNTimeDemix = infoDemix.ntime();
-
- // Update the overall Demixer DPInfo object.
- itsNTimeAvgSubtr = std::min (itsNTimeAvgSubtr, infoSel.ntime());
- itsNChanAvgSubtr = info.update (itsNChanAvgSubtr, itsNTimeAvgSubtr);
- itsNChanOutSubtr = info.nchan();
- ASSERTSTR (itsNChanAvg % itsNChanAvgSubtr == 0,
- "Demix frequency averaging " << itsNChanAvg
- << " must be a multiple of output averaging "
- << itsNChanAvgSubtr);
- ASSERTSTR (itsNTimeAvg % itsNTimeAvgSubtr == 0,
- "Demix time averaging " << itsNTimeAvg
- << " must be a multiple of output averaging "
- << itsNTimeAvgSubtr);
- ASSERTSTR (itsChunkSize % itsNTimeAvg == 0,
- "Demix predict time chunk size " << itsChunkSize
- << " must be a multiple of averaging time step "
- << itsNTimeAvg);
- itsNTimeOut = itsChunkSize / itsNTimeAvg;
- itsNTimeOutSubtr = itsChunkSize / itsNTimeAvgSubtr;
- // Store channel frequencies for the demix and subtract resolutions.
- itsFreqDemix = infoDemix.chanFreqs();
- itsFreqSubtr = info.chanFreqs();
-
- // Store phase center position in J2000.
- MDirection dirJ2000(MDirection::Convert(infoSel.phaseCenter(),
- MDirection::J2000)());
- Quantum<Vector<Double> > angles = dirJ2000.getAngle();
- itsPhaseRef = Position(angles.getBaseValue()[0],
- angles.getBaseValue()[1]);
-
- // Determine if the minimum distance (scaled with freq) of A-sources
- // to target is within the threshold.
- // First get the target position (average of its patches).
- BBS::PatchSumInfo sumInfo(0);
- for (size_t i=0; i<itsTargetList.size(); ++i) {
- sumInfo.add (itsTargetList[i]->position()[0],
- itsTargetList[i]->position()[1],
- 1.);
- }
- double targetRa = sumInfo.getRa();
- double targetDec = sumInfo.getDec();
- // Determine the minimum distance.
- double minDist = 1e30;
- double freqRatio = info.refFreq() / itsAngdistRefFreq;
- for (size_t i=0; i<itsAteamList.size(); ++i) {
- double dist = acos (getCosAngDist (itsAteamList[i]->position()[0],
- itsAteamList[i]->position()[1],
- targetRa, targetDec));
- dist *= freqRatio;
- if (verbose() > 10) {
- cout << "Target distance to " << itsAteamList[i]->name()
- << " = " << dist*180./C::pi << " deg" << endl;
- }
- if (dist < minDist) minDist = dist;
- }
- itsIsAteamNearby = cos(minDist) > cos(itsAngdistThreshold*C::pi/180.);
- }
-
- void DemixInfo::show (ostream& os) const
- {
- os << " estimate.skymodel: " << itsPredictModelName << endl;
- os << " ateam.skymodel: " << itsDemixModelName << endl;
- os << " target.skymodel: " << itsTargetModelName << endl;
- os << " sources: " << itsSourceNames << endl;
- os << " " << itsAteamRemoved
- << " removed from A-team model (in target)" << endl;
- os << " " << itsTargetReplaced
- << " replaced in target model (better A-team model)" << endl;
- os << " ratio1: " << itsRatio1 << endl;
- os << " ratio2: " << itsRatio2 << endl;
- os << " ateam.threshold: " << itsAteamAmplThreshold << endl;
- os << " target.threshold: " << itsTargetAmplThreshold << endl;
- os << " target.delta: "
- << acos(itsCosTargetDelta) * 3600. / casacore::C::pi * 180.
- << " arcsec" << endl;
- os << " distance.threshold: " << itsAngdistThreshold << " deg" << endl;
- os << " distance.reffreq: " << itsAngdistRefFreq << " Hz" << endl;
- os << " minnbaseline: " << itsMinNBaseline << endl;
- os << " minnstation: " << itsMinNStation << endl;
- os << " maxiter: " << itsMaxIter << endl;
- os << " defaultgain: " << itsDefaultGain << endl;
- os << " propagatesolutions: " << (itsPropagateSolution ? "True":"False")
- << endl;
- os << " applybeam: " << (itsApplyBeam ? "True":"False") << endl;
- os << " solveboth: " << (itsSolveBoth ? "True":"False") << endl;
- os << " subtract: " << (itsDoSubtract ? "True":"False")
- << endl;
- os << " freqstep: " << itsNChanAvgSubtr << endl;
- os << " timestep: " << itsNTimeAvgSubtr << endl;
- os << " demixfreqstep: " << itsNChanAvg << endl;
- os << " demixtimestep: " << itsNTimeAvg << endl;
- os << " chunksize: " << itsChunkSize << endl;
- os << " ntimechunk: " << itsNTimeChunk << endl;
- os << " target estimate";
- itsSelBLTarget.show (os, " ");
- os << " demix";
- itsSelBL.show (os, " ");
- }
-
- vector<Patch::ConstPtr>
- DemixInfo::makePatchList (const string& sdbName,
- const vector<string>& patchNames)
- {
- // Open the SourceDB.
- BBS::SourceDB sdb(BBS::ParmDBMeta(string(), sdbName));
- sdb.lock();
- // Get all patches from it.
- vector<string> names(sdb.getPatches());
- vector<string>::const_iterator pnamesIter = patchNames.begin();
- vector<string>::const_iterator pnamesEnd = patchNames.end();
- if (patchNames.empty()) {
- pnamesIter = names.begin();
- pnamesEnd = names.end();
- }
- // Create a patch component list for each matching patch name.
- vector<Patch::ConstPtr> patchList;
- patchList.reserve (patchNames.size());
- for (; pnamesIter != pnamesEnd; ++pnamesIter) {
- ASSERTSTR (std::find (names.begin(), names.end(), *pnamesIter)
- != names.end(),
- "Demixer: sourcename " << *pnamesIter
- << " not found in SourceDB " << sdbName);
- // Use this patch; get all its sources.
- vector<BBS::SourceData> patch = sdb.getPatchSourceData (*pnamesIter);
- vector<ModelComponent::Ptr> componentList;
- componentList.reserve (patch.size());
- for (vector<BBS::SourceData>::const_iterator iter=patch.begin();
- iter!=patch.end(); ++iter) {
- const BBS::SourceData& src = *iter;
- // Fetch position.
- ASSERT (src.getInfo().getRefType() == "J2000");
- Position position;
- position[0] = src.getRa();
- position[1] = src.getDec();
-
- // Fetch stokes vector.
- Stokes stokes;
- stokes.I = src.getI();
- stokes.V = src.getV();
- if (!src.getInfo().getUseRotationMeasure()) {
- stokes.Q = src.getQ();
- stokes.U = src.getU();
- }
-
- PointSource::Ptr source;
- switch (src.getInfo().getType()) {
- case BBS::SourceInfo::POINT:
- {
- source = PointSource::Ptr(new PointSource(position, stokes));
- }
- break;
- case BBS::SourceInfo::GAUSSIAN:
- {
- GaussianSource::Ptr gauss(new GaussianSource(position, stokes));
- const double deg2rad = (casacore::C::pi / 180.0);
- gauss->setPositionAngle(src.getOrientation() * deg2rad);
- const double arcsec2rad = (casacore::C::pi / 3600.0) / 180.0;
- gauss->setMajorAxis(src.getMajorAxis() * arcsec2rad);
- gauss->setMinorAxis(src.getMinorAxis() * arcsec2rad);
- source = gauss;
- }
- break;
- default:
- {
- ASSERTSTR(false, "Only point sources and Gaussian sources are"
- " supported at this time.");
- }
- }
-
- // Fetch spectral index attributes (if applicable).
- if (src.getSpectralTerms().size() > 0) {
- bool isLogarithmic = true;
- source->setSpectralTerms(src.getInfo().getSpectralTermsRefFreq(),
- isLogarithmic,
- src.getSpectralTerms().begin(),
- src.getSpectralTerms().end());
- }
-
- // Fetch rotation measure attributes (if applicable).
- if (src.getInfo().getUseRotationMeasure()) {
- source->setRotationMeasure(src.getPolarizedFraction(),
- src.getPolarizationAngle(),
- src.getRotationMeasure());
- }
-
- // Add the source definition.
- componentList.push_back(source);
- }
-
- // Add the component list as a patch to the list of patches.
- Patch::Ptr ppatch(new Patch(*pnamesIter,
- componentList.begin(),
- componentList.end()));
- vector<BBS::PatchInfo> patchInfo(sdb.getPatchInfo(-1, *pnamesIter));
- ASSERT (patchInfo.size() == 1);
- // Set the position and apparent flux of the patch.
- Position patchPosition;
- patchPosition[0] = patchInfo[0].getRa();
- patchPosition[1] = patchInfo[0].getDec();
- ppatch->setPosition (patchPosition);
- ppatch->setBrightness (patchInfo[0].apparentBrightness());
- patchList.push_back (ppatch);
- }
- return patchList;
- }
-
-
- } //# end namespace
-} //# end namespace
diff --git a/CEP/DP3/DPPP/src/DemixWorker.cc b/CEP/DP3/DPPP/src/DemixWorker.cc
deleted file mode 100644
index 132cc073e8b4334b557521735a183a85cbd594b3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DemixWorker.cc
+++ /dev/null
@@ -1,1346 +0,0 @@
-//# DemixerWorker.cc: Demixing helper class to demix a time chunk
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: Demixer.cc 24221 2013-03-12 12:24:48Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DemixWorker.h>
-#include <DPPP/Apply.h>
-#include <DPPP/Averager.h>
-#include <DPPP/CursorUtilCasa.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/PhaseShift.h>
-#include <DPPP/Simulate.h>
-#include <DPPP/Simulator.h>
-#include <DPPP/SubtractNew.h>
-#include <DPPP/DPLogger.h>
-
-#include <ParmDB/Axis.h>
-#include <ParmDB/SourceDB.h>
-#include <ParmDB/ParmDB.h>
-#include <ParmDB/ParmSet.h>
-#include <ParmDB/ParmCache.h>
-#include <ParmDB/Parm.h>
-
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <Common/OpenMP.h>
-#include <Common/StreamUtil.h>
-#include <Common/lofar_iomanip.h>
-#include <Common/lofar_iostream.h>
-#include <Common/lofar_fstream.h>
-
-#include <casacore/casa/Quanta/MVAngle.h>
-#include <casacore/casa/Quanta/MVEpoch.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Arrays/Matrix.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <casacore/casa/Arrays/MatrixMath.h>
-#include <casacore/casa/Arrays/MatrixIter.h>
-#include <casacore/casa/Containers/Record.h>
-#include <casacore/scimath/Mathematics/MatrixMathLA.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/measures/Measures/MeasureHolder.h>
-
-#include <sstream>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- using LOFAR::operator<<;
-
- namespace
- {
- string toString (double value);
- } //# end unnamed namespace
-
- DemixWorker::DemixWorker (DPInput* input,
- const string& prefix,
- const DemixInfo& mixInfo,
- const DPInfo& info,
- int workerNr)
- : itsWorkerNr (workerNr),
- itsMix (&mixInfo),
- itsFilter (input, mixInfo.selBL()),
- itsNrSolves (0),
- itsNrConverged (0),
- itsNrIter (0),
- itsNrNoDemix (0),
- itsNrIncludeStrongTarget (0),
- itsNrIncludeCloseTarget (0),
- itsNrIgnoreTarget (0),
- itsNrDeprojectTarget (0)
- {
- // Add a null step as the last step in the filter.
- DPStep::ShPtr nullStep(new NullStep());
- itsFilter.setNextStep (nullStep);
- // The worker will process up to chunkSize input time slots.
- // Size buffers accordingly.
- uint nsrc = itsMix->ateamList().size();
- itsFactors.resize (itsMix->ntimeOut());
- itsFactorsSubtr.resize (itsMix->ntimeOutSubtr());
- itsOrigPhaseShifts.reserve (nsrc);
- itsOrigFirstSteps.reserve (nsrc+1); // also needed for target direction
- itsAvgResults.reserve (nsrc+1);
- itsDemixList.resize (nsrc+1);
-
- // Read the antenna beam info from the MS.
- // Only take the stations actually used.
- input->fillBeamInfo (itsAntBeamInfo, itsMix->antennaNames());
-
- // Create the solve and subtract steps for the sources to be removed.
- // Solving consists of the following steps:
- // - select the requested baselines (longer baselines may need no demix)
- // - phaseshift selected data to each demix source direction
- // - average selected data in each direction, also original phasecenter.
- // - determine and average demix factors for all directions
- // - predict and solve in each direction. It is possible to predict
- // more directions than to solve (for strong sources in field).
- // Subtract consists of the following steps:
- // - average all data (possibly different averaging than used in solve)
- // - determine and average demix factors (using select output in solve)
- // - select the requested baselines
- // - subtract sources for selected data
- // - merge subtract result into averaged data. This is not needed if
- // no selection is done.
- // Note that multiple time chunks are handled jointly, so a
- // MultiResultStep is used to catch the results of all time chunks.
- const vector<Patch::ConstPtr>& patchList = itsMix->ateamDemixList();
- vector<string> sourceVec(2);
- for (uint i=0; i<nsrc; ++i) {
- // First make the phaseshift and average steps for each demix source.
- // The resultstep gets the result.
- // The phasecenter can be given in a parameter. Its name is the default.
- // Look up the source direction in the patch table.
- // If found, turn it into a vector of strings.
- sourceVec[0] = toString(patchList[i]->position()[0]);
- sourceVec[1] = toString(patchList[i]->position()[1]);
- PhaseShift* step1 = new PhaseShift(input, ParameterSet(),
- prefix+itsMix->sourceNames()[i]+'.',
- sourceVec);
- itsOrigFirstSteps.push_back (DPStep::ShPtr(step1));
- itsOrigPhaseShifts.push_back (step1);
- DPStep::ShPtr step2 (new Averager(input, prefix, itsMix->nchanAvg(),
- itsMix->ntimeAvg()));
- step1->setNextStep (step2);
- MultiResultStep* step3 = new MultiResultStep(itsMix->ntimeOut());
- step2->setNextStep (DPStep::ShPtr(step3));
- // There is a single demix factor step which needs to get all results.
- itsAvgResults.push_back (step3);
- }
-
- // Now create the step to average the data themselves.
- DPStep::ShPtr targetAvg(new Averager(input, prefix,
- itsMix->nchanAvg(),
- itsMix->ntimeAvg()));
- itsOrigFirstSteps.push_back (targetAvg);
- MultiResultStep* targetAvgRes = new MultiResultStep(itsMix->ntimeOut());
- targetAvg->setNextStep (DPStep::ShPtr(targetAvgRes));
- itsAvgResults.push_back (targetAvgRes);
-
- // Create the data average step for the subtract.
- // The entire average result is needed for the next NDPPP step.
- // Only the selected baselines need to be subtracted, so add a
- // filter step as the last one.
- itsAvgStepSubtr = DPStep::ShPtr(new Averager(input, prefix,
- itsMix->nchanAvgSubtr(),
- itsMix->ntimeAvgSubtr()));
- itsAvgResultFull = new MultiResultStep(itsMix->ntimeOutSubtr());
- itsFilterSubtr = new Filter(input, itsMix->selBL());
- itsAvgResultSubtr = new MultiResultStep(itsMix->ntimeOutSubtr());
- itsAvgStepSubtr->setNextStep (DPStep::ShPtr(itsAvgResultFull));
- itsAvgResultFull->setNextStep (DPStep::ShPtr(itsFilterSubtr));
- itsFilterSubtr->setNextStep (DPStep::ShPtr(itsAvgResultSubtr));
-
- // Let the internal steps update their data.
- itsFilter.setInfo (info);
- for (uint i=0; i<itsOrigFirstSteps.size(); ++i) {
- itsOrigFirstSteps[i]->setInfo (itsFilter.getInfo());
- }
- itsAvgStepSubtr->setInfo (info);
-
- // Size the various work buffers to the maximum size possibly needed.
- // As few as possible dynamic buffers are used, because each malloc
- // requires a thread-lock.
- itsFactorBuf.resize (IPosition(4, itsMix->ncorr(), itsMix->nchanIn(),
- itsMix->nbl(), nsrc*(nsrc+1)/2));
- itsFactorBufSubtr.resize (itsFactorBuf.shape());
- itsAvgUVW.resize (3, itsMix->nbl());
- itsStationUVW.resize (3, itsMix->nstation(), itsMix->ntimeOutSubtr());
- itsUVW.resize (3, itsMix->nstation());
- itsSrcSet.reserve (nsrc+1);
- itsStationsToUse.resize (nsrc);
- itsUnknownsIndex.resize (nsrc+1);
- for (uint dr=0; dr<nsrc+1; ++dr) {
- itsUnknownsIndex[dr].resize (itsMix->nstation());
- }
- itsSolveStation.resize (itsMix->nstation());
- itsNrSourcesDemixed.resize (nsrc);
- itsNrSourcesDemixed = 0;
- itsNrStationsDemixed.resize (itsMix->nstation());
- itsNrStationsDemixed = 0;
- itsStatSourceDemixed.resize (nsrc, itsMix->nstation());
- itsStatSourceDemixed = 0;
- itsPredictVis.resize (itsMix->ncorr(), itsMix->nchanOut(),
- itsMix->nbl());
- // itsModelVisDemix could be reused also for subtracting to save memory.
- // Not doing this now, so that the elements can be a Cube and do not need
- // to be resized.
- itsModelVisDemix.resize (nsrc+1);
- for (uint dr=0; dr<nsrc+1; ++dr) {
- itsModelVisDemix[dr].resize(itsMix->ncorr(),
- itsMix->nchanOut(),
- itsMix->nbl());
- }
- itsModelVisSubtr.resize (nsrc+1);
- for (uint dr=0; dr<nsrc+1; ++dr) {
- itsModelVisSubtr[dr].resize(itsMix->ncorr(),
- itsMix->nchanOutSubtr(),
- itsMix->nbl());
- }
- itsAteamAmpl.resize (nsrc);
- for (uint dr=0; dr<nsrc; ++dr) {
- itsAteamAmpl[dr].resize (itsMix->nchanOut(), itsMix->nbl(),
- itsMix->ntimeOut());
- }
- itsAteamAmplSel.resize (itsMix->nbl(), nsrc);
- itsTargetAmpl.resize (itsMix->nchanOut(), itsMix->nbl(),
- itsMix->ntimeOut());
- itsTmpAmpl.resize (itsTargetAmpl.size());
- itsObservedAmpl.resize (itsMix->nbl());
- itsSourceAmpl.resize (itsMix->nbl());
- itsSumSourceAmpl.resize (itsMix->nbl());
- itsAmplSubtrMean.resize (itsMix->nbl(), nsrc+1);
- itsAmplSubtrM2.resize (itsMix->nbl(), nsrc+1);
- itsAmplSubtrNr.resize (itsMix->nbl(), nsrc+1);
- itsAmplSubtrMean = 0.;
- itsAmplSubtrM2 = 0.;
- itsAmplSubtrNr = 0;
- itsEstimate.update (nsrc+1, itsMix->nbl(), itsMix->nstation(),
- itsMix->nchanOut(), itsMix->maxIter(),
- itsMix->propagateSolution());
- itsBeamValues.resize (itsMix->nchanOutSubtr() * itsMix->nstation());
- // Make a copy of the array position and directions, because using the
- // same Measure object in multiple threads is not safe.
- // The only way to make a deep copy if using a MeasureHolder which
- // gets converted to/from a Record.
- String msg;
- {
- Record rec;
- MeasureHolder mh1(info.arrayPos());
- ASSERT (mh1.toRecord (msg, rec));
- MeasureHolder mh2;
- ASSERT (mh2.fromRecord (msg, rec));
- itsArrayPos = mh2.asMPosition();
- }
- {
- Record rec;
- MeasureHolder mh1(info.delayCenter());
- ASSERT (mh1.toRecord (msg, rec));
- MeasureHolder mh2;
- ASSERT (mh2.fromRecord (msg, rec));
- itsDelayCenter = mh2.asMDirection();
- }
- {
- Record rec;
- MeasureHolder mh1(info.tileBeamDir());
- ASSERT (mh1.toRecord (msg, rec));
- MeasureHolder mh2;
- ASSERT (mh2.fromRecord (msg, rec));
- itsTileBeamDir = mh2.asMDirection();
- }
- // Create the Measure ITRF conversion info given the array position.
- // The time and direction are filled in later.
- itsMeasFrame.set (itsArrayPos);
- itsMeasFrame.set (MEpoch(MVEpoch(info.startTime()/86400), MEpoch::UTC));
- itsMeasConverter.set (MDirection::J2000,
- MDirection::Ref(MDirection::ITRF, itsMeasFrame));
- // Do a dummy conversion, because Measure initialization does not
- // seem to be thread-safe.
- dir2Itrf(itsDelayCenter);
- }
-
- void DemixWorker::process (const DPBuffer* bufin, uint nbufin,
- DPBuffer* bufout, vector<double>* solutions,
- uint chunkNr)
- {
- itsTimer.start();
- itsTimerCoarse.start();
- if (itsMix->verbose() > 10) {
- cout << endl << "Info for time chunk " << chunkNr
- << " ( "<< nbufin << " time slots)"
- << endl << "===================" << endl;
- }
- // Average and split the baseline UVW coordinates per station.
- // Do this at the demix time resolution.
- // The buffer has not been filtered yet, so the UVWs have to be filtered.
- uint ntime = avgSplitUVW (bufin, nbufin, itsMix->ntimeAvg(),
- itsFilter.getIndicesBL());
- double timeStep = bufin[0].getExposure();
- double time = bufin[0].getTime() + (itsMix->ntimeAvg()-1) * 0.5*timeStep;
- // First do the predict of the coarse A-team sources at demix resolution.
- // It fills the indices of the sources to demix.
- // It also fills in the baselines that do not need to used.
- predictAteam (itsMix->ateamList(), ntime, time, timeStep);
- // If no sources to demix, simply average the input buffers.
- if (itsSrcSet.empty()) {
- itsNrNoDemix++;
- // Set all solutions to 0.
- uint nout = (nbufin + itsMix->ntimeAvg() - 1) / itsMix->ntimeAvg();
- for (size_t i=0; i<nout; ++i) {
- solutions[i].resize ((itsMix->ateamList().size() + 1) *
- itsMix->nstation() * 8);
- std::fill (solutions[i].begin(), solutions[i].end(), 0.);
- }
- // Set statistics to 0.
- /// itsAmplSubtrMean = 0.;
- /// itsAmplSubtrM2 = 0.;
- /// itsAmplSubtrNr = 0;
- itsTimerCoarse.stop();
- itsTimerPhaseShift.start();
- average (bufin, nbufin, bufout);
- itsTimerPhaseShift.stop();
- itsTimer.stop();
- return;
- }
- // The target has to be predicted as well (also at demix resolution).
- predictTarget (itsMix->targetList(), ntime, time, timeStep);
- // Determine what needs to be done.
- // It fills in the steps to perform for the sources to demix.
- setupDemix (chunkNr);
- itsTimerCoarse.stop();
- // Loop over the buffers and process them.
- itsNTimeOut = 0;
- itsNTimeOutSubtr = 0;
- for (uint i=0; i<nbufin; ++i) {
- // Do the filter step first.
- itsFilter.process (bufin[i]);
- const DPBuffer& selBuf = itsFilter.getBuffer();
- // Do the next steps (phaseshift and average) on the filter output.
- itsTimerPhaseShift.start();
- for (uint j=0; j<itsFirstSteps.size(); ++j) {
- itsFirstSteps[j]->process (selBuf);
- }
- // Do the average and filter step for the output for all data.
- itsAvgStepSubtr->process (bufin[i]);
- // Finish averaging when last buffer.
- if (i == nbufin-1) {
- for (size_t j=0; j<itsFirstSteps.size(); ++j) {
- itsFirstSteps[j]->finish();
- }
- itsAvgStepSubtr->finish();
- }
- itsTimerPhaseShift.stop();
-
- // For each NTimeAvg times, calculate the phase rotation per direction
- // for the selected data.
- itsTimerDemix.start();
- addFactors (selBuf, itsFactorBuf);
- if ((i+1) % itsMix->ntimeAvg() == 0 || i == nbufin-1) {
- makeFactors (itsFactorBuf, itsFactors[itsNTimeOut],
- itsAvgResults[itsSrcSet[0]]->get()[itsNTimeOut].getWeights(),
- itsMix->nchanOut(),
- itsMix->nchanAvg());
- // Deproject sources without a model.
- deproject (itsFactors[itsNTimeOut], itsAvgResults, itsNTimeOut);
- itsFactorBuf = Complex(); // Clear summation buffer
- itsNTimeOut++;
- }
- // Subtract is done with different averaging parameters, so calculate
- // the factors for it (again for selected data only).
- addFactors (selBuf, itsFactorBufSubtr);
- if ((i+1) % itsMix->ntimeAvgSubtr() == 0 || i == nbufin-1) {
- makeFactors (itsFactorBufSubtr, itsFactorsSubtr[itsNTimeOutSubtr],
- itsAvgResultSubtr->get()[itsNTimeOutSubtr].getWeights(),
- itsMix->nchanOutSubtr(),
- itsMix->nchanAvgSubtr());
- itsFactorBufSubtr = Complex(); // Clear summation buffer
- itsNTimeOutSubtr++;
- }
- itsTimerDemix.stop();
- }
-
- // Estimate gains and subtract source contributions.
- handleDemix (bufout, solutions, time, timeStep);
- itsTimer.stop();
- }
-
- uint DemixWorker::avgSplitUVW (const DPBuffer* bufin, uint nbufin,
- uint ntimeAvg, const vector<uint>& selbl)
- {
- ASSERT (selbl.size() == size_t(itsAvgUVW.shape()[1]));
- // First average the UVWs to the predict time window.
- uint ntime = (nbufin + ntimeAvg - 1) / ntimeAvg;
- const DPBuffer* buf = bufin;
- uint nleft = nbufin;
- // Loop over nr of output time slots.
- MatrixIterator<double> uvwIter(itsStationUVW);
- for (uint i=0; i<ntime; ++i) {
- // Sum the times for this output time slot.
- // Only take the selected baselines into account.
- itsAvgUVW = 0.;
- uint ntodo = std::min(ntimeAvg, nleft);
- for (uint j=0; j<ntodo; ++j) {
- double* sumPtr = itsAvgUVW.data();
- for (uint k=0; k<selbl.size(); ++k) {
- const double* uvwPtr = buf->getUVW().data() + 3*selbl[k];
- for (int k1=0; k1<3; ++k1) {
- *sumPtr++ += uvwPtr[k1];
- }
- }
- buf++;
- }
- // Average the UVWs.
- itsAvgUVW /= double(ntodo);
- if (itsMix->verbose() > 12) {
- cout<<"avguvw="<<ntodo<<itsAvgUVW<<endl;
- }
- nleft -= ntodo;
- // Split the baseline UVW coordinates per station.
- nsplitUVW (itsMix->uvwSplitIndex(), itsMix->baselines(),
- itsAvgUVW, uvwIter.matrix());
- uvwIter.next();
- }
- if (itsMix->verbose() > 12) {
- cout<<"stationuwv="<<itsStationUVW;
- }
- return ntime;
- }
-
- void DemixWorker::predictTarget (const vector<Patch::ConstPtr>& patchList,
- uint ntime, double time, double timeStep)
- {
- // This is only needed for the short baselines, but it takes hardly
- // any time to do it for all selected baselines.
- // So no effort is spent on further selection.
- itsTargetAmpl = 0;
- MatrixIterator<float> miter(itsTargetAmpl);
- MatrixIterator<double> uvwiter(itsStationUVW);
- double t = time;
- for (uint j=0; j<ntime; ++j) {
- for (uint dr=0; dr<patchList.size(); ++dr) {
- itsPredictVis = dcomplex();
- Simulator simulator(itsMix->phaseRef(),
- itsMix->nstation(),
- itsMix->nbl(),
- itsMix->freqDemix().size(),
- itsMix->baselines(),
- itsMix->freqDemix(),
- uvwiter.matrix(),
- itsPredictVis);
- for(size_t i = 0; i < patchList[dr]->nComponents(); ++i)
- {
- simulator.simulate(patchList[dr]->component(i));
- }
- // Get and apply beam for target patch.
- applyBeam (t, patchList[dr]->position(), True);
- addStokesI (miter.matrix());
- }
- miter.next();
- uvwiter.next();
- t += timeStep;
- }
- }
-
- void DemixWorker::addStokesI (Matrix<float>& ampl)
- {
- ASSERT (ampl.contiguousStorage());
- // Calculate the StokesI ampl ((XX+YY)/2).
- Array<dcomplex>::const_contiter iterEnd = itsPredictVis.cend();
- float* amplp = ampl.data();
- for (Array<dcomplex>::const_contiter iter=itsPredictVis.cbegin();
- iter!=iterEnd; ++iter) {
- float a = abs(*iter);
- ++iter; ++iter; ++iter; // skip XY and YX
- // Add amplitude.
- *amplp++ += 0.5*(a + abs(*iter));
- }
- }
-
- void DemixWorker::predictAteam (const vector<Patch::ConstPtr>& patchList,
- uint ntime, double time, double timeStep)
- {
- itsAteamAmplSel = false;
- itsSolveStation = false;
- for (uint dr=0; dr<patchList.size(); ++dr) {
- itsAteamAmpl[dr] = 0;
- MatrixIterator<float> miter(itsAteamAmpl[dr]);
- MatrixIterator<double> uvwiter(itsStationUVW);
- double t = time;
- for (uint j=0; j<ntime; ++j) {
- itsPredictVis = dcomplex();
- Simulator simulator(itsMix->phaseRef(),
- itsMix->nstation(),
- itsMix->nbl(),
- itsMix->freqDemix().size(),
- itsMix->baselines(),
- itsMix->freqDemix(),
- uvwiter.matrix(),
- itsPredictVis);
- for(size_t i = 0; i < patchList[dr]->nComponents(); ++i)
- {
- simulator.simulate(patchList[dr]->component(i));
- }
- // Get and apply beam.
- applyBeam (t, patchList[dr]->position(), True);
- // Keep the StokesI ampl ((XX+YY)/2).
- addStokesI (miter.matrix());
- miter.next();
- uvwiter.next();
- t += timeStep;
- }
- }
- // Determine per A-source which stations to use.
- itsSrcSet.resize (0);
- vector<uint> antCount (itsMix->nstation());
- for (uint i=0; i<patchList.size(); ++i) {
- // Count the stations of baselines with sufficient amplitude.
- std::fill (antCount.begin(), antCount.end(), 0);
- MatrixIterator<float> miter(itsAteamAmpl[i], 0, 2);
- for (uint j=0; j<itsMix->nbl(); ++j, miter.next()) {
- if (itsMix->verbose() > 11) {
- cout<<"source "<<i<<", baseline "<<itsMix->getAnt1()[j]
- <<' '<<itsMix->getAnt2()[j]
- <<" has max ampl " << max(miter.matrix())<<endl;
- }
- if (max(miter.matrix()) >= itsMix->ateamAmplThreshold()) {
- antCount[itsMix->getAnt1()[j]]++;
- antCount[itsMix->getAnt2()[j]]++;
- itsAteamAmplSel(j,i) = true;
- }
- }
- // Determine which stations have sufficient occurrence.
- itsStationsToUse[i].resize (0);
- for (uint j=0; j<antCount.size(); ++j) {
- if (antCount[j] >= itsMix->minNBaseline()) {
- itsStationsToUse[i].push_back (j);
- } else {
- if (itsMix->verbose() > 10) {
- cout << "ignore station " << j << " for source "
- << patchList[i]->name()
- << " (occurs in " << antCount[j] << " baselines)" << endl;
- }
- }
- }
- // Use this A-team source if more than N stations have matched.
- // If used, count the stations for it.
- // For fewer stations there are insufficient baselines.
- if (itsStationsToUse[i].size() >= itsMix->minNStation()) {
- itsSrcSet.push_back (i);
- itsNrSourcesDemixed[i]++;
- for (uint j=0; j<itsStationsToUse[i].size(); ++j) {
- uint st = itsStationsToUse[i][j];
- itsStatSourceDemixed(i,st)++;
- itsSolveStation[st] = true;
- }
- } else {
- itsStationsToUse[i].clear();
- itsSrcSet.push_back (i);
- if (itsMix->verbose() > 10) {
- cout << "ignore source " << patchList[i]->name()
- << " (" << itsStationsToUse[i].size() << " stations)" << endl;
- }
- }
- }
- // Add to statistics if a station is solved for.
- for (size_t i=0; i<itsSolveStation.size(); ++i) {
- if (itsSolveStation[i]) {
- itsNrStationsDemixed[i]++;
- }
- }
- }
-
- void DemixWorker::average (const DPBuffer* bufin, uint nbufin,
- DPBuffer* bufout)
- {
- for (uint i=0; i<nbufin; ++i) {
- itsAvgStepSubtr->process (bufin[i]);
- }
- itsAvgStepSubtr->finish();
- ASSERT (itsAvgResultFull->size() <= itsMix->ntimeOutSubtr());
- for (uint i=0; i<itsAvgResultFull->size(); ++i) {
- bufout[i] = itsAvgResultFull->get()[i];
- }
- itsAvgResultFull->clear();
- }
-
- float DemixWorker::findMedian (const Cube<float>& ampl,
- const bool* selbl)
- {
- // Take the median for only the baselines with sufficient amplitude.
- // Copy the amplitudes of those baselines to make them contiguous.
- const IPosition& shp = ampl.shape();
- float* tmp = &(itsTmpAmpl[0]);
- uint nrtmp = 0;
- for (uint bl=0; bl<shp[1]; ++bl) {
- if (selbl[bl]) {
- for (uint tm=0; tm<shp[2]; ++tm) {
- for (uint ch=0; ch<shp[0]; ++ch) {
- tmp[nrtmp++] = ampl(ch,bl,tm);
- }
- }
- }
- }
- // Median is middle element.
- if (nrtmp == 0) {
- return 0;
- }
- return GenSort<float>::kthLargest (tmp, nrtmp, (nrtmp-1)/2);
- }
-
- void DemixWorker::setupDemix (uint chunkNr)
- {
- // Decide if the target has to be included, ignored, or deprojected
- // which depends on the ratio target/Ateam of the total ampl.
- // Fill in the steps to be executed by removing the too weak A-sources.
- uint nsrc = itsSrcSet.size();
- itsPhaseShifts.resize (nsrc);
- itsFirstSteps.resize (nsrc+1);
- // Add target.
- itsFirstSteps[nsrc] = itsOrigFirstSteps[itsOrigFirstSteps.size() - 1];
- float minMedAmpl = 1e30;
- float maxMedAmpl = 0;
- // Add sources to be demixed.
- // Determine their minimum and maximum amplitude.
- for (uint i=0; i<nsrc; ++i) {
- uint srcOrig = itsSrcSet[i];
- itsPhaseShifts[i] = itsOrigPhaseShifts[srcOrig];
- itsFirstSteps[i] = itsOrigFirstSteps[srcOrig];
- itsDemixList[i] = itsMix->ateamDemixList()[srcOrig];
- float medAmpl = findMedian (itsAteamAmpl[srcOrig],
- &(itsAteamAmplSel(0,srcOrig)));
- if (medAmpl < minMedAmpl) {
- minMedAmpl = medAmpl;
- }
- if (medAmpl > maxMedAmpl) {
- maxMedAmpl = medAmpl;
- }
- }
- float targetMedAmpl = findMedian(itsTargetAmpl, itsMix->selTarget().data());
- float targetMaxAmpl = max(itsTargetAmpl);
- itsIncludeTarget = false;
- itsIgnoreTarget = false;
- itsNModel = nsrc;
- itsNSubtr = nsrc;
- itsNDir = nsrc+1;
- // For special purposes (testing) it is possible to set target handling,
- // but normally it is dependent on the data.
- if (itsMix->targetHandling() == 1) {
- itsIncludeTarget = true;
- itsNrIncludeCloseTarget++;
- } else if (itsMix->targetHandling() == 2) {
- itsNrDeprojectTarget++;
- } else if (itsMix->targetHandling() == 3) {
- itsIgnoreTarget = true;
- itsNrIgnoreTarget++;
- } else {
- if (targetMedAmpl / maxMedAmpl > itsMix->ratio1() ||
- targetMaxAmpl > itsMix->targetAmplThreshold()) {
- itsIncludeTarget = true;
- itsNrIncludeStrongTarget++;
- if (itsMix->verbose() > 10) {
- cout << "include strong target" << endl;
- }
- } else if (! itsMix->isAteamNearby()) {
- if (itsMix->verbose() > 10) {
- cout << "deproject target" << endl;
- }
- itsNrDeprojectTarget++;
- } else if (targetMedAmpl / minMedAmpl > itsMix->ratio2()) {
- itsIncludeTarget = true;
- itsNrIncludeCloseTarget++;
- if (itsMix->verbose() > 10) {
- cout << "include close target" << endl;
- }
- } else {
- itsIgnoreTarget = true;
- itsNrIgnoreTarget++;
- if (itsMix->verbose() > 10) {
- cout << "ignore target" << endl;
- }
- }
- if (itsMix->verbose() > 10) {
- cout << " targetMedAmpl=" << targetMedAmpl
- << " targetMaxAmpl=" << targetMaxAmpl
- << " maxAteamMedAmpl=" << maxMedAmpl
- << " minAteamMedAmpl=" << minMedAmpl << endl;
- }
- }
- // Determine the unknowns to be solved.
- // The unknowns are complex values of a Jones matrix per source/station.
- // thus 8 real unknowns per source/station.
- // Their names are DirectionalGain:i:j:Type::Station::Source
- // where i:j gives the Jones element and Type is Real or Imag.
- // Fill the map of source/station to unknown seqnr (4 pol, ampl/phase).
- uint nUnknown = 0;
- for (uint dr=0; dr<itsNModel; ++dr) {
- uint drOrig = itsSrcSet[dr];
- // Initialize first.
- std::fill (itsUnknownsIndex[dr].begin(), itsUnknownsIndex[dr].end(), -1);
- if (itsMix->verbose() > 11) {
- cout << "stationstouse "<<drOrig<<" = "<<itsStationsToUse[drOrig]
- << endl;
- }
- for (uint j=0; j<itsStationsToUse[drOrig].size(); ++j) {
- uint st = itsStationsToUse[drOrig][j];
- itsUnknownsIndex[dr][st] = nUnknown;
- nUnknown += 8;
- }
- }
- if (itsMix->verbose() > 11) {
- cout<<"nunkb="<<nUnknown<<endl;
- }
- // Do the same for the target if it has to be solved as well.
- // Solve for all stations.
- // Also add the target step.
- std::fill (itsUnknownsIndex[itsNModel].begin(),
- itsUnknownsIndex[itsNModel].end(), -1);
- if (itsIncludeTarget) {
- itsSrcSet.push_back (itsMix->ateamList().size());
- for (uint j=0; j<itsUnknownsIndex[itsNModel].size(); ++j) {
- itsUnknownsIndex[itsNModel][j] = nUnknown;
- nUnknown += 8;
- }
- itsNModel++;
- }
- if (itsMix->verbose() > 11) {
- cout<<"nunka="<<nUnknown<<endl;
- }
- // Show info if needed.
- if (itsMix->verbose() > 0) {
- ostringstream os;
- os << "chunk" << setw(5) << chunkNr << ": ";
- if (itsIncludeTarget) {
- os << " include target ";
- } else if (itsIgnoreTarget) {
- os << " ignore target ";
- } else {
- os << " deproject target";
- }
- os << " ";
- for (size_t i=0; i<itsNSubtr; ++i) {
- if (i > 0) os << ", ";
- os << itsMix->ateamList()[itsSrcSet[i]]->name() << " ("
- << itsStationsToUse[itsSrcSet[i]].size() << " st)";
- }
- DPLOG_INFO (os.str(), true);
- }
- }
-
-
- void DemixWorker::applyBeam (double time, const Position& pos, bool apply)
- {
- // Get the beam for demix resolution and apply to itsPredictVis.
- applyBeam (time, pos, apply, itsMix->freqDemix(), itsPredictVis.data());
- }
-
- void DemixWorker::applyBeam (double time, const Position& pos, bool apply,
- const Vector<double>& chanFreqs,
- dcomplex* data)
- {
- // For test purposes applying the beam can be defeated.
- // In this way an exact comparison with the old demixer is possible.
- if (! apply) {
- return;
- }
- // Convert the directions to ITRF for the given time.
- itsMeasFrame.resetEpoch (MEpoch(MVEpoch(time/86400), MEpoch::UTC));
- StationResponse::vector3r_t refdir = dir2Itrf(itsDelayCenter);
- StationResponse::vector3r_t tiledir = dir2Itrf(itsTileBeamDir);
- MDirection dir (MVDirection(pos[0], pos[1]), MDirection::J2000);
- StationResponse::vector3r_t srcdir = dir2Itrf(dir);
- // Get the beam values for each station.
- uint nchan = chanFreqs.size();
- for (size_t st=0; st<itsMix->nstation(); ++st) {
- itsAntBeamInfo[st]->response (nchan, time, chanFreqs.cbegin(),
- srcdir, itsMix->getInfo().refFreq(),
- refdir, tiledir,
- &(itsBeamValues[nchan*st]));
- }
- // Apply the beam values of both stations to the predicted data.
- dcomplex tmp[4];
- for (size_t bl=0; bl<itsMix->nbl(); ++bl) {
- const StationResponse::matrix22c_t* left =
- &(itsBeamValues[nchan * itsMix->getAnt1()[bl]]);
- const StationResponse::matrix22c_t* right =
- &(itsBeamValues[nchan * itsMix->getAnt2()[bl]]);
- for (size_t ch=0; ch<nchan; ++ch) {
- dcomplex l[] = {left[ch][0][0], left[ch][0][1],
- left[ch][1][0], left[ch][1][1]};
- // Form transposed conjugate of right.
- dcomplex r[] = {conj(right[ch][0][0]), conj(right[ch][1][0]),
- conj(right[ch][0][1]), conj(right[ch][1][1])};
- // left*data
- tmp[0] = l[0] * data[0] + l[1] * data[2];
- tmp[1] = l[0] * data[1] + l[1] * data[3];
- tmp[2] = l[2] * data[0] + l[3] * data[2];
- tmp[3] = l[2] * data[1] + l[3] * data[3];
- // data*conj(right)
- data[0] = tmp[0] * r[0] + tmp[1] * r[2];
- data[1] = tmp[0] * r[1] + tmp[1] * r[3];
- data[2] = tmp[2] * r[0] + tmp[3] * r[2];
- data[3] = tmp[2] * r[1] + tmp[3] * r[3];
- data += 4;
- }
- }
- }
-
- StationResponse::vector3r_t DemixWorker::dir2Itrf (const MDirection& dir)
- {
- const MDirection& itrfDir = itsMeasConverter(dir);
- const Vector<Double>& itrf = itrfDir.getValue().getValue();
- StationResponse::vector3r_t vec;
- vec[0] = itrf[0];
- vec[1] = itrf[1];
- vec[2] = itrf[2];
- return vec;
- }
-
- void DemixWorker::addFactors (const DPBuffer& newBuf,
- Array<DComplex>& factorBuf)
- {
- // Nothing to do if only target direction.
- if (itsNDir <= 1) return;
- int ncorr = newBuf.getData().shape()[0];
- int nchan = newBuf.getData().shape()[1];
- int nbl = newBuf.getData().shape()[2];
- int ncc = ncorr*nchan;
- //# If ever in the future a time dependent phase center is used,
- //# the machine must be reset for each new time, thus each new call
- //# to process.
- // Add the weighted factors for each pair of directions.
- // The input factor is the phaseshift from target direction to
- // source direction. By combining them you get the shift from one
- // source direction to another.
- int dirnr = 0;
- for (uint i1=0; i1<itsNDir-1; ++i1) {
- for (uint i0=i1+1; i0<itsNDir; ++i0) {
- if (i0 == itsNDir-1) {
- // The last direction is the target direction, so no need to
- // combine the factors. Take conj to get shift source to target.
- for (int i=0; i<nbl; ++i) {
- const bool* flagPtr = newBuf.getFlags().data() + i*ncc;
- const float* weightPtr = newBuf.getWeights().data() + i*ncc;
- DComplex* factorPtr = factorBuf.data() + (dirnr*nbl + i)*ncc;
- const DComplex* phasor1 = itsPhaseShifts[i1]->getPhasors().data()
- + i*nchan;
- for (int j=0; j<nchan; ++j) {
- DBGASSERT (phasor1 < itsPhaseShifts[i1]->getPhasors().data()+itsPhaseShifts[i1]->getPhasors().size());
- DComplex factor = conj(*phasor1++);
- for (int k=0; k<ncorr; ++k) {
- DBGASSERT (flagPtr < newBuf.getFlags().data()+newBuf.getFlags().size());
- DBGASSERT (weightPtr < newBuf.getWeights().data()+newBuf.getWeights().size());
- if (! *flagPtr) {
- *factorPtr += factor * double(*weightPtr);
- }
- flagPtr++;
- weightPtr++;
- factorPtr++;
- }
- }
- }
- } else {
- // Different source directions; take both phase terms into account.
- for (int i=0; i<nbl; ++i) {
- const bool* flagPtr = newBuf.getFlags().data() + i*ncc;
- const float* weightPtr = newBuf.getWeights().data() + i*ncc;
- DComplex* factorPtr = factorBuf.data() + (dirnr*nbl + i)*ncc;
- const DComplex* phasor0 = itsPhaseShifts[i0]->getPhasors().data()
- + i*nchan;
- const DComplex* phasor1 = itsPhaseShifts[i1]->getPhasors().data()
- + i*nchan;
- for (int j=0; j<nchan; ++j) {
- DComplex factor = *phasor0++ * conj(*phasor1++);
- for (int k=0; k<ncorr; ++k) {
- if (! *flagPtr) {
- *factorPtr += factor * double(*weightPtr);
- }
- flagPtr++;
- weightPtr++;
- factorPtr++;
- }
- }
- }
- }
- // Next direction pair.
- dirnr++;
- }
- }
- }
-
- void DemixWorker::makeFactors (const Array<DComplex>& bufIn,
- Array<DComplex>& bufOut,
- const Cube<float>& weightSums,
- uint nChanOut,
- uint nChanAvg)
- {
- // Nothing to do if only target direction.
- if (itsNDir <= 1) return;
- ASSERT (! weightSums.empty());
- bufOut.resize (IPosition(5, itsNDir, itsNDir,
- itsMix->ncorr(), nChanOut, itsMix->nbl()));
- bufOut = DComplex(1,0);
- int ncc = itsMix->ncorr()*nChanOut;
- int nccdd = ncc*itsNDir*itsNDir;
- int nccin = itsMix->ncorr()*itsMix->nchanIn();
- // Fill the factors for each combination of different directions.
- uint dirnr = 0;
- for (uint d0=0; d0<itsNDir; ++d0) {
- for (uint d1=d0+1; d1<itsNDir; ++d1) {
- // Average factors by summing channels.
- // Note that summing in time is done in addFactors.
- // The sum per output channel is divided by the summed weight.
- // Note there is a summed weight per baseline,outchan,corr.
- for (int k=0; k<int(itsMix->nbl()); ++k) {
- const DComplex* phin = bufIn.data() + (dirnr*itsMix->nbl() + k)*nccin;
- DComplex* ph1 = bufOut.data() + k*nccdd + (d0*itsNDir + d1);
- DComplex* ph2 = bufOut.data() + k*nccdd + (d1*itsNDir + d0);
- const float* weightPtr = weightSums.data() + k*ncc;
- for (uint c0=0; c0<nChanOut; ++c0) {
- // Sum the factors for the input channels to average.
- DComplex sum[4];
- // In theory the last output channel could consist of fewer
- // input channels, so take care of that.
- uint nch = std::min(nChanAvg, itsMix->nchanIn()-c0*nChanAvg);
- for (uint c1=0; c1<nch; ++c1) {
- for (uint j=0; j<itsMix->ncorr(); ++j) {
- DBGASSERT (phin < bufIn.data()+bufIn.size());
- sum[j] += *phin++;
- }
- }
- for (uint j=0; j<itsMix->ncorr(); ++j) {
- DBGASSERT (ph1 < bufOut.data()+bufOut.size());
- DBGASSERT (ph2 < bufOut.data()+bufOut.size());
- DBGASSERT (weightPtr < weightSums.data() + weightSums.size());
- if (*weightPtr == 0) {
- *ph1 = 0;
- } else {
- *ph1 = sum[j] / double(*weightPtr++);
- }
- *ph2 = conj(*ph1);
- ph1 += itsNDir*itsNDir;
- ph2 += itsNDir*itsNDir;
- }
- }
- }
- // Next input direction pair.
- dirnr++;
- }
- }
- //cout<<"makefactors "<<weightSums<<bufOut;
- }
-
- void DemixWorker::deproject (Array<DComplex>& factors,
- vector<MultiResultStep*> avgResults,
- uint resultIndex)
- {
- // Sources without a model have to be deprojected.
- // Optionally no deprojection of target direction.
- uint nrDeproject = itsNDir - itsNModel;
- if (itsIgnoreTarget) {
- nrDeproject--;
- }
- // Nothing to do if only target direction or nothing to deproject.
- if (itsNDir <= 1 || nrDeproject == 0) return;
- // Get pointers to the data for the various directions.
- vector<Complex*> resultPtr(itsNDir);
- for (uint dr=0; dr<itsNDir; ++dr) {
- uint drOrig = avgResults.size() - 1;
- if (dr < itsSrcSet.size()) drOrig = itsSrcSet[dr];
- resultPtr[dr] = avgResults[drOrig]->get()[resultIndex].getData().data();
- }
- // The projection matrix is given by
- // P = I - A * inv(A.T.conj * A) * A.T.conj
- // where A is the last column of the demixing matrix M.
- // The BBS equations get:
- // P * M' * v_predict = P * v_averaged
- // where M' is obtained by removing the last column of demixing matrix M.
- // The dimensions of the matrices/vectors are:
- // P : NxN
- // M' : Nx(N-1)
- // v_predict : (N-1) x 1
- // v_averaged: N x 1
- // where N is the number of modeled sources to use in demixing.
- // In the general case S sources might not have a source model.
- // In that case A is the NxS matrix containing all these columns
- // from M and M' is the Nx(N-S) matrix without all these columns.
-
- // Calculate P for all baselines,channels,correlations.
- IPosition shape = factors.shape();
- int nvis = shape[2] * shape[3] * shape[4];
- shape[1] = itsNModel;
- Array<DComplex> newFactors (shape);
- IPosition inShape (2, itsNDir, itsNDir);
- IPosition outShape(2, itsNDir, itsNModel);
- {
- Matrix<DComplex> a(itsNDir, nrDeproject);
- Matrix<DComplex> ma(itsNDir, itsNModel);
- vector<DComplex> vec(itsNDir);
- for (int i=0; i<nvis; ++i) {
- // Split the matrix into the modeled and deprojected sources.
- // Copy the columns to the individual matrices.
- const DComplex* inptr = factors.data() + i*itsNDir*itsNDir;
- DComplex* outptr = newFactors.data() + i*itsNDir*itsNModel;
- Matrix<DComplex> out (outShape, outptr, SHARE);
- // Copying a bit of data is probably faster than taking a matrix
- // subset.
- objcopy (ma.data(), inptr, itsNDir*itsNModel);
- objcopy (a.data(), inptr + itsNDir*itsNModel, itsNDir*nrDeproject);
- // Calculate conjugated transpose of A, multiply with A, and invert.
- Matrix<DComplex> at(adjoint(a));
- Matrix<DComplex> ata(invert(product(at, a)));
- if (ata.empty()) {
- ata.resize (nrDeproject, nrDeproject);
- }
- DBGASSERT(ata.ncolumn()==nrDeproject && ata.nrow()==nrDeproject);
- // Calculate P = I - A * ata * A.T.conj
- Matrix<DComplex> aata(product(a,ata));
- Matrix<DComplex> p (-product(product(a, ata), at));
- Vector<DComplex> diag(p.diagonal());
- diag += DComplex(1,0);
- // Multiply the demixing factors with P (get stored in newFactors).
- out = product(p, ma);
- // Multiply the averaged data point with P.
- std::fill (vec.begin(), vec.end(), DComplex());
- for (uint j=0; j<itsNDir; ++j) {
- for (uint k=0; k<itsNDir; ++k) {
- vec[k] += DComplex(resultPtr[j][i]) * p(k,j);
- }
- }
- // Put result back in averaged data for those sources.
- for (uint j=0; j<itsNDir; ++j) {
- resultPtr[j][i] = vec[j];
- }
- }
- }
- // Set the new demixing factors.
- factors.reference (newFactors);
- }
-
- void DemixWorker::handleDemix (DPBuffer* bufout, vector<double>* solutions,
- double time, double timeStep)
- {
- demix (solutions, time, timeStep);
- // If selection was done, merge the subtract results back into the
- // buffer.
- if (itsMix->doSubtract() && itsMix->selBL().hasSelection()) {
- mergeSubtractResult();
- }
- // Clear the input buffers.
- for (size_t i=0; i<itsAvgResults.size(); ++i) {
- itsAvgResults[i]->clear();
- }
- // Store the output buffers.
- for (uint i=0; i<itsNTimeOutSubtr; ++i) {
- // DPBuffer copying uses reference semantics which is fine.
- // At the end the itsAvgResultxxx buffers get removed, thus the ones
- // in bufout cannot be overwritten.
- if (itsMix->doSubtract() && itsMix->selBL().hasSelection()) {
- bufout[i] = itsAvgResultFull->get()[i];
- } else {
- bufout[i] = itsAvgResultSubtr->get()[i];
- }
- }
- // Clear the output buffers.
- itsAvgResultFull->clear();
- itsAvgResultSubtr->clear();
- }
-
- void DemixWorker::mergeSubtractResult()
- {
- // Merge the selected baselines from the subtract buffer into the
- // full buffer. Do it for all timestamps.
- for (uint i=0; i<itsNTimeOutSubtr; ++i) {
- const Array<Complex>& arr = itsAvgResultSubtr->get()[i].getData();
- size_t nr = arr.shape()[0] * arr.shape()[1];
- const Complex* in = arr.data();
- Complex* out = itsAvgResultFull->get()[i].getData().data();
- for (size_t j=0; j<itsFilter.getIndicesBL().size(); ++j) {
- size_t inx = itsFilter.getIndicesBL()[j];
- memcpy (out+inx*nr, in+j*nr, nr*sizeof(Complex));
- }
- }
- }
-
- void DemixWorker::demix (vector<double>* solutions,
- double time, double timeStep)
- {
- // Determine the various sizes.
- const size_t nTime = itsAvgResults[itsSrcSet[0]]->size();
- const size_t nTimeSubtr = itsAvgResultSubtr->size();
- const size_t multiplier = itsMix->ntimeAvg() / itsMix->ntimeAvgSubtr();
- const size_t nSt = itsMix->nstation();
- const size_t nBl = itsMix->baselines().size();
- const size_t nCh = itsMix->freqDemix().size();
- const size_t nChSubtr = itsMix->freqSubtr().size();
- const size_t nCr = 4;
- // Define various cursors to iterate through arrays.
- const_cursor<Baseline> cr_baseline(&(itsMix->baselines()[0]));
- // Do a solve and subtract for each predict time slot.
- // Determine the time step for the subtract.
- double subtrTimeStep = timeStep / multiplier;
- for (size_t ts=0; ts<nTime; ++ts) {
- itsTimerPredict.start();
- // Compute the model visibilities for each A-team source.
- size_t stride_model[3] = {1, nCr, nCr*nCh};
- for (size_t dr=0; dr<itsNModel; ++dr) {
- uint drOrig = itsSrcSet[dr];
- // Split the baseline UVW coordinates per station.
- nsplitUVW (itsMix->uvwSplitIndex(), itsMix->baselines(),
- itsAvgResults[drOrig]->get()[ts].getUVW(), itsUVW);
- if (itsMix->verbose() > 13) {
- cout <<"uvw"<<dr<<'='<<itsUVW;
- }
- // Initialize this part of the buffer.
- itsModelVisDemix[dr]=dcomplex();
- if (dr == itsNSubtr) {
- // This is the target which consists of multiple components.
- // To each of them the beam must be applied.
- for (uint i=0; i<itsMix->targetDemixList().size(); ++i) {
- itsPredictVis = dcomplex();
- Simulator simulator(itsMix->phaseRef(), nSt, nBl, nCh,
- itsMix->baselines(), itsMix->freqDemix(),
- itsUVW, itsPredictVis);
- for(size_t j = 0; j < itsMix->targetDemixList()[i]->nComponents(); ++j)
- {
- simulator.simulate(itsMix->targetDemixList()[i]->component(j));
- }
- applyBeam (time, itsMix->targetDemixList()[i]->position(),
- itsMix->applyBeam());
- itsModelVisDemix[dr]+=itsPredictVis;
- }
- } else {
- itsModelVisDemix[dr]=dcomplex();
- Simulator simulator(itsDemixList[dr]->position(), nSt, nBl, nCh,
- itsMix->baselines(), itsMix->freqDemix(),
- itsUVW, itsModelVisDemix[dr]);
- for(size_t i = 0; i < itsDemixList[dr]->nComponents(); ++i)
- {
- simulator.simulate(itsDemixList[dr]->component(i));
- }
- applyBeam (time, itsMix->ateamDemixList()[drOrig]->position(),
- itsMix->applyBeam(), itsMix->freqDemix(),
- itsModelVisDemix[dr].data());
- }
- } // end nModel
- itsTimerPredict.stop();
- if (itsMix->verbose() > 13) {
- cout<<"modelvis="<<itsModelVisDemix<<endl;
- }
- // A Jones matrix will be estimated for each pair of stations and
- // direction.
- // A single (overdetermined) non-linear set of equations for all
- // stations and directions is solved iteratively. The influence of
- // each direction on each other direction is given by the mixing
- // matrix.
- itsTimerSolve.start();
- const_cursor<bool> cr_flag =
- casa_const_cursor(itsAvgResults[itsSrcSet[0]]->get()[ts].getFlags());
- const_cursor<float> cr_weight =
- casa_const_cursor(itsAvgResults[itsSrcSet[0]]->get()[ts].getWeights());
- const_cursor<dcomplex> cr_mix = casa_const_cursor(itsFactors[ts]);
- if (itsMix->verbose() > 14) {
- cout << "demixfactor "<<ts<<" = "<<itsFactors[ts]<<endl;
- }
- // Create a cursor per source.
- vector<const_cursor<fcomplex> > cr_data(itsNModel);
- vector<const_cursor<dcomplex> > cr_model(itsNModel);
- for (size_t dr=0; dr<itsNModel; ++dr) {
- uint drOrig = itsSrcSet[dr];
- cr_data[dr] =
- casa_const_cursor(itsAvgResults[drOrig]->get()[ts].getData());
- cr_model[dr] =
- const_cursor<dcomplex>(itsModelVisDemix[dr].data(), 3, stride_model);
- }
- // If solving the system succeeds, increment nconverged.
- bool converged = itsEstimate.estimate (itsUnknownsIndex,
- itsSrcSet,
- cr_baseline, cr_data,
- cr_model, cr_flag,
- cr_weight, cr_mix,
- itsMix->defaultGain(),
- itsMix->solveBoth(),
- itsMix->verbose());
- // Copy solutions to overall solution array.
- solutions[ts] = itsEstimate.getSolution();
- itsNrSolves++;
- if (converged) {
- itsNrConverged++;
- itsNrIter += itsEstimate.nIterations();
- }
- itsTimerSolve.stop();
-
- // Compute the residual.
- //
- // All the so-called "subtract sources" are subtracted from the
- // observed data. The previously estimated Jones matrices, as well as
- // the appropriate mixing weight are applied before subtraction.
- //
- // Note that the resolution of the residual can differ from the
- // resolution at which the Jones matrices were estimated.
- if (itsMix->doSubtract()) {
- itsTimerSubtract.start();
- // Get middle of first subtract time interval.
- double subtrTime = time - 0.5*(timeStep - subtrTimeStep);
- for (size_t ts_subtr = multiplier * ts,
- ts_subtr_end = min(ts_subtr + multiplier, nTimeSubtr);
- ts_subtr != ts_subtr_end; ++ts_subtr) {
- // Get the observed amplitude per baseline for the middle channel.
- const Complex* data =
- itsAvgResultSubtr->get()[ts_subtr].getData().data() + nChSubtr/2*4;
- for (size_t bl=0; bl<nBl; ++bl) {
- itsSumSourceAmpl[bl] = 0;
- itsObservedAmpl[bl] = 0.5 * (abs(data[0]) + abs(data[3]));
- data += 4*nChSubtr;
- }
- for (size_t dr=0; dr<itsNSubtr; ++dr) {
- uint drOrig = itsSrcSet[dr];
- // Re-use simulation used for estimating Jones matrices if possible.
- cursor<dcomplex> cr_model_subtr(itsModelVisSubtr[dr].data(),
- 3, stride_model);
- // Re-simulate if required.
- if (multiplier != 1 || nCh != nChSubtr) {
- nsplitUVW (itsMix->uvwSplitIndex(), itsMix->baselines(),
- itsAvgResultSubtr->get()[ts_subtr].getUVW(), itsUVW);
- // Rotate the UVW coordinates for the target direction to the
- // direction of source to subtract. This is required because at
- // the resolution of the residual the UVW coordinates for
- // directions other than the target are unavailable (unless the
- // resolution of the residual is equal to the resolution at
- // which the Jones matrices were estimated, of course).
- rotateUVW (itsMix->phaseRef(),
- itsMix->ateamList()[drOrig]->position(), nSt,
- itsUVW.data());
- // Initialize the visibility buffer.
- std::fill (itsModelVisSubtr.begin(), itsModelVisSubtr.end(), dcomplex());
- // Simulate visibilities at the resolution of the residual.
- size_t stride_model_subtr[3] = {1, nCr, nCr * nChSubtr};
- cr_model_subtr = cursor<dcomplex>(itsModelVisSubtr[0].data(), 3,
- stride_model_subtr);
-
- Simulator simulator(itsMix->ateamList()[drOrig]->position(),
- nSt, nBl, nChSubtr, itsMix->baselines(),
- itsMix->freqSubtr(), itsUVW,
- itsModelVisSubtr[0]);
- for(size_t i = 0; i < itsMix->ateamList()[drOrig]->nComponents(); ++i)
- {
- simulator.simulate(itsMix->ateamList()[drOrig]->component(i));
- }
-
- applyBeam (subtrTime,
- itsMix->ateamDemixList()[drOrig]->position(),
- itsMix->applyBeam(),
- itsMix->freqSubtr(),
- itsModelVisSubtr[0].data());
- }
-
- // Apply Jones matrices.
- size_t stride_unknowns[2] = {1, 8};
- const_cursor<double> cr_unknowns(&(solutions[ts][drOrig * nSt * 8]),
- 2, stride_unknowns);
- apply (nBl, nChSubtr, cr_baseline, cr_unknowns, cr_model_subtr);
-
- // Subtract the source contribution from the data.
- cursor<fcomplex> cr_residual =
- casa_cursor(itsAvgResultSubtr->get()[ts_subtr].getData());
-
- // Construct a cursor to iterate over a slice of the mixing matrix
- // at the resolution of the residual. The "to" and "from" direction
- // are fixed. Since the full mixing matrix is 5-D, the slice is
- // therefore 3-D. Each individual value in the slice quantifies the
- // influence of the source to subtract on the target direction for
- // a particular correlation, channel, and baseline.
- //
- // The target direction is the direction with the highest index by
- // convention, i.e. index itsNDir - 1. The directions to subtract
- // have the lowest indices by convention, i.e. indices
- // [0, nDrSubtr).
- const IPosition& stride_mix_subtr =
- itsFactorsSubtr[ts_subtr].steps();
- size_t stride_mix_subtr_slice[3] = {
- static_cast<size_t>(stride_mix_subtr[2]),
- static_cast<size_t>(stride_mix_subtr[3]),
- static_cast<size_t>(stride_mix_subtr[4])
- };
- ASSERT(stride_mix_subtr_slice[0] == itsNDir*itsNDir &&
- stride_mix_subtr_slice[1] == itsNDir*itsNDir*nCr &&
- stride_mix_subtr_slice[2] == itsNDir*itsNDir*nCr*nChSubtr);
-
- IPosition offset(5, itsNDir - 1, dr, 0, 0, 0);
- const_cursor<dcomplex> cr_mix_subtr
- (&(itsFactorsSubtr[ts_subtr](offset)), 3, stride_mix_subtr_slice);
-
- // Subtract the source.
- // It fills in the subtracted amplitude per baseline.
- subtract (nBl, nChSubtr, cr_baseline, cr_residual, cr_model_subtr,
- cr_mix_subtr, itsSourceAmpl);
- // Calculate the mean percentage amplitude subtracted per source.
- // This array is ordered [nbl,nsrc]
- for (size_t bl=0; bl<nBl; ++bl) {
- itsSumSourceAmpl[bl] += itsSourceAmpl[bl];
- }
- addMeanM2 (itsSourceAmpl, drOrig);
- } // end itsNSubtr
- // Calculate the totals per baseline.
- addMeanM2 (itsSumSourceAmpl, itsAmplSubtrNr.shape()[1] - 1);
- subtrTime += subtrTimeStep;
- } // end ts_subtr
- itsTimerSubtract.stop();
- } // end doSubtract
- time += timeStep;
- } // end nTime
- }
-
- void DemixWorker::addMeanM2 (const vector<float>& sourceAmpl, uint dr)
- {
- for (size_t bl=0; bl<sourceAmpl.size(); ++bl) {
- if (itsObservedAmpl[bl] != 0 && sourceAmpl[bl] != 0) {
- // Calculate mean and stddev in a running way using a
- // numerically stable algorithm
- // See en.wikipedia.org/wiki/Algorithms_for_calculating_variance
- itsAmplSubtrNr(bl,dr)++;
- double perc = sourceAmpl[bl] / itsObservedAmpl[bl];
- double delta = perc - itsAmplSubtrMean(bl,dr);
- itsAmplSubtrMean(bl,dr) += delta/itsAmplSubtrNr(bl,dr);
- itsAmplSubtrM2(bl,dr) += delta*(perc-itsAmplSubtrMean(bl,dr));
- }
- }
- }
-
-
- namespace
- {
- string toString (double value)
- {
- ostringstream os;
- os << setprecision(16) << value;
- return os.str();
- }
- } //# end unnamed namespace
-
- } //# end namespace DPPP
-} //# end namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/Demixer.cc b/CEP/DP3/DPPP/src/Demixer.cc
deleted file mode 100644
index 1ab55014f84cf3ee7a3344555043f2cc3c1020b0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Demixer.cc
+++ /dev/null
@@ -1,1161 +0,0 @@
-//# Demixer.cc: DPPP step class to subtract A-team sources
-//# Copyright (C) 2011
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Demixer.h>
-#include <DPPP/Apply.h>
-#include <DPPP/Averager.h>
-#include <DPPP/CursorUtilCasa.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/EstimateMixed.h>
-#include <DPPP/PhaseShift.h>
-#include <DPPP/Simulate.h>
-#include <DPPP/SourceDBUtil.h>
-#include <DPPP/SubtractMixed.h>
-#include <DPPP/MSReader.h>
-#include <DPPP/Simulator.h>
-
-#include <ParmDB/Axis.h>
-#include <ParmDB/SourceDB.h>
-#include <ParmDB/ParmDB.h>
-#include <ParmDB/ParmSet.h>
-#include <ParmDB/ParmCache.h>
-#include <ParmDB/Parm.h>
-
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <Common/OpenMP.h>
-#include <Common/StreamUtil.h>
-#include <Common/lofar_iomanip.h>
-#include <Common/lofar_iostream.h>
-#include <Common/lofar_fstream.h>
-
-#include <casacore/casa/Quanta/MVAngle.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Arrays/Matrix.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/MatrixMath.h>
-#include <casacore/scimath/Mathematics/MatrixMathLA.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- using LOFAR::operator<<;
-
- namespace
- {
- string toString (double value);
- } //# end unnamed namespace
-
- Demixer::Demixer (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsSkyName (parset.getString(prefix+"skymodel", "sky")),
- itsInstrumentName (parset.getString(prefix+"instrumentmodel",
- "instrument")),
- itsDefaultGain (parset.getDouble(prefix+"defaultgain",1.0)),
- itsMaxIter (parset.getInt(prefix+"maxiter",50)),
- itsSelBL (parset, prefix, false, "cross"),
- itsFilter (input, itsSelBL),
- itsAvgResultSubtr (0),
- itsIgnoreTarget (parset.getBool (prefix+"ignoretarget", false)),
- itsTargetSource (parset.getString(prefix+"targetsource", string())),
- itsSubtrSources (parset.getStringVector (prefix+"subtractsources")),
- itsModelSources (parset.getStringVector (prefix+"modelsources",
- vector<string>())),
- itsExtraSources (parset.getStringVector (prefix+"othersources",
- vector<string>())),
-// itsCutOffs (parset.getDoubleVector (prefix+"elevationcutoffs",
-// vector<double>())),
-// itsJointSolve (parset.getBool (prefix+"jointsolve", true)),
- itsPropagateSolutions(parset.getBool (prefix+"propagatesolutions",
- false)),
- itsNDir (0),
- itsNModel (0),
- itsNStation (0),
- itsNBl (0),
- itsNCorr (0),
- itsNChanIn (0),
- itsNTimeIn (0),
- itsNTimeDemix (0),
- itsNChanAvgSubtr (parset.getUint (prefix+"freqstep", 1)),
- itsNTimeAvgSubtr (parset.getUint (prefix+"timestep", 1)),
- itsNChanOutSubtr (0),
- itsNTimeOutSubtr (0),
- itsNTimeChunk (parset.getUint (prefix+"ntimechunk", 0)),
- itsNTimeChunkSubtr(0),
- itsNChanAvg (parset.getUint (prefix+"demixfreqstep",
- itsNChanAvgSubtr)),
- itsNTimeAvg (parset.getUint (prefix+"demixtimestep",
- itsNTimeAvgSubtr)),
- itsNChanOut (0),
- itsNTimeOut (0),
- itsTimeIntervalAvg(0),
- itsTimeIndex (0),
- itsNConverged (0)
- {
- // Get and set solver options.
-// itsSolveOpt.maxIter =
-// parset.getUint (prefix+"Solve.Options.MaxIter", 300);
-// itsSolveOpt.epsValue =
-// parset.getDouble(prefix+"Solve.Options.EpsValue", 1e-9);
-// itsSolveOpt.epsDerivative =
-// parset.getDouble(prefix+"Solve.Options.EpsDerivative", 1e-9);
-// itsSolveOpt.colFactor =
-// parset.getDouble(prefix+"Solve.Options.ColFactor", 1e-9);
-// itsSolveOpt.lmFactor =
-// parset.getDouble(prefix+"Solve.Options.LMFactor", 1.0);
-// itsSolveOpt.balancedEq =
-// parset.getBool (prefix+"Solve.Options.BalancedEqs", false);
-// itsSolveOpt.useSVD =
-// parset.getBool (prefix+"Solve.Options.UseSVD", true);
-
- // Note:
- // Directions of unknown sources can be given in the PhaseShift step like
- // demixstepname.sourcename.phasecenter
-
- ASSERTSTR (!(itsSkyName.empty() || itsInstrumentName.empty()),
- "An empty name is given for the sky and/or instrument model");
- ASSERTSTR (!itsIgnoreTarget || itsTargetSource.empty(),
- "Target source name cannot be given if ignoretarget=true");
- // Add a null step as last step in the filter.
- DPStep::ShPtr nullStep(new NullStep());
- itsFilter.setNextStep (nullStep);
- // Default nr of time chunks is maximum number of threads.
- if (itsNTimeChunk == 0) {
- itsNTimeChunk = OpenMP::maxThreads();
- }
- // Check that time windows fit integrally.
- ASSERTSTR ((itsNTimeChunk * itsNTimeAvg) % itsNTimeAvgSubtr == 0,
- "time window should fit final averaging integrally");
- itsNTimeChunkSubtr = (itsNTimeChunk * itsNTimeAvg) / itsNTimeAvgSubtr;
- // Collect all source names.
- itsNModel = itsSubtrSources.size() + itsModelSources.size();
- itsNDir = itsNModel + itsExtraSources.size() + 1;
- itsAllSources.reserve (itsNDir);
- itsAllSources.insert (itsAllSources.end(),
- itsSubtrSources.begin(), itsSubtrSources.end());
- itsAllSources.insert (itsAllSources.end(),
- itsModelSources.begin(), itsModelSources.end());
- itsAllSources.insert (itsAllSources.end(),
- itsExtraSources.begin(), itsExtraSources.end());
- itsAllSources.push_back (itsTargetSource);
-
- // Get the source info of all patches from the SourceDB table.
- BBS::SourceDB sourceDB(BBS::ParmDBMeta("", itsSkyName), false);
- vector<string> patchNames(itsAllSources);
- // If the target source is given, add it to the model.
- // Because the target source has to be the last direction, it means
- // that (for the time being) no extra sources can be given.
- if (! itsTargetSource.empty()) {
- patchNames[itsNModel++] = itsTargetSource;
- // The target has to be the last demix direction.
- // If it has a source model, there cannot be any extra source
- // because the sources to be predicted have to be a consecutive vector.
- ASSERTSTR (itsExtraSources.empty(), "Currently no extrasources can "
- "be given if the targetsource is given");
- }
- itsPatchList = makePatches (sourceDB, patchNames, itsNModel);
- ASSERT(itsPatchList.size() == itsNModel);
-
- // Size buffers.
- itsFactors.resize (itsNTimeChunk);
- itsFactorsSubtr.resize (itsNTimeChunkSubtr);
- itsPhaseShifts.reserve (itsNDir-1); // not needed for target direction
- itsFirstSteps.reserve (itsNDir);
- itsAvgResults.reserve (itsNDir);
-
- // Create the solve and subtract steps for the sources to be removed.
- // Solving consists of the following steps:
- // - select the requested baselines (longer baselines may need no demix)
- // - phaseshift selected data to each demix source
- // - average selected data in each direction, also original phasecenter.
- // - determine and average demix factors for all directions
- // - predict and solve in each direction. It is possible to predict
- // more directions than to solve (for strong sources in field).
- // Subtract consists of the following steps:
- // - average all data (possibly different averaging than used in solve)
- // - determine and average demix factors (using select output in solve)
- // - select the requested baselines
- // - subtract sources for selected data
- // - merge subtract result into averaged data. This is not needed if
- // no selection is done.
- // Note that multiple time chunks are handled jointly, so a
- // MultiResultStep is used to catch the results of all time chunks.
- for (uint i=0; i<itsNDir-1; ++i) {
- // First make the phaseshift and average steps for each demix source.
- // The resultstep gets the result.
- // The phasecenter can be given in a parameter. Its name is the default.
- // Look up the source direction in the patch table.
- // If found, turn it into a vector of strings.
- vector<string> sourceVec (1, itsAllSources[i]);
- if(i < itsNModel) {
- sourceVec[0] = toString(itsPatchList[i]->position()[0]);
- sourceVec.push_back(toString(itsPatchList[i]->position()[1]));
- }
- PhaseShift* step1 = new PhaseShift (input, parset,
- prefix + itsAllSources[i] + '.',
- sourceVec);
- itsFirstSteps.push_back (DPStep::ShPtr(step1));
- itsPhaseShifts.push_back (step1);
- DPStep::ShPtr step2 (new Averager(input, prefix, itsNChanAvg,
- itsNTimeAvg));
- step1->setNextStep (step2);
- MultiResultStep* step3 = new MultiResultStep(itsNTimeChunk);
- step2->setNextStep (DPStep::ShPtr(step3));
- // There is a single demix factor step which needs to get all results.
- itsAvgResults.push_back (step3);
- }
-
- // Now create the step to average the data themselves.
- DPStep::ShPtr targetAvg(new Averager(input, prefix,
- itsNChanAvg, itsNTimeAvg));
- itsFirstSteps.push_back (targetAvg);
- MultiResultStep* targetAvgRes = new MultiResultStep(itsNTimeChunk);
- targetAvg->setNextStep (DPStep::ShPtr(targetAvgRes));
- itsAvgResults.push_back (targetAvgRes);
-
- // Create the data average step for the subtract.
- // The entire average result is needed for the next NDPPP step.
- // Only the selected baselines need to be subtracted, so add a
- // filter step as the last one.
- itsAvgStepSubtr = DPStep::ShPtr(new Averager(input, prefix,
- itsNChanAvgSubtr,
- itsNTimeAvgSubtr));
- itsAvgResultFull = new MultiResultStep(itsNTimeChunkSubtr);
- itsFilterSubtr = new Filter(input, itsSelBL);
- itsAvgResultSubtr = new MultiResultStep(itsNTimeChunkSubtr);
- itsAvgStepSubtr->setNextStep (DPStep::ShPtr(itsAvgResultFull));
- itsAvgResultFull->setNextStep (DPStep::ShPtr(itsFilterSubtr));
- itsFilterSubtr->setNextStep (DPStep::ShPtr(itsAvgResultSubtr));
-
-// while(itsCutOffs.size() < itsNModel) {
-// itsCutOffs.push_back(0.0);
-// }
-// itsCutOffs.resize(itsNModel);
- }
-
- void Demixer::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
-
- // Get size info.
- itsNChanIn = infoIn.nchan();
- itsNCorr = infoIn.ncorr();
- ASSERTSTR (itsNCorr==4, "Demixing requires data with 4 polarizations");
-
- // Handle possible data selection.
- itsFilter.setInfo (infoIn);
- const DPInfo& infoSel = itsFilter.getInfo();
- // NB. The number of baselines and stations refer to the number of
- // selected baselines and the number of unique stations that participate
- // in the selected baselines.
- itsNBl = infoSel.nbaselines();
- itsNStation = infoSel.antennaUsed().size();
-
- // Re-number the station IDs in the selected baselines, removing gaps in
- // the numbering due to unused stations.
- const vector<int> &antennaMap = infoSel.antennaMap();
- for (uint i=0; i<itsNBl; ++i) {
- itsBaselines.push_back(Baseline(antennaMap[infoSel.getAnt1()[i]],
- antennaMap[infoSel.getAnt2()[i]]));
- }
-
- // Prepare conversion from relative to absolute UVW
- casacore::Vector<casacore::Int> newAnt1(itsNBl);
- casacore::Vector<casacore::Int> newAnt2(itsNBl);
- for (uint i=0; i<itsNBl; ++i) {
- newAnt1[i]=antennaMap[infoSel.getAnt1()[i]];
- newAnt2[i]=antennaMap[infoSel.getAnt2()[i]];
- }
- itsUVWSplitIndex = nsetupSplitUVW (itsNStation,newAnt1,newAnt2);
-
- // Allocate buffers used to compute the smearing factors.
- itsFactorBuf.resize (IPosition(4, itsNCorr, itsNChanIn, itsNBl,
- itsNDir*(itsNDir-1)/2));
- itsFactorBufSubtr.resize (IPosition(4, itsNCorr, itsNChanIn, itsNBl,
- itsNDir*(itsNDir-1)/2));
-
- // Adapt averaging to available nr of channels and times.
- // Use a copy of the DPInfo, otherwise it is updated multiple times.
- DPInfo infoDemix(infoSel);
- itsNTimeAvg = std::min (itsNTimeAvg, infoSel.ntime());
- itsNChanAvg = infoDemix.update (itsNChanAvg, itsNTimeAvg);
- itsNChanOut = infoDemix.nchan();
- itsTimeIntervalAvg = infoDemix.timeInterval();
- itsNTimeDemix = infoDemix.ntime();
-
- // Let the internal steps update their data.
- for (uint i=0; i<itsFirstSteps.size(); ++i) {
- itsFirstSteps[i]->setInfo (infoSel);
- }
- itsAvgStepSubtr->setInfo (infoIn);
- // Update the info of this object.
- info().setNeedVisData();
- info().setWriteData();
- info().setWriteFlags();
- itsNTimeAvgSubtr = std::min (itsNTimeAvgSubtr, infoSel.ntime());
- itsNChanAvgSubtr = info().update (itsNChanAvgSubtr, itsNTimeAvgSubtr);
- itsNChanOutSubtr = info().nchan();
- ASSERTSTR (itsNChanAvg % itsNChanAvgSubtr == 0,
- "Demix averaging " << itsNChanAvg
- << " must be multiple of output averaging "
- << itsNChanAvgSubtr);
- ASSERTSTR (itsNTimeAvg % itsNTimeAvgSubtr == 0,
- "Demix averaging " << itsNTimeAvg
- << " must be multiple of output averaging "
- << itsNTimeAvgSubtr);
- // Store channel frequencies for the demix and subtract resolutions.
- itsFreqDemix = infoDemix.chanFreqs();
- itsFreqSubtr = getInfo().chanFreqs();
-
- // Store phase center position in J2000.
- MDirection dirJ2000(MDirection::Convert(infoIn.phaseCenter(),
- MDirection::J2000)());
- Quantum<Vector<Double> > angles = dirJ2000.getAngle();
- itsPhaseRef = Position(angles.getBaseValue()[0],
- angles.getBaseValue()[1]);
-
- // Intialize the unknowns.
- itsUnknowns.resize(itsNTimeDemix * itsNModel * itsNStation * 8);
- itsPrevSolution.resize(itsNModel * itsNStation * 8);
- vector<double>::iterator it = itsPrevSolution.begin();
- vector<double>::iterator it_end = itsPrevSolution.end();
- while(it != it_end)
- {
- *it++ = itsDefaultGain;
- *it++ = 0.0;
- *it++ = 0.0;
- *it++ = 0.0;
- *it++ = 0.0;
- *it++ = 0.0;
- *it++ = itsDefaultGain;
- *it++ = 0.0;
- }
- // Initialize the flag counters.
- itsFlagCounter.init (getInfo());
- }
-
- void Demixer::show (std::ostream& os) const
- {
- os << "Demixer " << itsName << std::endl;
- os << " skymodel: " << itsSkyName << std::endl;
- os << " instrumentmodel: " << itsInstrumentName << std::endl;
- os << " default gain: " << itsDefaultGain << std::endl;
- os << " max iterations: " << itsMaxIter << std::endl;
- itsSelBL.show (os);
- if (itsSelBL.hasSelection()) {
- os << " demixing " << itsFilter.getInfo().nbaselines()
- << " out of " << getInfo().nbaselines() << " baselines ("
- << itsFilter.getInfo().antennaUsed().size()
- << " out of " << getInfo().antennaUsed().size()
- << " stations)" << std::endl;
- }
- os << " targetsource: " << itsTargetSource << std::endl;
- os << " subtractsources: " << itsSubtrSources << std::endl;
- uint inx=0;
- for (uint i=0; i<itsSubtrSources.size(); ++i ) {
- os << " "
- << itsPhaseShifts[inx++]->getPhaseCenter() << std::endl;
- }
- os << " modelsources: " << itsModelSources << std::endl;
- for (uint i=0; i<itsModelSources.size(); ++i ) {
- os << " "
- << itsPhaseShifts[inx++]->getPhaseCenter() << std::endl;
- }
- os << " extrasources: " << itsExtraSources << std::endl;
- for (uint i=0; i<itsExtraSources.size(); ++i ) {
- os << " "
- << itsPhaseShifts[inx++]->getPhaseCenter() << std::endl;
- }
-// os << " elevationcutoffs: " << itsCutOffs << std::endl;
-// os << " jointsolve: " << itsJointSolve << std::endl;
- os << " propagatesolutions: " << std::boolalpha << itsPropagateSolutions
- << std::noboolalpha << std::endl;
- os << " freqstep: " << itsNChanAvgSubtr << std::endl;
- os << " timestep: " << itsNTimeAvgSubtr << std::endl;
- os << " demixfreqstep: " << itsNChanAvg << std::endl;
- os << " demixtimestep: " << itsNTimeAvg << std::endl;
- os << " ntimechunk: " << itsNTimeChunk << std::endl;
-// os << " Solve.Options.MaxIter: " << itsSolveOpt.maxIter << endl;
-// os << " Solve.Options.EpsValue: " << itsSolveOpt.epsValue << endl;
-// os << " Solve.Options.EpsDerivative: " << itsSolveOpt.epsDerivative << endl;
-// os << " Solve.Options.ColFactor: " << itsSolveOpt.colFactor << endl;
-// os << " Solve.Options.LMFactor: " << itsSolveOpt.lmFactor << endl;
-// os << " Solve.Options.BalancedEqs: " << itsSolveOpt.balancedEq << endl;
-// os << " Solve.Options.UseSVD: " << itsSolveOpt.useSVD <<endl;
- }
-
- void Demixer::showCounts (std::ostream& os) const
- {
- os << endl << "Statistics for Demixer " << itsName;
- os << endl << "======================" << endl;
- os << endl << "Converged: " << itsNConverged << "/" << itsNTimeDemix
- << " cells" << endl;
- }
-
- void Demixer::showTimings (std::ostream& os, double duration) const
- {
- const double self = itsTimer.getElapsed();
-
- os << " ";
- FlagCounter::showPerc1 (os, self, duration);
- os << " Demixer " << itsName << endl;
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerPhaseShift.getElapsed(), self);
- os << " of it spent in phase shifting/averaging data" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerDemix.getElapsed(), self);
- os << " of it spent in calculating decorrelation factors" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerSolve.getElapsed(), self);
- os << " of it spent in estimating gains and computing residuals" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerDump.getElapsed(), self);
- os << " of it spent in writing gain solutions to disk" << endl;
- }
-
- bool Demixer::process (const DPBuffer& buf)
- {
- itsTimer.start();
- // Update the count.
- itsNTimeIn++;
- // Make sure all required data arrays are filled in.
- /// itsBufTmp.referenceFilled (buf);
- itsBufTmp.copy (buf);
- itsInput->fetchUVW (buf, itsBufTmp, itsTimer);
- itsInput->fetchWeights (buf, itsBufTmp, itsTimer);
- itsInput->fetchFullResFlags (buf, itsBufTmp, itsTimer);
-
- // Do the filter step first.
- itsFilter.process (itsBufTmp);
- const DPBuffer& selBuf = itsFilter.getBuffer();
- // Do the next steps (phaseshift and average) on the filter output.
- itsTimerPhaseShift.start();
- for (int i=0; i<int(itsFirstSteps.size()); ++i) {
- itsFirstSteps[i]->process(selBuf);
- }
- // Do the average and filter step for the output for all data.
- itsAvgStepSubtr->process (itsBufTmp);
- itsTimerPhaseShift.stop();
-
- // For each itsNTimeAvg times, calculate the phase rotation per direction
- // for the selected data.
- itsTimerDemix.start();
- addFactors (selBuf, itsFactorBuf);
- if (itsNTimeIn % itsNTimeAvg == 0) {
- makeFactors (itsFactorBuf, itsFactors[itsNTimeOut],
- itsAvgResults[0]->get()[itsNTimeOut].getWeights(),
- itsNChanOut,
- itsNChanAvg);
- // Deproject sources without a model.
- deproject (itsFactors[itsNTimeOut], itsAvgResults, itsNTimeOut);
- itsFactorBuf = Complex(); // Clear summation buffer
- itsNTimeOut++;
- }
- // Subtract is done with different averaging parameters, so calculate the
- // factors for it (again for selected data only).
- addFactors (selBuf, itsFactorBufSubtr);
- if (itsNTimeIn % itsNTimeAvgSubtr == 0) {
- makeFactors (itsFactorBufSubtr, itsFactorsSubtr[itsNTimeOutSubtr],
- itsAvgResultSubtr->get()[itsNTimeOutSubtr].getWeights(),
- itsNChanOutSubtr,
- itsNChanAvgSubtr);
- itsFactorBufSubtr = Complex(); // Clear summation buffer
- itsNTimeOutSubtr++;
- }
- itsTimerDemix.stop();
-
- // Estimate gains and subtract source contributions when sufficient time
- // slots have been collected.
- if (itsNTimeOut == itsNTimeChunk) {
- handleDemix();
- }
- itsTimer.stop();
- return true;
- }
-
- void Demixer::finish()
- {
- cerr << " " << itsNTimeIn << " time slots to finish in Demixer ..."
- << endl;
- itsTimer.start();
-
- // Process remaining entries.
- if (itsNTimeIn > 0) {
- // Finish the initial steps (phaseshift and average).
- itsTimerPhaseShift.start();
- for (int i=0; i<int(itsFirstSteps.size()); ++i) {
- itsFirstSteps[i]->finish();
- }
- itsAvgStepSubtr->finish();
- itsTimerPhaseShift.stop();
- // Only average if there is some unaveraged data.
- itsTimerDemix.start();
- if (itsNTimeIn % itsNTimeAvg != 0) {
- makeFactors (itsFactorBuf, itsFactors[itsNTimeOut],
- itsAvgResults[0]->get()[itsNTimeOut].getWeights(),
- itsNChanOut,
- itsNChanAvg);
- // Deproject sources without a model.
- deproject (itsFactors[itsNTimeOut], itsAvgResults, itsNTimeOut);
- itsNTimeOut++;
- }
- if (itsNTimeIn % itsNTimeAvgSubtr != 0) {
- makeFactors (itsFactorBufSubtr, itsFactorsSubtr[itsNTimeOutSubtr],
- itsAvgResultSubtr->get()[itsNTimeOutSubtr].getWeights(),
- itsNChanOutSubtr,
- itsNChanAvgSubtr);
- itsNTimeOutSubtr++;
- }
- itsTimerDemix.stop();
- // Resize lists of mixing factors to the number of valid entries.
- itsFactors.resize(itsNTimeOut);
- itsFactorsSubtr.resize(itsNTimeOutSubtr);
-
- // Demix the source directions.
- handleDemix();
- }
-
- // Write solutions to disk in ParmDB format.
- itsTimerDump.start();
- dumpSolutions();
- itsTimerDump.stop();
-
- itsTimer.stop();
-
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void Demixer::handleDemix()
- {
- if(itsNModel > 0) {
- itsTimerSolve.start();
- demix();
- itsTimerSolve.stop();
- // If selection was done, merge the subtract results back into the
- // buffer.
- }
- // If needed, merge in the deselected baselines.
- if (itsSelBL.hasSelection()) {
- mergeSubtractResult();
- }
-
- // Clear the input buffers.
- for (size_t i=0; i<itsAvgResults.size(); ++i) {
- itsAvgResults[i]->clear();
- }
- // Let the next step process the data.
- for (uint i=0; i<itsNTimeOutSubtr; ++i) {
- itsTimer.stop();
- DPBuffer* bufptr;
- if (itsSelBL.hasSelection()) {
- bufptr = &(itsAvgResultFull->get()[i]);
- } else {
- bufptr = &(itsAvgResultSubtr->get()[i]);
- }
- MSReader::flagInfNaN (bufptr->getData(), bufptr->getFlags(),
- itsFlagCounter);
- getNextStep()->process (*bufptr);
- itsTimer.start();
- }
-
- // Clear the output buffer.
- itsAvgResultFull->clear();
- itsAvgResultSubtr->clear();
-
- // Reset counters.
- itsNTimeIn = 0;
- itsNTimeOut = 0;
- itsNTimeOutSubtr = 0;
- itsTimeIndex += itsNTimeChunk;
- }
-
- void Demixer::mergeSubtractResult()
- {
- // Merge the selected baselines from the subtract buffer into the
- // full buffer. Do it for all timestamps.
- for (uint i=0; i<itsNTimeOutSubtr; ++i) {
- const Array<Complex>& arr = itsAvgResultSubtr->get()[i].getData();
- size_t nr = arr.shape()[0] * arr.shape()[1];
- const Complex* in = arr.data();
- Complex* out = itsAvgResultFull->get()[i].getData().data();
- for (size_t j=0; j<itsFilter.getIndicesBL().size(); ++j) {
- size_t inx = itsFilter.getIndicesBL()[j];
- memcpy (out+inx*nr, in+j*nr, nr*sizeof(Complex));
- }
- }
- }
-
- void Demixer::addFactors (const DPBuffer& newBuf,
- Array<DComplex>& factorBuf)
- {
- // Nothing to do if only target direction.
- if (itsNDir <= 1) return;
- int ncorr = newBuf.getData().shape()[0];
- int nchan = newBuf.getData().shape()[1];
- int nbl = newBuf.getData().shape()[2];
- int ncc = ncorr*nchan;
- //# If ever in the future a time dependent phase center is used,
- //# the machine must be reset for each new time, thus each new call
- //# to process.
- // Add the weighted factors for each pair of directions.
- // The input factor is the phaseshift from target direction to
- // source direction. By combining them you get the shift from one
- // source direction to another.
- int dirnr = 0;
- for (uint i1=0; i1<itsNDir-1; ++i1) {
- for (uint i0=i1+1; i0<itsNDir; ++i0) {
- if (i0 == itsNDir-1) {
- // The last direction is the target direction, so no need to
- // combine the factors. Take conj to get shift source to target.
-#pragma omp parallel for
- for (int i=0; i<nbl; ++i) {
- const bool* flagPtr = newBuf.getFlags().data() + i*ncc;
- const float* weightPtr = newBuf.getWeights().data() + i*ncc;
- DComplex* factorPtr = factorBuf.data() + (dirnr*nbl + i)*ncc;
- const DComplex* phasor1 = itsPhaseShifts[i1]->getPhasors().data()
- + i*nchan;
- for (int j=0; j<nchan; ++j) {
- DComplex factor = conj(*phasor1++);
- for (int k=0; k<ncorr; ++k) {
- if (! *flagPtr) {
- *factorPtr += factor * double(*weightPtr);
- }
- flagPtr++;
- weightPtr++;
- factorPtr++;
- }
- }
- } // end omp parallel for
- } else {
- // Different source directions; take both phase terms into account.
-#pragma omp parallel for
- for (int i=0; i<nbl; ++i) {
- const bool* flagPtr = newBuf.getFlags().data() + i*ncc;
- const float* weightPtr = newBuf.getWeights().data() + i*ncc;
- DComplex* factorPtr = factorBuf.data() + (dirnr*nbl + i)*ncc;
- const DComplex* phasor0 = itsPhaseShifts[i0]->getPhasors().data()
- + i*nchan;
- const DComplex* phasor1 = itsPhaseShifts[i1]->getPhasors().data()
- + i*nchan;
- for (int j=0; j<nchan; ++j) {
- DComplex factor = *phasor0++ * conj(*phasor1++);
- for (int k=0; k<ncorr; ++k) {
- if (! *flagPtr) {
- *factorPtr += factor * double(*weightPtr);
- }
- flagPtr++;
- weightPtr++;
- factorPtr++;
- }
- }
- } // end omp parallel for
- }
-
- // Next direction pair.
- dirnr++;
- }
- }
- }
-
- void Demixer::makeFactors (const Array<DComplex>& bufIn,
- Array<DComplex>& bufOut,
- const Cube<float>& weightSums,
- uint nChanOut,
- uint nChanAvg)
- {
- // Nothing to do if only target direction.
- if (itsNDir <= 1) return;
- ASSERT (! weightSums.empty());
- bufOut.resize (IPosition(5, itsNDir, itsNDir,
- itsNCorr, nChanOut, itsNBl));
- bufOut = DComplex(1,0);
- int ncc = itsNCorr*nChanOut;
- int nccdd = ncc*itsNDir*itsNDir;
- int nccin = itsNCorr*itsNChanIn;
- // Fill the factors for each combination of different directions.
- uint dirnr = 0;
- for (uint d0=0; d0<itsNDir; ++d0) {
- for (uint d1=d0+1; d1<itsNDir; ++d1) {
-#pragma omp parallel for
- // Average factors by summing channels.
- // Note that summing in time is done in addFactors.
- // The sum per output channel is divided by the summed weight.
- // Note there is a summed weight per baseline,outchan,corr.
- for (int k=0; k<int(itsNBl); ++k) {
- const DComplex* phin = bufIn.data() + (dirnr*itsNBl + k)*nccin;
- DComplex* ph1 = bufOut.data() + k*nccdd + (d0*itsNDir + d1);
- DComplex* ph2 = bufOut.data() + k*nccdd + (d1*itsNDir + d0);
- const float* weightPtr = weightSums.data() + k*ncc;
- for (uint c0=0; c0<nChanOut; ++c0) {
- // Sum the factors for the input channels to average.
- DComplex sum[4];
- // In theory the last output channel could consist of fewer
- // input channels, so take care of that.
- uint nch = std::min(nChanAvg, itsNChanIn-c0*nChanAvg);
- for (uint c1=0; c1<nch; ++c1) {
- for (uint j=0; j<itsNCorr; ++j) {
- sum[j] += *phin++;
- }
- }
- for (uint j=0; j<itsNCorr; ++j) {
- *ph1 = sum[j] / double(*weightPtr++);
- *ph2 = conj(*ph1);
- ph1 += itsNDir*itsNDir;
- ph2 += itsNDir*itsNDir;
- }
- }
- } // end omp parallel for
- // Next input direction pair.
- dirnr++;
- }
- }
- ///cout<<"makefactors "<<weightSums<<bufOut;
- }
-
- void Demixer::deproject (Array<DComplex>& factors,
- vector<MultiResultStep*> avgResults,
- uint resultIndex)
- {
- // Sources without a model have to be deprojected.
- // Optionally no deprojection of target direction.
- uint nrDeproject = itsNDir - itsNModel;
- if (itsIgnoreTarget) {
- nrDeproject--;
- }
- // Nothing to do if only target direction or nothing to deproject.
- if (itsNDir <= 1 || nrDeproject == 0) return;
- // Get pointers to the data for the various directions.
- vector<Complex*> resultPtr(itsNDir);
- for (uint j=0; j<itsNDir; ++j) {
- resultPtr[j] = avgResults[j]->get()[resultIndex].getData().data();
- }
- // The projection matrix is given by
- // P = I - A * inv(A.T.conj * A) * A.T.conj
- // where A is the last column of the demixing matrix M.
- // The BBS equations get:
- // P * M' * v_predict = P * v_averaged
- // where M' is obtained by removing the last column of demixing matrix M.
- // The dimensions of the matrices/vectors are:
- // P : NxN
- // M' : Nx(N-1)
- // v_predict : (N-1) x 1
- // v_averaged: N x 1
- // where N is the number of modeled sources to use in demixing.
- // In the general case S sources might not have a source model.
- // In that case A is the NxS matrix containing all these columns
- // from M and M' is the Nx(N-S) matrix without all these columns.
-
- // Calculate P for all baselines,channels,correlations.
- IPosition shape = factors.shape();
- int nvis = shape[2] * shape[3] * shape[4];
- shape[1] = itsNModel;
- Array<DComplex> newFactors (shape);
- IPosition inShape (2, itsNDir, itsNDir);
- IPosition outShape(2, itsNDir, itsNModel);
-///#pragma omp parallel
- {
- Matrix<DComplex> a(itsNDir, nrDeproject);
- Matrix<DComplex> ma(itsNDir, itsNModel);
- vector<DComplex> vec(itsNDir);
- ///#pragma omp for
- for (int i=0; i<nvis; ++i) {
- // Split the matrix into the modeled and deprojected sources.
- // Copy the columns to the individual matrices.
- const DComplex* inptr = factors.data() + i*itsNDir*itsNDir;
- DComplex* outptr = newFactors.data() + i*itsNDir*itsNModel;
- Matrix<DComplex> out (outShape, outptr, SHARE);
- // Copying a bit of data is probably faster than taking a matrix
- // subset.
- objcopy (ma.data(), inptr, itsNDir*itsNModel);
- objcopy (a.data(), inptr + itsNDir*itsNModel, itsNDir*nrDeproject);
- // Calculate conjugated transpose of A, multiply with A, and invert.
- Matrix<DComplex> at(adjoint(a));
- Matrix<DComplex> ata(invert(product(at, a)));
- if (ata.empty()) {
- ata.resize (nrDeproject, nrDeproject);
- }
- DBGASSERT(ata.ncolumn()==nrDeproject && ata.nrow()==nrDeproject);
- // Calculate P = I - A * ata * A.T.conj
- Matrix<DComplex> aata(product(a,ata));
- Matrix<DComplex> p (-product(product(a, ata), at));
- Vector<DComplex> diag(p.diagonal());
- diag += DComplex(1,0);
- // Multiply the demixing factors with P (get stored in newFactors).
- out = product(p, ma);
- // Multiply the averaged data point with P.
- std::fill (vec.begin(), vec.end(), DComplex());
- for (uint j=0; j<itsNDir; ++j) {
- for (uint k=0; k<itsNDir; ++k) {
- vec[k] += DComplex(resultPtr[j][i]) * p(k,j);
- }
- }
- // Put result back in averaged data for those sources.
- for (uint j=0; j<itsNDir; ++j) {
- resultPtr[j][i] = vec[j];
- }
- }
- }
- // Set the new demixing factors.
- factors.reference (newFactors);
- }
-
- namespace {
- struct ThreadPrivateStorage
- {
- vector<double> unknowns;
- casacore::Matrix<double> uvw;
- vector<casacore::Cube<dcomplex> > model;
- casacore::Cube<dcomplex> model_subtr;
- size_t count_converged;
- };
-
- void initThreadPrivateStorage(ThreadPrivateStorage &storage,
- size_t nDirection, size_t nStation, size_t nBaseline, size_t nChannel,
- size_t nChannelSubtr)
- {
- storage.unknowns.resize(nDirection * nStation * 8);
- storage.uvw.resize(3, nStation);
- storage.model.resize(nDirection);
- for (uint dir=0;dir<nDirection; ++dir) {
- storage.model[dir].resize(4, nChannel, nBaseline);
- }
- storage.model_subtr.resize(4, nChannelSubtr, nBaseline);
- storage.count_converged = 0;
- }
- } //# end unnamed namespace
-
- void Demixer::demix()
- {
- const size_t nThread = OpenMP::maxThreads();
- const size_t nTime = itsAvgResults[0]->size();
- const size_t nTimeSubtr = itsAvgResultSubtr->size();
- const size_t multiplier = itsNTimeAvg / itsNTimeAvgSubtr;
- const size_t nDr = itsNModel;
- const size_t nDrSubtr = itsSubtrSources.size();
- const size_t nSt = itsNStation;
- const size_t nBl = itsBaselines.size();
- const size_t nCh = itsFreqDemix.size();
- const size_t nChSubtr = itsFreqSubtr.size();
- const size_t nCr = 4;
-
- vector<ThreadPrivateStorage> threadStorage(nThread);
- for(vector<ThreadPrivateStorage>::iterator it = threadStorage.begin(),
- end = threadStorage.end(); it != end; ++it)
- {
- initThreadPrivateStorage(*it, nDr, nSt, nBl, nCh, nChSubtr);
-
- // Copy the previous solution to the thread private vectors of unknowns.
- // When solution propagation is disabled, itsPrevSolution is never
- // updated. It then contains 1.0+0.0i for the diagonal terms and
- // 0.0+0.0i for the off-diagonal terms. Thus, when solution propagation
- // is disabled this statement effectively re-initializes the thread
- // private vectors of unknowns.
- copy(itsPrevSolution.begin(), itsPrevSolution.end(),
- it->unknowns.begin());
- }
-
- const_cursor<Baseline> cr_baseline(&(itsBaselines[0]));
-
-#pragma omp parallel for
- for(size_t ts = 0; ts < nTime; ++ts)
- {
- const size_t thread = OpenMP::threadNum();
- ThreadPrivateStorage &storage = threadStorage[thread];
-
- // If solution propagation is disabled, re-initialize the thread-private
- // vector of unknowns.
- if(!itsPropagateSolutions)
- {
- copy(itsPrevSolution.begin(), itsPrevSolution.end(),
- storage.unknowns.begin());
- }
-
- // Simulate.
- //
- // Model visibilities for each direction of interest will be computed
- // and stored.
- size_t stride_model[3] = {1, nCr, nCr * nCh};
- fill(storage.model.begin(), storage.model.end(), 0.);
- for(size_t dr = 0; dr < nDr; ++dr)
- {
- nsplitUVW(itsUVWSplitIndex, itsBaselines, itsAvgResults[dr]->get()[ts].getUVW(), storage.uvw);
- ///cout<<"uvw"<<dr<<'='<<storage.uvw<<endl;
-
- Simulator simulator(itsPatchList[dr]->position(), nSt, nBl, nCh,
- itsBaselines, itsFreqDemix, storage.uvw,
- storage.model[dr]);
- for(size_t i = 0; i < itsPatchList[dr]->nComponents(); ++i)
- {
- simulator.simulate(itsPatchList[dr]->component(i));
- }
-
- }
- ///cout<<"modelvis="<<storage.model<<endl;
-
- // Estimate Jones matrices.
- //
- // A Jones matrix will be estimated for each pair of station and
- // direction.
- //
- // A single (overdetermined) non-linear set of equations for all
- // stations and directions is solved iteratively. The influence of
- // each direction on each other direction is given by the mixing
- // matrix.
- const_cursor<bool> cr_flag =
- casa_const_cursor(itsAvgResults[0]->get()[ts].getFlags());
- const_cursor<float> cr_weight =
- casa_const_cursor(itsAvgResults[0]->get()[ts].getWeights());
- const_cursor<dcomplex> cr_mix = casa_const_cursor(itsFactors[ts]);
- ///cout << "demixfactor "<<ts<<" = "<<itsFactors[ts]<<endl;
-
- vector<const_cursor<fcomplex> > cr_data(nDr);
- vector<const_cursor<dcomplex> > cr_model(nDr);
- for(size_t dr = 0; dr < nDr; ++dr)
- {
- cr_data[dr] =
- casa_const_cursor(itsAvgResults[dr]->get()[ts].getData());
- cr_model[dr] =
- const_cursor<dcomplex>(storage.model[dr].data(), 3,
- stride_model);
- }
-
- bool converged = estimate(nDr, nSt, nBl, nCh, cr_baseline, cr_data,
- cr_model, cr_flag, cr_weight, cr_mix, &(storage.unknowns[0]),
- itsMaxIter);
- if(converged)
- {
- ++storage.count_converged;
- }
-
- // Compute the residual.
- //
- // All the so-called "subtract sources" are subtracted from the
- // observed data. The previously estimated Jones matrices, as well as
- // the appropriate mixing weight are applied before subtraction.
- //
- // Note that the resolution of the residual can differ from the
- // resolution at which the Jones matrices were estimated.
- for(size_t ts_subtr = multiplier * ts, ts_subtr_end = min(ts_subtr
- + multiplier, nTimeSubtr); ts_subtr != ts_subtr_end; ++ts_subtr)
- {
- for(size_t dr = 0; dr < nDrSubtr; ++dr)
- {
- // Re-use simulation used for estimating Jones matrices if possible.
- cursor<dcomplex> cr_model_subtr(storage.model[dr].data(),
- 3, stride_model);
-
- // Re-simulate if required.
- if(multiplier != 1 || nCh != nChSubtr)
- {
- nsplitUVW(itsUVWSplitIndex, itsBaselines, itsAvgResultSubtr->get()[ts_subtr].getUVW(), storage.uvw);
-
- // Rotate the UVW coordinates for the target direction to the
- // direction of source to subtract. This is required because at
- // the resolution of the residual the UVW coordinates for
- // directions other than the target are unavailable (unless the
- // resolution of the residual is equal to the resolution at which
- // the Jones matrices were estimated, of course).
- rotateUVW(itsPhaseRef, itsPatchList[dr]->position(), nSt,
- storage.uvw.data());
-
- // Zero the visibility buffer.
- storage.model_subtr=dcomplex();
-
- // Simulate visibilities at the resolution of the residual.
- size_t stride_model_subtr[3] = {1, nCr, nCr * nChSubtr};
- cr_model_subtr = cursor<dcomplex>(storage.model_subtr.data(), 3,
- stride_model_subtr);
-
- Simulator simulator(itsPatchList[dr]->position(), nSt, nBl,
- nChSubtr, itsBaselines, itsFreqSubtr,
- storage.uvw, storage.model_subtr);
- for(size_t i = 0; i < itsPatchList[dr]->nComponents(); ++i)
- {
- simulator.simulate(itsPatchList[dr]->component(i));
- }
- }
-
- // Apply Jones matrices.
- size_t stride_unknowns[2] = {1, 8};
- const_cursor<double> cr_unknowns(&(storage.unknowns[dr * nSt * 8]),
- 2, stride_unknowns);
-
- apply(nBl, nChSubtr, cr_baseline, cr_unknowns, cr_model_subtr);
-
- // Subtract the source contribution from the data.
- cursor<fcomplex> cr_residual =
- casa_cursor(itsAvgResultSubtr->get()[ts_subtr].getData());
-
- // Construct a cursor to iterate over a slice of the mixing matrix
- // at the resolution of the residual. The "to" and "from" direction
- // are fixed. Since the full mixing matrix is 5-D, the slice is
- // therefore 3-D. Each individual value in the slice quantifies the
- // influence of the source to subtract on the target direction for
- // a particular correlation, channel, and baseline.
- //
- // The target direction is the direction with the highest index by
- // convention, i.e. index itsNDir - 1. The directions to subtract
- // have the lowest indices by convention, i.e. indices
- // [0, nDrSubtr).
- const IPosition &stride_mix_subtr =
- itsFactorsSubtr[ts_subtr].steps();
- size_t stride_mix_subtr_slice[3] = {
- static_cast<size_t>(stride_mix_subtr[2]),
- static_cast<size_t>(stride_mix_subtr[3]),
- static_cast<size_t>(stride_mix_subtr[4])
- };
- ASSERT(stride_mix_subtr_slice[0] == itsNDir * itsNDir
- && stride_mix_subtr_slice[1] == itsNDir * itsNDir * nCr
- && stride_mix_subtr_slice[2] == itsNDir * itsNDir * nCr * nChSubtr);
-
- IPosition offset(5, itsNDir - 1, dr, 0, 0, 0);
- const_cursor<dcomplex> cr_mix_subtr =
- const_cursor<dcomplex>(&(itsFactorsSubtr[ts_subtr](offset)), 3,
- stride_mix_subtr_slice);
-
- // Subtract the source.
- subtract(nBl, nChSubtr, cr_baseline, cr_residual, cr_model_subtr,
- cr_mix_subtr);
- }
- }
-
- // Copy solutions to global solution array.
- copy(storage.unknowns.begin(), storage.unknowns.end(),
- &(itsUnknowns[(itsTimeIndex + ts) * nDr * nSt * 8]));
- }
-
- // Store last known solutions.
- if(itsPropagateSolutions && nTime > 0)
- {
- copy(&(itsUnknowns[(itsTimeIndex + nTime - 1) * nDr * nSt * 8]),
- &(itsUnknowns[(itsTimeIndex + nTime) * nDr * nSt * 8]),
- itsPrevSolution.begin());
- }
-
- // Update convergence count.
- for(size_t i = 0; i < nThread; ++i)
- {
- itsNConverged += threadStorage[i].count_converged;
- }
- }
-
- void Demixer::dumpSolutions()
- {
- // Construct solution grid.
- const Vector<double>& freq = getInfo().chanFreqs();
- const Vector<double>& freqWidth = getInfo().chanWidths();
- BBS::Axis::ShPtr freqAxis(new BBS::RegularAxis(freq[0] - freqWidth[0]
- * 0.5, freqWidth[0], 1));
- BBS::Axis::ShPtr timeAxis(new BBS::RegularAxis(getInfo().startTime()
- - getInfo().timeInterval() * 0.5, itsTimeIntervalAvg, itsNTimeDemix));
- BBS::Grid solGrid(freqAxis, timeAxis);
-
- // Create and initialize ParmDB.
- BBS::ParmDB parmDB(BBS::ParmDBMeta("casa", itsInstrumentName), true);
- BBS::ParmSet parmSet;
- BBS::ParmCache parmCache(parmSet, solGrid.getBoundingBox());
-
- // Store the (freq, time) resolution of the solutions.
- vector<double> resolution(2);
- resolution[0] = freqWidth[0];
- resolution[1] = itsTimeIntervalAvg;
- parmDB.setDefaultSteps(resolution);
-
- // Map station indices in the solution array to the corresponding antenna
- // names. This is required because solutions are only produced for
- // stations that participate in one or more baselines. Due to the baseline
- // selection or missing baselines, solutions may be available for less
- // than the total number of station available in the observation.
- const DPInfo &info = itsFilter.getInfo();
- const vector<int> &antennaUsed = info.antennaUsed();
- const Vector<String> &antennaNames = info.antennaNames();
-
- vector<BBS::Parm> parms;
- for(size_t dr = 0; dr < itsNModel; ++dr) {
- for(size_t st = 0; st < itsNStation; ++st) {
- string name(antennaNames[antennaUsed[st]]);
- string suffix(name + ":" + itsAllSources[dr]);
-
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:0:0:Real:" + suffix)));
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:0:0:Imag:" + suffix)));
-
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:0:1:Real:" + suffix)));
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:0:1:Imag:" + suffix)));
-
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:1:0:Real:" + suffix)));
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:1:0:Imag:" + suffix)));
-
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:1:1:Real:" + suffix)));
- parms.push_back(BBS::Parm(parmCache, parmSet.addParm(parmDB,
- "DirectionalGain:1:1:Imag:" + suffix)));
- }
- }
-
- // Cache parameter values.
- parmCache.cacheValues();
-
- // Assign solution grid to parameters.
- for(size_t i = 0; i < parms.size(); ++i) {
- parms[i].setSolveGrid(solGrid);
- }
-
- // Write solutions.
- for(size_t ts = 0; ts < itsNTimeDemix; ++ts) {
- double *unknowns = &(itsUnknowns[ts * itsNModel * itsNStation * 8]);
- for(size_t i = 0; i < parms.size(); ++i) {
- parms[i].setCoeff(BBS::Location(0, ts), unknowns + i, 1);
- }
- }
-
- // Flush solutions to disk.
- parmCache.flush();
- }
-
- namespace
- {
- string toString (double value)
- {
- ostringstream os;
- os << setprecision(16) << value;
- return os.str();
- }
- } //# end unnamed namespace
-
- } //# end namespace DPPP
-} //# end namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/DemixerNew.cc b/CEP/DP3/DPPP/src/DemixerNew.cc
deleted file mode 100644
index e4024c203d04bf1c992e7d7763227dc6433693c1..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DemixerNew.cc
+++ /dev/null
@@ -1,517 +0,0 @@
-//# DemixerNew.cc: DPPP step class to subtract A-team sources in adaptive way
-//# Copyright (C) 2011
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: DemixerNew.cc 24221 2013-03-12 12:24:48Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DemixerNew.h>
-#include <DPPP/DemixWorker.h>
-#include <DPPP/DemixInfo.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <ParmDB/ParmDB.h>
-#include <ParmDB/ParmValue.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <Common/OpenMP.h>
-#include <Common/StreamUtil.h>
-#include <Common/lofar_iostream.h>
-#include <Common/lofar_iomanip.h>
-
-#include <casacore/casa/Arrays/ArrayPartMath.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- using LOFAR::operator<<;
-
- DemixerNew::DemixerNew (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsDemixInfo (parset, prefix),
- itsInstrumentName (parset.getString(prefix+"instrumentmodel",
- "instrument")),
- itsFilter (input, itsDemixInfo.selBL()),
- itsNTime (0),
- itsNTimeOut (0),
- itsNChunk (0)
- {
- ASSERTSTR (! itsInstrumentName.empty(),
- "An empty name is given for the instrument model");
- }
-
- void DemixerNew::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- // Update the info of this object.
- info().setNeedVisData();
- info().setWriteData();
- info().setWriteFlags();
- // Handle possible data selection.
- itsFilter.updateInfo (getInfo());
- // Update itsDemixInfo and info().
- itsDemixInfo.update (itsFilter.getInfo(), info());
- // Size the buffers.
- itsBufIn.resize (itsDemixInfo.ntimeChunk() * itsDemixInfo.chunkSize());
- itsBufOut.resize(itsDemixInfo.ntimeChunk() * itsDemixInfo.ntimeOutSubtr());
- itsSolutions.resize(itsDemixInfo.ntimeChunk() * itsDemixInfo.ntimeOut());
- // Create a worker per thread.
- int nthread = OpenMP::maxThreads();
- itsWorkers.reserve (nthread);
- for (int i=0; i<nthread; ++i) {
- itsWorkers.push_back (DemixWorker (itsInput, itsName, itsDemixInfo,
- infoIn, i));
- }
- }
-
- void DemixerNew::show (ostream& os) const
- {
- os << "DemixerNew " << itsName << endl;
- os << " instrumentmodel: " << itsInstrumentName << endl;
- itsDemixInfo.show (os);
- if (itsDemixInfo.selBL().hasSelection()) {
- os << " demixing " << itsFilter.getInfo().nbaselines()
- << " out of " << getInfo().nbaselines() << " baselines ("
- << itsFilter.getInfo().antennaUsed().size()
- << " out of " << getInfo().antennaUsed().size()
- << " stations)" << std::endl;
- }
- }
-
- void DemixerNew::showCounts (ostream& os) const
- {
- os << endl << "Statistics for SmartDemixer " << itsName;
- os << endl << "===========================" << endl;
- // Add the statistics of all workers.
- uint nsolves = 0;
- uint nconverged = 0;
- uint niter = 0;
- uint nnodemix = 0;
- uint nincludeStrong = 0;
- uint nincludeClose = 0;
- uint nignore = 0;
- uint ndeproject = 0;
- // Sum the counts from all workers.
- uint ndir = itsDemixInfo.ateamList().size();
- Vector<uint> nsources(ndir, 0);
- Vector<uint> nstation(itsDemixInfo.nstation(), 0);
- Matrix<uint> statsrcs(ndir, nstation.size(), 0);
- Matrix<double> amplSubtrMean (itsDemixInfo.nbl(), ndir+1, 0.);
- Matrix<double> amplSubtrM2 (itsDemixInfo.nbl(), ndir+1, 0.);
- Matrix<size_t> amplSubtrNr (itsDemixInfo.nbl(), ndir+1, 0);
- double* amplmean = amplSubtrMean.data();
- double* amplm2 = amplSubtrM2.data();
- size_t* amplnr = amplSubtrNr.data();
- for (size_t i=0; i<itsWorkers.size(); ++i) {
- nsolves += itsWorkers[i].nSolves();
- nconverged += itsWorkers[i].nConverged();
- niter += itsWorkers[i].nIterations();
- nnodemix += itsWorkers[i].nNoDemix();
- nincludeStrong += itsWorkers[i].nIncludeStrongTarget();
- nincludeClose += itsWorkers[i].nIncludeCloseTarget();
- nignore += itsWorkers[i].nIgnoreTarget();
- ndeproject += itsWorkers[i].nDeprojectTarget();
- nsources += itsWorkers[i].nsourcesDemixed();
- nstation += itsWorkers[i].nstationsDemixed();
- statsrcs += itsWorkers[i].statSourceDemixed();
- const double* partmean = itsWorkers[i].amplSubtrMean().data();
- const double* partm2 = itsWorkers[i].amplSubtrM2().data();
- const size_t* partnr = itsWorkers[i].amplSubtrNr().data();
- for (uint j=0; j<amplSubtrNr.size(); ++j) {
- // Calculate overall mean and m2 for each baseline/direction.
- addMeanM2 (amplmean[j], amplm2[j], amplnr[j],
- partmean[j], partm2[j], partnr[j]);
- }
- }
- uint ntimes = (nnodemix + nignore + ndeproject +
- nincludeStrong + nincludeClose);
- // Show statistics.
- showStat (os, nconverged, nsolves, "Converged solves: ", "cells");
- os << " Average nr of iterations in converged solves: "
- << uint(double(niter)/nconverged + 0.5) << endl;
- showStat (os, nnodemix, ntimes, "No demixing: ", "times");
- showStat (os, nignore, ntimes, "Target ignored: ", "times");
- showStat (os, ndeproject, ntimes, "Target deprojected: ", "times");
- showStat (os, nincludeStrong, ntimes, "Strong target included:", "times");
- showStat (os, nincludeClose, ntimes, "Close target included: ", "times");
- // Show how often a source/station is demixed.
- os << endl << "Percentage of times a station/source is demixed:" << endl;
- os << setw(15) << " ";
- for (size_t dr=0; dr<ndir; ++dr) {
- os << setw(8) << itsDemixInfo.ateamList()[dr]->name().substr(0,8);
- }
- os << " Overall" << endl;
- for (size_t st=0; st<nstation.size(); ++st) {
- os << setw(4) << st << ' ';
- // Show 10 characters of the source names; append with blanks as needed.
- uint inx = itsFilter.getInfo().antennaUsed()[st];
- string nm = itsFilter.getInfo().antennaNames()[inx];
- os << nm.substr(0,10);
- if (nm.size() < 10) {
- os << setw(10 - nm.size()) << ' ';
- }
- for (size_t dr=0; dr<ndir; ++dr) {
- os << " ";
- showPerc1 (os, statsrcs(dr,st) / double(ntimes));
- }
- os << " ";
- showPerc1 (os, nstation(st) / double(ntimes));
- os << endl;
- }
- os << " Overall" << setw(3) << ' ';
- for (size_t dr=0; dr<ndir; ++dr) {
- os << " ";
- showPerc1 (os, nsources[dr] / double(ntimes));
- }
- os << endl;
-
- // Show percentage subtracted.
- if (itsDemixInfo.doSubtract()) {
- os << endl << "Mean/stddev percentage of subtracted Stokes I amplitude"
- << " for the middle channel" << endl;
- os << setw(8) << ' ';
- for (size_t dr=0; dr<ndir; ++dr) {
- if (nsources[dr] > 0) {
- // Print name a bit right of the center.
- const string& nm = itsDemixInfo.ateamList()[dr]->name().substr(0,13);
- if (nm.size() > 10) {
- cout << setw(13) << nm;
- } else {
- int szws = 13 - nm.size(); // whitespace
- os << setw(szws/2+1) << ' ';
- os << nm;
- os << setw(szws-szws/2-1) << ' ';
- }
- }
- }
- os << setw(10) << "Total" << endl;
- os << "baseline";
- for (size_t dr=0; dr<ndir; ++dr) {
- if (nsources[dr] > 0) {
- os << " mean stddev";
- }
- }
- os << " mean stddev" << endl;
- vector<double> totsump(ndir+1, 0.);
- vector<double> totm2p (ndir+1, 0.);
- vector<size_t> totnrp (ndir+1, 0);
- for (int bl=0; bl<amplSubtrMean.shape()[0]; ++bl) {
- // Do not show if nothing subtracted for this baseline.
- // Last entry contains the sum over all directions!!
- if (amplSubtrNr(bl,ndir)) {
- os << setw(4) << itsDemixInfo.getAnt1()[bl] << '-'
- << setw(2) << itsDemixInfo.getAnt2()[bl] << " ";
- for (uint dr=0; dr<ndir+1; ++dr) {
- if (dr == ndir || nsources[dr] > 0) {
- showPerc1 (os, amplSubtrMean(bl,dr));
- double variance = 0;
- if (amplSubtrNr(bl,dr) > 1) {
- variance = amplSubtrM2(bl,dr) / (amplSubtrNr(bl,dr) - 1);
- }
- showPerc1 (os, sqrt(variance));
- os << ' ';
- addMeanM2 (totsump[dr], totm2p[dr], totnrp[dr],
- amplSubtrMean(bl,dr), amplSubtrM2(bl,dr),
- amplSubtrNr(bl,dr));
- }
- }
- os << endl;
- } // end if show
- } // end for bl
- os << " Total ";
- for (uint dr=0; dr<ndir+1; ++dr) {
- if (dr == ndir || nsources[dr] > 0) {
- double variance = 0;
- if (totnrp[dr] > 1) {
- variance = totm2p[dr] / (totnrp[dr] - 1);
- }
- showPerc1 (os, totsump[dr]);
- showPerc1 (os, sqrt(variance));
- os << ' ';
- }
- }
- os << endl;
- }
- }
-
- void DemixerNew::addMeanM2 (double& mean, double& m2, size_t& nr,
- double partmean, double partm2,
- size_t partnr) const
- {
- // Calculate overall mean and m2 (for stddev) from two partitions.
- // See en.wikipedia.org/wiki/Algorithms_for_calculating_variance
- if (partnr > 0) {
- double delta = partmean - mean;
- double mp = mean*nr + partmean*partnr;
- double nab = double(nr) / (nr+partnr) * partnr;
- nr += partnr;
- mean = mp / nr;
- m2 += partm2 + delta*nab*delta;
- }
- }
-
- void DemixerNew::showStat (ostream& os, double n, double ntot,
- const string& str1, const string& str2) const
- {
- os << str1 << ' ';
- showPerc1 (os, ntot==0 ? 0 : n/ntot);
- os << " ("<< n << ' '<< str2 << " out of " << ntot << ')' << endl;
- }
-
- void DemixerNew::showPerc1 (ostream& os, float perc) const
- {
- int p = int(1000*perc + 0.5);
- os << std::setw(3) << p/10 << '.' << p%10 << '%';
- }
-
- void DemixerNew::showTimings (std::ostream& os, double duration) const
- {
- double self = itsTimer.getElapsed();
- double demix = itsTimerDemix.getElapsed();
- double tottime = 0;
- double coatime = 0;
- double psatime = 0;
- double demtime = 0;
- double pretime = 0;
- double soltime = 0;
- double subtime = 0;
- for (uint i=0; i<itsWorkers.size(); ++i) {
- tottime += itsWorkers[i].getTotalTime();
- coatime += itsWorkers[i].getCoarseTime();
- psatime += itsWorkers[i].getPhaseShiftTime();
- demtime += itsWorkers[i].getDemixTime();
- pretime += itsWorkers[i].getPredictTime();
- soltime += itsWorkers[i].getSolveTime();
- subtime += itsWorkers[i].getSubtractTime();
- }
- os << " ";
- FlagCounter::showPerc1 (os, self, duration);
- os << " DemixerNew " << itsName << endl;
- os << " ";
- FlagCounter::showPerc1 (os, demix, self);
- os << " of it spent in demixing the data of which" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, coatime, tottime);
- os << " in predicting coarse source models" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, psatime, tottime);
- os << " in phase shifting/averaging data" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, demtime, tottime);
- os << " in calculating decorrelation factors" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, pretime, tottime);
- os << " in predicting demix source models" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, soltime, tottime);
- os << " in solving complex gains" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, subtime, tottime);
- os << " in subtracting source models" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerDump.getElapsed(), self);
- os << " of it spent in writing gain solutions to disk" << endl;
- }
-
- bool DemixerNew::process (const DPBuffer& buf)
- {
- itsTimer.start();
- // Collect sufficient data buffers.
- // Make sure all required data arrays are filled in.
- DPBuffer& newBuf = itsBufIn[itsNTime];
- newBuf.copy (buf);
- itsInput->fetchUVW(buf, newBuf, itsTimer);
- itsInput->fetchWeights(buf, newBuf, itsTimer);
- itsInput->fetchFullResFlags(buf, newBuf, itsTimer);
- // Process the data if entire buffer is filled.
- if (++itsNTime >= itsBufIn.size()) {
- processData();
- itsNTime = 0;
- }
- itsTimer.stop();
- return true;
- }
-
- void DemixerNew::processData()
- {
- itsTimerDemix.start();
- // Last batch might contain fewer time slots.
- uint timeWindowIn = itsDemixInfo.chunkSize();
- uint timeWindowOut = itsDemixInfo.ntimeOutSubtr();
- uint timeWindowSol = itsDemixInfo.ntimeOut();
- int lastChunk = (itsNTime - 1) / timeWindowIn;
- int lastNTimeIn = itsNTime - lastChunk*timeWindowIn;
- int ntimeOut = ((itsNTime + itsDemixInfo.ntimeAvgSubtr() - 1)
- / itsDemixInfo.ntimeAvgSubtr());
- int ntimeSol = ((itsNTime + itsDemixInfo.ntimeAvg() - 1)
- / itsDemixInfo.ntimeAvg());
-#pragma omp parallel for schedule(dynamic)
- for (int i=0; i<=lastChunk; ++i) {
- if (i == lastChunk) {
- itsWorkers[OpenMP::threadNum()].process
- (&(itsBufIn[i*timeWindowIn]), lastNTimeIn,
- &(itsBufOut[i*timeWindowOut]),
- &(itsSolutions[i*timeWindowSol]),
- itsNChunk+i);
- } else {
- itsWorkers[OpenMP::threadNum()].process
- (&(itsBufIn[i*timeWindowIn]), timeWindowIn,
- &(itsBufOut[i*timeWindowOut]),
- &(itsSolutions[i*timeWindowSol]),
- itsNChunk+i);
- }
- }
- itsNChunk += lastChunk+1;
- itsTimerDemix.stop();
- // Write the solutions into the instrument ParmDB.
- // Let the next steps process the results.
- // This could be done in parallel.
- ///#pragma omp parallel for num_thread(2)
- for (int i=0; i<2; ++i) {
- if (i == 0) {
- itsTimerDump.start();
- double startTime = (itsBufIn[0].getTime() +
- 0.5 * itsBufIn[0].getExposure());
- writeSolutions (startTime, ntimeSol);
- itsTimerDump.stop();
- } else {
- itsTimer.stop();
- itsTimerNext.start();
- for (int j=0; j<ntimeOut; ++j) {
- getNextStep()->process (itsBufOut[j]);
- itsNTimeOut++;
- }
- itsTimerNext.stop();
- itsTimer.start();
- }
- }
- }
-
- void DemixerNew::finish()
- {
- cerr << " " << itsNTime << " time slots to finish in SmartDemixer ..."
- << endl;
- itsTimer.start();
- // Process remaining entries.
- if (itsNTime > 0) {
- processData();
- }
- itsTimer.stop();
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void DemixerNew::writeSolutions (double startTime, int ntime)
- {
- if (itsDemixInfo.verbose() > 12) {
- for (int i=0; i<ntime; ++i) {
- cout << "solution " << i << endl;
- const double* sol = &(itsSolutions[i][0]);
- for (size_t dr=0; dr<itsSolutions[i].size()/(8*itsDemixInfo.nstation()); ++dr) {
- for (size_t st=0; st<itsDemixInfo.nstation(); ++ st) {
- cout << dr<<','<<st<<' ';
- print (cout, sol, sol+8);
- cout << endl;
- sol += 8;
- }
- }
- }
- }
- /// todo: skip solutions that are all 0.
- // Construct solution grid.
- const Vector<double>& freq = getInfo().chanFreqs();
- const Vector<double>& freqWidth = getInfo().chanWidths();
- BBS::Axis::ShPtr freqAxis(new BBS::RegularAxis(freq[0] - freqWidth[0]
- * 0.5, freqWidth[0], 1));
- BBS::Axis::ShPtr timeAxis(new BBS::RegularAxis
- (startTime,
- itsDemixInfo.timeIntervalAvg(), ntime));
- BBS::Grid solGrid(freqAxis, timeAxis);
- // Create domain grid.
- BBS::Axis::ShPtr tdomAxis(new BBS::RegularAxis
- (startTime,
- itsDemixInfo.timeIntervalAvg() * ntime, 1));
- BBS::Grid domainGrid(freqAxis, tdomAxis);
-
- // Open the ParmDB at the first write.
- // In that way the instrumentmodel ParmDB can be in the MS directory.
- if (! itsParmDB) {
- itsParmDB = boost::shared_ptr<BBS::ParmDB>
- (new BBS::ParmDB(BBS::ParmDBMeta("casa", itsInstrumentName),
- true));
- itsParmDB->lock();
- // Store the (freq, time) resolution of the solutions.
- vector<double> resolution(2);
- resolution[0] = freqWidth[0];
- resolution[1] = itsDemixInfo.timeIntervalAvg();
- itsParmDB->setDefaultSteps(resolution);
- }
- // Write the solutions per parameter.
- const char* str01[] = {"0:","1:"};
- const char* strri[] = {"Real:","Imag:"};
- Matrix<double> values(1, ntime);
- uint seqnr = 0;
- for (size_t dr=0; dr<itsDemixInfo.ateamList().size()+1; ++dr) {
- for (size_t st=0; st<itsDemixInfo.nstation(); ++st) {
- string suffix(itsDemixInfo.antennaNames()[st]);
- if (dr == itsDemixInfo.ateamList().size()) {
- suffix += ":Target";
- } else {
- suffix += ":" + itsDemixInfo.ateamList()[dr]->name();
- }
- for (int i=0; i<2; ++i) {
- for (int j=0; j<2; j++) {
- for (int k=0; k<2; ++k) {
- string name(string("DirectionalGain:") +
- str01[i] + str01[j] + strri[k] + suffix);
- // Collect its solutions for all times in a single array.
- for (int ts=0; ts<ntime; ++ts) {
- values(0, ts) = itsSolutions[ts][seqnr];
- }
- seqnr++;
- BBS::ParmValue::ShPtr pv(new BBS::ParmValue());
- pv->setScalars (solGrid, values);
- BBS::ParmValueSet pvs(domainGrid,
- vector<BBS::ParmValue::ShPtr>(1, pv));
- map<string,int>::const_iterator pit = itsParmIdMap.find(name);
- if (pit == itsParmIdMap.end()) {
- // First time, so a new nameId will be set.
- int nameId = -1;
- itsParmDB->putValues (name, nameId, pvs);
- itsParmIdMap[name] = nameId;
- } else {
- // Parm has been put before.
- int nameId = pit->second;
- itsParmDB->putValues (name, nameId, pvs);
- }
- }
- }
- }
- }
- }
- }
-
-} //# end namespace DPPP
-} //# end namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/DummyStep.cc b/CEP/DP3/DPPP/src/DummyStep.cc
deleted file mode 100644
index 4fb62624db78903e2161ae0d27f0a0446404d6c9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/DummyStep.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-//# GainCal.cc: DPPP step class to DummyStep visibilities
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/DummyStep.h>
-
-#include <iostream>
-#include <Common/ParameterSet.h>
-#include <Common/Timer.h>
-
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- DummyStep::DummyStep (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput(input)
- {
-
- }
-
- DummyStep::~DummyStep()
- {}
-
- void DummyStep::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- }
-
- void DummyStep::show (std::ostream& os) const
- {
- os << "DummyStep " << itsName << endl;
- }
-
- void DummyStep::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " DummyStep " << itsName << endl;
- }
-
- bool DummyStep::process (const DPBuffer& bufin)
- {
- itsTimer.start();
- itsBuffer.copy (bufin);
- itsInput->fetchUVW(bufin, itsBuffer, itsTimer);
- itsInput->fetchWeights(bufin, itsBuffer, itsTimer);
-
- itsTimer.stop();
- getNextStep()->process(itsBuffer);
- return false;
- }
-
-
- void DummyStep::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/EstimateMixed.cc b/CEP/DP3/DPPP/src/EstimateMixed.cc
deleted file mode 100644
index b2eb4218e61adabdd5398f5bd8e3b80d9af2ce9a..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/EstimateMixed.cc
+++ /dev/null
@@ -1,341 +0,0 @@
-//# EstimateMixed.cc: Estimate Jones matrices for several directions
-//# simultaneously. A separate data stream is used for each direction. The
-//# mixing coefficients quantify the influence of each direction on each of the
-//# other directions (including time and frequency smearing).
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/EstimateMixed.h>
-#include <Common/LofarLogger.h>
-#include <casacore/scimath/Fitting/LSQFit.h>
-#include <Common/StreamUtil.h> ///
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-namespace
-{
-// Compute a map that contains the index of the unknowns related to the
-// specified baseline in the list of all unknowns.
-void makeIndex(size_t nDirection, size_t nStation, const Baseline &baseline,
- unsigned int *index)
-{
- const size_t nCorrelation = 4;
- for(size_t cr = 0; cr < nCorrelation; ++cr)
- {
- size_t idx0 = baseline.first * 8 + (cr / 2) * 4; // row of P
- size_t idx1 = baseline.second * 8 + (cr % 2) * 4;// column of Q (row of Q^H)
-
- for(size_t dr = 0; dr < nDirection; ++dr)
- {
- *index++ = idx0;
- *index++ = idx0 + 1;
- *index++ = idx0 + 2;
- *index++ = idx0 + 3;
-
- *index++ = idx1;
- *index++ = idx1 + 1;
- *index++ = idx1 + 2;
- *index++ = idx1 + 3;
-
- idx0 += nStation * 8;
- idx1 += nStation * 8;
- }
- }
-}
-} // Unnamed namespace.
-
-
-bool estimate(size_t nDirection, size_t nStation, size_t nBaseline,
- size_t nChannel, const_cursor<Baseline> baselines,
- vector<const_cursor<fcomplex> > data, vector<const_cursor<dcomplex> > model,
- const_cursor<bool> flag, const_cursor<float> weight,
- const_cursor<dcomplex> mix, double *unknowns, size_t maxiter)
-{
- ASSERT(data.size() == nDirection && model.size() == nDirection);
- bool sh=false;
-
- // Initialize LSQ solver.
- const size_t nUnknowns = nDirection * nStation * 4 * 2;
- casacore::LSQFit solver(nUnknowns);
-
- // Each visibility provides information about two (complex) unknowns per
- // station per direction. A visibility is measured by a specific
- // interferometer, which is the combination of two stations. Thus, in total
- // each visibility provides information about (no. of directions) x 2 x 2
- // x 2 (scalar) unknowns = (no. of directions) x 8. For each of these
- // unknowns the value of the partial derivative of the model with respect
- // to the unknown has to be computed.
- const size_t nPartial = nDirection * 8;
- vector<unsigned int> dIndex(4 * nPartial);
-
- // Allocate space for intermediate results.
- vector<dcomplex> M(nDirection * 4), dM(nDirection * 16);
- vector<double> dR(nPartial), dI(nPartial);
-
- // Iterate until convergence.
- size_t nIterations = 0;
- while(!solver.isReady() && nIterations < maxiter)
- {
- if (sh) cout<<endl<<"iteration " << nIterations << endl;
- for(size_t bl = 0; bl < nBaseline; ++bl)
- {
- const size_t p = baselines->first;
- const size_t q = baselines->second;
-
- if(p != q)
- {
- // Create partial derivative index for current baseline.
- makeIndex(nDirection, nStation, *baselines, &(dIndex[0]));
- if (sh) cout<<"derinx="<<dIndex<<endl;
-
- for(size_t ch = 0; ch < nChannel; ++ch)
- {
- for(size_t dr = 0; dr < nDirection; ++dr)
- {
- // Jones matrix for station P.
- const double *Jp =
- &(unknowns[dr * nStation * 8 + p * 8]);
- const dcomplex Jp_00(Jp[0], Jp[1]);
- const dcomplex Jp_01(Jp[2], Jp[3]);
- const dcomplex Jp_10(Jp[4], Jp[5]);
- const dcomplex Jp_11(Jp[6], Jp[7]);
-
- // Jones matrix for station Q, conjugated.
- const double *Jq =
- &(unknowns[dr * nStation * 8 + q * 8]);
- const dcomplex Jq_00(Jq[0], -Jq[1]);
- const dcomplex Jq_01(Jq[2], -Jq[3]);
- const dcomplex Jq_10(Jq[4], -Jq[5]);
- const dcomplex Jq_11(Jq[6], -Jq[7]);
-
- // Fetch model visibilities for the current direction.
- const dcomplex xx = model[dr][0];
- const dcomplex xy = model[dr][1];
- const dcomplex yx = model[dr][2];
- const dcomplex yy = model[dr][3];
-
- // Precompute terms involving conj(Jq) and the model
- // visibilities.
- const dcomplex Jq_00xx_01xy = Jq_00 * xx + Jq_01 * xy;
- const dcomplex Jq_00yx_01yy = Jq_00 * yx + Jq_01 * yy;
- const dcomplex Jq_10xx_11xy = Jq_10 * xx + Jq_11 * xy;
- const dcomplex Jq_10yx_11yy = Jq_10 * yx + Jq_11 * yy;
-
- // Precompute (Jp x conj(Jq)) * vec(data), where 'x'
- // denotes the Kronecker product. This is the model
- // visibility for the current direction, with the
- // current Jones matrix estimates applied. This is
- // stored in M.
- // Also, precompute the partial derivatives of M with
- // respect to all 16 parameters (i.e. 2 Jones matrices
- // Jp and Jq, 4 complex scalars per Jones matrix, 2 real
- // scalars per complex scalar, 2 * 4 * 2 = 16). These
- // partial derivatives are stored in dM.
- M[dr * 4] = Jp_00 * Jq_00xx_01xy + Jp_01 * Jq_00yx_01yy;
- dM[dr * 16] = Jq_00xx_01xy; //dM_00/dJp_00
- dM[dr * 16 + 1] = Jq_00yx_01yy; //dM_00/dJp_01
- dM[dr * 16 + 2] = Jp_00 * xx + Jp_01 * yx;//dM_00/dJq_00
- dM[dr * 16 + 3] = Jp_00 * xy + Jp_01 * yy;//dM_00/dJq_01
-
- M[dr * 4 + 1] = Jp_00 * Jq_10xx_11xy + Jp_01
- * Jq_10yx_11yy;
- dM[dr * 16 + 4] = Jq_10xx_11xy; //dM_01/dJp_00
- dM[dr * 16 + 5] = Jq_10yx_11yy; //dM_01/dJp_01
- dM[dr * 16 + 6] = dM[dr * 16 + 2]; //dM_01/dJq_10
- dM[dr * 16 + 7] = dM[dr * 16 + 3]; //dM_01/dJq_11
-
- M[dr * 4 + 2] = Jp_10 * Jq_00xx_01xy + Jp_11
- * Jq_00yx_01yy;
- dM[dr * 16 + 8] = dM[dr * 16]; //dM_10/dJp_10
- dM[dr * 16 + 9] = dM[dr * 16 + 1]; //dM_10/dJp_11
- dM[dr * 16 + 10] =Jp_10 * xx + Jp_11 * yx;//dM_10/dJq_00
- dM[dr * 16 + 11] =Jp_10 * xy + Jp_11 * yy;//dM_10/dJq_01
-
- M[dr * 4 + 3] = Jp_10 * Jq_10xx_11xy + Jp_11
- * Jq_10yx_11yy;
- dM[dr * 16 + 12] = dM[dr * 16 + 4]; //dM_11/dJp_10
- dM[dr * 16 + 13] = dM[dr * 16 + 5]; //dM_11/dJp_11
- dM[dr * 16 + 14] = dM[dr * 16 + 10]; //dM_11/dJq_10
- dM[dr * 16 + 15] = dM[dr * 16 + 11]; //dM_11/dJq_11
- }
- if (sh) {
- cout<<"M="<<M<<endl;
- cout<<"dM="<<dM<<endl;
- }
-
- for(size_t cr = 0; cr < 4; ++cr) // correlation: 00,01,10,11
- {
- if(!flag[cr])
- {
- for(size_t tg = 0; tg < nDirection; ++tg)
- {
- dcomplex visibility(0.0, 0.0);
- for(size_t dr = 0; dr < nDirection; ++dr)
- {
- // Look-up mixing weight.
- const dcomplex mix_weight = *mix;
-
- // Weight model visibility.
- visibility += mix_weight * M[dr * 4 + cr];
-
- // Compute weighted partial derivatives.
- dcomplex derivative(0.0, 0.0);
- derivative =
- mix_weight * dM[dr * 16 + cr * 4];
- dR[dr * 8] = real(derivative); //for cr==0: Re(d/dRe(p_00)))
- dI[dr * 8] = imag(derivative); //for cr==0: Re(d/dIm(p_00)))
- dR[dr * 8 + 1] = -imag(derivative);//for cr==0: Im(d/dRe(p_00)))
- dI[dr * 8 + 1] = real(derivative); //for cr==0: Im(d/dIm(p_00)))
-
- derivative =
- mix_weight * dM[dr * 16 + cr * 4 + 1];
- dR[dr * 8 + 2] = real(derivative); //for cr==0: Re(d/dRe(p_01)))
- dI[dr * 8 + 2] = imag(derivative); //for cr==0: Re(d/dIm(p_01)))
- dR[dr * 8 + 3] = -imag(derivative);//for cr==0: Im(d/dRe(p_01)))
- dI[dr * 8 + 3] = real(derivative); //for cr==0: Im(d/dIm(p_01)))
-
- derivative =
- mix_weight * dM[dr * 16 + cr * 4 + 2];
- dR[dr * 8 + 4] = real(derivative); //for cr==0: Re(d/dRe(q_00)))
- dI[dr * 8 + 4] = imag(derivative); //for cr==0: Re(d/dIm(q_00)))
- dR[dr * 8 + 5] = imag(derivative); //for cr==0: Im(d/dRe(q_00)))
- dI[dr * 8 + 5] = -real(derivative);//for cr==0: Im(d/dIm(q_00)))
-
- derivative =
- mix_weight * dM[dr * 16 + cr * 4 + 3];
- dR[dr * 8 + 6] = real(derivative); //for cr==0: Re(d/dRe(q_01)))
- dI[dr * 8 + 6] = imag(derivative); //for cr==0: Re(d/dIm(q_01)))
- dR[dr * 8 + 7] = imag(derivative); //for cr==0: Im(d/dRe(q_01)))
- dI[dr * 8 + 7] = -real(derivative);//for cr==0: Im(d/dIm(q_01)))
-
- // Move to next source direction.
- mix.forward(1);
- } // Source directions.
-
- // Compute the residual.
- dcomplex residual =
- static_cast<dcomplex>(data[tg][cr])
- - visibility;
-
- // Update the normal equations.
- solver.makeNorm(nPartial,
- &(dIndex[cr * nPartial]), &(dR[0]),
- static_cast<double>(weight[cr]),
- real(residual));
- solver.makeNorm(nPartial,
- &(dIndex[cr * nPartial]), &(dI[0]),
- static_cast<double>(weight[cr]),
- imag(residual));
- if (sh) {
- cout<<"makeres "<<real(residual)<<' '<<weight[cr]<<' '<<nPartial;
- for (uint i=0; i<nPartial; ++i) {
- cout << ' '<<dIndex[cr*nPartial+i]<<' '<<dR[i];
- }
- cout<<endl;
- }
-
- // Move to next target direction.
- mix.backward(1, nDirection);
- mix.forward(0);
- } // Target directions.
-
- // Reset cursor to the start of the correlation.
- mix.backward(0, nDirection);
- }
-
- // Move to the next correlation.
- mix.forward(2);
- } // Correlations.
-
- // Move to the next channel.
- mix.backward(2, 4);
- mix.forward(3);
-
- for(size_t dr = 0; dr < nDirection; ++dr)
- {
- model[dr].forward(1);
- data[dr].forward(1);
- }
- flag.forward(1);
- weight.forward(1);
- } // Channels.
-
- // Reset cursors to the start of the baseline.
- for(size_t dr = 0; dr < nDirection; ++dr)
- {
- model[dr].backward(1, nChannel);
- data[dr].backward(1, nChannel);
- }
- flag.backward(1, nChannel);
- weight.backward(1, nChannel);
- mix.backward(3, nChannel);
- }
-
- // Move cursors to the next baseline.
- for(size_t dr = 0; dr < nDirection; ++dr)
- {
- model[dr].forward(2);
- data[dr].forward(2);
- }
- flag.forward(2);
- weight.forward(2);
- mix.forward(4);
- ++baselines;
- } // Baselines.
-
- // Reset all cursors for the next iteration.
- for(size_t dr = 0; dr < nDirection; ++dr)
- {
- model[dr].backward(2, nBaseline);
- data[dr].backward(2, nBaseline);
- }
- flag.backward(2, nBaseline);
- weight.backward(2, nBaseline);
- mix.backward(4, nBaseline);
- baselines -= nBaseline;
-
- // Perform LSQ iteration.
- casacore::uInt rank;
- bool status = solver.solveLoop(rank, unknowns, true);
- ASSERT(status);
- if (sh) {
- cout<<"solution=[";
- for (uint i=0; i<nUnknowns; ++i) {
- cout << unknowns[i]<<',';
- }
- cout<<endl;
- }
-
- // Update iteration count.
- ++nIterations;
- }
-
- const bool converged = (solver.isReady() == casacore::LSQFit::SOLINCREMENT
- || solver.isReady() == casacore::LSQFit::DERIVLEVEL);
- return converged;
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/EstimateNew.cc b/CEP/DP3/DPPP/src/EstimateNew.cc
deleted file mode 100644
index ac7c073a312051f0ed448f71c7a6c44d3bf00cd6..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/EstimateNew.cc
+++ /dev/null
@@ -1,475 +0,0 @@
-//# EstimateNew.cc: Estimate Jones matrices for several directions and stations
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/EstimateNew.h>
-#include <Common/LofarLogger.h>
-#include <Common/StreamUtil.h> ///
-
-#include <casacore/scimath/Fitting/LSQFit.h>
-
-
-namespace LOFAR {
- namespace DPPP {
-
- EstimateNew::EstimateNew()
- {}
-
- void EstimateNew::update (size_t maxndir, size_t nBaseline, size_t nStation,
- size_t nChannel, size_t maxIter,
- bool propagateSolution)
- {
- itsNrBaselines = nBaseline;
- itsNrStations = nStation;
- itsNrChannels = nChannel;
- itsMaxIter = maxIter;
- itsNrDir = maxndir;
- itsPropagateSolution = propagateSolution;
- itsSolveStation.resize (nStation);
- itsUnknowns.resize (maxndir * nStation * 4 * 2);
- itsSolution.resize (itsUnknowns.size());
- std::fill (itsSolution.begin(), itsSolution.end(), 0);
- itsDerivIndex.resize (maxndir*4*2*4);
- itsM.resize (maxndir*4);
- itsdM.resize (maxndir*4*4);
- itsdR.resize (maxndir*8);
- itsdI.resize (maxndir*8);
- }
-
- // Initialize the solution to the defaultGain for sources/stations not to solve.
- // Set to 0 and diagonal to defaultGaib for solvable ones if no propagation.
- void EstimateNew::initSolution (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet,
- double defaultGain)
- {
- uint dr=0;
- double* solution = &(itsSolution[0]);
- for (size_t i=0; i<itsNrDir; ++i) {
- if (dr < srcSet.size() && srcSet[dr] == i) {
- // This source is to be solved.
- for (size_t st=0; st<itsNrStations; ++st) {
- if (unknownsIndex[dr][st] >= 0) {
- if (! itsPropagateSolution) {
- std::fill (solution, solution+8, 0);
- }
- // Solvable; set diagonal to 1 if it is 0.
- if (solution[0] == 0) solution[0] = 1;
- if (solution[6] == 0) solution[6] = 1;
- } else {
- // Set non-solvable station-source to 0.
- std::fill (solution, solution+8, 0);
- solution[0] = solution[6] = defaultGain;
- }
- solution += 8;
- }
- dr++;
- } else {
- // Set entire non-solvable source to 0.
- std::fill (solution, solution + 8*itsNrStations, 0.);
- solution += 8*itsNrStations;
- }
- }
- }
-
- // Clear the solution for unsolvable stations
- // (essentially changing defaultGain to 0).
- void EstimateNew::clearNonSolvable (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet)
- {
- uint dr=0;
- double* solution = &(itsSolution[0]);
- for (size_t i=0; i<itsNrDir; ++i) {
- if (dr < srcSet.size() && srcSet[dr] == i) {
- // This source is to be solved.
- for (size_t st=0; st<itsNrStations; ++st) {
- if (unknownsIndex[dr][st] < 0) {
- // Set non-solvable station-source to 0.
- std::fill (solution, solution+8, 0);
- }
- solution += 8;
- }
- dr++;
- } else {
- // Skip entire source.
- solution += 8*itsNrStations;
- }
- }
- }
-
- void EstimateNew::fillSolution (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet)
- {
- // Copy the solution for the direction-stations to solve.
- // Note that the solution vector contains all possible sources/stations.
- const double* unknowns = &(itsUnknowns[0]);
- for (size_t dr=0; dr<srcSet.size(); ++dr) {
- size_t inx = srcSet[dr] * itsNrStations * 8;
- double* solution = &(itsSolution[inx]);
- for (size_t st=0; st<itsNrStations; ++st) {
- if (unknownsIndex[dr][st] >= 0) {
- for (size_t k=0; k<8; ++k) {
- *solution++ = *unknowns++;
- }
- } else {
- solution += 8;
- }
- }
- }
- }
-
- // Form the partial derivative index for a particular baseline.
- // Partial derivatives are only needed if direction-station is solved for.
- // Each visibility provides information about two (complex) unknowns per
- // station per direction. A visibility is measured by a specific
- // interferometer, which is the combination of two stations. Thus, in total
- // each visibility provides information about (no. of directions) x 2 x 2
- // x 2 (scalar) unknowns = (no. of directions) x 8. For each of these
- // unknowns the value of the partial derivative of the model with respect
- // to the unknown has to be computed.
- uint EstimateNew::fillDerivIndex (size_t ndir,
- const vector<vector<int> >& unknownsIndex,
- const Baseline& baseline)
- {
- // Per direction a baseline has 32 equations with information about
- // 16 unknowns: real and imag part of p00,p01,p10,p11,q00,q01,q10,q11
- // where p and q are the stations forming the baseline.
- // However, only fill if a station has to be solved.
- size_t n = 0;
- for (size_t cr=0; cr<4; ++cr) {
- for (size_t dr=0; dr<ndir; ++dr) {
- if (unknownsIndex[dr][baseline.first] >= 0) {
- size_t idx0 = unknownsIndex[dr][baseline.first] + (cr/2)*4;
- itsDerivIndex[n++] = idx0;
- itsDerivIndex[n++] = idx0 + 1;
- itsDerivIndex[n++] = idx0 + 2;
- itsDerivIndex[n++] = idx0 + 3;
- }
- if (unknownsIndex[dr][baseline.second] >= 0) {
- size_t idx1 = unknownsIndex[dr][baseline.second] + (cr%2)*4;
- itsDerivIndex[n++] = idx1;
- itsDerivIndex[n++] = idx1 + 1;
- itsDerivIndex[n++] = idx1 + 2;
- itsDerivIndex[n++] = idx1 + 3;
- }
- }
- }
- // Return nr of partial derivatives per correlation.
- return n/4;
- }
-
- // Note that the cursors are passed by value, so a copy is made.
- // In this way no explicit reset of the cursor is needed on a next call.
- bool EstimateNew::estimate (const vector<vector<int> >& unknownsIndex,
- const vector<uint>& srcSet,
- const_cursor<Baseline> baselines,
- vector<const_cursor<fcomplex> > data,
- vector<const_cursor<dcomplex> > model,
- const_cursor<bool> flag,
- const_cursor<float> weight,
- const_cursor<dcomplex> mix,
- double defaultGain,
- bool solveBoth,
- uint verbose)
- {
- initSolution (unknownsIndex, srcSet, defaultGain);
- // Determine if a station has to be solved for any source.
- itsSolveStation = false;
- size_t nUnknowns = 0;
- const size_t nDirection = srcSet.size();
- for (size_t dr=0; dr<nDirection; ++dr) {
- uint drOrig = srcSet[dr];
- const double* solution = &(itsSolution[drOrig*itsNrStations*8]);
- for (size_t st=0; st<itsNrStations; ++st) {
- if (unknownsIndex[dr][st] >= 0) {
- itsSolveStation[st] = true;
- std::copy (solution, solution+8, itsUnknowns.begin()+nUnknowns);
- nUnknowns += 8;
- }
- solution += 8;
- }
- }
- if (verbose > 12) {
- cout<<"unkindex="<<unknownsIndex<<endl;
- }
- // Initialize LSQ solver.
- casacore::LSQFit solver(nUnknowns);
- // Iterate until convergence.
- itsNrIter = 0;
- while (!solver.isReady() && itsNrIter < itsMaxIter) {
- if (verbose > 12) {
- cout<<endl<<"iteration " << itsNrIter << endl;
- }
- for (size_t bl=0; bl<itsNrBaselines; ++bl) {
- const size_t p = baselines->first;
- const size_t q = baselines->second;
- // Only compute if no autocorr and if stations need to be solved.
- if (p != q && ((itsSolveStation[p] || itsSolveStation[q]) &&
- (!solveBoth ||
- (itsSolveStation[p] && itsSolveStation[q])))) {
- // Create partial derivative index for current baseline.
- size_t nPartial = fillDerivIndex (srcSet.size(), unknownsIndex,
- *baselines);
- if (verbose > 13) {
- cout<<"derinx="<<itsDerivIndex<<endl;
- }
- // Generate equations for each channel.
- for (size_t ch=0; ch<itsNrChannels; ++ch) {
- for (size_t dr=0; dr<nDirection; ++dr) {
- uint drOrig = srcSet[dr];
- // Jones matrix for station P.
- const double *Jp = &(itsSolution[(drOrig*itsNrStations + p)*8]);
- const dcomplex Jp_00(Jp[0], Jp[1]);
- const dcomplex Jp_01(Jp[2], Jp[3]);
- const dcomplex Jp_10(Jp[4], Jp[5]);
- const dcomplex Jp_11(Jp[6], Jp[7]);
-
- // Jones matrix for station Q, conjugated.
- const double *Jq = &(itsSolution[(drOrig*itsNrStations + q)*8]);
- const dcomplex Jq_00(Jq[0], -Jq[1]);
- const dcomplex Jq_01(Jq[2], -Jq[3]);
-
- const dcomplex Jq_10(Jq[4], -Jq[5]);
- const dcomplex Jq_11(Jq[6], -Jq[7]);
-
- // Fetch model visibilities for the current direction.
- const dcomplex xx = model[dr][0];
- const dcomplex xy = model[dr][1];
- const dcomplex yx = model[dr][2];
- const dcomplex yy = model[dr][3];
-
- // Precompute terms involving conj(Jq) and the model
- // visibilities.
- const dcomplex Jq_00xx_01xy = Jq_00 * xx + Jq_01 * xy;
- const dcomplex Jq_00yx_01yy = Jq_00 * yx + Jq_01 * yy;
- const dcomplex Jq_10xx_11xy = Jq_10 * xx + Jq_11 * xy;
- const dcomplex Jq_10yx_11yy = Jq_10 * yx + Jq_11 * yy;
-
- // Precompute (Jp x conj(Jq)) * vec(data), where 'x'
- // denotes the Kronecker product. This is the model
- // visibility for the current direction, with the
- // current Jones matrix estimates applied. This is
- // stored in M.
- // Also, precompute the partial derivatives of M with
- // respect to all 16 parameters (i.e. 2 Jones matrices
- // Jp and Jq, 4 complex scalars per Jones matrix, 2 real
- // scalars per complex scalar, 2 * 4 * 2 = 16). These
- // partial derivatives are stored in dM.
- // Note that conj(Jq) is used and that q01 and q10 are swapped.
-
- itsM[dr * 4] = Jp_00 * Jq_00xx_01xy + Jp_01 * Jq_00yx_01yy;
- // Derivatives of M00 wrt p00, p01, q00, q01
- itsdM[dr * 16] = Jq_00xx_01xy;
- itsdM[dr * 16 + 1] = Jq_00yx_01yy;
- itsdM[dr * 16 + 2] = Jp_00 * xx + Jp_01 * yx;
- itsdM[dr * 16 + 3] = Jp_00 * xy + Jp_01 * yy;
-
- itsM[dr * 4 + 1] = Jp_00 * Jq_10xx_11xy + Jp_01 * Jq_10yx_11yy;
- // Derivatives of M01 wrt p00, p01, q10, q11
- itsdM[dr * 16 + 4] = Jq_10xx_11xy;
- itsdM[dr * 16 + 5] = Jq_10yx_11yy;
- itsdM[dr * 16 + 6] = itsdM[dr * 16 + 2];
- itsdM[dr * 16 + 7] = itsdM[dr * 16 + 3];
-
- itsM[dr * 4 + 2] = Jp_10 * Jq_00xx_01xy + Jp_11 * Jq_00yx_01yy;
- // Derivatives of M10 wrt p10, p11, q00, q01
- itsdM[dr * 16 + 8] = itsdM[dr * 16];
- itsdM[dr * 16 + 9] = itsdM[dr * 16 + 1];
- itsdM[dr * 16 + 10] = Jp_10 * xx + Jp_11 * yx;
- itsdM[dr * 16 + 11] = Jp_10 * xy + Jp_11 * yy;
-
- itsM[dr * 4 + 3] = Jp_10 * Jq_10xx_11xy + Jp_11 * Jq_10yx_11yy;
- // Derivatives of M11 wrt p10, p11, q10, q11
- itsdM[dr * 16 + 12] = itsdM[dr * 16 + 4];
- itsdM[dr * 16 + 13] = itsdM[dr * 16 + 5];
- itsdM[dr * 16 + 14] = itsdM[dr * 16 + 10];
- itsdM[dr * 16 + 15] = itsdM[dr * 16 + 11];
- }
- if (verbose > 14) {
- cout<<"M="<<itsM<<endl;
- cout<<"dM="<<itsdM<<endl;
- }
-
- // Now compute the equations (per pol) for D*M=A where
- // D is the NxN demixing weight matrix
- // M is the model visibilities vector for the N directions
- // A is the shifted observed visibilities vector for N directions
- // Note that each element in the vectors is a 2x2 matrix
- // (xx,xy,yx,yy) of complex values.
- // A complex multiplication of (a,b) and (c,d) gives (ac-bd,ad+bc)
- // Thus real partial derivatives wrt a,b,c,d are c,-d,a,-b.
- // Imaginary partial derivatives wrt a,b,c,d are d,c,b,a
- for (size_t cr=0; cr<4; ++cr) {
- // Only use visibility if not flagged.
- if (!flag[cr]) {
- // For each direction a set of equations is generated.
- for (size_t tg=0; tg<nDirection; ++tg) {
- dcomplex visibility(0.0, 0.0);
- // Each direction is dependent on all directions.
- size_t off = 0;
- for (size_t dr=0; dr<nDirection; ++dr) {
- bool do1 = unknownsIndex[dr][p] >= 0;
- bool do2 = unknownsIndex[dr][q] >= 0;
- // Only generate equations if a station has to be solved
- // for this direction.
- if ((do1 && do2) || (!solveBoth && (do1 || do2))) {
- // Look-up mixing weight.
- const dcomplex mix_weight = *mix;
- // Sum weighted model visibilities.
- visibility += mix_weight * itsM[dr * 4 + cr];
-
- // Compute weighted partial derivatives.
- if (do1) {
- dcomplex der(mix_weight * itsdM[dr * 16 + cr * 4]);
- itsdR[off] = real(der);
- itsdI[off] = imag(der);
- itsdR[off + 1] = -imag(der);
- itsdI[off + 1] = real(der);
- off += 2;
- }
- if (do2) {
- dcomplex der(mix_weight * itsdM[dr * 16 + cr * 4 + 1]);
- itsdR[off] = real(der);
- itsdI[off] = imag(der);
- itsdR[off + 1] = -imag(der);
- itsdI[off + 1] = real(der);
- off += 2;
- }
- if (do1) {
- dcomplex der(mix_weight * itsdM[dr * 16 + cr * 4 + 2]);
- itsdR[off] = real(der);
- itsdI[off] = imag(der);
- itsdR[off + 1] = imag(der); // conjugate
- itsdI[off + 1] = -real(der);
- off += 2;
- }
- if (do2) {
- dcomplex der(mix_weight * itsdM[dr * 16 + cr * 4 + 3]);
- itsdR[off] = real(der);
- itsdI[off] = imag(der);
- itsdR[off + 1] = imag(der);
- itsdI[off + 1] = -real(der);
- off += 2;
- }
- }
- // Move to next source direction.
- mix.forward(1);
- } // Source directions.
-
- // Compute the residual.
- dcomplex residual(data[tg][cr]);
- residual -= visibility;
-
- // Update the normal equations.
- solver.makeNorm(nPartial,
- &(itsDerivIndex[cr * nPartial]), &(itsdR[0]),
- static_cast<double>(weight[cr]),
- real(residual));
- solver.makeNorm(nPartial,
- &(itsDerivIndex[cr * nPartial]), &(itsdI[0]),
- static_cast<double>(weight[cr]),
- imag(residual));
- if (verbose > 14) {
- cout<<"makeres "<<real(residual)<<' '<<weight[cr]
- <<' '<<nPartial;
- for (uint i=0; i<nPartial; ++i) {
- cout << ' '<<itsDerivIndex[cr*nPartial+i]<<' '<<itsdR[i];
- }
- cout<<endl;
- }
-
- // Move to next target direction.
- mix.backward(1, nDirection);
- mix.forward(0);
- } // Target directions.
-
- // Reset cursor to the start of the correlation.
- mix.backward(0, nDirection);
- }
-
- // Move to the next correlation.
- mix.forward(2);
- } // Correlations.
-
- // Move to the next channel.
- mix.backward(2, 4);
- mix.forward(3);
-
- for (size_t dr=0; dr<nDirection; ++dr) {
- model[dr].forward(1);
- data[dr].forward(1);
- }
- flag.forward(1);
- weight.forward(1);
- } // Channels.
-
- // Reset cursors to the start of the baseline.
- for (size_t dr=0; dr<nDirection; ++dr) {
- model[dr].backward(1, itsNrChannels);
- data[dr].backward(1, itsNrChannels);
- }
- flag.backward(1, itsNrChannels);
- weight.backward(1, itsNrChannels);
- mix.backward(3, itsNrChannels);
- }
-
- // Move cursors to the next baseline.
- for (size_t dr=0; dr<nDirection; ++dr) {
- model[dr].forward(2);
- data[dr].forward(2);
- }
- flag.forward(2);
- weight.forward(2);
- mix.forward(4);
- ++baselines;
- } // Baselines.
-
- // Reset all cursors for the next iteration.
- for (size_t dr=0; dr<nDirection; ++dr) {
- model[dr].backward(2, itsNrBaselines);
- data[dr].backward(2, itsNrBaselines);
- }
- flag.backward(2, itsNrBaselines);
- weight.backward(2, itsNrBaselines);
- mix.backward(4, itsNrBaselines);
- baselines -= itsNrBaselines;
-
- // Perform LSQ iteration.
- casacore::uInt rank;
- bool status = solver.solveLoop(rank, &(itsUnknowns[0]), true);
- ASSERT(status);
- // Copy the unknowns to the full solution.
- fillSolution (unknownsIndex, srcSet);
- if (verbose > 13) {
- cout<<"unknowns="<<nUnknowns<<' '<<itsUnknowns<<endl;
- cout<<"solution="<<itsSolution<<endl;
- }
- // Update iteration count.
- itsNrIter++;
- }
- bool converged = (solver.isReady() == casacore::LSQFit::SOLINCREMENT ||
- solver.isReady() == casacore::LSQFit::DERIVLEVEL);
- /// clearNonSolvable (unknownsIndex, srcSet);
- return converged;
- }
-
- } //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/Filter.cc b/CEP/DP3/DPPP/src/Filter.cc
deleted file mode 100644
index dc4f46cf68d559da6db387eb67060c6f39663ff1..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Filter.cc
+++ /dev/null
@@ -1,342 +0,0 @@
-//# Filter.cc: DPPP step to filter out baselines and channels
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Filter.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPLogger.h>
-#include <Common/ParameterSet.h>
-
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/tables/Tables/TableRecord.h>
-#include <casacore/tables/TaQL/ExprNode.h>
-#include <casacore/tables/TaQL/RecordGram.h>
-#include <casacore/casa/Containers/Record.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- Filter::Filter (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsStartChanStr (parset.getString(prefix+"startchan", "0")),
- itsNrChanStr (parset.getString(prefix+"nchan", "0")),
- itsRemoveAnt (parset.getBool (prefix+"remove", false)),
- itsBaselines (parset, prefix),
- itsDoSelect (false)
- {}
-
- Filter::Filter (DPInput* input, const BaselineSelection& baselines)
- : itsInput (input),
- itsStartChanStr ("0"),
- itsNrChanStr ("0"),
- itsRemoveAnt (false),
- itsBaselines (baselines),
- itsDoSelect (false)
- {}
-
- Filter::~Filter()
- {}
-
- void Filter::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- info().setWriteFlags();
- if (itsRemoveAnt) {
- info().setMetaChanged();
- }
- // Parse the chan expressions.
- // Nr of channels can be used as 'nchan' in the expressions.
- Record rec;
- rec.define ("nchan", infoIn.nchan());
- TableExprNode node1 (RecordGram::parse(rec, itsStartChanStr));
- TableExprNode node2 (RecordGram::parse(rec, itsNrChanStr));
- // nchan=0 means until the last channel.
- double result;
- node1.get (rec, result);
- itsStartChan = uint(result+0.001);
- node2.get (rec, result);
- uint nrChan = uint(result+0.0001);
- uint nAllChan = getInfo().nchan();
- ASSERTSTR (itsStartChan < nAllChan,
- "startchan " << itsStartChan
- << " exceeds nr of available channels (" << nAllChan << ')');
- uint maxNrChan = nAllChan - itsStartChan;
- if (nrChan == 0) {
- nrChan = maxNrChan;
- } else {
- nrChan = std::min (nrChan, maxNrChan);
- }
- itsDoSelect = itsStartChan>0 || nrChan<maxNrChan;
- // Handle possible baseline selection.
- if (itsBaselines.hasSelection()) {
- Matrix<bool> selbl(itsBaselines.apply (infoIn));
- const Vector<Int>& ant1 = getInfo().getAnt1();
- const Vector<Int>& ant2 = getInfo().getAnt2();
- itsSelBL.reserve (ant1.size());
- for (uint i=0; i<ant1.size(); ++i) {
- if (selbl(ant1[i], ant2[i])) {
- itsSelBL.push_back (i);
- }
- }
- if (itsSelBL.size() < ant1.size()) {
- itsDoSelect = true;
- }
- }
- if (itsDoSelect || itsRemoveAnt) {
- // Update the DPInfo object.
- info().update (itsStartChan, nrChan, itsSelBL, itsRemoveAnt);
- if (itsDoSelect) {
- // Shape the arrays in the buffer.
- IPosition shape (3, infoIn.ncorr(), nrChan, getInfo().nbaselines());
- itsBuf.getData().resize (shape);
- itsBuf.getFlags().resize (shape);
- itsBuf.getWeights().resize (shape);
- if (! itsSelBL.empty()) {
- itsBuf.getUVW().resize (IPosition(2, 3, shape[2]));
- }
- }
- }
- }
-
- void Filter::show (std::ostream& os) const
- {
- os << "Filter " << itsName << std::endl;
- os << " startchan: " << itsStartChan << " (" << itsStartChanStr
- << ')' << std::endl;
- os << " nchan: " << getInfo().nchan() << " (" << itsNrChanStr
- << ')' << std::endl;
- itsBaselines.show (os);
- os << " remove: " << itsRemoveAnt << std::endl;
- }
-
- void Filter::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " Filter " << itsName << endl;
- }
-
- bool Filter::process (const DPBuffer& buf)
- {
- itsTimer.start();
- if (!itsDoSelect) {
- itsBuf.referenceFilled (buf);
- itsTimer.stop();
- getNextStep()->process (buf);
- return true;
- }
- // Get the various data arrays.
- itsBufTmp.referenceFilled (buf);
- const Array<Complex>& data = buf.getData();
- const Array<Bool>& flags = buf.getFlags();
- const Array<Float>& weights =
- itsInput->fetchWeights (buf, itsBufTmp, itsTimer);
- const Array<Double>& uvws =
- itsInput->fetchUVW (buf, itsBufTmp, itsTimer);
- const Array<Bool>& frFlags =
- itsInput->fetchFullResFlags (buf, itsBufTmp, itsTimer);
- // Size fullResFlags if not done yet.
- int frfAvg = frFlags.shape()[0] / data.shape()[1];
- if (itsBuf.getFullResFlags().empty()) {
- IPosition frfShp = frFlags.shape();
- frfShp[0] = getInfo().nchan() * frfAvg;
- frfShp[2] = getInfo().nbaselines();
- itsBuf.getFullResFlags().resize (frfShp);
- }
- // Form the blc and trc for the channel selection.
- IPosition first(3, 0);
- IPosition last (data.shape() - 1);
- first[1] = itsStartChan;
- last[1] = itsStartChan + getInfo().nchan() - 1;
- IPosition frfFirst(3,0);
- IPosition frfLast (frFlags.shape() - 1);
- frfFirst[0] = first[1] * frfAvg;
- frfLast[0] = (last[1] + 1) * frfAvg - 1;
- // Copy the data into the output buffer.
- if (itsSelBL.empty()) {
- // No baseline selection; copy all data for given channels to
- // make them contiguous.
- // UVW can be referenced, because not dependent on channel.
- itsBuf.getData().assign (data(first, last));
- itsBuf.getFlags().assign (flags(first, last));
- itsBuf.getWeights().assign (weights(first, last));
- itsBuf.getFullResFlags().assign (frFlags(frfFirst, frfLast));
- itsBuf.setUVW (buf.getUVW());
- itsBuf.setRowNrs (buf.getRowNrs());
- } else {
- Vector<uint> rowNrs;
- if (! buf.getRowNrs().empty()) {
- rowNrs.resize(getInfo().nbaselines());
- }
- // Copy the data of the selected baselines and channels.
- Complex* toData = itsBuf.getData().data();
- Bool* toFlag = itsBuf.getFlags().data();
- Float* toWeight = itsBuf.getWeights().data();
- Double* toUVW = itsBuf.getUVW().data();
- Bool* toFrf = itsBuf.getFullResFlags().data();
- uint off = data.shape()[0] * first[1]; // offset of first channel
- const Complex* frData = data.data() + off;
- const Bool* frFlag = flags.data() + off;
- const Float* frWeight = weights.data() + off;
- const Double* frUVW = uvws.data();
- int ndfr = data.shape()[0] * data.shape()[1];
- int ndto = itsBuf.getData().shape()[0] * itsBuf.getData().shape()[1];
- int nffr = frFlags.shape()[0];
- int nfto = itsBuf.getFullResFlags().shape()[0];
- for (uint i=0; i<itsSelBL.size(); ++i) {
- if (!buf.getRowNrs().empty()) {
- rowNrs[i] = buf.getRowNrs()[itsSelBL[i]];
- }
- objcopy (toData , frData + itsSelBL[i]*ndfr, ndto);
- toData += ndto;
- objcopy (toFlag , frFlag + itsSelBL[i]*ndfr, ndto);
- toFlag += ndto;
- objcopy (toWeight, frWeight + itsSelBL[i]*ndfr, ndto);
- toWeight += ndto;
- objcopy (toUVW , frUVW + itsSelBL[i]*3 , 3);
- toUVW += 3;
- // Copy FullResFlags for all times.
- const Bool* frFrf = (frFlags.data() + frfFirst[0] +
- itsSelBL[i]*nffr * frFlags.shape()[1]);
- for (int j=0; j<=frfLast[1]; ++j) {
- objcopy (toFrf, frFrf, nfto);
- toFrf += nfto;
- frFrf += nffr;
- }
- }
- itsBuf.setRowNrs(rowNrs);
- }
- itsBuf.setTime (buf.getTime());
- itsBuf.setExposure (buf.getExposure());
- itsTimer.stop();
- getNextStep()->process (itsBuf);
- return true;
- }
-
- void Filter::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void Filter::addToMS (const string& msName)
- {
- getPrevStep()->addToMS(msName);
- if (! itsRemoveAnt) {
- return;
- }
- // See if and which stations have been removed.
- Table antTab (msName + "/ANTENNA", Table::Update);
- Table selTab = antTab(! antTab.col("NAME").in (info().antennaNames()));
- if (selTab.nrow() == 0) {
- return;
- }
- // Remove these rows from the ANTENNA table.
- // Note that stations of baselines that have been filtered out before,
- // will also be removed.
- Vector<uInt> removedAnt = selTab.rowNumbers();
- Vector<Int> antMap = createIdMap (antTab.nrow(), removedAnt);
- antTab.removeRow (removedAnt);
- // Remove and renumber the stations in other subtables.
- Table ms(msName);
- uInt nr;
- renumberSubTable (ms, "FEED", "ANTENNA_ID", removedAnt, antMap, nr);
- renumberSubTable (ms, "POINTING", "ANTENNA_ID", removedAnt, antMap, nr);
- renumberSubTable (ms, "SYSCAL", "ANTENNA_ID", removedAnt, antMap, nr);
- renumberSubTable (ms, "QUALITY_BASELINE_STATISTIC", "ANTENNA1",
- removedAnt, antMap, nr);
- renumberSubTable (ms, "QUALITY_BASELINE_STATISTIC", "ANTENNA2",
- removedAnt, antMap, nr);
- // Finally remove and renumber in the beam tables.
- uInt nrAntFldId;
- Vector<uInt> remAntFldId = renumberSubTable (ms, "LOFAR_ANTENNA_FIELD",
- "ANTENNA_ID",
- removedAnt, antMap,
- nrAntFldId);
- if (! remAntFldId.empty()) {
- Vector<Int> antFldIdMap = createIdMap (nrAntFldId, remAntFldId);
- renumberSubTable (ms, "LOFAR_ELEMENT_FAILURE", "ANTENNA_FIELD_ID",
- remAntFldId, antFldIdMap, nr);
- }
- }
-
- Vector<Int> Filter::createIdMap (uInt nrId,
- const Vector<uInt>& removedIds) const
- {
- // Create the mapping from old to new id.
- Vector<Int> idMap (nrId);
- indgen (idMap); // fill with 0,1,2,...
- int nrrem = 0;
- for (uInt i=0; i<removedIds.size(); ++i) {
- idMap[removedIds[i]] = -1;
- nrrem++;
- if (i < removedIds.size() - 1) {
- for (uInt j=removedIds[i]+1; j<removedIds[i+1]; ++j) {
- idMap[j] -= nrrem;
- }
- }
- }
- for (uInt j=removedIds[removedIds.size()-1]+1; j<idMap.size(); ++j) {
- idMap[j] -= nrrem;
- }
- return idMap;
- }
-
- Vector<uInt> Filter::renumberSubTable (const Table& ms,
- const String& name,
- const String& colName,
- const Vector<uInt>& removedAnt,
- const Vector<Int>& antMap,
- uInt& nrId) const
- {
- // Exit if no such subtable.
- if (! ms.keywordSet().isDefined(name)) {
- return Vector<uInt>();
- }
- // Remove the rows of the removed stations.
- Table subTab (ms.tableName() + '/' + name, Table::Update);
- nrId = subTab.nrow();
- Table selTab = subTab(subTab.col(colName).in (removedAnt));
- subTab.removeRow (selTab.rowNumbers());
- // Renumber the rest.
- ScalarColumn<Int> antCol(subTab, colName);
- Vector<Int> antIds = antCol.getColumn();
- for (uint i=0; i<antIds.size(); ++i) {
- Int newId = antMap[antIds[i]];
- ASSERT (newId >= 0);
- antIds[i] = newId;
- }
- antCol.putColumn (antIds);
- return selTab.rowNumbers();
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/FlagCounter.cc b/CEP/DP3/DPPP/src/FlagCounter.cc
deleted file mode 100644
index ecbf2cd0b400b221c61f09ebc76118c6dbdfc044..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/FlagCounter.cc
+++ /dev/null
@@ -1,377 +0,0 @@
-//# FlagCounter.cc: Class to keep counts of nr of flagged visibilities
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/FlagCounter.h>
-#include <DPPP/DPInput.h>
-#include <Common/ParameterSet.h>
-#include <Common/StreamUtil.h>
-#include <Common/LofarLogger.h>
-#include <casacore/tables/Tables/Table.h>
-#include <casacore/tables/Tables/TableDesc.h>
-#include <casacore/tables/Tables/SetupNewTab.h>
-#include <casacore/tables/Tables/ScaColDesc.h>
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/casa/Arrays/Matrix.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <vector>
-#include <map>
-#include <iomanip>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- FlagCounter::FlagCounter()
- : itsInfo (0),
- itsShowFF (false)
- {}
-
- FlagCounter::FlagCounter (const string& msName,
- const ParameterSet& parset,
- const string& prefix)
- {
- itsWarnPerc = parset.getDouble (prefix+"warnperc", 0);
- itsShowFF = parset.getBool (prefix+"showfullyflagged", false);
- bool save = parset.getBool (prefix+"save", false);
- if (save) {
- // Percentages have to be saved, so form the table name to use.
- string path = parset.getString (prefix+"path", "");
- // Use the step name (without dot) as a name suffix.
- string suffix = prefix;
- string::size_type pos = suffix.find ('.');
- if (pos != string::npos) {
- suffix = suffix.substr(0, pos);
- }
- // Use the MS name as the name.
- // If no path is given, use the path of the MS (use . if no path).
- string name = msName;
- pos = name.rfind ('/');
- if (path.empty()) {
- if (pos == string::npos) {
- path = '.';
- } else {
- path = name.substr(0, pos);
- }
- }
- name = name.substr(pos+1);
- pos = name.find ('.');
- if (pos != string::npos) {
- name = name.substr(0, pos);
- }
- itsSaveName = path + '/' + name + '_' + suffix + ".flag";
- }
- }
-
- void FlagCounter::init (const DPInfo& info)
- {
- itsInfo = &info;
- itsBLCounts.resize (info.nbaselines());
- itsChanCounts.resize (info.nchan());
- itsCorrCounts.resize (info.ncorr());
- std::fill (itsBLCounts.begin(), itsBLCounts.end(), 0);
- std::fill (itsChanCounts.begin(),itsChanCounts.end(), 0);
- std::fill (itsCorrCounts.begin(),itsCorrCounts.end(), 0);
- }
-
- void FlagCounter::add (const FlagCounter& that)
- {
- // Add that to this after checking for equal sizes.
- ASSERT (itsBLCounts.size() == that.itsBLCounts.size());
- ASSERT (itsChanCounts.size() == that.itsChanCounts.size());
- ASSERT (itsCorrCounts.size() == that.itsCorrCounts.size());
- std::transform (itsBLCounts.begin(), itsBLCounts.end(),
- that.itsBLCounts.begin(), itsBLCounts.begin(),
- std::plus<int64>());
- std::transform (itsChanCounts.begin(), itsChanCounts.end(),
- that.itsChanCounts.begin(), itsChanCounts.begin(),
- std::plus<int64>());
- std::transform (itsCorrCounts.begin(), itsCorrCounts.end(),
- that.itsCorrCounts.begin(), itsCorrCounts.begin(),
- std::plus<int64>());
- }
-
- void FlagCounter::showBaseline (ostream& os, int64 ntimes) const
- {
- const Vector<Int>& ant1 = itsInfo->getAnt1();
- const Vector<Int>& ant2 = itsInfo->getAnt2();
- const Vector<String>& antNames = itsInfo->antennaNames();
- // Keep track of fully flagged baselines.
- std::vector<std::pair<int,int> > fullyFlagged;
- int64 npoints = ntimes * itsChanCounts.size();
- os << endl << "Percentage of visibilities flagged per baseline"
- " (antenna pair):";
- uint nrant = 1 + std::max(max(ant1), max(ant2));
- // Collect counts per baseline and antenna.
- Vector<int64> nusedAnt(nrant, 0);
- Vector<int64> countAnt(nrant, 0);
- Matrix<int64> nusedBL (nrant, nrant, 0);
- Matrix<int64> countBL (nrant, nrant, 0);
- for (uint i=0; i<itsBLCounts.size(); ++i) {
- countBL(ant1[i], ant2[i]) += itsBLCounts[i];
- nusedBL(ant1[i], ant2[i])++;
- countAnt[ant1[i]] += itsBLCounts[i];
- nusedAnt[ant1[i]]++;
- if (ant1[i] != ant2[i]) {
- countBL(ant2[i], ant1[i]) += itsBLCounts[i];
- nusedBL(ant2[i], ant1[i])++;
- countAnt[ant2[i]] += itsBLCounts[i];
- nusedAnt[ant2[i]]++;
- }
- }
- // Determine nr of antennae used.
- int nrused = 0;
- for (uint i=0; i<nrant; ++i) {
- if (nusedAnt[i] > 0) {
- nrused++;
- }
- }
- // Print 15 antennae per line.
- const int nantpl = 15;
- int nrl = (nrused + nantpl - 1) / nantpl;
- int ant = 0;
- // Loop over nr of lines needed for the antennae.
- for (int i=0; i<nrl; ++i) {
- int oldant = ant;
- // Determine nrant per line
- int nra = std::min(nantpl, nrused - i*nantpl);
- // Print the header for the antennae being used.
- // It also updates ant for the next iteration.
- os << endl << " ant";
- for (int j=0; j<nra;) {
- if (nusedAnt[ant] > 0) {
- os << std::setw(5) << ant;
- j++;
- }
- ant++;
- }
- os << endl;
- // Print the percentages per antenna pair.
- for (uint k=0; k<nrant; ++k) {
- if (nusedAnt[k] > 0) {
- os << std::setw(4) << k << " ";
- int ia = oldant;
- for (int j=0; j<nra;) {
- if (nusedAnt[ia] > 0) {
- if (nusedBL(k,ia) > 0) {
- os << std::setw(4)
- << int((100. * countBL(k,ia)) /
- (nusedBL(k,ia) * npoints) + 0.5)
- << '%';
- // Determine if baseline is fully flagged.
- // Do it only for ANT1<=ANT2
- if (int(k) <= ia) {
- if (countBL(k,ia) == nusedBL(k,ia) * npoints) {
- fullyFlagged.push_back (std::pair<int,int>(k,ia));
- }
- }
- } else {
- os << " ";
- }
- j++;
- }
- ia++;
- }
- os << endl;
- }
- }
- // Print the percentages per antenna.
- os << "TOTAL";
- int ia = oldant;
- for (int j=0; j<nra;) {
- if (nusedAnt[ia] > 0) {
- double perc = 100. * countAnt[ia] / (nusedAnt[ia] * npoints);
- os << std::setw(4) << int(perc + 0.5) << '%';
- j++;
- }
- ia++;
- }
- os << endl;
- }
- if (itsWarnPerc > 0) {
- for (uint i=0; i<nrant; ++i) {
- if (nusedAnt[i] > 0) {
- double perc = (100. * countAnt[i]) / (nusedAnt[i] * npoints);
- if (perc >= itsWarnPerc) {
- os << "** NOTE: ";
- showPerc1 (os, perc, 100);
- os << " of data are flagged for station " << i
- << " (" << antNames[i] << ')' << endl;
- }
- }
- }
- }
- if (itsShowFF) {
- os << "Fully flagged baselines: ";
- for (uint i=0; i<fullyFlagged.size(); ++i) {
- if (i>0) os << "; ";
- os << fullyFlagged[i].first << '&' << fullyFlagged[i].second;
- }
- os << endl;
- }
- if (! itsSaveName.empty()) {
- saveStation (npoints, nusedAnt, countAnt);
- }
- }
-
- void FlagCounter::showChannel (ostream& os, int64 ntimes) const
- {
- int64 npoints = ntimes * itsBLCounts.size();
- int64 nflagged = 0;
- os << endl << "Percentage of visibilities flagged per channel:" << endl;
- if (npoints == 0) {
- return;
- }
- // Print 10 channels per line.
- const int nchpl = 10;
- os << " channels ";
- for (int i=0; i<std::min(nchpl, int(itsChanCounts.size())); ++i) {
- os << std::setw(5) << i;
- }
- os << endl;
- int nrl = (itsChanCounts.size() + nchpl - 1) / nchpl;
- int ch = 0;
- for (int i=0; i<nrl; ++i) {
- int nrc = std::min(nchpl, int(itsChanCounts.size() - i*nchpl));
- os << std::setw(4) << ch << '-' << std::setw(4) << ch+nrc-1 << ": ";
- for (int j=0; j<nrc; ++j) {
- nflagged += itsChanCounts[ch];
- os << std::setw(4) << int((100. * itsChanCounts[ch]) / npoints + 0.5)
- << '%';
- ch++;
- }
- os << endl;
- }
- int64 totalnpoints = npoints * itsChanCounts.size();
- // Prevent division by zero
- if (totalnpoints == 0) {
- totalnpoints = 1;
- }
- os << "Total flagged: ";
- showPerc3 (os, nflagged, totalnpoints);
- os << " (" << nflagged << " out of " << totalnpoints
- << " visibilities)" << endl;
- if (itsWarnPerc > 0) {
- for (uint i=0; i<itsChanCounts.size(); ++i) {
- double perc = (100. * itsChanCounts[i]) / npoints;
- if (perc >= itsWarnPerc) {
- os << "** NOTE: ";
- showPerc1 (os, perc, 100);
- os << " of data are flagged for channel " << i << endl;
- }
- }
- }
- if (! itsSaveName.empty()) {
- saveChannel (npoints, itsChanCounts);
- }
- }
-
- void FlagCounter::showCorrelation (ostream& os, int64 ntimes) const
- {
- int64 ntotal = ntimes * itsBLCounts.size() * itsChanCounts.size();
- // Prevent division by zero
- if (ntotal == 0) {
- ntotal = 1;
- }
- os << endl
- << "Percentage of flagged visibilities detected per correlation:"
- << endl;
- os << " " << itsCorrCounts << " out of " << ntotal
- << " visibilities [";
- for (uint i=0; i<itsCorrCounts.size(); ++i) {
- if (i > 0) {
- os << ", ";
- }
- os << int(100. * itsCorrCounts[i] / ntotal + 0.5) << '%';
- }
- os << ']' << endl;
- }
-
- void FlagCounter::showPerc1 (ostream& os, double value, double total)
- {
- int perc = (total==0 ? 0 : int(1000. * value / total + 0.5));
- os << std::setw(3) << perc/10 << '.' << perc%10 << '%';
- }
-
- void FlagCounter::showPerc3 (ostream& os, double value, double total)
- {
- int perc = (total==0 ? 0 : int(100000. * value / total + 0.5));
- os << std::setw(5) << perc/1000 << '.';
- // It looks as if std::setfill keeps the fill character, so use
- // ios.fill to be able to reset it.
- char prev = os.fill ('0');
- os << std::setw(3) << perc%1000 << '%';
- os.fill (prev);
- }
-
- void FlagCounter::saveStation (int64 npoints, const Vector<int64>& nused,
- const Vector<int64>& count) const
- {
- // Create the table.
- TableDesc td;
- td.addColumn (ScalarColumnDesc<Int> ("Station"));
- td.addColumn (ScalarColumnDesc<String>("Name"));
- td.addColumn (ScalarColumnDesc<float> ("Percentage"));
- SetupNewTable newtab(itsSaveName+"stat", td, Table::New);
- Table tab(newtab);
- ScalarColumn<Int> statCol(tab, "Station");
- ScalarColumn<String> nameCol(tab, "Name");
- ScalarColumn<float> percCol(tab, "Percentage");
- const Vector<String>& antNames = itsInfo->antennaNames();
- // Write if an antenna is used.
- for (uint i=0; i<nused.size(); ++i) {
- if (nused[i] > 0) {
- int rownr = tab.nrow();
- tab.addRow();
- statCol.put (rownr, i);
- nameCol.put (rownr, antNames[i]);
- percCol.put (rownr, (100. * count[i]) / (nused[i] * npoints));
- }
- }
- }
-
- void FlagCounter::saveChannel (int64 npoints,
- const Vector<int64>& count) const
- {
- // Create the table.
- TableDesc td;
- td.addColumn (ScalarColumnDesc<double>("Frequency"));
- td.addColumn (ScalarColumnDesc<float> ("Percentage"));
- SetupNewTable newtab(itsSaveName+"freq", td, Table::New);
- Table tab(newtab);
- ScalarColumn<double> freqCol(tab, "Frequency");
- ScalarColumn<float> percCol(tab, "Percentage");
- // Get the channel frequencies.
- const Vector<double>& chanFreqs = itsInfo->chanFreqs();
- for (uint i=0; i<count.size(); ++i) {
- int rownr = tab.nrow();
- tab.addRow();
- freqCol.put (rownr, chanFreqs[i]);
- percCol.put (rownr, (100. * count[i]) / npoints);
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/GainCal.cc b/CEP/DP3/DPPP/src/GainCal.cc
deleted file mode 100644
index fa382903f8176a1e4e836173732d93b1ae2cf981..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/GainCal.cc
+++ /dev/null
@@ -1,1313 +0,0 @@
-//# GainCal.cc: DPPP step class to do a gain calibration
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/Package__Version.h>
-#include <DPPP/GainCal.h>
-#include <DPPP/Simulate.h>
-#include <DPPP/ApplyCal.h>
-#include <DPPP/PhaseFitter.h>
-#include <DPPP/CursorUtilCasa.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/SourceDBUtil.h>
-#include <DPPP/MSReader.h>
-#include <DPPP/DPLogger.h>
-#include <ParmDB/ParmDB.h>
-#include <ParmDB/ParmValue.h>
-#include <ParmDB/SourceDB.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <Common/LofarLogger.h>
-#include <Common/OpenMP.h>
-
-#include <fstream>
-#include <ctime>
-
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/MatrixMath.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/casa/OS/File.h>
-
-#include <vector>
-#include <algorithm>
-
-#include <limits>
-#include <iostream>
-#include <iomanip>
-
-using namespace casacore;
-using namespace LOFAR::BBS;
-
-namespace LOFAR {
- namespace DPPP {
-
- GainCal::GainCal (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsUseModelColumn(parset.getBool (prefix + "usemodelcolumn", false)),
- itsParmDBName (parset.getString (prefix + "parmdb", "")),
- itsUseH5Parm (itsParmDBName.find(".h5") != string::npos),
- itsDebugLevel (parset.getInt (prefix + "debuglevel", 0)),
- itsDetectStalling (parset.getBool (prefix + "detectstalling", true)),
- itsApplySolution (parset.getBool (prefix + "applysolution", false)),
- itsUVWFlagStep (input, parset, prefix),
- itsBaselineSelection (parset, prefix),
- itsMaxIter (parset.getInt (prefix + "maxiter", 50)),
- itsTolerance (parset.getDouble (prefix + "tolerance", 1.e-5)),
- itsPropagateSolutions
- (parset.getBool(prefix + "propagatesolutions", true)),
- itsSolInt (parset.getInt(prefix + "solint", 1)),
- itsNFreqCells (0),
- itsConverged (0),
- itsNonconverged (0),
- itsFailed (0),
- itsStalled (0),
- itsStepInParmUpdate (0),
- itsChunkStartTime(0),
- itsStepInSolInt (0),
- itsAllSolutions ()
- {
- stringstream ss;
- ss << parset;
- itsParsetString = ss.str();
-
- if (itsParmDBName=="") {
- itsParmDBName=parset.getString("msin")+"/instrument";
- }
-
- if (!itsUseH5Parm) {
- itsTimeSlotsPerParmUpdate = parset.getInt(prefix +
- "timeslotsperparmupdate",
- 500);
- } else {
- itsTimeSlotsPerParmUpdate = 0;
- }
-
- itsDataResultStep = ResultStep::ShPtr(new ResultStep());
- itsUVWFlagStep.setNextStep(itsDataResultStep);
-
- if (!itsUseModelColumn) {
- itsPredictStep=Predict(input, parset, prefix);
- itsResultStep = ResultStep::ShPtr(new ResultStep());
- itsPredictStep.setNextStep(itsResultStep);
- } else {
- itsApplyBeamToModelColumn=parset.getBool(prefix +
- "applybeamtomodelcolumn", false);
- if (itsApplyBeamToModelColumn) {
- itsApplyBeamStep=ApplyBeam(input, parset, prefix, true);
- ASSERT(!itsApplyBeamStep.invert());
- itsResultStep = ResultStep::ShPtr(new ResultStep());
- itsApplyBeamStep.setNextStep(itsResultStep);
- }
- }
-
- itsNIter.resize(4,0);
-
- if (itsApplySolution) {
- itsBuf.resize(itsSolInt);
- } else {
- itsBuf.resize(1);
- }
-
- string modestr = parset.getString (prefix + "caltype");
- itsMode = stringToCalType(modestr);
- uint defaultNChan = 0;
- if (itsMode == TECANDPHASE || itsMode == TEC) {
- defaultNChan = 1;
- }
- itsNChan = parset.getInt(prefix + "nchan", defaultNChan);
- ASSERT(itsMode!=TECSCREEN);
- }
-
- GainCal::~GainCal()
- {}
-
- GainCal::CalType GainCal::stringToCalType(const string &modestr) {
- if (modestr=="diagonal"||modestr=="complexgain") return COMPLEXGAIN;
- else if (modestr=="scalarcomplexgain") return SCALARCOMPLEXGAIN;
- else if (modestr=="fulljones") return FULLJONES;
- else if (modestr=="phaseonly") return PHASEONLY;
- else if (modestr=="scalarphase") return SCALARPHASE;
- else if (modestr=="amplitudeonly") return AMPLITUDEONLY;
- else if (modestr=="scalaramplitude") return SCALARAMPLITUDE;
- else if (modestr=="tecandphase") return TECANDPHASE;
- else if (modestr=="tec") return TEC;
- else if (modestr=="tecscreen") return TECSCREEN;
- else if (modestr=="rotation+diagonal") return ROTATIONANDDIAGONAL;
- else if (modestr=="rotation") return ROTATION;
- THROW(Exception, "Unknown mode: "<<modestr);
- }
-
- string GainCal::calTypeToString(GainCal::CalType caltype) {
- switch(caltype)
- {
- case COMPLEXGAIN: return "complexgain";
- case SCALARCOMPLEXGAIN: return "scalarcomplexgain";
- case FULLJONES: return "fulljones";
- case PHASEONLY: return "phaseonly";
- case SCALARPHASE: return "scalarphase";
- case AMPLITUDEONLY: return "amplitudeonly";
- case SCALARAMPLITUDE: return "scalaramplitude";
- case TECANDPHASE: return "tecandphase";
- case TEC: return "tec";
- case TECSCREEN: return "tecscreen";
- case ROTATION: return "rotation";
- case ROTATIONANDDIAGONAL: return "rotation+diagonal";
- default: THROW(Exception, "Unknown caltype: "<< caltype);
- }
- }
-
- void GainCal::setAntennaUsed() {
- Matrix<bool> selbl(itsBaselineSelection.apply (info()));
- uint nBl=info().getAnt1().size();
- itsAntennaUsed.resize(info().antennaNames().size());
- itsAntennaUsed=false;
- for (uint bl=0; bl<nBl; ++bl) {
- if (selbl(info().getAnt1()[bl], info().getAnt2()[bl])) {
- itsAntennaUsed[info().getAnt1()[bl]] = true;
- itsAntennaUsed[info().getAnt2()[bl]] = true;
- }
- }
- }
-
- void GainCal::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
-
- itsUVWFlagStep.updateInfo(infoIn);
-
- if (itsUseModelColumn) {
- if (itsApplyBeamToModelColumn) {
- itsApplyBeamStep.updateInfo(infoIn);
- }
- } else {
- itsPredictStep.updateInfo(infoIn);
- }
- if (itsApplySolution) {
- info().setWriteData();
- info().setWriteFlags();
- }
-
- if (itsSolInt==0) {
- itsSolInt=info().ntime();
- }
- if (itsTimeSlotsPerParmUpdate==0) {
- itsTimeSlotsPerParmUpdate = info().ntime();
- }
-
- if (itsNChan==0) {
- itsNChan = info().nchan();
- }
- if (itsNChan>info().nchan()) {
- itsNChan=info().nchan();
- }
- itsNFreqCells = info().nchan() / itsNChan;
- if (itsNChan*itsNFreqCells<info().nchan()) { // If last freq cell is smaller
- itsNFreqCells++;
- }
-
- itsSols.reserve(itsTimeSlotsPerParmUpdate);
-
- itsSelectedBL = itsBaselineSelection.applyVec(info());
- setAntennaUsed();
-
- // Compute average frequency for every freqcell
- itsFreqData.resize(itsNFreqCells);
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- double meanfreq=0;
- uint chmin=itsNChan*freqCell;
- uint chmax=min(info().nchan(), chmin+itsNChan);
-
- meanfreq = std::accumulate(info().chanFreqs().data()+chmin,
- info().chanFreqs().data()+chmax, 0.0);
-
- itsFreqData[freqCell] = meanfreq / (chmax-chmin);
- }
-
- // Initialize phase fitters, set their frequency data
- if (itsMode==TECANDPHASE || itsMode==TEC) {
- itsTECSols.reserve(itsTimeSlotsPerParmUpdate);
-
- itsPhaseFitters.reserve(itsNFreqCells); // TODO: could be numthreads instead
-
- uint nSt=info().antennaUsed().size();
- for (uint st=0; st<nSt; ++st) {
- itsPhaseFitters.push_back(CountedPtr<PhaseFitter>(new PhaseFitter(itsNFreqCells)));
- double* nu = itsPhaseFitters[st]->FrequencyData();
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- nu[freqCell] = itsFreqData[freqCell];
- }
- }
- }
-
- iS.reserve(itsNFreqCells);
- uint chMax = itsNChan;
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- if ((freqCell+1)*itsNChan>info().nchan()) { // Last cell can be smaller
- chMax-=((freqCell+1)*itsNChan)%info().nchan();
- }
-
- StefCal::StefCalMode smode;
- switch (itsMode)
- {
- case COMPLEXGAIN: smode = StefCal::DEFAULT; break;
- case FULLJONES: smode = StefCal::FULLJONES; break;
- case SCALARPHASE:
- case PHASEONLY:
- case TEC:
- case TECANDPHASE: smode = StefCal::PHASEONLY; break;
- case AMPLITUDEONLY:
- case SCALARAMPLITUDE: smode = StefCal::AMPLITUDEONLY; break;
- default: THROW(Exception, "Unhandled mode");
- }
-
- iS.push_back(StefCal(itsSolInt, chMax, smode, scalarMode(itsMode),
- itsTolerance, info().antennaUsed().size(),
- itsDetectStalling, itsDebugLevel));
- }
-
- itsFlagCounter.init(getInfo());
-
- itsChunkStartTime = info().startTime();
-
- if (itsDebugLevel>0) {
- ASSERT(OpenMP::maxThreads()==1);
- ASSERT(itsTimeSlotsPerParmUpdate >= info().ntime());
- itsAllSolutions.resize(IPosition(6,
- iS[0].numCorrelations(),
- info().antennaUsed().size(),
- (itsMode==TEC||itsMode==TECANDPHASE?2:1),
- itsNFreqCells,
- itsMaxIter,
- info().ntime()
- ));
- }
- }
-
- void GainCal::show (std::ostream& os) const
- {
- os << "GainCal " << itsName << endl;
- if (itsUseH5Parm) {
- os << " H5Parm: " << itsParmDBName;
- } else {
- os << " parmdb: " << itsParmDBName;
- if (Table::isReadable(itsParmDBName)) {
- os << " (existing)";
- } else {
- os << " (will be created)";
- }
- }
- os << endl;
- os << " solint: " << itsSolInt <<endl;
- os << " nchan: " << itsNChan <<endl;
- os << " max iter: " << itsMaxIter << endl;
- os << " tolerance: " << itsTolerance << endl;
- os << " caltype: " << calTypeToString(itsMode) << endl;
- os << " apply solution: " << boolalpha << itsApplySolution << endl;
- os << " propagate solutions: " << boolalpha << itsPropagateSolutions << endl;
- if (!itsUseH5Parm) {
- os << " timeslotsperparmupdate: " << itsTimeSlotsPerParmUpdate << endl;
- }
- os << " detect stalling: " << boolalpha << itsDetectStalling << endl;
- os << " use model column: " << boolalpha << itsUseModelColumn << endl;
- itsBaselineSelection.show (os);
- if (!itsUseModelColumn) {
- itsPredictStep.show(os);
- } else if (itsApplyBeamToModelColumn) {
- itsApplyBeamStep.show(os);
- }
- itsUVWFlagStep.show(os);
- }
-
- void GainCal::showTimings (std::ostream& os, double duration) const
- {
- double totaltime=itsTimer.getElapsed();
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " GainCal " << itsName << endl;
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerPredict.getElapsed(), totaltime);
- os << " of it spent in predict" << endl;
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerFill.getElapsed(), totaltime);
- os << " of it spent in reordering visibility data" << endl;
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerSolve.getElapsed(), totaltime);
- os << " of it spent in estimating gains and computing residuals" << endl;
-
- if (itsMode == TEC || itsMode == TECANDPHASE) {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerPhaseFit.getElapsed(), totaltime);
- os << " of it spent in fitting phases" << endl;
- }
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerWrite.getElapsed(), totaltime);
- os << " of it spent in writing gain solutions to disk" << endl;
-
- os << " ";
- os <<"Converged: "<<itsConverged<<", stalled: "<<itsStalled<<", non converged: "<<itsNonconverged<<", failed: "<<itsFailed<<endl;
- os << " ";
- os <<"Iters converged: " << (itsConverged==0?0:itsNIter[0]/itsConverged);
- os << ", stalled: "<< (itsStalled ==0?0:itsNIter[1]/itsStalled);
- os << ", non converged: "<<(itsNonconverged==0?0:itsNIter[2]/itsNonconverged);
- os << ", failed: "<<(itsFailed==0?0:itsNIter[3]/itsFailed)<<endl;
- }
-
- bool GainCal::process (const DPBuffer& bufin)
- {
- itsTimer.start();
-
- uint bufIndex=0;
-
- if (itsApplySolution) {
- // Need to keep a copy of all solint buffers in this step
- bufIndex=itsStepInSolInt;
- itsBuf[bufIndex].copy(bufin);
- } else {
- // We'll read the necessary info from the buffer and pass it on
- itsBuf[bufIndex].referenceFilled (bufin);
- }
- itsInput->fetchUVW(bufin, itsBuf[bufIndex], itsTimer);
- itsInput->fetchWeights(bufin, itsBuf[bufIndex], itsTimer);
- itsInput->fetchFullResFlags(bufin, itsBuf[bufIndex], itsTimer);
-
- // UVW flagging happens on a copy of the buffer, so these flags are not written
- itsUVWFlagStep.process(itsBuf[bufIndex]);
-
- Cube<Complex> dataCube=itsBuf[bufIndex].getData();
- Complex* data=dataCube.data();
- float* weight = itsBuf[bufIndex].getWeights().data();
- const Bool* flag=itsBuf[bufIndex].getFlags().data();
-
- // Simulate.
- //
- // Model visibilities for each direction of interest will be computed
- // and stored.
-
- itsTimerPredict.start();
-
- if (itsUseModelColumn) {
- itsInput->getModelData (itsBuf[bufIndex].getRowNrs(), itsModelData);
- if (itsApplyBeamToModelColumn) { // TODO: double check this
- // Temporarily put model data in data column for applybeam step
- // ApplyBeam step will copy the buffer so no harm is done
- itsBuf[bufIndex].getData()=itsModelData;
- itsApplyBeamStep.process(itsBuf[bufIndex]);
- //Put original data back in data column
- itsBuf[bufIndex].getData()=dataCube;
- }
- } else { // Predict
- itsPredictStep.process(itsBuf[bufIndex]);
- }
-
- itsTimerPredict.stop();
-
- itsTimerFill.start();
-
- if (itsStepInSolInt==0) {
- // Start new solution interval
-
- for (uint freqCell=0; freqCell<itsNFreqCells; freqCell++) {
- iS[freqCell].clearStationFlagged();
- iS[freqCell].resetVis();
- }
- }
-
- // Store data in the stefcal object
- if (itsUseModelColumn && !itsApplyBeamToModelColumn) {
- fillMatrices(itsModelData.data(),data,weight,flag);
- } else {
- fillMatrices(itsResultStep->get().getData().data(),data,weight,flag);
- }
- itsTimerFill.stop();
-
- if (itsStepInSolInt==itsSolInt-1) {
- // Solve past solution interval
- stefcal();
- itsStepInParmUpdate++;
-
- if (itsApplySolution) {
- Cube<DComplex> invsol = invertSol(itsSols.back());
- for (uint stepInSolInt=0; stepInSolInt<itsSolInt; stepInSolInt++) {
- applySolution(itsBuf[stepInSolInt], invsol);
- getNextStep()->process(itsBuf[stepInSolInt]);
- }
- }
-
- itsStepInSolInt=0;
- } else {
- itsStepInSolInt++;
- }
-
- itsTimer.stop();
-
- if (!itsUseH5Parm && (itsStepInParmUpdate == itsTimeSlotsPerParmUpdate)) {
- writeSolutionsParmDB(itsChunkStartTime);
- itsChunkStartTime += itsSolInt * itsTimeSlotsPerParmUpdate * info().timeInterval();
- itsSols.clear();
- itsTECSols.clear();
- itsStepInParmUpdate = 0;
- }
-
- if (!itsApplySolution) {
- getNextStep()->process(itsBuf[bufIndex]);
- }
- return false;
- }
-
- Cube<DComplex> GainCal::invertSol(const Cube<DComplex>& sol) {
- Cube<DComplex> invsol = sol.copy();
- uint nCr = invsol.shape()[0];
-
- // Invert copy of solutions
- uint nSt = invsol.shape()[1];
- for (uint st=0; st<nSt; ++st) {
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- if (nCr==4) {
- ApplyCal::invert(&invsol(0,st,freqCell));
- } else {
- for (uint cr=0; cr<nCr; ++cr) {
- invsol(cr, st, freqCell) = 1./invsol(cr, st, freqCell);
- }
- }
- }
- }
-
- return invsol;
- }
-
- void GainCal::applySolution(DPBuffer& buf, const Cube<DComplex>& invsol) {
- uint nbl = buf.getData().shape()[2];
- Complex* data = buf.getData().data();
- float* weight = buf.getWeights().data(); // Not initialized yet
- bool* flag = buf.getFlags().data();
- uint nchan = buf.getData().shape()[1];
-
- uint nCr = invsol.shape()[0];
-
- for (size_t bl=0; bl<nbl; ++bl) {
- for (size_t chan=0;chan<nchan;chan++) {
- uint antA = info().antennaMap()[info().getAnt1()[bl]];
- uint antB = info().antennaMap()[info().getAnt2()[bl]];
- uint freqCell = chan / itsNChan;
- if (nCr>2) {
- ApplyCal::applyFull( &invsol(0, antA, freqCell),
- &invsol(0, antB, freqCell),
- &data[bl * 4 * nchan + chan * 4 ],
- &weight[bl * 4 * nchan + chan * 4 ], // Not passing weights, any pointer should do
- &flag[ bl * 4 * nchan + chan * 4 ],
- bl, chan, false, itsFlagCounter); // Update weights is disabled here
- }
- else if (scalarMode(itsMode)) {
- ApplyCal::applyScalar( &invsol(0, antA, freqCell),
- &invsol(0, antB, freqCell),
- &data[bl * 4 * nchan + chan * 4 ],
- &weight[bl * 4 * nchan + chan * 4 ], // Not passing weights, any pointer should do
- &flag[ bl * 4 * nchan + chan * 4 ],
- bl, chan, false, itsFlagCounter); // Update weights is disabled here
- } else {
- ApplyCal::applyDiag( &invsol(0, antA, freqCell),
- &invsol(0, antB, freqCell),
- &data[bl * 4 * nchan + chan * 4 ],
- &weight[bl * 4 * nchan + chan * 4 ], // Not passing weights, any pointer should do
- &flag[ bl * 4 * nchan + chan * 4 ],
- bl, chan, false, itsFlagCounter); // Update weights is disabled here
- }
- }
- }
- }
-
- // Fills itsVis and itsMVis as matrices with all 00 polarizations in the
- // top left, all 11 polarizations in the bottom right, etc.
- // For TEC fitting, it also sets weights for the frequency cells
- void GainCal::fillMatrices (casacore::Complex* model, casacore::Complex* data, float* weight,
- const casacore::Bool* flag) {
- const size_t nBl = info().nbaselines();
- const size_t nCh = info().nchan();
- const size_t nCr = info().ncorr();
- ASSERT(nCr==4 || nCr==2 || nCr==1);
-
- for (uint ch=0;ch<nCh;++ch) {
- for (uint bl=0;bl<nBl;++bl) {
- if (itsSelectedBL[bl]) {
- int ant1=info().antennaMap()[info().getAnt1()[bl]];
- int ant2=info().antennaMap()[info().getAnt2()[bl]];
- DBGASSERT(ant1>=0 && ant2>=0);
- if (ant1==ant2 ||
- iS[ch/itsNChan].getStationFlagged()[ant1] ||
- iS[ch/itsNChan].getStationFlagged()[ant2] ||
- flag[bl*nCr*nCh+ch*nCr]) { // Only check flag of cr==0
- continue;
- }
-
- if (itsMode==TEC || itsMode==TECANDPHASE) {
- iS[ch/itsNChan].incrementWeight(weight[bl*nCr*nCh+ch*nCr]);
- }
-
- for (uint cr=0;cr<nCr;++cr) {
- // The nCrDiv is there such that for nCr==2 the visibilities end up at (0,0) for cr==0, (1,1) for cr==1
- uint nCrDiv = (nCr==4?2:1);
- iS[ch/itsNChan].getVis() (IPosition(6,ant1,cr/nCrDiv,itsStepInSolInt,ch%itsNChan,cr%2,ant2)) =
- DComplex(data [bl*nCr*nCh+ch*nCr+cr]) *
- DComplex(sqrt(weight[bl*nCr*nCh+ch*nCr+cr]));
- iS[ch/itsNChan].getMVis()(IPosition(6,ant1,cr/nCrDiv,itsStepInSolInt,ch%itsNChan,cr%2,ant2)) =
- DComplex(model[bl*nCr*nCh+ch*nCr+cr]) *
- DComplex(sqrt(weight[bl*nCr*nCh+ch*nCr+cr]));
-
- // conjugate transpose
- iS[ch/itsNChan].getVis() (IPosition(6,ant2,cr%2,itsStepInSolInt,ch%itsNChan,cr/nCrDiv,ant1)) =
- DComplex(conj(data [bl*nCr*nCh+ch*nCr+cr])) *
- DComplex(sqrt(weight[bl*nCr*nCh+ch*nCr+cr]));
- iS[ch/itsNChan].getMVis()(IPosition(6,ant2,cr%2,itsStepInSolInt,ch%itsNChan,cr/nCrDiv,ant1)) =
- DComplex(conj(model[bl*nCr*nCh+ch*nCr+cr] )) *
- DComplex(sqrt(weight[bl*nCr*nCh+ch*nCr+cr]));
- }
- }
- }
- }
- }
-
- bool GainCal::scalarMode(CalType caltype) {
- return (caltype==TECANDPHASE || caltype==TEC || caltype==SCALARPHASE ||
- caltype==SCALARAMPLITUDE);
- }
-
- bool GainCal::diagonalMode(CalType caltype) {
- return (caltype==COMPLEXGAIN || caltype==PHASEONLY ||
- caltype==AMPLITUDEONLY);
- }
-
- void GainCal::stefcal () {
- itsTimerSolve.start();
-
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- if (itsPropagateSolutions) {
- iS[freqCell].init(false);
- } else {
- iS[freqCell].init(true);
- }
- }
-
- uint iter=0;
-
- casacore::Matrix<double> tecsol(itsMode==TECANDPHASE?2:1,
- info().antennaUsed().size(), 0);
-
- vector<StefCal::Status> converged(itsNFreqCells,StefCal::NOTCONVERGED);
- for (;iter<itsMaxIter;++iter) {
- bool allConverged=true;
-#pragma omp parallel for
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- // Do another step when stalled and not all converged
- if (converged[freqCell]==StefCal::CONVERGED) {
- continue;
- }
- converged[freqCell] = iS[freqCell].doStep(iter);
- // Only continue if there are steps worth continuing
- // (so not converged, failed or stalled)
- if (converged[freqCell]==StefCal::NOTCONVERGED) {
- allConverged = false;
- }
- }
-
- if (itsDebugLevel>0) {
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- Matrix<DComplex> fullSolution = iS[freqCell].getSolution(false);
- std::copy(fullSolution.begin(),
- fullSolution.end(),
- &(itsAllSolutions(IPosition(6, 0,
- 0,
- 0,
- freqCell,
- iter,
- itsStepInParmUpdate
- ))));
- }
- }
-
- if (itsMode==TEC || itsMode==TECANDPHASE) {
- itsTimerSolve.stop();
- itsTimerPhaseFit.start();
- casacore::Matrix<casacore::DComplex> sols_f(itsNFreqCells, info().antennaUsed().size());
-
- uint nSt = info().antennaUsed().size();
-
- // TODO: set phase reference so something smarter that station 0
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- casacore::Matrix<casacore::DComplex> sol = iS[freqCell].getSolution(false);
- if (iS[freqCell].getStationFlagged()[0]) {
- // If reference station flagged, flag whole channel
- for (uint st=0; st<info().antennaUsed().size(); ++st) {
- iS[freqCell].getStationFlagged()[st] = true;
- }
- } else {
- for (uint st=0; st<info().antennaUsed().size(); ++st) {
- sols_f(freqCell, st) = sol(st, 0)/sol(0, 0);
- ASSERT(isFinite(sols_f(freqCell, st)));
- }
- }
- }
-
-#pragma omp parallel for
- for (uint st=0; st<nSt; ++st) {
- uint numpoints=0;
- double* phases = itsPhaseFitters[st]->PhaseData();
- double* weights = itsPhaseFitters[st]->WeightData();
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- if (iS[freqCell].getStationFlagged()[st%nSt] ||
- converged[freqCell]==StefCal::FAILED) {
- phases[freqCell] = 0;
- weights[freqCell] = 0;
- } else {
- phases[freqCell] = arg(sols_f(freqCell, st));
- if (!isFinite(phases[freqCell])) {
- cout<<"Yuk, phases[freqCell]="<<phases[freqCell]<<", sols_f(freqCell, st)="<<sols_f(freqCell, st)<<endl;
- ASSERT(isFinite(phases[freqCell]));
- }
- ASSERT(iS[freqCell].getWeight()>0);
- weights[freqCell] = iS[freqCell].getWeight();
- numpoints++;
- }
- }
-
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- ASSERT(isFinite(phases[freqCell]));
- }
-
- if (numpoints>1) { // TODO: limit should be higher
- //cout<<"tecsol(0,"<<st<<")="<<tecsol(0,st)<<", tecsol(1,"<<st<<")="<<tecsol(1,st)<<endl;
- if (itsMode==TECANDPHASE) {
- itsPhaseFitters[st]->FitDataToTEC2Model(tecsol(0, st), tecsol(1,st));
- } else { // itsMode==TEC
- itsPhaseFitters[st]->FitDataToTEC1Model(tecsol(0, st));
- }
- // Update solution in stefcal
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- ASSERT(isFinite(phases[freqCell]));
- iS[freqCell].getSolution(false)(st, 0) = polar(1., phases[freqCell]);
- }
- } else {
- tecsol(0, st) = 0; //std::numeric_limits<double>::quiet_NaN();
- if (itsMode==TECANDPHASE) {
- tecsol(1, st) = 0; //std::numeric_limits<double>::quiet_NaN();
- }
- }
-
- if (itsDebugLevel>0) {
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- Matrix<DComplex> fullSolution = iS[freqCell].getSolution(false);
- std::copy(fullSolution.begin(),
- fullSolution.end(),
- &(itsAllSolutions(IPosition(6, 0,
- 0,
- 1,
- freqCell,
- iter,
- itsStepInParmUpdate
- ))));
- }
- }
- }
- itsTimerPhaseFit.stop();
- itsTimerSolve.start();
- }
-
- if (allConverged) {
- break;
- }
-
- } // End niter
-
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- switch (converged[freqCell]) {
- case StefCal::CONVERGED: {itsConverged++; itsNIter[0]+=iter; break;}
- case StefCal::STALLED: {itsStalled++; itsNIter[1]+=iter; break;}
- case StefCal::NOTCONVERGED: {itsNonconverged++; itsNIter[2]+=iter; break;}
- case StefCal::FAILED: {itsFailed++; itsNIter[3]+=iter; break;}
- default:
- THROW(Exception, "Unknown converged status");
- }
- }
-
- // Stefcal terminated (either by maxiter or by converging)
-
- Cube<DComplex> sol(iS[0].numCorrelations(), info().antennaUsed().size(), itsNFreqCells);
-
- uint transpose[2][4] = { { 0, 1, 0, 0 }, { 0, 2, 1, 3 } };
-
- uint nSt = info().antennaUsed().size();
-
- for (uint freqCell=0; freqCell<itsNFreqCells; ++freqCell) {
- casacore::Matrix<casacore::DComplex> tmpsol = iS[freqCell].getSolution(true);
-
- for (uint st=0; st<nSt; st++) {
- for (uint cr=0; cr<iS[0].nCr(); ++cr) {
- uint crt=transpose[iS[0].numCorrelations()/4][cr]; // Conjugate transpose ! (only for numCorrelations = 4)
- sol(crt, st, freqCell) = conj(tmpsol(st, cr)); // Conjugate transpose
- if (itsMode==COMPLEXGAIN || itsMode==PHASEONLY || itsMode==AMPLITUDEONLY) {
- sol(crt+1, st, freqCell) = conj(tmpsol(st+nSt,cr)); // Conjugate transpose
- }
- }
- }
- }
- itsSols.push_back(sol);
- if (itsMode==TEC || itsMode==TECANDPHASE) {
- itsTECSols.push_back(tecsol);
- }
-
- itsTimerSolve.stop();
- } // End stefcal()
-
-
- void GainCal::initParmDB() {
- itsParmDB = boost::shared_ptr<BBS::ParmDB>
- (new BBS::ParmDB(BBS::ParmDBMeta("casa", itsParmDBName),
- false));
- itsParmDB->lock();
- // Store the (freq, time) resolution of the solutions.
-
- double freqWidth = getInfo().chanWidths()[0];
- if (getInfo().chanFreqs().size()>1) { // Handle data with evenly spaced gaps between channels
- freqWidth = info().chanFreqs()[1]-info().chanFreqs()[0];
- }
-
- vector<double> resolution(2);
- resolution[0] = freqWidth * itsNChan;
- resolution[1] = info().timeInterval() * itsSolInt;
- itsParmDB->setDefaultSteps(resolution);
-
- string parmname=parmName()+"*";
-
- if (!itsParmDB->getNames(parmname).empty()) {
- DPLOG_WARN_STR ("Solutions for "<<parmname<<" already in "<<itsParmDBName
- <<", these are removed");
- // Specify entire domain of this MS; only to specify that the existing
- // values should be deleted for this domain
- BBS::Axis::ShPtr tdomAxis(
- new BBS::RegularAxis(
- info().startTime(),
- info().ntime() * info().timeInterval(),
- 1));
- BBS::Axis::ShPtr fdomAxis(
- new BBS::RegularAxis(
- info().chanFreqs()[0] - freqWidth * 0.5,
- freqWidth * getInfo().chanFreqs().size(), 1));
-
- itsParmDB->deleteValues(parmname, BBS::Box(
- fdomAxis->start(), tdomAxis->start(),
- fdomAxis->end(), tdomAxis->end(), true));
- }
-
- // Write out default values, if they don't exist yet
- ParmMap parmset;
-
- // Write out default amplitudes
- if (itsMode==PHASEONLY || itsMode==SCALARPHASE) {
- itsParmDB->getDefValues(parmset, "Gain:0:0:Ampl");
- if (parmset.empty()) {
- ParmValueSet pvset(ParmValue(1.0));
- itsParmDB->putDefValue("Gain:0:0:Ampl",pvset);
- itsParmDB->putDefValue("Gain:1:1:Ampl",pvset);
- }
- }
-
- // Write out default phases
- if (itsMode==AMPLITUDEONLY || itsMode==SCALARAMPLITUDE) {
- itsParmDB->getDefValues(parmset, "Gain:0:0:Phase");
- if (parmset.empty()) {
- ParmValueSet pvset(ParmValue(0.0));
- itsParmDB->putDefValue("Gain:0:0:Phase",pvset);
- itsParmDB->putDefValue("Gain:1:1:Phase",pvset);
- }
- }
-
- // Write out default gains
- if (itsMode==COMPLEXGAIN || itsMode==FULLJONES) {
- itsParmDB->getDefValues(parmset, "Gain:0:0:Real");
- if (parmset.empty()) {
- ParmValueSet pvset(ParmValue(1.0));
- itsParmDB->putDefValue("Gain:0:0:Real",pvset);
- itsParmDB->putDefValue("Gain:1:1:Real",pvset);
- }
- }
- }
-
- string GainCal::parmName() {
- string name;
- if (itsMode==SCALARPHASE) {
- name=string("CommonScalarPhase:");
- } else if (itsMode==SCALARAMPLITUDE) {
- name=string("CommonScalarAmplitude:");
- } else if (itsMode==TEC || itsMode==TECANDPHASE) {
- name=string("TEC:");
- }
- else {
- name=string("Gain:");
- }
-
- return name;
- }
-
- void GainCal::writeSolutionsH5Parm(double) {
- itsTimer.start();
- itsTimerWrite.start();
-
- H5Parm h5parm(itsParmDBName, true);
-
- // Fill antenna info in H5Parm, need to convert from casa types to std types
- std::vector<std::string> allAntennaNames(info().antennaNames().size());
- std::vector<std::vector<double> > antennaPos(info().antennaPos().size());
- for (uint i=0; i<info().antennaNames().size(); ++i) {
- allAntennaNames[i]=info().antennaNames()[i];
- casacore::Quantum<casacore::Vector<double> > pos = info().antennaPos()[i].get("m");
- antennaPos[i].resize(3);
- antennaPos[i][0] = pos.getValue()[0];
- antennaPos[i][1] = pos.getValue()[1];
- antennaPos[i][2] = pos.getValue()[2];
- }
-
- h5parm.addAntennas(allAntennaNames, antennaPos);
-
- vector<pair<double, double> > pointingPosition(1);
- MDirection phasecenter = info().phaseCenter();
- pointingPosition[0].first = phasecenter.getValue().get()[0];
- pointingPosition[0].second = phasecenter.getValue().get()[1];
- vector<string> pointingName(1, "POINTING");
-
- h5parm.addSources(pointingName, pointingPosition);
-
- uint nPol;
- vector<string> polarizations;
- if (scalarMode(itsMode)) {
- nPol = 1;
- } else if (diagonalMode(itsMode)) {
- nPol = 2;
- polarizations.push_back("XX");
- polarizations.push_back("YY");
- } else {
- ASSERT(itsMode==FULLJONES);
- polarizations.push_back("XX");
- polarizations.push_back("XY");
- polarizations.push_back("YX");
- polarizations.push_back("YY");
- nPol = 4;
- }
-
- // Construct time axis
- uint nSolTimes = (info().ntime()+itsSolInt-1)/itsSolInt;
- vector<double> solTimes(nSolTimes);
- ASSERT(nSolTimes==itsSols.size());
- double starttime=info().startTime();
- for (uint t=0; t<nSolTimes; ++t) {
- solTimes[t] = starttime+(t+0.5)*info().timeInterval()*itsSolInt;
- }
-
- // Construct frequency axis
- uint nSolFreqs;
- if (itsMode==TEC || itsMode==TECANDPHASE) {
- nSolFreqs = 1;
- } else {
- nSolFreqs = itsNFreqCells;
- }
-
- vector<H5Parm::AxisInfo> axes;
- axes.push_back(H5Parm::AxisInfo("time", itsSols.size()));
- axes.push_back(H5Parm::AxisInfo("freq", nSolFreqs));
- axes.push_back(H5Parm::AxisInfo("ant", info().antennaUsed().size()));
- if (nPol>1) {
- axes.push_back(H5Parm::AxisInfo("pol", nPol));
- }
-
- vector<H5Parm::SolTab> soltabs = makeSolTab(h5parm, itsMode, axes);
-
- std::vector<std::string> antennaUsedNames;
- for (uint st = 0; st<info().antennaUsed().size(); ++st) {
- antennaUsedNames.push_back(info().antennaNames()[info().antennaUsed()[st]]);
- }
-
- vector<H5Parm::SolTab>::iterator soltabiter = soltabs.begin();
- for (; soltabiter != soltabs.end(); ++soltabiter) {
- (*soltabiter).setAntennas(antennaUsedNames);
- if (nPol>1) {
- (*soltabiter).setPolarizations(polarizations);
- }
- if (itsMode==TEC || itsMode==TECANDPHASE) {
- // Set channel to frequency of middle channel
- // TODO: fix this for nchan
- vector<double> oneFreq(1);
- oneFreq[0] = info().chanFreqs()[info().nchan()/2];
- (*soltabiter).setFreqs(oneFreq);
- } else {
- (*soltabiter).setFreqs(itsFreqData);
- }
- (*soltabiter).setTimes(solTimes);
- }
-
- // Put solutions in a contiguous piece of memory
- string historyString = "CREATE by DPPP\n" +
- Version::getInfo<DPPPVersion>("DPPP", "top") + "\n" +
- "step " + itsName + " in parset: \n" + itsParsetString;
-
- if (itsMode==TEC || itsMode==TECANDPHASE) {
- vector<double> tecsols(nSolFreqs*antennaUsedNames.size()*nSolTimes*nPol);
- vector<double> weights(nSolFreqs*antennaUsedNames.size()*nSolTimes*nPol, 1.);
- vector<double> phasesols;
- if (itsMode==TECANDPHASE) {
- phasesols.resize(nSolFreqs*antennaUsedNames.size()*nSolTimes*nPol);
- }
- size_t i=0;
- for (uint time=0; time<nSolTimes; ++time) {
- for (uint freqCell=0; freqCell<nSolFreqs; ++freqCell) {
- for (uint ant=0; ant<info().antennaUsed().size(); ++ant) {
- for (uint pol=0; pol<nPol; ++pol) {
- ASSERT(!itsTECSols[time].empty());
- tecsols[i] = itsTECSols[time](0, ant) / 8.44797245e9;
- if (!std::isfinite(tecsols[i])) {
- weights[i] = 0.;
- }
- if (itsMode==TECANDPHASE) {
- phasesols[i] = -itsTECSols[time](0, ant);
- }
- ++i;
- }
- }
- }
- }
- soltabs[0].setValues(tecsols, weights, historyString);
- if (itsMode==TECANDPHASE) {
- soltabs[1].setValues(phasesols, weights, historyString);
- }
- } else {
- vector<DComplex> sols(nSolFreqs*antennaUsedNames.size()*nSolTimes*nPol);
- vector<double> weights(nSolFreqs*antennaUsedNames.size()*nSolTimes*nPol, 1.);
- size_t i=0;
- for (uint time=0; time<nSolTimes; ++time) {
- for (uint freqCell=0; freqCell<nSolFreqs; ++freqCell) {
- for (uint ant=0; ant<info().antennaUsed().size(); ++ant) {
- for (uint pol=0; pol<nPol; ++pol) {
- ASSERT(!itsSols[time].empty());
- sols[i] = itsSols[time](pol, ant, freqCell);
- if (!std::isfinite(sols[i].real())) {
- weights[i] = 0.;
- }
- ++i;
- }
- }
- }
- }
-
- if (itsMode!=AMPLITUDEONLY) {
- soltabs[0].setComplexValues(sols, weights, false, historyString);
- } else {
- soltabs[0].setComplexValues(sols, weights, true, historyString);
- }
- if (soltabs.size()>1) {
- // Also write amplitudes
- soltabs[1].setComplexValues(sols, weights, true, historyString);
- }
- }
-
- itsTimerWrite.stop();
- itsTimer.stop();
- }
-
- vector<H5Parm::SolTab> GainCal::makeSolTab(H5Parm& h5parm, CalType caltype,
- vector<H5Parm::AxisInfo>& axes) {
- uint numsols = 1;
- // For [scalar]complexgain, store two soltabs: phase and amplitude
- if (caltype == GainCal::COMPLEXGAIN ||
- caltype == GainCal::SCALARCOMPLEXGAIN ||
- caltype == GainCal::TECANDPHASE ||
- caltype == GainCal::FULLJONES) {
- numsols = 2;
- }
- vector<H5Parm::SolTab> soltabs;
- for (uint solnum=0; solnum<numsols; ++solnum) {
- string solTabName;
- H5Parm::SolTab soltab;
- switch (caltype) {
- case GainCal::SCALARPHASE:
- case GainCal::PHASEONLY:
- solTabName = "phase000";
- soltab = h5parm.createSolTab(solTabName, "phase", axes);
- break;
- case GainCal::SCALARCOMPLEXGAIN:
- case GainCal::COMPLEXGAIN:
- case GainCal::FULLJONES:
- if (solnum==0) {
- solTabName = "phase000";
- soltab = h5parm.createSolTab(solTabName, "phase", axes);
- } else {
- solTabName = "amplitude000";
- soltab = h5parm.createSolTab(solTabName, "amplitude", axes);
- }
- break;
- case GainCal::SCALARAMPLITUDE:
- case GainCal::AMPLITUDEONLY:
- solTabName = "amplitude000";
- soltab = h5parm.createSolTab(solTabName, "amplitude", axes);
- break;
- case GainCal::TEC:
- case GainCal::TECANDPHASE:
- if (solnum==0) {
- solTabName = "tec000";
- soltab = h5parm.createSolTab(solTabName, "tec", axes);
- } else {
- solTabName = "phase000";
- soltab = h5parm.createSolTab(solTabName, "phase", axes);
- }
- break;
- default:
- THROW(Exception, "Unhandled mode in writing H5Parm output: "<<calTypeToString(caltype));
- }
- soltabs.push_back(soltab);
- }
- return soltabs;
- }
-
- void GainCal::writeSolutionsParmDB(double startTime) {
- itsTimer.start();
- itsTimerWrite.start();
-
- // Open the ParmDB at the first write.
- // In that way the instrumentmodel ParmDB can be in the MS directory.
- if (!itsParmDB) {
- initParmDB();
- } // End initialization of parmdb
-
- uint ntime=itsSols.size();
- uint nchan, nfreqs;
- if (itsMode==TEC || itsMode==TECANDPHASE) {
- nfreqs = 1;
- nchan = info().nchan();
- } else {
- nfreqs = itsNFreqCells;
- nchan = itsNChan;
- }
-
- // Construct solution grid for the current chunk
- double freqWidth = getInfo().chanWidths()[0];
- if (getInfo().chanFreqs().size()>1) { // Handle data with evenly spaced gaps between channels
- freqWidth = info().chanFreqs()[1]-info().chanFreqs()[0];
- }
-
- // Get end time of the current chunk. For the last chunk, this
- // is chopped off at the end of the MS (only if solint > 1)
- double endTime = min(startTime + ntime * info().timeInterval() * itsSolInt,
- info().startTime() + info().ntime() * info().timeInterval());
-
- // Make time axis (can be non regular for last chunk if solint > 1)
- vector<double> lowtimes(ntime), hightimes(ntime);
- for (uint t=0; t<ntime; ++t) {
- lowtimes[t] = startTime + info().timeInterval() * itsSolInt * t;
- hightimes[t] = min(startTime + info().timeInterval() * itsSolInt * (t+1),
- endTime);
- }
- BBS::Axis::ShPtr timeAxis = Axis::makeAxis(lowtimes, hightimes);
-
- BBS::Axis::ShPtr freqAxis(
- new BBS::RegularAxis(
- getInfo().chanFreqs()[0] - freqWidth * 0.5,
- freqWidth*nchan,
- nfreqs));
- BBS::Grid solGrid(freqAxis, timeAxis);
-
- // Construct domain grid for the current chunk
- BBS::Axis::ShPtr tdomAxis(
- new BBS::RegularAxis(
- startTime,
- endTime - startTime,
- 1));
- BBS::Axis::ShPtr fdomAxis(
- new BBS::RegularAxis(
- info().chanFreqs()[0] - freqWidth * 0.5,
- freqWidth * getInfo().chanFreqs().size(), 1));
- BBS::Grid domainGrid(fdomAxis, tdomAxis);
-
- // Write the solutions per parameter.
- const char* str0101[] = {"0:0:","0:1:","1:0:","1:1:"};
- const char* strri[] = {"Real:","Imag:"};
- Matrix<double> values(nfreqs, ntime);
-
- DComplex sol;
-
- uint nSt=info().antennaUsed().size();
-
- for (size_t st=0; st<nSt; ++st) {
- // Do not write NaN solutions for stations that were not used
- if (!itsAntennaUsed[info().antennaUsed()[st]]) {
- // itsAntennaUsed is indexed with real antenna numbers, so antennaUsed() is needed
- continue;
- }
- for (int pol=0; pol<4; ++pol) { // For 0101
- if (scalarMode(itsMode) && pol>0) {
- continue;
- } else if (diagonalMode(itsMode) && (pol==1||pol==2)) {
- continue;
- }
- int realimmax; // For tecandphase, this functions as dummy between tec and commonscalarphase
- if (itsMode==PHASEONLY || itsMode==SCALARPHASE ||
- itsMode==AMPLITUDEONLY || itsMode==SCALARAMPLITUDE || itsMode==TEC) {
- realimmax=1;
- } else {
- realimmax=2;
- }
- for (int realim=0; realim<realimmax; ++realim) { // For real and imaginary
- string name = parmName();
-
- if (itsMode!=SCALARPHASE && itsMode!=SCALARAMPLITUDE) {
- name+=str0101[pol];
- if (itsMode==PHASEONLY) {
- name=name+"Phase:";
- } else if (itsMode==AMPLITUDEONLY) {
- name=name+"Ampl:";
- } else {
- name=name+strri[realim];
- }
- }
- if (itsMode==TECANDPHASE || itsMode==TEC) {
- if (realim==0) {
- name="TEC:";
- } else {
- name="CommonScalarPhase:";
- }
- }
-
- name+=info().antennaNames()[info().antennaUsed()[st]];
-
- // Collect its solutions for all times and frequency cells in a single array.
- for (uint ts=0; ts<ntime; ++ts) {
- for (uint freqCell=0; freqCell<nfreqs; ++freqCell) {
- if (itsMode==FULLJONES) {
- if (realim==0) {
- values(freqCell, ts) = real(itsSols[ts](pol,st,freqCell));
- } else {
- values(freqCell, ts) = imag(itsSols[ts](pol,st,freqCell));
- }
- } else if (itsMode==COMPLEXGAIN) {
- if (realim==0) {
- values(freqCell, ts) = real(itsSols[ts](pol/3,st,freqCell));
- } else {
- values(freqCell, ts) = imag(itsSols[ts](pol/3,st,freqCell));
- }
- } else if (itsMode==SCALARPHASE || itsMode==PHASEONLY) {
- values(freqCell, ts) = arg(itsSols[ts](pol/3,st,freqCell));
- } else if (itsMode==SCALARAMPLITUDE || itsMode==AMPLITUDEONLY) {
- values(freqCell, ts) = abs(itsSols[ts](pol/3,st,freqCell));
- } else if (itsMode==TEC || itsMode==TECANDPHASE) {
- if (realim==0) {
- values(freqCell, ts) = itsTECSols[ts](realim,st) / 8.44797245e9;
- } else {
- values(freqCell, ts) = -itsTECSols[ts](realim,st); // TODO: why is there a minus here?
- }
- }
- else {
- THROW (Exception, "Unhandled mode");
- }
- }
- }
- BBS::ParmValue::ShPtr pv(new BBS::ParmValue());
- pv->setScalars (solGrid, values);
-
- BBS::ParmValueSet pvs(domainGrid,
- vector<BBS::ParmValue::ShPtr>(1, pv));
- map<string,int>::const_iterator pit = itsParmIdMap.find(name);
-
- if (pit == itsParmIdMap.end()) {
- // First time, so a new nameId will be set.
- // Check if the name was defined in the parmdb previously
- int nameId = itsParmDB->getNameId(name);
- itsParmDB->putValues (name, nameId, pvs);
- itsParmIdMap[name] = nameId;
- } else {
- // Parm has been put before.
- int nameId = pit->second;
- itsParmDB->putValues (name, nameId, pvs);
- }
- }
- }
- }
-
- itsTimerWrite.stop();
- itsTimer.stop();
- }
-
- void GainCal::finish()
- {
- itsTimer.start();
-
- //Solve remaining time slots if any
- if (itsStepInSolInt!=0) {
- stefcal();
-
- if (itsApplySolution) {
- Cube<DComplex> invsol = invertSol(itsSols.back());
- for (uint stepInSolInt=0; stepInSolInt<itsStepInSolInt; stepInSolInt++) {
- applySolution(itsBuf[stepInSolInt], invsol);
- getNextStep()->process(itsBuf[stepInSolInt]);
- }
- }
- }
-
- itsTimer.stop();
-
- if (!itsSols.empty()) {
- if (itsUseH5Parm) {
- writeSolutionsH5Parm(itsChunkStartTime);
- } else {
- writeSolutionsParmDB(itsChunkStartTime);
- }
- if (itsDebugLevel>0) {
- H5::H5File hdf5file = H5::H5File("debug.h5", H5F_ACC_TRUNC);
- vector<hsize_t> dims(6);
- for (uint i=0; i<6; ++i) {
- dims[i] = itsAllSolutions.shape()[5-i];
- }
- H5::DataSpace dataspace(dims.size(), &(dims[0]), NULL);
- H5::CompType complex_data_type(sizeof(DComplex));
- complex_data_type.insertMember( "r", 0, H5::PredType::IEEE_F64LE);
- complex_data_type.insertMember( "i", sizeof(double), H5::PredType::IEEE_F64LE);
- H5::DataSet dataset = hdf5file.createDataSet("val",
- complex_data_type,
- dataspace);
- dataset.write(itsAllSolutions.data(), complex_data_type);
- hdf5file.close();
- }
- }
-
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/GaussianSource.cc b/CEP/DP3/DPPP/src/GaussianSource.cc
deleted file mode 100644
index ff0e0411d0ac151dd5ff0e7a0996bd18499b7428..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/GaussianSource.cc
+++ /dev/null
@@ -1,69 +0,0 @@
-//# GaussianSource.cc: Gaussian source model component.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/GaussianSource.h>
-#include <DPPP/ModelComponentVisitor.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-GaussianSource::GaussianSource(const Position &position)
- : PointSource(position),
- itsPositionAngle(0.0),
- itsMajorAxis(0.0),
- itsMinorAxis(0.0)
-{
-}
-
-GaussianSource::GaussianSource(const Position &position, const Stokes &stokes)
- : PointSource(position, stokes),
- itsPositionAngle(0.0),
- itsMajorAxis(0.0),
- itsMinorAxis(0.0)
-{
-}
-
-void GaussianSource::setPositionAngle(double angle)
-{
- itsPositionAngle = angle;
-}
-
-void GaussianSource::setMajorAxis(double fwhm)
-{
- itsMajorAxis = fwhm;
-}
-
-void GaussianSource::setMinorAxis(double fwhm)
-{
- itsMinorAxis = fwhm;
-}
-
-void GaussianSource::accept(ModelComponentVisitor &visitor) const
-{
- visitor.visit(*this);
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/GridInterpolate.cc b/CEP/DP3/DPPP/src/GridInterpolate.cc
deleted file mode 100644
index 158451aad65db5f9ce2a068ea3fd4d7cbe67d5c3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/GridInterpolate.cc
+++ /dev/null
@@ -1,102 +0,0 @@
-//# GridInterpolate.cc: Interpolate data from regular 2d grid to another
-//# Copyright (C) 2018
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GridInterpolate.cc 37169 2017-04-19 12:41:21Z dijkema $
-//#
-
-#include <DPPP/GridInterpolate.h>
-
-#include <iostream>
-#include <vector>
-#include <cassert>
-#include <stdexcept>
-
-using namespace std;
-
-namespace LOFAR {
- void getAxisIndices(const vector<double>& ax_src,
- const vector<double>& ax_tgt,
- vector<size_t>& indices,
- bool nearest) {
- indices.resize(ax_tgt.size());
- if (ax_tgt.empty()) {
- return;
- }
- assert(!ax_src.empty());
-
- double lowmatch, highmatch;
-
- vector<double>::const_iterator src_val = ax_src.begin();
- vector<double>::const_iterator tgt_val = ax_tgt.begin();
- vector<size_t>::iterator index_val = indices.begin();
-
- while (tgt_val != ax_tgt.end()) {
- while (*src_val < *tgt_val && src_val != ax_src.end()) {
- src_val++;
- }
- if (src_val == ax_src.begin()) {
- *index_val = src_val - ax_src.begin();
- } else if (src_val == ax_src.end()) {
- *index_val = src_val - ax_src.begin() - 1;
- } else {
- if (nearest) {
- lowmatch = *(src_val-1);
- highmatch = *src_val;
-
- if (highmatch - *tgt_val < *tgt_val - lowmatch) {
- *index_val = src_val - ax_src.begin();
- } else {
- *index_val = src_val - ax_src.begin() - 1;
- }
- } else {
- *index_val = src_val - ax_src.begin() - 1;
- }
- }
- tgt_val++; index_val++;
- }
- }
-
- void gridNearestNeighbor(const vector<double>& x_src,
- const vector<double>& y_src,
- const vector<double>& x_tgt,
- const vector<double>& y_tgt,
- const double* vals_src,
- double* vals_tgt,
- bool nearest) {
- vector<size_t> x_indices;
- vector<size_t> y_indices;
- getAxisIndices(x_src, x_tgt, x_indices, nearest);
- getAxisIndices(y_src, y_tgt, y_indices, nearest);
-
- size_t nx = x_tgt.size();
- size_t ny = y_tgt.size();
- size_t ny_src = y_src.size();
- // y varies fastest
-
- if (nearest) {
- for (size_t i=0; i<nx; ++i) {
- for (size_t j=0; j<ny; ++j) {
- vals_tgt[i*ny+j] = vals_src[x_indices[i]*ny_src + y_indices[j]];
- }
- }
- } else {
- throw std::logic_error("Not implemented");
- }
- }
-}
diff --git a/CEP/DP3/DPPP/src/H5Parm.cc b/CEP/DP3/DPPP/src/H5Parm.cc
deleted file mode 100644
index afa6d4fd6631f6bc9730e4c3384f5aec9ec1aa68..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/H5Parm.cc
+++ /dev/null
@@ -1,180 +0,0 @@
-#include <lofar_config.h>
-#include <DPPP/H5Parm.h>
-#include <Common/Exception.h>
-#include <Common/StringUtil.h>
-#include <Common/LofarLogger.h>
-#include <cstring>
-#include <complex>
-#include <sstream>
-#include <iomanip>
-#include <sys/stat.h>
-
-#include <hdf5.h>
-
-using namespace std;
-
-namespace LOFAR {
- H5Parm::H5Parm(const std::string& filename, bool forceNew,
- bool forceNewSolSet, const std::string& solSetName):
- H5::H5File(filename,
- !forceNew&&access(filename.c_str(),F_OK)!=-1?H5F_ACC_RDWR:H5F_ACC_TRUNC
- )
- {
- if (forceNewSolSet || getNumObjs()==0) { // Create a new solSet
- if (solSetName=="") {
- // Get the name of first non-existing solset
- stringstream newSolSetName;
- H5::Group tryGroup;
- for (uint solSetNum=0; solSetNum<100; ++solSetNum) {
- try {
- H5::Exception::dontPrint();
- newSolSetName<<"sol"<<setfill('0')<<setw(3)<<solSetNum;
- tryGroup = openGroup(newSolSetName.str());
- newSolSetName.str("");
- }
- catch (H5::FileIException& not_found_error ) {
- // solSetName does not exist yet
- break;
- }
- tryGroup.close();
- }
- _solSet = createGroup("/"+newSolSetName.str(), H5P_DEFAULT);
- } else {
- // Create solset with the given name
- _solSet = createGroup("/"+solSetName, H5P_DEFAULT);
- }
- addVersionStamp(_solSet);
- } else {
- string solSetNameToOpen=solSetName;
- if (solSetNameToOpen=="") {
- if (this->getNumObjs()==1) {
- solSetNameToOpen=this->getObjnameByIdx(0);
- } else {
- THROW(Exception, "H5Parm "<<filename<<" contains more than one SolSet, "<<
- "please specify which one to use.");
- }
- }
-
- _solSet = openGroup(solSetNameToOpen);
-
- vector<string> solTabNames;
- for (uint i=0; i<_solSet.getNumObjs();++i) {
- if (_solSet.getObjTypeByIdx(i)==H5G_GROUP) {
- solTabNames.push_back(_solSet.getObjnameByIdx(i));
- }
- }
-
- for (vector<string>::iterator solTabName=solTabNames.begin();
- solTabName!=solTabNames.end(); ++solTabName) {
- H5::Group group = _solSet.openGroup(*solTabName);
- _solTabs.insert(
- std::map<std::string, SolTab>::value_type (*solTabName, SolTab(group)));
- }
- }
- }
-
-
- H5Parm::H5Parm() {
- }
-
- H5Parm::~H5Parm() {
- // Throw an error if the antenna or source table is not present
- //_solSet.openDataSet("antenna");
- //_solSet.openDataSet("source");
- _solSet.close();
- }
-
- string H5Parm::getSolSetName() const {
- char buffer[100];
- hsize_t namelen = H5Iget_name(_solSet.getId(),buffer,100);
- buffer[namelen+1]=0;
- // Strip leading '/'
- return buffer+1;
- }
-
- void H5Parm::addVersionStamp(H5::Group &node) {
- // Write an attribute with the h5parm version
- H5::Attribute attr = node.createAttribute("h5parm_version",
- H5::StrType(H5::PredType::C_S1, 3),
- H5::DataSpace());
- attr.write(H5::StrType(H5::PredType::C_S1, 3), "1.0");
- }
-
- void H5Parm::addSources (const std::vector<std::string>& names,
- const std::vector<std::pair<double, double> >& dirs) {
- hsize_t dims[1];
-
- // Create data type
- dims[0]=2; // For ra, dec in directions
- H5::CompType sourceType(sizeof(source_t));
- sourceType.insertMember("name", HOFFSET(antenna_t, name), H5::StrType(H5::PredType::C_S1, 128));
- sourceType.insertMember("dir", HOFFSET(source_t, dir), H5::ArrayType(H5::PredType::NATIVE_FLOAT, 1, dims));
-
- // Create dataset
- dims[0] = names.size();
- H5::DataSpace dataspace(1, dims, NULL);
- H5::DataSet dataset = _solSet.createDataSet("source", sourceType, dataspace);
-
- // Prepare data
- vector<source_t> sources(names.size());
- for (uint src=0; src<sources.size(); ++src) {
- std::strncpy(sources[src].name, names[src].c_str(), 128);
- sources[src].dir[0] = dirs[src].first;
- sources[src].dir[1] = dirs[src].second;
- }
-
- // Write data
- dataset.write(&(sources[0]), sourceType);
- }
-
- void H5Parm::addAntennas (const std::vector<std::string>& names,
- const std::vector<std::vector<double> >& positions) {
- hsize_t dims[1];
-
- // Create data type
- dims[0]=3; // For x,y,z in positions
- H5::CompType antennaType(sizeof(antenna_t));
- antennaType.insertMember("name", HOFFSET(antenna_t, name), H5::StrType(H5::PredType::C_S1, 16));
- antennaType.insertMember("position", HOFFSET(antenna_t, position), H5::ArrayType(H5::PredType::NATIVE_FLOAT, 1, dims));
-
- // Create dataset
- dims[0] = names.size();
- H5::DataSpace dataspace(1, dims, NULL);
- H5::DataSet dataset = _solSet.createDataSet("antenna", antennaType, dataspace);
-
- // Prepare data
- vector<antenna_t> ants(names.size());
- for (uint ant=0; ant<ants.size(); ++ant) {
- std::strncpy(ants[ant].name, names[ant].c_str(), 16);
- const std::vector<double>& pos = positions[ant];
- ants[ant].position[0] = pos[0];
- ants[ant].position[1] = pos[1];
- ants[ant].position[2] = pos[2];
- }
-
- dataset.write(&(ants[0]), antennaType);
- }
-
- H5Parm::SolTab& H5Parm::getSolTab(const std::string& name) {
- std::map<std::string, SolTab>::iterator item =
- _solTabs.find(name);
- if (item == _solTabs.end()) {
- THROW(Exception, "SolTab "<<name<<" does not exist in solset "<<
- getSolSetName());
- }
- return item->second;
- }
-
- bool H5Parm::hasSolTab(const string& solTabName) const {
- return _solTabs.find(solTabName) != _solTabs.end();
- }
-
- H5Parm::SolTab& H5Parm::createSolTab(const std::string& name,
- const std::string& type,
- const std::vector<H5Parm::AxisInfo> axes) {
- H5::Group newgroup = _solSet.createGroup(name);
- std::map<std::string, SolTab>::iterator newItem =
- _solTabs.insert(std::make_pair(name, SolTab(newgroup, type, axes))).first;
- return newItem->second;
- }
-}
diff --git a/CEP/DP3/DPPP/src/H5ParmPredict.cc b/CEP/DP3/DPPP/src/H5ParmPredict.cc
deleted file mode 100644
index 215b95bf367d54b0e970f2abc9242c371e8dd0b6..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/H5ParmPredict.cc
+++ /dev/null
@@ -1,157 +0,0 @@
-//# GainCal.cc: DPPP step class to H5ParmPredict visibilities
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/H5ParmPredict.h>
-
-#include <iostream>
-#include <Common/ParameterSet.h>
-#include <Common/Timer.h>
-
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-#include <Common/StreamUtil.h>
-#include <Common/StringUtil.h>
-
-using namespace casacore;
-using namespace LOFAR::BBS;
-
-namespace LOFAR {
- namespace DPPP {
-
- H5ParmPredict::H5ParmPredict (DPInput* input,
- const ParameterSet& parset,
- const string& prefix):
- itsInput(input),
- itsH5ParmName(parset.getString(prefix+"applycal.parmdb")),
- itsDirections(parset.getStringVector(
- prefix+"directions", vector<string> ()))
- {
- H5Parm h5parm = H5Parm(itsH5ParmName, false);
- H5Parm::SolTab soltab = h5parm.getSolTab(parset.getString(prefix+"applycal.correction"));
-
- vector<string> h5directions = soltab.getStringAxis("dir");
-
- string operation = parset.getString(prefix+"operation", "replace");
-
- if (itsDirections.empty()) {
- itsDirections = h5directions;
- } else {
- for (vector<string>::iterator it = itsDirections.begin();
- // Check that all specified directions are in the h5parm
- it != itsDirections.end(); ++it) {
- if (find(h5directions.begin(), h5directions.end(), *it) ==
- h5directions.end()) {
- THROW(Exception, "Direction "<<*it<<" not found in "<<itsH5ParmName);
- }
- }
- }
-
- ASSERT(!itsDirections.empty());
-
- for (uint i=0; i<itsDirections.size(); ++i) {
- string directionStr = itsDirections[i];
- vector<string> directionVec; // each direction should be like '[patch1,patch2]'
- ASSERT(directionStr.size()>2 && directionStr[0]=='[' &&
- directionStr[directionStr.size()-1]==']');
- directionVec = StringUtil::tokenize(directionStr.substr(1, directionStr.size()-2), ",");
- Predict* predictStep = new Predict(input, parset, prefix, directionVec);
-
- if (operation=="replace" && i>0) {
- predictStep->setOperation("add");
- } else {
- predictStep->setOperation(operation);
- }
-
- itsPredictSteps.push_back(Predict::ShPtr(predictStep));
- if (i>0) {
- itsPredictSteps[i-1]->setNextStep(itsPredictSteps[i]);
- }
- }
-
- itsResultStep=new ResultStep();
- itsPredictSteps[itsPredictSteps.size()-1]->setNextStep(DPStep::ShPtr(itsResultStep));
- }
-
- H5ParmPredict::~H5ParmPredict()
- {}
-
- void H5ParmPredict::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
-
- vector<Predict::ShPtr>::iterator predictstep;
- for (predictstep=itsPredictSteps.begin();
- predictstep!=itsPredictSteps.end();
- predictstep++) {
- (*predictstep)->updateInfo(infoIn);
- }
- }
-
- void H5ParmPredict::show (std::ostream& os) const
- {
- os << "H5ParmPredict " << itsName << endl;
- os << " H5Parm: " << itsH5ParmName << endl;
- os << " directions: " << itsDirections << endl;
- for (uint dir=0; dir<itsPredictSteps.size(); ++dir) {
- itsPredictSteps[dir]->show(os);
- }
- }
-
- void H5ParmPredict::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " H5ParmPredict " << itsName << endl;
- }
-
- bool H5ParmPredict::process (const DPBuffer& bufin)
- {
- itsTimer.start();
- itsBuffer.copy (bufin);
- itsInput->fetchUVW(bufin, itsBuffer, itsTimer);
- itsInput->fetchWeights(bufin, itsBuffer, itsTimer);
-
- itsPredictSteps[0]->process(itsBuffer);
- itsBuffer = itsResultStep->get();
-
- itsTimer.stop();
- getNextStep()->process(itsBuffer);
- return false;
- }
-
-
- void H5ParmPredict::finish()
- {
- // Let the next steps finish.
- itsPredictSteps[0]->finish();
- getNextStep()->finish();
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/MSReader.cc b/CEP/DP3/DPPP/src/MSReader.cc
deleted file mode 100644
index fd9af907f3146ed16eaf4af9f0f9ff2c8cfb6e65..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/MSReader.cc
+++ /dev/null
@@ -1,867 +0,0 @@
-//# MSReader.cc: DPPP step reading from an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/MSReader.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPLogger.h>
-#include <StationResponse/LofarMetaDataUtil.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-
-#include <casacore/tables/Tables/TableRecord.h>
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/tables/Tables/ArrayColumn.h>
-#include <casacore/tables/TaQL/ExprNode.h>
-#include <casacore/tables/TaQL/RecordGram.h>
-#include <casacore/measures/Measures/MeasTable.h>
-#include <casacore/measures/TableMeasures/ScalarMeasColumn.h>
-#include <casacore/measures/TableMeasures/ArrayMeasColumn.h>
-#include <casacore/ms/MeasurementSets/MeasurementSet.h>
-#if defined(HAVE_CASACORE)
-#include <casacore/ms/MSSel/MSSelection.h>
-#else
-#include <casacore/ms/MSSel/MSSelection.h>
-#endif
-#include <casacore/casa/Containers/Record.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Quanta/MVTime.h>
-#include <casacore/casa/OS/Conversion.h>
-#include <iostream>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- MSReader::MSReader()
- : itsReadVisData (False),
- itsLastMSTime (0),
- itsNrRead (0),
- itsNrInserted (0)
- {}
-
- MSReader::MSReader (const string& msName,
- const ParameterSet& parset,
- const string& prefix,
- bool missingData)
- : itsReadVisData (False),
- itsMissingData (missingData),
- itsLastMSTime (0),
- itsNrRead (0),
- itsNrInserted (0)
- {
- NSTimer::StartStop sstime(itsTimer);
- // Get info from parset.
- itsSpw = parset.getInt (prefix+"band", -1);
- itsStartChanStr = parset.getString (prefix+"startchan", "0");
- itsNrChanStr = parset.getString (prefix+"nchan", "0");
- string startTimeStr = parset.getString (prefix+"starttime", "");
- string endTimeStr = parset.getString (prefix+"endtime", "");
- uint nTimes = parset.getInt (prefix+"ntimes", 0);
- itsTimeTolerance = parset.getDouble (prefix+"timetolerance", 1e-2);
- itsUseFlags = parset.getBool (prefix+"useflag", true);
- itsDataColName = parset.getString (prefix+"datacolumn", "DATA");
- itsWeightColName = parset.getString (prefix+"weightcolumn",
- "WEIGHT_SPECTRUM");
- itsModelColName = parset.getString (prefix+"modelcolumn",
- "MODEL_DATA");
- itsAutoWeight = parset.getBool (prefix+"autoweight", false);
- itsAutoWeightForce = parset.getBool (prefix+"forceautoweight", false);
- itsNeedSort = parset.getBool (prefix+"sort", false);
- itsSelBL = parset.getString (prefix+"baseline", string());
- // Try to open the MS and get its full name.
- if (itsMissingData && !Table::isReadable (msName)) {
- DPLOG_WARN_STR ("MeasurementSet " << msName
- << " not found; dummy data used");
- return;
- }
- itsMS = MeasurementSet (msName, TableLock::AutoNoReadLocking);
- itsSelMS = itsMS;
- itsMSName = itsMS.tableName();
- // See if a selection on band needs to be done.
- // We assume that DATA_DESC_ID and SPW_ID map 1-1.
- if (itsSpw >= 0) {
- DPLOG_INFO_STR (" MSReader selecting spectral window " << itsSpw << " ...");
- Table subset = itsSelMS (itsSelMS.col("DATA_DESC_ID") == itsSpw);
- // If not all is selected, use the selection.
- if (subset.nrow() < itsSelMS.nrow()) {
- ASSERTSTR (subset.nrow() > 0, "Band " << itsSpw << " not found in "
- << itsMSName);
- itsSelMS = subset;
- }
- } else {
- itsSpw = 0;
- }
- // See if a selection on baseline needs to be done.
- if (! itsSelBL.empty()) {
- DPLOG_INFO_STR (" MSReader selecting baselines ...");
- MSSelection select;
- // Set given selection strings.
- select.setAntennaExpr (itsSelBL);
- // Create a table expression for an MS representing the selection.
- MeasurementSet ms(itsSelMS);
- TableExprNode node = select.toTableExprNode (&ms);
- Table subset = itsSelMS(node);
- // If not all is selected, use the selection.
- if (subset.nrow() < itsSelMS.nrow()) {
- ASSERTSTR (subset.nrow() > 0, "Baselines " << itsSelBL
- << "not found in " << itsMSName);
- itsSelMS = subset;
- }
- }
- // Prepare the MS access and get time info.
- double startTime=0., endTime=0.;
- prepare (startTime, endTime, itsTimeInterval);
- // Start and end time can be given in the parset in case leading
- // or trailing time slots are missing.
- // They can also be used to select part of the MS.
- Quantity qtime;
- itsFirstTime = startTime;
- if (!startTimeStr.empty()) {
- if (!MVTime::read (qtime, startTimeStr)) {
- THROW (LOFAR::Exception, startTimeStr << " is an invalid date/time");
- }
- itsFirstTime = qtime.getValue("s");
- ASSERT (itsFirstTime <= endTime);
- }
- itsLastTime = endTime;
- if (!endTimeStr.empty()) {
- if (!MVTime::read (qtime, endTimeStr)) {
- THROW (LOFAR::Exception, endTimeStr << " is an invalid date/time");
- }
- itsLastTime = qtime.getValue("s");
- }
- ASSERT (itsLastTime >= itsFirstTime);
- // If needed, skip the first times in the MS.
- // It also sets itsFirstTime properly (round to time/interval in MS).
- skipFirstTimes();
- if (nTimes > 0) {
- itsLastTime = itsFirstTime + (nTimes-1) * itsTimeInterval;
- }
- itsNextTime = itsFirstTime;
- itsStartTime = itsFirstTime - 0.5*itsTimeInterval;
- // Parse the chan expressions.
- // Nr of channels can be used as 'nchan' in the expressions.
- Record rec;
- rec.define ("nchan", itsNrChan);
- TableExprNode node1 (RecordGram::parse(rec, itsStartChanStr));
- TableExprNode node2 (RecordGram::parse(rec, itsNrChanStr));
- // nchan=0 means until the last channel.
- double result;
- node1.get (rec, result);
- itsStartChan = uint(result+0.001);
- node2.get (rec, result);
- uint nrChan = uint(result+0.0001);
- uint nAllChan = itsNrChan;
- ASSERTSTR (itsStartChan < nAllChan,
- "startchan " << itsStartChan
- << " exceeds nr of channels in MS (" << nAllChan << ')');
- uint maxNrChan = nAllChan - itsStartChan;
- if (nrChan == 0) {
- itsNrChan = maxNrChan;
- } else {
- itsNrChan = std::min (nrChan, maxNrChan);
- }
- // Are all channels used?
- itsUseAllChan = itsStartChan==0 && itsNrChan==nAllChan;
- // Do the rest of the preparation.
- prepare2();
- // Take subset of channel frequencies if needed.
- // Make sure to copy the subset to get a proper Vector.
- // Form the slicer to get channels and correlations from column.
- itsColSlicer = Slicer(IPosition(2, 0, itsStartChan),
- IPosition(2, itsNrCorr, itsNrChan));
- // Form the slicer to get channels, corrs, and baselines from array.
- itsArrSlicer = Slicer(IPosition(3, 0, itsStartChan, 0),
- IPosition(3, itsNrCorr, itsNrChan, itsNrBl));
- // Initialize the flag counters.
- itsFlagCounter.init (getInfo());
- }
-
- MSReader::~MSReader()
- {}
-
- void MSReader::updateInfo (const DPInfo&)
- {}
-
- casacore::String MSReader::msName() const
- {
- return itsMSName;
- }
-
- void MSReader::setReadVisData (bool readVisData)
- {
- itsReadVisData = readVisData;
- }
-
- bool MSReader::process (const DPBuffer&)
- {
- if (itsNrRead == 0) {
- if (itsReadVisData) {
- itsBuffer.getData().resize (itsNrCorr, itsNrChan, itsNrBl);
- }
- if (itsUseFlags) {
- itsBuffer.getFlags().resize (itsNrCorr, itsNrChan, itsNrBl);
- }
- ///cout<<(void*)(itsBuffer.getData().data())<<" upd"<<endl;
- }
- {
- NSTimer::StartStop sstime(itsTimer);
- /// itsBuffer.clear();
- // Use time from the current time slot in the MS.
- bool useIter = false;
- while (!itsIter.pastEnd()) {
- // Take time from row 0 in subset.
- double mstime = ROScalarColumn<double>(itsIter.table(), "TIME")(0);
- // Skip time slot and give warning if MS data is not in time order.
- if (mstime < itsLastMSTime) {
- LOG_WARN_STR ("Time at rownr "
- << itsIter.table().rowNumbers(itsMS)[0]
- << " of MS " << itsMSName
- << " is less than previous time slot");
- } else {
- // Use the time slot if near or < nexttime, but > starttime.
- // In this way we cater for irregular times in some WSRT MSs.
- if (nearAbs(mstime, itsNextTime, itsTimeTolerance)) {
- useIter = true;
- break;
- } else if (mstime > itsFirstTime && mstime < itsNextTime) {
- itsFirstTime -= itsNextTime-mstime;
- itsNextTime = mstime;
- useIter = true;
- break;
- }
- if (mstime > itsNextTime) {
- // A time slot seems to be missing; insert one.
- break;
- }
- }
- // Skip this time slot.
- itsLastMSTime = mstime;
- itsIter.next();
- }
- // Stop if at the end, or if there is no data at all
- if ((itsNextTime > itsLastTime && !near(itsNextTime, itsLastTime)) ||
- itsNextTime==0.) {
- return false;
- }
- // Fill the buffer.
- itsBuffer.setTime (itsNextTime);
- ///cout << "read time " <<itsBuffer.getTime() - 4472025855.0<<endl;
- if (!useIter) {
- // Need to insert a fully flagged time slot.
- itsBuffer.setRowNrs (Vector<uint>());
- itsBuffer.setExposure (itsTimeInterval);
- itsBuffer.getFlags() = true;
- if (itsReadVisData){
- itsBuffer.getData() = Complex();
- }
- itsNrInserted++;
- } else {
- itsBuffer.setRowNrs (itsIter.table().rowNumbers(itsMS, True));
- if (itsMissingData) {
- // Data column not present, so fill a fully flagged time slot.
- itsBuffer.setExposure (itsTimeInterval);
- itsBuffer.getFlags() = true;
- if (itsReadVisData) {
- itsBuffer.getData() = Complex();
- }
- } else {
- // Set exposure.
- itsBuffer.setExposure (ROScalarColumn<double>
- (itsIter.table(), "EXPOSURE")(0));
- // Get data and flags from the MS.
- /// if (itsNrRead%50 < 4) {
- /// cout<<(void*)(itsBuffer.getData().data())<<" rd1"<<endl;
- ///}
- if (itsReadVisData) {
- ROArrayColumn<Complex> dataCol(itsIter.table(), itsDataColName);
- if (itsUseAllChan) {
- dataCol.getColumn (itsBuffer.getData());
- } else {
- dataCol.getColumn (itsColSlicer, itsBuffer.getData());
- }
- }
- ///if (itsNrRead%50 < 4) {
- ///cout<<(void*)(itsBuffer.getData().data())<<" rd2"<<endl;
- ///}
- if (itsUseFlags) {
- ROArrayColumn<bool> flagCol(itsIter.table(), "FLAG");
- if (itsUseAllChan) {
- flagCol.getColumn (itsBuffer.getFlags());
- } else {
- flagCol.getColumn(itsColSlicer, itsBuffer.getFlags());
- }
- // Set flags if FLAG_ROW is set.
- ROScalarColumn<bool> flagrowCol(itsIter.table(), "FLAG_ROW");
- for (uint i=0; i<itsIter.table().nrow(); ++i) {
- if (flagrowCol(i)) {
- itsBuffer.getFlags()
- (IPosition(3,0,0,i),
- IPosition(3,itsNrCorr-1,itsNrChan-1,i)) = true;
- }
- }
- } else {
- // Do not use FLAG from the MS.
- itsBuffer.getFlags().resize (itsNrCorr, itsNrChan, itsNrBl);
- itsBuffer.getFlags() = false;
- }
- // Flag invalid data (NaN, infinite).
- flagInfNaN(itsBuffer.getData(), itsBuffer.getFlags(),
- itsFlagCounter);
- }
- itsLastMSTime = itsNextTime;
- itsNrRead++;
- itsIter.next();
- }
- ASSERTSTR (itsBuffer.getFlags().shape()[2] == int(itsNrBl),
- "#baselines is not the same for all time slots in the MS");
- } // end of scope stops the timer.
- // Let the next step in the pipeline process this time slot.
- getNextStep()->process (itsBuffer);
- /// cout << "Reader: " << itsNextTime-4.75e9<<endl;
- // Do not add to previous time, because it introduces round-off errors.
- itsNextTime = itsFirstTime + (itsNrRead+itsNrInserted) * itsTimeInterval;
- return true;
- }
-
- void MSReader::flagInfNaN(const casacore::Cube<casacore::Complex>& dataCube,
- casacore::Cube<bool>& flagsCube,
- FlagCounter& flagCounter) {
- int ncorr=dataCube.shape()[0];
- const Complex* dataPtr = dataCube.data();
- bool* flagPtr = flagsCube.data();
- for (uint i=0; i<dataCube.size();) {
- for (uint j=i; j<i+ncorr; ++j) {
- bool flag = (!isFinite(dataPtr[j].real()) ||
- !isFinite(dataPtr[j].imag()));
- if (flag) {
- flagCounter.incrCorrelation(j-i);
- }
- if (flag || flagPtr[j]) {
- // Flag all correlations if a single one is flagged.
- for (uint k=i; k<i+ncorr; ++k) {
- flagPtr[k] = true;
- }
- break;
- }
- }
- i += ncorr;
- }
- }
-
- void MSReader::finish()
- {
- getNextStep()->finish();
- }
-
- void MSReader::show (std::ostream& os) const
- {
- os << "MSReader" << std::endl;
- os << " input MS: " << itsMSName << std::endl;
- if (itsMS.isNull()) {
- os << " *** MS does not exist ***" << std::endl;
- } else {
- if (! itsSelBL.empty()) {
- os << " baseline: " << itsSelBL << std::endl;
- }
- os << " band " << itsSpw << std::endl;
- os << " startchan: " << itsStartChan << " (" << itsStartChanStr
- << ')' << std::endl;
- os << " nchan: " << getInfo().nchan() << " (" << itsNrChanStr
- << ')' << std::endl;
- os << " ncorrelations: " << getInfo().ncorr() << std::endl;
- uint nrbl = getInfo().nbaselines();
- os << " nbaselines: " << nrbl << std::endl;
- os << " ntimes: " << (nrbl==0 ? 0 : itsSelMS.nrow() / nrbl) << std::endl;
- os << " time interval: " << getInfo().timeInterval() << std::endl;
- os << " DATA column: " << itsDataColName;
- if (itsMissingData) {
- os << " (not present)";
- }
- os << std::endl;
- os << " WEIGHT column: " << itsWeightColName << std::endl;
- os << " autoweight: " << boolalpha << itsAutoWeight << std::endl;
- }
- }
-
- void MSReader::showCounts (std::ostream& os) const
- {
- os << endl << "NaN/infinite data flagged in reader";
- os << endl << "===================================" << endl;
- int64 nrtim = itsNrRead;
- itsFlagCounter.showCorrelation (os, nrtim);
- os << itsNrInserted << " missing time slots were inserted" << endl;
- }
-
- void MSReader::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " MSReader" << endl;
- }
-
- void MSReader::prepare (double& firstTime, double& lastTime,
- double& interval)
- {
- if (itsSelMS.nrow() == 0) {
- DPLOG_WARN_STR ("The selected input does not contain any data.");
- }
- TableDesc tdesc = itsMS.tableDesc();
-
- itsHasWeightSpectrum = false;
- // if weightcolname is specified to "WEIGHT" then this is used, even
- // if a weight_spectrum is present.
- if (itsWeightColName!="WEIGHT") {
- // Test if specified weight column or WEIGHT_SPECTRUM is present.
- if (tdesc.isColumn(itsWeightColName)) {
- // The column is there, but it might not contain values. Test row 0.
- itsHasWeightSpectrum =
- ROArrayColumn<float>(itsSelMS, itsWeightColName).isDefined(0);
- if (!itsHasWeightSpectrum && itsWeightColName!="WEIGHT_SPECTRUM") {
- LOG_WARN ("Specified weight column " + itsWeightColName +
- "is not a valid column, using WEIGHT instead");
- }
- }
- }
-
- // Test if the data column is present.
- if (tdesc.isColumn (itsDataColName)) {
- itsMissingData = false;
- } else {
- if (itsMissingData) {
- // Only give warning if a missing data column is allowed.
- LOG_WARN ("Data column " + itsDataColName +
- " is missing in " + itsMSName);
- } else {
- THROW (Exception, ("Data column " + itsDataColName +
- " is missing in " + itsMSName));
- }
- }
-
- // Test if the full resolution flags are present.
- itsHasFullResFlags = tdesc.isColumn("LOFAR_FULL_RES_FLAG");
- if (itsHasFullResFlags) {
- ROTableColumn fullResFlagCol(itsMS, "LOFAR_FULL_RES_FLAG");
- itsFullResNChanAvg = fullResFlagCol.keywordSet().asInt ("NCHAN_AVG");
- itsFullResNTimeAvg = fullResFlagCol.keywordSet().asInt ("NTIME_AVG");
- } else {
- itsFullResNChanAvg = 1;
- itsFullResNTimeAvg = 1;
- }
- // Get the main table in the correct order.
- // Determine if the data are stored using LofarStMan.
- // If so, we know it is in time order.
- // (sorting on TIME with LofarStMan can be expensive).
- bool needSort = itsNeedSort;
- bool useRaw = false;
- Record dminfo = itsMS.dataManagerInfo();
- for (unsigned i=0; i<dminfo.nfields(); ++i) {
- Record subrec = dminfo.subRecord(i);
- if (subrec.asString("TYPE") == "LofarStMan") {
- needSort = false;
- useRaw = true;
- break;
- }
- }
- // Give an error if autoweight is used for a non-raw MS.
- if (itsAutoWeightForce) {
- itsAutoWeight = true;
- } else if (!useRaw && itsAutoWeight) {
- THROW (Exception, "Using autoweight=true cannot be done on DPPP-ed MS");
- }
- // If not in order, sort the table selection (also on baseline).
- Table sortms(itsSelMS);
- Block<String> sortCols(3);
- sortCols[0] = "TIME";
- sortCols[1] = "ANTENNA1";
- sortCols[2] = "ANTENNA2";
- if (needSort) {
- sortms = itsSelMS.sort(sortCols);
- }
- // Get first and last time and interval from MS.
- if (itsSelMS.nrow() > 0) {
- firstTime = ROScalarColumn<double>(sortms, "TIME")(0);
- lastTime = ROScalarColumn<double>(sortms, "TIME")(sortms.nrow()-1);
- interval = ROScalarColumn<double>(sortms, "INTERVAL")(0);
- }
- // Create iterator over time. Do not sort again.
- itsIter = TableIterator (sortms, Block<String>(1, "TIME"),
- TableIterator::Ascending,
- TableIterator::NoSort);
- // Find the nr of corr, chan, and baseline.
- IPosition shp (ROArrayColumn<Complex>(itsSelMS, "DATA").shape(0));
- itsNrCorr = shp[0];
- itsNrChan = shp[1];
- itsNrBl = itsIter.table().nrow();
- // Ensure we have only one band by checking the nr of unique baselines.
- Table sortab = itsIter.table().sort(sortCols, Sort::Ascending,
- Sort::QuickSort + Sort::NoDuplicates);
- ASSERTSTR (sortab.nrow() == itsNrBl,
- "The MS appears to have multiple subbands");
- // Get the baseline columns.
- ROScalarColumn<Int> ant1col(itsIter.table(), "ANTENNA1");
- ROScalarColumn<Int> ant2col(itsIter.table(), "ANTENNA2");
- // Keep the row numbers of the first part to be used for the meta info
- // of possibly missing time slots.
- itsBaseRowNrs = itsIter.table().rowNumbers(itsMS, True);
- // Get the antenna names and positions.
- Table anttab(itsMS.keywordSet().asTable("ANTENNA"));
- ROScalarColumn<String> nameCol (anttab, "NAME");
- ROScalarColumn<Double> diamCol (anttab, "DISH_DIAMETER");
- uint nant = anttab.nrow();
- ROScalarMeasColumn<MPosition> antcol (anttab, "POSITION");
- vector<MPosition> antPos;
- antPos.reserve (nant);
- for (uint i=0; i<nant; ++i) {
- antPos.push_back (antcol(i));
- }
- // Set antenna/baseline info.
- info().set (nameCol.getColumn(), diamCol.getColumn(), antPos,
- ant1col.getColumn(), ant2col.getColumn());
-
- if (itsAutoWeight) {
- info().setNeedVisData();
- info().setWriteWeights();
- }
-
- // Read the phase reference position from the FIELD subtable.
- // Only use the main value from the PHASE_DIR array.
- // The same for DELAY_DIR and LOFAR_TILE_BEAM_DIR.
- // If LOFAR_TILE_BEAM_DIR does not exist, use DELAY_DIR.
- Table fldtab (itsMS.keywordSet().asTable ("FIELD"));
- AlwaysAssert (fldtab.nrow() == 1, AipsError);
- MDirection phaseCenter, delayCenter, tileBeamDir;
- ROArrayMeasColumn<MDirection> fldcol1 (fldtab, "PHASE_DIR");
- ROArrayMeasColumn<MDirection> fldcol2 (fldtab, "DELAY_DIR");
- phaseCenter = *(fldcol1(0).data());
- delayCenter = *(fldcol2(0).data());
-
- tileBeamDir = StationResponse::readTileBeamDirection(itsMS);
-
- // Get the array position using the telescope name from the OBSERVATION
- // subtable.
- Table obstab (itsMS.keywordSet().asTable ("OBSERVATION"));
- ROScalarColumn<String> telCol(obstab, "TELESCOPE_NAME");
- MPosition arrayPos;
- if (obstab.nrow() == 0 ||
- ! MeasTable::Observatory(arrayPos, telCol(0))) {
- // If not found, use the position of the middle antenna.
- arrayPos = antPos[antPos.size() / 2];
- }
- info().set (arrayPos, phaseCenter, delayCenter, tileBeamDir);
- // Create the UVW calculator.
- itsUVWCalc = UVWCalculator (phaseCenter, arrayPos, antPos);
- }
-
- void MSReader::prepare2()
- {
- // Set the info.
- uint ntime = uint((itsLastTime - itsFirstTime)/itsTimeInterval + 1.5);
- // Read the antenna set.
- Table obstab(itsMS.keywordSet().asTable("OBSERVATION"));
- string antennaSet;
- if (obstab.nrow() > 0 &&
- obstab.tableDesc().isColumn ("LOFAR_ANTENNA_SET")) {
- antennaSet = ROScalarColumn<String>(obstab, "LOFAR_ANTENNA_SET")(0);
- }
- info().init (itsNrCorr, itsNrChan, ntime, itsStartTime,
- itsTimeInterval, itsMSName, antennaSet);
- info().setDataColName(itsDataColName);
- info().setWeightColName(itsWeightColName);
- // Read the center frequencies of all channels.
- Table spwtab(itsMS.keywordSet().asTable("SPECTRAL_WINDOW"));
- ROArrayColumn<double> freqCol (spwtab, "CHAN_FREQ");
- ROArrayColumn<double> widthCol (spwtab, "CHAN_WIDTH");
- ROArrayColumn<double> resolCol (spwtab, "RESOLUTION");
- ROArrayColumn<double> effBWCol (spwtab, "EFFECTIVE_BW");
- ROScalarColumn<Double> refCol (spwtab, "REF_FREQUENCY");
- Vector<double> chanFreqs = freqCol(itsSpw);
- Vector<double> chanWidths = widthCol(itsSpw);
- Vector<double> resolutions = resolCol(itsSpw);
- Vector<double> effectiveBW = effBWCol(itsSpw);
- double refFreq = refCol(itsSpw);
- if (itsUseAllChan) {
- info().set (chanFreqs, chanWidths, resolutions, effectiveBW,
- sum(effectiveBW), refFreq);
- } else {
- Vector<double> cwSlice(effectiveBW(Slice(itsStartChan, itsNrChan)));
- info().set (chanFreqs (Slice(itsStartChan, itsNrChan)),
- chanWidths (Slice(itsStartChan, itsNrChan)),
- resolutions(Slice(itsStartChan, itsNrChan)),
- cwSlice,
- sum(cwSlice), refFreq);
- }
- }
-
- void MSReader::skipFirstTimes()
- {
- while (!itsIter.pastEnd()) {
- // Take time from row 0 in subset.
- double mstime = ROScalarColumn<double>(itsIter.table(), "TIME")(0);
- // Skip time slot and give warning if MS data is not in time order.
- if (mstime < itsLastMSTime) {
- LOG_WARN_STR ("Time at rownr "
- << itsIter.table().rowNumbers(itsMS)[0]
- << " of MS " << itsMSName
- << " is less than previous time slot");
- } else {
- // Stop skipping if time equal to itsFirstTime.
- if (near(mstime, itsFirstTime)) {
- itsFirstTime = mstime;
- break;
- }
- // Also stop if time > itsFirstTime.
- // In that case determine the true first time, because itsFirstTime
- // can be a time value that does not coincide with a true time.
- // Note that a time stamp might be missing at this point,
- // so do not simply assume that mstime can be used.
- if (mstime > itsFirstTime) {
- int nrt = int((mstime - itsFirstTime) / itsTimeInterval);
- mstime -= (nrt+1) * itsTimeInterval; // Add 1 for rounding errors
- if (near(mstime, itsFirstTime)) {
- itsFirstTime = mstime;
- } else {
- itsFirstTime = mstime + itsTimeInterval;
- }
- break;
- }
- }
- // Skip this time slot.
- itsLastMSTime = mstime;
- itsIter.next();
- }
- }
-
- void MSReader::calcUVW (double time, DPBuffer& buf)
- {
- Matrix<double>& uvws = buf.getUVW();
- uvws.resize (3, itsNrBl);
- const Vector<Int>& ant1 = getInfo().getAnt1();
- const Vector<Int>& ant2 = getInfo().getAnt2();
- for (uint i=0; i<itsNrBl; ++i) {
- uvws.column(i) = itsUVWCalc.getUVW (ant1[i], ant2[i], time);
- }
- }
-
- void MSReader::getUVW (const RefRows& rowNrs, double time, DPBuffer& buf)
- {
- NSTimer::StartStop sstime(itsTimer);
- // Calculate UVWs if empty rownrs (i.e., missing data).
- if (rowNrs.rowVector().empty()) {
- calcUVW (time, buf);
- } else {
- ROArrayColumn<double> dataCol(itsMS, "UVW");
- dataCol.getColumnCells (rowNrs, buf.getUVW());
- }
- }
-
- void MSReader::getWeights (const RefRows& rowNrs, DPBuffer& buf)
- {
- NSTimer::StartStop sstime(itsTimer);
- Cube<float>& weights = buf.getWeights();
- // Resize if needed (probably when called for first time).
- if (weights.empty()) {
- weights.resize (itsNrCorr, itsNrChan, itsNrBl);
- }
- if (rowNrs.rowVector().empty()) {
- // rowNrs can be empty if a time slot was inserted.
- weights = 0;
- } else {
- // Get weights for entire spectrum if present.
- if (itsHasWeightSpectrum) {
- ROArrayColumn<float> wsCol(itsMS, itsWeightColName);
- // Using getColumnCells(rowNrs,itsColSlicer) fails for LofarStMan.
- // Hence work around it.
- if (itsUseAllChan) {
- wsCol.getColumnCells (rowNrs, weights);
- } else {
- Cube<float> w = wsCol.getColumnCells (rowNrs);
- weights = w(itsArrSlicer);
- }
- } else {
- // No spectrum present; get global weights and assign to each channel.
- ROArrayColumn<float> wCol(itsMS, "WEIGHT");
- Matrix<float> inArr = wCol.getColumnCells (rowNrs);
- float* inPtr = inArr.data();
- float* outPtr = weights.data();
- for (uint i=0; i<itsNrBl; ++i) {
- // Set global weights to 1 if zero. Some old MSs need that.
- for (uint k=0; k<itsNrCorr; ++k) {
- if (inPtr[k] == 0.) {
- inPtr[k] = 1.;
- }
- }
- for (uint j=0; j<itsNrChan; ++j) {
- for (uint k=0; k<itsNrCorr; ++k) {
- *outPtr++ = inPtr[k];
- }
- }
- inPtr += itsNrCorr;
- }
- }
- if (itsAutoWeight) {
- // Adapt weights using autocorrelations.
- autoWeight (weights, buf);
- }
- }
- }
-
- void MSReader::autoWeight (Cube<float>& weights, const DPBuffer& buf)
- {
- const double* chanWidths = getInfo().chanWidths().data();
- uint npol = weights.shape()[0];
- uint nchan = weights.shape()[1];
- uint nbl = weights.shape()[2];
- // Get the autocorrelations indices.
- const vector<int>& autoInx = getInfo().getAutoCorrIndex();
- // Calculate the weight for each cross-correlation data point.
- const Vector<Int>& ant1 = getInfo().getAnt1();
- const Vector<Int>& ant2 = getInfo().getAnt2();
- const Complex* data = buf.getData().data();
- float* weight = weights.data();
- for (uint i=0; i<nbl; ++i) {
- // Can only be done if both autocorrelations are present.
- if (ant1[i] != ant2[i] &&
- autoInx[ant1[i]] >= 0 && autoInx[ant2[i]] >= 0) {
- // Get offset of both autocorrelations in data array.
- const Complex* auto1 = data + autoInx[ant1[i]]*nchan*npol;
- const Complex* auto2 = data + autoInx[ant2[i]]*nchan*npol;
- for (uint j=0; j<nchan; ++j) {
- if (auto1[0].real() != 0 && auto2[0].real() != 0) {
- double w = chanWidths[j] * itsTimeInterval;
- weight[0] *= w / (auto1[0].real() * auto2[0].real()); // XX
- if (npol == 4) {
- if (auto1[3].real() != 0 && auto2[3].real() != 0) {
- weight[1] *= w / (auto1[0].real() * auto2[3].real()); // XY
- weight[2] *= w / (auto1[3].real() * auto2[0].real()); // YX
- weight[3] *= w / (auto1[3].real() * auto2[3].real()); // YY
- }
- } else if (npol == 2) {
- if (auto1[1].real() != 0 && auto2[1].real() != 0) {
- weight[1] *= w / (auto1[1].real() * auto2[1].real()); // YY
- }
- }
- }
- // Set pointers to next channel.
- weight += npol;
- auto1 += npol;
- auto2 += npol;
- }
- } else {
- // No autocorrelations for this baseline, so skip it.
- weight += nchan*npol;
- }
- }
- }
-
- bool MSReader::getFullResFlags (const RefRows& rowNrs, DPBuffer& buf)
- {
- NSTimer::StartStop sstime(itsTimer);
- Cube<bool>& flags = buf.getFullResFlags();
- int norigchan = itsNrChan * itsFullResNChanAvg;
- // Resize if needed (probably when called for first time).
- if (flags.empty()) {
- flags.resize (norigchan, itsFullResNTimeAvg, itsNrBl);
- }
- // Return false if no fullRes flags available.
- if (!itsHasFullResFlags) {
- flags = false;
- return false;
- }
- // Flag everything if data rows are missing.
- if (rowNrs.rowVector().empty()) {
- flags = true;
- return true;
- }
- ROArrayColumn<uChar> fullResFlagCol(itsMS, "LOFAR_FULL_RES_FLAG");
- int origstart = itsStartChan * itsFullResNChanAvg;
- Array<uChar> chars = fullResFlagCol.getColumnCells (rowNrs);
- // The original flags are kept per channel, not per corr.
- // Per row the flags are stored as uchar[nchar,navgtime].
- // Each char contains a bit per channel, thus nchan/8 chars are needed.
- // ntimeavg is the nr of times used when averaging.
- // Return it as Cube<bool>[norigchan,ntimeavg,nrbl].
- IPosition chShape = chars.shape();
- ASSERT (chShape[1] == itsFullResNTimeAvg && chShape[2] == itsNrBl);
- // Now expand the bits to bools.
- // If all bits to convert are contiguous, do it all in one go.
- // Otherwise we have to iterate.
- if (norigchan == chShape[0]*8) {
- Conversion::bitToBool (flags.data(), chars.data(), flags.size());
- } else {
- ASSERT (norigchan < chShape[0]*8);
- const uChar* charsPtr = chars.data();
- bool* flagsPtr = flags.data();
- for (int i=0; i<chShape[1]*chShape[2]; ++i) {
- Conversion::bitToBool (flagsPtr, charsPtr, origstart, norigchan);
- flagsPtr += norigchan;
- charsPtr += chShape[0];
- }
- }
- return true;
- }
-
- void MSReader::getModelData (const casacore::RefRows& rowNrs,
- casacore::Cube<casacore::Complex>& arr)
- {
- NSTimer::StartStop sstime(itsTimer);
- if (rowNrs.rowVector().empty()) {
- arr.resize (itsNrCorr, itsNrChan, itsNrBl);
- arr = Complex();
- } else {
- ROArrayColumn<Complex> modelCol(itsMS, itsModelColName);
- if (itsUseAllChan) {
- modelCol.getColumnCells (rowNrs, arr);
- } else {
- modelCol.getColumnCells (rowNrs, itsColSlicer, arr);
- }
- }
- }
-
- void MSReader::fillBeamInfo (vector<StationResponse::Station::Ptr>& vec,
- const Vector<String>& antNames)
- {
- // Get the names of all stations in the MS.
- const Vector<String>& allNames = getInfo().antennaNames();
- // Create a vector holding the beam info of all stations.
- vector<StationResponse::Station::Ptr> beams (allNames.size());
- StationResponse::readStations (itsMS, beams.begin());
- // Copy only the ones for which the station name matches.
- // Note: the order of the station names in both vectors match.
- vec.resize (antNames.size());
- uint ant = 0;
- for (uint i=0; i<allNames.size(); ++i) {
- if (ant < antNames.size() && allNames[i] == antNames[ant]) {
- vec[ant] = beams[i];
- ant++;
- }
- }
- ASSERTSTR (ant == vec.size(), "MSReader::fillBeamInfo -"
- " some stations miss the beam info");
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/MSUpdater.cc b/CEP/DP3/DPPP/src/MSUpdater.cc
deleted file mode 100644
index 5b4d321fbb5e62fc3079cad8b4a8f74374f47f49..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/MSUpdater.cc
+++ /dev/null
@@ -1,368 +0,0 @@
-//# MSUpdater.cc: DPPP step updating an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/MSUpdater.h>
-#include <DPPP/MSReader.h>
-#include <DPPP/MSWriter.h>
-#include <DPPP/DPBuffer.h>
-#include <Common/ParameterSet.h>
-#include <casacore/tables/Tables/Table.h>
-#include <casacore/tables/Tables/ArrayColumn.h>
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/tables/Tables/ArrColDesc.h>
-#include <casacore/tables/Tables/ColumnDesc.h>
-#include <casacore/tables/DataMan/StandardStMan.h>
-#include <casacore/casa/Containers/Record.h>
-#include <casacore/casa/Utilities/LinearSearch.h>
-#include <casacore/ms/MeasurementSets/MeasurementSet.h>
-#include <iostream>
-#include <limits>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- MSUpdater::MSUpdater (MSReader* reader, String msName,
- const ParameterSet& parset,
- const string& prefix, bool writeHistory)
- : itsReader (reader),
- itsName (prefix),
- itsMSName (msName),
- itsParset (parset),
- itsWriteData (false),
- itsWriteWeights (false),
- itsWriteFlags (false),
- itsNrDone (0),
- itsDataColAdded (false),
- itsWeightColAdded (false),
- itsWriteHistory (writeHistory)
- {
- itsDataColName = parset.getString (prefix+"datacolumn", "");
- itsWeightColName = parset.getString (prefix+"weightcolumn","");
- itsNrTimesFlush = parset.getUint (prefix+"flush", 0);
- itsTileSize = parset.getUint (prefix+"tilesize", 1024);
- itsStManKeys.Set(parset, prefix);
- }
-
- MSUpdater::~MSUpdater()
- {}
-
- bool MSUpdater::addColumn (const string& colName, const casacore::DataType
- dataType, const ColumnDesc& cd)
- {
- if (itsMS.tableDesc().isColumn(colName)) {
- const ColumnDesc& cd = itsMS.tableDesc().columnDesc(colName);
- ASSERTSTR (cd.dataType() == dataType && cd.isArray(),
- "Column " << itsDataColName
- << " already exists, but is not of the right type");
- return false;
- }
-
- if (itsStManKeys.stManName == "dysco" && itsStManKeys.dyscoDataBitRate != 0) {
- casacore::Record dyscoSpec = itsStManKeys.GetDyscoSpec();
- DataManagerCtor dyscoConstructor = DataManager::getCtor("DyscoStMan");
- CountedPtr<DataManager> dyscoStMan(dyscoConstructor(colName + "_dm", dyscoSpec));
- ColumnDesc directColumnDesc(cd);
- directColumnDesc.setOptions(casacore::ColumnDesc::Direct | casacore::ColumnDesc::FixedShape);
- TableDesc td;
- td.addColumn (directColumnDesc, colName);
- itsMS.addColumn (td, *dyscoStMan);
- }
- else {
- // When no specific storage manager is requested, use the same
- // as for the DATA column.
- // Get the data manager info and find the DATA column in it.
- Record dminfo = itsMS.dataManagerInfo();
- Record colinfo;
- for (uInt i=0; i<dminfo.nfields(); ++i) {
- const Record& subrec = dminfo.subRecord(i);
- if (linearSearch1 (Vector<String>(subrec.asArrayString("COLUMNS")),
- "DATA") >= 0) {
- colinfo = subrec;
- break;
- }
- }
- ASSERT(colinfo.nfields()>0);
- // When the storage manager is compressed, do not implicitly (re)compress it. Use TiledStMan instead.
- std::string dmType = colinfo.asString("TYPE");
- TableDesc td;
- td.addColumn (cd, colName);
- if(dmType == "DyscoStMan")
- {
- IPosition tileShape(3, info().ncorr(), info().nchan(), 1);
- tileShape[2] = itsTileSize * 1024 / (8 * tileShape[0] * tileShape[1]);
- if (tileShape[2] < 1) {
- tileShape[2] = 1;
- }
- TiledColumnStMan tsm(colName + "_dm", tileShape);
- itsMS.addColumn (td, tsm);
- }
- else {
- colinfo.define ("NAME", colName + "_dm");
- itsMS.addColumn (td, colinfo);
- }
- }
- return true;
- }
-
- bool MSUpdater::process (const DPBuffer& buf)
- {
- NSTimer::StartStop sstime(itsTimer);
- if (itsWriteFlags) {
- putFlags (buf.getRowNrs(), buf.getFlags());
- }
- if (itsWriteData) {
- // If compressing, flagged values need to be set to NaN to decrease the dynamic range
- if (itsStManKeys.stManName == "dysco") {
- Cube<Complex> dataCopy = buf.getData().copy();
- Cube<Complex>::iterator dataIter = dataCopy.begin();
- for (Cube<bool>::const_iterator flagIter = buf.getFlags().begin();
- flagIter != buf.getFlags().end(); ++flagIter) {
- if(*flagIter) {
- *dataIter = Complex(std::numeric_limits<float>::quiet_NaN(),
- std::numeric_limits<float>::quiet_NaN());
- }
- ++dataIter;
- }
- putData (buf.getRowNrs(), dataCopy);
- }
- else {
- putData (buf.getRowNrs(), buf.getData());
- }
- }
- if (itsWriteWeights) {
- Cube<float> weights;
- if (!buf.getWeights().empty()) {
- // Use weights from buffer
- weights = buf.getWeights();
- } else {
- itsBuffer.referenceFilled (buf);
- weights = itsReader->fetchWeights(buf, itsBuffer, itsTimer);
- }
-
- // If compressing, set weights of flagged points to zero to decrease the
- // dynamic range
- if (itsStManKeys.stManName == "dysco") {
- Cube<float> weightsCopy = weights.copy();
- Cube<float>::iterator weightsIter = weightsCopy.begin();
- for (Cube<bool>::const_iterator flagIter = buf.getFlags().begin();
- flagIter != buf.getFlags().end(); ++flagIter) {
- if(*flagIter) {
- *weightsIter = 0.;
- }
- ++weightsIter;
- }
- putWeights (buf.getRowNrs(), weightsCopy);
- } else {
- putWeights (buf.getRowNrs(), weights);
- }
- }
- itsNrDone++;
- if (itsNrTimesFlush > 0 && itsNrDone%itsNrTimesFlush == 0) {
- itsMS.flush();
- }
- getNextStep()->process(buf);
- return true;
- }
-
- void MSUpdater::finish()
- {}
-
- void MSUpdater::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- itsWriteFlags = getInfo().writeFlags();
-
- String origDataColName = getInfo().getDataColName();
- if (itsDataColName.empty()) {
- itsDataColName = origDataColName;
- } else if (itsDataColName != origDataColName) {
- info().setNeedVisData();
- info().setWriteData();
- }
- itsWriteData = getInfo().writeData();
-
- String origWeightColName = getInfo().getWeightColName();
- if (itsWeightColName.empty()) {
- if (origWeightColName == "WEIGHT") {
- itsWeightColName = "WEIGHT_SPECTRUM";
- } else {
- itsWeightColName = origWeightColName;
- }
- }
- ASSERT(itsWeightColName != "WEIGHT");
- if (itsWeightColName != origWeightColName) {
- info().setWriteWeights();
- }
- itsWriteWeights = getInfo().writeWeights();
-
- if (getInfo().metaChanged()) {
- THROW(Exception, "Update step " + itsName +
- " is not possible because meta data changes"
- " (by averaging, adding/removing stations, etc.)");
- }
-
- if (itsWriteData || itsWriteFlags || itsWriteWeights) {
- NSTimer::StartStop sstime(itsTimer);
- itsMS = MeasurementSet (itsMSName, TableLock::AutoNoReadLocking,
- Table::Update);
- // Add the data + weight column if needed and if it does not exist yet.
- if (itsWriteData) {
- // use same layout as DATA column
- ColumnDesc cd = itsMS.tableDesc().columnDesc("DATA");
- itsDataColAdded = addColumn(itsDataColName, TpComplex, cd);
- }
- if (itsWriteWeights) {
- IPosition dataShape =
- itsMS.tableDesc().columnDesc("DATA").shape();
- ArrayColumnDesc<float> cd("WEIGHT_SPECTRUM", "weight per corr/chan",
- dataShape, ColumnDesc::FixedShape);
- itsWeightColAdded = addColumn(itsWeightColName, TpFloat, cd);
- }
- }
- // Subsequent steps have to set again if writes need to be done.
- info().clearWrites();
- info().clearMetaChanged();
- // Tell the reader if visibility data needs to be read.
- itsReader->setReadVisData (info().needVisData());
- }
-
- void MSUpdater::addToMS (const string&)
- {
- getPrevStep()->addToMS (itsMSName);
- if (itsWriteHistory) {
- MSWriter::writeHistory (itsMS, itsParset);
- }
- }
-
- void MSUpdater::show (std::ostream& os) const
- {
- os << "MSUpdater " << itsName << std::endl;
- os << " MS: " << itsMSName << std::endl;
- os << " datacolumn: " << itsDataColName;
- if (itsDataColAdded) {
- os << " (has been added to the MS)";
- }
- os << std::endl;
- os << " weightcolumn " << itsWeightColName;
- if (itsWeightColAdded) {
- os << " (has been added to the MS)";
- }
- os << std::endl;
- if (itsWriteData || itsWriteFlags || itsWriteWeights) {
- os << " writing: ";
- if (itsWriteData) os << " data";
- if (itsWriteFlags) os << " flags";
- if (itsWriteWeights) os << " weights";
- os << std::endl;
- }
- if(itsStManKeys.stManName == "dysco") {
- os
- << " Compressed: yes\n"
- << " Data bitrate: " << itsStManKeys.dyscoDataBitRate << '\n'
- << " Weight bitrate: " << itsStManKeys.dyscoWeightBitRate << '\n'
- << " Dysco mode: " << itsStManKeys.dyscoNormalization << ' '
- << itsStManKeys.dyscoDistribution << '(' << itsStManKeys.dyscoDistTruncation << ")\n";
- }
- else {
- os << " Compressed: no\n";
- }
- os << std::endl;
- os << " flush: " << itsNrTimesFlush << std::endl;
- }
-
- void MSUpdater::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " MSUpdater " << itsName << endl;
- }
-
- void MSUpdater::putFlags (const RefRows& rowNrs,
- const Cube<bool>& flags)
- {
- // Only put if rownrs are filled, thus if data were not inserted.
- if (! rowNrs.rowVector().empty()) {
- Slicer colSlicer(IPosition(2, 0, info().startchan()),
- IPosition(2, info().ncorr(), info().nchan()) );
- ArrayColumn<bool> flagCol(itsMS, "FLAG");
- ScalarColumn<bool> flagRowCol(itsMS, "FLAG_ROW");
- // Loop over all rows of this subset.
- // (it also avoids StandardStMan putCol with RefRows problem).
- Vector<uint> rows = rowNrs.convert();
- ReadOnlyArrayIterator<bool> flagIter (flags, 2);
- for (uint i=0; i<rows.size(); ++i) {
- flagCol.putSlice (rows[i], colSlicer, flagIter.array());
- // If a new flag in a row is clear, the ROW_FLAG should not be set.
- // If all new flags are set, we leave it because we might have a
- // subset of the channels, so other flags might still be clear.
- if (anyEQ (flagIter.array(), False)) {
- flagRowCol.put (rows[i], False);
- }
- flagIter.next();
- }
- }
- }
-
- void MSUpdater::putWeights (const RefRows& rowNrs,
- const Cube<float>& weights)
- {
- // Only put if rownrs are filled, thus if data were not inserted.
- if (! rowNrs.rowVector().empty()) {
- Slicer colSlicer(IPosition(2, 0, info().startchan()),
- IPosition(2, info().ncorr(), info().nchan()) );
- ArrayColumn<float> weightCol(itsMS, itsWeightColName);
- // Loop over all rows of this subset.
- // (it also avoids StandardStMan putCol with RefRows problem).
- Vector<uint> rows = rowNrs.convert();
- ReadOnlyArrayIterator<float> weightIter (weights, 2);
- for (uint i=0; i<rows.size(); ++i) {
- weightCol.putSlice (rows[i], colSlicer, weightIter.array());
- weightIter.next();
- }
- }
- }
-
-
- void MSUpdater::putData (const RefRows& rowNrs,
- const Cube<Complex>& data)
- {
- // Only put if rownrs are filled, thus if data were not inserted.
- if (! rowNrs.rowVector().empty()) {
- Slicer colSlicer(IPosition(2, 0, info().startchan()),
- IPosition(2, info().ncorr(), info().nchan()) );
- ArrayColumn<Complex> dataCol(itsMS, itsDataColName);
- // Loop over all rows of this subset.
- // (it also avoids StandardStMan putCol with RefRows problem).
- Vector<uint> rows = rowNrs.convert();
- ReadOnlyArrayIterator<Complex> dataIter (data, 2);
- for (uint i=0; i<rows.size(); ++i) {
- dataCol.putSlice (rows[i], colSlicer, dataIter.array());
- dataIter.next();
- }
- }
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/MSWriter.cc b/CEP/DP3/DPPP/src/MSWriter.cc
deleted file mode 100644
index 0c6106cb67ef917e40bfd94828ceabb3e9fd6303..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/MSWriter.cc
+++ /dev/null
@@ -1,699 +0,0 @@
-//# MSWriter.cc: DPPP step writing to an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Package__Version.h>
-#include <DPPP/MSWriter.h>
-#include <DPPP/MSUpdater.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPLogger.h>
-#include <MS/VdsMaker.h>
-#include <Common/ParameterSet.h>
-#include <casacore/tables/Tables/TableCopy.h>
-#if defined(HAVE_CASACORE)
-#include <casacore/tables/DataMan/DataManInfo.h>
-#else
-#include <casacore/tables/DataMan/DataManInfo.h>
-#endif
-#include <casacore/tables/Tables/SetupNewTab.h>
-#include <casacore/tables/Tables/ArrColDesc.h>
-#include <casacore/tables/DataMan/StandardStMan.h>
-#include <casacore/tables/DataMan/TiledStManAccessor.h>
-#include <casacore/measures/TableMeasures/ArrayMeasColumn.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Containers/Record.h>
-#include <casacore/casa/OS/Path.h>
-#include <casacore/casa/version.h>
-#include <iostream>
-#include <limits>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- MSWriter::MSWriter (MSReader* reader, const string& outName,
- const ParameterSet& parset, const string& prefix)
- : itsReader (reader),
- itsName (prefix),
- itsOutName (outName),
- itsParset (parset),
- itsNrDone (0)
- {
- // Get tile size (default 1024 KBytes).
- itsTileSize = parset.getUint (prefix+"tilesize", 1024);
- itsTileNChan = parset.getUint (prefix+"tilenchan", 0);
- itsOverwrite = parset.getBool (prefix+"overwrite", false);
- itsNrTimesFlush = parset.getUint (prefix+"flush", 60);
- itsCopyCorrData = parset.getBool (prefix+"copycorrecteddata", false);
- itsCopyModelData = parset.getBool (prefix+"copymodeldata", false);
- itsWriteFullResFlags = parset.getBool (prefix+"writefullresflag", true);
- itsDataColName = parset.getString (prefix+"datacolumn", "DATA");
- itsWeightColName = parset.getString (prefix+"weightcolumn",
- "WEIGHT_SPECTRUM");
- itsVdsDir = parset.getString (prefix+"vdsdir", string());
- itsClusterDesc = parset.getString (prefix+"clusterdesc", string());
- ASSERTSTR (itsDataColName == "DATA", "Currently only the DATA column"
- " can be used as output when writing a new MS");
- ASSERTSTR (itsWeightColName == "WEIGHT_SPECTRUM", "Currently only the "
- "WEIGHT_SPECTRUM column can be used as output when writing a new MS");
-
- itsStManKeys.Set(parset, prefix);
- }
-
- MSWriter::~MSWriter()
- {}
-
- bool MSWriter::process (const DPBuffer& buf)
- {
- NSTimer::StartStop sstime(itsTimer);
- // Form the vector of the output table containing new rows.
- Vector<uint> rownrs(itsNrBl);
- indgen (rownrs, itsMS.nrow());
- // Add the necessary rows to the table.
- itsMS.addRow (itsNrBl);
- // Form the subset of the tables containing the rows.
- // It can happen that a missing slot was inserted. In that case
- // the rownr vector is empty and we use the first itsNrBl input rows.
- // Time related info can only be copied if not averaging and if the
- // the time slot was not missing.
- Table out(itsMS(rownrs));
- // Copy the input columns that do not change.
- writeMeta (out, buf);
- // Now write the data and flags.
- writeData (out, buf);
- // Flush if sufficient time slots are written.
- itsNrDone++;
- if (itsNrTimesFlush > 0 && itsNrDone%itsNrTimesFlush == 0) {
- itsMS.flush();
- }
- // Replace the rownrs in the buffer which is needed if in a later
- // step the MS gets updated.
- itsBuffer.setRowNrs (rownrs);
- getNextStep()->process(itsBuffer);
- return true;
- }
-
- void MSWriter::finish()
- {
- NSTimer::StartStop sstime(itsTimer);
- itsMS.flush();
- ///ROTiledStManAccessor acc1(itsMS, "TiledData");
- ///acc1.showCacheStatistics (cout);
- ///ROTiledStManAccessor acc2(itsMS, "TiledFlag");
- ///acc2.showCacheStatistics (cout);
- ///ROTiledStManAccessor acc3(itsMS, "TiledUVW");
- ///acc3.showCacheStatistics (cout);
- ///ROTiledStManAccessor acc4(itsMS, "TiledFullResFlag");
- ///acc4.showCacheStatistics (cout);
- // Create the VDS file.
- if (! itsClusterDesc.empty()) {
- string vdsName = itsMS.tableName() + ".vds";
- if (! itsVdsDir.empty()) {
- if (itsVdsDir[itsVdsDir.size() - 1] != '/') {
- itsVdsDir.append ("/");
- }
- vdsName = itsVdsDir + string(casacore::Path(vdsName).baseName());
- }
- // Create VDS file without detailed time info.
- LOFAR::VdsMaker::create (itsMS.tableName(), vdsName,
- itsClusterDesc, "", false);
- }
- }
-
- void MSWriter::addToMS (const string&)
- {
- getPrevStep()->addToMS(itsOutName);
- }
-
- void MSWriter::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- itsInterval = info().timeInterval();
- itsNrCorr = info().ncorr();
- itsNrChan = info().nchan();
- itsNrBl = info().nbaselines();
- itsNrTimes = info().ntime();
- // Input can already be averaged, so take that into account.
- itsNChanAvg = itsReader->nchanAvgFullRes() * info().nchanAvg();
- itsNTimeAvg = itsReader->ntimeAvgFullRes() * info().ntimeAvg();
- // Create the MS.
- if (itsTileNChan <= 0 || itsTileNChan > getInfo().nchan()) {
- itsTileNChan = getInfo().nchan();
- }
- NSTimer::StartStop sstime(itsTimer);
- createMS (itsOutName, info(), itsTileSize, itsTileNChan);
- // Write the parset info into the history.
- writeHistory (itsMS, itsParset);
- itsMS.flush (true, true);
- DPLOG_INFO ("Finished preparing output MS", false);
- info().clearWrites();
- info().clearMetaChanged();
- }
-
- void MSWriter::show (std::ostream& os) const
- {
- os << "MSWriter " << itsName << std::endl;
- os << " output MS: " << itsMS.tableName() << std::endl;
- os << " nchan: " << itsNrChan << std::endl;
- os << " ncorrelations: " << itsNrCorr << std::endl;
- os << " nbaselines: " << itsNrBl << std::endl;
- os << " ntimes: " << itsNrTimes << std::endl;
- os << " time interval: " << itsInterval << std::endl;
- os << " DATA column: " << itsDataColName << std::endl;
- os << " WEIGHT column: " << itsWeightColName << std::endl;
- if (itsStManKeys.stManName == "dysco") {
- os
- << " Compressed: yes\n"
- << " Data bitrate: " << itsStManKeys.dyscoDataBitRate << '\n'
- << " Weight bitrate: " << itsStManKeys.dyscoWeightBitRate << '\n'
- << " Dysco mode: " << itsStManKeys.dyscoNormalization << ' '
- << itsStManKeys.dyscoDistribution << '('
- << itsStManKeys.dyscoDistTruncation << ")\n";
- }
- else {
- os << " Compressed: no\n";
- }
- }
-
- void MSWriter::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " MSWriter " << itsName << endl;
- }
-
- void MSWriter::makeArrayColumn (ColumnDesc desc, const IPosition& ipos,
- DataManager* dm, Table& table, bool makeDirectColumn)
- {
- desc.setOptions(0);
- desc.setShape(ipos);
- if (makeDirectColumn) {
- desc.setOptions(ColumnDesc::Direct | ColumnDesc::FixedShape);
- }
- else {
- desc.setOptions(ColumnDesc::FixedShape);
- }
- if (table.tableDesc().isColumn(desc.name())) {
- table.removeColumn(desc.name());
- }
- // Use storage manager if given.
- if (dm == 0) {
- table.addColumn (desc);
- } else {
- table.addColumn (desc, *dm);
- }
- }
-
- void MSWriter::createMS (const string& outName, const DPInfo& info,
- uint tileSize, uint tileNChan)
- {
- // Determine the data shape.
- IPosition dataShape(2, itsNrCorr, itsNrChan);
- // Obtain the MS description.
- TableDesc tdesc (itsReader->table().tableDesc());
- // Create the output table without the columns depending
- // on the nr of channels.
- // FLAG_CATEGORY is taken, but ignored when writing.
- Block<String> fixedColumns(20);
- fixedColumns[0] = "UVW";
- fixedColumns[1] = "FLAG_CATEGORY";
- fixedColumns[2] = "WEIGHT";
- fixedColumns[3] = "SIGMA";
- fixedColumns[4] = "ANTENNA1";
- fixedColumns[5] = "ANTENNA2";
- fixedColumns[6] = "ARRAY_ID";
- fixedColumns[7] = "DATA_DESC_ID";
- fixedColumns[8] = "EXPOSURE";
- fixedColumns[9] = "FEED1";
- fixedColumns[10] = "FEED2";
- fixedColumns[11] = "FIELD_ID";
- fixedColumns[12] = "FLAG_ROW";
- fixedColumns[13] = "INTERVAL";
- fixedColumns[14] = "OBSERVATION_ID";
- fixedColumns[15] = "PROCESSOR_ID";
- fixedColumns[16] = "SCAN_NUMBER";
- fixedColumns[17] = "STATE_ID";
- fixedColumns[18] = "TIME";
- fixedColumns[19] = "TIME_CENTROID";
- Table temptable = itsReader->table().project(fixedColumns);
- TableDesc newdesc = temptable.tableDesc();
- // Now quite some 'magic' is done to get the storage managers right.
- // Most of the columns do not change much and should be stored with
- // the IncrementalStMan. The ANTENNA columns change more often and
- // can best be stored with StandardStMan.
- // The new storage managers are only used for MSs stored with LofarStMan.
- // For 'normal' MSs they won't change.
- //
- // If needed, make WEIGHT and SIGMA a fixed shape, direct column.
- // In this way they are not written if the values are the same.
- {
- ColumnDesc& cdesc = newdesc.rwColumnDesc("WEIGHT");
- if (cdesc.shape().empty()) {
- cdesc.setShape (IPosition(1,itsNrCorr), true);
- }
- }
- {
- ColumnDesc& cdesc = newdesc.rwColumnDesc("SIGMA");
- if (cdesc.shape().empty()) {
- cdesc.setShape (IPosition(1,itsNrCorr), true);
- }
- }
- // Remove possible hypercolumn definitions.
-// Test for casacore version 3.1.1 or smaller
-#if CASACORE_MAJOR_VERSION<3 || (CASACORE_MAJOR_VERSION==3 && (CASACORE_MINOR_VERSION==0 || (CASACORE_MINOR_VERSION==1 && CASACORE_PATCH_VERSION < 2)))
- newdesc.adjustHypercolumns (SimpleOrderedMap<String,String>(String()));
-#else
- newdesc.adjustHypercolumns (std::map<String,String>());
-#endif
- // Set data manager info.
- Record dminfo = temptable.dataManagerInfo();
- // Determine the DATA tile shape. Use all corrs and the given #channels.
- // The given tile size (in kbytes) determines the nr of rows in a tile .
- IPosition tileShape(3, itsNrCorr, tileNChan, 1);
- tileShape[2] = tileSize * 1024 / (8 * tileShape[0] * tileShape[1]);
- if (tileShape[2] < 1) {
- tileShape[2] = 1;
- }
- // Replace all non-writable storage managers (i.e. LofarStMan) by ISM.
- dminfo = DataManInfo::adjustStMan (dminfo, "IncrementalStMan");
- // Remove ANTENNA1 and ANTENNA2 from the dminfo.
- // Don't remove them if already stored with StandardStMan.
- Vector<String> removeCols(2);
- removeCols[0] = "ANTENNA1";
- removeCols[1] = "ANTENNA2";
- DataManInfo::removeDminfoColumns (dminfo, removeCols, "StandardStMan");
- // Use TiledStMan for UVW.
- // Use as many rows as used for the DATA columns, but minimal 1024.
- int tsmnrow = tileShape[2];
- if (tsmnrow < 1024) {
- tsmnrow = 1024;
- }
- DataManInfo::setTiledStMan (dminfo, Vector<String>(1, "UVW"),
- "TiledColumnStMan", "TiledUVW",
- IPosition(2, 3, tsmnrow));
- // Test if SSMVar already exists.
- bool hasSSMVar = false;
- for (uint i=0; i<dminfo.nfields(); ++i) {
- if (dminfo.subRecord(i).asString("NAME") == "SSMVar") {
- hasSSMVar = true;
- break;
- }
- }
- // Setup table creation. Exception is thrown if it exists already.
- Table::TableOption opt = itsOverwrite ? Table::New : Table::NewNoReplace;
- SetupNewTable newtab(outName, newdesc, opt);
-
- // First bind all columns to SSM.
- // For all columns defined in dminfo the bindings will be overwritten.
- // In this way variable columns like ANTENNA1/2 are bound to SSM.
- // Only do it if SSMVar does not exist (otherwise duplicate StMan name).
- if (!hasSSMVar) {
- StandardStMan ssm("SSMVar", 32768);
- newtab.bindAll (ssm);
- }
-
- // Bind all columns according to dminfo.
- newtab.bindCreate (dminfo);
- DataManagerCtor dyscoConstructor = 0;
- Record dyscoSpec;
- if (itsStManKeys.stManName == "dysco") {
- dyscoSpec = itsStManKeys.GetDyscoSpec();
- dyscoConstructor = DataManager::getCtor("DyscoStMan");
- }
- itsMS = Table(newtab);
-
- if (itsStManKeys.stManName == "dysco" && itsStManKeys.dyscoDataBitRate != 0) {
- // Add DATA column using Dysco stman.
- CountedPtr<DataManager> dyscoStMan(dyscoConstructor("DyscoData", dyscoSpec));
- makeArrayColumn (tdesc["DATA"], dataShape, dyscoStMan.get(), itsMS, true);
- }
- else {
- // Add DATA column using tsm.
- TiledColumnStMan tsm("TiledData", tileShape);
- makeArrayColumn (tdesc["DATA"], dataShape, &tsm, itsMS);
- }
-
- // Add FLAG column using tsm.
- // Use larger tile shape because flags are stored as bits.
- IPosition tileShapeF(tileShape);
- tileShapeF[2] *= 8;
- TiledColumnStMan tsmf("TiledFlag", tileShapeF);
- makeArrayColumn(tdesc["FLAG"], dataShape, &tsmf, itsMS);
-
- if (itsWriteFullResFlags) {
- // Add LOFAR_FULL_RES_FLAG column using tsm.
- // The input can already be averaged and averaging can be done in
- // this run, so the full resolution is the combination of both.
- uint orignchan = itsNrChan * itsNChanAvg;
- IPosition dataShapeF(2, (orignchan+7)/8, itsNTimeAvg);
- IPosition tileShapeF(3, (orignchan+7)/8, 1024, tileShape[2]);
- TiledColumnStMan tsmf("TiledFullResFlag", tileShapeF);
- ArrayColumnDesc<uChar> padesc("LOFAR_FULL_RES_FLAG",
- "flags in original full resolution",
- dataShapeF, ColumnDesc::FixedShape);
- makeArrayColumn(padesc, dataShapeF, &tsmf, itsMS);
- }
- if (itsStManKeys.stManName == "dysco" && itsStManKeys.dyscoWeightBitRate != 0) {
- // Add WEIGHT_SPECTRUM column using Dysco stman.
- CountedPtr<DataManager> dyscoStMan(dyscoConstructor(
- "DyscoWeightSpectrum", dyscoSpec)
- );
- ArrayColumnDesc<float> wsdesc("WEIGHT_SPECTRUM", "weight per corr/chan",
- dataShape,
- ColumnDesc::FixedShape | ColumnDesc::Direct);
- makeArrayColumn (wsdesc, dataShape, dyscoStMan.get(), itsMS, true);
- }
- else {
- // Add WEIGHT_SPECTRUM column using tsm.
- TiledColumnStMan tsmw("TiledWeightSpectrum", tileShape);
- ArrayColumnDesc<float> wsdesc("WEIGHT_SPECTRUM", "weight per corr/chan",
- dataShape, ColumnDesc::FixedShape);
- makeArrayColumn(wsdesc, dataShape, &tsmw, itsMS);
- }
- // If present handle the CORRECTED_DATA and MODEL_DATA column.
- if (!tdesc.isColumn("CORRECTED_DATA")) {
- itsCopyCorrData = false;
- }
- if (!tdesc.isColumn("MODEL_DATA")) {
- itsCopyModelData = false;
- }
- if (itsCopyCorrData) {
- TiledColumnStMan tsmc("CorrectedData", tileShape);
- makeArrayColumn(tdesc["CORRECTED_DATA"], dataShape, &tsmc,
- itsMS);
-
- IPosition iwShape(1, dataShape[1]);
- IPosition iwShapeTile(2, tileShape[1], tileShape[2]);
- TiledColumnStMan tsmw("TiledImagingWeight", iwShapeTile);
- ColumnDesc iwdesc(ArrayColumnDesc<float>("IMAGING_WEIGHT"));
- makeArrayColumn(iwdesc, iwShape, &tsmw, itsMS);
- }
- if (itsCopyModelData) {
- ColumnDesc mdesc = tdesc.columnDesc("MODEL_DATA");
- TableRecord& keyset = mdesc.rwKeywordSet();
- // Redefine possible keywords used by the CASA VisSet classes.
- if (keyset.isDefined("CHANNEL_SELECTION")) {
- keyset.removeField("CHANNEL_SELECTION");
- }
- Matrix<int> selection(2, 1);
- selection(0, 0) = 0;
- selection(1, 0) = itsNrChan;
- keyset.define("CHANNEL_SELECTION", selection);
- TiledColumnStMan tsmm("ModelData", tileShape);
- makeArrayColumn(mdesc, dataShape, &tsmm, itsMS);
- }
- DPLOG_INFO (" copying info and subtables ...", false);
- // Copy the info and subtables.
- TableCopy::copyInfo(itsMS, temptable);
- TableCopy::copySubTables(itsMS, temptable);
- // Adjust the SPECTRAL_WINDOW and DATA_DESCRIPTION table as needed.
- updateSpw (outName, info);
- // Adjust the OBSERVATION table as needed.
- updateObs (outName);
- // Adjust the FIELD table as needed.
- if (! info.phaseCenterIsOriginal()) {
- updateField (outName, info);
- }
- }
-
- void MSWriter::updateSpw (const string& outName, const DPInfo& info)
- {
- // Fix the SPECTRAL_WINDOW values by updating the values in the subtable.
- IPosition shape(1,itsNrChan);
- Table inSPW = itsReader->table().keywordSet().asTable("SPECTRAL_WINDOW");
- Table outSPW = Table(outName + "/SPECTRAL_WINDOW", Table::Update);
- Table outDD = Table(outName + "/DATA_DESCRIPTION", Table::Update);
- ASSERT (outSPW.nrow() == outDD.nrow());
- uint spw = itsReader->spectralWindow();
- // Remove all rows before and after the selected band.
- // Do it from the end, otherwise row numbers change.
- for (uint i=outSPW.nrow(); i>0;) {
- if (--i != spw) {
- outSPW.removeRow (i);
- outDD.removeRow (i);
- }
- }
- ASSERT (outSPW.nrow() == 1);
- // Set nr of channels.
- ScalarColumn<Int> channum(outSPW, "NUM_CHAN");
- channum.fillColumn (itsNrChan);
- // Change the column shapes.
- TableDesc tdesc = inSPW.tableDesc();
- makeArrayColumn (tdesc["CHAN_FREQ"], shape, 0, outSPW);
- makeArrayColumn (tdesc["CHAN_WIDTH"], shape, 0, outSPW);
- makeArrayColumn (tdesc["EFFECTIVE_BW"], shape, 0, outSPW);
- makeArrayColumn (tdesc["RESOLUTION"], shape, 0, outSPW);
- // Create the required column objects.
- ArrayColumn<Double> outFREQ(outSPW, "CHAN_FREQ");
- ArrayColumn<Double> outWIDTH(outSPW, "CHAN_WIDTH");
- ArrayColumn<Double> outBW(outSPW, "EFFECTIVE_BW");
- ArrayColumn<Double> outRESOLUTION(outSPW, "RESOLUTION");
- ScalarColumn<Double> outTOTALBW(outSPW, "TOTAL_BANDWIDTH");
- ScalarColumn<Double> outREFFREQ(outSPW, "REF_FREQUENCY");
- outFREQ.put (0, info.chanFreqs());
- outWIDTH.put (0, info.chanWidths());
- outBW.put (0, info.effectiveBW());
- outRESOLUTION.put(0, info.resolutions());
- outTOTALBW.put (0, info.totalBW());
- outREFFREQ.put (0, info.refFreq());
- // Adjust the spwid in the DATA_DESCRIPTION.
- ScalarColumn<Int> spwCol(outDD, "SPECTRAL_WINDOW_ID");
- spwCol.put (0, 0);
- }
-
- void MSWriter::updateObs (const string& outName)
- {
- Table outObs = Table(outName + "/OBSERVATION", Table::Update);
- // Set nr of channels.
- ArrayColumn<Double> timeRange(outObs, "TIME_RANGE");
- Vector<double> times(2);
- times[0] = itsReader->firstTime() - 0.5 * itsReader->getInfo().timeInterval();
- times[1] = itsReader->lastTime() + 0.5 * itsReader->getInfo().timeInterval();
- // There should be one row, but loop in case of.
- for (uint i=0; i<outObs.nrow(); ++i) {
- timeRange.put (i, times);
- }
- }
-
- void MSWriter::updateField (const string& outName, const DPInfo& info)
- {
- Table outField = Table(outName + "/FIELD", Table::Update);
- // Write new phase center.
- ArrayMeasColumn<MDirection> phaseCol (outField, "PHASE_DIR");
- Vector<MDirection> dir(1, info.phaseCenter());
- phaseCol.put (0, dir);
- }
-
- void MSWriter::writeHistory (Table& ms, const ParameterSet& parset)
- {
- Table histtab(ms.keywordSet().asTable("HISTORY"));
- histtab.reopenRW();
- ScalarColumn<Double> time (histtab, "TIME");
- ScalarColumn<Int> obsId (histtab, "OBSERVATION_ID");
- ScalarColumn<String> message (histtab, "MESSAGE");
- ScalarColumn<String> application (histtab, "APPLICATION");
- ScalarColumn<String> priority (histtab, "PRIORITY");
- ScalarColumn<String> origin (histtab, "ORIGIN");
- ArrayColumn<String> parms (histtab, "APP_PARAMS");
- ArrayColumn<String> cli (histtab, "CLI_COMMAND");
- // Put all parset entries in a Vector<String>.
- // Some WSRT MSs have a FixedShape APP_PARAMS and CLI_COMMAND column.
- // For them, put all params in a single vector element (with newlines).
- bool fixedShaped =
- (parms.columnDesc().options() & ColumnDesc::FixedShape) != 0;
- Vector<String> appvec;
- Vector<String> clivec;
- if (fixedShaped) {
- appvec.resize(1);
- clivec.resize(1);
- ostringstream ostr;
- parset.writeStream (ostr);
- appvec[0] = ostr.str();
- } else {
- appvec.resize (parset.size());
- Array<String>::contiter viter = appvec.cbegin();
- for (ParameterSet::const_iterator iter = parset.begin();
- iter != parset.end(); ++iter, ++viter) {
- *viter = iter->first + '=' + iter->second.get();
- }
- }
- uint rownr = histtab.nrow();
- histtab.addRow();
- time.put (rownr, Time().modifiedJulianDay()*24.*3600.);
- obsId.put (rownr, 0);
- message.put (rownr, "parameters");
- application.put (rownr, "NDPPP");
- priority.put (rownr, "NORMAL");
- origin.put (rownr, Version::getInfo<DPPPVersion>("DPPP", "full"));
- parms.put (rownr, appvec);
- cli.put (rownr, clivec);
- }
-
- void MSWriter::writeData (Table& out, const DPBuffer& buf)
- {
- ArrayColumn<Complex> dataCol(out, itsDataColName);
- ArrayColumn<Bool> flagCol(out, "FLAG");
- ScalarColumn<Bool> flagRowCol(out, "FLAG_ROW");
-
- if (buf.getData().empty()) {
- return;
- }
-
- // Write WEIGHT_SPECTRUM and DATA
- ArrayColumn<Float> weightCol(out, "WEIGHT_SPECTRUM");
- itsBuffer.referenceFilled (buf);
- const Array<Float>& weights = itsReader->fetchWeights (buf, itsBuffer,
- itsTimer);
-
- // If compressing, flagged values need to be set to NaN, and flagged
- // weights to zero, to decrease the dynamic range
- if (itsStManKeys.stManName == "dysco") {
- Cube<Complex> dataCopy = buf.getData().copy();
- Cube<Complex>::iterator dataIter = dataCopy.begin();
- Cube<float> weightsCopy = weights.copy();
- Cube<float>::iterator weightsIter = weightsCopy.begin();
- for (Cube<bool>::const_iterator flagIter = buf.getFlags().begin();
- flagIter != buf.getFlags().end(); ++flagIter) {
- if (*flagIter) {
- *dataIter = Complex(std::numeric_limits<float>::quiet_NaN(),
- std::numeric_limits<float>::quiet_NaN());
- *weightsIter = 0.;
- }
- ++dataIter;
- ++weightsIter;
- }
- dataCol.putColumn (dataCopy);
- weightCol.putColumn (weightsCopy);
- }
- else {
- dataCol.putColumn (buf.getData());
- weightCol.putColumn (weights);
- }
-
- flagCol.putColumn (buf.getFlags());
- // A row is flagged if no flags in the row are False.
- Vector<Bool> rowFlags (partialNFalse(buf.getFlags(), IPosition(2,0,1)) == 0u);
- flagRowCol.putColumn (rowFlags);
- if (itsWriteFullResFlags) {
- writeFullResFlags (out, buf);
- }
-
- // Write UVW
- ArrayColumn<Double> uvwCol(out, "UVW");
- const Array<Double>& uvws = itsReader->fetchUVW (buf, itsBuffer,
- itsTimer);
- uvwCol.putColumn (uvws);
- }
-
- void MSWriter::writeFullResFlags (Table& out, const DPBuffer& buf)
- {
- // Get the flags.
- const Cube<bool>& flags = itsReader->fetchFullResFlags (buf, itsBuffer,
- itsTimer);
- const IPosition& ofShape = flags.shape();
- ASSERTSTR (uint(ofShape[0]) == itsNChanAvg * itsNrChan,
- ofShape<<itsNChanAvg<<'*'<<itsNrChan);
- ASSERTSTR (uint(ofShape[1]) == itsNTimeAvg, ofShape<<itsNTimeAvg);
- // Convert the bools to uChar bits.
- IPosition chShape(ofShape);
- chShape[0] = (ofShape[0] + 7) / 8;
- Cube<uChar> chars(chShape);
- // If their sizes match, do it all in one go.
- // Otherwise we have to iterate.
- if (ofShape[0] == chShape[0]*8) {
- Conversion::boolToBit (chars.data(), flags.data(), flags.size());
- } else {
- ASSERT (ofShape[0] < chShape[0]*8);
- const bool* flagsPtr = flags.data();
- uChar* charsPtr = chars.data();
- for (int i=0; i<ofShape[1]*ofShape[2]; ++i) {
- Conversion::boolToBit (charsPtr, flagsPtr, ofShape[0]);
- flagsPtr += ofShape[0];
- charsPtr += chShape[0];
- }
- }
- ArrayColumn<uChar> fullResCol(out, "LOFAR_FULL_RES_FLAG");
- if (! fullResCol.keywordSet().isDefined ("NCHAN_AVG")) {
- fullResCol.rwKeywordSet().define ("NCHAN_AVG", int(itsNChanAvg));
- fullResCol.rwKeywordSet().define ("NTIME_AVG", int(itsNTimeAvg));
- }
- fullResCol.putColumn (chars);
- }
-
- void MSWriter::writeMeta (Table& out, const DPBuffer& buf)
- {
- // Fill ANTENNA1/2.
- ScalarColumn<Int> ant1col(out, "ANTENNA1");
- ScalarColumn<Int> ant2col(out, "ANTENNA2");
- ant1col.putColumn (getInfo().getAnt1());
- ant2col.putColumn (getInfo().getAnt2());
- // Fill all rows that do not change.
- fillSca<Double> (buf.getTime(), out, "TIME");
- fillSca<Double> (buf.getTime(), out, "TIME_CENTROID");
- fillSca<Double> (buf.getExposure(), out, "EXPOSURE");
- fillSca<Double> (itsInterval, out, "INTERVAL");
- fillSca<Int> (0, out, "FEED1");
- fillSca<Int> (0, out, "FEED2");
- fillSca<Int> (0, out, "DATA_DESC_ID");
- fillSca<Int> (0, out, "PROCESSOR_ID");
- fillSca<Int> (0, out, "FIELD_ID");
- fillSca<Int> (0, out, "SCAN_NUMBER");
- fillSca<Int> (0, out, "ARRAY_ID");
- fillSca<Int> (0, out, "OBSERVATION_ID");
- fillSca<Int> (0, out, "STATE_ID");
- Array<Float> arr(IPosition(1, itsNrCorr));
- arr = 1;
- fillArr<Float> (arr, out, "SIGMA");
- fillArr<Float> (arr, out, "WEIGHT");
- }
-
- void MSWriter::copyMeta (const Table& in, Table& out, bool copyTimeInfo)
- {
- // Copy all rows that do not change.
- copySca<int> (in, out, "ANTENNA1");
- copySca<int> (in, out, "ANTENNA2");
- copySca<int> (in, out, "FEED1");
- copySca<int> (in, out, "FEED2");
- copySca<int> (in, out, "PROCESSOR_ID");
- copySca<int> (in, out, "FIELD_ID");
- copySca<int> (in, out, "SCAN_NUMBER");
- copySca<int> (in, out, "ARRAY_ID");
- copySca<int> (in, out, "OBSERVATION_ID");
- copySca<int> (in, out, "STATE_ID");
- copyArr<float> (in, out, "SIGMA");
- copyArr<float> (in, out, "WEIGHT");
- if (copyTimeInfo) {
- copySca<double>(in, out, "TIME");
- copySca<double>(in, out, "TIME_CENTROID");
- copySca<double>(in, out, "INTERVAL");
- copySca<double>(in, out, "EXPOSURE");
- copyArr<double>(in, out, "UVW");
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/MedFlagger.cc b/CEP/DP3/DPPP/src/MedFlagger.cc
deleted file mode 100644
index 5bcfb915efb2686c91a15c4ffc7c8c6b53d7ec9e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/MedFlagger.cc
+++ /dev/null
@@ -1,527 +0,0 @@
-//# MedFlagger.cc: DPPP step class to flag data based on median filtering
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/MedFlagger.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StreamUtil.h>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Containers/Record.h>
-#include <casacore/casa/Containers/RecordField.h>
-#include <casacore/tables/TaQL/ExprNode.h>
-#include <casacore/tables/TaQL/RecordGram.h>
-#include <iostream>
-#include <algorithm>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- MedFlagger::MedFlagger (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsThresholdStr (parset.getString (prefix+"threshold", "1")),
- itsFreqWindowStr (parset.getString (prefix+"freqwindow", "1")),
- itsTimeWindowStr (parset.getString (prefix+"timewindow", "1")),
- itsNTimes (0),
- itsNTimesDone (0),
- itsFlagCounter (input->msName(), parset, prefix+"count."),
- itsMoveTime (0),
- itsMedianTime (0)
- {
- itsFlagCorr = parset.getUintVector (prefix+"correlations",
- vector<uint>());
- itsApplyAutoCorr = parset.getBool (prefix+"applyautocorr", false);
- itsMinBLength = parset.getDouble(prefix+"blmin", -1);
- itsMaxBLength = parset.getDouble(prefix+"blmax", 1e30);
- }
-
- MedFlagger::~MedFlagger()
- {}
-
- void MedFlagger::show (std::ostream& os) const
- {
- os << "MADFlagger " << itsName << std::endl;
- os << " freqwindow: " << itsFreqWindowStr
- << " (max = " << itsFreqWindow << ')' << std::endl;
- os << " timewindow: " << itsTimeWindowStr
- << " (max = " << itsTimeWindow << ')' << std::endl;
- os << " threshold: " << itsThresholdStr
- << " (max = " << itsThreshold << ')' << std::endl;
- os << " correlations: " << itsFlagCorr << std::endl;
- os << " applyautocorr: " << itsApplyAutoCorr
- << " (nautocorr = " << itsNrAutoCorr << ')' << std::endl;
- os << " blmin: " << itsMinBLength << " m" << std::endl;
- os << " blmax: " << itsMaxBLength << " m" << std::endl;
- }
-
- void MedFlagger::showCounts (std::ostream& os) const
- {
- os << endl << "Flags set by MADFlagger " << itsName;
- os << endl << "=======================" << endl;
- itsFlagCounter.showBaseline (os, itsNTimes);
- itsFlagCounter.showChannel (os, itsNTimes);
- itsFlagCounter.showCorrelation (os, itsNTimes);
- }
-
- void MedFlagger::showTimings (std::ostream& os, double duration) const
- {
- double flagDur = itsTimer.getElapsed();
- os << " ";
- FlagCounter::showPerc1 (os, flagDur, duration);
- os << " MADFlagger " << itsName << endl;
- os << " ";
- // move time and median time are sum of all threads.
- // Scale them to a single elapsed time.
- double factor = (itsComputeTimer.getElapsed() /
- (itsMoveTime + itsMedianTime));
- FlagCounter::showPerc1 (os, itsMoveTime*factor, flagDur);
- os << " of it spent in shuffling data" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, itsMedianTime*factor, flagDur);
- os << " of it spent in calculating medians" << endl;
- }
-
- void MedFlagger::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteFlags();
- // Get baseline indices of autocorrelations.
- itsAutoCorrIndex = info().getAutoCorrIndex();
- itsNrAutoCorr = 0;
- for (uint i=0; i<itsAutoCorrIndex.size(); ++i) {
- if (itsAutoCorrIndex[i] >= 0) {
- itsNrAutoCorr++;
- }
- }
- if (itsApplyAutoCorr) {
- ASSERTSTR (itsNrAutoCorr > 0, "applyautocorr=True cannot be used if "
- "the data does not contain autocorrelations");
- }
- // Calculate the baseline lengths.
- itsBLength = info().getBaselineLengths();
- // Evaluate the window size expressions.
- getExprValues (infoIn.nchan(), infoIn.ntime());
- itsBuf.resize (itsTimeWindow);
- itsAmpl.resize (itsTimeWindow);
- for (size_t i=0; i<itsAmpl.size(); ++i) {
- itsAmpl[i].resize (infoIn.ncorr(), infoIn.nchan(),
- infoIn.nbaselines());
- }
- // Set or check the correlations to flag on.
- vector<uint> flagCorr;
- uint ncorr = infoIn.ncorr();
- if (itsFlagCorr.empty()) {
- // No correlations given means use them all.
- for (uint i=0; i<ncorr; ++i) {
- flagCorr.push_back (i);
- }
- } else {
- for (uint i=0; i<itsFlagCorr.size(); ++i) {
- // Only take valid corrrelations.
- if (itsFlagCorr[i] < ncorr) {
- flagCorr.push_back (itsFlagCorr[i]);
- }
- }
- // If no valid left, use first one.
- if (flagCorr.empty()) {
- LOG_INFO_STR ("No valid correlations given in MedFlagger "
- << itsName << "; first one will be used");
- flagCorr.push_back (0);
- }
- }
- itsFlagCorr = flagCorr;
- // Initialize the flag counters.
- itsFlagCounter.init (getInfo());
- }
-
- bool MedFlagger::process (const DPBuffer& buf)
- {
- itsTimer.start();
- // Accumulate in the time window.
- // The buffer is wrapped, thus oldest entries are overwritten.
- uint index = itsNTimes % itsTimeWindow;
- itsBuf[index].copy (buf);
- DPBuffer& dbuf = itsBuf[index];
- // Calculate amplitudes if needed.
- amplitude (itsAmpl[index], dbuf.getData());
- // Fill flags if needed.
- if (dbuf.getFlags().empty()) {
- dbuf.getFlags().resize (dbuf.getData().shape());
- dbuf.getFlags() = false;
- }
- itsNTimes++;
- ///cout << "medproc: " << itsNTimes << endl;
- // Flag if there are enough time entries in the buffer.
- if (itsNTimes > itsTimeWindow/2) {
- // Fill the vector telling which time entries to use for the medians.
- // Arrange indices such that any width can be used; thus first center,
- // then one left and right, second left and right, etc.
- // If window not entirely full, use copies as needed.
- // This is done as follows:
- // Suppose timewindow=9 and we only have entries 0,1,2,3,4.
- // The entries are mirrored, thus we get 4,3,2,1,0,1,2,3,4
- // to obtain sufficient time entries.
- vector<uint> timeEntries;
- timeEntries.reserve (itsTimeWindow);
- ///uint rinx = itsNTimesDone % itsTimeWindow;
- ///timeEntries.push_back (rinx); // center
- ///uint linx = rinx;
- ///for (uint i=1; i<=itsTimeWindow/2; ++i) {
- ///if (linx == 0) linx = itsTimeWindow;
- ///linx--;
- ///rinx++;
- ///if (rinx == itsTimeWindow) rinx = 0;
- ///if (i >= itsNTimes) rinx = linx;
- ///timeEntries.push_back (linx);
- ///timeEntries.push_back (rinx);
- timeEntries.push_back (itsNTimesDone % itsTimeWindow); // center
- for (uint i=1; i<=itsTimeWindow/2; ++i) {
- timeEntries.push_back
- (std::abs(int(itsNTimesDone) - int(i)) % itsTimeWindow);
- timeEntries.push_back ((itsNTimesDone + i) % itsTimeWindow);
- }
- flag (itsNTimesDone%itsTimeWindow, timeEntries);
- itsNTimesDone++;
- }
- itsTimer.stop();
- return true;
- }
-
- void MedFlagger::finish()
- {
- itsTimer.start();
- // Adjust window size if there are fewer time entries.
- if (itsNTimes < itsTimeWindow) {
- itsTimeWindow = 1 + ((itsNTimes-1)/2)*2; // make sure it is odd
- }
- uint halfWindow = itsTimeWindow/2;
- vector<uint> timeEntries(itsTimeWindow);
- // Process possible leading entries.
- // This can happen if the window was larger than number of times.
- while (itsNTimesDone < itsNTimes-halfWindow) {
- // Process in the same way as in process.
- uint inx = 0;
- timeEntries[inx++] = itsNTimesDone % itsTimeWindow; // center
- for (uint i=1; i<=halfWindow; ++i) {
- timeEntries[inx++] =
- std::abs(int(itsNTimesDone) - int(i)) % itsTimeWindow;
- timeEntries[inx++] = (itsNTimesDone + i) % itsTimeWindow;
- }
- flag (itsNTimesDone, timeEntries);
- itsNTimesDone++;
- }
- ASSERT (itsNTimes - itsNTimesDone == halfWindow);
- // Process the remaining time entries.
- while (itsNTimesDone < itsNTimes) {
- uint inx = 0;
- timeEntries[inx++] = itsNTimesDone % itsTimeWindow; // center
- for (uint i=1; i<=halfWindow; ++i) {
- timeEntries[inx++] =
- std::abs(int(itsNTimesDone) - int(i)) % itsTimeWindow;
- // Time entries might need to be mirrored at the end.
- uint ri = itsNTimesDone + i;
- if (ri >= itsNTimes) {
- ri = 2*(itsNTimes-1) - ri;
- }
- timeEntries[inx++] = ri % itsTimeWindow;
- }
- flag (itsNTimesDone%itsTimeWindow, timeEntries);
- itsNTimesDone++;
- }
- itsTimer.stop();
- // Let the next step finish its processing.
- getNextStep()->finish();
- }
-
- void MedFlagger::flag (uint index, const vector<uint>& timeEntries)
- {
- ///cout << "flag: " <<itsNTimes<<' '<<itsNTimesDone<<' ' <<index << timeEntries << endl;
- // Get antenna numbers in case applyautocorr is true.
- const Vector<Int>& ant1 = getInfo().getAnt1();
- const Vector<Int>& ant2 = getInfo().getAnt2();
- // Result is 'copy' of the entry at the given time index.
- DPBuffer buf (itsBuf[index]);
- IPosition shp = buf.getData().shape();
- uint ncorr = shp[0];
- uint nchan = shp[1];
- uint blsize = ncorr*nchan;
- int nrbl = shp[2]; // OpenMP 2.5 needs signed iteration variables
- uint ntime = timeEntries.size();
- // Get pointers to data and flags.
- const float* bufDataPtr = itsAmpl[index].data();
- bool* bufFlagPtr = buf.getFlags().data();
- float MAD = 1.4826; //# constant determined by Pandey
- itsComputeTimer.start();
- // Now flag each baseline, channel and correlation for this time window.
- // This can be done in parallel.
-#pragma omp parallel
- {
- // Create a temporary buffer (per thread) to hold data for determining
- // the medians.
- // Also create thread-private counter and timer objects.
- Block<float> tempBuf(itsFreqWindow*ntime);
- FlagCounter counter;
- counter.init (getInfo());
- NSTimer moveTimer;
- NSTimer medianTimer;
- float Z1, Z2;
- // The for loop can be parallellized. This must be done dynamically,
- // because the execution time of each iteration can vary a lot.
-#pragma omp for schedule(dynamic)
- // GCC-4.3 only supports OpenMP 2.5 that needs signed iteration
- // variables.
- for (int ib=0; ib<nrbl; ++ib) {
- const float* dataPtr = bufDataPtr + ib*blsize;
- bool* flagPtr = bufFlagPtr + ib*blsize;
- double threshold = itsThresholdArr[ib];
- // Do only autocorrelations if told so.
- // Otherwise do baseline only if length within min-max.
- if ((!itsApplyAutoCorr && itsBLength[ib] >= itsMinBLength &&
- itsBLength[ib] <= itsMaxBLength) ||
- (itsApplyAutoCorr && ant1[ib] == ant2[ib])) {
- for (uint ic=0; ic<nchan; ++ic) {
- bool corrIsFlagged = false;
- // Iterate over given correlations.
- for (vector<uint>::const_iterator iter = itsFlagCorr.begin();
- iter != itsFlagCorr.end(); ++iter) {
- uint ip = *iter;
- // If one correlation is flagged, all of them will be flagged.
- // So no need to check others.
- if (flagPtr[ip]) {
- corrIsFlagged = true;
- break;
- }
- // Calculate values from the median.
- computeFactors (timeEntries, ib, ic, ip, nchan, ncorr,
- Z1, Z2, tempBuf.storage(),
- moveTimer, medianTimer);
- if (dataPtr[ip] > Z1 + threshold * Z2 * MAD) {
- corrIsFlagged = true;
- counter.incrBaseline(ib);
- counter.incrChannel(ic);
- counter.incrCorrelation(ip);
- break;
- }
- }
- if (corrIsFlagged) {
- for (uint ip=0; ip<ncorr; ++ip) {
- flagPtr[ip] = true;
- }
- }
- dataPtr += ncorr;
- flagPtr += ncorr;
- }
- } else {
- dataPtr += nchan*ncorr;
- flagPtr += nchan*ncorr;
- }
- } // end of OMP for
-#pragma omp critical(medflagger_updatecounts)
- {
- // Add the counters to the overall object.
- itsFlagCounter.add (counter);
- // Add the timings.
- itsMoveTime += moveTimer.getElapsed();
- itsMedianTime += medianTimer.getElapsed();
- } // end of OMP critical
- } // end of OMP parallel
-
- itsComputeTimer.stop();
- // Apply autocorrelations flags if needed.
- // Only to baselines with length within min-max.
- if (itsApplyAutoCorr) {
- for (int ib=0; ib<nrbl; ++ib) {
- bool* flagPtr = bufFlagPtr + ib*blsize;
- // Flag crosscorr if at least one autocorr is present.
- // Only if baseline length within min-max.
- if (ant1[ib] != ant2[ib] && itsBLength[ib] >= itsMinBLength &&
- itsBLength[ib] <= itsMaxBLength) {
- int inx1 = itsAutoCorrIndex[ant1[ib]];
- int inx2 = itsAutoCorrIndex[ant2[ib]];
- if (inx1 >= 0 || inx2 >= 0) {
- // Find flags of the autocorrelations of both antennae.
- // Use other autocorr if one autocorr does not exist.
- // In this way inx does not need to be tested in the inner loop.
- if (inx1 < 0) {
- inx1 = inx2;
- } else if (inx2 < 0) {
- inx2 = inx1;
- }
- bool* flagAnt1 = buf.getFlags().data() + inx1*nchan*ncorr;
- bool* flagAnt2 = buf.getFlags().data() + inx2*nchan*ncorr;
- // Flag if not flagged yet and if one of autocorr is flagged.
- for (uint ic=0; ic<nchan; ++ic) {
- if (!*flagPtr && (*flagAnt1 || *flagAnt2)) {
- for (uint ip=0; ip<ncorr; ++ip) {
- flagPtr[ip] = true;
- }
- itsFlagCounter.incrBaseline(ib);
- itsFlagCounter.incrChannel(ic);
- }
- flagPtr += ncorr;
- flagAnt1 += ncorr;
- flagAnt2 += ncorr;
- }
- }
- } else {
- flagPtr += nchan*ncorr;
- }
- }
- }
- // Process the result in the next step.
- itsTimer.stop();
- getNextStep()->process (buf);
- itsTimer.start();
- }
-
- void MedFlagger::computeFactors (const vector<uint>& timeEntries,
- uint bl, int chan, int corr,
- int nchan, int ncorr,
- float& Z1, float& Z2,
- float* tempBuf,
- NSTimer& moveTimer, NSTimer& medianTimer)
- {
- moveTimer.start();
- // Collect all non-flagged data points for given baseline, channel,
- // and correlation in the window around the channel.
- uint np = 0;
- // At the beginning or end of the window the values are wrapped.
- // So we might need to move in two parts.
- // This little piece of code is tested in tMirror.cc.
- int hw = itsFreqWindowArr[bl]/2;
- int s1 = chan - hw;
- int e1 = chan + hw + 1;
- int s2 = 1;
- int e2 = 1;
- if (s1 < 0) {
- e2 = -s1 + 1;
- s1 = 0;
- } else if (e1 > nchan) {
- s2 = nchan + nchan - e1 - 1; // e1-nchan+1 too far, go back that amount
- e2 = nchan-1;
- e1 = nchan;
- }
- // Iterate over all time entries.
- const uint* iter = &(timeEntries[0]);
- const uint* endIter = iter + itsTimeWindowArr[bl];
- for (; iter!=endIter; ++iter) {
- const DPBuffer& inbuf = itsBuf[*iter];
- const Cube<float>& ampl = itsAmpl[*iter];
- // Get pointers to given baseline and correlation.
- uint offset = bl*nchan*ncorr + corr;
- const float* dataPtr = ampl.data() + offset;
- const bool* flagPtr = inbuf.getFlags().data() + offset;
- // Now move data from the two channel parts.
- for (int i=s1*ncorr; i<e1*ncorr; i+=ncorr) {
- if (!flagPtr[i]) {
- tempBuf[np++] = dataPtr[i];
- }
- }
- for (int i=s2*ncorr; i<e2*ncorr; i+=ncorr) {
- if (!flagPtr[i]) {
- tempBuf[np++] = dataPtr[i];
- }
- }
- }
- moveTimer.stop();
- // If only flagged data, don't do anything.
- if (np == 0) {
- Z1 = -1.0;
- Z2 = 0.0;
- } else {
- medianTimer.start();
- // Get median of data and get median of absolute difference.
- ///std::nth_element (tempBuf, tempBuf+np/2, tempBuf+np);
- ///Z1 = *(tempBuf+np/2);
- Z1 = GenSort<float>::kthLargest (tempBuf, np, np/2);
- for (uint i=0; i<np; ++i) {
- tempBuf[i] = std::abs(tempBuf[i] - Z1);
- }
- ///std::nth_element (tempBuf, tempBuf+np/2, tempBuf+np);
- ///Z2 = *(tempBuf+np/2);
- Z2 = GenSort<float>::kthLargest (tempBuf, np, np/2);
- medianTimer.stop();
- }
- }
-
- void MedFlagger::getExprValues (int maxNChan, int maxNTime)
- {
- // Parse the expressions.
- // Baseline length can be used as 'bl' in the expressions.
- Record rec;
- rec.define ("bl", double(0));
- TableExprNode node1 (RecordGram::parse(rec, itsFreqWindowStr));
- TableExprNode node2 (RecordGram::parse(rec, itsTimeWindowStr));
- TableExprNode node3 (RecordGram::parse(rec, itsThresholdStr));
- // Size the arrays.
- uInt nrbl = itsBLength.size();
- itsThresholdArr.reserve (nrbl);
- itsTimeWindowArr.reserve (nrbl);
- itsFreqWindowArr.reserve (nrbl);
- itsFreqWindow = 0;
- itsTimeWindow = 0;
- itsThreshold = -1e30;
- // Evaluate the expression for each baseline.
- double result;
- RecordFieldPtr<double> blref(rec, "bl");
- for (uint i=0; i<nrbl; ++i) {
- // Put the length of each baseline in the record used to evaluate.
- *blref = itsBLength[i];
- // Evaluate freqwindow size and make it odd if needed.
- node1.get (rec, result);
- int freqWindow = std::min (std::max(1, int(result+0.5)), maxNChan);
- if (freqWindow%2 == 0) {
- freqWindow--;
- }
- itsFreqWindowArr.push_back (freqWindow);
- itsFreqWindow = std::max(itsFreqWindow, uint(freqWindow));
- // Evaluate timewindow size and make it odd if needed.
- node2.get (rec, result);
- int timeWindow = std::max(1, int(result+0.5));
- if (maxNTime > 0 && timeWindow > maxNTime) {
- timeWindow = maxNTime;
- }
- if (timeWindow%2 == 0) {
- timeWindow--;
- }
- itsTimeWindowArr.push_back (timeWindow);
- itsTimeWindow = std::max (itsTimeWindow, uint(timeWindow));
- // Evaluate threshold.
- node3.get (rec, result);
- itsThresholdArr.push_back (result);
- if (result > itsThreshold) {
- itsThreshold = result;
- }
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/ModelComponent.cc b/CEP/DP3/DPPP/src/ModelComponent.cc
deleted file mode 100644
index 1bd8b0ad4580dac9562e2392f6405df76f6fbbee..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/ModelComponent.cc
+++ /dev/null
@@ -1,36 +0,0 @@
-//# ModelComponent.cc: Base class for model components.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/ModelComponent.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-ModelComponent::~ModelComponent()
-{
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/ModelComponentVisitor.cc b/CEP/DP3/DPPP/src/ModelComponentVisitor.cc
deleted file mode 100644
index 3cd0eb8a51a1f15d6b82ec1d10d2c2d23d905727..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/ModelComponentVisitor.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-//# ModelComponentVisitor.cc: Base class for visitors that visit model component
-//# hierarchies.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/ModelComponentVisitor.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-ModelComponentVisitor::~ModelComponentVisitor()
-{
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/MultiMSReader.cc b/CEP/DP3/DPPP/src/MultiMSReader.cc
deleted file mode 100644
index f62cdf9becd918fa214a922ab99c004da384d05d..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/MultiMSReader.cc
+++ /dev/null
@@ -1,457 +0,0 @@
-//# MultiMSReader.cc: DPPP step reading from multiple MSs
-//# Copyright (C) 2011
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: MSReader.cc 19257 2011-11-14 14:36:08Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/MultiMSReader.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPLogger.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StreamUtil.h>
-#include <Common/LofarLogger.h>
-#include <casacore/tables/Tables/TableRecord.h>
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/tables/Tables/ArrayColumn.h>
-#include <casacore/tables/TaQL/ExprNode.h>
-#include <casacore/tables/TaQL/RecordGram.h>
-#include <casacore/measures/Measures/MeasTable.h>
-#include <casacore/measures/TableMeasures/ScalarMeasColumn.h>
-#include <casacore/measures/TableMeasures/ArrayMeasColumn.h>
-#include <casacore/casa/Containers/Record.h>
-#include <casacore/casa/Quanta/MVTime.h>
-#include <casacore/casa/Utilities/GenSort.h>
-#include <casacore/casa/OS/Conversion.h>
-#include <iostream>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- MultiMSReader::MultiMSReader (const vector<string>& msNames,
- const ParameterSet& parset,
- const string& prefix)
- : itsFirst (-1),
- itsNMissing (0),
- itsMSNames (msNames),
- itsRegularChannels (true)
- {
- ASSERTSTR (msNames.size() > 0, "No names of MeasurementSets given");
- itsMSName = itsMSNames[0];
- itsStartChanStr = parset.getString (prefix+"startchan", "0");
- itsNrChanStr = parset.getString (prefix+"nchan", "0");
- itsUseFlags = parset.getBool (prefix+"useflag", true);
- itsDataColName = parset.getString (prefix+"datacolumn", "DATA");
- itsWeightColName = parset.getString (prefix+"weightcolumn",
- "WEIGHT_SPECTRUM"),
- itsMissingData = parset.getBool (prefix+"missingdata", false);
- itsAutoWeight = parset.getBool (prefix+"autoweight", false);
- itsNeedSort = parset.getBool (prefix+"sort", false);
- itsOrderMS = parset.getBool (prefix+"orderms", true);
- // Open all MSs.
- DPStep::ShPtr nullStep (new NullStep());
- itsReaders.reserve (msNames.size());
- itsSteps.reserve (msNames.size());
- for (uint i=0; i<msNames.size(); ++i) {
- itsReaders.push_back (new MSReader (msNames[i], parset, prefix,
- itsMissingData));
- // itsSteps takes care of deletion of the MSReader object.
- itsSteps.push_back (DPStep::ShPtr(itsReaders[i]));
- // Add a null step for the reader.
- itsSteps[i]->setNextStep (nullStep);
- // Ignore if the MS is missing.
- if (itsReaders[i]->table().isNull()) {
- itsReaders[i] = 0;
- itsNMissing++;
- } else if (itsFirst < 0) {
- itsFirst = i;
- }
- }
-
- // TODO: check if frequencies are regular, insert some empy readers
- // if necessary
-
- ASSERTSTR (itsFirst>=0, "All input MeasurementSets do not exist");
- itsBuffers.resize (itsReaders.size());
- }
-
- MultiMSReader::~MultiMSReader()
- {}
-
- void MultiMSReader::setReadVisData (bool readVisData)
- {
- itsReadVisData = readVisData;
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- itsReaders[i]->setReadVisData (readVisData);
- }
- }
- }
-
- void MultiMSReader::handleBands()
- {
- if (itsNMissing > 0) {
- fillBands();
- return;
- }
- if (itsOrderMS) {
- sortBands();
- }
-
- // Collect the channel info of all MSs.
- Vector<double> chanFreqs (itsNrChan);
- Vector<double> chanWidths (itsNrChan);
- Vector<double> resolutions(itsNrChan);
- Vector<double> effectiveBW(itsNrChan);
- uint inx = 0;
- for (uint i=0; i<itsReaders.size(); ++i) {
- uint nchan = itsReaders[i]->getInfo().nchan();
- objcopy (chanFreqs.data() + inx,
- itsReaders[i]->getInfo().chanFreqs().data(), nchan);
- objcopy (chanWidths.data() + inx,
- itsReaders[i]->getInfo().chanWidths().data(), nchan);
- objcopy (resolutions.data() + inx,
- itsReaders[i]->getInfo().resolutions().data(), nchan);
- objcopy (effectiveBW.data() + inx,
- itsReaders[i]->getInfo().effectiveBW().data(), nchan);
- inx += nchan;
- }
- info().set (chanFreqs, chanWidths, resolutions, effectiveBW, 0., 0.);
- }
-
- void MultiMSReader::sortBands()
- {
- // Order the bands (MSs) in order of frequency.
- int nband = itsReaders.size();
- Vector<double> freqs(nband);
- for (int i=0; i<nband; ++i) {
- freqs[i] = itsReaders[i]->getInfo().chanFreqs().data()[0];
- }
- Vector<uInt> index;
- GenSortIndirect<double>::sort (index, freqs);
- vector<MSReader*> oldReaders (itsReaders);
- for (int i=0; i<nband; ++i) {
- itsReaders[i] = oldReaders[index[i]];
- }
- }
-
- void MultiMSReader::fillBands()
- {
- ASSERTSTR (!itsOrderMS, "Cannot order the MSs if some are missing");
- // Get channel width (which should be the same for all bands).
- double chanw = itsReaders[itsFirst]->getInfo().chanWidths().data()[0];
- // Get frequency for first subband.
- double freq = itsReaders[itsFirst]->getInfo().chanFreqs().data()[0];
- freq -= itsFirst*itsFillNChan*chanw;
- // Add missing channels to the total nr.
- itsNrChan += itsNMissing*itsFillNChan;
- // Collect the channel info of all MSs.
- Vector<double> chanFreqs (itsNrChan);
- Vector<double> chanWidths(itsNrChan);
- uint inx = 0;
- // Data for a missing MS can only be inserted if all other MSs have
- // the same nr of channels and are in increasing order of freq.
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- ASSERTSTR (itsReaders[i]->getInfo().nchan() == itsFillNChan,
- "An MS is missing; the others should have equal nchan");
- // Check if all channels have the same width and are consecutive.
- const Vector<double>& freqs = itsReaders[i]->getInfo().chanFreqs();
- const Vector<double>& width = itsReaders[i]->getInfo().chanWidths();
- ASSERTSTR (freqs[0] > freq || near(freqs[0], freq, 1e-5),
- "Subbands should be in increasing order of frequency; found "
- << freqs[0] << ", expected " << freq << " (diff="
- << freqs[0]-freq << ')');
- freq = freqs[itsFillNChan-1] + width[itsFillNChan-1];
- objcopy (chanFreqs.data() + inx, freqs.data(), itsFillNChan);
- objcopy (chanWidths.data() + inx, width.data(), itsFillNChan);
- inx += itsFillNChan;
- } else {
- // Insert channel info for missing MSs.
- for (uint j=0; j<itsFillNChan; ++j) {
- chanFreqs[inx] = freq;
- chanWidths[inx] = chanw;
- freq += chanw;
- inx++;
- }
- }
- }
-
- info().set (chanFreqs, chanWidths);
- }
-
- bool MultiMSReader::process (const DPBuffer& buf)
- {
- // Stop if at end.
- if (! itsReaders[itsFirst]->process (buf)) {
- return false; // end of input
- }
- const DPBuffer& buf1 = itsReaders[itsFirst]->getBuffer();
- itsBuffer.setTime (buf1.getTime());
- itsBuffer.setExposure (buf1.getExposure());
- itsBuffer.setRowNrs (buf1.getRowNrs());
- // Size the buffers.
- if (itsBuffer.getFlags().empty()) {
- if (itsReadVisData) {
- itsBuffer.getData().resize (IPosition(3, itsNrCorr,
- itsNrChan, itsNrBl));
- }
- itsBuffer.getFlags().resize (IPosition(3, itsNrCorr,
- itsNrChan, itsNrBl));
- }
- // Loop through all readers and get data and flags.
- IPosition s(3, 0, 0, 0);
- IPosition e(3, itsNrCorr-1, 0, itsNrBl-1);
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- if (int(i) != itsFirst) {
- itsReaders[i]->process (buf);
- }
- const DPBuffer& msBuf = itsReaders[i]->getBuffer();
- ASSERTSTR (! msBuf.getRowNrs().empty(),
- "When using multiple MSs, the times in all MSs have to be "
- "consecutive; this is not the case for MS " << i);
- // Copy data and flags.
- e[1] = s[1] + itsReaders[i]->getInfo().nchan() - 1;
- if (itsReadVisData) {
- itsBuffer.getData()(s,e) = msBuf.getData();
- }
- itsBuffer.getFlags()(s,e) = msBuf.getFlags();
- } else {
- e[1] = s[1] + itsFillNChan - 1;
- if (itsReadVisData) {
- itsBuffer.getData()(s,e) = Complex();
- }
- itsBuffer.getFlags()(s,e) = true;
- }
- s[1] = e[1] + 1;
- }
- getNextStep()->process (itsBuffer);
- return true;
- }
-
- void MultiMSReader::finish()
- {
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- itsReaders[i]->finish();
- }
- }
- getNextStep()->finish();
- }
-
- void MultiMSReader::updateInfo (const DPInfo& infoIn)
- {
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- itsReaders[i]->updateInfo (infoIn);
- }
- }
- info() = itsReaders[itsFirst]->getInfo();
- // Use the first valid MS as the standard MS (for meta data)
- // Get meta data and check they are equal for all MSs.
- itsMS = itsReaders[itsFirst]->table();
- itsStartTime = getInfo().startTime();
- itsFirstTime = itsReaders[itsFirst]->firstTime();
- itsLastTime = itsReaders[itsFirst]->lastTime();
- itsTimeInterval = getInfo().timeInterval();
- itsSelBL = itsReaders[itsFirst]->baselineSelection();
- itsSpw = itsReaders[itsFirst]->spectralWindow();
- itsNrCorr = getInfo().ncorr();
- itsNrBl = getInfo().nbaselines();
- itsNrChan = 0;
- itsFillNChan = getInfo().nchan();
- itsStartChan = itsReaders[itsFirst]->startChan();
- itsFullResNChanAvg = itsReaders[itsFirst]->nchanAvgFullRes();
- itsFullResNTimeAvg = itsReaders[itsFirst]->ntimeAvgFullRes();
- itsHasFullResFlags = itsReaders[itsFirst]->hasFullResFlags();
- itsBaseRowNrs = itsReaders[itsFirst]->getBaseRowNrs();
- for (uint i=0; i<itsMSNames.size(); ++i) {
- if (itsReaders[i]) {
- const DPInfo& rdinfo = itsReaders[i]->getInfo();
- ASSERTSTR (near(itsStartTime, rdinfo.startTime()) &&
- near(itsLastTime, itsReaders[i]->lastTime()) &&
- near(itsTimeInterval, rdinfo.timeInterval()) &&
- itsNrCorr == rdinfo.ncorr() &&
- itsNrBl == rdinfo.nbaselines() &&
- itsFullResNChanAvg == itsReaders[i]->nchanAvgFullRes() &&
- itsFullResNTimeAvg == itsReaders[i]->ntimeAvgFullRes() &&
- getInfo().antennaSet() == rdinfo.antennaSet() &&
- allEQ (getInfo().getAnt1(), rdinfo.getAnt1()) &&
- allEQ (getInfo().getAnt2(), rdinfo.getAnt2()),
- "Meta data of MS " << itsMSNames[i]
- << " differs from " << itsMSNames[itsFirst]);
- itsNrChan += rdinfo.nchan();
- itsHasFullResFlags = (itsHasFullResFlags &&
- itsReaders[i]->hasFullResFlags());
- }
- }
- // Handle the bands and take care of missing MSs.
- // Sort them if needed.
- handleBands();
-
- // check that channels are regularly spaced, give warning otherwise
- if (itsNrChan>1) {
- Vector<Double> upFreq = info().chanFreqs()(
- Slicer(IPosition(1,1),
- IPosition(1,itsNrChan-1)));
- Vector<Double> lowFreq = info().chanFreqs()(
- Slicer(IPosition(1,0),
- IPosition(1,itsNrChan-1)));
- Double freqstep0=upFreq(0)-lowFreq(0);
- // Compare up to 1kHz accuracy
- itsRegularChannels=allNearAbs(upFreq-lowFreq, freqstep0, 1.e3) &&
- allNearAbs(info().chanWidths(),
- info().chanWidths()(0), 1.e3);
- }
-
- // Set correct nr of channels.
- info().setNChan (itsNrChan);
- // Initialize the flag counters.
- itsFlagCounter.init (getInfo());
- }
-
- void MultiMSReader::show (std::ostream& os) const
- {
- os << "MultiMSReader" << std::endl;
- os << " input MSs: " << itsMSNames[0] << std::endl;
- for (uint i=1; i<itsMSNames.size(); ++i) {
- os << " " << itsMSNames[i] << std::endl;
- }
- if (! itsSelBL.empty()) {
- os << " baseline: " << itsSelBL << std::endl;
- }
- os << " band " << itsSpw << std::endl;
- os << " startchan: " << itsStartChan << " (" << itsStartChanStr
- << ')' << std::endl;
- os << " nchan: " << itsNrChan << " (" << itsNrChanStr
- << ')';
- if (itsRegularChannels) {
- os <<" (regularly spaced)" << std::endl;
- } else {
- os <<" (NOT regularly spaced)" << std::endl;
- }
- os << " ncorrelations: " << itsNrCorr << std::endl;
- os << " nbaselines: " << itsNrBl << std::endl;
- os << " ntimes: " << itsMS.nrow() / itsNrBl << std::endl;
- os << " time interval: " << itsTimeInterval << std::endl;
- os << " DATA column: " << itsDataColName << std::endl;
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- if (itsReaders[i]->missingData()) {
- os << " column missing in " << itsMSNames[i] << std::endl;
- }
- } else {
- os << " MS missing " << itsMSNames[i] << std::endl;
- }
- }
- os << " WEIGHT column: " << itsWeightColName << std::endl;
- os << " autoweight: " << boolalpha << itsAutoWeight << std::endl;
- }
-
- void MultiMSReader::showCounts (std::ostream& os) const
- {
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- itsReaders[i]->showCounts (os);
- }
- }
- }
-
- void MultiMSReader::showTimings (std::ostream& os, double duration) const
- {
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- itsReaders[i]->showTimings (os, duration);
- }
- }
- }
-
- void MultiMSReader::getUVW (const RefRows& rowNrs,
- double time, DPBuffer& buf)
- {
- // All MSs have the same UVWs, so use first one.
- itsReaders[itsFirst]->getUVW (rowNrs, time, buf);
- }
-
- void MultiMSReader::getWeights (const RefRows& rowNrs, DPBuffer& buf)
- {
- Cube<float>& weights = buf.getWeights();
- // Resize if needed (probably when called for first time).
- if (weights.empty()) {
- weights.resize (itsNrCorr, itsNrChan, itsNrBl);
- }
- IPosition s(3, 0, 0, 0);
- IPosition e(3, itsNrCorr-1, 0, itsNrBl-1);
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- uint nchan = itsReaders[i]->getInfo().nchan();
- e[1] = s[1] + nchan-1;
- itsReaders[i]->getWeights (rowNrs, itsBuffers[i]);
- weights(s,e) = itsBuffers[i].getWeights();
- } else {
- e[1] = s[1] + itsFillNChan-1;
- weights(s,e) = float(0);
- }
- s[1] = e[1] + 1;
- }
- }
-
- bool MultiMSReader::getFullResFlags (const RefRows& rowNrs,
- DPBuffer& buf)
- {
- Cube<bool>& flags = buf.getFullResFlags();
- // Resize if needed (probably when called for first time).
- if (flags.empty()) {
- int norigchan = itsNrChan * itsFullResNChanAvg;
- flags.resize (norigchan, itsFullResNTimeAvg, itsNrBl);
- }
- // Return false if no fullRes flags available.
- if (!itsHasFullResFlags) {
- flags = false;
- return false;
- }
- // Flag everything if data rows are missing.
- if (rowNrs.rowVector().empty()) {
- flags = true;
- return true;
- }
- // Get the flags from all MSs and combine them.
- IPosition s(3, 0);
- IPosition e(flags.shape() - 1);
- for (uint i=0; i<itsReaders.size(); ++i) {
- if (itsReaders[i]) {
- itsReaders[i]->getFullResFlags (rowNrs, itsBuffers[i]);
- e[0] = s[0] + itsBuffers[i].getFullResFlags().shape()[0] - 1;
- flags(s,e) = itsBuffers[i].getFullResFlags();
- } else {
- e[0] = s[0] + itsFillNChan - 1;
- flags(s,e) = true;
- }
- s[0] = e[0] + 1;
- }
- return true;
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/NDPPP.cc b/CEP/DP3/DPPP/src/NDPPP.cc
deleted file mode 100644
index 17a3ad8ba7f9e12cb57abb2836401cd9ce5d6660..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/NDPPP.cc
+++ /dev/null
@@ -1,106 +0,0 @@
-//# DPPP.cc: Program to execute steps like averaging and flagging on an MS
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DPRun.h>
-#include <DPPP/Package__Version.h>
-#include <Common/LofarLogger.h>
-#include <Common/SystemUtil.h>
-#include <Common/Exception.h>
-#include <iostream>
-#include <stdexcept>
-
-#include <casacore/casa/OS/File.h>
-
-using namespace LOFAR::DPPP;
-using namespace LOFAR;
-
-// Define handler that tries to print a backtrace.
-Exception::TerminateHandler t(Exception::terminate);
-
-void showUsage() {
- std::cout<<"Usage: DPPP [-v] [parsetfile] [parsetkeys...]"<<std::endl;
- std::cout<<" parsetfile: a file containing one parset key=value pair "<<
- "per line"<<std::endl;
- std::cout<<" parsetkeys: any number of parset key=value pairs, e.g. "<<
- "msin=my.MS"<<std::endl<<std::endl;
- std::cout<<"If both a file and command-line keys are specified, the "<<
- "keys on the command line override those in the file."<<std::endl;
- std::cout<<"If no arguments are specified, the program tries to read "<<
- "\"NDPPP.parset\" or \"DPPP.parset\" as a default."<<std::endl;
- std::cout<<"-v will show version info and exit."<<std::endl;
- std::cout<<"Documentation is at http://www.lofar.org/wiki/doku.php?id="<<
- "public:user_software:documentation:ndppp"<<std::endl;
-}
-
-int main(int argc, char *argv[])
-{
- try
- {
- TEST_SHOW_VERSION (argc, argv, DPPP);
- INIT_LOGGER("DPPP");
- // Get the name of the parset file.
- if (argc>1 && (
- string(argv[1])=="--help" ||
- string(argv[1])=="-help" || string(argv[1])=="-h" ||
- string(argv[1])=="--usage" || string(argv[1])=="-usage")) {
- showUsage();
- return 0;
- }
-
- string parsetName;
- if (argc > 1 && string(argv[1]).find('=') == string::npos) {
- // First argument is parset name (except if it's a key-value pair)
- parsetName = argv[1];
- } else if (argc==1) {
- // No arguments given: try to load [N]DPPP.parset
- if (casacore::File("NDPPP.parset").exists()) {
- parsetName="NDPPP.parset";
- } else if (casacore::File("DPPP.parset").exists()) {
- parsetName="DPPP.parset";
- } else { // No default file, show usage and exit
- showUsage();
- return 0;
- }
- }
-
- // Execute the parset file.
- DPRun::execute (parsetName, argc, argv);
- } catch (LOFAR::APSException& err) {
- // just send err.what() to the error stream
- // this is just the error message, not a full backtrace
- std::cerr << std::endl;
- std::cerr << "ParameterSet Exception detected: "<< err.what() << std::endl;
- return 1;
- } catch (LOFAR::Exception& err) {
- std::cerr << "LOFAR Exception detected: " << err << std::endl;
- return 1;
-#ifdef __clang__
- } catch (std::exception& err) {
- std::cerr << std::endl;
- std::cerr << "std exception detected: " << err.what() << std::endl;
- return 1;
-#endif
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/src/OneApplyCal.cc b/CEP/DP3/DPPP/src/OneApplyCal.cc
deleted file mode 100644
index 5c1a9d93e584e7540b9966f0a8496a6a4acb4672..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/OneApplyCal.cc
+++ /dev/null
@@ -1,764 +0,0 @@
-//# OneApplyCal.cc: DPPP step class to apply a calibration correction to the data
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: OneApplyCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/OneApplyCal.h>
-#include <DPPP/ApplyCal.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/MSReader.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/OS/File.h>
-#include <iostream>
-#include <limits>
-#include <algorithm>
-#include <iomanip>
-
-using namespace casacore;
-using namespace LOFAR::BBS;
-
-/// Look at BBSKernel MeasurementExprLOFARUtil.cc and Apply.cc
-
-namespace LOFAR {
- namespace DPPP {
-
- OneApplyCal::OneApplyCal (DPInput* input,
- const ParameterSet& parset,
- const string& prefix,
- const string& defaultPrefix,
- bool substep,
- string predictDirection
- )
- : itsInput (input),
- itsName (prefix),
- itsParmDBName (
- parset.isDefined(prefix+"parmdb") ?
- parset.getString(prefix + "parmdb") :
- parset.getString(defaultPrefix + "parmdb")),
- itsUseH5Parm (itsParmDBName.find(".h5") != string::npos),
- itsSolSetName (
- parset.isDefined(prefix + "solset") ?
- parset.getString(prefix + "solset") :
- parset.getString(defaultPrefix + "solset", "")),
- itsSigmaMMSE (
- parset.isDefined(prefix + "MMSE.Sigma") ?
- parset.getDouble(prefix + "MMSE.Sigma") :
- parset.getDouble(defaultPrefix + "MMSE.Sigma", 0.)),
- itsUpdateWeights (
- parset.isDefined(prefix + "updateweights") ?
- parset.getBool (prefix + "updateweights") :
- parset.getBool (defaultPrefix + "updateweights", false)),
- itsCount (0),
- itsTimeStep (0),
- itsNCorr (0),
- itsTimeInterval (-1),
- itsLastTime (-1),
- itsUseAP (false)
- {
-
- ASSERT (!itsParmDBName.empty());
-
- if (substep) {
- itsInvert=false;
- } else {
- itsInvert = (parset.isDefined(prefix + "invert") ?
- parset.getBool (prefix + "invert") :
- parset.getBool (defaultPrefix + "invert", true));
- }
-
- if (itsUseH5Parm) {
- itsTimeSlotsPerParmUpdate = 0;
- string directionStr;
- directionStr = (parset.isDefined(prefix + "direction") ?
- parset.getString(prefix + "direction") :
- parset.getString(defaultPrefix + "direction",
- predictDirection));
- itsH5Parm = H5Parm(itsParmDBName, false, false, itsSolSetName);
-
- itsSolTabName = (parset.isDefined(prefix + "correction") ?
- parset.getString(prefix + "correction") :
- parset.getString(defaultPrefix + "correction"));
- if(itsSolTabName == "fulljones")
- {
- itsSolTab = itsH5Parm.getSolTab("amplitude000");
- itsSolTab2 = itsH5Parm.getSolTab("phase000");
- itsSolTabName = "amplitude000, phase000"; // this is only so that show() shows these tables
- itsCorrectType = FULLJONES;
- }
- else {
- itsSolTab = itsH5Parm.getSolTab(itsSolTabName);
- itsCorrectType = stringToCorrectType(itsSolTab.getType());
- }
- if (itsCorrectType==PHASE && nPol("")==1) {
- itsCorrectType = SCALARPHASE;
- }
- if (itsCorrectType==AMPLITUDE && nPol("")==1) {
- itsCorrectType = SCALARAMPLITUDE;
- }
- itsDirection = 0;
- if (directionStr=="") {
- ASSERT(!itsSolTab.hasAxis("dir") || itsSolTab.getAxis("dir").size==1);
- // If there is only one direction, silently assume it is the right one
- } else if (itsSolTab.hasAxis("dir") && itsSolTab.getAxis("dir").size>1) {
- itsDirection = itsSolTab.getDirIndex(directionStr);
- }
- } else {
- itsTimeSlotsPerParmUpdate =
- parset.isDefined(prefix + "timeslotsperparmupdate") ?
- parset.getInt (prefix + "timeslotsperparmupdate") :
- parset.getInt (defaultPrefix + "timeslotsperparmupdate", 500);
- string correctTypeStr = toLower(
- parset.isDefined(prefix + "correction") ?
- parset.getString(prefix + "correction") :
- parset.getString (defaultPrefix + "correction", "gain"));
- itsCorrectType = stringToCorrectType(correctTypeStr);
- }
-
- if (itsCorrectType==FULLJONES && itsUpdateWeights) {
- ASSERTSTR (itsInvert, "Updating weights has not been implemented for invert=false and fulljones");
- }
- }
-
- string OneApplyCal::correctTypeToString(CorrectType ct) {
- if (ct==GAIN) return "gain";
- if (ct==FULLJONES) return "fulljones";
- if (ct==TEC) return "tec";
- if (ct==CLOCK) return "clock";
- if (ct==SCALARPHASE) return "scalarphase";
- if (ct==SCALARAMPLITUDE) return "scalaramplitude";
- if (ct==ROTATIONANGLE) return "rotationangle";
- if (ct==ROTATIONMEASURE) return "rotationmeasure";
- if (ct==PHASE) return "phase";
- if (ct==AMPLITUDE) return "amplitude";
- THROW(Exception, "Unknown correction type: "<<ct);
- return "";
- }
-
- OneApplyCal::CorrectType OneApplyCal::stringToCorrectType(const string& ctStr) {
- if (ctStr=="gain") return GAIN;
- if (ctStr=="fulljones") return FULLJONES;
- if (ctStr=="tec") return TEC;
- if (ctStr=="clock") return CLOCK;
- if (ctStr=="scalarphase" || ctStr=="commonscalarphase") return SCALARPHASE;
- if (ctStr=="scalaramplitude" || ctStr=="commonscalaramplitude") return SCALARAMPLITUDE;
- if (ctStr=="phase") return PHASE;
- if (ctStr=="amplitude") return AMPLITUDE;
- if (ctStr=="rotationangle" || ctStr=="commonrotationangle" || ctStr=="rotation") return ROTATIONANGLE;
- if (ctStr=="rotationmeasure") return ROTATIONMEASURE;
- THROW(Exception, "Unknown correction type: "<<ctStr);
- return GAIN;
- }
-
- OneApplyCal::~OneApplyCal()
- {}
-
- void OneApplyCal::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- info().setWriteFlags();
- if (itsUpdateWeights) {
- info().setWriteWeights();
- }
- itsTimeInterval = infoIn.timeInterval();
- itsNCorr = infoIn.ncorr();
-
- ASSERT(itsNCorr==4);
-
- if (itsUseH5Parm) {
- itsTimeSlotsPerParmUpdate = info().ntime();
- } else { // Use ParmDB
- itsParmDB.reset(new BBS::ParmFacade(itsParmDBName));
- }
-
- // Detect if full jones solutions are present
- if (!itsUseH5Parm &&
- (itsCorrectType == GAIN || itsCorrectType==FULLJONES) &&
- (itsParmDB->getNames("Gain:0:1:*").size() +
- itsParmDB->getDefNames("Gain:0:1:*").size() >0 )) {
- itsCorrectType=FULLJONES;
- }
-
- // Detect if solutions are saved as Real/Imag or Ampl/Phase
- if (itsCorrectType == GAIN || itsCorrectType == FULLJONES ){
- if (itsUseH5Parm) {
- // H5Parm uses amplitude / phase by definition
- itsUseAP = true;
- } else {
- // Determine from values present in parmdb what to use
- if (!itsParmDB->getNames("Gain:0:0:Real*").empty()) {
- // Values with :Real present
- itsUseAP = false;
- } else if (!itsParmDB->getNames("Gain:0:0:Ampl*").empty() ||
- !itsParmDB->getNames("Phase:0:0:Ampl*").empty()) {
- // Values with :Ampl present
- itsUseAP = true;
- } else if (!itsParmDB->getDefNames("Gain:0:0:Real*").empty()) {
- // Defvalues with :Real present
- itsUseAP = false;
- } else if (!itsParmDB->getDefNames("Gain:0:0:Ampl*").empty() ||
- !itsParmDB->getDefNames("Gain:0:0:Phase*").empty()) {
- // Defvalues with :Ampl present
- itsUseAP = true;
- } else {
- THROW (Exception, "No gains found in parmdb "+itsParmDBName);
- }
- }
- }
-
- if (itsCorrectType == GAIN) {
- if (itsUseAP) {
- itsParmExprs.push_back("Gain:0:0:Ampl");
- itsParmExprs.push_back("Gain:0:0:Phase");
- itsParmExprs.push_back("Gain:1:1:Ampl");
- itsParmExprs.push_back("Gain:1:1:Phase");
- } else {
- itsParmExprs.push_back("Gain:0:0:Real");
- itsParmExprs.push_back("Gain:0:0:Imag");
- itsParmExprs.push_back("Gain:1:1:Real");
- itsParmExprs.push_back("Gain:1:1:Imag");
- }
- } else if (itsCorrectType == FULLJONES) {
- if (itsUseAP) {
- itsParmExprs.push_back("Gain:0:0:Ampl");
- itsParmExprs.push_back("Gain:0:0:Phase");
- itsParmExprs.push_back("Gain:0:1:Ampl");
- itsParmExprs.push_back("Gain:0:1:Phase");
- itsParmExprs.push_back("Gain:1:0:Ampl");
- itsParmExprs.push_back("Gain:1:0:Phase");
- itsParmExprs.push_back("Gain:1:1:Ampl");
- itsParmExprs.push_back("Gain:1:1:Phase");
- } else {
- itsParmExprs.push_back("Gain:0:0:Real");
- itsParmExprs.push_back("Gain:0:0:Imag");
- itsParmExprs.push_back("Gain:0:1:Real");
- itsParmExprs.push_back("Gain:0:1:Imag");
- itsParmExprs.push_back("Gain:1:0:Real");
- itsParmExprs.push_back("Gain:1:0:Imag");
- itsParmExprs.push_back("Gain:1:1:Real");
- itsParmExprs.push_back("Gain:1:1:Imag");
- }
- } else if (itsCorrectType == TEC) {
- if (nPol("TEC")==1) {
- itsParmExprs.push_back("TEC");
- }
- else {
- itsParmExprs.push_back("TEC:0");
- itsParmExprs.push_back("TEC:1");
- }
- } else if (itsCorrectType == CLOCK) {
- if (nPol("Clock")==1) {
- itsParmExprs.push_back("Clock");
- }
- else {
- itsParmExprs.push_back("Clock:0");
- itsParmExprs.push_back("Clock:1");
- }
- } else if (itsCorrectType == ROTATIONANGLE) {
- itsParmExprs.push_back("{Common,}RotationAngle");
- } else if (itsCorrectType == SCALARPHASE) {
- itsParmExprs.push_back("{Common,}ScalarPhase");
- } else if (itsCorrectType == ROTATIONMEASURE) {
- itsParmExprs.push_back("RotationMeasure");
- } else if (itsCorrectType == SCALARAMPLITUDE) {
- itsParmExprs.push_back("{Common,}ScalarAmplitude");
- } else if (itsCorrectType == PHASE) {
- ASSERT(itsUseH5Parm);
- itsParmExprs.push_back("Phase:0");
- itsParmExprs.push_back("Phase:1");
- } else if (itsCorrectType == AMPLITUDE) {
- ASSERT(itsUseH5Parm);
- itsParmExprs.push_back("Amplitude:0");
- itsParmExprs.push_back("Amplitude:1");
- } else {
- THROW (Exception, "Correction type "<<
- correctTypeToString(itsCorrectType)<<" is unknown");
- }
-
- initDataArrays();
- itsFlagCounter.init(getInfo());
-
- // Check that channels are evenly spaced
- if (info().nchan()>1) {
- Vector<Double> upFreq = info().chanFreqs()(
- Slicer(IPosition(1,1),
- IPosition(1,info().nchan()-1)));
- Vector<Double> lowFreq = info().chanFreqs()(
- Slicer(IPosition(1,0),
- IPosition(1,info().nchan()-1)));
- Double freqstep0=upFreq(0)-lowFreq(0);
- // Compare up to 1kHz accuracy
- bool regularChannels=allNearAbs(upFreq-lowFreq, freqstep0, 1.e3) &&
- allNearAbs(info().chanWidths(),
- info().chanWidths()(0), 1.e3);
-
- if (!itsUseH5Parm) {
- ASSERTSTR(regularChannels,
- "ApplyCal with parmdb requires evenly spaced channels.");
- }
- }
- }
-
- void OneApplyCal::show (std::ostream& os) const
- {
- os << "ApplyCal " << itsName << std::endl;
- if (itsUseH5Parm) {
- os << " H5Parm: " << itsParmDBName << endl;
- os << " SolSet: " << itsH5Parm.getSolSetName() << endl;
- os << " SolTab: " << itsSolTabName << endl;
- } else {
- os << " parmdb: " << itsParmDBName << endl;
- }
- os << " correction: " << correctTypeToString(itsCorrectType) << endl;
- if (itsCorrectType==GAIN || itsCorrectType==FULLJONES) {
- os << " Ampl/Phase: " << boolalpha << itsUseAP << endl;
- }
- os << " update weights: " << boolalpha << itsUpdateWeights << endl;
- os << " invert: " << boolalpha << itsInvert <<endl;
- if (itsInvert) {
- os << " sigmaMMSE: " << itsSigmaMMSE << endl;
- }
- os << " timeSlotsPerParmUpdate: " << itsTimeSlotsPerParmUpdate <<endl;
- }
-
- void OneApplyCal::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " OneApplyCal " << itsName << endl;
- }
-
- bool OneApplyCal::process (const DPBuffer& bufin)
- {
- itsTimer.start();
- itsBuffer.copy (bufin);
-
- if (bufin.getTime() > itsLastTime) {
- updateParms(bufin.getTime());
- itsTimeStep=0;
- }
- else {
- itsTimeStep++;
- }
-
- // Loop through all baselines in the buffer.
- size_t nbl = itsBuffer.getData().shape()[2];
-
- Complex* data = itsBuffer.getData().data();
-
- itsInput->fetchWeights (bufin, itsBuffer, itsTimer);
- float* weight = itsBuffer.getWeights().data();
-
- bool* flag = itsBuffer.getFlags().data();
-
- size_t nchan = itsBuffer.getData().shape()[1];
-
-#pragma omp parallel for
- for (size_t bl=0; bl<nbl; ++bl) {
- for (size_t chan=0;chan<nchan;chan++) {
- uint timeFreqOffset=(itsTimeStep*info().nchan())+chan;
- uint antA = info().getAnt1()[bl];
- uint antB = info().getAnt2()[bl];
- if (itsParms.shape()[0]>2) {
- ApplyCal::applyFull( &itsParms(0, antA, timeFreqOffset),
- &itsParms(0, antB, timeFreqOffset),
- &data[bl * itsNCorr * nchan + chan * itsNCorr ],
- &weight[bl * itsNCorr * nchan + chan * itsNCorr ],
- &flag[ bl * itsNCorr * nchan + chan * itsNCorr ],
- bl, chan, itsUpdateWeights, itsFlagCounter);
- }
- else {
- ApplyCal::applyDiag( &itsParms(0, antA, timeFreqOffset),
- &itsParms(0, antB, timeFreqOffset),
- &data[bl * itsNCorr * nchan + chan * itsNCorr ],
- &weight[bl * itsNCorr * nchan + chan * itsNCorr ],
- &flag[ bl * itsNCorr * nchan + chan * itsNCorr ],
- bl, chan, itsUpdateWeights, itsFlagCounter);
- }
- }
- }
-
- itsTimer.stop();
- getNextStep()->process(itsBuffer);
-
- itsCount++;
- return true;
- }
-
- void OneApplyCal::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void OneApplyCal::applyFlags(vector<double>& values,
- const vector<double>& weights) {
- ASSERT(values.size() == weights.size());
- vector<double>::iterator values_it = values.begin();
- vector<double>::const_iterator weights_it = weights.begin();
-
- for (; values_it != values.end(); ++values_it) {
- if (*weights_it == 0.) {
- *values_it = std::numeric_limits<float>::quiet_NaN();
- }
- weights_it++;
- }
- }
-
- void OneApplyCal::updateParms (const double bufStartTime)
- {
- uint numAnts = info().antennaNames().size();
-
- // itsParms contains the parameters to a grid, first for all parameters
- // (e.g. Gain:0:0 and Gain:1:1), next all antennas, next over freq * time
- // as returned by ParmDB
- vector<vector<vector<double> > > parmvalues;
- parmvalues.resize(itsParmExprs.size());
- for (size_t i=0;i<parmvalues.size();++i) {
- parmvalues[i].resize(numAnts);
- }
-
- uint numFreqs (info().chanFreqs().size());
- double freqInterval (info().chanWidths()[0]);
- if (numFreqs>1) { // Handle data with evenly spaced gaps between channels
- freqInterval = info().chanFreqs()[1]-info().chanFreqs()[0];
- }
- double minFreq (info().chanFreqs()[0]-0.5*freqInterval);
- double maxFreq (info().chanFreqs()[numFreqs-1]+0.5*freqInterval);
- itsLastTime = bufStartTime - 0.5*itsTimeInterval +
- itsTimeSlotsPerParmUpdate * itsTimeInterval;
- uint numTimes = itsTimeSlotsPerParmUpdate;
-
- double lastMSTime = info().startTime() + info().ntime() * itsTimeInterval;
- if (itsLastTime > lastMSTime && !nearAbs(itsLastTime, lastMSTime, 1.e-3)) {
- itsLastTime = lastMSTime;
- numTimes = info().ntime() % itsTimeSlotsPerParmUpdate;
- }
-
- map<string, vector<double> > parmMap;
- map<string, vector<double> >::iterator parmIt;
-
- uint tfDomainSize=numTimes*numFreqs;
-
- // Fill parmvalues here, get raw data from H5Parm or ParmDB
- if (itsUseH5Parm) {
-#pragma omp critical(updateH5ParmValues)
-{
- // TODO: understand polarization etc.
- // ASSERT(itsParmExprs.size()==1 || itsParmExprs.size()==2);
-
- // Figure out whether time or frequency is first axis
- bool freqvariesfastest = true;
- if (itsSolTab.hasAxis("freq") && itsSolTab.hasAxis("time") &&
- itsSolTab.getAxisIndex("freq") < itsSolTab.getAxisIndex("time")) {
- freqvariesfastest = false;
- }
- ASSERT(freqvariesfastest);
-
- vector<double> times(info().ntime());
- for (uint t=0; t<times.size(); ++t) {
- // time centroids
- times[t] = info().startTime() + (t+0.5) * info().timeInterval();
- }
- vector<double> freqs(info().chanFreqs().size());
- for (uint ch=0; ch<info().chanFreqs().size(); ++ch) {
- freqs[ch] = info().chanFreqs()[ch];
- }
-
- vector<double> weights;
- for (uint ant = 0; ant < numAnts; ++ant) {
- if(itsCorrectType == FULLJONES)
- {
- for (uint pol=0; pol<4; ++pol) {
- // Place amplitude in even and phase in odd elements
- parmvalues[pol*2][ant] = itsSolTab.getValuesOrWeights("val",
- info().antennaNames()[ant],
- times, freqs,
- pol, itsDirection);
- weights = itsSolTab.getValuesOrWeights("weight",
- info().antennaNames()[ant], times, freqs, pol, itsDirection);
- applyFlags(parmvalues[pol*2][ant], weights);
- parmvalues[pol*2+1][ant] = itsSolTab2.getValuesOrWeights("val",
- info().antennaNames()[ant],
- times, freqs,
- pol, itsDirection);
- weights = itsSolTab2.getValuesOrWeights("weight",
- info().antennaNames()[ant], times, freqs, pol, itsDirection);
- applyFlags(parmvalues[pol*2+1][ant], weights);
- }
- }
- else {
- for (uint pol=0; pol<itsParmExprs.size(); ++pol) {
- parmvalues[pol][ant] = itsSolTab.getValuesOrWeights("val",
- info().antennaNames()[ant],
- times, freqs,
- pol, itsDirection);
- weights = itsSolTab.getValuesOrWeights("weight",
- info().antennaNames()[ant], times, freqs, pol, itsDirection);
- applyFlags(parmvalues[pol][ant], weights);
- }
- }
- }
-} // End pragma omp critical
- } else { // Use ParmDB
- for (uint parmExprNum = 0; parmExprNum<itsParmExprs.size();++parmExprNum) {
- // parmMap contains parameter values for all antennas
- parmMap = itsParmDB->getValuesMap( itsParmExprs[parmExprNum] + "{:phase_center,}*",
- minFreq, maxFreq, freqInterval,
- bufStartTime-0.5*itsTimeInterval, itsLastTime,
- itsTimeInterval, true);
-
- string parmExpr = itsParmExprs[parmExprNum];
-
- // Resolve {Common,}Bla to CommonBla or Bla
- if (!parmMap.empty() &&
- parmExpr.find("{Common,}") != string::npos) {
- // Take the name of the first parm, e.g. Bla:CS001, and remove the antenna name
- uint colonPos = (parmMap.begin()->first).find(":");
- parmExpr = (parmMap.begin()->first).substr(0, colonPos);
- }
-
- string name_postfix = "";
- // Remove :phase_center postfix
- if (!parmMap.empty()) {
- // Take the name of the first parm, e.g. Bla:CS001, and remove the antenna name
- // If necessary, append :phase_center
- if ((parmMap.begin()->first).find(":phase_center") != string::npos) {
- name_postfix = ":phase_center";
- }
- }
-
- for (uint ant = 0; ant < numAnts; ++ant) {
- parmIt = parmMap.find(parmExpr + ":" + string(info().antennaNames()[ant])
- + name_postfix);
-
- if (parmIt != parmMap.end()) {
- parmvalues[parmExprNum][ant].swap(parmIt->second);
- ASSERT(parmvalues[parmExprNum][ant].size()==tfDomainSize);
- } else {// No value found, try default
- Array<double> defValues;
- double defValue;
-
- if (itsParmDB->getDefValues(parmExpr + ":" +
- string(info().antennaNames()[ant]) + name_postfix).size()==1) { // Default for antenna
- itsParmDB->getDefValues(string(itsParmExprs[parmExprNum]) + ":" +
- string(info().antennaNames()[ant]) + name_postfix).get(0,defValues);
- ASSERT(defValues.size()==1);
- defValue=defValues.data()[0];
- }
- else if (itsParmDB->getDefValues(parmExpr).size() == 1) { //Default value
- itsParmDB->getDefValues(parmExpr).get(0,defValues);
- ASSERT(defValues.size()==1);
- defValue=defValues.data()[0];
- } else if (parmExpr.substr(0,5)=="Gain:") {
- defValue=0.;
- }
- else {
- THROW (Exception, "No parameter value found for "+
- parmExpr + ":" + string(info().antennaNames()[ant]) + name_postfix);
- }
-
- parmvalues[parmExprNum][ant].resize(tfDomainSize);
- for (uint tf=0; tf<tfDomainSize;++tf) {
- parmvalues[parmExprNum][ant][tf]=defValue;
- }
- }
- }
- }
- }
-
- ASSERT(parmvalues[0][0].size() <= tfDomainSize); // Catches multiple matches
-
- double freq;
-
- // Make parameters complex
- for (uint tf=0;tf<tfDomainSize;++tf) {
- for (uint ant=0;ant<numAnts;++ant) {
-
- freq=info().chanFreqs()[tf % numFreqs];
-
- if (itsCorrectType==GAIN) {
- if (itsUseAP) { // Data as Amplitude / Phase
- itsParms(0, ant, tf) = polar(parmvalues[0][ant][tf],
- parmvalues[1][ant][tf]);
- itsParms(1, ant, tf) = polar(parmvalues[2][ant][tf],
- parmvalues[3][ant][tf]);
- } else { // Data as Real / Imaginary
- itsParms(0, ant, tf) = DComplex(parmvalues[0][ant][tf],
- parmvalues[1][ant][tf]);
- itsParms(1, ant, tf) = DComplex(parmvalues[2][ant][tf],
- parmvalues[3][ant][tf]);
- }
- }
- else if (itsCorrectType==FULLJONES) {
- if (itsUseAP) { // Data as Amplitude / Phase
- itsParms(0, ant, tf) = polar(parmvalues[0][ant][tf],
- parmvalues[1][ant][tf]);
- itsParms(1, ant, tf) = polar(parmvalues[2][ant][tf],
- parmvalues[3][ant][tf]);
- itsParms(2, ant, tf) = polar(parmvalues[4][ant][tf],
- parmvalues[5][ant][tf]);
- itsParms(3, ant, tf) = polar(parmvalues[6][ant][tf],
- parmvalues[7][ant][tf]);
- } else { // Data as Real / Imaginary
- itsParms(0, ant, tf) = DComplex(parmvalues[0][ant][tf],
- parmvalues[1][ant][tf]);
- itsParms(1, ant, tf) = DComplex(parmvalues[2][ant][tf],
- parmvalues[3][ant][tf]);
- itsParms(2, ant, tf) = DComplex(parmvalues[4][ant][tf],
- parmvalues[5][ant][tf]);
- itsParms(3, ant, tf) = DComplex(parmvalues[6][ant][tf],
- parmvalues[7][ant][tf]);
- }
- }
- else if (itsCorrectType==TEC) {
- itsParms(0, ant, tf)=polar(1.,
- parmvalues[0][ant][tf] * -8.44797245e9 / freq);
- if (itsParmExprs.size() == 1) { // No TEC:0, only TEC:
- itsParms(1, ant, tf)=polar(1.,
- parmvalues[0][ant][tf] * -8.44797245e9 / freq);
- }
- else { // TEC:0 and TEC:1
- itsParms(1, ant, tf)=polar(1.,
- parmvalues[1][ant][tf] * -8.44797245e9 / freq);
- }
- }
- else if (itsCorrectType==CLOCK) {
- itsParms(0, ant, tf)=polar(1.,
- parmvalues[0][ant][tf] * freq * casacore::C::_2pi);
- if (itsParmExprs.size() == 1) { // No Clock:0, only Clock:
- itsParms(1, ant, tf)=polar(1.,
- parmvalues[0][ant][tf] * freq * casacore::C::_2pi);
- }
- else { // Clock:0 and Clock:1
- itsParms(1, ant, tf)=polar(1.,
- parmvalues[1][ant][tf] * freq * casacore::C::_2pi);
- }
- }
- else if (itsCorrectType==ROTATIONANGLE) {
- double phi=parmvalues[0][ant][tf];
- if (itsInvert) {
- phi = -phi;
- }
- double sinv=sin(phi);
- double cosv=cos(phi);
- itsParms(0, ant, tf) = cosv;
- itsParms(1, ant, tf) = -sinv;
- itsParms(2, ant, tf) = sinv;
- itsParms(3, ant, tf) = cosv;
- }
- else if (itsCorrectType==ROTATIONMEASURE) {
- double lambda2 = casacore::C::c / freq;
- lambda2 *= lambda2;
- double chi = parmvalues[0][ant][tf] * lambda2;
- if (itsInvert) {
- chi = -chi;
- }
- double sinv = sin(chi);
- double cosv = cos(chi);
- itsParms(0, ant, tf) = cosv;
- itsParms(1, ant, tf) = -sinv;
- itsParms(2, ant, tf) = sinv;
- itsParms(3, ant, tf) = cosv;
- }
- else if (itsCorrectType==PHASE || itsCorrectType==SCALARPHASE) {
- itsParms(0, ant, tf) = polar(1., parmvalues[0][ant][tf]);
- if (itsCorrectType==SCALARPHASE) { // Same value for x and y
- itsParms(1, ant, tf) = polar(1., parmvalues[0][ant][tf]);
- } else { // Different value for x and y
- itsParms(1, ant, tf) = polar(1., parmvalues[1][ant][tf]);
- }
- }
- else if (itsCorrectType==AMPLITUDE || itsCorrectType==SCALARAMPLITUDE) {
- itsParms(0, ant, tf) = parmvalues[0][ant][tf];
- if (itsCorrectType==SCALARAMPLITUDE) { // Same value for x and y
- itsParms(1, ant, tf) = parmvalues[0][ant][tf];
- } else { // Different value for x and y
- itsParms(1, ant, tf) = parmvalues[1][ant][tf];
- }
- }
-
- // Invert
- if (itsInvert) {
- if (itsParms.shape()[0]==2) {
- itsParms(0, ant, tf) = 1./itsParms(0, ant, tf);
- itsParms(1, ant, tf) = 1./itsParms(1, ant, tf);
- } else if (itsCorrectType==FULLJONES) {
- ApplyCal::invert(&itsParms(0, ant, tf),itsSigmaMMSE);
- } else {
- ASSERT (itsCorrectType==ROTATIONMEASURE || itsCorrectType==ROTATIONANGLE);
- // rotationmeasure and commonrotationangle are already inverted above
- }
- }
- }
- }
- }
-
- uint OneApplyCal::nPol(const string& parmName) {
- if (itsUseH5Parm) {
- if (!itsSolTab.hasAxis("pol")) {
- return 1;
- } else {
- return itsSolTab.getAxis("pol").size;
- }
- } else { // Use ParmDB
- if (itsParmDB->getNames(parmName+":0:*").empty() &&
- itsParmDB->getDefNames(parmName+":0:*").empty() ) {
- return 1;
- } else {
- return 2;
- }
- }
- }
-
- void OneApplyCal::initDataArrays() {
- uint numAnts=info().antennaNames().size();
- uint tfDomainSize=itsTimeSlotsPerParmUpdate*info().chanFreqs().size();
-
- uint numParms;
- if (itsCorrectType==FULLJONES ||
- itsCorrectType==ROTATIONANGLE ||
- itsCorrectType==ROTATIONMEASURE) {
- numParms = 4;
- }
- else {
- numParms = 2;
- }
-
- itsParms.resize(numParms, numAnts, tfDomainSize);
- }
-
- void OneApplyCal::showCounts (std::ostream& os) const
- {
- os << endl << "Flags set by OneApplyCal " << itsName;
- os << endl << "=======================" << endl;
- itsFlagCounter.showBaseline (os, itsCount);
- itsFlagCounter.showChannel (os, itsCount);
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/Patch.cc b/CEP/DP3/DPPP/src/Patch.cc
deleted file mode 100644
index 82bb727be48b9018ddd1fca86fc4f18f823eb5eb..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Patch.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-//# Patch.cc: A set of sources for which direction dependent effects are assumed to be equal.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/Patch.h>
-#include <DPPP/ModelComponentVisitor.h>
-#include <Common/lofar_math.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-void Patch::computePosition()
-{
- itsPosition = Position();
-
- if(!itsComponents.empty())
- {
- double x = 0.0, y = 0.0, z = 0.0;
- for(const_iterator it = begin(), it_end = end(); it != it_end; ++it)
- {
- const Position &position = (*it)->position();
- double cosDec = cos(position[1]);
- x += cos(position[0]) * cosDec;
- y += sin(position[0]) * cosDec;
- z += sin(position[1]);
- }
-
- x /= itsComponents.size();
- y /= itsComponents.size();
- z /= itsComponents.size();
-
- itsPosition[0] = atan2(y, x);
- itsPosition[1] = asin(z);
- }
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/PhaseFitter.cc b/CEP/DP3/DPPP/src/PhaseFitter.cc
deleted file mode 100644
index c883ec004a6256b9a4b4a9d933ff2cc2a5b78a12..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/PhaseFitter.cc
+++ /dev/null
@@ -1,232 +0,0 @@
-//# phasefitter.cc: Class to perform phase fitting (TEC), allowing phase wraps
-//# Copyright (C) 2016
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: phasefitter.cc 21598 2012-07-16 08:07:34Z offringa $
-//#
-//# @author Andre Offringa
-
-#ifdef AOPROJECT
-#include "PhaseFitter.h"
-#else
-#include <lofar_config.h>
-#include <DPPP/PhaseFitter.h>
-#endif
-
-#include <limits>
-
-double PhaseFitter::TEC2ModelCost(double alpha, double beta) const
-{
- double costVal = 0.0, weightSum = 0.0;
- for(size_t i=0; i!=Size(); ++i) {
- double estphase = TEC2ModelFuncWrapped(_frequencies[i], alpha, beta);
- double dCost = fmod(std::fabs(estphase - _phases[i]), 2.0*M_PI);
- if(dCost > M_PI)
- dCost = 2.0*M_PI - dCost;
- dCost *= _weights[i];
- costVal += dCost;
- weightSum += _weights[i];
- }
- if(weightSum == 0.0)
- return 0.0;
- else
- return costVal / weightSum;
-}
-
-double PhaseFitter::fitTEC2ModelBeta(double alpha, double betaEstimate) const {
- double weightSum = 0.0;
- for(size_t iter=0; iter!=3; ++iter) {
- double sum = 0.0;
- for(size_t i=0; i!=Size(); ++i) {
- double p = _phases[i], e = TEC2ModelFunc(_frequencies[i], alpha, betaEstimate);
- double dist = fmod(p - e, 2.0*M_PI);
- if(dist < -M_PI)
- dist += 2.0*M_PI;
- else if(dist > M_PI)
- dist -= 2.0*M_PI;
- sum += dist * _weights[i];
- weightSum += _weights[i];
- }
- if(weightSum != 0.0)
- betaEstimate = betaEstimate + sum / weightSum;
- }
- return fmod(betaEstimate, 2.0*M_PI);
-}
-
-void PhaseFitter::bruteForceSearchTEC2Model(double& lowerAlpha, double& upperAlpha, double& beta) const
-{
- double minCost = std::numeric_limits<double>::max();
- double alphaOversampling = 256;
- //size_t betaOversampling = 16;
- double dAlpha = upperAlpha - lowerAlpha;
- int alphaIndex = 0;
- for(int i=0; i!=alphaOversampling; ++i) {
- // make r between [0, 1]
- double r = double(i)/alphaOversampling;
- double alpha = lowerAlpha + r*dAlpha;
- double curBeta = fitTEC2ModelBeta(alpha, beta);
- double costVal = TEC2ModelCost(alpha, curBeta);
- if(costVal < minCost) {
- beta = curBeta;
- minCost = costVal;
- alphaIndex = i;
- }
- }
- double newLowerAlpha = double(alphaIndex-1)/alphaOversampling*dAlpha + lowerAlpha;
- upperAlpha = double(alphaIndex+1)/alphaOversampling*dAlpha + lowerAlpha;
- lowerAlpha = newLowerAlpha;
-}
-
-double PhaseFitter::ternarySearchTEC2ModelAlpha(double startAlpha, double endAlpha, double& beta) const
-{
- size_t iter = 0;
- double dCost, lAlpha, rAlpha;
- do {
- lAlpha = startAlpha + (endAlpha - startAlpha) * (1.0/3.0);
- rAlpha = startAlpha + (endAlpha - startAlpha) * (2.0/3.0);
- double lBeta = fitTEC2ModelBeta(lAlpha, beta);
- double rBeta = fitTEC2ModelBeta(rAlpha, beta);
- double lCost = TEC2ModelCost(lAlpha, lBeta);
- double rCost = TEC2ModelCost(rAlpha, rBeta);
- if(lCost < rCost) {
- endAlpha = rAlpha;
- beta = lBeta;
- } else {
- startAlpha = lAlpha;
- beta = rBeta;
- }
- dCost = std::fabs(lCost - rCost);
- ++iter;
- } while(dCost > _fittingAccuracy && iter < 100);
- double finalAlpha = (lAlpha + rAlpha) * 0.5;
- beta = fitTEC2ModelBeta(finalAlpha, beta);
- return finalAlpha;
-}
-
-void PhaseFitter::fillDataWithTEC2Model(double alpha, double beta)
-{
- for(size_t ch=0; ch!=Size(); ++ch)
- _phases[ch] = TEC2ModelFunc(_frequencies[ch], alpha, beta);
-}
-
-void PhaseFitter::fillDataWithTEC1Model(double alpha)
-{
- for(size_t ch=0; ch!=Size(); ++ch)
- _phases[ch] = TEC1ModelFuncWrapped(_frequencies[ch], alpha);
-}
-
-void PhaseFitter::FitTEC2ModelParameters(double& alpha, double& beta) const
-{
- double lowerAlpha = -40000.0e6, upperAlpha = 40000.0e6;
- bruteForceSearchTEC2Model(lowerAlpha, upperAlpha, beta);
- alpha = (lowerAlpha + upperAlpha) * 0.5;
- //beta = fitBeta(alpha, beta);
- alpha = ternarySearchTEC2ModelAlpha(lowerAlpha, upperAlpha, beta);
-}
-
-double PhaseFitter::FitDataToTEC2Model(double& alpha, double& beta)
-{
- FitTEC2ModelParameters(alpha, beta);
- double cost = TEC2ModelCost(alpha, beta);
- fillDataWithTEC2Model(alpha, beta);
- return cost;
-}
-
-double PhaseFitter::FitDataToTEC1Model(double& alpha)
-{
- FitTEC1ModelParameters(alpha);
- double cost = TEC1ModelCost(alpha);
- fillDataWithTEC1Model(alpha);
- return cost;
-}
-
-void PhaseFitter::FitTEC1ModelParameters(double& alpha) const
-{
- double lowerAlpha = -40000.0e6, upperAlpha = 40000.0e6;
- bruteForceSearchTEC1Model(lowerAlpha, upperAlpha);
- alpha = ternarySearchTEC1ModelAlpha(lowerAlpha, upperAlpha);
-}
-
-#include <iostream>
-void PhaseFitter::bruteForceSearchTEC1Model(double& lowerAlpha, double& upperAlpha) const
-{
- double minCost = std::numeric_limits<double>::max();
- double alphaOversampling = 256;
- double dAlpha = upperAlpha - lowerAlpha;
- int alphaIndex = 0;
- for(int i=0; i!=alphaOversampling; ++i) {
- // make r between [0, 1]
- double r = double(i)/alphaOversampling;
- double alpha = lowerAlpha + r*dAlpha;
- // We have to have some freedom in the fit to make sure
- // we do rule out an area with an unwripping that is correct
- // Hence we use the two-parameter model and allow beta to be fitted.
- // The ternary search will fix alpha to accomodate a zero beta.
- double curBeta = fitTEC2ModelBeta(alpha, 0.0);
- double costVal = TEC2ModelCost(alpha, curBeta);
- if(costVal < minCost) {
- minCost = costVal;
- alphaIndex = i;
- }
- }
- double newLowerAlpha = double(alphaIndex-1)/alphaOversampling*dAlpha + lowerAlpha;
- upperAlpha = double(alphaIndex+1)/alphaOversampling*dAlpha + lowerAlpha;
- lowerAlpha = newLowerAlpha;
- //std::cout << "alpha in " << lowerAlpha << "-" << upperAlpha << '\n';
-}
-
-double PhaseFitter::TEC1ModelCost(double alpha) const
-{
- double costVal = 0.0, weightSum = 0.0;
- for(size_t i=0; i!=Size(); ++i) {
- double estphase = TEC1ModelFuncWrapped(_frequencies[i], alpha);
- double dCost = fmod(std::fabs(estphase - _phases[i]), 2.0*M_PI);
- if(dCost > M_PI)
- dCost = 2.0*M_PI - dCost;
- dCost *= _weights[i];
- costVal += dCost;
- weightSum += _weights[i];
- }
- if(weightSum == 0.0)
- return 0.0;
- else
- return costVal / weightSum;
-}
-
-double PhaseFitter::ternarySearchTEC1ModelAlpha(double startAlpha, double endAlpha) const
-{
- size_t iter = 0;
- double dCost, lAlpha, rAlpha;
- do {
- lAlpha = startAlpha + (endAlpha - startAlpha) * (1.0/3.0);
- rAlpha = startAlpha + (endAlpha - startAlpha) * (2.0/3.0);
- double lCost = TEC1ModelCost(lAlpha);
- double rCost = TEC1ModelCost(rAlpha);
- if(lCost < rCost) {
- endAlpha = rAlpha;
- } else {
- startAlpha = lAlpha;
- }
- dCost = std::fabs(lCost - rCost);
- ++iter;
- //std::cout << iter << '\t' << startAlpha << '\t' << endAlpha << '\n';
- } while(dCost > _fittingAccuracy && iter < 100);
- double finalAlpha = (lAlpha + rAlpha) * 0.5;
- return finalAlpha;
-}
-
diff --git a/CEP/DP3/DPPP/src/PhaseShift.cc b/CEP/DP3/DPPP/src/PhaseShift.cc
deleted file mode 100644
index c465a33bc65167f81cd1a9d5f3681239fb69c156..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/PhaseShift.cc
+++ /dev/null
@@ -1,233 +0,0 @@
-//# PhaseShift.cc: DPPP step class to shift the data to another phase center
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/PhaseShift.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <Common/StreamUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/MatrixMath.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/casa/Quanta/MVAngle.h>
-#include <casacore/casa/BasicSL/Constants.h>
-#include <iostream>
-#include <iomanip>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- PhaseShift::PhaseShift (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsCenter (parset.getStringVector(prefix+"phasecenter"))
- {}
-
- PhaseShift::PhaseShift (DPInput* input,
- const ParameterSet& parset,
- const string& prefix,
- const vector<string>& defVal)
- : itsInput (input),
- itsName (prefix),
- itsCenter (parset.getStringVector(prefix+"phasecenter", defVal))
- {}
-
- PhaseShift::~PhaseShift()
- {}
-
- void PhaseShift::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- info().setMetaChanged();
- // Default phase center is the original one.
- MDirection newDir(itsInput->getInfo().phaseCenter());
- //// bool original = true;
- bool original = false;
- if (! itsCenter.empty()) {
- newDir = handleCenter();
- original = false;
- }
- double newRa = newDir.getValue().get()[0];
- double newDec = newDir.getValue().get()[1];
- double oldRa = infoIn.phaseCenter().getValue().get()[0];
- double oldDec = infoIn.phaseCenter().getValue().get()[1];
- Matrix<double> oldUVW(3,3);
- Matrix<double> newUVW(3,3);
- fillTransMatrix (oldUVW, oldRa, oldDec);
- fillTransMatrix (newUVW, newRa, newDec);
-
- itsMat1.reference (product(transpose(newUVW), oldUVW));
- Matrix<double> wold(oldUVW(IPosition(2,0,2),IPosition(2,2,2)));
- Matrix<double> wnew(newUVW(IPosition(2,0,2),IPosition(2,2,2)));
- Matrix<double> tt= product(transpose(Matrix<double>(wold-wnew)), oldUVW);
- itsXYZ[0] = tt(0,0);
- itsXYZ[1] = tt(0,1);
- itsXYZ[2] = tt(0,2);
- /// cout << itsXYZ[0]<<' '<<itsXYZ[1]<<' '<<itsXYZ[2]<<" ps"<<endl;
-
- info().setPhaseCenter (newDir, original);
- // Calculate 2*pi*freq/C to get correct phase term (in wavelengths).
- const Vector<double>& freq = infoIn.chanFreqs();
- itsFreqC.reserve (freq.size());
- for (uint i=0; i<freq.size(); ++i) {
- itsFreqC.push_back (2. * C::pi * freq[i] / C::c);
- }
- itsPhasors.resize (infoIn.nchan(), infoIn.nbaselines());
- }
-
- void PhaseShift::show (std::ostream& os) const
- {
- os << "PhaseShift " << itsName << std::endl;
- os << " phasecenter: " << itsCenter << std::endl;
- }
-
- void PhaseShift::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " PhaseShift " << itsName << endl;
- }
-
- bool PhaseShift::process (const DPBuffer& buf)
- {
- itsTimer.start();
- ///itsBuf.referenceFilled (buf);
- itsBuf.copy (buf);
- itsInput->fetchUVW (buf, itsBuf, itsTimer);
- int ncorr = itsBuf.getData().shape()[0];
- int nchan = itsBuf.getData().shape()[1];
- int nbl = itsBuf.getData().shape()[2];
- DBGASSERT (itsPhasors.nrow() == uint(nchan) &&
- itsPhasors.ncolumn() == uint(nbl));
- const double* mat1 = itsMat1.data();
- //# If ever in the future a time dependent phase center is used,
- //# the machine must be reset for each new time, thus each new call
- //# to process.
-#pragma omp parallel for
- for (int i=0; i<nbl; ++i) {
- Complex* __restrict__ data = itsBuf.getData().data() + i*nchan*ncorr;
- double* __restrict__ uvw = itsBuf.getUVW().data() + i*3;
- DComplex* __restrict__ phasors = itsPhasors.data() + i*nchan;
- double u = uvw[0]*mat1[0] + uvw[1]*mat1[3] + uvw[2]*mat1[6];
- double v = uvw[0]*mat1[1] + uvw[1]*mat1[4] + uvw[2]*mat1[7];
- double w = uvw[0]*mat1[2] + uvw[1]*mat1[5] + uvw[2]*mat1[8];
- double phase = itsXYZ[0]*uvw[0] + itsXYZ[1]*uvw[1] + itsXYZ[2]*uvw[2];
- for (int j=0; j<nchan; ++j) {
- // Shift the phase of the data of this baseline.
- // Converting the phase term to wavelengths (and applying 2*pi)
- // u_wvl = u_m / wvl = u_m * freq / c
- // has been done once in the beginning (in updateInfo).
- double phasewvl = phase * itsFreqC[j];
- DComplex phasor(cos(phasewvl), sin(phasewvl));
- *phasors++ = phasor;
- for (int k=0; k<ncorr; ++k) {
- *data = DComplex(*data) * phasor;
- data++;
- }
- }
- uvw[0] = u;
- uvw[1] = v;
- uvw[2] = w;
- uvw += 3;
- } //# end omp parallel for
- itsTimer.stop();
- getNextStep()->process (itsBuf);
- return true;
- }
-
- void PhaseShift::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- MDirection PhaseShift::handleCenter()
- {
- // A case-insensitive name can be given for a moving source (e.g. SUN)
- // or a known source (e.g. CygA).
- if (itsCenter.size() == 1) {
- return MDirection::makeMDirection (itsCenter[0]);
- }
- // The phase center must be given in J2000 as two values (ra,dec).
- // In the future time dependent frames can be done as in UVWFlagger.
- ASSERTSTR (itsCenter.size() == 2,
- "2 values must be given in PhaseShift phasecenter");
- ///ASSERTSTR (itsCenter.size() < 4,
- ///"Up to 3 values can be given in PhaseShift phasecenter");
- MDirection phaseCenter;
- if (itsCenter.size() == 1) {
- string str = toUpper(itsCenter[0]);
- MDirection::Types tp;
- ASSERTSTR (MDirection::getType(tp, str),
- str << " is an invalid source type"
- " in PhaseShift phasecenter");
- return MDirection(tp);
- }
- Quantity q0, q1;
- ASSERTSTR (MVAngle::read (q0, itsCenter[0]),
- itsCenter[0] << " is an invalid RA or longitude"
- " in PhaseShift phasecenter");
- ASSERTSTR (MVAngle::read (q1, itsCenter[1]),
- itsCenter[1] << " is an invalid DEC or latitude"
- " in PhaseShift phasecenter");
- MDirection::Types type = MDirection::J2000;
- if (itsCenter.size() > 2) {
- string str = toUpper(itsCenter[2]);
- MDirection::Types tp;
- ASSERTSTR (MDirection::getType(tp, str),
- str << " is an invalid direction type"
- " in PhaseShift phasecenter");
- }
- return MDirection(q0, q1, type);
- }
-
- void PhaseShift::fillTransMatrix (Matrix<double>& mat,
- double ra, double dec)
- {
- DBGASSERT (mat.nrow()==3 && mat.ncolumn()==3);
- double sinra = sin(ra);
- double cosra = cos(ra);
- double sindec = sin(dec);
- double cosdec = cos(dec);
- mat(0,0) = cosra;
- mat(1,0) = -sinra;
- mat(2,0) = 0;
- mat(0,1) = -sinra*sindec;
- mat(1,1) = -cosra*sindec;
- mat(2,1) = cosdec;
- mat(0,2) = sinra*cosdec;
- mat(1,2) = cosra*cosdec;
- mat(2,2) = sindec;
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/PointSource.cc b/CEP/DP3/DPPP/src/PointSource.cc
deleted file mode 100644
index 4d86f34698c26dbfaea224a0ad298ae855db690e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/PointSource.cc
+++ /dev/null
@@ -1,147 +0,0 @@
-//# PointSource.cc: Point source model component with optional spectral index
-//# and rotation measure.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/PointSource.h>
-#include <DPPP/ModelComponentVisitor.h>
-#include <Common/lofar_math.h>
-#include <casacore/casa/BasicSL/Constants.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-PointSource::PointSource(const Position &position)
- : itsPosition(position),
- itsRefFreq(0.0),
- itsPolarizedFraction(0.0),
- itsPolarizationAngle(0.0),
- itsRotationMeasure(0.0),
- itsHasRotationMeasure(false),
- itsHasLogarithmicSI(true)
-{
-}
-
-PointSource::PointSource(const Position &position, const Stokes &stokes)
- : itsPosition(position),
- itsStokes(stokes),
- itsRefFreq(0.0),
- itsPolarizedFraction(0.0),
- itsPolarizationAngle(0.0),
- itsRotationMeasure(0.0),
- itsHasRotationMeasure(false),
- itsHasLogarithmicSI(true)
-{
-}
-
-void PointSource::setPosition(const Position &position)
-{
- itsPosition = position;
-}
-
-void PointSource::setStokes(const Stokes &stokes)
-{
- itsStokes = stokes;
-}
-
-void PointSource::setRotationMeasure(double fraction, double angle, double rm)
-{
- itsPolarizedFraction = fraction;
- itsPolarizationAngle = angle;
- itsRotationMeasure = rm;
- itsHasRotationMeasure = true;
-}
-
-Stokes PointSource::stokes(double freq) const
-{
- Stokes stokes(itsStokes);
-
- if(hasSpectralTerms())
- {
- if(itsHasLogarithmicSI)
- {
- // Compute spectral index as:
- // (v / v0) ^ (c0 + c1 * log10(v / v0) + c2 * log10(v / v0)^2 + ...)
- // Where v is the frequency and v0 is the reference frequency.
-
- // Compute log10(v / v0).
- double base = log10(freq) - log10(itsRefFreq);
-
- // Compute c0 + log10(v / v0) * c1 + log10(v / v0)^2 * c2 + ...
- // using Horner's rule.
- double exponent = 0.0;
- typedef vector<double>::const_reverse_iterator iterator_type;
- for(iterator_type it = itsSpectralTerms.rbegin(),
- end = itsSpectralTerms.rend(); it != end; ++it)
- {
- exponent = exponent * base + *it;
- }
-
- // Compute I * (v / v0) ^ exponent, where I is the value of Stokes
- // I at the reference frequency.
- stokes.I *= pow(10., base * exponent);
- }
- else {
- double x = freq / itsRefFreq - 1.0;
- typedef vector<double>::const_reverse_iterator iterator_type;
- double val = 0.0;
- for(iterator_type it = itsSpectralTerms.rbegin(),
- end = itsSpectralTerms.rend(); it != end; ++it)
- {
- val = val * x + *it;
- }
- stokes.I += val * x;
- }
- }
-
- if(hasRotationMeasure())
- {
- double lambda = casacore::C::c / freq;
- double chi = 2.0 * (itsPolarizationAngle + itsRotationMeasure
- * lambda * lambda);
- double stokesQU = stokes.I * itsPolarizedFraction;
- stokes.Q = stokesQU * cos(chi);
- stokes.U = stokesQU * sin(chi);
- }
-
- return stokes;
-}
-
-void PointSource::accept(ModelComponentVisitor &visitor) const
-{
- visitor.visit(*this);
-}
-
-bool PointSource::hasSpectralTerms() const
-{
- return !itsSpectralTerms.empty();
-}
-
-bool PointSource::hasRotationMeasure() const
-{
- return itsHasRotationMeasure;
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/Position.cc b/CEP/DP3/DPPP/src/Position.cc
deleted file mode 100644
index 4104695b14e7cca6f3c77d052df372b5ba958bd8..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Position.cc
+++ /dev/null
@@ -1,44 +0,0 @@
-//# Position.cc: A position on the celestial sphere.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/Position.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-Position::Position()
-{
- itsPosition[0] = 0.0;
- itsPosition[1] = 0.0;
-}
-
-Position::Position(double alpha, double delta)
-{
- itsPosition[0] = alpha;
- itsPosition[1] = delta;
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/PreFlagger.cc b/CEP/DP3/DPPP/src/PreFlagger.cc
deleted file mode 100644
index 77b51bd7e586421b4ed5f0df900389b6b3423cac..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/PreFlagger.cc
+++ /dev/null
@@ -1,1194 +0,0 @@
-//# PreFlagger.cc: DPPP step class to (un)flag data on channel, baseline, time
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/PreFlagger.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPLogger.h>
-#include <Common/ParameterSet.h>
-#include <Common/StreamUtil.h>
-#include <Common/LofarLogger.h>
-
-#include <casacore/tables/TaQL/ExprNode.h>
-#include <casacore/tables/TaQL/RecordGram.h>
-#include <casacore/casa/Containers/Record.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/casa/Quanta/MVTime.h>
-#include <casacore/casa/Quanta/MVAngle.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/measures/Measures/MeasFrame.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/measures/Measures/MCEpoch.h>
-#include <iostream>
-#include <stack>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- PreFlagger::PreFlagger (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsName (prefix),
- itsInput (input),
- itsMode (SetFlag),
- itsPSet (input, parset, prefix),
- itsCount (0),
- itsFlagCounter (input->msName(), parset, prefix+"count.")
- {
- string mode = toLower(parset.getString(prefix+"mode", "set"));
- if (mode == "clear") {
- itsMode = ClearFlag;
- } else if (mode == "setcomplement" || mode == "setother") {
- itsMode = SetComp;
- } else if (mode == "clearcomplement" || mode == "clearother") {
- itsMode = ClearComp;
- } else {
- ASSERTSTR (mode=="set",
- "invalid preflagger mode: "
- "only set, clear, and set/clearcomplement/other are valid");
- }
- }
-
- PreFlagger::~PreFlagger()
- {}
-
- void PreFlagger::show (std::ostream& os) const
- {
- os << "PreFlagger " << itsName << std::endl;
- os << " mode: ";
- switch (itsMode) {
- case SetFlag:
- os << "set";
- break;
- case ClearFlag:
- os << "clear";
- break;
- case SetComp:
- os << "setcomplement";
- break;
- case ClearComp:
- os << "clearcomplement";
- break;
- }
- os << endl;
- itsPSet.show (os, false);
- }
-
- void PreFlagger::showCounts (std::ostream& os) const
- {
- os << endl << "Flags set by PreFlagger " << itsName;
- os << endl << "=======================" << endl;
- itsFlagCounter.showBaseline (os, itsCount);
- itsFlagCounter.showChannel (os, itsCount);
- }
-
- void PreFlagger::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " PreFlagger " << itsName << endl;
- }
-
- void PreFlagger::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteFlags();
- itsPSet.updateInfo (getInfo());
- // Initialize the flag counters.
- itsFlagCounter.init (getInfo());
- }
-
- bool PreFlagger::process (const DPBuffer& buf)
- {
- itsTimer.start();
- // Because no buffers are kept, we can reference the filled arrays
- // in the input buffer instead of copying them.
- itsBuffer.referenceFilled (buf);
- // Do the PSet steps and combine the result with the current flags.
- // Only count if the flag changes.
- Cube<bool>* flags = itsPSet.process (buf, itsBuffer, itsCount,
- Block<bool>(), itsTimer);
- const IPosition& shape = flags->shape();
- uint nrcorr = shape[0];
- uint nrchan = shape[1];
- uint nrbl = shape[2];
- const bool* inPtr = flags->data();
- bool* outPtr = itsBuffer.getFlags().data();
- switch (itsMode) {
- case SetFlag:
- setFlags (inPtr, outPtr, nrcorr, nrchan, nrbl, true);
- break;
- case ClearFlag:
- clearFlags (inPtr, outPtr, nrcorr, nrchan, nrbl, true, buf);
- break;
- case SetComp:
- setFlags (inPtr, outPtr, nrcorr, nrchan, nrbl, false);
- break;
- case ClearComp:
- clearFlags (inPtr, outPtr, nrcorr, nrchan, nrbl, false, buf);
- break;
- }
- itsTimer.stop();
- // Let the next step do its processing.
- getNextStep()->process (itsBuffer);
- itsCount++;
- return true;
- }
-
- void PreFlagger::setFlags (const bool* inPtr, bool* outPtr,
- uint nrcorr, uint nrchan, uint nrbl,
- bool mode)
- {
- for (uint i=0; i<nrbl; ++i) {
- for (uint j=0; j<nrchan; ++j) {
- if (*inPtr == mode && !*outPtr) {
- // Only 1st corr is counted.
- itsFlagCounter.incrBaseline(i);
- itsFlagCounter.incrChannel(j);
- for (uint k=0; k<nrcorr; ++k) {
- outPtr[k] = true;
- }
- }
- inPtr += nrcorr;
- outPtr += nrcorr;
- }
- }
- }
-
- void PreFlagger::clearFlags (const bool* inPtr, bool* outPtr,
- uint nrcorr, uint nrchan, uint nrbl,
- bool mode, const DPBuffer& buf)
- {
- const Complex* dataPtr = buf.getData().data();
- Cube<float> weights = itsInput->fetchWeights (buf, itsBuffer,
- itsTimer);
- const float* weightPtr = weights.data();
- for (uint i=0; i<nrbl; ++i) {
- for (uint j=0; j<nrchan; ++j) {
- if (*inPtr == mode) {
- bool flag = false;
- // Flags for invalid data are not cleared.
- for (uint k=0; k<nrcorr; ++k) {
- if (!isFinite(dataPtr[k].real()) ||
- !isFinite(dataPtr[k].imag()) ||
- weightPtr[k] == 0) {
- flag = true;
- break;
- }
- }
- if (*outPtr != flag) {
- itsFlagCounter.incrBaseline(i);
- itsFlagCounter.incrChannel(j);
- for (uint k=0; k<nrcorr; ++k) {
- outPtr[k] = flag;
- }
- }
- }
- inPtr += nrcorr;
- outPtr += nrcorr;
- dataPtr += nrcorr;
- weightPtr += nrcorr;
- }
- }
- }
-
- void PreFlagger::finish()
- {
- // Let the next step finish its processing.
- getNextStep()->finish();
- }
-
-
- PreFlagger::PSet::PSet (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsFlagOnUV (false),
- itsFlagOnBL (false),
- itsFlagOnAmpl (false),
- itsFlagOnPhase (false),
- itsFlagOnReal (false),
- itsFlagOnImag (false),
- itsFlagOnAzEl (false),
- itsSelBL (parset, prefix, true)
- {
- // Read all possible parameters.
- itsStrTime = parset.getStringVector (prefix+"timeofday",
- vector<string>());
- itsStrLST = parset.getStringVector (prefix+"lst",
- vector<string>());
- itsStrATime = parset.getStringVector (prefix+"abstime",
- vector<string>());
- itsStrRTime = parset.getStringVector (prefix+"reltime",
- vector<string>());
- itsTimeSlot = parset.getUintVector (prefix+"timeslot",
- vector<uint>(), true); // expand ..
- itsStrAzim = parset.getStringVector (prefix+"azimuth",
- vector<string>());
- itsStrElev = parset.getStringVector (prefix+"elevation",
- vector<string>());
- itsMinUV = parset.getDouble (prefix+"uvmmin", -1);
- itsMaxUV = parset.getDouble (prefix+"uvmmax", -1);
- itsStrFreq = parset.getStringVector (prefix+"freqrange",
- vector<string>());
- itsStrChan = parset.getStringVector (prefix+"chan", vector<string>());
- itsAmplMin = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"amplmin", string())),-1e30,
- itsFlagOnAmpl);
- itsAmplMax = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"amplmax", string())), 1e30,
- itsFlagOnAmpl);
- itsPhaseMin = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"phasemin", string())),-1e30,
- itsFlagOnPhase);
- itsPhaseMax = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"phasemax", string())), 1e30,
- itsFlagOnPhase);
- itsRealMin = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"realmin", string())),-1e30,
- itsFlagOnReal);
- itsRealMax = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"realmax", string())), 1e30,
- itsFlagOnReal);
- itsImagMin = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"imagmin", string())),-1e30,
- itsFlagOnImag);
- itsImagMax = fillValuePerCorr
- (ParameterValue (parset.getString (prefix+"imagmax", string())), 1e30,
- itsFlagOnImag);
- itsStrExpr = parset.getString (prefix+"expr", string());
- // Parse the possible pset expression and convert to RPN form.
- if (! itsStrExpr.empty()) {
- vector<string> names = exprToRpn (itsStrExpr);
- // Create PSet objects for all operands.
- itsPSets.reserve (names.size());
- for (uint i=0; i<names.size(); ++i) {
- itsPSets.push_back
- (PSet::ShPtr(new PSet(itsInput, parset, prefix+names[i]+'.')));
- }
- }
- }
-
- void PreFlagger::PSet::updateInfo (const DPInfo& info)
- {
- itsInfo = &info;
- // Fill the matrix with the baselines to flag.
- fillBLMatrix();
- // Handle the possible date/time parameters.
- itsTimes = fillTimes (itsStrTime, true, true);
- itsLST = fillTimes (itsStrLST, true, true);
- itsATimes = fillTimes (itsStrATime, false, false);
- itsRTimes = fillTimes (itsStrRTime, true, false);
- itsFlagOnTime = !(itsTimeSlot.empty() && itsTimes.empty() &&
- itsLST.empty() && itsATimes.empty() &&
- itsRTimes.empty());
- // Handle possible azimuth/elevation ranges.
- itsAzimuth = fillTimes (itsStrAzim, true, true);
- itsElevation = fillTimes (itsStrElev, true, true);
- itsFlagOnAzEl = !(itsAzimuth.empty() && itsElevation.empty());
- // Determine if to flag on UV distance.
- // If so, square the distances to avoid having to take the sqrt in flagUV.
- if (itsMinUV >= 0) {
- itsFlagOnUV = true;
- itsMinUV *= itsMinUV;
- }
- if (itsMaxUV > 0) {
- itsFlagOnUV = true;
- itsMaxUV *= itsMaxUV;
- } else {
- // Make it a very high number.
- itsMaxUV = 1e30;
- }
- ASSERTSTR (itsMinUV<itsMaxUV, "PreFlagger uvmmin should be < uvmmax");
- // Determine if only flagging on time info is done.
- itsFlagOnTimeOnly = ( !(itsFlagOnUV || itsFlagOnBL || itsFlagOnAzEl ||
- itsFlagOnAmpl || itsFlagOnPhase ||
- itsFlagOnReal || itsFlagOnImag) &&
- itsPSets.empty());
- // Size the object's buffers (used in process) correctly.
- uint nrcorr = info.ncorr();
- uint nrchan = info.nchan();
- itsFlags.resize (nrcorr, nrchan, info.nbaselines());
- itsMatchBL.resize (info.nbaselines());
- // Determine the channels to be flagged.
- if (!(itsStrChan.empty() && itsStrFreq.empty())) {
- fillChannels (info);
- if (! itsChannels.empty()) {
- itsFlagOnTimeOnly = false;
- }
- }
- // Do the same for the child steps.
- for (uint i=0; i<itsPSets.size(); ++i) {
- itsPSets[i]->updateInfo (info);
- }
- }
-
- void PreFlagger::PSet::fillChannels (const DPInfo& info)
- {
- uint nrcorr = info.ncorr();
- uint nrchan = info.nchan();
- Vector<bool> selChan(nrchan);
- if (itsStrChan.empty()) {
- selChan = true;
- } else {
- // Set selChan for channels not exceeding nr of channels.
- selChan = false;
- Record rec;
- rec.define ("nchan", nrchan);
- double result;
- for (uint i=0; i<itsStrChan.size(); ++i) {
- // Evaluate possible expressions.
- // Split the value if start..end is given.
- uint startch, endch;
- string::size_type pos = itsStrChan[i].find ("..");
- if (pos == string::npos) {
- TableExprNode node (RecordGram::parse(rec, itsStrChan[i]));
- node.get (rec, result);
- startch = uint(result+0.001);
- endch = startch;
- } else {
- ASSERTSTR (pos != 0 && pos < itsStrChan[i].size() - 2,
- "No start or end given in PreFlagger channel range "
- << itsStrChan[i]);
- TableExprNode node1
- (RecordGram::parse(rec, itsStrChan[i].substr(0,pos)));
- node1.get (rec, result);
- startch = uint(result+0.001);
- TableExprNode node2
- (RecordGram::parse(rec, itsStrChan[i].substr(pos+2)));
- node2.get (rec, result);
- endch = uint(result+0.001);
- ASSERTSTR (startch <= endch,
- "Start " << startch << " must be <= end " << endch
- << " in PreFlagger channel range " << itsStrChan[i]);
- }
- if (startch < nrchan) {
- for (uint ch=startch; ch<std::min(endch+1, nrchan); ++ch) {
- selChan[ch] = true;
- }
- }
- }
- }
- // Now determine which channels to use from given frequency ranges.
- // AND it with the channel selection given above.
- if (! itsStrFreq.empty()) {
- selChan = selChan && handleFreqRanges (itsInfo->chanFreqs());
- }
- // Turn the channels into a mask.
- itsChannels.clear();
- itsChanFlags.resize (nrcorr, nrchan);
- itsChanFlags = false;
- for (uint i=0; i<nrchan; ++i) {
- if (selChan[i]) {
- itsChannels.push_back (i);
- for (uint j=0; j<nrcorr; ++j) {
- itsChanFlags(j,i) = true;
- }
- }
- }
- }
-
- void PreFlagger::PSet::show (std::ostream& os, bool showName) const
- {
- if (showName) {
- os << " pset " << itsName << endl;
- }
- if (! itsStrExpr.empty()) {
- os << " expr: " << itsStrExpr << std::endl;
- }
- if (! itsStrLST.empty()) {
- os << " lst: " << itsStrLST << std::endl;
- }
- if (! itsStrTime.empty()) {
- os << " timeofday: " << itsStrTime << std::endl;
- }
- if (! itsStrATime.empty()) {
- os << " abstime: " << itsStrATime << std::endl;
- }
- if (! itsStrRTime.empty()) {
- os << " reltime: " << itsStrRTime << std::endl;
- }
- if (! itsTimeSlot.empty()) {
- os << " timeslot: " << itsTimeSlot << std::endl;
- }
- if (itsFlagOnBL) {
- itsSelBL.show (os);
- }
- if (itsFlagOnUV) {
- if (itsMinUV >= 0) {
- os << " uvmmin: " << sqrt(itsMinUV) << std::endl;
- } else {
- os << " uvmmin: " << itsMinUV << std::endl;
- }
- os << " uvmmax: " << sqrt(itsMaxUV) << std::endl;
- }
- if (itsFlagOnAzEl) {
- os << " azimuth: " << itsStrAzim << std::endl;
- os << " elevation: " << itsStrElev << std::endl;
- }
- if (! itsChannels.empty()) {
- os << " channel: " << itsStrChan << std::endl;
- os << " freqrange: " << itsStrFreq << std::endl;
- os << " chan to flag: " << itsChannels << std::endl;
- }
- if (itsFlagOnAmpl) {
- os << " amplmin: " << itsAmplMin << std::endl;
- os << " amplmax: " << itsAmplMax << std::endl;
- }
- if (itsFlagOnPhase) {
- os << " phasemin: " << itsPhaseMin << std::endl;
- os << " phasemax: " << itsPhaseMax << std::endl;
- }
- if (itsFlagOnReal) {
- os << " realmin: " << itsRealMin << std::endl;
- os << " realmax: " << itsRealMax << std::endl;
- }
- if (itsFlagOnImag) {
- os << " imagmin: " << itsImagMin << std::endl;
- os << " imagmax: " << itsImagMax << std::endl;
- }
- // Do it for the child steps.
- for (uint i=0; i<itsPSets.size(); ++i) {
- itsPSets[i]->show (os, true);
- }
- }
-
- Cube<bool>* PreFlagger::PSet::process (const DPBuffer& in,
- DPBuffer& out,
- uint timeSlot,
- const Block<bool>& matchBL,
- NSTimer& timer)
- {
- // No need to process it if the time mismatches or if only time selection.
- if (itsFlagOnTime) {
- if (! matchTime (out.getTime(), timeSlot)) {
- itsFlags = false;
- return &itsFlags;
- }
- }
- if (itsFlagOnTimeOnly) {
- itsFlags = itsFlagOnTime;
- return &itsFlags;
- }
- // Initialize the flags.
- itsFlags = false;
- const IPosition& shape = out.getFlags().shape();
- uint nr = shape[0] * shape[1];
- // Take over the baseline info from the parent. Default is all.
- if (matchBL.empty()) {
- itsMatchBL = true;
- } else {
- itsMatchBL = matchBL;
- }
- // The PSet tree is a combination of ORs and ANDs.
- // Depth is AND, breadth is OR.
- // In a PSet flagging is done in two stages.
- // First it is determined which baselines are not flagged. It is kept
- // in the itsMatchBL block.
- // This is passed to the children who do their flagging. In this way
- // a child can minimize the amount of work to do.
- // The resulting flags of the children are handled according to the
- // operators in the RPN list.
-
- // First flag on baseline if necessary. Stop if no matches.
- if (itsFlagOnBL && !flagBL()) {
- return &itsFlags;
- }
- // Flag on UV distance if necessary.
- if (itsFlagOnUV && !flagUV (itsInput->fetchUVW (in, out,
- timer))) {
- return &itsFlags;
- }
- // Flag on AzEl is necessary.
- if (itsFlagOnAzEl && !flagAzEl (out.getTime())) {
- return &itsFlags;
- }
- // Convert each baseline flag to a flag per correlation/channel.
- bool* flagPtr = itsFlags.data();
- for (uint i=0; i<itsMatchBL.size(); ++i) {
- if (itsMatchBL[i]) {
- std::fill (flagPtr, flagPtr+nr, itsMatchBL[i]);
- }
- flagPtr += nr;
- }
- // Flag on channel if necessary.
- if (! itsChannels.empty()) {
- flagChannels();
- }
- // Flag on amplitude, phase or real/imaginary if necessary.
- if (itsFlagOnAmpl) {
- flagAmpl (amplitude(out.getData()));
- }
- if (itsFlagOnReal) {
- flagReal (out.getData());
- }
- if (itsFlagOnImag) {
- flagImag (out.getData());
- }
- if (itsFlagOnPhase) {
- flagPhase (out.getData());
- }
- // Evaluate the PSet expression.
- // The expression is in RPN notation. A stack of array pointers is used
- // to keep track of intermediate results. The arrays (in the PSet objects)
- // are reused to AND or OR subexpressions. This can be done harmlessly
- // and saves the creation of too many arrays.
- if (! itsPSets.empty()) {
- std::stack<Cube<bool>*> results;
- for (vector<int>::const_iterator oper = itsRpn.begin();
- oper != itsRpn.end(); ++oper) {
- if (*oper >= 0) {
- results.push (itsPSets[*oper]->process (in, out, timeSlot,
- itsMatchBL, timer));
- } else if (*oper == OpNot) {
- Cube<bool>* left = results.top();
- // No ||= operator exists, so use the transform function.
- transformInPlace (left->cbegin(), left->cend(),
- std::logical_not<bool>());
- } else {
- ASSERT (*oper==OpOr || *oper==OpAnd);
- Cube<bool>* right = results.top();
- results.pop();
- Cube<bool>* left = results.top();
- // No ||= operator exists, so use the transform function.
- if (*oper == OpOr) {
- transformInPlace (left->cbegin(), left->cend(),
- right->cbegin(), std::logical_or<bool>());
- } else {
- transformInPlace (left->cbegin(), left->cend(),
- right->cbegin(), std::logical_and<bool>());
- }
- }
- }
- // Finally AND the children's flags with the flags of this pset.
- ASSERT (results.size() == 1);
- Cube<bool>* mflags = results.top();
- transformInPlace (itsFlags.cbegin(), itsFlags.cend(),
- mflags->cbegin(), std::logical_and<bool>());
- }
- return &itsFlags;
- }
-
- bool PreFlagger::PSet::matchTime (double time, uint timeSlot) const
- {
- if (!itsATimes.empty() &&
- !matchRange (time, itsATimes)) {
- return false;
- }
- if (!itsRTimes.empty() &&
- !matchRange (time-itsInfo->startTime(), itsRTimes)) {
- return false;
- }
- if (!itsTimes.empty()) {
- MVTime mvtime(time/86400); // needs time in days
- double timeofday = time - int(mvtime.day()) * 86400.;
- if (!matchRange (timeofday, itsTimes)) {
- return false;
- }
- }
- if (!itsTimeSlot.empty()) {
- if (std::find (itsTimeSlot.begin(), itsTimeSlot.end(), timeSlot) ==
- itsTimeSlot.end()) {
- return false;
- }
- }
- if (!itsLST.empty()) {
- // Convert time from UTC to Local Apparent Sidereal Time.
- MeasFrame frame;
- frame.set (itsInfo->arrayPos());
- Quantity qtime(time, "s");
- MEpoch lst = MEpoch::Convert (MEpoch(MVEpoch(qtime), MEpoch::UTC),
- MEpoch::Ref(MEpoch::LAST, frame))();
- double lstSec = lst.getValue().get(); // in days
- lstSec -= int(lstSec); // time of day
- if (!matchRange (lstSec*86400, itsLST)) { // use seconds
- return false;
- }
- }
- return true;
- }
-
- bool PreFlagger::PSet::matchRange (double v,
- const vector<double>& ranges) const
- {
- for (uint i=0; i<ranges.size(); i+=2) {
- if (v > ranges[i] && v < ranges[i+1]) {
- return true;
- }
- }
- return false;
- }
-
- bool PreFlagger::PSet::flagUV (const Matrix<double>& uvw)
- {
- bool match = false;
- uint nrbl = itsMatchBL.size();
- const double* uvwPtr = uvw.data();
- for (uint i=0; i<nrbl; ++i) {
- if (itsMatchBL[i]) {
- // UV-distance is sqrt(u^2 + v^2).
- // The sqrt is not needed because minuv and maxuv are squared.
- double uvdist = uvwPtr[0] * uvwPtr[0] + uvwPtr[1] * uvwPtr[1];
- if (uvdist >= itsMinUV && uvdist <= itsMaxUV) {
- // UV-dist mismatches, so do not flag baseline.
- itsMatchBL[i] = false;
- } else {
- match = true;
- }
- }
- uvwPtr += 3;
- }
- return match;
- }
-
- bool PreFlagger::PSet::flagBL()
- {
- bool match = false;
- uint nrbl = itsMatchBL.size();
- const Int* ant1Ptr = itsInfo->getAnt1().data();
- const Int* ant2Ptr = itsInfo->getAnt2().data();
- for (uint i=0; i<nrbl; ++i) {
- if (itsMatchBL[i]) {
- if (! itsFlagBL(ant1Ptr[i], ant2Ptr[i])) {
- // do not flag this baseline
- itsMatchBL[i] = false;
- } else {
- match = true;
- }
- }
- }
- return match;
- }
-
- bool PreFlagger::PSet::flagAzEl (double time)
- {
- bool match = false;
- uint nrbl = itsMatchBL.size();
- const Int* ant1Ptr = itsInfo->getAnt1().data();
- const Int* ant2Ptr = itsInfo->getAnt2().data();
- // Calculate AzEl for each flagged antenna for this time slot.
- MeasFrame frame;
- Quantity qtime(time, "s");
- MEpoch epoch(MVEpoch(qtime), MEpoch::UTC);
- frame.set (epoch);
- MDirection::Convert converter (itsInfo->phaseCenter(),
- MDirection::Ref(MDirection::AZEL, frame));
- uint nrant = itsInfo->antennaNames().size();
- Block<bool> done(nrant, false);
- for (uint i=0; i<nrbl; ++i) {
- if (itsMatchBL[i]) {
- // If needed, check if ant1 matches AzEl criterium.
- // If not matching, itsMatchBL is cleared for this baseline and all
- // subsequent baselines with this antenna.
- int a1 = ant1Ptr[i];
- int a2 = ant2Ptr[i];
- if (!done[a1]) {
- frame.set (itsInfo->antennaPos()[a1]);
- testAzEl (converter, i, a1, ant1Ptr, ant2Ptr);
- done[a1]= true;
- }
- // If needed, check if ant2 matches AzEl criterium.
- if (itsMatchBL[i] && !done[a2]) {
- frame.set (itsInfo->antennaPos()[a2]);
- testAzEl (converter, i, a2, ant1Ptr, ant2Ptr);
- done[a2] = true;
- }
- if (itsMatchBL[i]) {
- match = true;
- }
- }
- }
- return match;
- }
-
- void PreFlagger::PSet::testAzEl (MDirection::Convert& converter,
- uint blnr, int ant,
- const int* ant1, const int* ant2)
- {
- // Calculate AzEl (n seconds because ranges are in seconds too).
- MVDirection mvAzel (converter().getValue());
- Vector<double> azel = mvAzel.getAngle("s").getValue();
- double az = azel[0];
- double el = azel[1];
- if (az < 0) az += 86400;
- if (el < 0) el += 86400;
- // If outside the ranges, there is no match.
- // Set no match for all baselines containing this antenna.
- // It needs to be done from this baseline on, because the earlier
- // baselines have already been handled.
- bool res = ((itsAzimuth.empty() || matchRange(az, itsAzimuth)) &&
- (itsElevation.empty() || matchRange(el, itsElevation)));
- if (!res) {
- for (uint i=blnr; i<itsMatchBL.size(); ++i) {
- if (ant1[i] == ant || ant2[i] == ant) {
- itsMatchBL[i] = false;
- }
- }
- }
- }
-
- void PreFlagger::PSet::flagAmpl (const Cube<float>& values)
- {
- const IPosition& shape = values.shape();
- uint nrcorr = shape[0];
- uint nr = shape[1] * shape[2];
- const float* valPtr = values.data();
- bool* flagPtr = itsFlags.data();
- for (uint i=0; i<nr; ++i) {
- bool flag = false;
- for (uint j=0; j<nrcorr; ++j) {
- if (valPtr[j] < itsAmplMin[j] || valPtr[j] > itsAmplMax[j]) {
- flag = true;
- break;
- }
- }
- if (!flag) {
- for (uint j=0; j<nrcorr; ++j) {
- flagPtr[j] = false;
- }
- }
- valPtr += nrcorr;
- flagPtr += nrcorr;
- }
- }
-
- void PreFlagger::PSet::flagPhase (const Cube<Complex>& values)
- {
- const IPosition& shape = values.shape();
- uint nrcorr = shape[0];
- uint nr = shape[1] * shape[2];
- const Complex* valPtr = values.data();
- bool* flagPtr = itsFlags.data();
- for (uint i=0; i<nr; ++i) {
- bool flag = false;
- for (uint j=0; j<nrcorr; ++j) {
- float phase = arg(valPtr[j]);
- if (phase < itsPhaseMin[j] || phase > itsPhaseMax[j]) {
- flag = true;
- break;
- }
- }
- if (!flag) {
- for (uint j=0; j<nrcorr; ++j) {
- flagPtr[j] = false;
- }
- }
- valPtr += nrcorr;
- flagPtr += nrcorr;
- }
- }
-
- void PreFlagger::PSet::flagReal (const Cube<Complex>& values)
- {
- const IPosition& shape = values.shape();
- uint nrcorr = shape[0];
- uint nr = shape[1] * shape[2];
- const Complex* valPtr = values.data();
- bool* flagPtr = itsFlags.data();
- for (uint i=0; i<nr; ++i) {
- bool flag = false;
- for (uint j=0; j<nrcorr; ++j) {
- if (valPtr[j].real() < itsRealMin[j] ||
- valPtr[j].real() > itsRealMax[j]) {
- flag = true;
- break;
- }
- }
- if (!flag) {
- for (uint j=0; j<nrcorr; ++j) {
- flagPtr[j] = false;
- }
- }
- valPtr += nrcorr;
- flagPtr += nrcorr;
- }
- }
-
- void PreFlagger::PSet::flagImag (const Cube<Complex>& values)
- {
- const IPosition& shape = values.shape();
- uint nrcorr = shape[0];
- uint nr = shape[1] * shape[2];
- const Complex* valPtr = values.data();
- bool* flagPtr = itsFlags.data();
- for (uint i=0; i<nr; ++i) {
- bool flag = false;
- for (uint j=0; j<nrcorr; ++j) {
- if (valPtr[j].imag() < itsImagMin[j] ||
- valPtr[j].imag() > itsImagMax[j]) {
- flag = true;
- break;
- }
- }
- if (!flag) {
- for (uint j=0; j<nrcorr; ++j) {
- flagPtr[j] = false;
- }
- }
- valPtr += nrcorr;
- flagPtr += nrcorr;
- }
- }
-
- void PreFlagger::PSet::flagChannels()
- {
- const IPosition& shape = itsFlags.shape();
- uint nr = shape[0] * shape[1];
- uint nrbl = shape[2];
- bool* flagPtr = itsFlags.data();
- for (uint i=0; i<nrbl; ++i) {
- transformInPlace (flagPtr, flagPtr+nr,
- itsChanFlags.cbegin(), std::logical_and<bool>());
- flagPtr += nr;
- }
- }
-
- // See http://montcs.bloomu.edu/~bobmon/Information/RPN/infix2rpn.shtml
- // for the algorithm used here.
- // Some code was added to check if no two subsequent operators or names
- // are given.
- vector<string> PreFlagger::PSet::exprToRpn (const string& origExpr)
- {
- // Operators & (or &&) | (or ||) and , are used as well as parentheses.
- // The operators must have a value in order of precedence, thus &&
- // has a higher precedence than || (as in C).
- string expr = toUpper(origExpr);
- std::stack<int> tokens;
- vector<string> names;
- uint i=0;
- bool hadName = false; // the last token was a name.
- while (i < expr.size()) {
- int oper = 0;
- // skip whitespace.
- // Look for parenthesis or operator.
- if (expr[i] == ' ' || expr[i] == '\t') {
- i++;
- } else if (expr[i] == '(') {
- ASSERTSTR (!hadName, "no operator before opening parenthesis at pos. "
- << i << " in expression: " << origExpr);
- oper = OpParen;
- tokens.push (oper);
- i++;
- } else if (expr[i] == '|') {
- oper = OpOr;
- i++;
- if (i < expr.size() && expr[i] == '|') i++;
- } else if (expr[i] == ',') {
- oper = OpOr;
- i++;
- } else if (expr[i] == '&') {
- oper = OpAnd;
- i++;
- if (i < expr.size() && expr[i] == '&') i++;
- } else if (expr[i] == '!') {
- oper = OpNot;
- i++;
- } else if (expr.size()-i >= 3 && (expr.substr(i,3) == "OR " ||
- expr.substr(i,3) == "OR\t" ||
- expr.substr(i,3) == "OR!" ||
- expr.substr(i,3) == "OR(")) {
- oper = OpOr;
- i+=2;
- } else if (expr.size()-i == 2 && (expr.substr(i,2) == "OR")) {
- oper = OpOr;
- i+=2;
- } else if (expr.size()-i >= 4 && (expr.substr(i,4) == "AND " ||
- expr.substr(i,4) == "AND\t" ||
- expr.substr(i,4) == "AND!" ||
- expr.substr(i,4) == "AND(")) {
- oper = OpAnd;
- i+=3;
- } else if (expr.size()-i == 3 && (expr.substr(i,3) == "AND")) {
- oper = OpAnd;
- i+=3;
- } else if (expr.size()-i >= 4 && (expr.substr(i,4) == "NOT " ||
- expr.substr(i,4) == "NOT\t" ||
- expr.substr(i,4) == "NOT(")) {
- oper = OpNot;
- i+=3;
- } else if (expr.size()-i == 3 && (expr.substr(i,3) == "NOT")) {
- oper = OpNot;
- i+=3;
- } else if (expr[i] == ')') {
- // Closing parenthesis. Push till opening parenthesis found.
- ASSERTSTR (hadName, "no set name before closing parenthesis at pos. "
- << i << " in expression: " << origExpr);
- while (true) {
- ASSERTSTR (!tokens.empty(), "mismatched parentheses at pos. "
- << i << " in expression: " << origExpr);
- if (tokens.top() == OpParen) {
- tokens.pop();
- break;
- }
- itsRpn.push_back (tokens.top());
- tokens.pop();
- }
- i++;
- } else {
- // No operator, thus it must be an operand (a set name).
- int st=i;
- ASSERTSTR (!hadName, "No operator between set names at pos. "
- << i << " in expression: " << origExpr);
- while (i < expr.size() &&
- expr[i] != ' ' && expr[i] != '\t' &&
- expr[i] != '(' && expr[i] != ')' && expr[i] != '!' &&
- expr[i] != ',' && expr[i] != '&' && expr[i] != '|') {
- i++;
- }
- hadName = true;
- itsRpn.push_back (names.size());
- String setName (origExpr.substr(st, i-st));
- // Check the name is valid (no special characters).
- ASSERTSTR (setName.matches (RXidentifier),
- "Invalid set name " << setName
- << " used in set expression " << origExpr);
- names.push_back (setName);
- }
- if (oper < OpParen) {
- // Check if an operator was preceeded correctly.
- if (oper == OpNot) {
- ASSERTSTR (!hadName, "No set name before operator ! at pos. "
- << i << " in expression: " << origExpr);
- } else {
- ASSERTSTR (hadName, "No set name before operator at pos. "
- << i << " in expression: " << origExpr);
- }
- hadName = false;
- // Push till lower precedence found.
- while (!tokens.empty() && tokens.top() < oper) {
- itsRpn.push_back (tokens.top());
- tokens.pop();
- }
- tokens.push (oper);
- }
- }
- ASSERTSTR (hadName, "no set name after last operator in expression: "
- << origExpr);
- while (!tokens.empty()) {
- ASSERTSTR (tokens.top()<OpParen,
- "mismatched parentheses in expression: " << origExpr);
- itsRpn.push_back (tokens.top());
- tokens.pop();
- }
- return names;
- }
-
- vector<double> PreFlagger::PSet::fillTimes (const vector<string>& vec,
- bool asTime,
- bool canEndBeforeStart)
- {
- vector<double> result;
- result.reserve (2*vec.size());
- // A time range can be given as time..time or time+-value.
- for (vector<string>::const_iterator str = vec.begin();
- str != vec.end(); ++str) {
- // Find the .. or +- token.
- bool usepm = false;
- string::size_type pos = str->find ("..");
- if (pos == string::npos) {
- usepm = true;
- pos = str->find ("+-");
- ASSERTSTR (pos != string::npos, "PreFlagger time range '" << *str
- << "' should be range using .. or +-");
- }
- // Get the time or datetime in seconds. The values must be positive.
- double v1 = getSeconds (str->substr(0, pos), asTime, false);
- double v2 = getSeconds (str->substr(pos+2), asTime, usepm);
- ASSERTSTR (v1>=0 && v2>=0, "PreFlagger time range " << *str
- << " must have positive values");
- if (usepm) {
- double pm = v2;
- v2 = v1 + pm;
- v1 -= pm;
- }
- // If time is used, values around midnight can be given.
- // Note there are 86400 seconds in a day.
- // They are split in 2 ranges.
- if (!canEndBeforeStart) {
- ASSERTSTR (v1<v2, "PreFlagger time range " << *str << " is invalid");
- } else {
- if (v1 < 0) {
- v1 += 86400;
- }
- if (v2 > 86400) {
- v2 -= 86400;
- }
- }
- if (v1 < v2) {
- result.push_back (v1);
- result.push_back (v2);
- } else {
- result.push_back (-1);
- result.push_back (v2);
- result.push_back (v1);
- result.push_back (86401);
- }
- }
- return result;
- }
-
- double PreFlagger::PSet::getSeconds (const string& str, bool asTime,
- bool usepm)
- {
- Quantity q;
- if (asTime || usepm) {
- ASSERTSTR (MVAngle::read(q, str, true),
- "PreFlagger time " << str << " is invalid");
- } else {
- // It should be a proper date/time, so MVAngle::read should fail.
- ASSERTSTR (!MVAngle::read(q, str, true),
- "PreFlagger datetime " << str
- << " is not a proper date/time");
- ASSERTSTR (MVTime::read(q, str, true),
- "PreFlagger datetime " << str << " is invalid"
- << " is not a proper date/time");
- }
- double v = q.getValue ("s");
- if (usepm) {
- ASSERTSTR (v>0, "Preflagger time plusminus value " << str
- << " must be positive");
- }
- return v;
- }
-
- vector<float> PreFlagger::PSet::fillValuePerCorr
- (const ParameterValue& value, float defVal, bool& doFlag)
- {
- // Initialize with the default value per correlation.
- vector<float> result(4);
- std::fill (result.begin(), result.end(), defVal);
- if (! value.get().empty()) {
- if (value.isVector()) {
- // Defined as a vector, take the values given.
- vector<string> valstr = value.getStringVector();
- uint sz = std::min(valstr.size(), result.size());
- if (sz > 0) {
- // It contains a value, so set that flagging is done.
- doFlag = true;
- for (uint i=0; i<sz; ++i) {
- if (! valstr[i].empty()) {
- result[i] = strToFloat(valstr[i]);
- }
- }
- }
- } else {
- // A single value means use it for all correlations.
- doFlag = true;
- std::fill (result.begin(), result.end(), value.getFloat());
- }
- }
- return result;
- }
-
- void PreFlagger::PSet::fillBLMatrix()
- {
- itsFlagOnBL = itsSelBL.hasSelection();
- if (itsFlagOnBL) {
- itsFlagBL.reference (itsSelBL.apply (*itsInfo));
- }
- }
-
- Vector<bool> PreFlagger::PSet::handleFreqRanges
- (const Vector<double>& chanFreqs)
- {
- uint nrchan = chanFreqs.size();
- Vector<bool> selChan(nrchan, false);
- // A frequency range can be given as value..value or value+-value.
- // Units can be given for each value; if one is given it applies to both.
- // Default unit is MHz.
- for (vector<string>::const_iterator str = itsStrFreq.begin();
- str != itsStrFreq.end(); ++str) {
- // Find the .. or +- token.
- bool usepm = false;
- string::size_type pos = str->find ("..");
- if (pos == string::npos) {
- usepm = true;
- pos = str->find ("+-");
- ASSERTSTR (pos != string::npos, "PreFlagger freqrange '" << *str
- << "' should be range using .. or +-");
- }
- string str1 = str->substr (0, pos);
- string str2 = str->substr (pos+2);
- String u1, u2;
- double v1, v2;
- getValue (str1, v1, u1);
- // Default unit for 2nd value is that of 1st value.
- u2 = u1;
- getValue (str2, v2, u2);
- // If no unit, use MHz.
- if (u2.empty()) {
- u2 = "MHz";
- }
- // Default unit of 1st value is that of 2nd value.
- if (u1.empty()) {
- u1 = u2;
- }
- v1 = getFreqHz (v1, u1);
- v2 = getFreqHz (v2, u2);
- if (usepm) {
- double pm = v2;
- v2 = v1 + pm;
- v1 -= pm;
- }
- // Add any channel inside this range.
- for (uint i=0; i<chanFreqs.size(); ++i) {
- if (chanFreqs[i] > v1 && chanFreqs[i] < v2) {
- selChan[i] = true;
- }
- }
- }
- return selChan;
- }
-
- void PreFlagger::PSet::getValue (const string& str, double& value,
- String& unit)
- {
- // See if a unit is given at the end.
- String v(str);
- // Remove possible trailing blanks.
- rtrim(v);
- Regex regex("[a-zA-Z]+$");
- string::size_type pos = v.index (regex);
- if (pos != String::npos) {
- unit = v.from (pos);
- v = v.before (pos);
- }
- // Set value and unit.
- value = strToDouble(v);
- }
-
- double PreFlagger::PSet::getFreqHz (double value, const String& unit)
- {
- Quantity q(value, unit);
- return q.getValue ("Hz");
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/Predict.cc b/CEP/DP3/DPPP/src/Predict.cc
deleted file mode 100644
index d74170976bd3398fc33e53f3d605c67f78fbaaf8..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Predict.cc
+++ /dev/null
@@ -1,429 +0,0 @@
-//# GainCal.cc: DPPP step class to predict visibilities
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/Predict.h>
-
-#include <iostream>
-//#include <iomanip>
-#include <Common/ParameterSet.h>
-#include <Common/Timer.h>
-#include <Common/OpenMP.h>
-#include <Common/StreamUtil.h>
-#include <ParmDB/ParmDBMeta.h>
-#include <ParmDB/PatchInfo.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/FlagCounter.h>
-#include <DPPP/Position.h>
-#include <DPPP/ApplyBeam.h>
-#include <DPPP/Simulator.h>
-
-#include <DPPP/Stokes.h>
-#include <DPPP/PointSource.h>
-#include <DPPP/GaussianSource.h>
-#include <ParmDB/SourceDB.h>
-
-#include <casacore/casa/Arrays/Array.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/OS/File.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/tables/Tables/RefRows.h>
-
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-using namespace casacore;
-using namespace LOFAR::BBS;
-
-namespace LOFAR {
- namespace DPPP {
-
- Predict::Predict (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- {
- init(input, parset, prefix, parset.getStringVector(prefix + "sources",
- vector<string>()));
- }
-
- Predict::Predict (DPInput* input,
- const ParameterSet& parset,
- const string& prefix,
- const vector<string>& sourcePatterns)
- {
- init(input, parset, prefix, sourcePatterns);
- }
-
- void Predict::init(DPInput* input,
- const ParameterSet& parset,
- const string& prefix, const vector<string>& sourcePatterns) {
- itsInput = input;
- itsName = prefix;
- itsSourceDBName = parset.getString (prefix + "sourcedb");
- setOperation(parset.getString (prefix + "operation", "replace"));
- itsApplyBeam = parset.getBool (prefix + "usebeammodel", false);
- itsDebugLevel = parset.getInt (prefix + "debuglevel", 0);
- itsPatchList = vector<Patch::ConstPtr> ();
-
- ASSERT(File(itsSourceDBName).exists());
- BBS::SourceDB sourceDB(BBS::ParmDBMeta("", itsSourceDBName), false);
-
- // Save directions specifications to pass to applycal
- stringstream ss;
- ss << sourcePatterns;
- itsDirectionsStr = ss.str();
-
- vector<string> patchNames=makePatchList(sourceDB, sourcePatterns);
- itsPatchList = makePatches (sourceDB, patchNames, patchNames.size());
-
- if (itsApplyBeam) {
- itsUseChannelFreq=parset.getBool (prefix + "usechannelfreq", true);
- itsOneBeamPerPatch=parset.getBool (prefix + "onebeamperpatch", true);
-
- string mode=toLower(parset.getString(prefix + "beammode","default"));
- ASSERT (mode=="default" || mode=="array_factor" || mode=="element");
- if (mode=="default") {
- itsBeamMode=ApplyBeam::DEFAULT;
- } else if (mode=="array_factor") {
- itsBeamMode=ApplyBeam::ARRAY_FACTOR;
- } else if (mode=="element") {
- itsBeamMode=ApplyBeam::ELEMENT;
- } else {
- THROW(Exception, "Beammode should be DEFAULT, ARRAY_FACTOR or ELEMENT");
- }
-
- // Rework patch list to contain a patch for every source
- if (!itsOneBeamPerPatch) {
- itsPatchList = makeOnePatchPerComponent(itsPatchList);
- }
- }
-
- // If called from h5parmpredict, applycal gets set by that step,
- // so must not be read from parset
- if (parset.isDefined(prefix + "applycal.parmdb") ||
- parset.isDefined(prefix + "applycal.steps")) {
- setApplyCal(input, parset, prefix + "applycal.");
- } else {
- itsDoApplyCal=false;
- }
-
- itsSourceList = makeSourceList(itsPatchList);
-
- // Determine whether any sources are polarized. If not, enable Stokes-I-
- // only mode (note that this mode cannot be used with itsApplyBeam)
- if (itsApplyBeam) {
- itsStokesIOnly = false;
- } else {
- itsStokesIOnly = !checkPolarized(sourceDB, patchNames, patchNames.size());
- }
- }
-
- void Predict::setApplyCal(DPInput* input,
- const ParameterSet& parset,
- const string& prefix) {
- itsDoApplyCal=true;
- itsApplyCalStep=ApplyCal(input, parset, prefix, true,
- itsDirectionsStr);
- ASSERT(!(itsOperation!="replace" &&
- parset.getBool(prefix + "applycal.updateweights", false)));
- itsResultStep=new ResultStep();
- itsApplyCalStep.setNextStep(DPStep::ShPtr(itsResultStep));
- }
-
- Predict::Predict()
- {}
-
- Predict::~Predict()
- {}
-
- void Predict::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
-
- uint nBl=info().nbaselines();
- for (uint i=0; i<nBl; ++i) {
- itsBaselines.push_back (Baseline(info().getAnt1()[i],
- info().getAnt2()[i]));
- }
-
- MDirection dirJ2000(MDirection::Convert(infoIn.phaseCenter(),
- MDirection::J2000)());
- Quantum<Vector<Double> > angles = dirJ2000.getAngle();
- itsPhaseRef = Position(angles.getBaseValue()[0],
- angles.getBaseValue()[1]);
-
- //const size_t nDr = itsPatchList.size();
- const size_t nSt = info().nantenna();
- const size_t nCh = info().nchan();
- const size_t nCr = info().ncorr();
-
- itsUVW.resize(3,nSt);
- itsUVWSplitIndex = nsetupSplitUVW (info().nantenna(), info().getAnt1(),
- info().getAnt2());
-
- itsModelVis.resize(OpenMP::maxThreads());
- itsModelVisPatch.resize(OpenMP::maxThreads());
- itsBeamValues.resize(OpenMP::maxThreads());
-
- // Create the Measure ITRF conversion info given the array position.
- // The time and direction are filled in later.
- itsMeasConverters.resize(OpenMP::maxThreads());
- itsMeasFrames.resize(OpenMP::maxThreads());
- itsAntBeamInfo.resize(OpenMP::maxThreads());
-
- for (uint thread=0;thread<OpenMP::maxThreads();++thread) {
- if (itsStokesIOnly) {
- itsModelVis[thread].resize(1,nCh,nBl);
- } else {
- itsModelVis[thread].resize(nCr,nCh,nBl);
- }
- if (itsApplyBeam) {
- itsModelVisPatch[thread].resize(nCr,nCh,nBl);
- itsBeamValues[thread].resize(nSt*nCh);
- itsMeasFrames[thread].set (info().arrayPosCopy());
- itsMeasFrames[thread].set (MEpoch(MVEpoch(info().startTime()/86400),
- MEpoch::UTC));
- itsMeasConverters[thread].set (MDirection::J2000,
- MDirection::Ref(MDirection::ITRF, itsMeasFrames[thread]));
- itsInput->fillBeamInfo (itsAntBeamInfo[thread], info().antennaNames());
- }
- }
-
- if (itsDoApplyCal) {
- info()=itsApplyCalStep.setInfo(info());
- }
- }
-
- std::pair<double, double> Predict::getFirstDirection() const {
- std::pair<double, double> res;
- res.first = itsPatchList[0]->position()[0];
- res.second = itsPatchList[0]->position()[1];
- return res;
- }
-
- void Predict::setOperation(const std::string& operation) {
- itsOperation=operation;
- ASSERT(itsOperation=="replace" || itsOperation=="add" ||
- itsOperation=="subtract");
- }
-
-
- void Predict::show (std::ostream& os) const
- {
- os << "Predict " << itsName << endl;
- os << " sourcedb: " << itsSourceDBName << endl;
- os << " number of patches: " << itsPatchList.size() << endl;
- os << " number of sources: " << itsSourceList.size() << endl;
- os << " all unpolarized: " << boolalpha << itsStokesIOnly << endl;
- os << " apply beam: " << boolalpha << itsApplyBeam << endl;
- if (itsApplyBeam) {
- os << " mode: ";
- if (itsBeamMode==ApplyBeam::DEFAULT)
- os<<"default";
- else if (itsBeamMode==ApplyBeam::ARRAY_FACTOR)
- os<<"array_factor";
- else os<<"element";
- os << endl;
- os << " use channelfreq: " << boolalpha << itsUseChannelFreq << endl;
- os << " one beam per patch:" << boolalpha << itsOneBeamPerPatch << endl;
- }
- os << " operation: "<<itsOperation << endl;
- os << " threads: "<<OpenMP::maxThreads()<<endl;
- if (itsDoApplyCal) {
- itsApplyCalStep.show(os);
- }
- }
-
- void Predict::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " Predict " << itsName << endl;
- }
-
- bool Predict::process (const DPBuffer& bufin)
- {
- itsTimer.start();
- itsTempBuffer.copy (bufin);
- itsInput->fetchUVW(bufin, itsTempBuffer, itsTimer);
- itsInput->fetchWeights(bufin, itsTempBuffer, itsTimer);
-
- // Determine the various sizes.
- //const size_t nDr = itsPatchList.size();
- const size_t nSt = info().nantenna();
- const size_t nBl = info().nbaselines();
- const size_t nCh = info().nchan();
- const size_t nCr = info().ncorr();
- const size_t nSamples = nBl * nCh * nCr;
-
- double time = itsTempBuffer.getTime();
-
- itsTimerPredict.start();
-
- nsplitUVW(itsUVWSplitIndex, itsBaselines, itsTempBuffer.getUVW(), itsUVW);
-
- //Set up directions for beam evaluation
- StationResponse::vector3r_t refdir, tiledir;
-
- if (itsApplyBeam) {
- for (uint thread=0;thread<OpenMP::maxThreads();++thread) {
-#pragma omp critical(initialmeasuresconversion)
- {
- itsMeasFrames[thread].resetEpoch (MEpoch(MVEpoch(time/86400),
- MEpoch::UTC));
- //Do a conversion on all threads, because converters are not
- //thread safe and apparently need to be used at least once
- refdir = dir2Itrf(info().delayCenter(),itsMeasConverters[thread]);
- tiledir = dir2Itrf(info().tileBeamDir(),itsMeasConverters[thread]);
- }
- }
- }
-
-#pragma omp parallel
-{
- uint thread=OpenMP::threadNum();
- itsModelVis[thread]=dcomplex();
- itsModelVisPatch[thread]=dcomplex();
-
- //When applying beam, simulate into patch vector,
- Cube<dcomplex>& simulatedest=(itsApplyBeam?itsModelVisPatch[thread]
- :itsModelVis[thread]);
-
- Simulator simulator(itsPhaseRef, nSt, nBl, nCh, itsBaselines,
- info().chanFreqs(), itsUVW, simulatedest,
- itsStokesIOnly);
-
- Patch::ConstPtr curPatch;
-#pragma omp for
- for (uint i=0;i<itsSourceList.size();++i) {
- // Keep on predicting, only apply beam when an entire patch is done
- if (itsApplyBeam && curPatch!=itsSourceList[i].second && curPatch!=0) {
- addBeamToData (curPatch, time, refdir, tiledir, thread, nSamples,
- itsModelVisPatch[thread].data());
- }
- simulator.simulate(itsSourceList[i].first);
- curPatch=itsSourceList[i].second;
- }
-
- // Apply beam to the last patch
- if (itsApplyBeam && curPatch!=0) {
- addBeamToData (curPatch, time, refdir, tiledir, thread, nSamples,
- itsModelVisPatch[thread].data());
- }
-}
-
- // Add all thread model data to one buffer
- itsTempBuffer.getData()=Complex();
- Complex* tdata=itsTempBuffer.getData().data();
- for (uint thread=0;thread<OpenMP::maxThreads();++thread) {
- if (itsStokesIOnly) {
- for (uint i=0,j=0;i<nSamples;i+=nCr,j++) {
- tdata[i] += itsModelVis[thread].data()[j];
- tdata[i+nCr-1] += itsModelVis[thread].data()[j];
- }
- } else {
- std::transform(tdata, tdata+nSamples, itsModelVis[thread].data(),
- tdata, std::plus<dcomplex>());
- }
- }
-
- // Call ApplyCal step
- if (itsDoApplyCal) {
- itsApplyCalStep.process(itsTempBuffer);
- itsTempBuffer=itsResultStep->get();
- tdata=itsTempBuffer.getData().data();
- }
-
- // Put predict result from temp buffer into the 'real' buffer
- if (itsOperation=="replace") {
- itsBuffer=itsTempBuffer;
- } else {
- itsBuffer.copy(bufin);
- Complex* data=itsBuffer.getData().data();
- if (itsOperation=="add") {
- std::transform(data, data+nSamples, tdata,
- data, std::plus<dcomplex>());
- } else if (itsOperation=="subtract") {
- std::transform(data, data+nSamples, tdata,
- data, std::minus<dcomplex>());
- }
- }
-
- itsTimerPredict.stop();
-
- itsTimer.stop();
- getNextStep()->process(itsBuffer);
- return false;
- }
-
- StationResponse::vector3r_t Predict::dir2Itrf (const MDirection& dir,
- MDirection::Convert& measConverter) {
- const MDirection& itrfDir = measConverter(dir);
- const Vector<Double>& itrf = itrfDir.getValue().getValue();
- StationResponse::vector3r_t vec;
- vec[0] = itrf[0];
- vec[1] = itrf[1];
- vec[2] = itrf[2];
- return vec;
- }
-
- void Predict::addBeamToData (Patch::ConstPtr patch, double time,
- const StationResponse::vector3r_t& refdir,
- const StationResponse::vector3r_t& tiledir,
- uint thread, uint nSamples, dcomplex* data0) {
- //Apply beam for a patch, add result to itsModelVis
- MDirection dir (MVDirection(patch->position()[0],
- patch->position()[1]),
- MDirection::J2000);
- StationResponse::vector3r_t srcdir = dir2Itrf(dir,itsMeasConverters[thread]);
-
- float* dummyweight = 0;
-
- ApplyBeam::applyBeam(info(), time, data0, dummyweight, srcdir, refdir,
- tiledir, itsAntBeamInfo[thread],
- itsBeamValues[thread], itsUseChannelFreq, false,
- itsBeamMode, false);
-
- //Add temporary buffer to itsModelVis
- std::transform(itsModelVis[thread].data(),
- itsModelVis[thread].data()+nSamples,
- data0,
- itsModelVis[thread].data(), std::plus<dcomplex>());
- //threadoutput<<"thread "<<thread<<" has "<<itsModelVis[thread]<<endl;
- itsModelVisPatch[thread]=dcomplex();
- }
-
- void Predict::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/ProgressMeter.cc b/CEP/DP3/DPPP/src/ProgressMeter.cc
deleted file mode 100644
index cd8de38170d6ebf0b10d8cff515a26caadd163a9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/ProgressMeter.cc
+++ /dev/null
@@ -1,140 +0,0 @@
-//# ProgressMeter.cc: Visual indication of a task's progress.
-//# Copyright (C) 1997,2000,2001,2002
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <DPPP/ProgressMeter.h>
-#include <Common/lofar_iostream.h>
-#include <Common/lofar_vector.h>
-
-namespace LOFAR {
-
-// First implement a simple stderr based progress meter that just prints out
-// 0%....10....20....30....40....50....60....70....80....90....100%
-// cerr is better than cout because it isn't buffered usually, so the above
-// will come out right away. Also, one often wants to direct "real" output
-// to a file, but see informative messages on the screen.
-
-// If we have lots and lots of progress meters we should figure out
-// a way to reclaim the following storage.
-static vector<double> stderr_min, stderr_max, stderr_last;
-static int stderr_creation_function(double min, double max,
- const string&, const string&,
- const string&, const string&,
- bool)
-{
- stderr_min.push_back (min);
- stderr_max.push_back (max);
- stderr_last.push_back (min);
- cerr << "\n0%";
- return stderr_min.size();
-}
-
-static void stderr_update_function(int id, double value)
-{
- if (id < 0 || id > int(stderr_min.size())) {
- cerr << __FILE__ << " illegal id " << id << endl;
- return;
- }
- id--; // 0-relative
- int percent = int((value - stderr_min[id]) /
- (stderr_max[id] - stderr_min[id]) * 100.0);
- int lastpercent = int((stderr_last[id] - stderr_min[id]) /
- (stderr_max[id] - stderr_min[id]) * 100.0);
- if (percent > lastpercent) {
- stderr_last[id] = value;
- // Probably we could do this more efficiently. We need to get all the
- // "missing" ..'s etc if we have jumped a lot since our last updated.
- for (int i=lastpercent+1; i<=percent; i++) {
- if (i%2 == 0 && i%10 != 0) {
- cerr << ".";
- } else if (i %10 == 0) {
- cerr << i;
- if (i >= 100) {
- cerr << "%\n";
- }
- }
- }
- }
-
-}
-
-int (*ProgressMeter::creation_function_p)(double, double,
- const string&, const string&,
- const string&, const string&,
- bool) = stderr_creation_function;
-void (*ProgressMeter::update_function_p)(int, double) = stderr_update_function;
-
-ProgressMeter::ProgressMeter()
- : id_p(-1), min_p(0.0), max_p(1.0), update_every_p(1), update_count_p(0)
-{
-}
-
-ProgressMeter::ProgressMeter(double min, double max,
- const string& title, const string& subtitle,
- const string& minlabel, const string& maxlabel,
- bool estimateTime, int updateEvery)
- : id_p(-1), min_p(min), max_p(max), update_every_p(updateEvery),
- update_count_p(0)
-{
- // Correct silently
- if (update_every_p <= 0) {
- update_every_p = 1;
- }
- if (creation_function_p) {
- id_p = creation_function_p(min, max, title, subtitle, minlabel, maxlabel,
- estimateTime);
- }
-}
-
-ProgressMeter::~ProgressMeter()
-{
- update_count_p++;
- update(max_p, true);
-}
-
-void ProgressMeter::update(double value, bool force)
-{
- update_count_p++;
- // Always force the first one through
- if (update_count_p == 1) {
- force = true;
- }
- if((value >= min_p) && (value <= max_p)){
- if(update_count_p == 1 || force || ((update_count_p%update_every_p)== 0))
- {
- // Do the update if we have a "sink" and a valid id
- if (id_p > 0 && update_function_p) {
- update_function_p(id_p, value);
- } else {
- // If we have more than one progress meter active at once
- // this might look pretty confusing. We can decide what to
- // do if that ever actually happens.
-
- }
- }
- }
- else{
-
- //cerr << "WARNING: progress meter trying to update beyond range" << endl;//The user does not need to know that the programmer does not know how to add.
-
- }
-}
-
-} //# NAMESPACE LOFAR - END
diff --git a/CEP/DP3/DPPP/src/ScaleData.cc b/CEP/DP3/DPPP/src/ScaleData.cc
deleted file mode 100644
index 962165b421c60c3535d3abea46773a0204dcd1c4..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/ScaleData.cc
+++ /dev/null
@@ -1,251 +0,0 @@
-//# ScaleData.cc: DPPP step class for freq-dependent scaling of the data
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: ScaleData.cc 23223 2012-12-07 14:09:42Z schoenmakers $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/ScaleData.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/ParameterValue.h>
-#include <Common/StreamUtil.h>
-#include <Common/LofarLogger.h>
-
-#include <casacore/tables/Tables/Table.h>
-#include <casacore/tables/Tables/TableRecord.h>
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/tables/Tables/ArrayColumn.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Utilities/Regex.h>
-
-#include <iostream>
-
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- ScaleData::ScaleData (DPInput*,
- const ParameterSet& parset,
- const string& prefix)
- : itsName (prefix),
- itsScaleSizeGiven (False),
- itsScaleSize (False),
- itsStationExp (parset.getStringVector(prefix+"stations",
- vector<string>())),
- itsCoeffStr (parset.getStringVector(prefix+"coeffs",
- vector<string>()))
- {
- ASSERTSTR (itsStationExp.size() == itsCoeffStr.size(),
- "ScaleData parameters stations and coeffs differ in size");
- // Determine if scaling for size is explicitly given.
- if (parset.isDefined (prefix+"scalesize")) {
- itsScaleSizeGiven = True;
- itsScaleSize = parset.getBool (prefix+"scalesize");
- }
- }
-
- ScaleData::~ScaleData()
- {}
-
- void ScaleData::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- // Find out if the observation has LBA or HBA data.
- // Add the default factors to itsCoeffStr as being valid for all stations.
- // In that way they will be used if a station matches no others.
- string antennaSet (infoIn.antennaSet());
- itsStationExp.push_back ("*");
- vector<double> defCoeff(5);
- uint nNominal = 0;
- if (antennaSet.substr(0,3) == "LBA") {
- nNominal = 48;
- itsCoeffStr.push_back ("[1.02306755e+06, -7.31426342e+04,"
- " 2.05537660e+03, -2.61310245e+01,"
- " 1.26031118e-01]");
- } else {
- nNominal = 24;
- itsCoeffStr.push_back ("[2.35166277e+05, -6.20793100e+03,"
- " 6.22685124e+01, -2.78418826e-01,"
- " 4.67920578e-04]");
- }
- // Get the frequencies.
- const Vector<double>& freqs = infoIn.chanFreqs();
- // Convert the coefficients to scale factors per freq per station regex.
- vector<vector<double> > scaleVec(itsStationExp.size());
- vector<Regex> stationRegex(itsStationExp.size());
- for (uint i=0; i<scaleVec.size(); ++i) {
- // Convert the station string to a proper Regex object.
- stationRegex[i] = Regex(Regex::fromPattern(itsStationExp[i]));
- // Convert coefficients from string to double.
- ParameterValue coeffPar(itsCoeffStr[i]);
- vector<double> coeff (coeffPar.getDoubleVector());
- ASSERTSTR (coeff.size() > 0, "A ScaleData coeffs vector is empty");
- vector<double>& scales = scaleVec[i];
- scales.reserve (freqs.size());
- // Evaluate the polynomial for each frequency giving the scale factors.
- for (uint j=0; j<freqs.size(); ++j) {
- double fact = coeff[coeff.size() -1];
- for (uint k=coeff.size()-1; k>0; --k) {
- fact *= freqs[j] / 1e6; // use freq in MHz
- fact += coeff[k-1];
- }
- scales.push_back (fact);
- }
- }
- // If needed, find the nr of tiles/dipoles used for each station and
- // fill the size scale factors.
- uint nant = infoIn.antennaNames().size();
- vector<double> extraFactors(nant, 1.);
- if (itsScaleSize || !itsScaleSizeGiven) {
- fillSizeScaleFactors (nNominal, extraFactors);
- ASSERTSTR (extraFactors.size() == nant,
- "Maybe stations have been added before doing the scaling; "
- "that should not be done");
- }
- // Find the scale factors for each station.
- // The first matching regex is used.
- // The nr of tiles in use gives an extra scale factor.
- itsStationFactors.reserve (nant);
- for (uint i=0; i<nant; ++i) {
- for (uint j=0; j<stationRegex.size(); ++j) {
- if (infoIn.antennaNames()[i].matches (stationRegex[j])) {
- itsStationFactors.push_back (scaleVec[j]);
- // If needed, scale with the nr of dipoles/tiles actually used.
- // Do that if explicitly told so or if default coeffs are used.
- if (itsScaleSize ||
- (!itsScaleSizeGiven && j == stationRegex.size()-1)) {
- for (uint k=0; k<itsStationFactors[i].size(); ++k) {
- itsStationFactors[i][k] *= extraFactors[i];
- }
- }
- break;
- }
- }
- }
- ASSERT (itsStationFactors.size() == nant);
- // Now calculate the factors per baseline,freq,pol.
- uint nb = infoIn.nbaselines();
- uint nf = freqs.size();
- uint nc = infoIn.ncorr();
- itsFactors.resize (nc, nf, nb);
- double* factPtr = itsFactors.data();
- for (uint i=0; i<nb; ++i) {
- const vector<double>& f1 = itsStationFactors[infoIn.getAnt1()[i]];
- const vector<double>& f2 = itsStationFactors[infoIn.getAnt2()[i]];
- for (uint j=0; j<nf; ++j) {
- double fact = sqrt(f1[j] * f2[j]);
- for (uint k=0; k<nc; ++k) {
- *factPtr++ = fact;
- }
- }
- }
- }
-
- void ScaleData::show (std::ostream& os) const
- {
- os << "ScaleData " << itsName << std::endl;
- os << " stations: " << itsStationExp << std::endl;
- os << " coeffs: " << itsCoeffStr << std::endl;
- os << " scalesize ";
- if (itsScaleSizeGiven) {
- os << itsScaleSize;
- } else if (itsCoeffStr.size() == 1) {
- os << True;
- } else {
- os << True << " for stations using default coeffs, otherwise "
- << False;
- }
- os << endl;
- os << " Scale factors per station/frequency:" << endl;
- for (uint i=0; i<itsStationFactors.size(); ++i) {
- os << " " << getInfo().antennaNames()[i] << ' '
- << itsStationFactors[i] << endl;
- }
- }
-
- void ScaleData::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " ScaleData " << itsName << endl;
- }
-
- bool ScaleData::process (const DPBuffer& buf)
- {
- itsTimer.start();
- // Apply the scale factors.
- DPBuffer bufNew(buf);
- const IPosition shp = itsFactors.shape();
- ASSERT (buf.getData().shape() == shp);
- // Multiply the data and factors giving a new data array.
- Array<Complex> data(shp);
- arrayContTransform (static_cast<const Array<Complex>&>(buf.getData()),
- static_cast<const Array<double>&>(itsFactors),
- data,
- casacore::Multiplies<Complex,double,Complex>());
- bufNew.setData (data);
- itsTimer.stop();
- getNextStep()->process (bufNew);
- return true;
- }
-
- void ScaleData::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void ScaleData::fillSizeScaleFactors (uint nNominal, vector<double>& fact)
- {
- Table ms(getInfo().msName());
- ASSERTSTR (ms.keywordSet().isDefined ("LOFAR_ANTENNA_FIELD"),
- "ScaleData: subtable LOFAR_ANTENNA_FIELD is missing, but "
- "is needed unless scalesize=false is given");
- Table tab(ms.keywordSet().asTable ("LOFAR_ANTENNA_FIELD"));
- // Get nr of antennae from the table to be sure it matches the
- // contents of LOFAR_ANTENNA_FIELD. Later it is checked if it matches
- // the actual nr of antennae.
- uint nant = ms.keywordSet().asTable("ANTENNA").nrow();
- fact.resize (nant);
- for (uint i=0; i<nant; ++i) {
- fact[i] = 0;
- }
- // Count the nr of used tiles (for which ELEMENT_FLAG is false).
- // A station can have multiple fields (e.g. both ears for HBA_JOINED).
- ROScalarColumn<Int> antId (tab, "ANTENNA_ID");
- ROArrayColumn<Bool> elemFlag (tab, "ELEMENT_FLAG");
- for (uint i=0; i<tab.nrow(); ++i) {
- fact[antId(i)] += 0.5*nfalse(elemFlag(i)); // X and Y are separate
- }
- // Determine the scale factor.
- for (uint i=0; i<nant; ++i) {
- fact[i] = nNominal / fact[i];
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/Simulate.cc b/CEP/DP3/DPPP/src/Simulate.cc
deleted file mode 100644
index c6346a359ea90ea4edbab2858a8cff61817c0b64..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Simulate.cc
+++ /dev/null
@@ -1,165 +0,0 @@
-//# Simulate.cc: Simulate visibilities for a patch of sources.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/Simulate.h>
-#include <Common/LofarLogger.h>
-
-// Only required for rotateUVW().
-#include <DPPP/PhaseShift.h>
-#include <casacore/casa/Arrays/Vector.h>
-#include <casacore/casa/Arrays/Matrix.h>
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Arrays/MatrixMath.h>
-
-using namespace casacore;
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-vector<int> nsetupSplitUVW (uint nant, const Vector<int>& ant1,
- const Vector<int>& ant2)
-{
- // Get the indices of the baselines needed to split the baseline UVWs into
- // station UVWs. They are in such an order that the UVW of a station is known
- // before used in another baseline to derive the UVW of the other station.
- // It can handle cases where baselines occur in disjoint station groups
- // like 0-1, 0-2, 1-2 and 3-4, 4-5, 5-6.
- // Note that the first station of a group gets UVW=0. All other station UVWs
- // are relative to it using the baseline UVWs.
- // Also note that nr of groups can be derived from the size of the returned
- // vector (because it contains no entry for the first antenna in a group).
- vector<int> uvwbl;
- uvwbl.reserve (nant);
- Block<bool> known(nant, false);
- uint nset = 0;
- // Loop until all stations are set.
- while (nset < nant) {
- // Disjoint groups might exist, so keep a vector containing related antennae
- // which are members of the same group.
- vector<uint> members(1);
- // Set first unset station as the reference station (which gets UVW=0).
- for (uint i=0; i<nant; ++i) {
- if (!known[i]) {
- members[0] = i;
- known[i] = true;
- ++nset;
- break;
- }
- }
- // Loop through all members in the group.
- // Note that new members can be appended in this loop.
- for (uint j=0; j<members.size(); ++j) {
- int refst = members[j];
- // Find all stations having a baseline with the reference station.
- for (uint i=0; i<ant1.size(); ++i) {
- int a1 = ant1[i];
- int a2 = ant2[i];
- // Only take baselines into account for which one station is known,
- // so the other can be derived from it.
- // The unknown station becomes member of the group.
- if (known[a1] != known[a2]) {
- if (a1 == refst) {
- uvwbl.push_back (i);
- members.push_back (a2);
- known[a2] = true;
- ++nset;
- } else if (a2 == refst) {
- uvwbl.push_back (-(i+1));
- members.push_back (a1);
- known[a1] = true;
- ++nset;
- }
- }
- }
- }
- }
- return uvwbl;
-}
-
-void nsplitUVW (const vector<int>& blindex,
- const vector<Baseline>& baselines,
- const Matrix<double>& uvwbl,
- Matrix<double>& uvwant)
-{
- uvwant = 0.;
- double* uvwl;
- double* uvwr;
- const double* uvwb;
- for (uint i=0; i<blindex.size(); ++i) {
- int inx = blindex[i];
- if (inx < 0) {
- // Ant2 is known.
- inx = -inx - 1;
- //ASSERT(uint(uvwant.shape()[1])>baselines[inx].first);
- //ASSERT(uint(uvwant.shape()[1])>baselines[inx].second);
- uvwl = uvwant.data() + 3 * baselines[inx].first;
- uvwr = uvwant.data() + 3 * baselines[inx].second;
- uvwb = uvwbl.data() + 3*inx;
- for (int j=0; j<3; ++j) {
- uvwl[j] = uvwr[j] - uvwb[j];
- }
- } else {
- //ASSERT(uint(uvwant.shape()[1])>baselines[inx].first);
- //ASSERT(uint(uvwant.shape()[1])>baselines[inx].second);
- // Ant1 is known.
- uvwl = uvwant.data() + 3 * baselines[inx].first;
- uvwr = uvwant.data() + 3 * baselines[inx].second;
- uvwb = uvwbl.data() + 3*inx;
- for (int j=0; j<3; ++j) {
- uvwr[j] = uvwl[j] + uvwb[j];
- }
- }
- }
-}
-
-
-void rotateUVW(const Position &from, const Position &to, size_t nUVW,
- double *uvw)
-{
- casacore::Matrix<double> oldUVW(3,3);
- casacore::Matrix<double> newUVW(3,3);
- PhaseShift::fillTransMatrix(oldUVW, from[0], from[1]);
- PhaseShift::fillTransMatrix(newUVW, to[0], to[1]);
-
- casacore::Matrix<double> tmp(casacore::product(casacore::transpose(newUVW), oldUVW));
- const double *R = tmp.data();
-
- for(size_t i = 0; i < 3*nUVW; i+=3)
- {
- // Compute rotated UVW.
- double u = uvw[i+0] * R[0] + uvw[i+1] * R[3] + uvw[i+2] * R[6];
- double v = uvw[i+0] * R[1] + uvw[i+1] * R[4] + uvw[i+2] * R[7];
- double w = uvw[i+0] * R[2] + uvw[i+1] * R[5] + uvw[i+2] * R[8];
-
- uvw[i+0] = u;
- uvw[i+1] = v;
- uvw[i+2] = w;
-
- // Move to next station.
- } // Stations.
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/Simulator.cc b/CEP/DP3/DPPP/src/Simulator.cc
deleted file mode 100644
index 9a431294b354ee3fb17fd6d3e8610ac089f78878..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Simulator.cc
+++ /dev/null
@@ -1,312 +0,0 @@
-//# Simulator.cc: Compute visibilities for different model components types
-//# (implementation of ModelComponentVisitor).
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/Simulator.h>
-#include <DPPP/GaussianSource.h>
-#include <DPPP/PointSource.h>
-#include <casacore/casa/BasicSL/Constants.h>
-#include <Common/StreamUtil.h> ///
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-namespace
-{
-// Compute LMN coordinates of \p position relative to \p reference.
-//
-// \param[in] reference
-// Reference position on the celestial sphere.
-// \param[in] position
-// Position of interest on the celestial sphere.
-// \param[in] lmn
-// Pointer to a buffer of (at least) length three into which the computed LMN
-// coordinates will be written.
-void radec2lmn(const Position &reference, const Position &position,
- double* lmn);
-
-void phases(size_t nStation, size_t nChannel, const double* lmn,
- const casacore::Matrix<double>& uvw, const casacore::Vector<double>& freq,
- Simulator::Matrix<dcomplex>& shift);
-
-void spectrum(const PointSource &component, size_t nChannel,
- const casacore::Vector<double>& freq,
- Simulator::Matrix<dcomplex>& spectrum, bool stokesIOnly);
-} // Unnamed namespace.
-
-Simulator::Simulator(const Position &reference, size_t nStation,
- size_t nBaseline, size_t nChannel, const casacore::Vector<Baseline>& baselines,
- const casacore::Vector<double>& freq, const casacore::Matrix<double>& uvw,
- casacore::Cube<dcomplex>& buffer, bool stokesIOnly)
- : itsReference(reference),
- itsNStation(nStation),
- itsNBaseline(nBaseline),
- itsNChannel(nChannel),
- itsStokesIOnly(stokesIOnly),
- itsBaselines(baselines),
- itsFreq(freq),
- itsUVW(uvw),
- itsBuffer(buffer),
- itsShiftBuffer(),
- itsSpectrumBuffer()
-{
- itsShiftBuffer.resize(nChannel,nStation);
- if (stokesIOnly) {
- itsSpectrumBuffer.resize(1,nChannel);
- } else {
- itsSpectrumBuffer.resize(4,nChannel);
- }
-}
-
-void Simulator::simulate(const ModelComponent::ConstPtr &component)
-{
- component->accept(*this);
-}
-
-
-void Simulator::visit(const PointSource &component) {
- // Compute LMN coordinates.
- double lmn[3];
- radec2lmn(itsReference, component.position(), lmn);
-
- // Compute station phase shifts.
- phases(itsNStation, itsNChannel, lmn, itsUVW, itsFreq, itsShiftBuffer);
-
- // Compute component spectrum.
- spectrum(component, itsNChannel, itsFreq, itsSpectrumBuffer, itsStokesIOnly);
-
- // Set number of correlations
- int nCorr = 4;
- if (itsStokesIOnly) {
- nCorr = 1;
- }
-
- // Compute visibilities.
-#pragma omp parallel for
- for(size_t bl = 0; bl < itsNBaseline; ++bl) {
- dcomplex* buffer=&itsBuffer(0,0,bl);
- const size_t p = itsBaselines[bl].first;
- const size_t q = itsBaselines[bl].second;
-
- if(p == q) {
- buffer+=itsNChannel*nCorr;
- } else {
- const dcomplex *shiftP = &(itsShiftBuffer(0,p));
- const dcomplex *shiftQ = &(itsShiftBuffer(0,q));
- const dcomplex *spectrum = itsSpectrumBuffer.data();
-
- if (itsStokesIOnly) {
- for (size_t ch = 0; ch < itsNChannel; ++ch)
- {
- // Compute baseline phase shift.
- // Compute visibilities.
- *buffer++ += (*shiftQ) * conj(*shiftP) * (*spectrum++);
- ++shiftP;
- ++shiftQ;
- } // Channels.
- } else {
- for(size_t ch = 0; ch < itsNChannel; ++ch)
- {
- // Compute baseline phase shift.
- const dcomplex blShift = (*shiftQ) * conj(*shiftP);
- ++shiftP;
- ++shiftQ;
-
- // Compute visibilities.
- *buffer++ += blShift * (*spectrum++);
- *buffer++ += blShift * (*spectrum++);
- *buffer++ += blShift * (*spectrum++);
- *buffer++ += blShift * (*spectrum++);
- } // Channels.
- }
- }
- } // Baselines.
-}
-
-void Simulator::visit(const GaussianSource &component)
-{
- // Compute LMN coordinates.
- double lmn[3];
- radec2lmn(itsReference, component.position(), lmn);
-
- // Compute station phase shifts.
- phases(itsNStation, itsNChannel, lmn, itsUVW, itsFreq, itsShiftBuffer);
-
- // Compute component spectrum.
- spectrum(component, itsNChannel, itsFreq, itsSpectrumBuffer, itsStokesIOnly);
-
- // Convert position angle from North over East to the angle used to
- // rotate the right-handed UV-plane.
- // TODO: Can probably optimize by changing the rotation matrix instead.
- const double phi = casacore::C::pi_2 + component.positionAngle() + casacore::C::pi;
- const double cosPhi = cos(phi);
- const double sinPhi = sin(phi);
-
- // Take care of the conversion of axis lengths from FWHM in radians to
- // sigma.
- // TODO: Shouldn't the projection from the celestial sphere to the
- // UV-plane be taken into account here?
- const double fwhm2sigma = 1.0 / (2.0 * std::sqrt(2.0 * std::log(2.0)));
- const double uScale = component.majorAxis() * fwhm2sigma;
- const double vScale = component.minorAxis() * fwhm2sigma;
-
- // Set number of correlations
- int nCorr = 4;
- if (itsStokesIOnly) {
- nCorr = 1;
- }
-
- for(size_t bl = 0; bl < itsNBaseline; ++bl)
- {
- dcomplex* buffer=&itsBuffer(0,0,bl);
- const size_t p = itsBaselines[bl].first;
- const size_t q = itsBaselines[bl].second;
-
- if(p == q) {
- buffer+=itsNChannel*nCorr;
- } else {
- double u = itsUVW(0,q);
- double v = itsUVW(1,q);
-
- u -= itsUVW(0,p);
- v -= itsUVW(1,p);
-
- // Rotate (u, v) by the position angle and scale with the major
- // and minor axis lengths (FWHM in rad).
- const double uPrime = uScale * (u * cosPhi - v * sinPhi);
- const double vPrime = vScale * (u * sinPhi + v * cosPhi);
-
- // Compute uPrime^2 + vPrime^2 and pre-multiply with -2.0 * PI^2
- // / C^2.
- const double uvPrime = (-2.0 * casacore::C::pi * casacore::C::pi)
- * (uPrime * uPrime + vPrime * vPrime);
-
- const dcomplex *shiftP = &(itsShiftBuffer(0,p));
- const dcomplex *shiftQ = &(itsShiftBuffer(0,q));
- const dcomplex *spectrum = itsSpectrumBuffer.data();
-
- if (itsStokesIOnly) {
- for (size_t ch = 0; ch < itsNChannel; ++ch)
- {
- // Compute baseline phase shift.
- dcomplex blShift = (*shiftQ) * conj(*shiftP);
- ++shiftP;
- ++shiftQ;
-
- const double ampl = exp((itsFreq[ch] * itsFreq[ch])
- / (casacore::C::c * casacore::C::c) * uvPrime);
-
- blShift *= ampl;
-
- // Compute visibilities.
- *buffer++ += blShift * (*spectrum++);
- } // Channels.
- } else {
- for(size_t ch = 0; ch < itsNChannel; ++ch)
- {
- // Compute baseline phase shift.
- dcomplex blShift = (*shiftQ) * conj(*shiftP);
- ++shiftP;
- ++shiftQ;
-
- const double ampl = exp((itsFreq[ch] * itsFreq[ch])
- / (casacore::C::c * casacore::C::c) * uvPrime);
-
- blShift *= ampl;
-
- // Compute visibilities.
- *buffer++ += blShift * (*spectrum++);
- *buffer++ += blShift * (*spectrum++);
- *buffer++ += blShift * (*spectrum++);
- *buffer++ += blShift * (*spectrum++);
- } // Channels.
- }
- }
- } // Baselines.
-}
-
-namespace
-{
-inline void radec2lmn(const Position &reference, const Position &position,
- double* lmn)
-{
- const double dRA = position[0] - reference[0];
- const double pDEC = position[1];
- const double rDEC = reference[1];
- const double cDEC = cos(pDEC);
-
- const double l = cDEC * sin(dRA);
- const double m = sin(pDEC) * cos(rDEC) - cDEC * sin(rDEC) * cos(dRA);
-
- lmn[0] = l;
- lmn[1] = m;
- lmn[2] = sqrt(1.0 - l * l - m * m);
-}
-
-// Compute station phase shifts.
-inline void phases(size_t nStation, size_t nChannel, const double* lmn,
- const casacore::Matrix<double>& uvw,
- const casacore::Vector<double>& freq,
- Simulator::Matrix<dcomplex>& shift)
-{
- dcomplex* shiftdata=shift.data();
- for(size_t st = 0; st < nStation; ++st)
- {
- const double phase = casacore::C::_2pi * (uvw(0,st) * lmn[0]
- + uvw(1,st) * lmn[1] + uvw(2,st) * (lmn[2] - 1.0));
-
- for(size_t ch = 0; ch < nChannel; ++ch)
- {
- const double chPhase = phase * freq[ch] / casacore::C::c;
- *shiftdata = dcomplex(cos(chPhase), sin(chPhase));
- ++shiftdata;
- } // Channels.
- } // Stations.
-}
-
-
-// Compute component spectrum.
-inline void spectrum(const PointSource &component, size_t nChannel,
- const casacore::Vector<double>& freq,
- Simulator::Matrix<dcomplex>& spectrum, bool stokesIOnly=false)
-{
- for(size_t ch = 0; ch < nChannel; ++ch)
- {
- Stokes stokes = component.stokes(freq[ch]);
-
- if (stokesIOnly) {
- spectrum(0,ch) = dcomplex(stokes.I, 0.0);
- } else {
- spectrum(0,ch) = dcomplex(stokes.I + stokes.Q, 0.0);
- spectrum(1,ch) = dcomplex(stokes.U, stokes.V);
- spectrum(2,ch) = dcomplex(stokes.U, -stokes.V);
- spectrum(3,ch) = dcomplex(stokes.I - stokes.Q, 0.0);
- }
- }
-}
-} // Unnamed namespace.
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/SolTab.cc b/CEP/DP3/DPPP/src/SolTab.cc
deleted file mode 100644
index b96b4361fb821a30c606b9487f584c52b020dd4d..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/SolTab.cc
+++ /dev/null
@@ -1,560 +0,0 @@
-#include <lofar_config.h>
-#include <DPPP/H5Parm.h>
-#include <DPPP/GridInterpolate.h>
-#include <Common/Exception.h>
-#include <Common/StringUtil.h>
-#include <Common/LofarLogger.h>
-#include <stdlib.h>
-#include <cstring>
-#include <iostream>
-#include <ctime>
-#include <iomanip>
-
-#include <hdf5.h>
-
-using namespace std;
-
-namespace LOFAR {
- H5Parm::SolTab::SolTab(H5::Group group,
- const string& type,
- const vector<AxisInfo> axes):
- H5::Group(group),
- _type(type),
- _axes(axes)
- {
- H5::Attribute attr = createAttribute("TITLE",
- H5::StrType(H5::PredType::C_S1, _type.size()),
- H5::DataSpace());
- attr.write(H5::StrType(H5::PredType::C_S1, _type.size()), _type);
- addVersionStamp(*this);
- }
-
- H5Parm::SolTab::SolTab(H5::Group& group):
- H5::Group(group)
- {
- // Read the type from the "TITLE" attribute
- H5::Attribute typeattr = openAttribute("TITLE");
- hsize_t typenamelen = typeattr.getDataType().getSize();
- char typecstr[typenamelen+1];
- typecstr[typenamelen]='\0';
- typeattr.read(typeattr.getDataType(), &typecstr);
- _type = typecstr;
-
- readAxes();
- }
-
- H5Parm::SolTab::~SolTab() {
- }
-
- H5Parm::AxisInfo H5Parm::SolTab::getAxis(uint i) const {
- return _axes[i];
- }
-
- H5Parm::AxisInfo H5Parm::SolTab::getAxis(const string& axisName) const {
- for (uint i=0; i<_axes.size(); ++i) {
- if (_axes[i].name == axisName) {
- return _axes[i];
- }
- }
- THROW(Exception, "Axis "<<axisName<<" does not exist in "<<getName());
- }
-
- bool H5Parm::SolTab::hasAxis(const string& axisName) {
- for (size_t i=0; i<_axes.size(); ++i) {
- if (_axes[i].name==axisName)
- return true;
- }
- return false;
- }
-
- size_t H5Parm::SolTab::getAxisIndex(const string& axisName) {
- for (uint i=0; i<_axes.size(); ++i) {
- if (_axes[i].name == axisName) {
- return i;
- }
- }
- THROW(Exception, "Axis "<<axisName<<" does not exist in "<<getName());
- }
-
- void H5Parm::SolTab::setValues(const vector<double>& vals,
- const vector<double>& weights,
- const string& history) {
- // Convert axes to comma separated string, fill dims
- size_t expectedsize = 1;
- string axesstr = _axes[0].name;
- vector<hsize_t> dims(_axes.size());
- for (uint i=0; i<_axes.size(); ++i) {
- dims[i] = _axes[i].size;
- expectedsize *= dims[i];
- if (i>0) {
- axesstr += ","+_axes[i].name;
- }
- }
-
- ASSERTSTR(expectedsize == vals.size(), "Values for H5Parm do not have the expected size: they have size "<<vals.size()<<", expected is "<<expectedsize);
-
- H5::DataSpace dataspace(dims.size(), &(dims[0]), NULL);
- H5::DataSet dataset = createDataSet("val", H5::PredType::IEEE_F64LE,
- dataspace);
-
- dataset.write(&(vals[0]), H5::PredType::IEEE_F64LE);
-
- H5::Attribute attr = dataset.createAttribute("AXES",
- H5::StrType(H5::PredType::C_S1, axesstr.size()),
- H5::DataSpace());
- attr.write(H5::StrType(H5::PredType::C_S1, axesstr.size()), axesstr);
-
- // Write history if given
- if (history.size()>0) {
- time_t rawtime;
- struct tm* timeinfo;
- char timebuffer[80];
-
- time(&rawtime);
- timeinfo = localtime(&rawtime);
-
- strftime(timebuffer, sizeof(timebuffer), "%d-%m-%Y %H:%M:%S", timeinfo);
-
- string historyline = string(timebuffer) + ": " + history;
-
- H5::StrType historytype = H5::StrType(H5::PredType::C_S1,
- historyline.size());
- H5::Attribute attr = dataset.createAttribute("HISTORY000",
- historytype,
- H5::DataSpace());
- attr.write(historytype, historyline);
- }
-
- // Add weights
- // Do not use half float data type because typical weights range can be 1.e-14
- /*
- hid_t halffloat = H5Tcopy(H5T_IEEE_F32BE);
- H5Tset_fields(halffloat, 15, 10, 5, 0, 10);
- H5Tset_size(halffloat, 2);
- H5Tset_ebias(halffloat, 15);
- H5Tlock(halffloat);
- */
- H5::DataSet weightset = createDataSet("weight", H5::PredType::IEEE_F32LE,
- dataspace);
-
- // If weights are empty, write ones everywhere
- if (weights.empty()) {
- vector<double> fullweights(vals.size(), 1);
- weightset.write(&(fullweights[0]), H5::PredType::IEEE_F64LE);
- } else {
- weightset.write(&(weights[0]), H5::PredType::IEEE_F64LE);
- }
-
- attr = weightset.createAttribute("AXES",
- H5::StrType(H5::PredType::C_S1, axesstr.size()),
- H5::DataSpace());
- attr.write(H5::StrType(H5::PredType::C_S1, axesstr.size()), axesstr);
- }
-
- void H5Parm::SolTab::setComplexValues(const vector<complex<double> >& vals,
- const vector<double>& weights,
- bool toAmplitudes, const string& history) {
- // Convert values to real numbers by taking amplitude or argument
- vector<double> realvals(vals.size());
-
- if (toAmplitudes) {
- transform(vals.begin(), vals.end(), realvals.begin(), takeAbs);
- } else { // Phase only
- transform(vals.begin(), vals.end(), realvals.begin(), takeArg);
- }
-
- setValues(realvals, weights, history);
- }
-
- void H5Parm::SolTab::readAxes() {
- H5::DataSet val;
- try {
- val = openDataSet("val");
- } catch (H5::GroupIException& e) {
- THROW(Exception, "SolTab "<<getName()<<" has no values");
- }
-
- H5::Attribute axesattr;
- try {
- axesattr = val.openAttribute("AXES");
- } catch (H5::AttributeIException& e) {
- THROW(Exception, "Values of SolTab "<<getName()<<" has no AXIS attribute");
- }
-
- hsize_t axesstrlen = axesattr.getDataType().getSize();
- char axescstr[axesstrlen+1];
- axescstr[axesstrlen]='\0';
- axesattr.read(axesattr.getDataType(), &axescstr);
- vector<string> axesnames = StringUtil::tokenize(axescstr,",");
-
- uint ndims = axesnames.size();
-
- // Get number of dimensions and size of all dimensions
- H5::DataSpace ds = val.getSpace();
- ASSERT (ds.getSimpleExtentNdims() == int(ndims));
- hsize_t dims_out[ndims];
- ds.getSimpleExtentDims(dims_out);
-
- for (uint i=0; i<axesnames.size(); ++i) {
- AxisInfo a(axesnames[i], dims_out[i]);
- _axes.push_back(a);
- }
- }
-
- string H5Parm::SolTab::getName() const {
- size_t len = H5Iget_name(getId(),NULL,0);
- char buffer[len];
- H5Iget_name(getId(),buffer,len+1);
- // Strip leading /
- return buffer+1;
- }
-
- vector<double> H5Parm::SolTab::getValuesOrWeights(
- const string& valOrWeight,
- const string& antName,
- const vector<double>& times,
- const vector<double>& freqs,
- uint pol, uint dir) {
- vector<double> res(times.size()*freqs.size());
-
- uint startTimeSlot = 0;
- uint ntimeH5 = 1;
-
- ASSERT(!freqs.empty());
- uint startFreq = 0;
- uint nfreqH5 = 1;
-
- vector<double> interpolated(times.size()*freqs.size());
-
- vector<double> freqAxisH5(1, 0.);
- vector<double> timeAxisH5(1, 0.);
- if (hasAxis("time")) {
- timeAxisH5 = getRealAxis("time");
- ntimeH5 = timeAxisH5.size();
- }
- if (hasAxis("freq")) {
- vector<double> fullFreqAxisH5 = getRealAxis("freq");
- startFreq = getFreqIndex(freqs[0]);
- nfreqH5 = getFreqIndex(freqs[freqs.size()-1])-startFreq+1;
- freqAxisH5 = vector<double>(fullFreqAxisH5.begin()+startFreq, fullFreqAxisH5.begin()+startFreq+nfreqH5);
- }
-
- vector<double> h5values = getValuesOrWeights(valOrWeight,
- antName,
- startTimeSlot, ntimeH5, 1,
- startFreq, nfreqH5, 1,
- pol, dir);
-
- gridNearestNeighbor(timeAxisH5, freqAxisH5,
- times, freqs,
- &(h5values[0]),
- &(interpolated[0]));
-
- return interpolated;
- }
-
- vector<double> H5Parm::SolTab::getValuesOrWeights(
- const string& valOrWeight,
- const string& antName,
- uint starttimeslot, uint ntime, uint timestep,
- uint startfreq, uint nfreq, uint freqstep,
- uint pol, uint dir) {
- vector<double> res(ntime*nfreq);
- H5::DataSet val = openDataSet(valOrWeight);
-
- // Set offsets and strides
- hsize_t memdims[_axes.size()];
- hsize_t offset[_axes.size()];
- hsize_t count[_axes.size()];
- hsize_t stride[_axes.size()];
-
- for (uint i=0; i<_axes.size(); ++i) {
- stride[i] = 1;
- count[i] = 1;
- memdims[i] = 1;
- if (_axes[i].name=="time") {
- offset[i] = starttimeslot;
- stride[i] = timestep;
- count[i] = ntime;
- memdims[i] = ntime;
- } else if (_axes[i].name=="freq") {
- offset[i] = startfreq;
- stride[i] = freqstep;
- count[i] = nfreq;
- memdims[i] = nfreq;
- } else if (_axes[i].name=="ant") {
- offset[i] = getAntIndex(antName);
- } else if (_axes[i].name=="dir") {
- offset[i] = dir;
- } else if (_axes[i].name=="pol") {
- offset[i] = pol;
- } else {
- ASSERT(_axes[i].size == 1);
- offset[i] = 0;
- }
- }
-
- H5::DataSpace dataspace = val.getSpace();
-
- dataspace.selectHyperslab(H5S_SELECT_SET, count, offset, stride);
-
- // Setup memory dataspace
- H5::DataSpace memspace(_axes.size(), memdims);
- try {
- val.read(&(res[0]), H5::PredType::NATIVE_DOUBLE, memspace, dataspace);
- } catch (H5::DataSetIException& e) {
- e.printError();
- THROW(Exception, "Could not read data");
- }
- return res;
- }
-
- void H5Parm::SolTab::setAntennas(const vector<string>& solAntennas) {
- // TODO: assert that antenna is present in antenna table in solset
- hsize_t dims[1];
- dims[0]=solAntennas.size();
-
- // Create dataset
- H5::DataSpace dataspace(1, dims, NULL);
- H5::DataSet dataset = createDataSet("ant", H5::StrType(H5::PredType::C_S1, 16), dataspace);
-
- // Prepare data
- char antArray[solAntennas.size()][16];
- for (uint i=0; i<solAntennas.size(); ++i) {
- strncpy(antArray[i], solAntennas[i].c_str(), 16);
- }
-
- dataset.write(antArray, H5::StrType(H5::PredType::C_S1, 16));
- }
-
- void H5Parm::SolTab::setAxisMeta(const string& metaName,
- size_t nChar,
- const vector<string>& metaVals) {
- hsize_t dims[1]; // Only a name
- dims[0]=metaVals.size();
-
- // Create dataset
- H5::DataSpace dataspace(1, dims, NULL);
- H5::DataSet dataset = createDataSet(metaName,
- H5::StrType(H5::PredType::C_S1, nChar),
- dataspace);
-
- if (metaVals.size()>0) {
- // Prepare data
- char srcArray[metaVals.size()][nChar];
- for (uint i=0; i<metaVals.size(); ++i) {
- strncpy(srcArray[i], metaVals[i].c_str(), nChar);
- }
-
- dataset.write(srcArray, H5::StrType(H5::PredType::C_S1, nChar));
- }
- }
-
- void H5Parm::SolTab::setSources(const vector<string>& solSources) {
- setAxisMeta("dir", 128, solSources);
- }
-
- void H5Parm::SolTab::setPolarizations(const vector<string>& polarizations) {
- setAxisMeta("pol", 2, polarizations);
- }
-
- void H5Parm::SolTab::setFreqs(const vector<double>& freqs) {
- setAxisMeta("freq", freqs);
- }
-
- void H5Parm::SolTab::setTimes(const vector<double>& times) {
- setAxisMeta("time", times);
- }
-
-
- void H5Parm::SolTab::setAxisMeta(const string& metaName,
- const vector<double>& metaVals) {
- hsize_t dims[1];
- dims[0]=metaVals.size();
-
- // Create dataset
- H5::DataSpace dataspace(1, dims, NULL);
- H5::DataSet dataset = createDataSet(metaName,
- H5::PredType::IEEE_F64LE, dataspace);
-
- if (metaVals.size() > 0) {
- dataset.write(&(metaVals[0]), H5::PredType::IEEE_F64LE);
- }
- }
-
- hsize_t H5Parm::SolTab::getAntIndex(const string& antName) {
- return getNamedIndex(_antMap, "ant", antName);
- }
-
- void H5Parm::SolTab::fillCache(map<string, hsize_t>& cache,
- const string& tableName) {
- H5::DataSet dataset;
- H5::DataSpace dataspace;
- try {
- dataset = openDataSet(tableName);
- dataspace = dataset.getSpace();
- } catch (H5::GroupIException& e) {
- THROW(Exception, "SolTab has no table "<<tableName);
- }
- ASSERT(dataspace.getSimpleExtentNdims()==1);
- hsize_t dims[1];
- dataspace.getSimpleExtentDims(dims);
-
- // TODO: assert that DataType is String
- hsize_t strLen = dataset.getDataType().getSize();
-
- char elNames[strLen*dims[0]];
- dataset.read(elNames, H5::StrType(H5::PredType::C_S1, strLen));
-
- for (hsize_t elNum=0; elNum<dims[0];++elNum) {
- char elNamecstr[strLen+1];
- copy(elNames+elNum*strLen,elNames+(elNum+1)*strLen, elNamecstr);
- elNamecstr[strLen]='\0';
- cache[elNamecstr]=elNum;
- }
- }
-
- hsize_t H5Parm::SolTab::getNamedIndex(map<string, hsize_t>& cache,
- const string& tableName,
- const string& elementName) {
- // Initialize _antMap on first use
- if (cache.empty()) {
- fillCache(cache, tableName);
- }
- map<string, hsize_t>::iterator it = cache.find(elementName);
- if (it == cache.end()) {
- THROW(Exception, "SolTab has no element "<<elementName <<
- " in "<<tableName);
- }
- return cache.find(elementName)->second;
- }
-
- hsize_t H5Parm::SolTab::getFreqIndex(double freq) const {
- if (getAxis("freq").size == 1) {
- return 0;
- }
- vector<double> freqs = getRealAxis("freq");
-
- double freqInterval = getFreqInterval(0);
-
- // Half a cell width before the first frequency
- if (abs(freqs[0]-freq)<0.501*freqInterval) {
- return 0;
- }
- // No assumptions on regular spacing here
- for (size_t i = 0; i<freqs.size()-1; ++i) {
- if (freqs[i]-0.001<=freq && freq<freqs[i+1]) { // Some tolerance
- // Nearest neighbor: i or i+1
- if (freq-freqs[i] < freqs[i+1]-freq) {
- return i;
- } else {
- return i+1;
- }
- }
- }
-
- // Half a cell width after the last frequency
- freqInterval = getFreqInterval(freqs.size()-2);
- if (abs(freqs[freqs.size()-1]-freq)<0.501*freqInterval) {
- return freqs.size()-1;
- }
-
- THROW(Exception,"Frequency "<<fixed<<freq<<" not found in "<<getName());
- return 0;
- }
-
- vector<double> H5Parm::SolTab::getRealAxis(const string& axisname) const {
- H5::DataSet dataset;
- H5::DataSpace dataspace;
- try {
- dataset = openDataSet(axisname);
- dataspace = dataset.getSpace();
- } catch (H5::GroupIException& e) {
- THROW(Exception, "SolTab "<<getName()<<" has no axis '"<<axisname<<"'");
- }
- ASSERT(dataspace.getSimpleExtentNdims()==1);
-
- hsize_t dims[1];
- dataspace.getSimpleExtentDims(dims);
-
- vector<double> data(dims[0]);
- dataset.read(&(data[0]), H5::PredType::NATIVE_DOUBLE);
-
- return data;
- }
-
- vector<string> H5Parm::SolTab::getStringAxis(const string& axisName) {
- map<string, hsize_t> cachemap;
-
- if (axisName=="dir") {
- if (_dirMap.empty()) {
- fillCache(_dirMap, "dir");
- }
- cachemap = _dirMap;
- } else if (axisName=="ant") {
- if (_antMap.empty()) {
- fillCache(_antMap, "ant");
- }
- cachemap = _antMap;
- } else {
- THROW(Exception, "Only string axes 'ant' and 'dir' supported for now.");
- }
-
- // Get the keys of the cache map and put them in a vector
- vector<string> res;
- for(map<string,hsize_t>::iterator it = cachemap.begin();
- it != cachemap.end(); ++it) {
- res.push_back(it->first);
- }
- return res;
- }
-
-
- hsize_t H5Parm::SolTab::getTimeIndex(double time) const {
- if (getAxis("time").size == 1) {
- return 0;
- }
- vector<double> times = getRealAxis("time");
-
- double timeInterval = getTimeInterval();
-
- for (size_t i = 0; i<times.size(); ++i) {
- if (abs(times[i]-time)<timeInterval*0.501) // 0.5 with some tolerance
- return i;
- }
- THROW(Exception,"Time "<<fixed<<time<<" not found in "<<getName());
- return 0;
- }
-
- hsize_t H5Parm::SolTab::getDirIndex(const string& dirName) {
- return getNamedIndex(_dirMap, "dir", dirName);
- }
-
- double H5Parm::SolTab::getInterval(const string& axisName, size_t start) const {
- H5::DataSet dataset;
- H5::DataSpace dataspace;
- try {
- dataset = openDataSet(axisName);
- dataspace = dataset.getSpace();
- } catch (H5::GroupIException& e) {
- THROW(Exception, "SolTab "<<getName()<<" has no axis table for "<<axisName);
- }
- ASSERT(dataspace.getSimpleExtentNdims()==1);
-
- hsize_t dims[1];
- dataspace.getSimpleExtentDims(dims);
- ASSERTSTR(dims[0]>start+1, "For reading the " + axisName + " interval, more than one value is required.");
-
- hsize_t count[1], offset[1], memoffset[1];
- count[0]=2; offset[0]=start; memoffset[0]=0;
- dataspace.selectHyperslab(H5S_SELECT_SET, count, offset);
-
- H5::DataSpace memspace(1, count);
- memspace.selectHyperslab(H5S_SELECT_SET, count, memoffset);
-
- // Get only two values
- vector<double> values(2);
- dataset.read(&(values[0]), H5::PredType::NATIVE_DOUBLE, memspace, dataspace);
- return values[1]-values[0];
- }
-}
diff --git a/CEP/DP3/DPPP/src/SourceDBUtil.cc b/CEP/DP3/DPPP/src/SourceDBUtil.cc
deleted file mode 100644
index 5bca4865d833c4b11a999a43390870d6058599a1..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/SourceDBUtil.cc
+++ /dev/null
@@ -1,270 +0,0 @@
-//# SourceDBUtil.cc: Helper functions to extract patch and source information
-//# from a SourceDB.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/SourceDBUtil.h>
-#include <DPPP/PointSource.h>
-#include <DPPP/GaussianSource.h>
-#include <ParmDB/SourceDB.h>
-#include <Common/LofarLogger.h>
-#include <Common/lofar_vector.h>
-#include <sstream>
-#include <set>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-using BBS::SourceDB;
-using BBS::SourceData;
-using BBS::SourceInfo;
-
-
-vector<Patch::ConstPtr> makePatches(SourceDB &sourceDB,
- const vector<string> &patchNames,
- uint nModel)
-{
- // Create a component list for each patch name.
- vector<vector<ModelComponent::Ptr> > componentsList(nModel);
-
- // Loop over all sources.
- sourceDB.lock();
- sourceDB.rewind();
- SourceData src;
- while (! sourceDB.atEnd()) {
- sourceDB.getNextSource (src);
- // Use the source if its patch matches a patch name.
- for (uint i=0; i<nModel; ++i) {
- if (src.getPatchName() == patchNames[i]) {
- // Fetch position.
- ASSERT (src.getInfo().getRefType() == "J2000");
- Position position;
- position[0] = src.getRa();
- position[1] = src.getDec();
-
- // Fetch stokes vector.
- Stokes stokes;
- stokes.I = src.getI();
- stokes.V = src.getV();
- if(!src.getInfo().getUseRotationMeasure())
- {
- stokes.Q = src.getQ();
- stokes.U = src.getU();
- }
-
- PointSource::Ptr source;
- switch(src.getInfo().getType())
- {
- case SourceInfo::POINT:
- {
- source = PointSource::Ptr(new PointSource(position, stokes));
- }
- break;
-
- case SourceInfo::GAUSSIAN:
- {
- GaussianSource::Ptr gauss(new GaussianSource(position, stokes));
-
- const double deg2rad = (casacore::C::pi / 180.0);
- gauss->setPositionAngle(src.getOrientation() * deg2rad);
-
- const double arcsec2rad = (casacore::C::pi / 3600.0) / 180.0;
- gauss->setMajorAxis(src.getMajorAxis() * arcsec2rad);
- gauss->setMinorAxis(src.getMinorAxis() * arcsec2rad);
- source = gauss;
- }
- break;
-
- default:
- {
- ASSERTSTR(false, "Only point sources and Gaussian sources are"
- " supported at this time.");
- }
- }
-
- // Fetch spectral index attributes (if applicable).
- bool isLogarithmic = src.getInfo().getHasLogarithmicSI();
- if (src.getSpectralTerms().size() > 0) {
- source->setSpectralTerms(src.getInfo().getSpectralTermsRefFreq(),
- isLogarithmic,
- src.getSpectralTerms().begin(),
- src.getSpectralTerms().end());
- }
-
- // Fetch rotation measure attributes (if applicable).
- if(src.getInfo().getUseRotationMeasure())
- {
- source->setRotationMeasure(src.getPolarizedFraction(),
- src.getPolarizationAngle(), src.getRotationMeasure());
- }
-
- componentsList[i].push_back(source);
- break;
- }
- }
- }
- sourceDB.unlock();
-
- vector<Patch::ConstPtr> patchList;
- patchList.reserve (componentsList.size());
- for (uint i=0; i<componentsList.size(); ++i) {
- ASSERTSTR (!componentsList[i].empty(), "No sources found for patch "
- << patchNames[i]);
- Patch::Ptr ppatch(new Patch(patchNames[i],
- componentsList[i].begin(),
- componentsList[i].end()));
- vector<BBS::PatchInfo> patchInfo(sourceDB.getPatchInfo(-1, patchNames[i]));
- ASSERT (patchInfo.size() == 1);
- // Set the position and apparent flux of the patch.
- Position patchPosition;
- patchPosition[0] = patchInfo[0].getRa();
- patchPosition[1] = patchInfo[0].getDec();
- ppatch->setPosition (patchPosition);
- ppatch->setBrightness (patchInfo[0].apparentBrightness());
- /// ppatch->computePosition();
- patchList.push_back (ppatch);
- }
- return patchList;
-}
-
-vector<pair<ModelComponent::ConstPtr,Patch::ConstPtr> >
-makeSourceList (const vector<Patch::ConstPtr>& patchList) {
- vector<Patch::ConstPtr>::const_iterator pIter=patchList.begin();
- vector<Patch::ConstPtr>::const_iterator pEnd =patchList.end();
-
- uint nSources=0;
- for (; pIter!=pEnd; ++pIter) {
- nSources+=(*pIter)->nComponents();
- }
-
- vector<pair<ModelComponent::ConstPtr,Patch::ConstPtr> > sourceList;
- sourceList.reserve(nSources);
-
- pIter=patchList.begin();
-
- for (; pIter!=pEnd; ++pIter) {
- Patch::const_iterator sIter=(*pIter)->begin();
- Patch::const_iterator sEnd =(*pIter)->end();
- for (; sIter!=sEnd; ++sIter) {
- sourceList.push_back(make_pair(*sIter,*pIter));
- }
- }
-
- return sourceList;
-}
-
-
-vector<Patch::ConstPtr> makeOnePatchPerComponent(
- const vector<Patch::ConstPtr>& patchList) {
- size_t numComponents=0;
- vector<Patch::ConstPtr>::const_iterator patchIt;
-
- for (patchIt=patchList.begin();patchIt!=patchList.end();++patchIt) {
- numComponents+=(*patchIt)->nComponents();
- }
-
- vector<Patch::ConstPtr> largePatchList;
- largePatchList.reserve(numComponents);
-
- for (patchIt=patchList.begin();patchIt!=patchList.end();++patchIt) {
- Patch::const_iterator compIt;
-
- size_t compNum=0;
- for (compIt=(*patchIt)->begin();compIt!=(*patchIt)->end();++compIt) {
- // convert compNum to string (blegh)
- stringstream ss;
- ss<<compNum;
-
- Patch::Ptr ppatch(new Patch((*patchIt)->name()+"_"+ss.str(),
- compIt,
- compIt+1));
- ppatch->setPosition((*compIt)->position());
- largePatchList.push_back(ppatch);
- compNum++;
- }
- }
-
- return largePatchList;
-}
-
-
-vector<string> makePatchList(SourceDB &sourceDB, vector<string> patterns)
-{
- if(patterns.empty())
- {
- patterns.push_back("*");
- }
-
- std::set<string> patches;
- vector<string>::iterator it = patterns.begin();
- while(it != patterns.end())
- {
- if(!it->empty() && (*it)[0] == '@')
- {
- patches.insert(*it);
- it = patterns.erase(it);
- }
- else
- {
- vector<string> match(sourceDB.getPatches(-1, *it));
- patches.insert(match.begin(), match.end());
- ++it;
- }
- }
-
- return vector<string>(patches.begin(), patches.end());
-}
-
-
-bool checkPolarized(SourceDB &sourceDB,
- const vector<string> &patchNames,
- uint nModel)
-{
- bool polarized = false;
-
- // Loop over all sources.
- sourceDB.lock();
- sourceDB.rewind();
- SourceData src;
- while (! sourceDB.atEnd()) {
- sourceDB.getNextSource (src);
- // Use the source if its patch matches a patch name.
- for (uint i=0; i<nModel; ++i) {
- if (src.getPatchName() == patchNames[i]) {
- // Determine whether source is unpolarized.
- if (src.getV() > 0.0 || src.getQ() > 0.0 || src.getU() > 0.0) {
- polarized = true;
- break;
- }
- }
- }
- if (polarized) {
- break;
- }
- }
- sourceDB.unlock();
- return polarized;
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/Split.cc b/CEP/DP3/DPPP/src/Split.cc
deleted file mode 100644
index 4f114295394cf92346fd22ce537d7f1eeae20f4e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Split.cc
+++ /dev/null
@@ -1,140 +0,0 @@
-//# Split.cc: DPPP step class to Split visibilities
-//# Copyright (C) 2018
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/Split.h>
-#include <DPPP/DPRun.h>
-
-#include <iostream>
-#include <Common/ParameterSet.h>
-#include <Common/Timer.h>
-
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- Split::Split (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- {
- itsReplaceParms = parset.getStringVector(prefix + "replaceparms");
- vector<vector<string> > replaceParmValues; // For each of the parameters, the values for each substep
- replaceParmValues.resize(itsReplaceParms.size());
-
- vector<vector<string> >::iterator replaceParmValueIt = replaceParmValues.begin();
- vector<string>::iterator replaceParmIt;
- uint numSteps = 0;
- for (replaceParmIt = itsReplaceParms.begin();
- replaceParmIt != itsReplaceParms.end(); ++replaceParmIt) {
- vector<string> parmValues = parset.getStringVector(*replaceParmIt);
- *(replaceParmValueIt++) = parmValues;
- if (numSteps > 0) {
- ASSERTSTR(parmValues.size() == numSteps, "Each parameter in replaceparms should have the same number of items (expected "<<
- numSteps <<", got "<<parmValues.size() <<" for step "<<(*replaceParmIt));
- } else {
- numSteps = parmValues.size();
- }
- }
-
- // Make a shallow copy to work around constness of parset
- ParameterSet parsetCopy(parset);
-
- // Create the substeps
- uint numParameters = itsReplaceParms.size();
- for (uint i = 0; i<numSteps; ++i) {
- for (uint j = 0; j<numParameters; ++j) {
- parsetCopy.replace(itsReplaceParms[j], replaceParmValues[j][i]);
- }
- DPStep::ShPtr firstStep = DPRun::makeSteps (parsetCopy, prefix, input);
- firstStep->setPrevStep(this);
- itsSubsteps.push_back(firstStep);
- }
- ASSERT(itsSubsteps.size()>0);
- }
-
- Split::~Split()
- {}
-
- void Split::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
-
- vector<DPStep::ShPtr>::iterator it;
- for (it=itsSubsteps.begin(); it!=itsSubsteps.end(); ++it) {
- (*it)->setInfo(infoIn);
- }
- }
-
- void Split::show (std::ostream& os) const
- {
- os << "Split " << itsName << endl;
- os << " replace parameters:" << itsReplaceParms << endl;
- // Show the steps.
- for (uint i=0; i<itsSubsteps.size(); ++i) {
- os << "Split substep "<<(i+1)<<" of "<<itsSubsteps.size()<<endl;
- DPStep::ShPtr step = itsSubsteps[0];
- DPStep::ShPtr lastStep;
- while (step) {
- step->show (os);
- lastStep = step;
- step = step->getNextStep();
- }
- }
- }
-
- void Split::showTimings (std::ostream& os, double duration) const
- {
- for (uint i=0; i<itsSubsteps.size(); ++i) {
- DPStep::ShPtr step = itsSubsteps[i];
- while (step) {
- step->showTimings(os, duration);
- step = step->getNextStep();
- }
- }
- }
-
- bool Split::process (const DPBuffer& bufin)
- {
- for (uint i=0; i<itsSubsteps.size(); ++i) {
- itsSubsteps[i]->process(bufin);
- }
- return false;
- }
-
-
- void Split::finish()
- {
- // Let the next steps finish.
- for (uint i=0; i<itsSubsteps.size(); ++i) {
- itsSubsteps[i]->finish();
- }
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/StationAdder.cc b/CEP/DP3/DPPP/src/StationAdder.cc
deleted file mode 100644
index 5cb92fd69fd599e984ce31e732dd2a23c76f9edb..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/StationAdder.cc
+++ /dev/null
@@ -1,634 +0,0 @@
-//# StationAdder.cc: DPPP step class to add stations as a superstation
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/StationAdder.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPLogger.h>
-#include <Common/ParameterSet.h>
-#include <Common/ParameterRecord.h>
-
-#include <casacore/measures/Measures/MPosition.h>
-#include <casacore/measures/Measures/MCPosition.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/TableMeasures/ScalarMeasColumn.h>
-#include <casacore/tables/Tables/ScalarColumn.h>
-#include <casacore/tables/Tables/ArrayColumn.h>
-#include <casacore/tables/Tables/TableRecord.h>
-#include <casacore/tables/Tables/TableRow.h>
-#include <casacore/casa/Utilities/LinearSearch.h>
-#include <casacore/casa/Utilities/Regex.h>
-#include <iostream>
-#include <iomanip>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- StationAdder::StationAdder (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsStatRec (parset.getRecord(prefix+"stations")),
- itsMinNPoint (parset.getUint (prefix+"minpoints", 1)),
- itsMakeAutoCorr (parset.getBool (prefix+"autocorr", false)),
- itsSumAutoCorr (parset.getBool (prefix+"sumauto", true)),
- itsDoAverage (parset.getBool (prefix+"average", true)),
- itsUseWeight (parset.getBool (prefix+"useweights", true))
- {
- }
-
- StationAdder::~StationAdder()
- {}
-
- vector<int> StationAdder::getMatchingStations
- (const Vector<String>& antennaNames, const vector<string>& patterns)
- {
- Vector<bool> used(antennaNames.size(), false);
- for (vector<string>::const_iterator iter=patterns.begin();
- iter!=patterns.end(); ++iter) {
- int n=0;
- if (iter->size() > 1 && ((*iter)[0] == '!' || (*iter)[0] == '^')) {
- Regex regex(Regex::fromPattern(iter->substr(1)));
- for (uint i=0; i<antennaNames.size(); ++i) {
- if (antennaNames[i].matches (regex)) {
- used[i] = false;
- n++;
- }
- }
- } else {
- Regex regex(Regex::fromPattern(*iter));
- for (uint i=0; i<antennaNames.size(); ++i) {
- if (antennaNames[i].matches (regex)) {
- used[i] = true;
- n++;
- }
- }
- }
- if (n == 0) {
- DPLOG_WARN_STR
- ("StationAdder: no matching stations found for pattern " << *iter);
- }
- }
- vector<int> parts;
- parts.reserve (12); // Usually up to 12 stations are used
- for (uint i=0; i<used.size(); ++i) {
- if (used[i]) {
- parts.push_back (i);
- }
- }
- return parts;
- }
-
- void StationAdder::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- info().setMetaChanged();
- // Check the superstation definition(s).
- // They are specified as a ParameterRecord like:
- // stations = {new1:[s1,s2,s3], new2:[s4,s5,s6]}
- // where s1, etc. can be glob patterns.
- Vector<String> antennaNames (infoIn.antennaNames());
- Vector<Double> antennaDiam (infoIn.antennaDiam());
- vector<MPosition> antennaPos(info().antennaPos());
- // For each existing station, give id of new superstation it is used in.
- vector<int> newStations (antennaNames.size());
- std::fill (newStations.begin(), newStations.end(), -1);
- vector<string> newNames; // Names of new superstations
- vector<double> newDiam;
- vector<MPosition> newPoss;
- for (ParameterRecord::const_iterator iter = itsStatRec.begin();
- iter != itsStatRec.end(); ++iter) {
- if (std::find(antennaNames.begin(), antennaNames.end(),
- String(iter->first)) != antennaNames.end() ||
- std::find(newNames.begin(),
- newNames.end(), iter->first) != newNames.end()) {
- THROW (Exception, "StationAdder: new station name " + iter->first +
- " already exists");
- }
- // Get the ids of the stations forming the new superstation.
- // Expand possible .. in the parameter value.
- vector<int> parts = getMatchingStations
- (antennaNames, iter->second.expand().getStringVector());
- if (parts.empty()) {
- continue;
- }
- MVPosition newPosition;
- // Check if the stations exist and not used for other superstations.
- // Add their ITRF positions.
- for (uint i=0; i<parts.size(); ++i) {
- int inx = parts[i];
- ASSERTSTR (newStations[inx] < 0, "Station " + antennaNames[inx] +
- " is used in multiple superstations");
- newStations[inx] = newNames.size();
- newPosition += MPosition::Convert (antennaPos[inx],
- MPosition::ITRF)().getValue();
- }
- // The superstation position is the average of its parts.
- newNames.push_back (iter->first);
- newPosition *= 1./parts.size();
- newPoss.push_back (MPosition(newPosition, MPosition::ITRF));
- itsParts.push_back (Vector<int>(parts));
- // Set the diameter of the new station by determining the
- // maximum distance to the center.
- double maxdist = 0;
- for (uint i=0; i<parts.size(); ++i) {
- int inx = parts[i];
- MVPosition mvdiff = newPosition -
- MPosition::Convert (antennaPos[inx], MPosition::ITRF)().getValue();
- const Vector<Double>& diff = mvdiff.getValue();
- double dist = sqrt(std::accumulate(diff.cbegin(), diff.cend(), 0.,
- casacore::SumSqr<Double>()));
- // Add the radius of the station used.
- maxdist = max (maxdist, dist + 0.5*antennaDiam[inx]);
- }
- newDiam.push_back (2*maxdist);
- }
- // Add the new stations to the info's vectors.
- Vector<Int> ant1 (info().getAnt1());
- Vector<Int> ant2 (info().getAnt2());
- uint nrold = antennaNames.size();
- uint nrnew = nrold + newNames.size();
- antennaNames.resize (nrnew, True);
- antennaDiam.resize (nrnew, True);
- antennaPos.reserve (nrnew);
- for (uint i=0; i<newNames.size(); ++i) {
- antennaNames[nrold+i] = newNames[i];
- antennaDiam[nrold+i] = newDiam[i];
- antennaPos.push_back (newPoss[i]);
- }
- // Now determine the new baselines.
- // A new baseline is formed if ANTENNA1 or ANTENNA2 is in the list
- // of stations, but not both.
- // However, for auto-correlations they can be the same.
- vector<int> newbl(nrnew);
- // Loop over the superstations.
- // Note that by making this the outer loop, the baselines between
- // superstations are also formed.
- // At the end the new baselines are added to itsAnt1 and itsAnt2.
- // itsBufRows contains for each new baseline the rownrs in the DPBuffer
- // to be added for the new baseline. If rownr<0, the conjugate has to be
- // added (1 is added to rownr, otherwise 0 is ambiguous).
- // Note that a rownr can be the rownr of a new baseline.
- for (uint j=0; j<itsParts.size(); ++j) {
- std::fill (newbl.begin(), newbl.end(), -1);
- vector<int> newAnt1;
- vector<int> newAnt2;
- // Loop through all baselines and find out if a baseline should
- // be used for a superstation.
- for (uint i=0; i<ant1.size(); ++i) {
- bool havea1 = linearSearch1 (itsParts[j], ant1[i]) >= 0;
- bool havea2 = linearSearch1 (itsParts[j], ant2[i]) >= 0;
- int ant = nrold+j;
- int take = 0;
- if (havea1) {
- // If both stations are in same superstation, only use them
- // if autocorrelations have to be made.
- // Taking auto or cross depends on summing mode.
- if (havea2) {
- take = itsMakeAutoCorr && itsSumAutoCorr == (ant1[i]==ant2[i]);
- } else {
- ant = ant2[i];
- take = -1; // conjugate has to be added
- }
- } else if (havea2) {
- ant = ant1[i];
- take = 1;
- }
- if (take != 0) {
- // We have a baseline for the superstation.
- // Get its index; create it if not used before.
- int blinx = newbl[ant];
- if (blinx < 0) {
- blinx = newbl[ant] = itsBufRows.size();
- itsBufRows.push_back (vector<int>());
- newAnt1.push_back (ant);
- newAnt2.push_back (nrold+j);
- }
- itsBufRows[blinx].push_back (take*int(i+1));
- }
- }
- // Copy the new baselines for this superstation to the baseline list.
- // Give a warning if nothing found.
- if (newAnt1.empty()) {
- DPLOG_WARN_STR ("StationAdder: no baseline found for superstation");
- } else {
- uint oldsz = ant1.size();
- ant1.resize (oldsz + newAnt1.size(), True);
- ant2.resize (oldsz + newAnt1.size(), True);
- for (uint i=0; i<newAnt1.size(); ++i) {
- ant1[oldsz+i] = newAnt1[i];
- ant2[oldsz+i] = newAnt2[i];
- }
- }
- }
- // Set the new info.
- info().set (antennaNames, antennaDiam, antennaPos, ant1, ant2);
- // Setup the UVW calculator (for new baselines).
- itsUVWCalc = UVWCalculator (infoIn.phaseCenter(), infoIn.arrayPos(),
- antennaPos);
- // Size the buffer to cater for the new baselines.
- IPosition dataShp (3, getInfo().ncorr(), getInfo().nchan(),
- getInfo().nbaselines());
- IPosition uvwShp (2, 3, getInfo().nbaselines());
- itsBuf.getData().resize (dataShp);
- itsBuf.getFlags().resize (dataShp);
- itsBuf.getWeights().resize (dataShp);
- itsBuf.getUVW().resize (uvwShp);
- }
-
- void StationAdder::show (std::ostream& os) const
- {
- os << "StationAdder " << itsName << std::endl;
- os << " stations: " << itsStatRec << std::endl;
- // Show the stations used for the new stations.
- uint nold = getInfo().antennaNames().size() - itsParts.size();
- for (uint i=0; i<itsParts.size(); ++i) {
- os << " " << getInfo().antennaNames()[nold+i] << ": [";
- for (uint j=0; j<itsParts[i].size(); ++j) {
- if (j > 0) os << ", ";
- os << getInfo().antennaNames()[itsParts[i][j]];
- }
- os << ']' << endl;
- }
- os << " minpoints: " << itsMinNPoint << std::endl;
- os << " autocorr: " << boolalpha << itsMakeAutoCorr << std::endl;
- os << " sumauto: " << boolalpha << itsSumAutoCorr << std::endl;
- os << " average: " << boolalpha << itsDoAverage << std::endl;
- os << " useweights: " << boolalpha << itsUseWeight << std::endl;
- }
-
- void StationAdder::showTimings (std::ostream& os, double duration) const
- {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " StationAdder " << itsName << endl;
- }
-
- bool StationAdder::process (const DPBuffer& buf)
- {
- itsTimer.start();
- // Get the various data arrays.
- itsBufTmp.referenceFilled (buf);
- const Array<Complex>& data = buf.getData();
- const Array<Bool>& flags = buf.getFlags();
- const Array<Float>& weights =
- itsInput->fetchWeights (buf, itsBufTmp, itsTimer);
- const Array<Double>& uvws =
- itsInput->fetchUVW (buf, itsBufTmp, itsTimer);
- const Array<Bool>& frFlags =
- itsInput->fetchFullResFlags (buf, itsBufTmp, itsTimer);
- // Size fullResFlags if not done yet.
- if (itsBuf.getFullResFlags().empty()) {
- IPosition frfShp = frFlags.shape();
- frfShp[2] = getInfo().nbaselines();
- itsBuf.getFullResFlags().resize (frfShp);
- }
- // Copy the data; only the first baselines will be filled.
- std::copy (data.data(), data.data() + data.size(),
- itsBuf.getData().data());
- std::copy (flags.data(), flags.data() + flags.size(),
- itsBuf.getFlags().data());
- std::copy (weights.data(), weights.data() + weights.size(),
- itsBuf.getWeights().data());
- std::copy (uvws.data(), uvws.data() + uvws.size(),
- itsBuf.getUVW().data());
- std::copy (frFlags.data(), frFlags.data() + frFlags.size(),
- itsBuf.getFullResFlags().data());
- // Now calculate the data pointers of the new baselines.
- uint nrOldBL = data.shape()[2];
- uint nrcc = data.shape()[0] * data.shape()[1];
- uint nrfr = frFlags.shape()[0] * frFlags.shape()[1];
- Complex* dataPtr = itsBuf.getData().data() + data.size();
- Bool* flagPtr = itsBuf.getFlags().data() + data.size();
- Float* wghtPtr = itsBuf.getWeights().data() + data.size();
- Double* uvwPtr = itsBuf.getUVW().data() + uvws.size();
- Bool* frfPtr = itsBuf.getFullResFlags().data() + frFlags.size();
- vector<uint> npoints(nrcc);
- vector<Complex> dataFlg(nrcc);
- vector<Float> wghtFlg(nrcc);
- // Loop over all new baselines.
- for (uint i=0; i<itsBufRows.size(); ++i) {
- // Clear the data for the new baseline.
- for (uint k=0; k<nrcc; ++k) {
- dataPtr[k] = Complex();
- wghtPtr[k] = 0.;
- npoints[k] = 0;
- dataFlg[k] = Complex();
- wghtFlg[k] = 0.;
- }
- for (uint k=0; k<nrfr; ++k) {
- frfPtr[k] = true;
- }
-
- for (uint k=0; k<3; ++k) {
- uvwPtr[k] = 0.;
- }
- double uvwWghtSum = 0.;
-
- // Sum the baselines forming the new baselines.
- for (uint j=0; j<itsBufRows[i].size(); ++j) {
- // Get the baseline number to use.
- // A negative one means using the conjugate.
- int blnr = itsBufRows[i][j];
- bool useConj = false;
- if (blnr < 0) {
- blnr = -blnr;
- useConj = true;
- }
- blnr--; // decrement because blnr+1 is stored in itsBufRows
- // Get pointers to the input baseline data.
- const Complex* inDataPtr = (itsBuf.getData().data() +
- blnr*nrcc);
- const Bool* inFlagPtr = (itsBuf.getFlags().data() +
- blnr*nrcc);
- const Float* inWghtPtr = (itsBuf.getWeights().data() +
- blnr*nrcc);
- const Bool* inFrfPtr = (itsBuf.getFullResFlags().data() +
- blnr*nrfr);
- const Double* inUvwPtr = (itsBuf.getUVW().data() +
- blnr*3);
- // Add the data, uvw, and weights if not flagged.
- // Write 4 loops to avoid having to test inside the loop.
- // Count the flagged points separately, so it can be used
- // if too many points are flagged.
- if (useConj) {
- if (itsUseWeight) {
- for (uint k=0; k<nrcc; ++k) {
- if (inFlagPtr[k]) {
- dataFlg[k] += conj(inDataPtr[k]) * inWghtPtr[k];
- wghtFlg[k] += inWghtPtr[k];
- } else {
- npoints[k]++;
- dataPtr[k] += conj(inDataPtr[k]) * inWghtPtr[k];
- wghtPtr[k] += inWghtPtr[k];
- for (int ui=0; ui<3; ++ui) {
- uvwPtr[ui] -= inUvwPtr[ui] * inWghtPtr[k];
- }
- uvwWghtSum += inWghtPtr[k];
- }
- }
- } else {
- for (uint k=0; k<nrcc; ++k) {
- if (inFlagPtr[k]) {
- dataFlg[k] += conj(inDataPtr[k]);
- wghtFlg[k] += 1.;
- } else {
- npoints[k]++;
- dataPtr[k] += conj(inDataPtr[k]);
- wghtPtr[k] += 1.;
- for (int ui=0; ui<3; ++ui) {
- uvwPtr[ui] -= inUvwPtr[ui];
- }
- uvwWghtSum += 1;
- }
- }
- }
- } else {
- if (itsUseWeight) {
- for (uint k=0; k<nrcc; ++k) {
- if (inFlagPtr[k]) {
- dataFlg[k] += inDataPtr[k] * inWghtPtr[k];
- wghtFlg[k] += inWghtPtr[k];
- } else {
- npoints[k]++;
- dataPtr[k] += inDataPtr[k] * inWghtPtr[k];
- wghtPtr[k] += inWghtPtr[k];
- for (int ui=0; ui<3; ++ui) {
- uvwPtr[ui] += inUvwPtr[ui] * inWghtPtr[k];
- }
- uvwWghtSum += inWghtPtr[k];
- }
- }
- } else {
- for (uint k=0; k<nrcc; ++k) {
- if (inFlagPtr[k]) {
- dataFlg[k] += inDataPtr[k];
- wghtFlg[k] += 1.;
- } else {
- npoints[k]++;
- dataPtr[k] += inDataPtr[k];
- wghtPtr[k] += 1.;
- for (int ui=0; ui<3; ++ui) {
- uvwPtr[ui] += inUvwPtr[ui];
- }
- uvwWghtSum += 1;
- }
- }
- }
- }
- // It is a bit hard to say what to do with FULL_RES_FLAGS.
- // Set it to true (=flagged) if the flag of all baselines is true.
- for (uint k=0; k<nrfr; ++k) {
- frfPtr[k] = frfPtr[k] && inFrfPtr[k];
- }
- }
- // Set the resulting flags. Average if needed.
- // Set flag if too few unflagged data points; use flagged data too.
- for (uint k=0; k<nrcc; ++k) {
- if (wghtPtr[k] == 0 || npoints[k] < itsMinNPoint) {
- flagPtr[k] = true;
- dataPtr[k] += dataFlg[k];
- wghtPtr[k] += wghtFlg[k];
- } else {
- flagPtr[k] = false;
- }
- if (itsDoAverage) {
- dataPtr[k] /= wghtPtr[k];
- }
- }
-
- // Average or calculate the UVW coordinate of the new station.
- if (itsDoAverage && uvwWghtSum != 0) {
- for (int ui=0; ui<3; ++ui) {
- uvwPtr[ui] /= uvwWghtSum;
- }
- } else {
- uint blnr = nrOldBL + i;
- Vector<Double> uvws = itsUVWCalc.getUVW (getInfo().getAnt1()[blnr],
- getInfo().getAnt2()[blnr],
- buf.getTime());
- uvwPtr[0] = uvws[0];
- uvwPtr[1] = uvws[1];
- uvwPtr[2] = uvws[2];
- }
-
- dataPtr += nrcc;
- flagPtr += nrcc;
- wghtPtr += nrcc;
- uvwPtr += 3;
- frfPtr += nrfr;
- }
- itsBuf.setTime (buf.getTime());
- itsBuf.setExposure (buf.getExposure());
- itsTimer.stop();
- getNextStep()->process (itsBuf);
- return true;
- }
-
- void StationAdder::finish()
- {
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- void StationAdder::addToMS (const string& msName)
- {
- getPrevStep()->addToMS(msName);
- // Add the new stations to the ANTENNA subtable.
- Table antTab (msName + "/ANTENNA", Table::Update);
- ScalarColumn<String> nameCol (antTab, "NAME");
- ScalarColumn<String> typeCol (antTab, "TYPE");
- ScalarColumn<String> mountCol (antTab, "MOUNT");
- ArrayColumn<Double> offsetCol (antTab, "OFFSET");
- ScalarColumn<Double> diamCol (antTab, "DISH_DIAMETER");
- ScalarColumn<Bool> flagCol (antTab, "FLAG_ROW");
- ScalarColumn<String> statCol;
- ScalarColumn<Int> stidCol;
- ScalarMeasColumn<MPosition> phrefCol;
- ScalarMeasColumn<MPosition> posCol (antTab, "POSITION");
- // LOFAR columns are optional.
- if (antTab.tableDesc().isColumn ("STATION")) {
- statCol.attach (antTab, "STATION");
- }
- if (antTab.tableDesc().isColumn ("LOFAR_STATION_ID")) {
- stidCol.attach (antTab, "LOFAR_STATION_ID");
- }
- if (antTab.tableDesc().isColumn ("LOFAR_PHASE_REFERENCE")) {
- phrefCol.attach (antTab, "LOFAR_PHASE_REFERENCE");
- }
- uint origNant = antTab.nrow();
- // Take common info from the first row.
- String type, mount, stat;
- if (origNant > 0) {
- type = typeCol(0);
- mount = mountCol(0);
- if (! statCol.isNull()) {
- stat = statCol(0);
- }
- }
- Vector<Double> offset(3, 0.);
- // Put the data for each new antenna.
- for (uint i=origNant; i<getInfo().antennaNames().size(); ++i) {
- antTab.addRow();
- nameCol.put (i, getInfo().antennaNames()[i]);
- typeCol.put (i, type);
- mountCol.put (i, mount);
- offsetCol.put (i, offset);
- diamCol.put (i, getInfo().antennaDiam()[i]);
- flagCol.put (i, false);
- posCol.put (i, getInfo().antennaPos()[i]);
- if (! statCol.isNull()) {
- statCol.put (i, stat);
- }
- if (! stidCol.isNull()) {
- stidCol.put (i, -1);
- }
- if (! phrefCol.isNull()) {
- phrefCol.put (i, getInfo().antennaPos()[i]);
- }
- }
- // For each new station, add a row to the FEED subtable.
- // It is a copy of the first row, except for the station id.
- Table feedTab (msName + "/FEED", Table::Update);
- TableRow feedRow(feedTab);
- ScalarColumn<Int> antidCol(feedTab, "ANTENNA_ID");
- for (uint i=origNant; i<getInfo().antennaNames().size(); ++i) {
- uInt rownr = feedTab.nrow();
- feedTab.addRow();
- feedRow.put (rownr, feedRow.get(0));
- antidCol.put (rownr, i);
- }
- // Now update the BeamInfo tables if they are present.
- Table ms(msName);
- if (ms.keywordSet().isDefined("LOFAR_ANTENNA_FIELD")) {
- updateBeamInfo (msName, origNant, antTab);
- }
- }
-
- void StationAdder::updateBeamInfo (const string& msName, uint origNant,
- Table& antTab)
- {
- // Update the LOFAR_ANTENNA_FIELD table.
- // Copy all rows where ANTENNA_ID matches one of the stations used in
- // a superstation.
- Table afTab (msName + "/LOFAR_ANTENNA_FIELD", Table::Update);
- TableRow afRow (afTab);
- ScalarColumn<Int> antIdCol(afTab, "ANTENNA_ID");
- for (uint i=0; i<afTab.nrow(); ++i) {
- for (uint j=0; j<itsParts.size(); ++j) {
- int inx = linearSearch1 (itsParts[j], antIdCol(i));
- if (inx >= 0) {
- int rownr = afTab.nrow();
- afTab.addRow();
- afRow.put (rownr, afRow.get(i));
- antIdCol.put (rownr, j+origNant);
- }
- }
- }
- // Flush the table.
- afTab.flush();
- // Update the LOFAR_STATION table.
- Table statTab (msName + "/LOFAR_STATION", Table::Update);
- ScalarColumn<String> nameCol (statTab, "NAME");
- ScalarColumn<Int> clockCol (statTab, "CLOCK_ID");
- ScalarColumn<Bool> flagCol (statTab, "FLAG_ROW");
- // LOFAR_STATION_ID in the ANTENNA table needs to be updated.
- ScalarColumn<Int> stidCol (antTab, "LOFAR_STATION_ID");
- // Get station-ids for each antenna-id.
- // Get clock-ids for each station-id.
- Vector<Int> statIds (stidCol.getColumn());
- Vector<Int> clockIds (clockCol.getColumn());
- int nextClockId = max(clockIds);
- // Loop over all new antennae.
- for (uint i=0; i<itsParts.size(); ++i) {
- if (itsParts[i].empty()) {
- break;
- }
- // Do all antennae of a new antenna share the same clock?
- // If so, use that clock-id, otherwise make a new one.
- int cid = clockIds[statIds[itsParts[i][0]]];
- for (uint j=1; j<itsParts[i].size(); ++j) {
- if (clockIds[statIds[itsParts[i][j]]] != cid) {
- cid = ++nextClockId;
- break;
- }
- }
- // Update LOFAR_STATION_ID for this new antenna.
- int rownr = statTab.nrow();
- stidCol.put (origNant+i, rownr);
- // Add new station.
- statTab.addRow();
- nameCol.put (rownr, getInfo().antennaNames()[origNant+i]);
- clockCol.put (rownr, cid);
- flagCol.put (rownr, False);
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/StefCal.cc b/CEP/DP3/DPPP/src/StefCal.cc
deleted file mode 100644
index 0df7cca267481f61532a219e80c78c6aa6f0fb2b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/StefCal.cc
+++ /dev/null
@@ -1,510 +0,0 @@
-//# StefCal.cc: Perform StefCal algorithm for gain calibration
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: StefCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/StefCal.h>
-#include <DPPP/DPInput.h>
-
-#include <vector>
-#include <algorithm>
-#include <limits>
-
-#include <iostream>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- StefCal::StefCal(uint solInt, uint nChan, StefCalMode mode,
- bool scalar, double tolerance, uint maxAntennas,
- bool detectStalling, uint debugLevel)
- : _nSt (maxAntennas),
- _badIters (0),
- _veryBadIters (0),
- _solInt (solInt),
- _nChan (nChan),
- _mode (mode),
- _scalar (scalar),
- _tolerance (tolerance),
- _totalWeight (0.),
- _detectStalling (detectStalling),
- _debugLevel (debugLevel)
- {
- resetVis();
-
- _nSt = maxAntennas;
- if (_mode==FULLJONES) {
- ASSERT(!_scalar);
- _nCr=4;
- _nSp=1;
- _savedNCr=4;
- } else if (_scalar) {
- _nCr=1;
- _nSp=2;
- _savedNCr=1;
- } else {
- _nCr=1;
- _nSp=1;
- _savedNCr=2;
- }
-
- _vis.resize(IPosition(6,_nSt,2,_solInt,_nChan,2,_nSt));
- _mvis.resize(IPosition(6,_nSt,2,_solInt,_nChan,2,_nSt));
-
- if (_scalar || _mode==FULLJONES) {
- _nUn = _nSt;
- } else {
- _nUn = 2*_nSt;
- }
-
- _g.resize(_nUn,_nCr);
- _gold.resize(_nUn,_nCr);
- _gx.resize(_nUn,_nCr);
- _gxx.resize(_nUn,_nCr);
- _h.resize(_nUn,_nCr);
- _z.resize(_nUn*_nChan*_solInt*_nSp,_nCr);
-
- _stationFlagged.resize(_nSt, false);
-
- init(true);
- }
-
- void StefCal::resetVis() {
- _vis=0;
- _mvis=0;
- _totalWeight = 0.;
- }
-
- void StefCal::clearStationFlagged() {
- _stationFlagged=false;
- }
-
- void StefCal::init(bool initSolutions) {
- _dg = 1.0e29;
- _dgx = 1.0e30;
- _dgs.clear();
-
- _badIters = 0;
- _veryBadIters = 0;
-
- if (initSolutions) {
- double ginit=1.0;
- if (_mode != PHASEONLY) {
- // Initialize solution with sensible amplitudes
- double fronormvis=0;
- double fronormmod=0;
-
- DComplex* t_vis_p=_vis.data();
- DComplex* t_mvis_p=_mvis.data();
-
- uint vissize=_vis.size();
- for (uint i=0;i<vissize;++i) {
- fronormvis+=norm(t_vis_p[i]);
- fronormmod+=norm(t_mvis_p[i]);
- }
-
- fronormvis=sqrt(fronormvis);
- fronormmod=sqrt(fronormmod);
- if (abs(fronormmod)>1.e-15) {
- ginit=sqrt(fronormvis/fronormmod);
- } else {
- ginit=1.0;
- }
- }
-
- if (_nCr==4) {
- for (uint ant=0;ant<_nUn;++ant) {
- _g(ant,0)=ginit;
- _g(ant,1)=0.;
- _g(ant,2)=0.;
- _g(ant,3)=ginit;
- }
- } else {
- _g=ginit;
- }
- } else { // Take care of NaNs in solution
- double ginit=0.;
- bool ginitcomputed=false;
- for (uint ant=0; ant<_nUn; ++ant) {
- if (!isFinite(_g(ant,0).real()) ) {
- if (!ginitcomputed && !_stationFlagged[ant%_nSt]) {
- // Avoid calling getAverageUnflaggedSolution for stations that are always flagged
- ginit = getAverageUnflaggedSolution();
- ginitcomputed = true;
- if (ginit==0) {
- init(true);
- return;
- }
- }
- if (_nCr==4) {
- _g(ant,0)=ginit;
- _g(ant,1)=0.;
- _g(ant,2)=0.;
- _g(ant,3)=ginit;
- } else {
- _g(ant,0)=ginit;
- }
- }
- }
- }
- }
-
- double StefCal::getAverageUnflaggedSolution() {
- // Get average solution of unflagged antennas only once
- // Unflagged means unflagged in previous time slot, so
- // look at NaNs, don't look at stationFlagged (that's for
- // the current timeslot).
- double total=0.;
- uint unflaggedstations=0;
- for (uint ant2=0; ant2<_nUn; ++ant2) {
- if (isFinite(_g(ant2,0).real())) {
- total += abs(_g(ant2,0));
- unflaggedstations++;
- if (_nCr==4) {
- total += abs(_g(ant2,3));
- unflaggedstations++;
- }
- }
- }
- if (unflaggedstations==0) {
- return 0.;
- } else {
- return total/unflaggedstations;
- }
- }
-
- StefCal::Status StefCal::doStep(uint iter) {
- _gxx = _gx;
- _gx = _g;
-
- bool allFlagged=true;
- for (uint st1=0;st1<_nSt;++st1) {
- if (!_stationFlagged[st1]) {
- allFlagged=false;
- break;
- }
- }
- if (allFlagged) {
- return CONVERGED;
- }
-
- if (_mode==FULLJONES) {
- doStep_polarized();
- doStep_polarized();
- return relax(2*iter);
- } else {
- doStep_unpolarized();
- doStep_unpolarized();
- return relax(2*iter);
- }
- }
-
- void StefCal::doStep_polarized() {
- _gold = _g;
-
- for (uint st=0;st<_nSt;++st) {
- _h(st,0)=conj(_g(st,0));
- _h(st,1)=conj(_g(st,1));
- _h(st,2)=conj(_g(st,2));
- _h(st,3)=conj(_g(st,3));
- }
-
- for (uint st1=0;st1<_nSt;++st1) {
- if (_stationFlagged[st1]) {
- continue;
- }
-
- DComplex* vis_p;
- DComplex* mvis_p;
- Vector<DComplex> w(_nCr);
- Vector<DComplex> t(_nCr);
-
- for (uint time=0;time<_solInt;++time) {
- for (uint ch=0;ch<_nChan;++ch) {
- uint zoff=_nSt*ch+_nSt*_nChan*time;
- mvis_p=&_mvis(IPosition(6,0,0,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,0) = _h(st2,0) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,0,time,ch,0,st1))
- mvis_p=&_mvis(IPosition(6,0,1,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,0) += _h(st2,2) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,0,time,ch,1,st1))
- mvis_p=&_mvis(IPosition(6,0,0,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,1) = _h(st2,0) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,1,time,ch,0,st1))
- mvis_p=&_mvis(IPosition(6,0,1,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,1) += _h(st2,2) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,1,time,ch,1,st1))
- mvis_p=&_mvis(IPosition(6,0,0,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,2) = _h(st2,1) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,0,time,ch,0,st1))
- mvis_p=&_mvis(IPosition(6,0,1,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,2) += _h(st2,3) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,0,time,ch,1,st1))
- mvis_p=&_mvis(IPosition(6,0,0,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,3) = _h(st2,1) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,1,time,ch,0,st1))
- mvis_p=&_mvis(IPosition(6,0,1,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { _z(st2+zoff,3) += _h(st2,3) * mvis_p[st2]; } // itsMVis(IPosition(6,st2,1,time,ch,1,st1))
- }
- }
-
- w=0;
-
- for (uint time=0;time<_solInt;++time) {
- for (uint ch=0;ch<_nChan;++ch) {
- for (uint st2=0;st2<_nSt;++st2) {
- uint zoff=st2+_nSt*ch+_nSt*_nChan*time;
- w(0) += conj(_z(zoff,0))*_z(zoff,0) + conj(_z(zoff,2))*_z(zoff,2);
- w(1) += conj(_z(zoff,0))*_z(zoff,1) + conj(_z(zoff,2))*_z(zoff,3);
- w(3) += conj(_z(zoff,1))*_z(zoff,1) + conj(_z(zoff,3))*_z(zoff,3);
- }
- }
- }
- w(2)=conj(w(1));
-
- t=0;
-
- for (uint time=0;time<_solInt;++time) {
- for (uint ch=0;ch<_nChan;++ch) {
- vis_p=&_vis(IPosition(6,0,0,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { t(0) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,0)) * vis_p[st2]; }// itsVis(IPosition(6,st2,0,time,ch,0,st1))
- vis_p=&_vis(IPosition(6,0,1,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { t(0) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,2)) * vis_p[st2]; }// itsVis(IPosition(6,st2,0,time,ch,1,st1))
- vis_p=&_vis(IPosition(6,0,0,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { t(1) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,0)) * vis_p[st2]; }// itsVis(IPosition(6,st2,1,time,ch,0,st1))
- vis_p=&_vis(IPosition(6,0,1,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { t(1) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,2)) * vis_p[st2]; }// itsVis(IPosition(6,st2,1,time,ch,1,st1))
- vis_p=&_vis(IPosition(6,0,0,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { t(2) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,1)) * vis_p[st2]; }// itsVis(IPosition(6,st2,0,time,ch,0,st1))
- vis_p=&_vis(IPosition(6,0,1,time,ch,0,st1)); for (uint st2=0;st2<_nSt;++st2) { t(2) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,3)) * vis_p[st2]; }// itsVis(IPosition(6,st2,0,time,ch,1,st1))
- vis_p=&_vis(IPosition(6,0,0,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { t(3) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,1)) * vis_p[st2]; }// itsVis(IPosition(6,st2,1,time,ch,0,st1))
- vis_p=&_vis(IPosition(6,0,1,time,ch,1,st1)); for (uint st2=0;st2<_nSt;++st2) { t(3) += conj(_z(st2+_nSt*ch+_nSt*_nChan*time,3)) * vis_p[st2]; }// itsVis(IPosition(6,st2,1,time,ch,1,st1))
- }
- }
- DComplex invdet= w(0) * w (3) - w(1)*w(2);
- if (abs(invdet)==0) {
- _stationFlagged[st1] = true;
- _g(st1,0) = 0;
- continue;
- }
- invdet= 1./invdet;
- _g(st1,0) = invdet * ( w(3) * t(0) - w(1) * t(2) );
- _g(st1,1) = invdet * ( w(3) * t(1) - w(1) * t(3) );
- _g(st1,2) = invdet * ( w(0) * t(2) - w(2) * t(0) );
- _g(st1,3) = invdet * ( w(0) * t(3) - w(2) * t(1) );
- }
- }
-
- void StefCal::doStep_unpolarized() {
- _gold=_g;
-
- for (uint ant=0;ant<_nUn;++ant) {
- _h(ant,0)=conj(_g(ant,0));
- }
-
- for (uint st1=0;st1<_nUn;++st1) {
- if (_stationFlagged[st1%_nSt]) {
- continue;
- }
- DComplex* vis_p;
- DComplex* mvis_p;
- double ww=0; // Same as w, but specifically for pol==false
- DComplex tt=0; // Same as t, but specifically for pol==false
-
- mvis_p=&_mvis(IPosition(6,0,0,0,0,st1/_nSt,st1%_nSt));
- vis_p = &_vis(IPosition(6,0,0,0,0,st1/_nSt,st1%_nSt));
- for (uint st1pol=0;st1pol<_nSp;++st1pol) {
- for (uint ch=0;ch<_nChan;++ch) {
- for (uint time=0;time<_solInt;++time) {
- for (uint st2pol=0;st2pol<_nSp;++st2pol) {
- DComplex* h_p=_h.data();
- for (uint st2=0;st2<_nUn;++st2) {
- DComplex z(h_p[st2] * *mvis_p); //itsMVis(IPosition(6,st2%nSt,st2/nSt,time,ch,st1/nSt,st1%nSt));
- ASSERT(isFinite(z));
- ww+=norm(z);
- tt+=conj(z) * *vis_p; //itsVis(IPosition(6,st2%nSt,st2/nSt,time,ch,st1/nSt,st1%nSt));
- mvis_p++;
- vis_p++;
- }
- }
- }
- }
- }
-
- // Flag a station if all baselines are flagged or all data is zero
- if (ww==0 || abs(tt)==0) {
- _stationFlagged[st1%_nSt]=true;
- _g(st1,0)=0;
- continue;
- }
- _g(st1,0)=tt/ww;
-
-
- // Constrain solutions
- if (_mode==PHASEONLY) {
- ASSERT(abs(_g(st1,0))!=0);
- _g(st1,0)/=abs(_g(st1,0));
- ASSERT(isFinite(_g(st1,0)));
- } else if (_mode==AMPLITUDEONLY) {
- _g(st1,0) = abs(_g(st1,0));
- }
- }
- }
-
- void StefCal::incrementWeight(float weight) {
- _totalWeight += weight;
- }
-
- casacore::Matrix<casacore::DComplex> StefCal::getSolution(bool setNaNs) {
- if (setNaNs) {
- for (uint ant=0; ant<_nUn; ++ant) {
- if (_stationFlagged[ant%_nSt]) {
- for (uint cr=0; cr<_nCr; ++cr) {
- _g(ant,cr)=std::numeric_limits<double>::quiet_NaN();
- }
- }
- }
- }
-
- return _g;
- }
-
- StefCal::Status StefCal::relax(uint iter) {
- if (_nSt==0) {
- return CONVERGED;
- }
-
- double f2 = -1.0;
- double f3 = -0.5;
- double f1 = 1 - f2 - f3;
- double f2q = -0.5;
- double f1q = 1 - f2q;
- double omega = 0.5;
- uint nomega = 24;
- double c1 = 0.5;
- double c2 = 1.2;
- double dgxx;
- bool threestep = false;
- uint maxBadIters=3;
-
- int sstep=0;
-
- if (_detectStalling && iter > 3) {
- double improvement = _dgx-_dg;
-
- if (abs(improvement) < 5.0e-2*_dg) {
- // This iteration did not improve much upon the previous
- // Stalling detection only after 4 iterations, to account for
- // ''startup problems'' (not for tec, where stalling happens very soon)
- if (_debugLevel>3) {
- cout<<"**"<<endl;
- }
- _badIters++;
- } else if (improvement < 0) {
- _veryBadIters++;
- } else {
- _badIters=0;
- }
-
- if (_badIters>=maxBadIters) {
- if (_debugLevel>3) {
- cout<<"Detected stall"<<endl;
- }
- return STALLED;
- } else if (_veryBadIters > maxBadIters) {
- if (_debugLevel>3) {
- cout<<"Detected fail"<<endl;
- }
- return STALLED;
- }
- }
-
- dgxx = _dgx;
- _dgx = _dg;
-
- double fronormdiff=0;
- double fronormg=0;
- for (uint ant=0;ant<_nUn;++ant) {
- for (uint cr=0;cr<_nCr;++cr) {
- DComplex diff=_g(ant,cr)-_gold(ant,cr);
- fronormdiff+=abs(diff*diff);
- fronormg+=abs(_g(ant,cr)*_g(ant,cr));
- }
- }
- fronormdiff=sqrt(fronormdiff);
- fronormg=sqrt(fronormg);
-
- _dg = fronormdiff/fronormg;
- if (_debugLevel>2) {
- _dgs.push_back(_dg);
- }
-
- if (_dg <= _tolerance) {
- return CONVERGED;
- }
-
- if (_debugLevel>7) {
- cout<<"Averaged"<<endl;
- }
-
- for (uint ant=0;ant<_nUn;++ant) {
- for (uint cr=0;cr<_nCr;++cr) {
- _g(ant,cr) = (1-omega) * _g(ant,cr) +
- omega * _gold(ant,cr);
- }
- }
-
- if (!threestep) {
- threestep = (iter+1 >= nomega) ||
- ( max(_dg,max(_dgx,dgxx)) <= 1.0e-3 && _dg<_dgx && _dgx<dgxx);
- if (_debugLevel>7) {
- cout<<"Threestep="<<boolalpha<<threestep<<endl;
- }
- }
-
- if (threestep) {
- if (sstep <= 0) {
- if (_dg <= c1 * _dgx) {
- if (_debugLevel>7) {
- cout<<"dg<=c1*dgx"<<endl;
- }
- for (uint ant=0;ant<_nUn;++ant) {
- for (uint cr=0;cr<_nCr;++cr) {
- _g(ant,cr) = f1q * _g(ant,cr) +
- f2q * _gx(ant,cr);
- }
- }
- } else if (_dg <= _dgx) {
- if (_debugLevel>7) {
- cout<<"dg<=dgx"<<endl;
- }
- for (uint ant=0;ant<_nUn;++ant) {
- for (uint cr=0;cr<_nCr;++cr) {
- _g(ant,cr) = f1 * _g(ant,cr) +
- f2 * _gx(ant,cr) +
- f3 * _gxx(ant,cr);
- }
- }
- } else if (_dg <= c2 *_dgx) {
- if (_debugLevel>7) {
- cout<<"dg<=c2*dgx"<<endl;
- }
- _g = _gx;
- sstep = 1;
- } else {
- //cout<<"else"<<endl;
- _g = _gxx;
- sstep = 2;
- }
- } else {
- if (_debugLevel>7) {
- cout<<"no sstep"<<endl;
- }
- sstep = sstep - 1;
- }
- }
- return NOTCONVERGED;
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/Stokes.cc b/CEP/DP3/DPPP/src/Stokes.cc
deleted file mode 100644
index ca3010550f1b3fa424170992f0a63bffff7f9998..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Stokes.cc
+++ /dev/null
@@ -1,40 +0,0 @@
-//# Stokes.cc: Complex Stokes vector.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/Stokes.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-Stokes::Stokes()
-: I(0.0),
- Q(0.0),
- U(0.0),
- V(0.0)
-{
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/SubtractMixed.cc b/CEP/DP3/DPPP/src/SubtractMixed.cc
deleted file mode 100644
index 3359f4d0723820beb60006c40a03b3a61f4db0d0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/SubtractMixed.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-//# SubtractMixed.cc: Subtract visibilities from a buffer after weighting by
-//# mixing coefficients.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/SubtractMixed.h>
-
-namespace LOFAR
-{
-namespace DPPP
-{
-
-void subtract(size_t nBaseline, size_t nChannel,
- const_cursor<Baseline> baselines, cursor<fcomplex> data,
- const_cursor<dcomplex> model, const_cursor<dcomplex> weight)
-{
- for(size_t bl = 0; bl < nBaseline; ++bl)
- {
- const size_t p = baselines->first;
- const size_t q = baselines->second;
-
- if(p != q)
- {
- for(size_t ch = 0; ch < nChannel; ++ch)
- {
- // Subtract weighted model from data.
- *data -= static_cast<fcomplex>((*weight) * (*model));
- ++weight;
- ++model;
- ++data;
- *data -= static_cast<fcomplex>((*weight) * (*model));
- ++weight;
- ++model;
- ++data;
- *data -= static_cast<fcomplex>((*weight) * (*model));
- ++weight;
- ++model;
- ++data;
- *data -= static_cast<fcomplex>((*weight) * (*model));
- ++weight;
- ++model;
- ++data;
-
- // Move to the next channel.
- weight -= 4;
- weight.forward(1);
- model -= 4;
- model.forward(1);
- data -= 4;
- data.forward(1);
- } // Channels.
-
- weight.backward(1, nChannel);
- model.backward(1, nChannel);
- data.backward(1, nChannel);
- }
-
- // Move to the next baseline.
- weight.forward(2);
- model.forward(2);
- data.forward(2);
- ++baselines;
- } // Baselines.
-}
-
-} //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/SubtractNew.cc b/CEP/DP3/DPPP/src/SubtractNew.cc
deleted file mode 100644
index 19e265d62c4e09ee0b37fc8a583e824a14ae0ad0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/SubtractNew.cc
+++ /dev/null
@@ -1,72 +0,0 @@
-//# SubtractNew.cc: Subtract visibilities from a buffer after weighting by
-//# mixing coefficients.
-//#
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-#include <DPPP/SubtractNew.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- void subtract (size_t nBaseline, size_t nChannel,
- const_cursor<Baseline> baselines, cursor<fcomplex> data,
- const_cursor<dcomplex> model, const_cursor<dcomplex> weight,
- vector<float>& ampl)
- {
- dcomplex vis[4];
- for (size_t bl=0; bl<nBaseline; ++bl) {
- // Only for cross correlations.
- if (baselines->first != baselines->second) {
- for (size_t ch=0; ch<nChannel; ++ch) {
- for (size_t cr=0; cr<4; ++cr) {
- vis[cr] = (*weight++) * (*model++);
- *data++ -= static_cast<fcomplex>(vis[cr]);
- }
- // Return subtracted amplitude for middle channel.
- if (ch == nChannel/2) {
- ampl[bl] = (abs(vis[0]) + abs(vis[3])) * 0.5;
- }
- // Move to the next channel.
- weight -= 4;
- weight.forward(1);
- model -= 4;
- model.forward(1);
- data -= 4;
- data.forward(1);
- } // Channels.
- weight.backward(1, nChannel);
- model.backward(1, nChannel);
- data.backward(1, nChannel);
- } else {
- ampl[bl] = 0.;
- }
-
- // Move to the next baseline.
- weight.forward(2);
- model.forward(2);
- data.forward(2);
- ++baselines;
- } // Baselines.
- }
-
- } //# namespace DPPP
-} //# namespace LOFAR
diff --git a/CEP/DP3/DPPP/src/UVWCalculator/UVWCalculator.cc b/CEP/DP3/DPPP/src/UVWCalculator/UVWCalculator.cc
deleted file mode 100644
index 05ad175e3cfe7c46f87e25c6b4ef107655df3242..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/UVWCalculator/UVWCalculator.cc
+++ /dev/null
@@ -1,120 +0,0 @@
-//# UVWCalculator.cc: Class to calculate UVW coordinates
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-// Note: this code is used by LOFAR and APERTIF software.
-
-#include <lofar_config.h>
-#include <DPPP/UVWCalculator.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/measures/Measures/Muvw.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- UVWCalculator::UVWCalculator()
- {}
-
- UVWCalculator::UVWCalculator (const MDirection& phaseDir,
- const MPosition& arrayPos,
- const vector<MPosition>& stationPositions)
- {
- // Convert the station positions to a baseline in ITRF.
- int nrant = stationPositions.size();
- Vector<Double> pos0;
- for (int i=0; i<nrant; ++i) {
- // Get antenna positions and convert to ITRF.
- MPosition mpos = MPosition::Convert (stationPositions[i],
- MPosition::ITRF)();
- if (i == 0) {
- pos0 = mpos.getValue().getVector();
- }
- Vector<Double> pos = mpos.getValue().getVector();
- MVPosition mvpos((pos[0] - pos0[0]),
- (pos[1] - pos0[1]),
- (pos[2] - pos0[2]));
- itsAntMB.push_back (MBaseline (MVBaseline(mvpos), MBaseline::ITRF));
- }
- // Initialize the converters.
- // Set up the frame for epoch and antenna position.
- itsFrame.set (arrayPos, MDirection());
- // Create converter for phase center direction to J2000.
- itsDirToJ2000.set (phaseDir,
- MDirection::Ref(MDirection::J2000,itsFrame));
- // We can already try to convert the phase reference direction to J2000.
- // If it fails, it is a time-dependent direction (like SUN).
- itsMovingPhaseDir = false;
- try {
- itsPhaseDir = itsDirToJ2000();
- itsFrame.resetDirection (itsPhaseDir);
- } catch (...) {
- itsMovingPhaseDir = true;
- }
- itsFrame.set (MEpoch());
- // Create converter for MBaseline ITRF to J2000.
- itsBLToJ2000.set (MBaseline(), MBaseline::Ref(MBaseline::J2000,itsFrame));
- // Initialize the rest which is used to cache the UVW per antenna.
- // The cache is only useful if the MS is accessed in time order, but that
- // is normally the case.
- itsLastTime = 0;
- itsAntUvw.resize (nrant);
- itsUvwFilled.resize (nrant);
- itsUvwFilled = false;
- }
-
- Vector<double> UVWCalculator::getUVW (uint ant1, uint ant2, double time)
- {
- // If a different time, we have to calculate the UVWs.
- if (time != itsLastTime) {
- itsLastTime = time;
- Quantum<Double> tm(time, "s");
- itsFrame.resetEpoch
- (MEpoch(MVEpoch(tm.get("d").getValue()), MEpoch::UTC));
- itsUvwFilled = false;
- // If phase dir is moving, calculate it for this time.
- if (itsMovingPhaseDir) {
- itsPhaseDir = itsDirToJ2000();
- itsFrame.resetDirection (itsPhaseDir);
- }
- }
- // Calculate the UVWs for this timestamp if not done yet.
- int ant = ant1;
- for (int i=0; i<2; ++i) {
- if (!itsUvwFilled[ant]) {
- MBaseline& mbl = itsAntMB[ant];
- mbl.getRefPtr()->set(itsFrame); // attach frame
- MBaseline::Convert mcvt(mbl, MBaseline::J2000);
- MVBaseline bas = mcvt().getValue();
- MVuvw jvguvw(bas, itsPhaseDir.getValue());
- itsAntUvw[ant] = Muvw(jvguvw, Muvw::J2000).getValue().getVector();
- itsUvwFilled[ant] = true;
- }
- ant = ant2;
- }
- // The UVW of the baseline is the difference of the antennae.
- return itsAntUvw[ant2] - itsAntUvw[ant1];
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/UVWCalculator/UVWCalculator.h b/CEP/DP3/DPPP/src/UVWCalculator/UVWCalculator.h
deleted file mode 100644
index 252f899de4e116aef24228ac3fee1e867a4023a0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/UVWCalculator/UVWCalculator.h
+++ /dev/null
@@ -1,86 +0,0 @@
-//# UVWCalculator.h: Class to calculate UVW coordinates
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-// Note: this code is used by LOFAR and APERTIF software.
-
-#ifndef DPPP_UVWCALCULATOR_H
-#define DPPP_UVWCALCULATOR_H
-
-// @file
-// @brief Class to calculate UVW coordinates
-
-#include <Common/LofarLogger.h>
-#include <Common/lofar_vector.h>
-#include <casacore/measures/Measures/MeasFrame.h>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MPosition.h>
-#include <casacore/measures/Measures/MBaseline.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/measures/Measures/MCPosition.h>
-#include <casacore/measures/Measures/MCBaseline.h>
-#include <casacore/casa/Arrays/Vector.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- // @ingroup NDPPP
-
- // This class calculates the UVW coordinates for a given baseline and
- // time stamp in the same way as done in LofarStMan.
- //
- // It calculates and caches the UVW coordinates per antenna and combines
- // them to get the baseline UVW coordinates. This is much faster than
- // calculating baseline UVW coordinates directly.
-
- class UVWCalculator
- {
- public:
- // The default constructor creates an empty object.
- UVWCalculator();
-
- // Construct the object for the given phase direction, array position,
- // and station positions.
- UVWCalculator (const casacore::MDirection& phaseDir,
- const casacore::MPosition& arrayPosition,
- const vector<casacore::MPosition>& stationPositions);
-
- // get the UVW coordinates for the given baseline and time.
- casacore::Vector<double> getUVW (uint ant1, uint ant2, double time);
-
- private:
- casacore::MDirection itsPhaseDir;
- bool itsMovingPhaseDir;
- casacore::MDirection::Convert itsDirToJ2000; //# direction to J2000
- casacore::MBaseline::Convert itsBLToJ2000; //# convert ITRF to J2000
- casacore::MeasFrame itsFrame;
- vector<casacore::MBaseline> itsAntMB;
- vector<casacore::Vector<double> > itsAntUvw;
- casacore::Block<bool> itsUvwFilled;
- double itsLastTime;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP/src/UVWFlagger.cc b/CEP/DP3/DPPP/src/UVWFlagger.cc
deleted file mode 100644
index b02efa355e3c93a3675b9173b887dda34617a412..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/UVWFlagger.cc
+++ /dev/null
@@ -1,354 +0,0 @@
-//# UVWFlagger.cc: DPPP step class to flag data on UVW coordinates
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/UVWFlagger.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StreamUtil.h>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/casa/Quanta/MVAngle.h>
-#include <casacore/casa/Utilities/GenSort.h>
-#include <iostream>
-#include <algorithm>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- UVWFlagger::UVWFlagger (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsNTimes (0),
- itsFlagCounter (input->msName(), parset, prefix+"count.")
- {
- itsRangeUVm = fillUVW (parset, prefix, "uvm", true);
- itsRangeUm = fillUVW (parset, prefix, "um", false);
- itsRangeVm = fillUVW (parset, prefix, "vm", false);
- itsRangeWm = fillUVW (parset, prefix, "wm", false);
- itsRangeUVl = fillUVW (parset, prefix, "uvlambda", false);
- itsRangeUl = fillUVW (parset, prefix, "ulambda", false);
- itsRangeVl = fillUVW (parset, prefix, "vlambda", false);
- itsRangeWl = fillUVW (parset, prefix, "wlambda", false);
- itsIsDegenerate = itsRangeUVm.size() + itsRangeUVl.size() +
- itsRangeUm.size() + itsRangeVm.size() +
- itsRangeWm.size() + itsRangeUl.size() +
- itsRangeVl.size() + itsRangeWl.size() == 0;
- itsCenter = parset.getStringVector (prefix+"phasecenter",
- vector<string>());
- }
-
- UVWFlagger::~UVWFlagger()
- {}
-
- void UVWFlagger::show (std::ostream& os) const
- {
- if (itsIsDegenerate) {
- return;
- }
-
- os << "UVWFlagger " << itsName << std::endl;
- vector<double> uvm(itsRangeUVm);
- for (uint i=0; i<uvm.size(); ++i) {
- if (uvm[i] > 0) {
- uvm[i] = sqrt(uvm[i]);
- }
- }
- os << " uvm: " << uvm << std::endl;
- os << " um: " << itsRangeUm << std::endl;
- os << " vm: " << itsRangeVm << std::endl;
- os << " wm: " << itsRangeWm << std::endl;
- os << " uvlambda: " << itsRangeUVl << std::endl;
- os << " ulambda: " << itsRangeUl << std::endl;
- os << " vlambda: " << itsRangeVl << std::endl;
- os << " wlambda: " << itsRangeWl << std::endl;
- os << " phasecenter: " << itsCenter << std::endl;
- }
-
- void UVWFlagger::showCounts (std::ostream& os) const
- {
- if (itsIsDegenerate) {
- return;
- }
- os << endl << "Flags set by UVWFlagger " << itsName;
- os << endl << "=======================" << endl;
- itsFlagCounter.showBaseline (os, itsNTimes);
- itsFlagCounter.showChannel (os, itsNTimes);
- }
-
- void UVWFlagger::showTimings (std::ostream& os, double duration) const
- {
- double flagDur = itsTimer.getElapsed();
- os << " ";
- FlagCounter::showPerc1 (os, flagDur, duration);
- os << " UVWFlagger " << itsName << endl;
- if (! itsCenter.empty()) {
- os << " ";
- FlagCounter::showPerc1 (os, itsUVWTimer.getElapsed(), flagDur);
- os << " of it spent in calculating UVW coordinates" << endl;
- }
- }
-
- void UVWFlagger::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setWriteFlags();
- // Convert the given frequencies to possibly averaged frequencies.
- // Divide it by speed of light to get reciproke of wavelengths.
- itsRecWavel = infoIn.chanFreqs() / casacore::C::c;
- // Handle the phase center (if given).
- if (! itsCenter.empty()) {
- handleCenter();
- }
- // Initialize the flag counters.
- itsFlagCounter.init (getInfo());
- }
-
- bool UVWFlagger::process (const DPBuffer& buf)
- {
- if (itsIsDegenerate) {
- getNextStep()->process (buf);
- return true;
- }
-
- itsTimer.start();
- // Because no buffers are kept, we can reference the filled arrays
- // in the input buffer instead of copying them.
- itsBuffer.referenceFilled (buf);
- Cube<bool>& flags = itsBuffer.getFlags();
- // Loop over the baselines and flag as needed.
- const IPosition& shape = flags.shape();
- uint nrcorr = shape[0];
- uint nrchan = shape[1];
- uint nrbl = shape[2];
- uint nr = nrcorr*nrchan;
- ASSERT (nrchan == itsRecWavel.size());
- // Input uvw coordinates are only needed if no new phase center is used.
- Matrix<double> uvws;
- if (itsCenter.empty()) {
- uvws.reference (itsInput->fetchUVW (buf, itsBuffer, itsTimer));
- }
- const double* uvwPtr = uvws.data();
- bool* flagPtr = flags.data();
- const bool* origPtr = buf.getFlags().data();
- for (uint i=0; i<nrbl; ++i) {
- if (! itsCenter.empty()) {
- // A different phase center is given, so calculate UVW for it.
- NSTimer::StartStop ssuvwtimer(itsUVWTimer);
- Vector<double> uvw = itsUVWCalc.getUVW (getInfo().getAnt1()[i],
- getInfo().getAnt2()[i],
- buf.getTime());
- uvwPtr = uvw.data();
- ///cout << "uvw = " << uvw << endl;
- }
- double uvdist = uvwPtr[0] * uvwPtr[0] + uvwPtr[1] * uvwPtr[1];
- bool flagBL = false;
- if (! itsRangeUVm.empty()) {
- // UV-distance is sqrt(u^2 + v^2).
- // The sqrt is not needed because itsRangeUVm is squared.
- flagBL = testUVWm (uvdist, itsRangeUVm);
- }
- if (!(flagBL || itsRangeUm.empty())) {
- flagBL = testUVWm (uvwPtr[0], itsRangeUm);
- }
- if (!(flagBL || itsRangeVm.empty())) {
- flagBL = testUVWm (uvwPtr[1], itsRangeVm);
- }
- if (!(flagBL || itsRangeWm.empty())) {
- flagBL = testUVWm (uvwPtr[2], itsRangeWm);
- }
- if (flagBL) {
- // Flag entire baseline.
- std::fill (flagPtr, flagPtr+nr, true);
- } else {
- if (! itsRangeUVl.empty()) {
- // UV-distance is sqrt(u^2 + v^2).
- testUVWl (sqrt(uvdist), itsRangeUVl, flagPtr, nrcorr);
- }
- if (! itsRangeUl.empty()) {
- testUVWl (uvwPtr[0], itsRangeUl, flagPtr, nrcorr);
- }
- if (! itsRangeVl.empty()) {
- testUVWl (uvwPtr[1], itsRangeVl, flagPtr, nrcorr);
- }
- if (! itsRangeWl.empty()) {
- testUVWl (uvwPtr[2], itsRangeWl, flagPtr, nrcorr);
- }
- }
- // Count the flags set newly.
- for (uint j=0; j<nrchan; ++j) {
- if (*flagPtr && !*origPtr) {
- itsFlagCounter.incrBaseline(i);
- itsFlagCounter.incrChannel(j);
- }
- flagPtr += nrcorr;
- origPtr += nrcorr;
- }
- uvwPtr += 3;
- }
- // Let the next step do its processing.
- itsTimer.stop();
- itsNTimes++;
- getNextStep()->process (itsBuffer);
- return true;
- }
-
- void UVWFlagger::finish()
- {
- // Let the next step finish its processing.
- getNextStep()->finish();
- }
-
- bool UVWFlagger::testUVWm (double uvw, const vector<double>& ranges)
- {
- for (size_t i=0; i<ranges.size(); i+=2) {
- if (uvw > ranges[i] && uvw < ranges[i+1]) {
- return true;
- }
- }
- return false;
- }
-
- void UVWFlagger::testUVWl (double uvw, const vector<double>& ranges,
- bool* flagPtr, uint nrcorr)
- {
- // This loop could be made more efficient if it is guaranteed that
- // itsRecWavel is in strict ascending or descending order.
- // It is expected that the nr of ranges is so small that it is not
- // worth the trouble, but it could be done if ever needed.
- for (uint j=0; j<itsRecWavel.size(); ++j) {
- double uvwl = uvw * itsRecWavel[j];
- for (size_t i=0; i<ranges.size(); i+=2) {
- if (uvwl > ranges[i] && uvwl < ranges[i+1]) {
- std::fill (flagPtr, flagPtr+nrcorr, true);
- break;
- }
- }
- flagPtr += nrcorr;
- }
- }
-
- vector<double> UVWFlagger::fillUVW (const ParameterSet& parset,
- const string& prefix,
- const string& name,
- bool square)
- {
- // Get possible range, minimum, and maximum.
- vector<string> uvs = parset.getStringVector (prefix + name + "range",
- vector<string>());
- double minuv = parset.getDouble (prefix + name + "min", 0.);
- double maxuv = parset.getDouble (prefix + name + "max", 0.);
- // Process the ranges.
- vector<double> vals;
- vals.reserve (2*uvs.size());
- for (vector<string>::const_iterator str = uvs.begin();
- str != uvs.end(); ++str) {
- // Each range can be given as st..end or val+-halfwidth.
- // Find the .. or +- token.
- bool usepm = false;
- string::size_type pos;
- pos = str->find ("..");
- if (pos == string::npos) {
- usepm = true;
- pos = str->find ("+-");
- ASSERTSTR (pos != string::npos, "UVWFlagger " << name << "range '"
- << *str << "' should be range using .. or +-");
- }
- string str1 = str->substr (0, pos);
- string str2 = str->substr (pos+2);
- double v1 = strToDouble(str1);
- double v2 = strToDouble(str2);
- if (usepm) {
- double hw = v2;
- v2 = v1 + hw;
- v1 -= hw;
- }
- vals.push_back (v1);
- vals.push_back (v2);
- }
- // If minimum or maximum is given, add them as a range as well.
- if (minuv > 0) {
- vals.push_back (-1e15);
- vals.push_back (minuv);
- }
- if (maxuv > 0) {
- vals.push_back (maxuv);
- vals.push_back (1e15);
- }
- if (square) {
- for (vector<double>::iterator iter = vals.begin();
- iter != vals.end(); ++iter) {
- if (*iter != -1e15) {
- *iter = *iter * *iter;
- }
- }
- }
- return vals;
- }
-
- void UVWFlagger::handleCenter()
- {
- // The phase center can be given as one, two, or three values.
- // I.e., as source name, ra,dec or ra,dec,frame.
- ASSERTSTR (itsCenter.size() < 4,
- "Up to 3 values can be given in UVWFlagger phasecenter");
- MDirection phaseCenter;
- if (itsCenter.size() == 1) {
- string str = toUpper(itsCenter[0]);
- MDirection::Types tp;
- ASSERTSTR (MDirection::getType(tp, str),
- str << " is an invalid source type"
- " in UVWFlagger phasecenter");
- phaseCenter = MDirection(tp);
- } else {
- Quantity q0, q1;
- ASSERTSTR (MVAngle::read (q0, itsCenter[0]),
- itsCenter[0] << " is an invalid RA or longitude"
- " in UVWFlagger phasecenter");
- ASSERTSTR (MVAngle::read (q1, itsCenter[1]),
- itsCenter[1] << " is an invalid DEC or latitude"
- " in UVWFlagger phasecenter");
- MDirection::Types type = MDirection::J2000;
- if (itsCenter.size() > 2) {
- string str = toUpper(itsCenter[2]);
- MDirection::Types tp;
- ASSERTSTR (MDirection::getType(tp, str),
- str << " is an invalid direction type in UVWFlagger"
- " in UVWFlagger phasecenter");
- }
- phaseCenter = MDirection(q0, q1, type);
- }
- // Create the UVW calculator.
- itsUVWCalc = UVWCalculator (phaseCenter, getInfo().arrayPos(),
- getInfo().antennaPos());
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/Upsample.cc b/CEP/DP3/DPPP/src/Upsample.cc
deleted file mode 100644
index b15280b32baf5f90ab657f9294be675622e690d7..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/Upsample.cc
+++ /dev/null
@@ -1,152 +0,0 @@
-//# Upsample.cc: DPPP step class to Upsample visibilities
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: GainCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/Upsample.h>
-
-#include <iostream>
-#include <Common/ParameterSet.h>
-
-#include <casacore/casa/BasicMath/Math.h> // nearAbs
-#include <casacore/casa/Arrays/ArrayLogical.h> // anyTrue
-
-#include <iomanip>
-#include <stddef.h>
-#include <string>
-#include <sstream>
-#include <utility>
-#include <vector>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- Upsample::Upsample (DPInput*,
- const ParameterSet& parset,
- const string& prefix)
- : itsOldTimeInterval(0),
- itsTimeStep(parset.getInt(prefix + "timestep")),
- itsFirstToFlush(0)
- {
- itsBuffers.resize(itsTimeStep);
- }
-
- Upsample::~Upsample()
- {}
-
- void Upsample::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
-
- itsOldTimeInterval = info().timeInterval();
- info().setTimeInterval(itsOldTimeInterval / itsTimeStep);
-
- info().setMetaChanged();
- }
-
- void Upsample::show (std::ostream& os) const
- {
- os << "Upsample " << itsName << endl;
- os << " time step : " << itsTimeStep <<endl;
- }
-
- bool Upsample::process (const DPBuffer& bufin)
- {
-
- double time0 = bufin.getTime() - 0.5 * itsOldTimeInterval;
- double exposure = bufin.getExposure() / itsTimeStep;
-
- // Duplicate the input buffer itsTimeStep times
- for (uint i=0; i<itsTimeStep; ++i) {
- itsBuffers[i].copy (bufin);
- // Update the time centroid and time exposure
- itsBuffers[i].setTime(time0 + info().timeInterval() * (i+0.5));
- itsBuffers[i].setExposure(exposure);
- }
-
- if (itsPrevBuffers.empty()) {
- // First time slot, ask for next time slot first
- itsPrevBuffers.resize(itsTimeStep);
- for (uint i=0; i<itsTimeStep; ++i) {
- itsPrevBuffers[i].copy(itsBuffers[i]); // No shallow copy
- }
- return false;
- }
-
- // Flush the itsPrevBuffers. Skip parts at the beginning as determined
- // in the previous call to process. Skip parts at the end as determined
- // now. Also, determine at which step the next buffer should start to
- // flush in the next call of process.
- uint curIndex = 0; // Index in the current buffers
- for (uint prevIndex=itsFirstToFlush; prevIndex<itsTimeStep; prevIndex++) {
- itsFirstToFlush = 0; // reset for next use
- // Advance curIndex until
- // buffers[curIndex].time >= prevBuffers[prevIndex].time
- while (itsPrevBuffers[prevIndex].getTime() > itsBuffers[curIndex].getTime() &&
- !nearAbs(itsPrevBuffers[prevIndex].getTime(),
- itsBuffers[curIndex].getTime(),
- 0.4*info().timeInterval()) ) {
- curIndex++;
- }
- if (nearAbs(itsPrevBuffers[prevIndex].getTime(),
- itsBuffers[curIndex].getTime(),
- 0.4*info().timeInterval())) {
- // Found double buffer, choose which one to use
- // If both totally flagged, prefer prevbuffer
- if (allTrue(itsBuffers[curIndex].getFlags())) {
- // Use prevBuffer
- itsFirstToFlush = curIndex+1;
- getNextStep()->process(itsPrevBuffers[prevIndex]);
- } else {
- // Use next buffer; stop processing prevbuffer.
- // This will uncorrectly give flagged if data has been flagged
- // and a time slot has been inserted and itsTimeStep > 2.
- break;
- }
- } else {
- // No double buffer, just flush the prevbuffer
- getNextStep()->process(itsPrevBuffers[prevIndex]);
- }
- }
-
- itsPrevBuffers.swap(itsBuffers); // itsBuffers will be overwritten later
-
- return false;
- }
-
-
- void Upsample::finish()
- {
- // Flush itsPrevBuffers
- for (uint i=itsFirstToFlush; i<itsTimeStep; ++i) {
- getNextStep()->process(itsPrevBuffers[i]);
- }
-
- // Let the next steps finish.
- getNextStep()->finish();
- }
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP/src/__init__.py b/CEP/DP3/DPPP/src/__init__.py
deleted file mode 100644
index 5adf25334be561f849232729497a031a8b93caa9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/__init__.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# __init__.py: Top level .py file for DPPP flagging results plotting
-#
-# Copyright (C) 2007
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: __init__.py 17341 2011-02-07 07:15:52Z diepen $
-
-
-# Plot the NDPPP count results by frequency or baseline.
-# Frequencies are concatenated; baselines are averaged.
-
-""" Module to plot NDPPP count results. """
-
-import pyrap.tables as pt
-import numpy
-import pylab
-
-def plotflags (tabnames):
- """Plot NDPPP Count results
-
- A flagging or count step in NDPPP can save the flagging percentages per
- frequency or station. They are saved in a table (per subband) with the
- extension ''.flagfreq'' or ''.flagstat''.
- The flag percentages of a subband can be plotted by giving the name of
- the table containing the results.
-
- It is also possible to plot the results of multiple subbands by giving
- a list of table names. Frequency results will be sorted in order of
- frequency, while station results are averaged over the subbands.
-
- """
- t = pt.table(tabnames)
- if 'Frequency' in t.colnames():
- t1 = t.sort ('Frequency')
- pylab.plot (t1.getcol('Frequency'), t1.getcol('Percentage'))
- elif 'Station' in t.colnames():
- percs = []
- names = []
- for t1 in t.iter ('Station'):
- percs.append (t1.getcol('Percentage').mean())
- names.append (t1.getcell('Name', 0))
- pylab.plot (numpy.array(percs), '+')
- else:
- raise RuntimeError('Table appears not to be a NDPPP Count result; it does not contain a Frequency or Station column')
-
diff --git a/CEP/DP3/DPPP/src/taqlflagger b/CEP/DP3/DPPP/src/taqlflagger
deleted file mode 100755
index 6a4c4e8f0a9a44ac6dbbbba17c72889c2bab866b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/src/taqlflagger
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/sh
-
-# Skip first argument if it is a rank (a numeric value).
-res=`echo $1 | sed 's/[0-9]*//'`
-test "$res" = "" && shift
-
-# Test if an MS name is given.
-err=1
-ms=$1
-if test "$ms" != ""; then
- shift
- err=0
-
- # Test if -dry or dry is given.
- dry=0
- if test "$1" = "-dry" -o "$1" = "dry"; then
- dry=1
- fi
- # Test if flag or unflag is given. flag means use value T, otherwise F.
- val=T
- if test "$1" = "flag"; then
- shift
- elif test "$1" = "unflag"; then
- val=F
- shift
- fi
-
- pol=
- chan=
- where=
- while test $# != 0 -a $err = 0
- do
- sel=$1
- # Check if a pol or chan selection is given.
- case $sel in
- chan=*)
- test "$chan" = "" || err=1
- chan=`echo $sel | sed -e 's/chan=//'`
- cmd="update $ms set FLAG[$chan,]=T"
- ;;
- pol=*)
- test "$pol" = "" || err=1
- pol=`echo $sel | sed -e 's/pol=//'`
- cmd="update $ms set FLAG[,$pol]=T"
- ;;
- *)
- if test "$where" = ""; then
- where="$sel"
- else
- where="($where) && ($sel)"
- fi
- ;;
- esac
- shift
- done
-fi
-
-if test $err = 0; then
- cmd="update $ms set FLAG[$chan,$pol]=$val"
- if test "$where" != ""; then
- cmd="$cmd where $where"
- fi
- echo "taql '$cmd'"
- if test $dry = 0; then
- taql "$cmd" || err=1
- fi
-fi
-
-if test $err != 0; then
- echo ""
- echo "Run as: taqlflagger [rank] ms [-dry] [flag|unflag] [selection1 ...]"
- echo "The rank is a dummy argument meant for rundist."
- echo "ms is the name of the MS to be (un)flagged."
- echo "dry tells to do a dry run; it only shows the command to execute."
- echo "flag or unflag tells what to do (default is flag)."
- echo "The selections must be a TaQL WHERE part like"
- echo " ANTENNA1=1"
- echo " where multiple such parts are anded"
- echo "or a polarization or channel selection like"
- echo " pol=0 or chan=0:4"
- echo " where the end is exclusive (a la python)"
- echo "E.g."
- echo " taqlflagger unflag ~/my.ms chan=0:32 pol=0 ANTENNA1=1 'ANTENNA2 in [1:4]'"
- echo "unflags XX for channel 0 till 31 for baseline 1,1, 1,2 and 1,3"
- echo ""
- exit 1
-fi
-exit 0
diff --git a/CEP/DP3/DPPP/test/CMakeLists.txt b/CEP/DP3/DPPP/test/CMakeLists.txt
deleted file mode 100644
index 3cb53f56cacd01291d3a9d22b31a5a40795f8057..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/CMakeLists.txt
+++ /dev/null
@@ -1,40 +0,0 @@
-# $Id$
-
-# Generate the actual test script.
-configure_file(findenv.run_tmpl findenv.run_script)
-configure_file(tDemix.run_tmpl tDemix.run_script)
-
-include(LofarCTest)
-
-lofar_add_test(tMirror tMirror.cc)
-lofar_add_test(tMedian tMedian.cc)
-lofar_add_test(tAverager tAverager.cc)
-lofar_add_test(tMedFlagger tMedFlagger.cc)
-lofar_add_test(tPreFlagger tPreFlagger.cc)
-lofar_add_test(tPSet tPSet.cc)
-lofar_add_test(tUVWFlagger tUVWFlagger.cc)
-lofar_add_test(tPhaseShift tPhaseShift.cc)
-lofar_add_test(tStationAdder tStationAdder.cc)
-lofar_add_test(tScaleData tScaleData.cc)
-lofar_add_test(tApplyCal tApplyCal.cc)
-lofar_add_test(tApplyCalH5 tApplyCalH5.cc)
-lofar_add_test(tApplyCal2)
-lofar_add_test(tMultiApplyCal)
-lofar_add_test(tFilter tFilter.cc)
-#lofar_add_test(tDemixer tDemixer.cc)
-lofar_add_test(tNDPPP tNDPPP.cc)
-lofar_add_test(tparse tparse.cc)
-lofar_add_test(tBaselineSelection tBaselineSelection.cc)
-lofar_add_test(tDemix)
-lofar_add_test(tPredict)
-lofar_add_test(tApplyBeam)
-lofar_add_test(tGainCal)
-lofar_add_test(tH5Parm tH5Parm)
-lofar_add_test(tGainCalH5Parm)
-lofar_add_test(tUpsample tUpsample.cc)
-if(CMAKE_CXX_FLAGS MATCHES ".*\\+\\+11.*")
- lofar_add_test(tGridInterpolate tGridInterpolate.cc)
-endif()
-# lofar_add_test(tExpr tExpr.cc)
-# lofar_add_test(tmeqarray tmeqarray.cc)
-# lofar_add_test(test_flaggers test_flaggers.cc)
diff --git a/CEP/DP3/DPPP/test/findenv.run_tmpl b/CEP/DP3/DPPP/test/findenv.run_tmpl
deleted file mode 100644
index ca6b5eaf16fdc7f764abd8c53447fa495523cdb9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/findenv.run_tmpl
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# Find taql program.
-taqlexe=@TAQL_EXECUTABLE@
-# Find srcdir in the runctest file.
-if test -f runctest.sh; then
- rt_srcdir=`grep 'srcdir=' runctest.sh | sed -e 's/srcdir="//' -e 's/";.*//'`
-fi
diff --git a/CEP/DP3/DPPP/test/tApplyBeam.run b/CEP/DP3/DPPP/test/tApplyBeam.run
deleted file mode 100755
index 23c96214f0edebf8163fad21a1ffebf17c09e11f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyBeam.run
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/bash
-
-# Get the taql executable and srcdir (script created by cmake's CONFIGURE_FILE).
-source findenv.run_script
-echo "srcdirx=$rt_srcdir"
-
-# Set srcdir if not defined (in case run by hand).
-if test "$srcdir" = ""; then
- srcdir="$rt_srcdir"
-fi
-
-if test ! -f ${srcdir}/tNDPPP-generic.in_MS.tgz; then
- exit 3 # untested
-fi
-
-rm -rf tApplyBeam_tmp
-mkdir -p tApplyBeam_tmp
-# Unpack the MS and other files and do the DPPP run.
-cd tApplyBeam_tmp
-tar zxf ${srcdir}/tNDPPP-generic.in_MS.tgz
-tar zxf ${srcdir}/tApplyBeam.tab.tgz
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-
-echo; echo "Test with invert=true and usechannelfreq=false"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=outinv.ms steps=[applybeam] applybeam.usechannelfreq=false applybeam.invert=true'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the BBS reference output.
-$taqlexe 'select from outinv.ms t1, tApplyBeam.tab t2 where not all(near(t1.DATA,t2.DATA_noucf,5e-5) || (isnan(t1.DATA) && isnan(t2.DATA_noucf)))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "### Test with invert=false on the output of the previous step"; echo
-cmd='NDPPP msin=outinv.ms msout=out.ms steps=[applybeam] applybeam.usechannelfreq=false applybeam.invert=false'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the original MS.
-$taqlexe 'select from out.ms t1, tNDPPP-generic.MS t2 where not all(near(t1.DATA,t2.DATA,5e-5) || (isnan(t1.DATA) && isnan(t2.DATA)))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test with invert=true and usechannelfreq=true"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=outinv.ms msout.overwrite=true steps=[applybeam] applybeam.usechannelfreq=true applybeam.invert=true'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the BBS reference output.
-$taqlexe 'select from outinv.ms t1, tApplyBeam.tab t2 where not all(near(t1.DATA,t2.DATA_ucf,5e-5) || (isnan(t1.DATA) && isnan(t2.DATA_ucf)))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test with invert=false on the output of the previous step"; echo
-cmd='NDPPP msin=outinv.ms msout=out.ms msout.overwrite=true steps=[applybeam] applybeam.usechannelfreq=true applybeam.invert=false'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the original MS.
-$taqlexe 'select from out.ms t1, tNDPPP-generic.MS t2 where not all(near(t1.DATA,t2.DATA,5e-5) || (isnan(t1.DATA) && isnan(t2.DATA)))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test with beammode=ARRAY_FACTOR"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=outinv.ms msout.overwrite=true steps=[applybeam] applybeam.usechannelfreq=true applybeam.invert=true applybeam.beammode=ARRAY_FACTOR'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the BBS reference output.
-$taqlexe 'select from outinv.ms t1, tApplyBeam.tab t2 where not all(near(t1.DATA,t2.DATA_ARRAY_FACTOR,5e-5) || (isnan(t1.DATA) && isnan(t2.DATA_ARRAY_FACTOR)))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test with beammode=ELEMENT"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=outinv.ms msout.overwrite=true steps=[applybeam] applybeam.usechannelfreq=true applybeam.invert=true applybeam.beammode=ELEMENT'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the BBS reference output.
-$taqlexe 'select from outinv.ms t1, tApplyBeam.tab t2 where not all(near(t1.DATA,t2.DATA_ELEMENT,5e-5) || (isnan(t1.DATA) && isnan(t2.DATA_ELEMENT)))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test with updateweights=true"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. steps=[applybeam] applybeam.updateweights=truue msout.weightcolumn=NEW_WEIGHT_SPECTRUM'
-echo $cmd
-$cmd
-# Check that the weights have changed
-$taqlexe 'select from tNDPPP-generic.MS where all(near(WEIGHT_SPECTRUM, NEW_WEIGHT_SPECTRUM))' > taql.out
-diff taql.out taql.ref || exit 1
diff --git a/CEP/DP3/DPPP/test/tApplyBeam.sh b/CEP/DP3/DPPP/test/tApplyBeam.sh
deleted file mode 100755
index aee9fa411608edd260c6fa692f990902527a6c75..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyBeam.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tApplyBeam
diff --git a/CEP/DP3/DPPP/test/tApplyBeam.tab.tgz b/CEP/DP3/DPPP/test/tApplyBeam.tab.tgz
deleted file mode 100644
index a5c6d870b79a183f9953ec1356cc1c2ac7d8a25d..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP/test/tApplyBeam.tab.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP/test/tApplyBeam_ref b/CEP/DP3/DPPP/test/tApplyBeam_ref
deleted file mode 100755
index 967693ea9958672fbb0ee85b4b6fe4d7bfd2ffd7..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyBeam_ref
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/bash
-
-# Generate reference data tApplyBeam.tab for tApplyBeam
-# This script uses BBS. Note that casacore data should be up-to-date
-# Run the script in the source directory of CEP/DP3/DPPP/test
-
-tar xf tNDPPP-generic.in_MS.tgz
-
-rm -rf tApplyBeam.tab.tgz tApplyBeam.tab
-
-#### Run BBS, correct data for beam with UseChannelFreq
-
-cat > tApplyBeam-bbs-ucf.parset <<EOF
-Strategy.ChunkSize=100
-Strategy.Steps=[correct_ucf]
-
-Step.correct_ucf.Operation=CORRECT
-Step.correct_ucf.Model.Beam.Enable=True
-Step.correct_ucf.Model.Beam.UseChannelFreq=True
-Step.correct_ucf.Output.Column=DATA_ucf
-EOF
-
-calibrate-stand-alone -n -f tNDPPP-generic.MS tApplyBeam-bbs-ucf.parset tApplyBeam-bbs-ucf.parset
-
-rm tApplyBeam-bbs-ucf.parset
-
-#### Run BBS, correct data for beam with UseChannelFreq
-
-cat > tApplyBeam-bbs-noucf.parset <<EOF
-Strategy.ChunkSize=100
-Strategy.Steps=[correct_ucf]
-
-Step.correct_ucf.Operation=CORRECT
-Step.correct_ucf.Model.Beam.Enable=True
-Step.correct_ucf.Model.Beam.UseChannelFreq=False
-Step.correct_ucf.Output.Column=DATA_noucf
-EOF
-
-calibrate-stand-alone -n -f tNDPPP-generic.MS tApplyBeam-bbs-noucf.parset tApplyBeam-bbs-noucf.parset
-
-rm tApplyBeam-bbs-noucf.parset
-
-#### Run BBS, correct data for beam with UseChannelFreq, ARRAY_FACTOR
-
-cat > tApplyBeam-bbs-arrayfactor.parset <<EOF
-Strategy.ChunkSize=100
-Strategy.Steps=[correct_ucf]
-
-Step.correct_ucf.Operation=CORRECT
-Step.correct_ucf.Model.Beam.Enable=True
-Step.correct_ucf.Model.Beam.Mode=ARRAY_FACTOR
-Step.correct_ucf.Model.Beam.UseChannelFreq=True
-Step.correct_ucf.Output.Column=DATA_ARRAY_FACTOR
-EOF
-
-calibrate-stand-alone -n -f tNDPPP-generic.MS tApplyBeam-bbs-arrayfactor.parset tApplyBeam-bbs-arrayfactor.parset
-
-rm tApplyBeam-bbs-arrayfactor.parset
-
-#### Run BBS, correct data for beam with UseChannelFreq, ELEMENT beam only
-
-cat > tApplyBeam-bbs-element.parset <<EOF
-Strategy.ChunkSize=100
-Strategy.Steps=[correct_ucf]
-
-Step.correct_ucf.Operation=CORRECT
-Step.correct_ucf.Model.Beam.Enable=True
-Step.correct_ucf.Model.Beam.Mode=ELEMENT
-Step.correct_ucf.Model.Beam.UseChannelFreq=True
-Step.correct_ucf.Output.Column=DATA_ELEMENT
-EOF
-
-calibrate-stand-alone -n -f tNDPPP-generic.MS tApplyBeam-bbs-element.parset tApplyBeam-bbs-element.parset
-
-rm tApplyBeam-bbs-element.parset
-
-#### Store output from BBS in separate table
-
-taql 'select from (select DATA_ucf, DATA_noucf, DATA_ARRAY_FACTOR, DATA_ELEMENT from tNDPPP-generic.MS giving tApplyBeam.tab as plain)'
-
-tar czf tApplyBeam.tab.tgz tApplyBeam.tab
diff --git a/CEP/DP3/DPPP/test/tApplyCal.cc b/CEP/DP3/DPPP/test/tApplyCal.cc
deleted file mode 100644
index 36ee1c1f08d4fc6b588f5a1344579af6d04cd719..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCal.cc
+++ /dev/null
@@ -1,350 +0,0 @@
-//# tApplyCal.cc: Test program for class AORFlagger
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tApplyCal.cc 24221 2013-08-02 12:24:48Z tammo $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/ApplyCal.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <Common/StreamUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// 9 baselines, 3 antennas, 4 correlations
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nchan)
- : itsCount(0), itsNTime(ntime), itsNChan(nchan), itsNBl(9), itsNCorr(4),
- itsTimeInterval(5.)
- {
- info().init (itsNCorr, nchan, ntime, 4472025740.0, itsTimeInterval,
- string(), string());
- // Fill the baseline stations; use 3 stations.
- // So they are called 00 01 02 10 11 12 20 21 22, etc.
-
- Vector<Int> ant1(itsNBl);
- Vector<Int> ant2(itsNBl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<itsNBl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 3) {
- st2 = 0;
- if (++st1 == 3) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "ant1";
- antNames[1] = "ant2";
- antNames[2] = "ant3";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanWidth(nchan, 1000000.);
- Vector<double> chanFreqs(nchan);
- indgen (chanFreqs, 10500000., 1000000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(1,0);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- weights=1.;
-
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- DPBuffer buf;
- buf.setTime (itsCount*itsTimeInterval + 4472025740.0);
- buf.setData (data);
- buf.setWeights (weights);
- buf.setUVW (uvw);
- Cube<bool> flags(data.shape());
- flags = false;
- buf.setFlags (flags);
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = false;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&) {}
-
- int itsCount, itsNTime, itsNChan, itsNBl, itsNCorr, itsTimeInterval;
-};
-
-
-
-// Class to check result of TestInput run by tests.
-class TestOutput: public DPStep
-{
-public:
- enum tests {WeightsNotChanged=1, DataNotChanged=2, DataChanged=4,
- DataEquals=8, WeightEquals=16};
- TestOutput(int ntime, int nchan, int doTest)
- : itsCount(0), itsTimeStep(0), itsNTime(ntime), itsNBl(9), itsNChan(nchan),
- itsNCorr(4), itsTimeInterval(5.), itsDoTest(doTest)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // Fill data and scale as needed.
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
-
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(1,0);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- indgen (weights, 1.0f, 0.0f);
-
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
-
- // The same gain corrections as in tApplyCal_tmp.parmdb
- vector<Cube<Complex> > gains(4); // cube for every corr
- for (int corr=0; corr<4; ++corr) {
- gains[corr].resize(IPosition(3,2,2,3)); //freq,time,ant;
- }
-
- gains[0](Slice(0,2),Slice(0,2),Slice(0,3))=1;
- gains[1](Slice(0,2),Slice(0,2),Slice(0,3))=0;
- gains[2](Slice(0,2),Slice(0,2),Slice(0,3))=0;
- gains[3](Slice(0,2),Slice(0,2),Slice(0,3))=1;
- // ant2
- gains[0](0,0,1)=2;
- gains[3](0,0,1)=3;
- gains[0](1,1,1)=Complex(3.,4.);
- // ant3
- gains[2](1,0,2)=.5;
-
- if (itsDoTest & DataEquals) {
- //cout<<"weights="<<buf.getWeights()<<endl;
- for (int bl=0; bl<itsNBl; ++bl) {
- for (int chan=0; chan<itsNChan; ++chan) {
- for (int corr=0; corr<itsNCorr; ++corr) {
- data(corr,chan,bl) /=
- (gains[corr/2*3](chan/(itsNChan/2),
- itsTimeStep/(itsNTime/2), info().getAnt1()[bl]) *
- conj(gains[corr%2*3](chan/(itsNChan/2),
- itsTimeStep/(itsNTime/2), info().getAnt2()[bl])));
- }
- if (info().getAnt2()[bl]==2 && (itsTimeStep/(itsNTime/2))==0
- && (chan/(itsNChan/2))==1) {
- data(0,chan,bl)-=0.5*data(1,chan,bl);
- data(2,chan,bl)-=0.5*data(3,chan,bl);
- }
- if (info().getAnt1()[bl]==2 && (itsTimeStep/(itsNTime/2))==0
- && (chan/(itsNChan/2))==1) {
- data(0,chan,bl)-=0.5*data(2,chan,bl);
- data(1,chan,bl)-=0.5*data(3,chan,bl);
- }
- }
- }
-
- }
-
- if (itsDoTest & WeightEquals) {
- ASSERT ( near(buf.getWeights()(0,0,1),4.));
- ASSERT ( near(buf.getWeights()(1,0,1),9.));
- ASSERT ( near(buf.getWeights()(2,0,1),4.));
- ASSERT ( near(buf.getWeights()(3,0,1),9.));
- ASSERT ( near(buf.getWeights()(0,31,5),0.8));
- }
-
- if (itsDoTest & DataEquals) {
- ASSERT (allNear (buf.getData(), data, 1.e-7));
- }
-
- if (itsDoTest & DataNotChanged) {
- ASSERT (allNear (buf.getData(), data, 1.e-7));
- }
- if (itsDoTest & DataChanged) {
- ASSERT (!(allNear (buf.getData(), data, 1.e-7)));
- }
- if (itsDoTest & WeightsNotChanged) {
- ASSERT (allNear (buf.getWeights(), weights, 1.e-7));
- }
- itsCount++;
- itsTimeStep++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==itsTimeInterval);
- ASSERT (int(infoIn.nbaselines())==itsNBl);
- }
-
- int itsCount;
- int itsTimeStep;
- int itsNTime, itsNBl, itsNChan, itsNCorr, itsTimeInterval, itsDoTest;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
-
- const DPStep::ShPtr& step=step1->getNextStep();
-
- // TODO: do line below for any step that is an ApplyCal
- step->show (cout);
-
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test clock + tec, and test two ApplyCals in sequence
-void testclocktec(int ntime, int nchan)
-{
- cout << "testclocktec: ntime=" << ntime << " nchan=" << nchan << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nchan);
- DPStep::ShPtr step1(in);
-
- ParameterSet parset1;
- parset1.add ("correction", "tec");
- parset1.add ("parmdb", "tApplyCal_tmp.parmdb");
- parset1.add ("timeslotsperparmupdate", "5");
- parset1.add ("updateweights", "true");
- DPStep::ShPtr step2(new ApplyCal(in, parset1, ""));
-
- ParameterSet parset2;
- parset2.add ("correction", "clock");
- parset2.add ("parmdb", "tApplyCal_tmp.parmdb");
- parset2.add ("timeslotsperparmupdate", "5");
- parset2.add ("updateweights", "true");
- DPStep::ShPtr step3(new ApplyCal(in, parset2, ""));
-
- ParameterSet parset3;
- parset3.add ("correction", "commonscalarphase");
- parset3.add ("parmdb", "tApplyCal_tmp.parmdb");
- parset3.add ("timeslotsperparmupdate", "1");
- parset3.add ("udpateweights", "true");
- DPStep::ShPtr step4(new ApplyCal(in, parset3, ""));
-
- DPStep::ShPtr step5(new TestOutput(ntime, nchan,
- TestOutput::DataChanged | TestOutput::WeightsNotChanged));
-
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- step3->setNextStep (step4);
- step4->setNextStep (step5);
- execute (step1);
- cout<<"finished testclocktec"<<endl;
-}
-
-// Test gain
-void testgain(int ntime, int nchan)
-{
- cout << "testgain: ntime=" << ntime << " nchan=" << nchan << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nchan);
- DPStep::ShPtr step1(in);
-
- ParameterSet parset1;
- parset1.add ("correction", "gain");
- parset1.add ("parmdb", "tApplyCal_tmp.parmdb");
- parset1.add ("timeslotsperparmupdate", "5");
- parset1.add ("updateweights", "true");
- DPStep::ShPtr step2(new ApplyCal(in, parset1, ""));
-
- DPStep::ShPtr step3(new TestOutput(ntime, nchan,
- TestOutput::DataEquals));
-
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-
-int main()
-{
- INIT_LOGGER ("tApplyCal");
- try {
- testclocktec (10, 32);
- testgain (10, 32);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tApplyCal.in_parmdb.tgz b/CEP/DP3/DPPP/test/tApplyCal.in_parmdb.tgz
deleted file mode 100644
index 4c14cc17721364d02af8f9a7c73d3815929b7429..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP/test/tApplyCal.in_parmdb.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP/test/tApplyCal.run b/CEP/DP3/DPPP/test/tApplyCal.run
deleted file mode 100755
index 64910a681ba7b4f19d6da563dfc072154eb424b7..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCal.run
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-if test ! -f tApplyCal.in_parmdb.tgz; then
- return 3 # untested
-fi
-
-tar zxf tApplyCal.in_parmdb.tgz
-./tApplyCal
diff --git a/CEP/DP3/DPPP/test/tApplyCal.sh b/CEP/DP3/DPPP/test/tApplyCal.sh
deleted file mode 100755
index 52e5cec93291c8fd10fa1d969b44bbe6317448ab..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCal.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tApplyCal
diff --git a/CEP/DP3/DPPP/test/tApplyCal2.run b/CEP/DP3/DPPP/test/tApplyCal2.run
deleted file mode 100755
index 6505b604075aad6f511b36f38cd367b0c03f4e2c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCal2.run
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/bin/bash
-
-# Get the taql executable and srcdir (script created by cmake's CONFIGURE_FILE).
-source findenv.run_script
-echo "srcdirx=$rt_srcdir"
-
-# Set srcdir if not defined (in case run by hand).
-if test "$srcdir" = ""; then
- srcdir="$rt_srcdir"
-fi
-
-if test ! -f ${srcdir}/tNDPPP-generic.in_MS.tgz; then
- exit 3 # untested
-fi
-
-set -e # Stop on any error
-
-rm -rf tApplyCal2_tmp
-mkdir -p tApplyCal2_tmp
-# Unpack the MS and other files and do the DPPP run.
-cd tApplyCal2_tmp
-tar zxf ${srcdir}/tNDPPP-generic.in_MS.tgz
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-
-echo; echo "Creating parmdb with defvalues 3"
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-adddef Gain:0:0:Real values=3.
-adddef Gain:1:1:Real values=3.
-EOL
-
-echo; echo "Testing without updateweights"
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 msout.weightcolumn=WEIGHTS_NEW steps=[applycal] applycal.parmdb=tApplyCal.parmdb showcounts=false'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DATA~=9*DATA3))' > taql.out
-diff taql.out taql.ref || exit 1
-$taqlexe 'select from tNDPPP-generic.MS where not(all(WEIGHTS_NEW~=WEIGHT_SPECTRUM))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Testing with updateweights"
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 msout.weightcolumn=WEIGHTS_NEW steps=[applycal] applycal.parmdb=tApplyCal.parmdb showcounts=false applycal.updateweights=true'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(WEIGHTS_NEW~=81*WEIGHT_SPECTRUM))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Testing CommonScalarPhase"
-rm -rf tApplyCal.parmdb
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-adddef CommonScalarPhase values=0
-EOL
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.parmdb=tApplyCal.parmdb applycal.correction=commonscalarphase showcounts=false'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DATA~=DATA3))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Testing ScalarPhase"
-rm -rf tApplyCal.parmdb
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-adddef ScalarPhase values=0
-EOL
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.parmdb=tApplyCal.parmdb applycal.correction=commonscalarphase showcounts=false'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DATA~=DATA3))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Testing ScalarPhase values"
-rm -rf tApplyCal.parmdb
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-add ScalarPhase:CS001HBA0 values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarPhase:CS002HBA0 values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarPhase:CS002HBA1 values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarPhase:CS004HBA1 values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarPhase:RS106HBA values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarPhase:RS208HBA values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarPhase:RS305HBA values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarPhase:RS307HBA values=[0.,0.,0.,0.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-EOL
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.parmdb=tApplyCal.parmdb applycal.correction=commonscalarphase showcounts=false'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DATA~=DATA3))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Testing ScalarAmplitude values"
-rm -rf tApplyCal.parmdb
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-add ScalarAmplitude:CS001HBA0 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarAmplitude:CS002HBA0 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarAmplitude:CS002HBA1 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarAmplitude:CS004HBA1 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarAmplitude:RS106HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarAmplitude:RS208HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarAmplitude:RS305HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add ScalarAmplitude:RS307HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-EOL
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.parmdb=tApplyCal.parmdb applycal.correction=scalaramplitude showcounts=false'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DATA~=9*DATA3))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Testing RotationAngle:*:phase_center values"
-rm -rf tApplyCal.parmdb
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-add RotationMeasure:CS001HBA0:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:CS002HBA0:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:CS002HBA1:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:CS004HBA1:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS106HBA:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS208HBA:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS305HBA:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS307HBA:phase_center values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-EOL
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.parmdb=tApplyCal.parmdb applycal.correction=rotationmeasure showcounts=false'
-echo $cmd
-eval $cmd
-
-echo; echo "Testing RotationAngle:* values"
-rm -rf tApplyCal.parmdb
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-add RotationMeasure:CS001HBA0 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:CS002HBA0 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:CS002HBA1 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:CS004HBA1 values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS106HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS208HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS305HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-add RotationMeasure:RS307HBA values=[3.,3.,3.,3.], domain=[10e6, 200e6, 4472025735, 4972025795], shape=[2,2], shape=[2,2]
-EOL
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.parmdb=tApplyCal.parmdb applycal.correction=rotationmeasure showcounts=false'
-echo $cmd
-eval $cmd
-
diff --git a/CEP/DP3/DPPP/test/tApplyCal2.sh b/CEP/DP3/DPPP/test/tApplyCal2.sh
deleted file mode 100755
index 2c166e188f8c8671ea007c9e084af4ce70fde811..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCal2.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tApplyCal2
diff --git a/CEP/DP3/DPPP/test/tApplyCalH5.cc b/CEP/DP3/DPPP/test/tApplyCalH5.cc
deleted file mode 100644
index 9a5ebe79d0882bb8844f64af4fd8a2431304c8e3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCalH5.cc
+++ /dev/null
@@ -1,382 +0,0 @@
-//# tApplyCalH5.cc: Test program for class ApplyCal
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tApplyCalH5.cc 24221 2013-08-02 12:24:48Z tammo $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/ApplyCal.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/H5Parm.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <Common/StreamUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// 9 baselines, 3 antennas, 4 correlations
-class TestInput: public DPInput
-{
-public:
- TestInput(uint ntime, uint nchan)
- : itsCount(0), itsNTime(ntime), itsNChan(nchan), itsNBl(9), itsNCorr(4),
- itsTimeInterval(5.), itsFirstTime(4472025740.0)
- {
- info().init (itsNCorr, nchan, ntime, itsFirstTime, itsTimeInterval,
- string(), string());
- // Fill the baseline stations; use 3 stations.
- // So they are called 00 01 02 10 11 12 20 21 22, etc.
-
- Vector<Int> ant1(itsNBl);
- Vector<Int> ant2(itsNBl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<itsNBl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 3) {
- st2 = 0;
- if (++st1 == 3) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(3);
- antNames[0] = "ant1";
- antNames[1] = "ant2";
- antNames[2] = "ant3";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(3);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(3, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanWidth(nchan, 100.e6);
- Vector<double> chanFreqs(nchan);
- for (uint ch=0; ch<nchan; ++ch) {
- double freq = 100.e6 + ch*10.e6;
- if (ch>2) {
- // Make frequencies unevenly spaced
- freq += 4.e6;
- }
- if (ch>4) {
- freq += 1.e6;
- }
- if (ch>5) {
- freq += 15.e6;
- }
- chanFreqs[ch] = freq;
- }
- if (nchan==2) {
- chanFreqs[0] = 100.e6;
- chanFreqs[1] = 101.e6;
- }
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(1,0);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- weights=1.;
-
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- DPBuffer buf;
- buf.setTime (itsCount*itsTimeInterval + itsFirstTime);
- buf.setData (data);
- buf.setWeights (weights);
- buf.setUVW (uvw);
- Cube<bool> flags(data.shape());
- flags = false;
- buf.setFlags (flags);
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = false;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&) {}
-
- int itsCount, itsNTime, itsNChan, itsNBl, itsNCorr, itsTimeInterval;
- double itsFirstTime;
-};
-
-
-
-// Class to check result of TestInput run by tests.
-class TestOutput: public DPStep
-{
-public:
- enum tests {WeightsNotChanged=1, DataNotChanged=2, DataChanged=4,
- DataEquals=8, WeightEquals=16};
- TestOutput(int ntime, int nchan, int doTest, bool solsHadFreqAxis=true,
- bool solsHadTimeAxis=true)
- : itsCount(0), itsTimeStep(0), itsNTime(ntime), itsNBl(9), itsNChan(nchan),
- itsNCorr(4), itsTimeInterval(5.), itsDoTest(doTest),
- itsSolsHadFreqAxis(solsHadFreqAxis), itsSolsHadTimeAxis(solsHadTimeAxis)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(1,0);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- indgen (weights, 1.0f, 0.0f);
-
- vector<double> rightTimes(max(itsNTime, 5));
- rightTimes[0] = 0;
- rightTimes[1] = 2;
- rightTimes[2] = 3;
- for (int t=3; t<itsNTime; ++t) {
- rightTimes[t] = 4;
- }
- if (!itsSolsHadTimeAxis) {
- rightTimes.assign(itsNTime, 0);
- }
-
- vector<double> rightFreqs(max(itsNChan, 5));
- rightFreqs[0] = 1;
- rightFreqs[1] = 1;
- rightFreqs[2] = 2;
- rightFreqs[3] = 2;
- rightFreqs[4] = 2;
- for (int f=5; f<itsNChan; ++f) {
- rightFreqs[f] = 3;
- }
- if (!itsSolsHadFreqAxis) {
- rightFreqs.assign(itsNChan, 1);
- }
-
- if (itsDoTest) {
- //cout<<endl;
- for (uint bl=0; bl<info().nbaselines(); ++bl) {
- for (int chan=0; chan<itsNChan; ++chan) {
- uint ant1 = info().getAnt1()[bl];
- uint ant2 = info().getAnt2()[bl];
- // Square root of autocorrelation for first antenna
- complex<float> val = sqrt(buf.getData().data()[bl*itsNCorr*itsNChan + chan*itsNCorr]);
-
- bool flag = buf.getFlags().data()[bl*itsNCorr*itsNChan + chan*itsNCorr];
- if ((ant1==1 || ant2==1) && rightTimes[itsTimeStep]==2 && rightFreqs[chan]==2) {
- ASSERT(flag);
- } else {
- ASSERT(!flag);
- ASSERT(near(rightTimes[itsTimeStep]*100 + rightFreqs[chan], val));
- }
- }
- }
- }
-
- if (itsDoTest & DataEquals) {
- ASSERT (allNear (buf.getData(), data, 1.e-7));
- }
-
- if (itsDoTest & DataNotChanged) {
- ASSERT (allNear (buf.getData(), data, 1.e-7));
- }
- if (itsDoTest & DataChanged) {
- ASSERT (!(allNear (buf.getData(), data, 1.e-7)));
- }
- if (itsDoTest & WeightsNotChanged) {
- ASSERT (allNear (buf.getWeights(), weights, 1.e-7));
- }
- itsCount++;
- itsTimeStep++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==itsTimeInterval);
- ASSERT (int(infoIn.nbaselines())==itsNBl);
- }
-
- int itsCount;
- int itsTimeStep;
- int itsNTime, itsNBl, itsNChan, itsNCorr, itsTimeInterval, itsDoTest;
- bool itsSolsHadFreqAxis, itsSolsHadTimeAxis;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
-
- const DPStep::ShPtr& step=step1->getNextStep();
-
- // TODO: do line below for any step that is an ApplyCal
- step->show (cout);
-
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test amplitude correction
-void testampl(int ntime, int nchan, bool freqaxis, bool timeaxis)
-{
- cout << "testampl: ntime=" << ntime << " nchan=" << nchan << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nchan);
- DPStep::ShPtr step1(in);
-
- ParameterSet parset1;
- parset1.add ("correction", "myampl");
- parset1.add ("parmdb", "tApplyCalH5_tmp.h5");
- DPStep::ShPtr step2(new ApplyCal(in, parset1, ""));
-
- DPStep::ShPtr step3(new TestOutput(ntime, nchan,
- TestOutput::WeightsNotChanged, freqaxis, timeaxis));
-
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
- cout<<endl;
-}
-
-
-// Write a temporary H5Parm
-void createH5Parm(vector<double> times, vector<double> freqs) {
- H5Parm h5parm("tApplyCalH5_tmp.h5", true);
-
- // Add antenna metadata
- vector<string> antNames;
- vector<vector<double> > antPositions;
- vector<double> oneAntPos(3, 42.);
- for (uint i=0; i<3; ++i) {
- stringstream antNameStr;
- antNameStr<<"ant"<<(i+1);
- antNames.push_back(antNameStr.str());
- antPositions.push_back(oneAntPos);
- }
- h5parm.addAntennas(antNames, antPositions);
-
- vector<H5Parm::AxisInfo> axes;
- axes.push_back(H5Parm::AxisInfo("ant",3));
- if (!times.empty()) {
- axes.push_back(H5Parm::AxisInfo("time", times.size()));
- }
- if (!freqs.empty()) {
- axes.push_back(H5Parm::AxisInfo("freq", freqs.size()));
- }
-
- H5Parm::SolTab soltab = h5parm.createSolTab("myampl","amplitude",axes);
- ASSERT(h5parm.nSolTabs() == 1);
- ASSERT(h5parm.hasSolTab("myampl"));
- soltab.setTimes(times);
- soltab.setFreqs(freqs);
- soltab.setAntennas(antNames);
-
- uint ntimes = max(times.size(), 1);
- uint nfreqs = max(freqs.size(), 1);
- vector<double> values(ntimes*nfreqs*3);
- vector<double> weights(ntimes*nfreqs*3);
- for (uint ant=0; ant<3; ++ant) {
- for (uint t=0; t<ntimes; ++t) {
- for (uint f=0; f<nfreqs; ++f) {
- values[ant*ntimes*nfreqs+t*nfreqs + f] = 1./(100.*(t%100)+(1+f));
- weights[ant*ntimes*nfreqs+t*nfreqs + f] = 1.;
- if (ant==1 && t==2 && f==1) {
- weights[ant*ntimes*nfreqs+t*nfreqs + f] = 0.;
- }
- }
- }
- }
- soltab.setValues(values, weights, "CREATE with DPPP tApplyCalH5");
-}
-
-int main()
-{
- INIT_LOGGER ("tApplyCalH5");
-
- vector<double> times;
- times.push_back(4472025742.0);
- times.push_back(4472025745.0);
- times.push_back(4472025747.5);
- times.push_back(4472025748.0);
- times.push_back(4472025762.0);
- vector<double> freqs;
- freqs.push_back(90.e6);
- freqs.push_back(139.e6);
- freqs.push_back(170.e6);
-
- try {
- createH5Parm(times, freqs);
- testampl(5, 7, true, true);
- createH5Parm(times, freqs);
- testampl(5, 2, true, true);
- createH5Parm(times, vector<double>());
- testampl(8, 9, false, true);
- createH5Parm(vector<double>(), freqs);
- testampl(13, 3, true, false);
- createH5Parm(vector<double>(), vector<double>());
- testampl(9, 2, false, false);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tApplyCalH5.run b/CEP/DP3/DPPP/test/tApplyCalH5.run
deleted file mode 100755
index 81ee2db8f59099c3b4bd1b29b84edbadf92a96aa..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCalH5.run
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-./tApplyCalH5
diff --git a/CEP/DP3/DPPP/test/tApplyCalH5.sh b/CEP/DP3/DPPP/test/tApplyCalH5.sh
deleted file mode 100755
index 189bdc360b952a79420d8b65466ab5c5a930bd25..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCalH5.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tApplyCalH5
diff --git a/CEP/DP3/DPPP/test/tApplyCal_parmdbscript b/CEP/DP3/DPPP/test/tApplyCal_parmdbscript
deleted file mode 100644
index 01414cdf6bcfe147dd6f32006f2b5ff0f54213f2..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tApplyCal_parmdbscript
+++ /dev/null
@@ -1,29 +0,0 @@
-# List of parmdbm commands to generate tApplyCal_tmp.parmdb
-
-open tablename='tApplyCal_tmp.parmdb'
-
-clear
-
-adddef Gain:0:0:Real values=1
-adddef Gain:0:0:Imag values=0
-adddef Gain:1:1:Real values=1
-adddef Gain:0:1:Real values=0
-adddef Gain:0:1:Imag values=0
-adddef Gain:1:0:Imag values=0
-
-adddef Clock values=0.0000037
-adddef TEC values=0.000013
-adddef CommonRotationAngle values=0.785
-adddef CommonScalarPhase values=0.00003
-
-adddef Clock:ant1 values=0
-adddef TEC:ant1 values=0
-adddef CommonRotationAngle:ant1 values=0
-adddef CommonScalarPhase:ant1 values=0
-
-add Gain:0:0:Real:ant2 values=[2,1,1,3], domain=[10000000, 43000000, 4472025735, 4472025795], shape=[2,2]
-add Gain:0:0:Imag:ant2 values=[0,0,0,4], domain=[10000000, 43000000, 4472025735, 4472025795], shape=[2,2]
-add Gain:1:1:Real:ant2 values=[3,1,1,1], domain=[10000000, 43000000, 4472025735, 4472025795], shape=[2,2]
-add Gain:1:1:Imag:ant2 values=[0,0,0,0], domain=[10000000, 43000000, 4472025735, 4472025795], shape=[2,2]
-
-add Gain:0:1:Real:ant3 values=[0,0.5,0,0], domain=[10000000, 43000000, 4472025735, 4472025795], shape=[2,2]
diff --git a/CEP/DP3/DPPP/test/tAverager.cc b/CEP/DP3/DPPP/test/tAverager.cc
deleted file mode 100644
index 31a0bd2a20b0a6c576a07f859b3628582e9944a5..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tAverager.cc
+++ /dev/null
@@ -1,623 +0,0 @@
-//# tAverager.cc: Test program for class Averager
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Averager.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-
-#include <casacore/casa/Quanta/Quantum.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, double(itsCount*100));
- buf.setUVW (uvw);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&)
- {
- // Use timeInterval=5
- info().init (itsNCorr, itsNChan, itsNTime, 100, 5, string(), string());
- // Define the frequencies.
- Vector<double> chanFreqs(itsNChan);
- Vector<double> chanWidth(itsNChan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result of averaging TestInput.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr,
- int navgtime, int navgchan, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsNAvgTime(navgtime), itsNAvgChan(navgchan),
- itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- int nchan = 1+(itsNChan-1)/itsNAvgChan;
- int navgtime = std::min(itsNAvgTime, itsNTime-itsCount*itsNAvgTime);
- // Fill expected result in similar way as TestInput.
- Cube<Complex> data(itsNCorr,itsNChan,itsNBl);
- Cube<float> weights(itsNCorr,itsNChan,itsNBl);
- Cube<bool> fullResFlags(itsNChan,itsNAvgTime,itsNBl);
- fullResFlags = true; // takes care of missing times at the end
- weights = 0;
- for (int j=itsCount*itsNAvgTime; j<itsCount*itsNAvgTime+navgtime; ++j) {
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] += Complex(i+j*10,i-1000+j*6);
- weights.data()[i] += float(1);
- }
- fullResFlags(Slicer(IPosition(3,0,0,0),
- IPosition(3,itsNChan,navgtime,itsNBl))) = itsFlag;
- }
- Cube<Complex> result(itsNCorr,nchan,itsNBl);
- Cube<float> resultw(itsNCorr,nchan,itsNBl);
- resultw = 0;
- // Average to get the true expected result.
- for (int k=0; k<itsNBl; ++k) {
- for (int i=0; i<itsNCorr; ++i) {
- for (int j=0; j<nchan; ++j) {
- int jc;
- for (jc=j*itsNAvgChan;
- jc<std::min((j+1)*itsNAvgChan, itsNChan); ++jc) {
- result(i,j,k) += data(i,jc,k);
- resultw(i,j,k) += weights(i,jc,k);
- }
- result(i,j,k) /= float(navgtime*(jc-j*itsNAvgChan));
- }
- }
- }
- // Check the averaged result.
- ASSERT (allNear(real(buf.getData()), real(result), 1e-5));
- ///cout << imag(buf.getData()) << endl<<imag(result);
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (allEQ(buf.getFlags(), itsFlag));
- ASSERT (near(buf.getTime(),
- 2+5*(itsCount*itsNAvgTime + (itsNAvgTime-1)/2.)));
- ASSERT (allNear(buf.getWeights(), resultw, 1e-5));
- if (navgtime == itsNAvgTime) {
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, 100*(itsCount*itsNAvgTime + 0.5*(itsNAvgTime-1)));
- ASSERT (allNear(buf.getUVW(), uvw, 1e-5));
- }
- cout <<buf.getFullResFlags()<< fullResFlags;
- ASSERT (allEQ(buf.getFullResFlags(), fullResFlags));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNChan);
- ASSERT (int(info.nchan())==1+(itsNChan-1)/itsNAvgChan);
- ASSERT (int(info.ntime())==1+(itsNTime-1)/itsNAvgTime);
- ASSERT (info.timeInterval()==5*itsNAvgTime);
- ASSERT (int(info.nchanAvg())==itsNAvgChan);
- ASSERT (int(info.ntimeAvg())==itsNAvgTime);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr, itsNAvgTime, itsNAvgChan;
- bool itsFlag;
-};
-
-
-// More elaborate class which can set different flags and weights.
-class TestInput3: public DPInput
-{
-public:
- TestInput3(int nrtime, int nrbl, int nrchan, int nrcorr)
- : itsCount(0),
- itsNrTime(nrtime), itsNrBl(nrbl), itsNrChan(nrchan), itsNrCorr(nrcorr)
- {
- itsFullResFlags.resize (itsNrChan,1,nrbl);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNrTime) {
- return false;
- }
- Cube<Complex> data(itsNrCorr,itsNrChan,itsNrBl);
- Cube<float> weights(itsNrCorr,itsNrChan,itsNrBl);
- Cube<bool> flags(itsNrCorr,itsNrChan,itsNrBl);
- int i = 0;
- for (int ib=0; ib<itsNrBl; ++ib) {
- for (int ic=0; ic<itsNrChan; ++ic) {
- for (int ip=0; ip<itsNrCorr; ++ip) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- weights.data()[i] = (1 + (itsCount+ib+ic)%5) / 5.;
- flags.data()[i] = ((itsCount+2*ib+3*ic) % 7 == 0);
- i++;
- }
- itsFullResFlags(ic,0,ib) = ((itsCount+2*ib+3*ic) % 7 == 0);
- }
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- buf.setWeights (weights);
- buf.setFlags (flags);
- Vector<uint> rownrs(1,itsCount);
- buf.setRowNrs (rownrs);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void getUVW (const casacore::RefRows&, double, DPBuffer& buf)
- {
- buf.getUVW().resize (3, itsNrBl);
- indgen (buf.getUVW());
- }
- virtual bool getFullResFlags (const casacore::RefRows&, DPBuffer& buf)
- {
- buf.getFullResFlags().assign (itsFullResFlags);
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&)
- {
- // Use timeInterval=5
- info().init (itsNrCorr, itsNrChan, itsNrTime, 100, 5, string(), string());
- // Define the frequencies.
- Vector<double> chanFreqs(itsNrChan);
- Vector<double> chanWidth(itsNrChan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
- int itsCount, itsNrTime, itsNrBl, itsNrChan, itsNrCorr;
- Cube<bool> itsFullResFlags;
-};
-
-// Class to check result of averaging TestInput3.
-// All input must be averaged (in one or more steps) to a single value
-// per corr/baseline.
-class TestOutput3: public DPStep
-{
-public:
- TestOutput3(int nrtime, int nrbl, int nrchan, int nrcorr)
- : itsNrTime(nrtime), itsNrBl(nrbl), itsNrChan(nrchan), itsNrCorr(nrcorr)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- Cube<Complex> result(itsNrCorr,1,itsNrBl);
- Cube<float> weights(itsNrCorr,1,itsNrBl);
- Cube<bool> flags(itsNrCorr,1,itsNrBl);
- Cube<bool> fullResFlags(itsNrChan,itsNrTime,itsNrBl);
- weights = float(0);
- flags = true;
- fullResFlags = true;
- // Create data in the same way as in TestInput3.
- for (int it=0; it<itsNrTime; ++it) {
- int i = 0;
- for (int ib=0; ib<itsNrBl; ++ib) {
- for (int ic=0; ic<itsNrChan; ++ic) {
- for (int ip=0; ip<itsNrCorr; ++ip) {
- if ((it+2*ib+3*ic) % 7 != 0) {
- float weight = (1 + (it+ib+ic)%5) / 5.;
- result(ip,0,ib) += weight * Complex(i+it*10,i-1000+it*6);
- weights(ip,0,ib) += weight;
- /// cout << result(ip,0,ib) << weight << endl;
- flags(ip,0,ib) = false;
- fullResFlags(ic,it,ib) = false;
- }
- i++;
- }
- }
- }
- }
- ASSERT (allNE(weights, float(0.)));
- for (uint i=0; i<result.size(); ++i) {
- result.data()[i] /= weights.data()[i];
- }
- // Check the averaged result.
- ///cout << real(buf.getData()) << endl<<real(result);
- ASSERT (allNear(real(buf.getData()), real(result), 1e-5));
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (allEQ(buf.getFlags(), flags));
- ASSERT (near(buf.getTime(), 2.+5*(itsNrTime-1)/2.));
- ASSERT (allNear(buf.getWeights(), weights, 1e-5));
- Matrix<double> uvw(3,itsNrBl);
- indgen (uvw);
- ASSERT (allNear(buf.getUVW(), uvw, 1e-5));
- ///cout <<buf.getFullResFlags()<< fullResFlags;
- ASSERT (allEQ(buf.getFullResFlags(), fullResFlags));
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNrChan);
- ASSERT (info.nchan()==1);
- ASSERT (info.ntime()==1);
- ASSERT (info.timeInterval()==5*itsNrTime);
- ASSERT (int(info.nchanAvg())==itsNrChan);
- ASSERT (int(info.ntimeAvg())==itsNrTime);
- }
-
- int itsNrTime, itsNrBl, itsNrChan, itsNrCorr;
-};
-
-// Simple class to flag every step-th XX point.
-class TestFlagger: public DPStep
-{
-public:
- TestFlagger(int step)
- : itsCount(0), itsStep(step)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- DPBuffer buf2(buf);
- int ncorr = buf2.getFlags().shape()[0];
- int np = buf2.getFlags().size() / ncorr;
- bool* flagPtr = buf2.getFlags().data();
- for (int i=0; i<np; ++i) {
- if ((i+itsCount)%itsStep == 0) {
- ///cout << "flagged " <<itsCount <<' '<< i << endl;
- for (int j=0; j<ncorr; ++j) {
- flagPtr[i*ncorr + j] = true;
- }
- }
- }
- getNextStep()->process (buf2);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
-
- int itsCount, itsStep;
-};
-
-// Class to check result of averaging and flagging TestInput3.
-// First the data are averaged from 8,4 to 4,2, then every step-th point
-// is flagged, and finally it is averaged to 1,1.
-class TestOutput4: public DPStep
-{
-public:
- TestOutput4(int nrtime, int nrbl, int nrchan, int nrcorr, int step)
- : itsNrTime(nrtime), itsNrBl(nrbl), itsNrChan(nrchan), itsNrCorr(nrcorr),
- itsStep(step)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- Cube<Complex> result(itsNrCorr,1,itsNrBl);
- Cube<float> weights(itsNrCorr,1,itsNrBl);
- Cube<bool> flags(itsNrCorr,1,itsNrBl);
- Cube<bool> fullResFlags(itsNrChan,itsNrTime,itsNrBl);
- weights = float(0);
- flags = true;
- fullResFlags = true;
- // Create data in the same way as in TestInput3.
- for (int it=0; it<itsNrTime; ++it) {
- int i = 0;
- for (int ib=0; ib<itsNrBl; ++ib) {
- for (int ic=0; ic<itsNrChan; ++ic) {
- // TestFlagger flags every step-th point of 2x2 averaged data.
- int tf = it/2; // same as itsCount in testFlagger
- if (((ib*itsNrChan + ic)/2 + tf) % itsStep == 0) {
- ///cout << "out4 flagged "<< tf<<' '<< i/itsNrCorr<<' ' <<ib<<' '<<ic/2 << endl;
- i += itsNrCorr;
- } else {
- for (int ip=0; ip<itsNrCorr; ++ip) {
- if ((it+2*ib+3*ic) % 7 != 0) {
- float weight = (1 + (it+ib+ic)%5) / 5.;
- result(ip,0,ib) += weight * Complex(i+it*10,i-1000+it*6);
- weights(ip,0,ib) += weight;
- /// cout << result(ip,0,ib) << weight << endl;
- flags(ip,0,ib) = false;
- fullResFlags(ic,it,ib) = false;
- }
- i++;
- }
- }
- }
- }
- }
- for (uint i=0; i<result.size(); ++i) {
- if (!flags.data()[i]) {
- result.data()[i] /= weights.data()[i];
- }
- }
- // Check the averaged result.
- ///cout << real(buf.getData()) << endl<<real(result);
- ASSERT (allNear(real(buf.getData()), real(result), 1e-5));
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (allEQ(buf.getFlags(), flags));
- ASSERT (near(buf.getTime(), 2.+5*(itsNrTime-1)/2.));
- ASSERT (allNear(buf.getWeights(), weights, 1e-5));
- Matrix<double> uvw(3,itsNrBl);
- indgen (uvw);
- ASSERT (allNear(buf.getUVW(), uvw, 1e-5));
- ///cout <<buf.getFullResFlags()<< fullResFlags;
- ASSERT (allEQ(buf.getFullResFlags(), fullResFlags));
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNrChan);
- ASSERT (info.nchan()==1);
- ASSERT (info.ntime()==1);
- ASSERT (info.timeInterval()==5*itsNrTime);
- ASSERT (int(info.nchanAvg())==itsNrChan);
- ASSERT (int(info.ntimeAvg())==itsNrTime);
- }
-
- int itsNrTime, itsNrBl, itsNrChan, itsNrCorr, itsStep;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test simple averaging without flagged points.
-void test1(int ntime, int nbl, int nchan, int ncorr,
- int navgtime, int navgchan, bool flag)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " navgtime=" << navgtime
- << " navgchan=" << navgchan << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("freqstep", toString(navgchan));
- parset.add ("timestep", toString(navgtime));
- DPStep::ShPtr step2(new Averager(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr,
- navgtime, navgchan, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Like test 1, but specify target resolution
-void test1resolution(int ntime, int nbl, int nchan, int ncorr,
- double timeresolution, double freqresolution,
- string frequnit, bool flag)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " timeresolution=" << timeresolution
- << " freqresolution=" << freqresolution << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("freqresolution", toString(freqresolution)+frequnit);
- parset.add ("timeresolution", toString(timeresolution));
- DPStep::ShPtr step2(new Averager(in, parset, ""));
-
- if (!frequnit.empty()) {
- Quantity q(freqresolution, frequnit);
- freqresolution = q.getValue("Hz", true);
- }
-
- int navgchan = std::max(1, int(freqresolution / 100000 + 0.5));
- int navgtime = std::max(1, int(timeresolution / 5. + 0.5));
-
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr,
- navgtime, navgchan, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Like test1, but the averaging is done in two steps.
-void test2(int ntime, int nbl, int nchan, int ncorr, bool flag)
-{
- cout << "test2: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " navgtime=2"
- << " navgchan=4" << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset1, parset2;
- parset1.add ("freqstep", "4");
- parset2.add ("timestep", "2");
- DPStep::ShPtr step2a(new Averager(in, parset1, ""));
- DPStep::ShPtr step2b(new Averager(in, parset2, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr, 2, 4, flag));
- step1->setNextStep (step2a);
- step2a->setNextStep (step2b);
- step2b->setNextStep (step3);
- execute (step1);
-}
-
-// Do tests with weighting and some flagged points.
-void test3(int nrbl, int nrcorr)
-{
- {
- cout << "test3: ntime=2 nrbl=" << nrbl << " nchan=2 ncorr=" << nrcorr
- << endl;
- cout << " navgtime=2 navgchan=2" << endl;
- // Create the steps.
- TestInput3* in = new TestInput3(2, nrbl, 2, nrcorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset1;
- parset1.add ("freqstep", "2");
- parset1.add ("timestep", "2");
- DPStep::ShPtr step2a(new Averager(in, parset1, ""));
- DPStep::ShPtr step3(new TestOutput3(2, nrbl, 2, nrcorr));
- step1->setNextStep (step2a);
- step2a->setNextStep (step3);
- execute (step1);
- }
- {
- cout << "test3: ntime=4 nrbl=" << nrbl << " nchan=8 ncorr=" << nrcorr
- << endl;
- cout << " [navgtime=2 navgchan=4], [navgtime=2 navgchan=2]" << endl;
- // Create the steps.
- TestInput3* in = new TestInput3(4, nrbl, 8, nrcorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset1, parset2;
- parset1.add ("freqstep", "4");
- parset1.add ("timestep", "2");
- parset2.add ("freqstep", "2");
- parset2.add ("timestep", "2");
- DPStep::ShPtr step2a(new Averager(in, parset1, ""));
- DPStep::ShPtr step2b(new Averager(in, parset2, ""));
- DPStep::ShPtr step3(new TestOutput3(4, nrbl, 8, nrcorr));
- step1->setNextStep (step2a);
- step2a->setNextStep (step2b);
- step2b->setNextStep (step3);
- execute (step1);
- }
-}
-
-// Do tests with averaging and flagging steps to see if the flags are
-// promoted to the FULLRES flags.
-void test4(int nrbl, int nrcorr, int flagstep)
-{
- {
- cout << "test4: ntime=4 nrbl=" << nrbl << " nchan=8 ncorr=" << nrcorr
- << endl;
- cout << " [navgtime=2 navgchan=2], [flagstep=" << flagstep
- << "] [navgtime=2 navgchan=4]" << endl;
- // Create the steps.
- TestInput3* in = new TestInput3(4, nrbl, 8, nrcorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset1, parset2;
- parset1.add ("freqstep", "2");
- parset1.add ("timestep", "2");
- parset2.add ("freqstep", "4");
- parset2.add ("timestep", "2");
- DPStep::ShPtr step2a(new Averager(in, parset1, ""));
- DPStep::ShPtr step2b(new TestFlagger(flagstep));
- DPStep::ShPtr step2c(new Averager(in, parset2, ""));
- DPStep::ShPtr step3(new TestOutput4(4, nrbl, 8, nrcorr, flagstep));
- step1->setNextStep (step2a);
- step2a->setNextStep (step2b);
- step2b->setNextStep (step2c);
- step2c->setNextStep (step3);
- execute (step1);
- }
-}
-
-
-int main()
-{
- try {
- test1(10, 3, 32, 4, 2, 4, false);
- test1(10, 3, 30, 1, 3, 3, true);
- test1(10, 3, 30, 1, 3, 3, false);
- test1(11, 3, 30, 2, 3, 3, false);
- test1(10, 3, 32, 4, 1, 32, false);
- test1(10, 3, 32, 1, 1, 1, false);
-
- test1resolution(10, 3, 32, 4, 10., 100000, "Hz", false);
- test1resolution(11, 3, 32, 4, 1., 800, "kHz", false);
- test1resolution(11, 3, 32, 4, 15., 0.4, "MHz", false);
- test2(10, 3, 32, 2, true);
- test2(10, 3, 32, 2, false);
- test3(1, 1);
- test3(10, 4);
- test4(1, 4, 3);
- test4(20, 4, 5);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tAverager.sh b/CEP/DP3/DPPP/test/tAverager.sh
deleted file mode 100755
index 2e4fb8ee3c59528acfbb7e05e3d46943fd0dac97..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tAverager.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tAverager
diff --git a/CEP/DP3/DPPP/test/tBaselineSelection.cc b/CEP/DP3/DPPP/test/tBaselineSelection.cc
deleted file mode 100644
index 81a9ab12a06d1cd2dffbade1ab09591962d902eb..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tBaselineSelection.cc
+++ /dev/null
@@ -1,228 +0,0 @@
-//# tBaselineSelection.cc: Test program for class BaselneSelection
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/BaselineSelection.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <Common/StreamUtil.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-
-// Define the info object containing the antenna/baseline info.
-DPInfo makeInfo (int nbl)
-{
- DPInfo info;
- info.init (4, 16, 1, 0.5, 5., string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- // Baseline lengths are:
- // 0-1 144.01
- // 0-2 288.021
- // 0-3 431.914
- // 1-2 144.01
- // 1-3 287.904
- // 2-3 143.896
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info.set (antNames, antDiam, antPos, ant1, ant2);
- return info;
-}
-
-void test1 (const DPInfo& info)
-{
- cout << "test empty" << endl;
- ParameterSet ps;
- ps.add ("baseline", "[]");
- BaselineSelection selBL (ps, "");
- selBL.show (cout);
- Matrix<bool> res = selBL.apply (info);
- ASSERT (allEQ(res, true));
-}
-
-void test2 (const DPInfo& info)
-{
- cout << "test auto" << endl;
- ParameterSet ps;
- ps.add ("baseline", "[]");
- ps.add ("corrtype", "AUTO");
- BaselineSelection selBL (ps, "");
- selBL.show (cout);
- Matrix<bool> res = selBL.apply (info);
- ASSERT (allEQ(res.diagonal(), true));
- res.diagonal() = false;
- ASSERT (allEQ(res, false));
-}
-
-void test3 (const DPInfo& info)
-{
- cout << "test cross" << endl;
- ParameterSet ps;
- ps.add ("corrtype", "CROSS");
- BaselineSelection selBL (ps, "");
- selBL.show (cout);
- Matrix<bool> res = selBL.apply (info);
- ASSERT (allEQ(res.diagonal(), false));
- res.diagonal() = true;
- ASSERT (allEQ(res, true));
-}
-
-void test4 (const DPInfo& info)
-{
- cout << "test length" << endl;
- Matrix<double> blength(4,4, 0.);
- blength(0,1) = blength(1,0) = 144.01;
- blength(0,2) = blength(2,0) = 288.021;
- blength(0,3) = blength(3,0) = 431.914;
- blength(1,2) = blength(2,1) = 144.01;
- blength(1,3) = blength(3,1) = 287.904;
- blength(2,3) = blength(3,2) = 143.896;
- ParameterSet ps;
- ps.add ("blmin", "145");
- BaselineSelection selBL (ps, "", true);
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), blength<=145.));
- ps.add ("blmax", "288");
- selBL = BaselineSelection(ps, "", true);
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), blength<=145. || blength>=288.));
- ps.add ("corrtype", "cross");
- selBL = BaselineSelection(ps, "", true);
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), (blength > 0. &&
- (blength<=145. || blength>=288.))));
- ps.add ("blrange", "[0,144,288,430]");
- selBL = BaselineSelection(ps, "", true);
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), (blength > 0. &&
- (blength<=145. || blength>=288.))));
- selBL = BaselineSelection(ps, "", false);
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), ((blength > 0. && blength<=144.) ||
- (blength>=288. && blength<=430.))));
- ps.replace ("corrtype", "");
- selBL = BaselineSelection(ps, "", false);
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), ((blength >=0. && blength<=144.) ||
- (blength>=288. && blength<=430.))));
-}
-
-void test5 (const DPInfo& info)
-{
- cout << "test baseline" << endl;
- //antNames[0] = "rs01.s01";
- //antNames[1] = "rs02.s01";
- //antNames[2] = "cs01.s01";
- //antNames[3] = "cs01.s02";
- Matrix<int> bl(4,4);
- bl(0,0) = 00;
- bl(1,1) = 11;
- bl(2,2) = 22;
- bl(3,3) = 33;
- bl(0,1) = bl(1,0) = 01;
- bl(0,2) = bl(2,0) = 02;
- bl(0,3) = bl(3,0) = 03;
- bl(1,2) = bl(2,1) = 12;
- bl(1,3) = bl(3,1) = 13;
- bl(2,3) = bl(3,2) = 23;
- ParameterSet ps;
- ps.add ("baseline", "[[rs01.s01]]");
- BaselineSelection selBL (ps, "");
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), bl==00 || bl==01 || bl==02 || bl==03));
- ps.replace ("baseline", "[[rs01.s01,rs*]]");
- selBL = BaselineSelection(ps, "");
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), bl==00 || bl==01));
- ps.replace ("baseline", "[[rs01.s01],[rs02*]]");
- selBL = BaselineSelection(ps, "");
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), !(bl==22 || bl==23 || bl==33)));
- ps.replace ("baseline", "[rs01.s01,'rs02*']");
- selBL = BaselineSelection(ps, "");
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), !(bl==22 || bl==23 || bl==33)));
- ps.replace ("corrtype", "cross");
- selBL = BaselineSelection(ps, "");
- selBL.show (cout);
- ASSERT (allEQ(selBL.apply(info), !(bl==00 || bl==11 || bl==22 ||
- bl==23 || bl==33)));
- // Note that the MSSelection syntax is not tested, because it requires
- // a MeasurementSet.
-}
-
-int main()
-{
- try {
- DPInfo info(makeInfo(16));
- test1(info);
- test2(info);
- test3(info);
- test4(info);
- test5(info);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tDemix.in_MS.tgz b/CEP/DP3/DPPP/test/tDemix.in_MS.tgz
deleted file mode 100644
index 7caa4601cd5b7c4d991a9b706f95674b7edc49a6..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP/test/tDemix.in_MS.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP/test/tDemix.run b/CEP/DP3/DPPP/test/tDemix.run
deleted file mode 100755
index b43af0c3613a20d21a221eb59beb22d1e0d80ea0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tDemix.run
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh
-
-if test ! -f tDemix.in_MS.tgz; then
- exit 3 # untested
-fi
-
-rm -rf tDemix_tmp
-# Unpack the MS and other files and do the DPPP run.
-tar zxf tDemix.in_MS.tgz
-cd tDemix_tmp
-
-# Run the test script (created by cmake's CONFIGURE_FILE).
-sh ../tDemix.run_script
diff --git a/CEP/DP3/DPPP/test/tDemix.run_tmpl b/CEP/DP3/DPPP/test/tDemix.run_tmpl
deleted file mode 100644
index bfb0b9789d06794b9076872d98f111908f5f8a11..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tDemix.run_tmpl
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/sh
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-# Find taql program.
-taqlexe=@TAQL_EXECUTABLE@
-
-echo; echo "### Test without target."; echo
-rm -rf instrument
-NDPPP tDemix.parset demix.ignoretarget=true
-# Compare some columns of the output MS with the reference output.
-$taqlexe 'select from tDemix_out.MS t1, tDemix_ref1.MS t2 where not all(near(t1.DATA,t2.DATA,1e-3) || (isnan(t1.DATA) && isnan(t2.DATA))) || not all(t1.FLAG = t2.FLAG) || not all(near(t1.WEIGHT_SPECTRUM, t2.WEIGHT_SPECTRUM)) || not all(t1.LOFAR_FULL_RES_FLAG = t2.LOFAR_FULL_RES_FLAG) || t1.ANTENNA1 != t2.ANTENNA1 || t1.ANTENNA2 != t2.ANTENNA2 || t1.TIME !~= t2.TIME' > taql.out
-diff taql.out taql.ref || exit 1
-# Compare the instrument table.
-##$taqlexe 'select from instrument t1, instrument_ref1 t2 where t1.NAMEID!=t1.NAMEID || t1.STARTX!~=t2.STARTX || t1.ENDX!~=t2.ENDX || t1.STARTY!~=t2.STARTY || t1.ENDY!~=t2.ENDY || not all(near(t1.VALUES, t2.VALUES))' > taql.out
-##diff taql.out taql.ref || exit 1
-
-echo; echo "### Test with target projected away."; echo
-rm -rf instrument
-NDPPP tDemix.parset demix.ignoretarget=false
-# Compare some columns of the output MS with the reference output.
-$taqlexe 'select from tDemix_out.MS t1, tDemix_ref2.MS t2 where not all(near(t1.DATA,t2.DATA,1e-3) || (isnan(t1.DATA) && isnan(t2.DATA))) || not all(t1.FLAG = t2.FLAG) || not all(near(t1.WEIGHT_SPECTRUM, t2.WEIGHT_SPECTRUM)) || not all(t1.LOFAR_FULL_RES_FLAG = t2.LOFAR_FULL_RES_FLAG) || t1.ANTENNA1 != t2.ANTENNA1 || t1.ANTENNA2 != t2.ANTENNA2 || t1.TIME !~= t2.TIME' > taql.out
-diff taql.out taql.ref || exit 1
-# Compare the instrument table.
-##$taqlexe 'select from instrument t1, instrument_ref2 t2 where t1.NAMEID!=t1.NAMEID || t1.STARTX!~=t2.STARTX || t1.ENDX!~=t2.ENDX || t1.STARTY!~=t2.STARTY || t1.ENDY!~=t2.ENDY || not all(near(t1.VALUES, t2.VALUES))' > taql.out
-##diff taql.out taql.ref || exit 1
-
-echo; echo "### Test with target."; echo
-rm -rf instrument
-NDPPP tDemix.parset demix.target=CIZA.SP1A.FITS.pbcor_patch_s537 demix.freqstep=32 demix.timestep=5
-# Compare some columns of the output MS with the reference output.
-$taqlexe 'select from tDemix_out.MS t1, tDemix_ref3.MS t2 where not all(near(t1.DATA,t2.DATA,1e-3) || (isnan(t1.DATA) && isnan(t2.DATA))) || not all(t1.FLAG = t2.FLAG) || not all(near(t1.WEIGHT_SPECTRUM, t2.WEIGHT_SPECTRUM)) || not all(t1.LOFAR_FULL_RES_FLAG = t2.LOFAR_FULL_RES_FLAG) || t1.ANTENNA1 != t2.ANTENNA1 || t1.ANTENNA2 != t2.ANTENNA2 || t1.TIME !~= t2.TIME' > taql.out
-diff taql.out taql.ref || exit 1
-# Compare the instrument table.
-##$taqlexe 'select from instrument t1, instrument_ref3 t2 where t1.NAMEID!=t1.NAMEID || t1.STARTX!~=t2.STARTX || t1.ENDX!~=t2.ENDX || t1.STARTY!~=t2.STARTY || t1.ENDY!~=t2.ENDY || not all(near(t1.VALUES, t2.VALUES))' > taql.out
-##diff taql.out taql.ref || exit 1
-
-exit 0
diff --git a/CEP/DP3/DPPP/test/tDemix.sh b/CEP/DP3/DPPP/test/tDemix.sh
deleted file mode 100755
index cb5dad0c9ab5994e3e6a09d65fe147edec29bbe4..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tDemix.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tDemix
diff --git a/CEP/DP3/DPPP/test/tDemixer.cc b/CEP/DP3/DPPP/test/tDemixer.cc
deleted file mode 100644
index 463b22753e30694e6f962da17bd8db2e9e764a33..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tDemixer.cc
+++ /dev/null
@@ -1,233 +0,0 @@
-//# tDemixer.cc: Test program for class Demixer
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Demixer.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, double(itsCount*100));
- buf.setUVW (uvw);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (DPInfo& info)
- // Use startchan=8 and timeInterval=5
- { info.init (itsNCorr, 8, itsNChan, itsNBl, itsNTime, 5); }
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result of averaging TestInput.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr,
- int navgtime, int navgchan, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsNAvgTime(navgtime), itsNAvgChan(navgchan),
- itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- int nchan = 1+(itsNChan-1)/itsNAvgChan;
- int navgtime = std::min(itsNAvgTime, itsNTime-itsCount*itsNAvgTime);
- // Fill expected result in similar way as TestInput.
- Cube<Complex> data(itsNCorr,itsNChan,itsNBl);
- Cube<float> weights(itsNCorr,itsNChan,itsNBl);
- Cube<bool> fullResFlags(itsNChan,itsNAvgTime,itsNBl);
- fullResFlags = true; // takes care of missing times at the end
- weights = 0;
- if (!itsFlag) {
- for (int j=itsCount*itsNAvgTime; j<itsCount*itsNAvgTime+navgtime; ++j) {
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] += Complex(i+j*10,i-1000+j*6);
- weights.data()[i] += float(1);
- }
- }
- fullResFlags(Slicer(IPosition(3,0,0,0),
- IPosition(3,itsNChan,navgtime,itsNBl))) = itsFlag;
- }
- Cube<Complex> result(itsNCorr,nchan,itsNBl);
- Cube<float> resultw(itsNCorr,nchan,itsNBl);
- resultw = 0;
- // Average to get the true expected result.
- for (int k=0; k<itsNBl; ++k) {
- for (int i=0; i<itsNCorr; ++i) {
- for (int j=0; j<nchan; ++j) {
- int jc;
- for (jc=j*itsNAvgChan;
- jc<std::min((j+1)*itsNAvgChan, itsNChan); ++jc) {
- result(i,j,k) += data(i,jc,k);
- resultw(i,j,k) += weights(i,jc,k);
- }
- result(i,j,k) /= float(navgtime*(jc-j*itsNAvgChan));
- }
- }
- }
- // Check the averaged result.
- ASSERT (allNear(real(buf.getData()), real(result), 1e-5));
- ///cout << imag(buf.getData()) << endl<<imag(result);
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (allEQ(buf.getFlags(), itsFlag));
- ASSERT (near(buf.getTime(),
- 2+5*(itsCount*itsNAvgTime + (itsNAvgTime-1)/2.)));
- ASSERT (allNear(buf.getWeights(), resultw, 1e-5));
- if (navgtime == itsNAvgTime) {
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, 100*(itsCount*itsNAvgTime + 0.5*(itsNAvgTime-1)));
- ASSERT (allNear(buf.getUVW(), uvw, 1e-5));
- }
- ///cout <<buf.getFullResFlags()<< fullResFlags;
- ASSERT (allEQ(buf.getFullResFlags(), fullResFlags));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (DPInfo& info)
- {
- ASSERT (info.startChan()==8);
- ASSERT (int(info.origNChan())==itsNChan);
- ASSERT (int(info.nchan())==1+(itsNChan-1)/itsNAvgChan);
- ASSERT (int(info.ntime())==1+(itsNTime-1)/itsNAvgTime);
- ASSERT (info.timeInterval()==5*itsNAvgTime);
- ASSERT (int(info.nchanAvg())==itsNAvgChan);
- ASSERT (int(info.ntimeAvg())==itsNAvgTime);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr, itsNAvgTime, itsNAvgChan;
- bool itsFlag;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- DPInfo info;
- DPStep::ShPtr step = step1;
- while (step) {
- step->updateInfo (info);
- step = step->getNextStep();
- }
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test simple averaging without flagged points.
-void test1(int ntime, int nbl, int nchan, int ncorr,
- int navgtime, int navgchan, bool flag)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " navgtime=" << navgtime
- << " navgchan=" << navgchan << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("freqstep", toString(navgchan));
- parset.add ("timestep", toString(navgtime));
- parset.add ("sources" , "CasA");
- DPStep::ShPtr step2(new Demixer(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr,
- navgtime, navgchan, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-
-int main()
-{
- try {
- test1(10, 3, 32, 4, 2, 4, false);
- test1(10, 3, 30, 1, 3, 3, true);
- test1(10, 3, 30, 1, 3, 3, false);
- test1(11, 3, 30, 2, 3, 3, false);
- test1(10, 3, 32, 4, 1, 32, false);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tFilter.cc b/CEP/DP3/DPPP/test/tFilter.cc
deleted file mode 100644
index daf4cffde4584f5fa9d6a02ac249218d3c2a6506..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tFilter.cc
+++ /dev/null
@@ -1,302 +0,0 @@
-//# tFilter.cc: Test program for class Filter
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Filter.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {
- // Define start time 0.5 (= 3 - 0.5*5) and time interval 5.
- info().init (ncorr, nchan, ntime, 0.5, 5., string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanWidth(nchan, 100000.);
- Vector<double> chanFreqs(nchan);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2);
- buf.setExposure (0.1*(itsCount+1));
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- buf.setData (data);
- Cube<float> weights(data.shape());
- buf.setWeights (weights);
- indgen (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- // Set part of the flags to another value.
- flags(IPosition(3,0), flags.shape()-1, IPosition(3,1,3,4)) = !itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // Assume they are averaged for 2 chan, 2 time.
- Cube<bool> fullResFlags;
- if (itsNCorr == 4) {
- fullResFlags = flags.copy().reform(IPosition(3,2*itsNChan,2,itsNBl));
- } else {
- fullResFlags.resize (IPosition(3, itsNChan, 1, itsNBl));
- fullResFlags = true;
- }
- buf.setFullResFlags (fullResFlags);
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, double(itsCount*100));
- buf.setUVW (uvw);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&)
- {
- // Use timeInterval=5
- info().init (itsNCorr, itsNChan, itsNTime, 100, 5, string(), string());
- // Define the frequencies.
- Vector<double> chanFreqs(itsNChan);
- Vector<double> chanWidth(itsNChan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result of averaging TestInput.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr,
- int nblout, int stchan, int nchanOut, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsNBlOut(nblout),
- itsStChan(stchan), itsNChanOut(nchanOut),
- itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // Fill expected result in similar way as TestInput.
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- Cube<float> weights(data.shape());
- indgen (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- // Set part of the flags to another value.
- flags(IPosition(3,0), flags.shape()-1, IPosition(3,1,3,4)) = !itsFlag;
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // Assume they are averaged for 2 chan, 2 time.
- Cube<bool> fullResFlags;
- if (itsNCorr == 4) {
- fullResFlags = flags.copy().reform(IPosition(3,2*itsNChan,2,itsNBl));
- } else {
- fullResFlags.resize (IPosition(3, itsNChan, 1, itsNBl));
- fullResFlags = true;
- }
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, double(itsCount*100));
- Slicer slicer(IPosition(3,0,itsStChan,0),
- IPosition(3,itsNCorr,itsNChanOut,itsNBlOut));
- // Check the expected result.
- ASSERT (allEQ(buf.getData(), data(slicer)));
- ASSERT (allEQ(buf.getFlags(), flags(slicer)));
- ASSERT (allEQ(buf.getWeights(), weights(slicer)));
- ASSERT (allEQ(buf.getUVW(), uvw(IPosition(2,0,0),
- IPosition(2,2,itsNBlOut-1))));
- if (itsNCorr == 4) {
- ASSERT (allEQ(buf.getFullResFlags(),
- fullResFlags(Slicer(IPosition(3,itsStChan*2,0,0),
- IPosition(3,2*itsNChanOut,2,itsNBlOut)))));
- } else {
- ASSERT (allEQ(buf.getFullResFlags(),
- fullResFlags(Slicer(IPosition(3,itsStChan,0,0),
- IPosition(3,itsNChanOut,1,itsNBlOut)))));
- }
- ASSERT (near(buf.getTime(), itsCount*5.+2));
- ASSERT (near(buf.getExposure(), 0.1*(itsCount+1)));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNChan);
- ASSERT (int(info.nchan())==itsNChanOut);
- ASSERT (int(info.nbaselines())==itsNBlOut);
- ASSERT (int(info.ntime())==itsNTime);
- ASSERT (info.timeInterval()==5.);
- ASSERT (int(info.nchanAvg())==1);
- ASSERT (int(info.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr, itsNBlOut, itsStChan, itsNChanOut;
- bool itsFlag;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test filtering of channels only.
-void test1(int ntime, int nbl, int nchan, int ncorr,
- int startchan, int nchanout, bool flag)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " startchan=" << startchan
- << " nchanout=" << nchanout << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("startchan", toString(startchan));
- parset.add ("nchan", toString(nchanout)+"+nchan-nchan");
- DPStep::ShPtr step2(new Filter(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr,
- nbl, startchan, nchanout, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Test filtering of baselines and channels.
-void test2(int ntime, int nbl, int nchan, int ncorr,
- int startchan, int nchanout, bool flag)
-{
- ASSERT (nbl<=4); // otherwise baseline selection removes more than the first
- cout << "test2: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " startchan=" << startchan
- << " nchanout=" << nchanout << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("startchan", toString(startchan)+"+nchan-nchan");
- parset.add ("nchan", toString(nchanout));
- // This removes the first baseline.
- parset.add ("baseline", "[[rs01.s01,rs*]]");
- DPStep::ShPtr step2(new Filter(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr,
- 2, startchan, nchanout, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-
-int main()
-{
- try {
- test1(10, 3, 32, 4, 2, 24, false);
- test1(10, 10, 30, 1, 3, 3, true);
- test1(10, 10, 1, 4, 0, 1, true);
- test2(10, 4, 32, 4, 2, 24, false);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tGainCal.run b/CEP/DP3/DPPP/test/tGainCal.run
deleted file mode 100755
index d1b549cbbcbc194c81526808f66fd9434142c288..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tGainCal.run
+++ /dev/null
@@ -1,120 +0,0 @@
-#!/bin/bash
-
-# Get the taql executable and srcdir (script created by cmake's CONFIGURE_FILE).
-source findenv.run_script
-echo "srcdirx=$rt_srcdir"
-
-# Set srcdir if not defined (in case run by hand).
-if test "$srcdir" = ""; then
- srcdir="$rt_srcdir"
-fi
-
-if test ! -f ${srcdir}/tNDPPP-generic.in_MS.tgz; then
- exit 3 # untested
-fi
-
-set -e # Stop on any error
-
-rm -rf tGainCal_tmp
-mkdir -p tGainCal_tmp
-# Unpack the MS and other files and do the DPPP run.
-cd tGainCal_tmp
-tar zxf ${srcdir}/tNDPPP-generic.in_MS.tgz
-tar zxf ${srcdir}/tGainCal.tab.tgz
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-
-echo "Creating MODEL_DATA so that residual can be computed"
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=MODEL_DATA steps=[predict] predict.sourcedb=tNDPPP-generic.MS/sky predict.usebeammodel=false
-
-echo; echo "Test caltype=diagonal"; echo
-NDPPP msin=tNDPPP-generic.MS msout= steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-diagonal gaincal.usebeammodel=false gaincal.caltype=diagonal gaincal.propagatesolutions=true gaincal.solint=1
-
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-diagonal
-
-echo "Comparing the bbs residual with the dppp residual (solutions will not be equal, but residual should be equal). This avoids issues with local minima."
-$taqlexe 'select from (select gsumsqr(sumsqr(abs(iif(t1.FLAG,0,t1.DPPP_DIAGONAL-t1.MODEL_DATA)))) as dpppres, gsumsqr(sumsqr(abs(iif(FLAG,0,t2.BBS_DIAGONAL-t1.MODEL_DATA)))) as bbsres from tNDPPP-generic.MS t1, tGainCal.tab t2) where dpppres>bbsres*1.02' > taql.out
-diff taql.out taql.ref || exit 1
-echo "Checking that not everything was flagged"
-$taqlexe 'select from tNDPPP-generic.MS where all(FLAG) groupby true having gcount()>100' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=diagonal with timeslotsperparmupdate=4"; echo
-NDPPP msin=tNDPPP-generic.MS msout= steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-diagonal-tpp gaincal.usebeammodel=false gaincal.caltype=diagonal gaincal.solint=4 gaincal.timeslotsperparmupdate=1 gaincal.propagatesolutions=false
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_TPP steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-diagonal-tpp
-$taqlexe 'select from tNDPPP-generic.MS where not all(near(DPPP_DIAGONAL, DPPP_DIAGONAL_TPP))'
-
-echo "Comparing the difference between applying with timeslotsperparmupdate = default and timeslotsperparmupdate=1"
-$taqlexe 'select from (select gsumsqr(sumsqr(abs(iif(t1.FLAG,0,t1.DPPP_DIAGONAL-t1.MODEL_DATA)))) as dpppres, gsumsqr(sumsqr(abs(iif(FLAG,0,t2.BBS_DIAGONAL-t1.MODEL_DATA)))) as bbsres from tNDPPP-generic.MS t1, tGainCal.tab t2) where dpppres>bbsres*1.02' > taql.out
-diff taql.out taql.ref || exit 1
-echo "Checking that not everything was flagged"
-$taqlexe 'select from tNDPPP-generic.MS where all(FLAG) groupby true having gcount()>100' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=fulljones"; echo
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_FULLJONES_GAINCAL steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-fulljones gaincal.usebeammodel=false gaincal.caltype=fulljones gaincal.solint=1 gaincal.applysolution=true
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_FULLJONES steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-fulljones
-
-echo "Comparing the bbs residual with the dppp residual (solutions will not be equal, but residual should be equal). This avoids issues with local minima."
-$taqlexe 'select from (select gsumsqr(sumsqr(abs(iif(t1.FLAG,0,t1.DPPP_FULLJONES-t1.MODEL_DATA)))) as dpppres, gsumsqr(sumsqr(abs(iif(FLAG,0,t2.BBS_FULLJONES-t1.MODEL_DATA)))) as bbsres from tNDPPP-generic.MS t1, tGainCal.tab t2) where dpppres>bbsres*1.02' > taql.out
-diff taql.out taql.ref || exit 1
-echo "Comparing the solutions from gaincal + applycal with gaincal directly"
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DPPP_FULLJONES ~= DPPP_FULLJONES))' > taql.out
-diff taql.out taql.ref || exit 1
-echo "Checking that not everything was flagged"
-$taqlexe 'select from tNDPPP-generic.MS where all(FLAG) groupby true having gcount()>100' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=diagonal, nchan=2"; echo
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN_GAINCAL steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan gaincal.usebeammodel=false gaincal.caltype=diagonal gaincal.solint=4 gaincal.nchan=2 gaincal.applysolution=true
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan
-
-echo "Comparing the bbs residual with the dppp residual (solutions will not be equal, but residual should be equal). This avoids issues with local minima."
-$taqlexe 'select from (select gsumsqr(sumsqr(abs(iif(t1.FLAG,0,t1.DPPP_DIAGONAL_NCHAN-t1.MODEL_DATA)))) as dpppres, gsumsqr(sumsqr(abs(iif(FLAG,0,t2.BBS_DIAGONAL_NCHAN-t1.MODEL_DATA)))) as bbsres from tNDPPP-generic.MS t1, tGainCal.tab t2) where dpppres>bbsres*1.02' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo "Comparing the solutions from gaincal + applycal with gaincal directly"
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DPPP_DIAGONAL_NCHAN_GAINCAL ~= DPPP_DIAGONAL_NCHAN))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo "Checking that not everything was flagged"
-$taqlexe 'select from tNDPPP-generic.MS where all(FLAG) groupby true having gcount()>100' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=diagonal, nchan=2, solint=7"; echo
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN_7_GAINCAL steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan gaincal.usebeammodel=false gaincal.caltype=diagonal gaincal.solint=4 gaincal.nchan=2 gaincal.applysolution=true
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN_7 steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan
-
-echo "Comparing the solutions from gaincal + applycal with gaincal directly"
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DPPP_DIAGONAL_NCHAN_7_GAINCAL ~= DPPP_DIAGONAL_NCHAN_7))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=tec"; echo
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_TEC steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-tec gaincal.caltype=tec gaincal.solint=2
-# For now, only testing that the right parameter names are in the output
-echo " select result of 1 rows" > taql1.ref
-$taqlexe 'select from tNDPPP-generic.MS/inst-tec where (select NAME from ::NAMES)[NAMEID]=="TEC:CS001HBA0"' > taql.out
-diff taql.out taql1.ref || exit 1
-$taqlexe 'select from tNDPPP-generic.MS/inst-tec where (select NAME from ::NAMES)[NAMEID]=="CommonScalarPhase:CS001HBA0"' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=tecandphase"; echo
-NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_TEC steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-tecandphase gaincal.caltype=tecandphase gaincal.solint=2
-# For now, only testing that the right parameter names are in the output
-echo " select result of 1 rows" > taql1.ref
-$taqlexe 'select from tNDPPP-generic.MS/inst-tecandphase where (select NAME from ::NAMES)[NAMEID]=="TEC:CS001HBA0"' > taql.out
-diff taql.out taql1.ref || exit 1
-$taqlexe 'select from tNDPPP-generic.MS/inst-tecandphase where (select NAME from ::NAMES)[NAMEID]=="CommonScalarPhase:CS001HBA0"' > taql.out
-diff taql.out taql1.ref || exit 1
-
-echo; echo "Test filter"; echo
-NDPPP msin=tNDPPP-generic.MS msout=tNDPPP-filtered.MS steps=[filter,gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-filter filter.baseline='!CS001HBA0&&*' gaincal.baseline='!CS002HBA1,RS305HBA&&*' gaincal.caltype=diagonal
-$taqlexe 'select from tNDPPP-generic.MS/inst-filter::NAMES where NAME LIKE "CS001HBA0%" OR NAME LIKE "%CS002HBA1%" OR NAME LIKE "%RS305HBA%"' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test debug output"; echo
-NDPPP msin=tNDPPP-generic.MS msout=. numthreads=1 steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-debug gaincal.caltype=diagonal gaincal.debuglevel=1
-[[ -f debug.h5 ]]
-diff taql.out taql.ref || exit 1
-
diff --git a/CEP/DP3/DPPP/test/tGainCal.sh b/CEP/DP3/DPPP/test/tGainCal.sh
deleted file mode 100755
index 17ddb87fa0e8d572e6a065437ad79010d7252954..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tGainCal.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tGainCal
diff --git a/CEP/DP3/DPPP/test/tGainCal.tab.tgz b/CEP/DP3/DPPP/test/tGainCal.tab.tgz
deleted file mode 100644
index 2b93f5b9d5af30ffc0d38ec677d6e1108382aab2..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP/test/tGainCal.tab.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP/test/tGainCalH5Parm.run b/CEP/DP3/DPPP/test/tGainCalH5Parm.run
deleted file mode 100755
index 6799152789ce0a560ac73bd08c2cbe42f74f788a..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tGainCalH5Parm.run
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/bin/bash
-
-# Get the taql executable and srcdir (script created by cmake's CONFIGURE_FILE).
-source findenv.run_script
-echo "srcdirx=$rt_srcdir"
-
-# Set srcdir if not defined (in case run by hand).
-if test "$srcdir" = ""; then
- srcdir="$rt_srcdir"
-fi
-
-if test ! -f ${srcdir}/tNDPPP-generic.in_MS.tgz; then
- exit 3 # untested
-fi
-
-set -e # Stop on any error
-
-rm -rf tGainCalH5Parm_tmp
-mkdir -p tGainCalH5Parm_tmp
-# Unpack the MS and other files and do the DPPP run.
-cd tGainCalH5Parm_tmp
-tar zxf ${srcdir}/tNDPPP-generic.in_MS.tgz
-tar zxf ${srcdir}/tGainCal.tab.tgz
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-
-echo "Creating MODEL_DATA so that residual can be computed"
-cmd='NDPPP checkparset=1 showprogress=false msin=tNDPPP-generic.MS msout=. msout.datacolumn=MODEL_DATA steps=[predict] predict.sourcedb=tNDPPP-generic.MS/sky predict.usebeammodel=false'
-echo $cmd
-$cmd
-
-echo; echo "Test caltype=diagonal"; echo
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout= steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-diagonal.h5 gaincal.usebeammodel=false gaincal.caltype=diagonal gaincal.propagatesolutions=true gaincal.solint=1'
-echo $cmd
-$cmd
-
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-diagonal.h5 applycal.steps=[amplitude,phase] applycal.phase.correction=phase000 applycal.amplitude.correction=amplitude000 applycal.amplitude.correction=amplitude000'
-echo $cmd
-$cmd
-
-echo "Comparing the bbs residual with the dppp residual (solutions will not be equal, but residual should be equal). This avoids issues with local minima."
-$taqlexe 'select from (select gsumsqr(sumsqr(abs(iif(t1.FLAG,0,t1.DPPP_DIAGONAL-t1.MODEL_DATA)))) as dpppres, gsumsqr(sumsqr(abs(iif(FLAG,0,t2.BBS_DIAGONAL-t1.MODEL_DATA)))) as bbsres from tNDPPP-generic.MS t1, tGainCal.tab t2) where dpppres>bbsres*1.02' > taql.out
-diff taql.out taql.ref || exit 1
-echo "Checking that not everything was flagged"
-$taqlexe 'select from tNDPPP-generic.MS where all(FLAG) groupby true having gcount()>100' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=fulljones"; echo
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_FULLJONES_GAINCAL steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-fulljones.h5 gaincal.usebeammodel=false gaincal.caltype=fulljones gaincal.solint=1 gaincal.applysolution=true'
-echo $cmd
-$cmd
-
-echo; echo "Test caltype=diagonal, nchan=2"; echo
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN_GAINCAL steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan.h5 gaincal.usebeammodel=false gaincal.caltype=diagonal gaincal.solint=4 gaincal.nchan=2 gaincal.applysolution=true'
-echo $cmd
-$cmd
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan.h5 applycal.steps=[phase,amplitude] applycal.phase.correction=phase000 applycal.amplitude.correction=amplitude000'
-echo $cmd
-$cmd
-
-echo "Comparing the bbs residual with the dppp residual (solutions will not be equal, but residual should be equal). This avoids issues with local minima."
-$taqlexe 'select from (select gsumsqr(sumsqr(abs(iif(t1.FLAG,0,t1.DPPP_DIAGONAL_NCHAN-t1.MODEL_DATA)))) as dpppres, gsumsqr(sumsqr(abs(iif(FLAG,0,t2.BBS_DIAGONAL_NCHAN-t1.MODEL_DATA)))) as bbsres from tNDPPP-generic.MS t1, tGainCal.tab t2) where dpppres>bbsres*1.02' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo "Comparing the solutions from gaincal + applycal with gaincal directly"
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DPPP_DIAGONAL_NCHAN_GAINCAL ~= DPPP_DIAGONAL_NCHAN))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo "Checking that not everything was flagged"
-$taqlexe 'select from tNDPPP-generic.MS where all(FLAG) groupby true having gcount()>100' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=diagonal, nchan=2, solint=7"; echo
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN_7_GAINCAL steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan7.h5 gaincal.usebeammodel=false gaincal.caltype=diagonal gaincal.solint=4 gaincal.nchan=2 gaincal.applysolution=true'
-echo $cmd
-$cmd
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_DIAGONAL_NCHAN_7 steps=[applycal] applycal.parmdb=tNDPPP-generic.MS/inst-diagonal-nchan7.h5 applycal.steps=[amplitude,phase] applycal.amplitude.correction=amplitude000 applycal.phase.correction=phase000'
-echo $cmd
-$cmd
-
-echo "Comparing the solutions from gaincal + applycal with gaincal directly"
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DPPP_DIAGONAL_NCHAN_7_GAINCAL ~= DPPP_DIAGONAL_NCHAN_7))' > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test caltype=tec"; echo
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_TEC steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-tec.h5 gaincal.caltype=tec gaincal.solint=2'
-echo $cmd
-$cmd
-
-echo; echo "Test caltype=tecandphase"; echo
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=. msout.datacolumn=DPPP_TEC steps=[gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-tecandphase.h5 gaincal.caltype=tecandphase gaincal.solint=2'
-echo $cmd
-$cmd
-
-echo; echo "Test filter"; echo
-cmd='NDPPP checkparset=1 msin=tNDPPP-generic.MS msout=tNDPPP-filtered.MS steps=[filter,gaincal] gaincal.sourcedb=tNDPPP-generic.MS/sky gaincal.parmdb=tNDPPP-generic.MS/inst-filter.h5 filter.baseline="!CS001HBA0&&*" gaincal.baseline="!CS002HBA1,RS305HBA&&*" gaincal.caltype=diagonal'
-echo $cmd
-$cmd
-
diff --git a/CEP/DP3/DPPP/test/tGainCalH5Parm.sh b/CEP/DP3/DPPP/test/tGainCalH5Parm.sh
deleted file mode 100755
index 6fc019f8b380e3ab806ed38809cbb372fec01a54..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tGainCalH5Parm.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tGainCalH5Parm
diff --git a/CEP/DP3/DPPP/test/tGainCal_ref b/CEP/DP3/DPPP/test/tGainCal_ref
deleted file mode 100755
index 1c1549410ad8aa29f4cab9c03840f1f70a5040e0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tGainCal_ref
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/bin/bash
-
-# Generate reference data tGainCal.tab for tGainCal
-# This script uses BBS. Note that casacore data should be up-to-date
-# Run the script in the source directory of CEP/DP3/DPPP/test
-
-rm -rf tNDPPP-generic.MS
-tar xf tNDPPP-generic.in_MS.tgz
-
-#### Save instrument table with only defaultvalues
-cp -r tNDPPP-generic.MS/instrument tNDPPP-generic.MS/definstrument
-
-#### DIAGONAL
-
-cat > tGainCal-bbs-diagonal.parset <<EOF
-Strategy.ChunkSize = 0
-Strategy.Steps = [solve,correct]
-
-Step.solve.Operation = SOLVE
-Step.solve.Model.Sources = []
-Step.solve.Model.Beam.Enable = F
-Step.solve.Model.Beam.UseChannelFreq = T
-Step.solve.Model.Gain.Enable = T
-Step.solve.Solve.Parms = ["Gain:0:0:*","Gain:1:1:*"]
-Step.solve.Solve.CellSize.Freq = 0
-Step.solve.Solve.CellSize.Time = 4
-Step.solve.Solve.CellChunkSize = 10
-Step.solve.Solve.Options.UseSVD = T
-
-Step.correct.Operation = CORRECT
-Step.correct.Model.Gain.Enable = T
-Step.correct.Output.Column = BBS_DIAGONAL
-EOF
-
-bbs-reducer tNDPPP-generic.MS tGainCal-bbs-diagonal.parset
-
-rm tGainCal-bbs-diagonal.parset
-
-#### Restore instrument table with only defaultvalues
-rm -rf tNDPPP-generic.MS/instrument
-cp -r tNDPPP-generic.MS/definstrument tNDPPP-generic.MS/instrument
-
-#### DIAGONAL-NCHAN
-
-cat > tGainCal-bbs-diagonal-nchan.parset <<EOF
-Strategy.ChunkSize = 0
-Strategy.Steps = [solve,correct]
-
-Step.solve.Operation = SOLVE
-Step.solve.Model.Sources = []
-Step.solve.Model.Beam.Enable = F
-Step.solve.Model.Beam.UseChannelFreq = T
-Step.solve.Model.Gain.Enable = T
-Step.solve.Solve.Parms = ["Gain:0:0:*","Gain:1:1:*"]
-Step.solve.Solve.CellSize.Freq = 2
-Step.solve.Solve.CellSize.Time = 4
-Step.solve.Solve.CellChunkSize = 10
-Step.solve.Solve.Options.UseSVD = T
-
-Step.correct.Operation = CORRECT
-Step.correct.Model.Gain.Enable = T
-Step.correct.Output.Column = BBS_DIAGONAL_NCHAN
-EOF
-
-bbs-reducer tNDPPP-generic.MS tGainCal-bbs-diagonal-nchan.parset
-
-rm tGainCal-bbs-diagonal-nchan.parset
-
-#### Restore instrument table with only defaultvalues
-rm -rf tNDPPP-generic.MS/instrument
-cp -r tNDPPP-generic.MS/definstrument tNDPPP-generic.MS/instrument
-
-#### FULLJONES
-
-cat > tGainCal-bbs-fulljones.parset <<EOF
-Strategy.ChunkSize = 0
-Strategy.Steps = [solve,correct]
-
-Step.solve.Operation = SOLVE
-Step.solve.Model.Sources = []
-Step.solve.Model.Gain.Enable = T
-Step.solve.Model.Beam.Enable = F
-Step.solve.Model.Beam.UseChannelFreq = T
-Step.solve.Solve.Parms = ["Gain:0:0:*","Gain:1:1:*","Gain:0:1:*","Gain:1:0:*"]
-Step.solve.Solve.CellSize.Freq = 0
-Step.solve.Solve.CellSize.Time = 1
-Step.solve.Solve.CellChunkSize = 10
-Step.solve.Solve.Options.UseSVD = T
-
-Step.correct.Operation = CORRECT
-Step.correct.Model.Gain.Enable = T
-Step.correct.Output.Column = BBS_FULLJONES
-EOF
-
-bbs-reducer tNDPPP-generic.MS tGainCal-bbs-fulljones.parset
-
-rm tGainCal-bbs-fulljones.parset
-
-#### Restore instrument table with only defaultvalues
-rm -rf tNDPPP-generic.MS/instrument
-cp -r tNDPPP-generic.MS/definstrument tNDPPP-generic.MS/instrument
-
-
-
-#### Store output from BBS in separate table
-
-taql 'select from (select BBS_DIAGONAL, BBS_DIAGONAL_NCHAN, BBS_FULLJONES from tNDPPP-generic.MS giving tGainCal.tab as plain)'
-
-tar czf tGainCal.tab.tgz tGainCal.tab
diff --git a/CEP/DP3/DPPP/test/tGridInterpolate.cc b/CEP/DP3/DPPP/test/tGridInterpolate.cc
deleted file mode 100644
index 02a5814d9f453d50fa3345848ab730a52c8ff129..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tGridInterpolate.cc
+++ /dev/null
@@ -1,64 +0,0 @@
-//# tGridInterpolate.cc: test program for GridInterpolate
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tGridInterpolate.cc 31423 2015-04-03 14:06:21Z dijkema $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/GridInterpolate.h>
-#include <cassert>
-
-using namespace LOFAR;
-using namespace std;
-
-int main() {
- vector<double> ax_src = {1,3};
- vector<double> ax_tgt = {0.5, 1.5, 2.5, 3.5};
-
- vector<size_t> indices;
- getAxisIndices(ax_src, ax_tgt, indices);
- assert(indices.size()==ax_tgt.size());
- assert(indices[0]==0 && indices[1]==0 && indices[2]==1 && indices[3]==1);
-
- vector<double> x_src = {2,4,8,10};
- vector<double> y_src = {3,6,12};
- vector<double> x_tgt = {1,3.5,9.5,10};
- vector<double> y_tgt = {4,10};
- vector<double> vals_src(x_src.size()*y_src.size());
- vector<double> vals_tgt(x_tgt.size()*y_tgt.size());
- for (size_t i=0; i<vals_src.size(); ++i) {
- vals_src[i] = i;
- }
-
- getAxisIndices(x_src, x_tgt, indices);
- assert(indices.size() == x_tgt.size());
- assert(indices[0]==0 && indices[1]==1 && indices[2]==3 && indices[3]==3);
-
- gridNearestNeighbor(x_src, y_src, x_tgt, y_tgt, vals_src.data(), vals_tgt.data());
-
- assert(vals_tgt[0] == vals_src[0]);
- assert(vals_tgt[1] == vals_src[2]);
- assert(vals_tgt[2] == vals_src[3]);
- assert(vals_tgt[3] == vals_src[5]);
- assert(vals_tgt[4] == vals_src[9]);
- assert(vals_tgt[5] == vals_src[11]);
- assert(vals_tgt[6] == vals_src[9]);
- assert(vals_tgt[7] == vals_src[11]);
-}
diff --git a/CEP/DP3/DPPP/test/tH5Parm.cc b/CEP/DP3/DPPP/test/tH5Parm.cc
deleted file mode 100644
index 886369b302fe40eb58a244b25b835d791e6d53f1..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tH5Parm.cc
+++ /dev/null
@@ -1,198 +0,0 @@
-#include <DPPP/H5Parm.h>
-#include <iostream>
-#include <sstream>
-#include <stdio.h>
-#include <vector>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/BasicMath/Math.h>
-
-using namespace std;
-using namespace LOFAR;
-
-void checkAxes(H5Parm::SolTab& soltab, size_t ntimes) {
- ASSERT(soltab.nAxes()==3);
- ASSERT(soltab.hasAxis("ant"));
- ASSERT(soltab.hasAxis("time"));
- ASSERT(soltab.hasAxis("bla"));
- ASSERT(soltab.getAxis(0).name=="ant");
- ASSERT(soltab.getAxis(1).name=="time");
- ASSERT(soltab.getAxis(2).name=="bla");
- ASSERT(soltab.getAxis(0).size==3);
- ASSERT(soltab.getAxis(1).size==ntimes);
- ASSERT(soltab.getAxis(2).size==1);
-}
-
-int main(int, char**) {
- {
- size_t ntimes=7;
- {
- // Create a new H5Parm
- cout<<"Create tH5Parm_tmp.h5"<<endl;
- H5Parm h5parm("tH5Parm_tmp.h5", true);
-
- // Check that something is created
- ASSERT(((H5::H5File&)(h5parm)).getNumObjs()==1);
-
- // Check the name of the new solset "sol000"
- ASSERT(h5parm.getSolSetName()=="sol000");
-
- // Add some metadata
- vector<string> antNames;
- vector<double> oneAntPos(3);
- vector<vector<double> > antPositions;
- for (uint i=0; i<5; ++i) {
- stringstream antNameStr;
- antNameStr<<"Antenna"<<i;
- antNames.push_back(antNameStr.str());
- antPositions.push_back(oneAntPos);
- }
- h5parm.addAntennas(antNames, antPositions);
-
- vector<H5Parm::AxisInfo> axes;
- axes.push_back(H5Parm::AxisInfo("ant",3));
- axes.push_back(H5Parm::AxisInfo("time",ntimes));
- axes.push_back(H5Parm::AxisInfo("bla",1));
-
- cout<<"Create new SolTab"<<endl;
- H5Parm::SolTab a = h5parm.createSolTab("mysol","mytype",axes);
-
- // Check that the soltab exists
- ASSERT(h5parm.nSolTabs() == 1);
- ASSERT(h5parm.hasSolTab("mysol"));
-
- // Check the axes
- H5Parm::SolTab soltab = h5parm.getSolTab("mysol");
- ASSERT(soltab.getType()=="mytype");
- checkAxes(soltab, ntimes);
-
- // Add some data
- vector<double> vals(3*ntimes);
- vector<double> weights(3*ntimes);
- for (size_t ant=0; ant<3; ++ant) {
- for (size_t time=0; time<ntimes; ++time) {
- vals[ant*ntimes+time]=10*ant+time;
- weights[ant*ntimes+time]=0.4;
- }
- }
-
- soltab.setValues(vals, weights, "CREATE with DPPP");
-
- // Add metadata for stations
- vector<string> someAntNames;
- someAntNames.push_back("Antenna1");
- someAntNames.push_back("Antenna2");
- someAntNames.push_back("Antenna3");
- soltab.setAntennas(someAntNames);
-
- // Add metadata for times
- vector<double> times;
- for (size_t time=0; time<ntimes; ++time) {
- times.push_back(57878.5+2.0*time);
- }
- soltab.setTimes(times);
-
- // Add metadata for freqs;
- vector<double> freqs;
- freqs.push_back(130e6);
- freqs.push_back(131e6);
- freqs.push_back(135e6);
- freqs.push_back(137e6);
- soltab.setFreqs(freqs);
- }
-
- {
- cout<<"opening tH5Parm_tmp.h5 again, force a new soltab"<<endl;
- H5Parm h5parm("tH5Parm_tmp.h5", false, true);
- ASSERT(h5parm.getSolSetName()=="sol001");
- }
-
- {
- cout<<"opening tH5Parm_tmp.h5 again, force a new solset with name"<<endl;
- H5Parm h5parm("tH5Parm_tmp.h5", false, true, "harry");
- ASSERT(h5parm.getSolSetName()=="harry");
- }
-
- {
- cout<<"opening tH5Parm_tmp.h5 again, read existing soltab"<<endl;
- H5Parm h5parm("tH5Parm_tmp.h5", false, false, "sol000");
- ASSERT(h5parm.getSolSetName()=="sol000");
- ASSERT(h5parm.nSolTabs() == 1);
- ASSERT(h5parm.hasSolTab("mysol"));
- ASSERT(!h5parm.hasSolTab("nonexistingsol"));
-
- // Check the axes
- H5Parm::SolTab soltab = h5parm.getSolTab("mysol");
- ASSERT(soltab.getType()=="mytype");
- checkAxes(soltab, ntimes);
-
- cout<<"read some data"<<endl;
- double starttime = 57878.49999;
- hsize_t starttimeindex = soltab.getTimeIndex(starttime);
- cout<<"starttimeindex="<<starttimeindex<<endl;
- vector<double> val = soltab.getValues("Antenna2", starttimeindex, ntimes);
- ASSERT(casacore::near(val[0],10.));
- ASSERT(casacore::near(val[1],11.));
- ASSERT(casacore::near(val[2],12.));
- ASSERT(casacore::near(val[3],13.));
- cout<<"read some data with stride 2"<<endl;
- starttime = 57880.5;
- starttimeindex = soltab.getTimeIndex(starttime);
- ASSERT(starttimeindex==1);
- vector<double> val2 = soltab.getValues("Antenna3", starttimeindex, 2, 2);
- ASSERT(casacore::near(val2[0],21.));
- ASSERT(casacore::near(val2[1],23.));
- cout<<"testing stride"<<endl;
- ASSERT(casacore::near(soltab.getTimeInterval(),2.));
- cout<<"reading the antennas into a vector"<<endl;
- vector<string> antennas = soltab.getStringAxis("ant");
- ASSERT(antennas.size()==3);
- ASSERT(antennas[0]=="Antenna1");
- ASSERT(antennas[1]=="Antenna2");
- ASSERT(antennas[2]=="Antenna3");
- cout<<"Check frequency widths"<<endl;
- ASSERT(casacore::near(soltab.getFreqInterval(0),1e6));
- ASSERT(casacore::near(soltab.getFreqInterval(1),4e6));
- ASSERT(casacore::near(soltab.getFreqInterval(2),2e6));
-
- cout<<"Checking interpolation (on input time axis)"<<endl;
- vector<double> freqs;
- freqs.push_back(130e6);
- freqs.push_back(131e6);
-
- vector<double> times;
- for (size_t time=0; time<ntimes; ++time) {
- times.push_back(57878.5+2.0*time);
- }
-
- vector<double> newgridvals = soltab.getValuesOrWeights("val", "Antenna1",
- times, freqs, 0, 0);
- ASSERT(newgridvals.size() == times.size() * freqs.size());
- size_t idx=0;
- for (size_t time=0; time<times.size(); ++time) {
- for (size_t freq=0; freq<freqs.size(); ++freq) {
- ASSERT(casacore::near(newgridvals[idx++], double(time)));
- }
- }
-
- times.clear();
- cout<<"Checking interpolation, upsampled 3 times, add 2 time slots at end"<<endl;
- for (size_t time=0; time<3*ntimes+2; ++time) {
- times.push_back(57878.5+2.0*time/3.);
- }
- newgridvals = soltab.getValuesOrWeights("val", "Antenna1",
- times, freqs, 0, 0);
- ASSERT(newgridvals.size() == times.size() * freqs.size());
- idx=0;
- for (int time=0; time<int(times.size()); ++time) {
- for (size_t freq=0; freq<freqs.size(); ++freq) {
- ASSERT(casacore::near(newgridvals[idx++], min(double((time+1)/3),double(ntimes-1))));
- }
- }
-
- }
- // Remove the file
-// remove("tH5Parm_tmp.h5");
- }
-
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tH5Parm.sh b/CEP/DP3/DPPP/test/tH5Parm.sh
deleted file mode 100755
index dacccfa6967e51e30a9e9464f4f981f8c9fa7472..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tH5Parm.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-rm -f tH5Parm_tmp.h5
-./runctest.sh tH5Parm
diff --git a/CEP/DP3/DPPP/test/tMedFlagger.cc b/CEP/DP3/DPPP/test/tMedFlagger.cc
deleted file mode 100644
index ffe71936cc8b681e9dbc7b198d6cfe8cf08836aa..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMedFlagger.cc
+++ /dev/null
@@ -1,291 +0,0 @@
-//# tMedFlagger.cc: Test program for class MedFlagger
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/MedFlagger.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nant, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nant*(nant+1)/2), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- // Use timeInterval=5
- info().init (itsNCorr, itsNChan, itsNTime, 100, 5, string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(itsNBl);
- Vector<Int> ant2(itsNBl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<itsNBl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanFreqs(itsNChan);
- Vector<double> chanWidth(itsNChan, 100000);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nant, int nchan, int ncorr,
- bool flag, bool useAutoCorr, bool shortbl)
- : itsCount(0), itsNTime(ntime), itsNBl(nant*(nant+1)/2), itsNChan(nchan),
- itsNCorr(ncorr),
- itsFlag(flag),
- itsUseAutoCorr(useAutoCorr),
- itsShortBL(shortbl)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // Fill expected result in similar way as TestInput.
- Cube<Complex> result(itsNCorr,itsNChan,itsNBl);
- for (int i=0; i<int(result.size()); ++i) {
- result.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- // Check the result.
- ASSERT (allNear(real(buf.getData()), real(result), 1e-10));
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-10));
- // Check the flags.
- // If autocorrs are used, only the last channel is flagged, but the first
- // channel also for the first time stamp. Thus is only true for a limited
- // nr of baselines (thus do not use nant>2 in test2 with flag=false).
- // If short baselines are used, bl 2,3,7,8,12,13 are not flagged.
- // The others have length 0 or 144.
- Cube<bool> expFlag(itsNCorr,itsNChan,itsNBl);
- expFlag = itsFlag;
- if (itsUseAutoCorr) {
- for (int i=0; i<itsNBl; ++i) {
- if (!itsShortBL || !(i==2 || i==3 || i==7 || i==8 || i==12 || i==13)) {
- for (int j=0; j<itsNCorr; ++j) {
- expFlag(j,0,i) = itsFlag || itsCount==0;
- expFlag(j,itsNChan-1,i) = true;
- }
- }
- }
- }
- ASSERT (allEQ(buf.getFlags(), expFlag));
- ASSERT (near(buf.getTime(), 2+5.*itsCount));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNChan);
- ASSERT (int(info.nchan())==itsNChan);
- ASSERT (int(info.ntime())==itsNTime);
- ASSERT (info.timeInterval()==5);
- ASSERT (int(info.nchanAvg())==1);
- ASSERT (int(info.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag, itsUseAutoCorr, itsShortBL;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
- DPStep::ShPtr step = step1;
- while (step) {
- step->showCounts (cout);
- step = step->getNextStep();
- }
-}
-
-// Test simple flagging with or without preflagged points.
-void test1(int ntime, int nant, int nchan, int ncorr, bool flag, int threshold,
- bool shortbl)
-{
- cout << "test1: ntime=" << ntime << " nrant=" << nant << " nchan=" << nchan
- << " ncorr=" << ncorr << " threshold=" << threshold
- << " shortbl=" << shortbl << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nant, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("freqwindow", "1");
- parset.add ("timewindow", "1");
- parset.add ("threshold", toString(threshold));
- if (shortbl) {
- parset.add ("blmin", "0");
- parset.add ("blmax", "145");
- }
- DPStep::ShPtr step2(new MedFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nant, nchan, ncorr, flag, false,
- shortbl));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- step2->show (cout);
- execute (step1);
-}
-
-// Test applyautocorr flagging with or without preflagged points.
-void test2(int ntime, int nant, int nchan, int ncorr, bool flag, int threshold,
- bool shortbl)
-{
- cout << "test2: ntime=" << ntime << " nrant=" << nant << " nchan=" << nchan
- << " ncorr=" << ncorr << " threshold=" << threshold
- << " shortbl=" << shortbl << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nant, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("freqwindow", "3");
- parset.add ("timewindow", "min(1,max(1,bl))");
- parset.add ("threshold", toString(threshold));
- parset.add ("applyautocorr", "True");
- if (shortbl) {
- parset.add ("blmax", "145");
- }
- DPStep::ShPtr step2(new MedFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nant, nchan, ncorr, flag, true,
- shortbl));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-
-int main()
-{
- INIT_LOGGER ("tMedFlagger");
- try {
-
- for (uint i=0; i<2; ++i) {
- test1(10, 2, 32, 4, false, 1, i>0);
- test1(10, 5, 32, 4, true, 1, i>0);
- test2( 4, 2, 8, 4, false, 100, i>0);
- test2(10, 5, 32, 4, true, 1, i>0);
- test2( 4, 2, 8, 4, false, 100, i>0);
- }
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tMedFlagger.sh b/CEP/DP3/DPPP/test/tMedFlagger.sh
deleted file mode 100755
index 56257ab02b4637a1745c879d28dd5db20aa4be21..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMedFlagger.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tMedFlagger
diff --git a/CEP/DP3/DPPP/test/tMedian.cc b/CEP/DP3/DPPP/test/tMedian.cc
deleted file mode 100644
index 5d318ef9bd2beb3f0fbe29544badf3357a0d2249..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMedian.cc
+++ /dev/null
@@ -1,71 +0,0 @@
-//# tMedian.cc: Program to test performance of kthLargest and nth_element
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <casacore/casa/OS/Timer.h>
-#include <casacore/casa/Utilities/GenSort.h>
-#include <vector>
-#include <algorithm>
-#include <iostream>
-
-using namespace casacore;
-using namespace std;
-
-void testCasa (size_t sz, size_t n)
-{
- Timer timer;
- for (uint i=0; i<n; ++i) {
- vector<float> vec(sz);
- uint j=0;
- for (vector<float>::iterator iter=vec.begin(); iter!=vec.end(); ++iter) {
- *iter = j++;
- }
- GenSort<float>::kthLargest (&(vec[0]), sz, sz/2);
- }
- timer.show ("casa");
-}
-
-void testStl (size_t sz, size_t n)
-{
- Timer timer;
- for (uint i=0; i<n; ++i) {
- vector<float> vec(sz);
- uint j=0;
- for (vector<float>::iterator iter=vec.begin(); iter!=vec.end(); ++iter) {
- *iter = j++;
- }
- nth_element (vec.begin(), vec.begin()+sz/2, vec.end());
- }
- timer.show ("stl ");
-}
-
-int main (int argc, char* argv[])
-{
- size_t window = 155;
- size_t ntimes = 1000;
- if (argc > 1) window = atoi(argv[1]);
- if (argc > 2) ntimes = atoi(argv[2]);
- testCasa (window, ntimes);
- testStl (window, ntimes);
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tMedian.sh b/CEP/DP3/DPPP/test/tMedian.sh
deleted file mode 100755
index be461d0df19da54e2f515c9c04be030686e69315..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMedian.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tMedian
diff --git a/CEP/DP3/DPPP/test/tMirror.cc b/CEP/DP3/DPPP/test/tMirror.cc
deleted file mode 100644
index c32d9f091c9e6ac6fdd2a36e1258b4b6375d9eac..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMirror.cc
+++ /dev/null
@@ -1,171 +0,0 @@
-//# tMirror.cc: Test if the way of mirroring done in MedFlagger is fine
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <Common/LofarTypes.h>
-#include <Common/LofarLogger.h>
-
-#include <casacore/casa/Arrays/Matrix.h>
-#include <casacore/casa/Utilities/LinearSearch.h>
-#include <Common/StreamUtil.h>
-#include <Common/lofar_vector.h>
-
-using namespace casacore;
-using namespace std;
-using namespace LOFAR;
-
-void doChan (int windowSize, int nchan, int chan)
-{
- // At the beginning or end of the window the values are wrapped.
- // So we might need to move in two parts.
- int hw = windowSize/2;
- int s1 = chan - hw;
- int e1 = chan + hw + 1;
- int s2 = 1;
- int e2 = 1;
- if (s1 < 0) {
- e2 = -s1 + 1;
- s1 = 0;
- } else if (e1 > nchan) {
- s2 = nchan + nchan - e1 - 1; // e1-nchan+1 too far, so go back that amount
- e2 = nchan-1;
- e1 = nchan;
- }
- std::cout <<"wdw,nch=" << windowSize << ',' << nchan << " chan=" << chan
- << ' ' << s1 << '-' << e1 << ' ' << s2 << '-' << e2 << std::endl;
- ASSERT (e1-s1 + e2-s2 == windowSize);
-}
-
-void testAdd()
-{
- uint nrold = 5;
- uint nrnew = 7;
- vector<int> itsAnt1(15);
- vector<int> itsAnt2(15);
- int inx=0;
- for (int a1=0; a1<5; ++a1) {
- for (int a2=a1; a2<5; ++a2) {
- itsAnt1[inx] = a1;
- itsAnt2[inx] = a2;
- ++inx;
- }
- }
- vector<Vector<int> > itsParts(2);
- itsParts[0].resize (2);
- itsParts[0][0] = 0;
- itsParts[0][1] = 1;
- itsParts[1].resize (2);
- itsParts[1][0] = 3;
- itsParts[1][1] = 4;
-
-
- vector<int> newbl(nrnew);
- vector<vector<int> > itsBufRows;
- bool itsMakeAutoCorr = true;
- // Loop over the superstations.
- // Note that by making this the outer loop, the baselines between
- // superstations are also formed.
- // At the end the new baselines are added to itsAnt1 and itsAnt2.
- // itsBufRows contains for each new baseline the rownrs in the DPBuffer
- // to be added for the new baseline. If rownr<0, the conjugate has to be
- // added (1 is added to rownr, otherwise 0 is ambiguous).
- // Note that a rownr can be the rownr of a new baseline.
- for (uint j=0; j<itsParts.size(); ++j) {
- std::fill (newbl.begin(), newbl.end(), -1);
- vector<int> newAnt1;
- vector<int> newAnt2;
- // Loop through all baselines and find out if a baseline should
- // be used for a superstation.
- for (uint i=0; i<itsAnt1.size(); ++i) {
- bool havea1 = linearSearch1 (itsParts[j], itsAnt1[i]) >= 0;
- bool havea2 = linearSearch1 (itsParts[j], itsAnt2[i]) >= 0;
- int ant = nrold+j;
- int take = 0;
- if (havea1) {
- // If both stations are in same superstation, only use them
- // if it is an autocorrelation.
- if (havea2) {
- if (itsMakeAutoCorr && itsAnt1[i] == itsAnt2[i]) {
- take = 1;
- }
- } else {
- ant = itsAnt2[i];
- take = -1; // conjugate has to be added
- }
- } else if (havea2) {
- ant = itsAnt1[i];
- take = 1;
- }
- if (take != 0) {
- // We have a baseline for the superstation.
- // Get its index; create it if not used before.
- int blinx = newbl[ant];
- if (blinx < 0) {
- blinx = newbl[ant] = itsBufRows.size();
- itsBufRows.push_back (vector<int>());
- newAnt1.push_back (ant);
- newAnt2.push_back (nrold+j);
- }
- itsBufRows[blinx].push_back (take*(i+1));
- }
- }
- // Copy the new baselines for this superstation to the baseline list.
- // Give a warning if nothing found.
- if (newAnt1.empty()) {
- // DPLOG_WARN_STR ("StationAdder: no baseline found for superstation");
- cout << "StationAdder: no baseline found for superstation" << endl;
- } else {
- uint oldsz = itsAnt1.size();
- itsAnt1.resize (oldsz + newAnt1.size());
- itsAnt2.resize (oldsz + newAnt1.size());
- for (uint i=0; i<newAnt1.size(); ++i) {
- itsAnt1[oldsz+i] = newAnt1[i];
- itsAnt2[oldsz+i] = newAnt2[i];
- }
- }
- }
- cout << itsAnt1<<endl<<itsAnt2<<endl;
- writeVector (cout, itsBufRows);
- cout<<endl;
-}
-
-int main (int argc, char* argv[])
-{
- uint windowSize = 5;
- uint nchan = 8;
- if (argc > 1) {
- windowSize = atoi(argv[1]);
- }
- if (argc > 2) {
- nchan = atoi(argv[2]);
- }
- if (windowSize == 0) windowSize = 1;
- if (windowSize > nchan) windowSize = nchan;
- if (windowSize%2 == 0) windowSize--;
-
- for (uint i=0; i<nchan; ++i) {
- doChan (windowSize, nchan, i);
- }
-
- testAdd();
-}
diff --git a/CEP/DP3/DPPP/test/tMirror.sh b/CEP/DP3/DPPP/test/tMirror.sh
deleted file mode 100755
index 09a5523297bf25bd56b00de3e24dae11fcbe3e37..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMirror.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tMirror
diff --git a/CEP/DP3/DPPP/test/tMultiApplyCal.run b/CEP/DP3/DPPP/test/tMultiApplyCal.run
deleted file mode 100755
index 31762901986ee3d5555093f4f2774538b5cd84e9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMultiApplyCal.run
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-# Get the taql executable and srcdir (script created by cmake's CONFIGURE_FILE).
-source findenv.run_script
-echo "srcdirx=$rt_srcdir"
-
-# Set srcdir if not defined (in case run by hand).
-if test "$srcdir" = ""; then
- srcdir="$rt_srcdir"
-fi
-
-if test ! -f ${srcdir}/tNDPPP-generic.in_MS.tgz; then
- exit 3 # untested
-fi
-
-set -e # Stop on any error
-
-rm -rf tMultiApplyCal_tmp
-mkdir -p tMultiApplyCal_tmp
-# Unpack the MS and other files and do the DPPP run.
-cd tMultiApplyCal_tmp
-tar zxf ${srcdir}/tNDPPP-generic.in_MS.tgz
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-
-echo; echo "Creating parmdb with defvalues 3"
-parmdbm <<EOL
-open table="tApplyCal.parmdb"
-adddef Gain:0:0:Real values=3.
-adddef Gain:1:1:Real values=3.
-adddef CommonScalarPhase values=0
-EOL
-
-cmd='NDPPP msin=tNDPPP-generic.MS checkparset=1 msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.steps="[gain,csp]" applycal.gain.parmdb=tApplyCal.parmdb applycal.gain.correction=gain applycal.csp.parmdb=tApplyCal.parmdb applycal.csp.correction=commonscalarphase showcounts=false'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DATA~=9*DATA3))' > taql.out
-diff taql.out taql.ref || exit 1
-
-cmd='NDPPP msin=tNDPPP-generic.MS checkparset=1 msout=. msout.datacolumn=DATA3 steps=[applycal] applycal.steps="[gain,csp]" applycal.parmdb=tApplyCal.parmdb applycal.gain.correction=gain applycal.csp.correction=commonscalarphase showcounts=false'
-echo $cmd
-eval $cmd
-$taqlexe 'select from tNDPPP-generic.MS where not(all(DATA~=9*DATA3))' > taql.out
-diff taql.out taql.ref || exit 1
-
diff --git a/CEP/DP3/DPPP/test/tMultiApplyCal.sh b/CEP/DP3/DPPP/test/tMultiApplyCal.sh
deleted file mode 100755
index 12b39f281e6f9ca66c4a064ebd0f89b5422d5b63..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tMultiApplyCal.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tMultiApplyCal
diff --git a/CEP/DP3/DPPP/test/tNDPPP-generic.in_MS.tgz b/CEP/DP3/DPPP/test/tNDPPP-generic.in_MS.tgz
deleted file mode 100644
index e6244abfc464786d4f22ce43ebffbd5ec609703c..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP/test/tNDPPP-generic.in_MS.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP/test/tNDPPP.cc b/CEP/DP3/DPPP/test/tNDPPP.cc
deleted file mode 100644
index a42b0916993ae83c241450d090721ee7954e7d6f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tNDPPP.cc
+++ /dev/null
@@ -1,1027 +0,0 @@
-//# tNDPPP.cc: test program for NDPPP
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/DPRun.h>
-#include <casacore/tables/Tables.h>
-#include <casacore/tables/Tables/TableIter.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayPartMath.h>
-#include <Common/LofarLogger.h>
-#include <iostream>
-#include <stdexcept>
-
-using namespace LOFAR::DPPP;
-using namespace casacore;
-
-// This test program uses the MS in tNDPPP.in_MS.tgz.
-// The MS contains 4 corr, 16 freq, 6 baselines, 18 time slots of 30 sec.
-// Two time slots are missing between time slot 2 and 3.
-
-void checkCopy (const String& in, const String& out, int nms)
-{
- Table tin(in);
- Table tout(out);
- ASSERT (tout.nrow() == 6*24);
- for (int j=0; j<nms; ++j) {
- // A few dummy time slots were inserted, so ignore those.
- Table t1 = tableCommand
- ("using style python "
- "select from $1 where rownumber() not in [0:6, 4*6:6*6, 21*6:24*6]",
- tout);
- ROArrayColumn<Complex> data(t1, "DATA");
- ROArrayColumn<Bool> flag(t1, "FLAG");
- ROArrayColumn<uChar> oflag(t1, "LOFAR_FULL_RES_FLAG");
- IPosition dshape (2, 4, 16);
- IPosition dst (2, 0, j*16);
- Slicer dslicer (dst, dshape);
- ASSERT (data(0).shape() == IPosition(2,4,16*nms));
- ASSERT (flag(0).shape() == IPosition(2,4,16*nms));
- ASSERT (oflag(0).shape() == IPosition(2,16*nms/8,1));
- ASSERT (allEQ(data.getColumn(dslicer),
- ROArrayColumn<Complex>(tin,"DATA").getColumn()));
- ASSERT (allEQ(flag.getColumn(), false));
- ASSERT (allEQ(oflag.getColumn(), uChar(0)));
- ASSERT (allEQ(ROArrayColumn<float>(t1,"WEIGHT_SPECTRUM").getColumn(),
- float(1)));
- // cout<<ROArrayColumn<double>(t1,"UVW").getColumn()<<
- // ROArrayColumn<double>(tin,"UVW").getColumn();
- // ASSERT (allEQ(ROArrayColumn<double>(t1,"UVW").getColumn(),
- // ROArrayColumn<double>(tin,"UVW").getColumn()));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"TIME").getColumn(),
- ROScalarColumn<double>(tin,"TIME").getColumn()));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"TIME_CENTROID").getColumn(),
- ROScalarColumn<double>(tin,"TIME_CENTROID").getColumn()));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"INTERVAL").getColumn(),
- ROScalarColumn<double>(tin,"INTERVAL").getColumn()));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"EXPOSURE").getColumn(),
- ROScalarColumn<double>(tin,"EXPOSURE").getColumn()));
- }
- {
- // Check the inserted time slots.
- // The MS misses a few time slots (3 and 4).
- Table t1 = tableCommand
- ("using style python "
- "select from $1 where rownumber() in [0:6, 4*6:6*6, 21*6:24*6]",
- tout);
- ROArrayColumn<Complex> data(t1, "DATA");
- ROArrayColumn<Bool> flag(t1, "FLAG");
- ROArrayColumn<uChar> oflag(t1, "LOFAR_FULL_RES_FLAG");
- ASSERT (data(0).shape() == IPosition(2,4,16*nms));
- ASSERT (flag(0).shape() == IPosition(2,4,16*nms));
- ASSERT (oflag(0).shape() == IPosition(2,16*nms/8,1));
- ASSERT (allEQ(data.getColumn(), Complex()));
- ASSERT (allEQ(flag.getColumn(), true));
- ASSERT (allEQ(oflag.getColumn(), uChar(0xff)));
- ASSERT (allEQ(ROArrayColumn<float>(t1,"WEIGHT_SPECTRUM").getColumn(),
- float(0)));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"INTERVAL").getColumn(), 30.));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"EXPOSURE").getColumn(), 30.));
- double time = ROScalarColumn<double>(tin,"TIME")(0);
- for (uint i=0; i<6; ++i) {
- double timec = time - 30;
- ASSERT (near(ROScalarColumn<double>(t1,"TIME")(i), timec));
- ASSERT (near(ROScalarColumn<double>(t1,"TIME_CENTROID")(i), timec));
- }
- time = ROScalarColumn<double>(tin,"TIME")(2*6);
- for (uint i=6; i<18; ++i) {
- double timec = time + (i/6)*30.;
- ASSERT (near(ROScalarColumn<double>(t1,"TIME")(i), timec));
- ASSERT (near(ROScalarColumn<double>(t1,"TIME_CENTROID")(i), timec));
- }
- time = ROScalarColumn<double>(tin,"TIME")(17*6);
- for (uint i=18; i<36; ++i) {
- double timec = time + (i/6-2)*30.;
- ASSERT (near(ROScalarColumn<double>(t1,"TIME")(i), timec));
- ASSERT (near(ROScalarColumn<double>(t1,"TIME_CENTROID")(i), timec));
- }
- }
- // Now check if the SPECTRAL_WINDOW table is fine.
- Table spwin(tin.keywordSet().asTable("SPECTRAL_WINDOW"));
- Table spwout(tout.keywordSet().asTable("SPECTRAL_WINDOW"));
- for (int j=0; j<nms; ++j) {
- IPosition dshape (1, 16);
- IPosition dst (1, j*16);
- Slicer dslicer (dst, dshape);
- ASSERT (allEQ (ROArrayColumn<double>(spwin, "CHAN_FREQ").getColumn(),
- ROArrayColumn<double>(spwout,"CHAN_FREQ").getColumn(dslicer)));
- ASSERT (allEQ (ROArrayColumn<double>(spwin, "CHAN_WIDTH").getColumn(),
- ROArrayColumn<double>(spwout,"CHAN_WIDTH").getColumn(dslicer)));
- ASSERT (allEQ (ROArrayColumn<double>(spwin, "EFFECTIVE_BW").getColumn(),
- ROArrayColumn<double>(spwout,"EFFECTIVE_BW").getColumn(dslicer)));
- ASSERT (allEQ (ROArrayColumn<double>(spwin, "RESOLUTION").getColumn(),
- ROArrayColumn<double>(spwout,"RESOLUTION").getColumn(dslicer)));
- }
- ASSERT (allEQ (double(nms)*ROScalarColumn<double>(spwin, "TOTAL_BANDWIDTH").getColumn(),
- ROScalarColumn<double>(spwout,"TOTAL_BANDWIDTH").getColumn()));
- if (nms == 1) {
- ASSERT (allEQ (ROScalarColumn<double>(spwin, "REF_FREQUENCY").getColumn(),
- ROScalarColumn<double>(spwout,"REF_FREQUENCY").getColumn()));
- }
- // Check the TIME_RANGE in the OBSERVATION table.
- Table obsout(tout.keywordSet().asTable("OBSERVATION"));
- Vector<double> timeRange
- (ROArrayColumn<double>(obsout, "TIME_RANGE").getColumn());
- ASSERT (near(timeRange(0), ROScalarColumn<double>(tout,"TIME")(0) - 15));
- ASSERT (near(timeRange(1), ROScalarColumn<double>(tout,"TIME")(143) + 15));
-}
-
-void checkCopyColumn (const String& in)
-{
- Table tin(in);
- ASSERT (tin.nrow() == 6*24);
- ROArrayColumn<Complex> data1(tin, "DATA");
- ROArrayColumn<Complex> data2(tin, "COPY_DATA");
- ROArrayColumn<float> weight1(tin, "NEW_WEIGHT_SPECTRUM");
- ROArrayColumn<float> weight2(tin, "COPY_NEW_WEIGHT_SPECTRUM");
- ASSERT (allEQ(data1.getColumn(), data2.getColumn()));
- ASSERT (allEQ(weight1.getColumn(), weight2.getColumn()));
-}
-
-void testCopy()
-{
- cout << endl << "** testCopy **" << endl;
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- // Give starttime 35 sec before actual, hence 1 missing timeslot.
- ostr << "msin.starttime=03-Aug-2000/13:21:45" << endl;
- // Give endtime 90 sec after actual, hence 3 missing timeslots.
- ostr << "msin.endtime=03-Aug-2000/13:33:15" << endl;
- ostr << "msout=tNDPPP_tmp.MS1" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- checkCopy ("tNDPPP_tmp.MS", "tNDPPP_tmp.MS1", 1);
-}
-
-void testCopyColumn()
-{
- cout << endl << "** testCopyColumn 1 **" << endl;
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS1" << endl;
- ostr << "msout=." << endl;
- ostr << "msout.datacolumn=COPY_DATA" << endl;
- ostr << "msout.weightcolumn=NEW_WEIGHT_SPECTRUM" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
-
- cout << endl << "** testCopyColumn 2 **" << endl;
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS1" << endl;
- ostr << "msin.datacolumn=COPY_DATA" << endl;
- ostr << "msin.weightcolumn=NEW_WEIGHT_SPECTRUM" << endl;
- ostr << "msout=." << endl;
- ostr << "msout.datacolumn=DATA" << endl;
- ostr << "msout.weightcolumn=COPY_NEW_WEIGHT_SPECTRUM" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
-
- checkCopyColumn ("tNDPPP_tmp.MS1");
-}
-
-void testMultiIn()
-{
- cout << endl << "** testMultiIn **" << endl;
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=[tNDPPP_tmp.MS1, tNDPPP_tmp.MS1]" << endl;
- ostr << "msout=tNDPPP_tmp.MS1a" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- checkCopy ("tNDPPP_tmp.MS", "tNDPPP_tmp.MS1a", 2);
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=[tNDPPP_tmp.MS1, tNDPPP_tmp.MS1]" << endl;
- ostr << "msin.datacolumn=CORRECTED_DATA" << endl;
- ostr << "msin.weightcolumn=NEW_WEIGHT_SPECTRUM" << endl;
- ostr << "msin.missingdata=true" << endl;
- ostr << "msin.baseline=0,2&6" << endl;
- ostr << "msout=tNDPPP_tmp.MS1a" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- Table tab("tNDPPP_tmp.MS1a");
- ASSERT (tab.nrow() == 48);
- ASSERT (allEQ (ROArrayColumn<Complex>(tab,"DATA").getColumn(), Complex()));
- ASSERT (tab.tableDesc().isColumn("WEIGHT_SPECTRUM"));
- ASSERT (allEQ (ROArrayColumn<Bool>(tab,"FLAG").getColumn(), True));
- ASSERT (allEQ (ROScalarColumn<Int>(tab,"ANTENNA2").getColumn(), 6));
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=[notexist, tNDPPP_tmp.MS1, notexist, notexist]" << endl;
- ostr << "msin.datacolumn=CORRECTED_DATA" << endl;
- ostr << "msin.missingdata=true" << endl;
- ostr << "msin.orderms=false" << endl;
- ostr << "msin.baseline=0,2&6" << endl;
- ostr << "msout=tNDPPP_tmp.MS1b" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- tab = Table("tNDPPP_tmp.MS1b");
- ASSERT (tab.nrow() == 48);
- ASSERT (allEQ (ROArrayColumn<Complex>(tab,"DATA").getColumn(), Complex()));
- ASSERT (allEQ (ROArrayColumn<Bool>(tab,"FLAG").getColumn(), True));
- ASSERT (allEQ (ROScalarColumn<Int>(tab,"ANTENNA2").getColumn(), 6));
-}
-
-void checkAvg (const String& outName)
-{
- Table tin("tNDPPP_tmp.MS");
- Table tout(outName);
- ROScalarColumn<double> timeCol(tin, "TIME");
- double time = 0.5 * (timeCol(tin.nrow()-1) + timeCol(0));
- ASSERT (tout.nrow() == 6);
- Block<String> colNames(2);
- colNames[0] = "ANTENNA1";
- colNames[1] = "ANTENNA2";
- TableIterator iterin(tin, colNames);
- TableIterator iterout(tout, colNames);
- // Iterate over baseline to be able to average the input in an easy way.
- while (! iterin.pastEnd()) {
- Table t2(iterin.table());
- Table t1(iterout.table());
- ROArrayColumn<Complex> data(t1, "DATA");
- ROArrayColumn<Bool> flag(t1, "FLAG");
- ROArrayColumn<uChar> oflag(t1, "LOFAR_FULL_RES_FLAG");
- ASSERT (data(0).shape() == IPosition(2,4,1));
- ASSERT (flag(0).shape() == IPosition(2,4,1));
- ASSERT (oflag(0).shape() == IPosition(2,16/8,20));
- // Average the original data over all channels and times.
- Array<Complex> dataAvg = partialMeans
- (ROArrayColumn<Complex>(t2,"DATA").getColumn(), IPosition(2,1,2));
- Array<Complex> dataRes = dataAvg.reform (IPosition(3,4,1,1));
- ASSERT (allNear(data.getColumn(), dataRes, 1e-5));
- ASSERT (allNear(ROArrayColumn<float>(t1,"WEIGHT_SPECTRUM").getColumn(),
- float(18*16), 1e-5));
- ASSERT (allEQ(flag.getColumn(), false));
- ASSERT (allNear(ROScalarColumn<double>(t1,"TIME").getColumn(), time, 1e-5));
- ASSERT (allNear(ROScalarColumn<double>(t1,"TIME_CENTROID").getColumn(),
- time, 1e-5));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"INTERVAL").getColumn(), 20*30.));
- ASSERT (allEQ(ROScalarColumn<double>(t1,"EXPOSURE").getColumn(), 20*30.));
- // Two time entries should be flagged.
- Array<uChar> of = oflag.getColumn();
- ASSERT (allEQ(of(Slicer(IPosition(3,0,0,0),IPosition(3,2,3,1))),uChar(0)));
- ASSERT (allEQ(of(Slicer(IPosition(3,0,3,0),IPosition(3,2,2,1))),uChar(0xff)));
- ASSERT (allEQ(of(Slicer(IPosition(3,0,5,0),IPosition(3,2,15,1))),uChar(0)));
- iterin.next();
- iterout.next();
- }
- // Now check if the SPECTRAL_WINDOW table is fine.
- Table spwin(tin.keywordSet().asTable("SPECTRAL_WINDOW"));
- Table spwout(tout.keywordSet().asTable("SPECTRAL_WINDOW"));
- Matrix<double> cw = ROArrayColumn<double>(spwout, "CHAN_WIDTH").getColumn();
- ASSERT (cw.size() == 1);
- ASSERT (near(cw(0,0),
- sum(ROArrayColumn<double>(spwin, "CHAN_WIDTH").getColumn())));
- Matrix<double> cfi = ROArrayColumn<double>(spwin, "CHAN_FREQ").getColumn();
- Matrix<double> cfo = ROArrayColumn<double>(spwout, "CHAN_FREQ").getColumn();
- ASSERT (near(cfo(0,0), 0.5*(cfi(0,0) + cfi(15,0))));
- Matrix<double> ce = ROArrayColumn<double>(spwout, "EFFECTIVE_BW").getColumn();
- ASSERT (ce.size() == 1);
- ASSERT (near(ce(0,0), cw(0,0)));
- Matrix<double> cr = ROArrayColumn<double>(spwout, "RESOLUTION").getColumn();
- ASSERT (cr.size() == 1);
- ASSERT (near(cr(0,0), cw(0,0)));
- ASSERT (near(ROScalarColumn<double>(spwin, "TOTAL_BANDWIDTH")(0),
- cw(0,0)));
- ASSERT (allEQ (ROScalarColumn<double>(spwin, "REF_FREQUENCY").getColumn(),
- ROScalarColumn<double>(spwout,"REF_FREQUENCY").getColumn()));
- // Check the TIME_RANGE in the OBSERVATION table.
- Table obsout(tout.keywordSet().asTable("OBSERVATION"));
- Vector<double> timeRange
- (ROArrayColumn<double>(obsout, "TIME_RANGE").getColumn());
- ASSERT (near(timeRange(0), ROScalarColumn<double>(tout,"TIME")(0) - 300));
- ASSERT (near(timeRange(1), ROScalarColumn<double>(tout,"TIME")(0) + 295));
-}
-
-void testAvg1()
-{
- cout << endl << "** testAvg1 **" << endl;
- {
- // Average in a single step.
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin.name=tNDPPP_tmp.MS" << endl;
- // Give start and end time as actual, hence no missing timeslots.
- ostr << "msin.starttime=03-Aug-2000/13:22:20" << endl;
- ostr << "msin.endtime=03-Aug-2000/13:31:45" << endl;
- ostr << "msout.name=tNDPPP_tmp.MS2" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[avg,count]" << endl;
- ostr << "avg.type=average" << endl;
- ostr << "avg.timestep=20" << endl;
- ostr << "avg.freqstep=100" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- checkAvg ("tNDPPP_tmp.MS2");
-}
-
-void testAvg2()
-{
- cout << endl << "** testAvg2 **" << endl;
- // Averaging in multiple steps should be the same as above.
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msout=tNDPPP_tmp.MS3" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[avg1,avg2,avg3,avg4]" << endl;
- ostr << "avg1.type=average" << endl;
- ostr << "avg1.timestep=5" << endl;
- ostr << "avg1.freqstep=2" << endl;
- ostr << "avg2.type=average" << endl;
- ostr << "avg2.timestep=1" << endl;
- ostr << "avg2.freqstep=2" << endl;
- ostr << "avg3.type=average" << endl;
- ostr << "avg3.timestep=2" << endl;
- ostr << "avg3.freqstep=1" << endl;
- ostr << "avg4.type=average" << endl;
- ostr << "avg4.timestep=2" << endl;
- ostr << "avg4.freqstep=4" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- checkAvg ("tNDPPP_tmp.MS3");
-}
-
-void testAvg3()
-{
- cout << endl << "** testAvg3 **" << endl;
- // Averaging in multiple steps with multiple outputs should be the same
- // as above.
- {
- ofstream ostr1("tNDPPP_tmp.parset1");
- ofstream ostr2("tNDPPP_tmp.parset2");
- ofstream ostr3("tNDPPP_tmp.parset3");
- ofstream ostr4("tNDPPP_tmp.parset4");
- ostr1 << "msin=tNDPPP_tmp.MS" << endl;
- ostr1 << "msout=tNDPPP_tmp.MS4a" << endl;
- ostr1 << "msout.overwrite=true" << endl;
- ostr1 << "steps=[avg1]" << endl;
- ostr1 << "avg1.type=average" << endl;
- ostr1 << "avg1.timestep=5" << endl;
- ostr1 << "avg1.freqstep=2" << endl;
- ostr2 << "msin=tNDPPP_tmp.MS4a" << endl;
- ostr2 << "msout=tNDPPP_tmp.MS4b" << endl;
- ostr2 << "msout.overwrite=true" << endl;
- ostr2 << "steps=[avg2]" << endl;
- ostr2 << "avg2.type=average" << endl;
- ostr2 << "avg2.timestep=1" << endl;
- ostr2 << "avg2.freqstep=2" << endl;
- ostr3 << "msin=tNDPPP_tmp.MS4b" << endl;
- ostr3 << "msout=tNDPPP_tmp.MS4c" << endl;
- ostr3 << "msout.overwrite=true" << endl;
- ostr3 << "steps=[avg3]" << endl;
- ostr3 << "avg3.type=average" << endl;
- ostr3 << "avg3.timestep=2" << endl;
- ostr3 << "avg3.freqstep=1" << endl;
- ostr4 << "msin=tNDPPP_tmp.MS4c" << endl;
- ostr4 << "msout=tNDPPP_tmp.MS4d" << endl;
- ostr4 << "msout.overwrite=true" << endl;
- ostr4 << "steps=[avg4]" << endl;
- ostr4 << "avg4.type=average" << endl;
- ostr4 << "avg4.timestep=2" << endl;
- ostr4 << "avg4.freqstep=4" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset1");
- DPRun::execute ("tNDPPP_tmp.parset2");
- DPRun::execute ("tNDPPP_tmp.parset3");
- DPRun::execute ("tNDPPP_tmp.parset4");
- checkAvg ("tNDPPP_tmp.MS4d");
-}
-
-// This function tests if the correct start time is used when selecting times.
-void testAvg4()
-{
- cout << endl << "** testAvg4 **" << endl;
- {
- // Average in a single step.
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- // Give start a few seconds after first one, hence skip first time slot.
- ostr << "msin.starttime=03-Aug-2000/13:22:25" << endl;
- ostr << "msout=tNDPPP_tmp.MS5" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[avg]" << endl;
- ostr << "avg.type=average" << endl;
- ostr << "avg.timestep=2" << endl;
- ostr << "avg.freqstep=100" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- {
- // Only check the times;
- Table tab ("tNDPPP_tmp.MS");
- // First time to be used.
- double time = ROScalarColumn<double>(tab, "TIME")(6) + 15;
- Table t2 ("tNDPPP_tmp.MS5");
- ASSERT (t2.nrow() == 6*10);
- ROScalarColumn<double> timeCol(t2, "TIME");
- for (uint i=0; i<t2.nrow(); ++i) {
- ASSERT (near(timeCol(i), time));
- if (i%6 == 5) time += 60;
- }
- }
-}
-
-void testUpdate1()
-{
- cout << endl << "** testUpdate1 **" << endl;
- // Test if update works fine.
- // In fact, it does not do anything apart from rewriting the current flags.
- // However, it should ignore the inserted time slots.
- Array<bool> flags;
- {
- Table tab("tNDPPP_tmp.MS");
- flags = ROArrayColumn<bool>(tab,"FLAG").getColumn();
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msout=''" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- // Check that the flags did not change.
- {
- Table tab("tNDPPP_tmp.MS");
- ASSERT (allEQ(ROArrayColumn<bool>(tab,"FLAG").getColumn(), flags));
- }
-}
-
-void testUpdate2()
-{
- cout << endl << "** testUpdate2 **" << endl;
- // Test if update all flags works fine.
- {
- Table tab("tNDPPP_tmp.MS");
- tab.deepCopy ("tNDPPP_tmp.MS_copy1", Table::New);
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS_copy1" << endl;
- ostr << "msout=." << endl;
- ostr << "steps=[preflag]" << endl;
- ostr << "preflag.blmin=1e6" << endl; // should flag all data
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- // Check that all flags are true.
- {
- Table tab("tNDPPP_tmp.MS_copy1");
- ASSERT (allEQ(ROArrayColumn<bool>(tab,"FLAG").getColumn(), true));
- }
-}
-
-void testUpdateScale()
-{
- cout << endl << "** testUpdateScale **" << endl;
- // Test if update data works fine.
- Array<Complex> data;
- Array<bool> flags;
- {
- Table tab("tNDPPP_tmp.MS");
- data = ROArrayColumn<Complex>(tab,"DATA").getColumn();
- flags = ROArrayColumn<bool>(tab,"FLAG").getColumn();
- tab.deepCopy ("tNDPPP_tmp.MS_copy1", Table::New);
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS_copy1" << endl;
- ostr << "msout=tNDPPP_tmp.MS_copy1" << endl; // same name means update
- ostr << "steps=[scaledata]" << endl;
- ostr << "scaledata.coeffs=2" << endl;
- ostr << "scaledata.stations=*" << endl;
- ostr << "scaledata.scalesize=false" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- // Check that all data is doubled.
- {
- Table tab("tNDPPP_tmp.MS_copy1");
- data *= Complex(2,0);
- ASSERT (allNear(ROArrayColumn<Complex>(tab,"DATA").getColumn(), data, 1e-5));
- ASSERT (allEQ(ROArrayColumn<bool>(tab,"FLAG").getColumn(), flags));
- }
-}
-
-void checkFlags (const string& outName)
-{
- // Only check the FULL_RES_FLAGS and table size.
- // The flags are created in various ways, but should be the same in all cases.
- // 3 time slots are averaged to 1.
- Table tout(outName);
- ASSERT (tout.nrow() == 6*4);
- // Check the full-res-flags.
- // Channels 0,2,6,7,8 are flagged everywhere.
- {
- // Input time slots 3,4 are inserted, thus flagged.
- Table t1 = tableCommand
- ("using style python "
- "select from $1 where rownumber() in [0:6]",
- tout);
- ROArrayColumn<uChar> oflag(t1, "LOFAR_FULL_RES_FLAG");
- ASSERT (oflag(0).shape() == IPosition(2,2,6));
- Array<uChar> flags = oflag.getColumn();
- ASSERT (allEQ(flags(IPosition(3,0,0,0), IPosition(3,0,2,5)), uChar(0xc5)));
- ASSERT (allEQ(flags(IPosition(3,1,0,0), IPosition(3,1,2,5)), uChar(0x01)));
- ASSERT (allEQ(flags(IPosition(3,0,3,0), IPosition(3,0,4,5)), uChar(0xff)));
- ASSERT (allEQ(flags(IPosition(3,1,3,0), IPosition(3,1,4,5)), uChar(0x0f)));
- ASSERT (allEQ(flags(IPosition(3,0,5,0), IPosition(3,0,5,5)), uChar(0xc5)));
- ASSERT (allEQ(flags(IPosition(3,1,5,0), IPosition(3,1,5,5)), uChar(0x01)));
- }
- {
- // Input time slots 10,11 are flagged.
- Table t1 = tableCommand
- ("using style python "
- "select from $1 where rownumber() in [6:12]",
- tout);
- ROArrayColumn<uChar> oflag(t1, "LOFAR_FULL_RES_FLAG");
- ASSERT (oflag(0).shape() == IPosition(2,2,6));
- Array<uChar> flags = oflag.getColumn();
- ASSERT (allEQ(flags(IPosition(3,0,0,0), IPosition(3,0,3,5)), uChar(0xc5)));
- ASSERT (allEQ(flags(IPosition(3,1,0,0), IPosition(3,1,3,5)), uChar(0x01)));
- ASSERT (allEQ(flags(IPosition(3,0,4,0), IPosition(3,0,5,5)), uChar(0xff)));
- ASSERT (allEQ(flags(IPosition(3,1,4,0), IPosition(3,1,5,5)), uChar(0x0f)));
- }
- {
- // Input time slots 12-17 are not flagged.
- Table t1 = tableCommand
- ("using style python "
- "select from $1 where rownumber() in [12:18]",
- tout);
- ROArrayColumn<uChar> oflag(t1, "LOFAR_FULL_RES_FLAG");
- ASSERT (oflag(0).shape() == IPosition(2,2,6));
- Array<uChar> flags = oflag.getColumn();
- ASSERT (allEQ(flags(IPosition(3,0,0,0), IPosition(3,0,5,5)), uChar(0xc5)));
- ASSERT (allEQ(flags(IPosition(3,1,0,0), IPosition(3,1,5,5)), uChar(0x01)));
- }
- {
- // Input time slot 20-23 did not exist, thus flagged in average.
- Table t1 = tableCommand
- ("using style python "
- "select from $1 where rownumber() in [18:24]",
- tout);
- ROArrayColumn<uChar> oflag(t1, "LOFAR_FULL_RES_FLAG");
- ASSERT (oflag(0).shape() == IPosition(2,2,6));
- Array<uChar> flags = oflag.getColumn();
- ASSERT (allEQ(flags(IPosition(3,0,0,0), IPosition(3,0,1,5)), uChar(0xc5)));
- ASSERT (allEQ(flags(IPosition(3,1,0,0), IPosition(3,1,1,5)), uChar(0x01)));
- ASSERT (allEQ(flags(IPosition(3,0,2,0), IPosition(3,0,5,5)), uChar(0xff)));
- ASSERT (allEQ(flags(IPosition(3,1,2,0), IPosition(3,1,5,5)), uChar(0x0f)));
- }
-}
-
-void testFlags1()
-{
- cout << endl << "** testFlags1 **" << endl;
- {
- // Most simple case.
- // Just flag some channels and time stamps.
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msin.startchan=1" << endl;
- ostr << "msin.nchan=12" << endl;
- ostr << "msout=tNDPPP_tmp.MS5" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[preflag,average]" << endl;
- ostr << "preflag.expr='flag1 or flag2'" << endl;
- ostr << "preflag.flag1.timeslot=[10,11]" << endl;
- ostr << "preflag.flag2.chan=[0,2,6..8]" << endl;
- ostr << "average.timestep=6" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- checkFlags ("tNDPPP_tmp.MS5");
-}
-
-void testFlags2()
-{
- cout << endl << "** testFlags2 **" << endl;
- {
- // A more advanced case in two NDPPP steps.
- // Flag the channels, but shifted 2. In the next step the first 2 channels
- // are skipped, thus the channel numbers shift back 2.
- // An averaged time slot is flagged, so the original time slots should
- // come out flagged.
- ofstream ostr1("tNDPPP_tmp.parset1");
- ostr1 << "msin=tNDPPP_tmp.MS" << endl;
- ostr1 << "msin.nchan=15" << endl;
- ostr1 << "msout=tNDPPP_tmp.MS6a" << endl;
- ostr1 << "msout.overwrite=true" << endl;
- ostr1 << "steps=[preflag,average]" << endl;
- ostr1 << "preflag.chan=[2,4,8..10]" << endl;
- ostr1 << "average.timestep=2" << endl;
- ofstream ostr2("tNDPPP_tmp.parset2");
- ostr2 << "msin=tNDPPP_tmp.MS6a" << endl;
- ostr2 << "msin.startchan=2*1" << endl; // output chan 0,2 are now flagged
- ostr2 << "msin.nchan=nchan-3" << endl;
- ostr2 << "msout=tNDPPP_tmp.MS6b" << endl;
- ostr2 << "msout.overwrite=true" << endl;
- ostr2 << "steps=[preflag,average]" << endl;
- ostr2 << "preflag.timeslot=5" << endl; // is 10,11 in input
- ostr2 << "average.timestep=3" << endl;
- ostr2 << "average.freqstep=2" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset1");
- DPRun::execute ("tNDPPP_tmp.parset2");
- checkFlags ("tNDPPP_tmp.MS6b");
-}
-
-void testFlags3()
-{
- cout << endl << "** testFlags3 **" << endl;
- {
- // Even a bit more advanced, also in two NDPPP runs.
- // Input channels 6,7,8 are flagged by flagging their averaged channel.
- // This is done at the end of run 1, so the averager of run 2 should pick
- // up those flags.
- ofstream ostr1("tNDPPP_tmp.parset1");
- ostr1 << "msin=tNDPPP_tmp.MS" << endl;
- ostr1 << "msin.nchan=15" << endl;
- ostr1 << "msout=tNDPPP_tmp.MS7a" << endl;
- ostr1 << "msout.overwrite=true" << endl;
- ostr1 << "steps=[preflag,average,pre2]" << endl;
- ostr1 << "preflag.chan=[0,2]" << endl;
- ostr1 << "average.timestep=2" << endl;
- ostr1 << "average.freqstep=3" << endl;
- ostr1 << "pre2.type=preflag" << endl;
- ostr1 << "pre2.chan=2" << endl; // is input channel 6,7,8
- ofstream ostr2("tNDPPP_tmp.parset2");
- ostr2 << "msin=tNDPPP_tmp.MS7a" << endl;
- ostr2 << "msin.nchan=4" << endl;
- ostr2 << "msout=tNDPPP_tmp.MS7b" << endl;
- ostr2 << "msout.overwrite=true" << endl;
- ostr2 << "steps=[preflag,average]" << endl;
- ostr2 << "preflag.timeslot=5" << endl; // is 10,11 in input
- ostr2 << "average.timestep=3" << endl;
- ostr2 << "average.freqstep=2" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset1");
- DPRun::execute ("tNDPPP_tmp.parset2");
- checkFlags ("tNDPPP_tmp.MS7b");
-}
-
-void testStationAdd()
-{
- cout << endl << "** testStationAdd **" << endl;
- // Add station RT0, 1 and 2.
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msout=tNDPPP_tmp.MSa" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[stationadd]" << endl;
- ostr << "stationadd.stations={RTnew:[RT0..2]}" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- Table t1("tNDPPP_tmp.MS/ANTENNA");
- Table t2("tNDPPP_tmp.MSa/ANTENNA");
- ASSERT (t2.nrow() == t1.nrow()+1); // 1 antenna has been added
- ASSERT (ROScalarColumn<String>(t2,"NAME")(t2.nrow()-1) == "RTnew");
- Int oldNant = t1.nrow();
- t1 = Table("tNDPPP_tmp.MS/FEED");
- t2 = Table("tNDPPP_tmp.MSa/FEED");
- ASSERT (t2.nrow() == t1.nrow()+1); // 1 antenna has been added
- ASSERT (ROScalarColumn<Int>(t2,"ANTENNA_ID")(t2.nrow()-1) == oldNant);
- t1 = Table("tNDPPP_tmp.MS");
- t2 = Table("tNDPPP_tmp.MSa");
- ASSERT (t2.nrow() == t1.nrow()+40+12); // 2 baselines and 2 time slots added
-}
-
-void testFilter1()
-{
- cout << endl << "** testFilter1 **" << endl;
- // Remove all baselines containing station RT1 or 6.
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msout=tNDPPP_tmp.MSa" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[filter]" << endl;
- ostr << "filter.baseline=!RT[16]&&*" << endl;
- ostr << "filter.remove=True" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- Table t1("tNDPPP_tmp.MS/ANTENNA");
- Table t2("tNDPPP_tmp.MSa/ANTENNA");
- // Note: the ANTENNA table also contained RT8, RT9, etc., but they do not
- // have baselines. So these were removed as well meaning only 0,2,7 are left.
- Vector<uInt> rownrs(3);
- rownrs[0]=0; rownrs[1]=2; rownrs[2]=7;
- Table t1s = t1(rownrs);
- ASSERT (t2.nrow() == t1s.nrow());
- ASSERT (allEQ (ROScalarColumn<String>(t2,"NAME").getColumn(),
- ROScalarColumn<String>(t1s,"NAME").getColumn()));
- t1 = Table("tNDPPP_tmp.MS/FEED");
- t2 = Table("tNDPPP_tmp.MSa/FEED");
- t1s = t1(rownrs);
- ASSERT (t2.nrow() == t1s.nrow());
- // The ANTENNA_IDs in the FEED table must be 0,1,2.
- Vector<Int> ids(t2.nrow());
- indgen (ids);
- ASSERT (allEQ (ROScalarColumn<Int>(t2,"ANTENNA_ID").getColumn(), ids));
- // Check the main table.
- t1 = Table("tNDPPP_tmp.MS");
- t2 = Table("tNDPPP_tmp.MSa");
- ASSERT (t2.nrow() == t1.nrow()-72+4); // 4 baselines removed, 2 timeslots added
- t1s = t1((t1.col("ANTENNA1")==0 || t1.col("ANTENNA1")==2) &&
- t1.col("ANTENNA2")==7);
- // A few dummy time slots were inserted, so ignore those.
- Table t2s = t2(t2.nodeRownr() < 6 || t2.nodeRownr() >= 10);
- ASSERT (allEQ (ROArrayColumn<Complex>(t2s,"DATA").getColumn(),
- ROArrayColumn<Complex>(t1s,"DATA").getColumn()));
- t2s = t2(t2.nodeRownr() % 2 == 0);
- ASSERT (allEQ (ROScalarColumn<Int>(t2s,"ANTENNA1").getColumn(), 0));
- ASSERT (allEQ (ROScalarColumn<Int>(t2s,"ANTENNA2").getColumn(), 2));
- t2s = t2(t2.nodeRownr() % 2 == 1);
- ASSERT (allEQ (ROScalarColumn<Int>(t2s,"ANTENNA1").getColumn(), 1));
- ASSERT (allEQ (ROScalarColumn<Int>(t2s,"ANTENNA2").getColumn(), 2));
-}
-
-void testFilter2()
-{
- cout << endl << "** testFilter2 **" << endl;
- // Keep all baselines.
- // First by not specifying baseline selection, second by all baselines.
- // Also alter between remove and !remove.
- for (int iter=0; iter<4; ++iter) {
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msout=tNDPPP_tmp.MSa" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[filter]" << endl;
- if (iter%2 == 1) {
- ostr << "filter.baseline=*&&*" << endl;
- }
- if (iter/2 == 1) {
- ostr << "filter.remove=True" << endl;
- }
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- //cout << "check ANTENNA"<<endl;
- Table t1("tNDPPP_tmp.MS/ANTENNA");
- Table t2("tNDPPP_tmp.MSa/ANTENNA");
- // Note: the ANTENNA table also contained RT8, RT9, etc., but they do not
- // have baselines. So these were removed meaning only 0,1,2,6,7 are left.
- Vector<uInt> rownrs(5);
- rownrs[0]=0; rownrs[1]=1; rownrs[2]=2; rownrs[3]=6; rownrs[4]=7;
- Table t1s(t1);
- if (iter/2 == 1) {
- t1s = t1(rownrs);
- }
- ASSERT (t2.nrow() == t1s.nrow());
- ASSERT (allEQ (ROScalarColumn<String>(t2,"NAME").getColumn(),
- ROScalarColumn<String>(t1s,"NAME").getColumn()));
- //cout << "check FEED"<<endl;
- t1 = Table("tNDPPP_tmp.MS/FEED");
- t2 = Table("tNDPPP_tmp.MSa/FEED");
- t1s = t1;
- if (iter/2 == 1) {
- t1s = t1(rownrs);
- }
- ASSERT (t2.nrow() == t1s.nrow());
- // The ANTENNA_IDs in the FEED table must be 0,1,2.
- Vector<Int> ids(t2.nrow());
- indgen (ids);
- ASSERT (allEQ (ROScalarColumn<Int>(t2,"ANTENNA_ID").getColumn(), ids));
- // Check the main table.
- t1 = Table("tNDPPP_tmp.MS");
- t2 = Table("tNDPPP_tmp.MSa");
- ASSERT (t2.nrow() == t1.nrow()+12); // 2 timeslots added
- // A few dummy time slots were inserted, so ignore those.
- Table t2s = t2(t2.nodeRownr() < 18 || t2.nodeRownr() >= 30);
- ASSERT (allEQ (ROArrayColumn<Complex>(t2s,"DATA").getColumn(),
- ROArrayColumn<Complex>(t1,"DATA").getColumn()));
- int ant1[] = {0,0,1,1,2,2};
- int ant2[] = {6,7,6,7,6,7};
- int sub = (iter/2 == 0 ? 0:3); // if remove, ant2 6->3 and 7->4
- for (int i=0; i<6; ++i) {
- t2s = t2(t2.nodeRownr() % 6 == i);
- ASSERT (allEQ (ROScalarColumn<Int>(t2s,"ANTENNA1").getColumn(),
- ant1[i]));
- ASSERT (allEQ (ROScalarColumn<Int>(t2s,"ANTENNA2").getColumn(),
- ant2[i]-sub));
- }
- }
-}
-
-
-void testFilter3()
-{
- cout << endl << "** testFilter3 **" << endl;
- // Remove some baselines, update original file with different data column
- // This test justs tests if it runs without throwing exceptions
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msout=." << endl;
- ostr << "msout.datacolumn=DATA_FILTER" << endl;
- ostr << "steps=[filter]" << endl;
- ostr << "filter.baseline=!RT[16]&&*" << endl;
- ostr << "filter.remove=False" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
-}
-
-
-void testClear()
-{
- cout << endl << "** testClear **" << endl;
- Array<bool> flags;
- // First flag in the same way as testFlags1.
- testFlags1();
- // Get the resulting flags.
- {
- Table tab("tNDPPP_tmp.MS5");
- flags.reference (ROArrayColumn<bool>(tab, "FLAG").getColumn());
- }
- // Flag all data.
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS5" << endl;
- ostr << "msout=tNDPPP_tmp.MS5a" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[preflag]" << endl;
- ostr << "preflag.baseline=[[*]]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- checkFlags ("tNDPPP_tmp.MS5a");
- {
- Table tab("tNDPPP_tmp.MS5a");
- ASSERT (allEQ(ROArrayColumn<bool>(tab, "FLAG").getColumn(), True));
- }
- // Clear the flags.
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS5a" << endl;
- ostr << "msout=tNDPPP_tmp.MS5b" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[preflag]" << endl;
- ostr << "preflag.mode=clear" << endl;
- ostr << "preflag.baseline=[[*]]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- checkFlags ("tNDPPP_tmp.MS5b");
- {
- Table tab("tNDPPP_tmp.MS5b");
- ASSERT (allEQ(ROArrayColumn<bool>(tab, "FLAG").getColumn(), false));
- }
-}
-
-
-void testMultiOut()
-{
- cout << endl << "** testMultiOut **" << endl;
- {
- // First make the reference output MS.
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "msout=tNDPPP_tmp.MS_copy" << endl;
- ostr << "msout.overwrite=true" << endl;
- ostr << "steps=[]" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- // Test if update data works fine with multiple outputs:
- // read from tNDPPP_tmp.MS, write to copy3, update to copy3
- Array<Complex> data;
- Array<bool> flags;
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "steps=[scaledata,out1,scaledata,out2]" << endl;
- ostr << "scaledata.coeffs=2" << endl;
- ostr << "scaledata.stations=*" << endl;
- ostr << "scaledata.scalesize=false" << endl;
- ostr << "out1.type=out" << endl;
- ostr << "out1.name=tNDPPP_tmp.MS_copy3" << endl;
- ostr << "out1.overwrite=true" << endl;
- ostr << "out2.type=out" << endl;
- ostr << "out2.name=." << endl; // Defaults to the previous out, so _copy3
- ostr << "out2.datacolumn=DATA_2" << endl;
- ostr << "msout=tNDPPP_tmp.MS_copy4" << endl; // same name means update
- ostr << "msout.overwrite=true" << endl;
- }
- DPRun::execute ("tNDPPP_tmp.parset");
- // Check that tables exist, contain the specified columns
- {
- Table tab1("tNDPPP_tmp.MS_copy");
- Table tab2("tNDPPP_tmp.MS_copy3");
- ///cout<<ROArrayColumn<Complex>(tab2,"DATA_2").getColumn();
- ///cout<<Complex(2,0)*ROArrayColumn<Complex>(tab1,"DATA").getColumn();
- ASSERT (allNear(ROArrayColumn<Complex>(tab2,"DATA").getColumn(),
- Complex(2,0)*ROArrayColumn<Complex>(tab1,"DATA").getColumn(),
- 1e-5));
- ASSERT (allNear(ROArrayColumn<Complex>(tab2,"DATA_2").getColumn(),
- Complex(4,0)*ROArrayColumn<Complex>(tab1,"DATA").getColumn(),
- 1e-5));
- ASSERT (allNear(ROArrayColumn<Float>(tab2,"WEIGHT_SPECTRUM").getColumn(),
- ROArrayColumn<Float>(tab1,"WEIGHT_SPECTRUM").getColumn(),
- 1e-5));
- ASSERT (allEQ(ROArrayColumn<Bool>(tab2,"FLAG").getColumn(),
- ROArrayColumn<Bool>(tab1,"FLAG").getColumn()));
- Table tab3("tNDPPP_tmp.MS_copy4");
- ASSERT (allNear(ROArrayColumn<Complex>(tab3,"DATA").getColumn(),
- Complex(4,0)*ROArrayColumn<Complex>(tab1,"DATA").getColumn(),
- 1e-5));
- ASSERT (allNear(ROArrayColumn<Float>(tab3,"WEIGHT_SPECTRUM").getColumn(),
- ROArrayColumn<Float>(tab1,"WEIGHT_SPECTRUM").getColumn(),
- 1e-5));
- ASSERT (allEQ(ROArrayColumn<Bool>(tab3,"FLAG").getColumn(),
- ROArrayColumn<Bool>(tab1,"FLAG").getColumn()));
- }
-
-}
-
-void tryErr (const string& parsetName)
-{
- bool err = false;
- try {
- DPRun::execute (parsetName);
- } catch (const std::exception& x) {
- err = true;
- cout << "Expected exception: " << x.what() << endl;
- }
- ASSERT (err);
-}
-
-void testErrorOut()
-{
- cout << endl << "Trying some incorrect DPPP runs ..." << endl;
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "steps=[filter,out1,average,out2]" << endl;
- ostr << "out1.type=out" << endl;
- ostr << "out1.name=''" << endl;
- ostr << "out2.type=out" << endl;
- ostr << "out2.name=." << endl; // update not possible when avg
- ostr << "msout=''" << endl;
- tryErr ("tNDPPP_tmp.parset");
- }
- {
- ofstream ostr("tNDPPP_tmp.parset");
- ostr << "msin=tNDPPP_tmp.MS" << endl;
- ostr << "steps=[average,out1,filter,out2]" << endl;
- ostr << "out1.type=out" << endl;
- ostr << "out1.name=tNDPPP_tmp.MSx" << endl;
- ostr << "out1.overwrite=true" << endl;
- ostr << "filter.remove=true" << endl;
- ostr << "out2.type=out" << endl;
- ostr << "out2.name=./tNDPPP_tmp.MSx" << endl; // update not possible (filter)
- ostr << "msout=''" << endl;
- tryErr ("tNDPPP_tmp.parset");
- }
-}
-
-
-int main()
-{
- try
- {
- testCopy();
- testCopyColumn();
- testMultiIn();
- testAvg1();
- testAvg2();
- testAvg3();
- testAvg4();
- testUpdate1();
- testUpdate2();
- testUpdateScale();
- testFlags1();
- testFlags2();
- testFlags3();
- testStationAdd();
- testFilter1();
- testFilter2();
- testFilter3();
- testClear();
- testMultiOut();
- testErrorOut();
- } catch (std::exception& err) {
- std::cerr << "Error detected: " << err.what() << std::endl;
- return 1;
- }
-}
diff --git a/CEP/DP3/DPPP/test/tNDPPP.in_MS.tgz b/CEP/DP3/DPPP/test/tNDPPP.in_MS.tgz
deleted file mode 100644
index 57bef2b6821d647bf45bc685c4c224c13ba99312..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP/test/tNDPPP.in_MS.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP/test/tNDPPP.run b/CEP/DP3/DPPP/test/tNDPPP.run
deleted file mode 100755
index e3ae3b22b327d4b054f479d81d67256210a1a1c0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tNDPPP.run
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-if test ! -f tNDPPP.in_MS.tgz; then
- return 3 # untested
-fi
-
-tar zxf tNDPPP.in_MS.tgz
-./tNDPPP
diff --git a/CEP/DP3/DPPP/test/tNDPPP.sh b/CEP/DP3/DPPP/test/tNDPPP.sh
deleted file mode 100755
index feac3b39893761b798f0376df89e74d8027c1194..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tNDPPP.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tNDPPP
diff --git a/CEP/DP3/DPPP/test/tPSet.cc b/CEP/DP3/DPPP/test/tPSet.cc
deleted file mode 100644
index 0e24a6d301f57864dbc83b7d357c5927f469a8ff..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPSet.cc
+++ /dev/null
@@ -1,435 +0,0 @@
-//# tPSet.cc: Test program for class PreFlagger::PSet
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/PreFlagger.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <casacore/casa/Quanta/MVTime.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int nbl, int nchan, int ncorr)
- : itsNChan(nchan), itsNCorr(ncorr)
- {
- info().init (itsNCorr, itsNChan, 0, 0, 50, string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- vector<MPosition> antPos(4);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- Vector<double> chanWidth(nchan, 100000);
- Vector<double> chanFreqs(nchan);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&) { return false; }
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
-
- int itsNChan, itsNCorr;
-};
-
-
-namespace LOFAR {
- namespace DPPP {
- // This class name should match the friend in PreFlagger.
- class TestPSet
- {
- public:
- static void testNone();
- static void testBL();
- static void testChan();
- static void testTime();
- static void testMinMax();
- };
- }
-}
-
-void TestPSet::testNone()
-{
- TestInput* in = new TestInput(16, 8, 4);
- DPStep::ShPtr step1(in);
- cout << "testNone" << endl;
- ParameterSet parset;
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (!(pset.itsFlagOnBL || pset.itsFlagOnAmpl || pset.itsFlagOnPhase ||
- pset.itsFlagOnReal || pset.itsFlagOnImag ||
- pset.itsFlagOnAzEl || pset.itsFlagOnUV));
-}
-
-void TestPSet::testBL()
-{
- TestInput* in = new TestInput(16, 8, 4);
- DPStep::ShPtr step1(in);
- {
- cout << "testBL 1" << endl;
- ParameterSet parset;
- parset.add ("baseline", "[rs01.*, rs02.s01]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (!(pset.itsFlagOnAmpl || pset.itsFlagOnPhase || pset.itsFlagOnReal ||
- pset.itsFlagOnImag || pset.itsFlagOnAzEl || pset.itsFlagOnUV) &&
- pset.itsFlagOnBL);
- // Make sure the matrix is correct.
- const Matrix<bool>& mat = pset.itsFlagBL;
- ASSERT (mat.shape() == IPosition(2,4,4));
- ASSERT ( mat(0,0) && mat(0,1) && mat(0,2) && mat(0,3));
- ASSERT ( mat(1,0) && mat(1,1) && mat(1,2) && mat(1,3));
- ASSERT ( mat(2,0) && mat(2,1) && !mat(2,2) && !mat(2,3));
- ASSERT ( mat(3,0) && mat(3,1) && !mat(3,2) && !mat(3,3));
- }
- {
- cout << "testBL 2" << endl;
- ParameterSet parset;
- parset.add ("corrtype", "auto");
- parset.add ("baseline", "[rs01.*, [*s*.*2], rs02.s01]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- // Make sure the matrix is correct.
- const Matrix<bool>& mat = pset.itsFlagBL;
- ASSERT (mat.shape() == IPosition(2,4,4));
- ASSERT ( mat(0,0) && !mat(0,1) && !mat(0,2) && !mat(0,3));
- ASSERT (!mat(1,0) && mat(1,1) && !mat(1,2) && !mat(1,3));
- ASSERT (!mat(2,0) && !mat(2,1) && !mat(2,2) && !mat(2,3));
- ASSERT (!mat(3,0) && !mat(3,1) && !mat(3,2) && mat(3,3));
- }
- {
- cout << "testBL 3" << endl;
- ParameterSet parset;
- parset.add ("corrtype", "CROSS");
- parset.add ("baseline", "[[rs*, *s*.*1], [cs01.s01,cs01.s02]]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- // Make sure the matrix is correct.
- const Matrix<bool>& mat = pset.itsFlagBL;
- ASSERT (mat.shape() == IPosition(2,4,4));
- ASSERT (!mat(0,0) && mat(0,1) && mat(0,2) && !mat(0,3));
- ASSERT ( mat(1,0) && !mat(1,1) && mat(1,2) && !mat(1,3));
- ASSERT ( mat(2,0) && mat(2,1) && !mat(2,2) && mat(2,3));
- ASSERT (!mat(3,0) && !mat(3,1) && mat(3,2) && !mat(3,3));
- }
- // Some erronous ones.
- cout << "testBL expected error 1" << endl;
- bool err = false;
- try {
- ParameterSet parset;
- parset.add ("corrtype", "crossx");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- } catch (std::exception& x) {
- err = true;
- cout << " " << x.what() << endl;
- }
- ASSERT (err);
- cout << "testBL expected error 2" << endl;
- err = false;
- try {
- ParameterSet parset;
- parset.add ("baseline", "[[a,b,c]]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- } catch (std::exception& x) {
- err = true;
- cout << " " << x.what() << endl;
- }
- ASSERT (err);
- cout << "testBL expected error 3" << endl;
- err = false;
- try {
- ParameterSet parset;
- parset.add ("baseline", "[[a,b], [ ] ]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- } catch (std::exception& x) {
- err = true;
- cout << " " << x.what() << endl;
- }
- ASSERT (err);
-}
-
-void TestPSet::testChan()
-{
- TestInput* in = new TestInput(16, 32, 4);
- DPStep::ShPtr step1(in);
- {
- cout << "testChan 1" << endl;
- ParameterSet parset;
- parset.add ("chan", "[11..13, 4]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsChannels.size() == 4);
- ASSERT (pset.itsChannels[0] == 4);
- ASSERT (pset.itsChannels[1] == 11);
- ASSERT (pset.itsChannels[2] == 12);
- ASSERT (pset.itsChannels[3] == 13);
- ASSERT (pset.itsChanFlags.shape() == IPosition(2,4,32));
- for (uint i=0; i<32; ++i) {
- if (i==4 || i==11 || i==12 || i==13) {
- ASSERT (allEQ(pset.itsChanFlags.column(i), true));
- } else {
- ASSERT (allEQ(pset.itsChanFlags.column(i), false));
- }
- }
- }
- {
- cout << "testChan 2" << endl;
- ParameterSet parset;
- parset.add ("freqrange", "[ 1.1 .. 1.2 MHz, 1.5MHz+-65000Hz]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsChannels.size() == 3);
- ASSERT (pset.itsChannels[0] == 1);
- ASSERT (pset.itsChannels[1] == 4);
- ASSERT (pset.itsChannels[2] == 5);
- }
- {
- cout << "testChan 3" << endl;
- ParameterSet parset;
- parset.add ("chan", "[11..13, 4]");
- parset.add ("freqrange", "[ 1.1 .. 1.2 MHz, 1.5MHz+-65000Hz]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsChannels.size() == 1);
- ASSERT (pset.itsChannels[0] == 4);
- }
-}
-
-void TestPSet::testTime()
-{
- TestInput* in = new TestInput(16, 8, 4);
- DPStep::ShPtr step1(in);
- {
- cout << "testTime 1" << endl;
- ParameterSet parset;
- parset.add ("abstime", "[1mar2009/12:00:00..2mar2009/13:00:00]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsATimes.size() == 2);
- Quantity q;
- MVTime::read (q, "1mar2009/12:00:00");
- ASSERT (q.getValue("s") == pset.itsATimes[0]);
- ASSERT (pset.itsATimes[1] - pset.itsATimes[0] == 86400+3600);
- }
- {
- cout << "testTime 2" << endl;
- ParameterSet parset;
- parset.add ("reltime", "[12:00:00..13:00:00, 16:00 +- 2min ]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsRTimes.size() == 4);
- ASSERT (pset.itsRTimes[0] == 12*3600);
- ASSERT (pset.itsRTimes[1] == 13*3600);
- ASSERT (pset.itsRTimes[2] == 16*3600-120);
- ASSERT (pset.itsRTimes[3] == 16*3600+120);
- }
- {
- cout << "testTime 3" << endl;
- ParameterSet parset;
- parset.add ("timeofday", "[22:00:00..2:00:00, 23:30 +- 1h ]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsTimes.size() == 8);
- ASSERT (pset.itsTimes[0] == -1);
- ASSERT (pset.itsTimes[1] == 2*3600);
- ASSERT (pset.itsTimes[2] == 22*3600);
- ASSERT (pset.itsTimes[3] == 24*3600+1);
- ASSERT (pset.itsTimes[4] == -1);
- ASSERT (pset.itsTimes[5] == 1800);
- ASSERT (pset.itsTimes[6] == 22*3600+1800);
- ASSERT (pset.itsTimes[7] == 24*3600+1);
- }
- {
- cout << "testTime 4" << endl;
- ParameterSet parset;
- parset.add ("timeslot", "[2..4, 10]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsTimeSlot.size() == 4);
- ASSERT (pset.itsTimeSlot[0] == 2);
- ASSERT (pset.itsTimeSlot[1] == 3);
- ASSERT (pset.itsTimeSlot[2] == 4);
- ASSERT (pset.itsTimeSlot[3] == 10);
- }
- // Some erronous ones.
- cout << "testTime expected error 1" << endl;
- bool err = false;
- try {
- ParameterSet parset;
- parset.add ("reltime", "[12:00:00]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- } catch (std::exception& x) {
- err = true;
- cout << " " << x.what() << endl;
- }
- ASSERT (err);
- cout << "testTime expected error 2" << endl;
- err = false;
- try {
- ParameterSet parset;
- parset.add ("reltime", "[12:00:00..11:00:00]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- } catch (std::exception& x) {
- err = true;
- cout << " " << x.what() << endl;
- }
- ASSERT (err);
- cout << "testTime expected error 3" << endl;
- err = false;
- try {
- ParameterSet parset;
- parset.add ("abstime", "[12:00:00..13:00:00]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- } catch (std::exception& x) {
- err = true;
- cout << " " << x.what() << endl;
- }
- ASSERT (err);
-}
-
-void TestPSet::testMinMax()
-{
- TestInput* in = new TestInput(16, 8, 4);
- DPStep::ShPtr step1(in);
- {
- cout << "testMinMax 1" << endl;
- ParameterSet parset;
- parset.add ("amplmin", "[23,,,45]");
- parset.add ("amplmax", "112.5");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsFlagOnAmpl);
- ASSERT (pset.itsAmplMin.size() == 4);
- ASSERT (pset.itsAmplMax.size() == 4);
- ASSERT (near(pset.itsAmplMin[0], 23.));
- ASSERT (near(pset.itsAmplMin[1], -1e30));
- ASSERT (near(pset.itsAmplMin[2], -1e30));
- ASSERT (near(pset.itsAmplMin[3], 45.));
- ASSERT (near(pset.itsAmplMax[0], 112.5));
- ASSERT (near(pset.itsAmplMax[1], 112.5));
- ASSERT (near(pset.itsAmplMax[2], 112.5));
- ASSERT (near(pset.itsAmplMax[3], 112.5));
- }
- {
- cout << "testMinMax 2" << endl;
- ParameterSet parset;
- parset.add ("phasemin", "[23]");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsFlagOnPhase);
- ASSERT (pset.itsAmplMin.size() == 4);
- ASSERT (pset.itsAmplMax.size() == 4);
- ASSERT (near(pset.itsPhaseMin[0], 23.));
- ASSERT (near(pset.itsPhaseMin[1], -1e30));
- ASSERT (near(pset.itsPhaseMin[2], -1e30));
- ASSERT (near(pset.itsPhaseMin[3], -1e30));
- ASSERT (near(pset.itsPhaseMax[0], 1e30));
- ASSERT (near(pset.itsPhaseMax[1], 1e30));
- ASSERT (near(pset.itsPhaseMax[2], 1e30));
- ASSERT (near(pset.itsPhaseMax[3], 1e30));
- }
- {
- cout << "testMinMax 3" << endl;
- ParameterSet parset;
- parset.add ("uvmmin", "23");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsFlagOnUV);
- ASSERT (near(pset.itsMinUV, 23.*23.));
- ASSERT (near(pset.itsMaxUV, 1e30));
- }
- {
- cout << "testMinMax 4" << endl;
- ParameterSet parset;
- parset.add ("uvmmax", "23");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsFlagOnUV);
- ASSERT (pset.itsMinUV < 0.);
- ASSERT (near(pset.itsMaxUV, 23.*23.));
- }
- {
- cout << "testMinMax 5" << endl;
- ParameterSet parset;
- parset.add ("uvmmin", "23");
- parset.add ("uvmmax", "123");
- PreFlagger::PSet pset (in, parset, "");
- pset.updateInfo (in->getInfo());
- ASSERT (pset.itsFlagOnUV);
- ASSERT (near(pset.itsMinUV, 23.*23.));
- ASSERT (near(pset.itsMaxUV, 123.*123.));
- }
-}
-
-int main()
-{
- INIT_LOGGER ("tPSet");
- try {
- TestPSet::testNone();
- TestPSet::testBL();
- TestPSet::testChan();
- TestPSet::testTime();
- TestPSet::testMinMax();
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tPSet.sh b/CEP/DP3/DPPP/test/tPSet.sh
deleted file mode 100755
index 2f3e0c17fdcf24eea376956444f1ab0bdc6a402c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPSet.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tPSet
diff --git a/CEP/DP3/DPPP/test/tPhaseShift.cc b/CEP/DP3/DPPP/test/tPhaseShift.cc
deleted file mode 100644
index 7638d5549a40b9a0f1b1320d282da01a59528394..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPhaseShift.cc
+++ /dev/null
@@ -1,330 +0,0 @@
-//# tPhaseShift.cc: Test program for class PhaseShift
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/PhaseShift.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {
- info().init (ncorr, nchan, ntime, 0., 10., string(), string());
- MDirection phaseCenter(Quantity(45,"deg"), Quantity(30,"deg"),
- MDirection::J2000);
- info().set (MPosition(), phaseCenter, phaseCenter, phaseCenter);
- // Define the frequencies.
- Vector<double> chanWidth (nchan, 100000.);
- Vector<double> chanFreqs (nchan);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- // Fill the baseline stations.
- // Determine nr of stations using: na*(na+1)/2 = nbl
- // If many baselines, divide into groups of 6 to test if
- // PhaseShift disentangles it correctly.
- int nant = int(-0.5 + sqrt(0.25 + 2*nbl));
- if (nant*(nant+1)/2 < nbl) ++nant;
- int grpszant = 3;
- int grpszbl = grpszant*(grpszant+1)/2;
- if (nbl > grpszbl) {
- nant = grpszant*(nbl+grpszbl-1)/grpszbl;
- } else {
- grpszant = nant;
- grpszbl = nbl;
- }
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- int lastant = grpszant;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (i%grpszbl == grpszbl-1) {
- st1 = lastant;
- st2 = lastant;
- lastant += grpszant;
- } else {
- if (++st2 == lastant) {
- st2 = ++st1;
- }
- }
- }
- Vector<String> antNames(nant);
- vector<MPosition> antPos(nant);
- Vector<double> antDiam(nant, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- itsStatUVW.resize (3, nant);
- for (int i=0; i<nant; ++i) {
- itsStatUVW(0,i) = 0.01 + i*0.02;
- itsStatUVW(1,i) = 0.05 + i*0.03;
- itsStatUVW(2,i) = 0.015 + i*0.025;
- }
- }
-
- void fillUVW (Matrix<double>& uvw, int count)
- {
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = (itsStatUVW(0,getInfo().getAnt2()[i]) + count*0.002 -
- (itsStatUVW(0,getInfo().getAnt1()[i]) + count*0.002));
- uvw(1,i) = (itsStatUVW(1,getInfo().getAnt2()[i]) + count*0.004 -
- (itsStatUVW(1,getInfo().getAnt1()[i]) + count*0.004));
- uvw(2,i) = (itsStatUVW(2,getInfo().getAnt2()[i]) + count*0.006 -
- (itsStatUVW(2,getInfo().getAnt1()[i]) + count*0.006));
- cout <<getInfo().getAnt1()[i]<<' '<<getInfo().getAnt2()[i]<<' '
- <<uvw(0,i)<<' '<<uvw(1,i)<<' '<<uvw(2,i)<<endl;
- }
- }
-
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- Matrix<double> uvw(3,itsNBl);
- fillUVW (uvw, itsCount);
- buf.setUVW (uvw);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void updateInfo (const DPInfo&) {}
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
- Matrix<double> itsStatUVW;
-};
-
-// Class to check result of null phase-shifted TestInput.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(TestInput* input,
- int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsInput(input),
- itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> result(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(result.size()); ++i) {
- result.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- Matrix<double> uvw(3,itsNBl);
- itsInput->fillUVW (uvw, itsCount);
- // Check the result.
- ASSERT (allNear(real(buf.getData()), real(result), 1e-7));
- ///cout << imag(buf.getData()) << endl<<imag(result);
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-7));
- ASSERT (allEQ(buf.getFlags(), itsFlag));
- ASSERT (near(buf.getTime(), 2.+5*itsCount));
- ASSERT (allNear(buf.getUVW(), uvw, 1e-7));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- MVDirection dir = infoIn.phaseCenter().getValue();
- ASSERT (near(dir.getLong("deg").getValue(), 45.));
- ASSERT (near(dir.getLat("deg").getValue(), 30.));
- }
-
- TestInput* itsInput;
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result of null phase-shifted TestInput.
-class TestOutput1: public DPStep
-{
-public:
- TestOutput1(TestInput* input,
- int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsInput(input),
- itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> result(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(result.size()); ++i) {
- result.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- Matrix<double> uvw(3,itsNBl);
- itsInput->fillUVW (uvw, itsCount);
- // Check the result.
- ASSERT (! allNear(real(buf.getData()), real(result), 1e-5));
- ASSERT (! allEQ(real(buf.getData()), real(result)));
- ///cout << imag(buf.getData()) << endl<<imag(result);
- ASSERT (! allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (! allEQ(imag(buf.getData()), imag(result)));
- ASSERT (allEQ(buf.getFlags(), itsFlag));
- ASSERT (near(buf.getTime(), 2.+5*itsCount));
- ASSERT (! allNear(buf.getUVW(), uvw, 1e-5));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- MVDirection dir = infoIn.phaseCenter().getValue();
- ASSERT (near(dir.getLong("deg").getValue(), 50.));
- ASSERT (near(dir.getLat("deg").getValue(), 35.));
- }
-
- TestInput* itsInput;
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test with a shift to the original center.
-void test1(int ntime, int nbl, int nchan, int ncorr, bool flag)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " flag=" << flag << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- // Keep phase center the same to be able to check if data are correct.
- parset.add ("phasecenter", "[45deg, 30deg]");
- DPStep::ShPtr step2(new PhaseShift(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(in, ntime, nbl, nchan, ncorr, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Test with a shift to another and then to the original phase center.
-void test2(int ntime, int nbl, int nchan, int ncorr, bool flag)
-{
- cout << "test2: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " flag=" << flag << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- // First shift to another center, then back to original.
- ParameterSet parset;
- parset.add ("phasecenter", "[50deg, 35deg]");
- ParameterSet parset1;
- parset1.add ("phasecenter", "[]");
- DPStep::ShPtr step2(new PhaseShift(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput1(in, ntime, nbl, nchan, ncorr, flag));
- DPStep::ShPtr step4(new PhaseShift(in, parset1, ""));
- DPStep::ShPtr step5(new TestOutput(in, ntime, nbl, nchan, ncorr, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- step3->setNextStep (step4);
- step4->setNextStep (step5);
- execute (step1);
-}
-
-
-int main()
-{
- try {
- test1(10, 3, 32, 4, false);
- test1(10, 10, 30, 1, true);
- test2(10, 6, 32, 4, false);
- test2(10, 6, 30, 1, true);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tPreFlagger.cc b/CEP/DP3/DPPP/test/tPreFlagger.cc
deleted file mode 100644
index f080f77abb4893522c3f47305a67d3d96f12d5bd..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPreFlagger.cc
+++ /dev/null
@@ -1,606 +0,0 @@
-//# tPreFlagger.cc: Test program for class PreFlagger
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/PreFlagger.h>
-#include <DPPP/Counter.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- DPStep::ShPtr step = step1;
- while (step) {
- step->show (cout);
- step = step->getNextStep();
- }
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {
- // Define start time 0.5 (= 3 - 0.5*5) and time interval 5.
- info().init (ncorr, nchan, ntime, 0.5, 5., string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanWidth(nchan, 100000.);
- Vector<double> chanFreqs(nchan);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 3); //same interval as in updateAveragInfo
- buf.setData (data);
- buf.setUVW (uvw);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&) {}
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result of flagged, unaveraged TestInput run by test1/3.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr,
- bool flag, bool clear, bool useComplement)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr),
- itsFlag(flag),
- itsClear(clear),
- itsUseComplement(useComplement)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // The result of the selected and complement data depends on the
- // settings of itsFlag, itsClear, and itsUseComplement as follows:
- // itsFlag itsUseComp itsClear sel comp
- // T T T T F *
- // F T T F F
- // T F T F T
- // F F T F F
- // T T F T T
- // F T F F T *
- // T F F T T
- // F F F T F
- // The lines marked with * are the cases in the if below.
- Cube<bool> result(itsNCorr,itsNChan,itsNBl);
- bool compFlag = itsFlag;
- bool selFlag = !itsClear;
- if (itsUseComplement && itsFlag == itsClear) {
- compFlag = !itsFlag;
- selFlag = itsClear;
- }
- result = compFlag;
- // All baselines except 2-2 should be selected.
- // Of them only channel 1,4,5 are selected.
- for (int i=0; i<itsNBl; ++i) {
- if (i%16 != 10) {
- for (int j=0; j<itsNChan; ++j) {
- if (j==1 || j==4 || j==5) {
- for (int k=0; k<itsNCorr; ++k) {
- result(k,j,i) = selFlag;
- }
- }
- }
- }
- }
- /// cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), result));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag, itsClear, itsUseComplement;
-};
-
-// Test flagging a few antennae and freqs.
-void test1(int ntime, int nbl, int nchan, int ncorr, bool flag,
- bool clear, bool useComplement)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " flag=" << flag
- << " clear=" << clear << " useComplement=" << useComplement << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- string mode;
- if (clear) {
- mode = (useComplement ? "clearComplement" : "clear");
- } else {
- mode = (useComplement ? "setComplement" : "set");
- }
- parset.add ("mode", mode);
- parset.add ("freqrange", "[ 1.1 .. 1.2 MHz, 1.5MHz+-65000Hz]");
- parset.add ("baseline", "[rs01.*, *s*.*2, rs02.s01]");
- parset.add ("countflag", "true");
- DPStep::ShPtr step2(new PreFlagger(in, parset, ""));
- DPStep::ShPtr step3(new Counter(in, parset, "cnt"));
- DPStep::ShPtr step4(new TestOutput(ntime, nbl, nchan, ncorr, flag,
- clear, useComplement));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- step3->setNextStep (step4);
- execute (step1);
- step2->showCounts (cout);
- step3->showCounts (cout);
-}
-
-
-// Class to check result of flagged, unaveraged TestInput run by test2.
-class TestOutput2: public DPStep
-{
-public:
- TestOutput2(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // A few baselines should be flagged (0, 7, 13, 15)
- // Furthermore channel 1,4,5,11,12,13 are flagged.
- Cube<bool> result(itsNCorr,itsNChan,itsNBl);
- result = false;
- for (int i=0; i<itsNBl; ++i) {
- if (i%16 == 0 || i%16 == 7 || i%16 == 13 || i%16 == 15) {
- for (int j=0; j<itsNChan; ++j) {
- if (j==4) {
- for (int k=0; k<itsNCorr; ++k) {
- result(k,j,i) = true;
- }
- }
- }
- }
- }
- ///cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), result));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-// Test flagging a few baselines, freqs, and channels.
-void test2(int ntime, int nbl, int nchan, int ncorr)
-{
- cout << "test2: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, false);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("freqrange", "[ 1.1 .. 1.2 MHz, 1.5MHz+-65000Hz]");
- parset.add ("chan", "[11..13, 4, 11, nchan/1000+1000..1000*nchan]");
- parset.add ("baseline", "[[rs01.*,rs01.*],[*s*.*2,*s*.*2],[*s*.*2,rs02.*]]");
- DPStep::ShPtr step2(new PreFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput2(ntime, nbl, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Test flagging a few antennae or freqs by using multiple steps.
-void test3(int ntime, int nbl, int nchan, int ncorr, bool flag)
-{
- cout << "test3: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " flag=" << flag << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("expr", "s1");
- parset.add ("s1.freqrange", "[ 1.1 .. 1.2 MHz, 1.5MHz+-65000Hz]");
- parset.add ("s1.expr", "s2");
- parset.add ("s1.s2.baseline", "[rs01.*, *s*.*2, rs02.s01]");
- DPStep::ShPtr step2(new PreFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr, flag,
- false, false));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-
-// Class to check result of flagged, unaveraged TestInput run by test4.
-class TestOutput4: public DPStep
-{
-public:
- TestOutput4(int ntime, int nbl, int nchan, int ncorr,
- bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr),
- itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // All baselines except autocorr should be flagged.
- // Furthermore channel 1,4,5 are flagged.
- Cube<bool> result(itsNCorr,itsNChan,itsNBl);
- result = true;
- for (int i=0; i<itsNBl; ++i) {
- if (i%16==0 || i%16==5 || i%16==10 || i%16==15) {
- for (int j=0; j<itsNChan; ++j) {
- if (j!=1 && j!=4 && j!=5) {
- for (int k=0; k<itsNCorr; ++k) {
- result(k,j,i) = itsFlag;
- }
- }
- }
- }
- }
- ///cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), result));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Test flagging a few antennae and freqs by using multiple steps.
-void test4(int ntime, int nbl, int nchan, int ncorr, bool flag)
-{
- cout << "test4: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " flag=" << flag << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("expr", "(s1&s1),(s2|s2)");
- parset.add ("s1.freqrange", "[ 1.1 .. 1.2 MHz, 1.5MHz+-65000Hz]");
- parset.add ("s2.baseline", "[rs01.*, *s*.*2, rs02.s01]");
- parset.add ("s2.corrtype", "cross");
- DPStep::ShPtr step2(new PreFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput4(ntime, nbl, nchan, ncorr, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-typedef bool CheckFunc (Complex value, double time, int ant1, int ant2,
- const double* uvw);
-
-// Class to check result of flagged, unaveraged TestInput run by test5.
-class TestOutput5: public DPStep
-{
-public:
- TestOutput5 (CheckFunc* cfunc)
- : itsCount(0),
- itsCFunc (cfunc)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- const Cube<Complex>& data = buf.getData();
- const double* uvw = buf.getUVW().data();
- const IPosition& shp = data.shape();
- Cube<bool> result(shp);
- for (int i=0; i<shp[2]; ++i) {
- int a1 = i/4;
- int a2 = i%4;
- for (int j=0; j<shp[1]; ++j) {
- bool flag = false;
- for (int k=0; k<shp[0]; ++k) {
- if (!flag) flag = itsCFunc(data(k,j,i), buf.getTime(), a1, a2,
- uvw+3*i);
- }
- for (int k=0; k<shp[0]; ++k) {
- result(k,j,i) = flag;
- }
- }
- }
- ///cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), result));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&) {}
-
- int itsCount;
- CheckFunc* itsCFunc;
-};
-
-// Test flagging on a single parameter.
-void test5(const string& key, const string& value, CheckFunc* cfunc)
-{
- cout << "test5: " << key << '=' << value << endl;
- // Create the steps.
- TestInput* in = new TestInput(2, 6, 5, 4, false);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add (key, value);
- DPStep::ShPtr step2(new PreFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput5(cfunc));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Test flagging on multiple parameters.
-void test6(const string& key1, const string& value1,
- const string& key2, const string& value2, CheckFunc* cfunc)
-{
- cout << "test6: " << key1 << '=' << value1 << ' '
- << key2 << '=' << value2 << endl;
- // Create the steps.
- TestInput* in = new TestInput(6, 10, 8, 4, false);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add (key1, value1);
- parset.add (key2, value2);
- DPStep::ShPtr step2(new PreFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput5(cfunc));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-bool checkBL(Complex, double, int a1, int a2, const double*)
- { return a1==a2; }
-bool checkAmplMin (Complex data, double, int, int, const double*)
- { return abs(data) < 9.5; }
-bool checkAmplMax (Complex data, double, int, int, const double*)
- { return abs(data) > 31.5; }
-bool checkPhaseMin (Complex data, double, int, int, const double*)
- { return arg(data) < 1.4; }
-bool checkPhaseMax (Complex data, double, int, int, const double*)
- { return arg(data) > 2.1; }
-bool checkRealMin (Complex data, double, int, int, const double*)
- { return real(data) < 5.5; }
-bool checkRealMax (Complex data, double, int, int, const double*)
- { return real(data) > 29.4; }
-bool checkImagMin (Complex data, double, int, int, const double*)
- { return imag(data) < -1.4; }
-bool checkImagMax (Complex data, double, int, int, const double*)
- { return imag(data) > 20.5; }
-bool checkUVMin (Complex, double, int, int, const double* uvw)
- { return sqrt(uvw[0]*uvw[0] + uvw[1]*uvw[1]) <= 30; }
-bool checkUVBL (Complex, double, int a1, int a2, const double* uvw)
- { return sqrt(uvw[0]*uvw[0] + uvw[1]*uvw[1]) >= 30 && (a1==0 || a2==0); }
-bool checkBLMin (Complex, double, int a1, int a2, const double*)
- { return abs(a1-a2) < 2; } // adjacent ant have bl<145
-bool checkBLMinMax (Complex, double, int a1, int a2, const double*)
- { return abs(a1-a2) != 1; } // adjacent ant have bl<145
-bool checkTimeSlot (Complex, double time, int, int, const double*)
- { return time<5; }
-bool checkNone (Complex, double, int, int, const double*)
- { return false; }
-bool checkAll (Complex, double, int, int, const double*)
- { return true; }
-
-// Test flagging on various fields.
-void testMany()
-{
- test5("corrtype", "auto", &checkBL);
- test5("amplmin", "9.5", &checkAmplMin);
- test5("amplmax", "31.5", &checkAmplMax);
- test5("phasemin", "1.4", &checkPhaseMin);
- test5("phasemax", "2.1", &checkPhaseMax);
- test5("realmin", "5.5", &checkRealMin);
- test5("realmax", "29.4", &checkRealMax);
- test5("imagmin", "-1.4", &checkImagMin);
- test5("imagmax", "20.5", &checkImagMax);
- test5("uvmmin", "30", &checkUVMin);
- test6("uvmmax", "30", "baseline", "[rs01.s01]", &checkUVBL);
- test5("timeslot", "0", &checkTimeSlot);
- test5("abstime", "17-nov-1858/0:0:2..17nov1858/0:0:4", &checkTimeSlot);
- test5("abstime", "17-nov-1858/0:0:3+-1s", &checkTimeSlot);
- test5("reltime", "0:0:0..0:0:6", &checkTimeSlot);
- test5("reltime", "0:0:2+-1s", &checkTimeSlot);
- test5("reltime", "0:0:2+-20s", &checkAll);
- test6("abstime", "17-nov-1858/0:0:20+-19s",
- "reltime", "0:0:20+-20s", &checkAll);
- test6("abstime", "17-nov-1858/0:0:3+-2s",
- "reltime", "0:0:20+-20s", &checkTimeSlot);
- test6("abstime", "17-nov-1858/0:0:20+-19s",
- "reltime", "0:0:3+-2s", &checkTimeSlot);
- test6("abstime", "17-nov-1858/0:0:20+-9s",
- "reltime", "0:0:3+-2s", &checkNone);
- // Elevation is 12738s; azimuth=86121s
- test5("elevation", "180deg..190deg", &checkNone);
- test5("elevation", "12730s..12740s", &checkAll);
- test5("azimuth", "180deg..190deg", &checkNone);
- test5("azimuth", "86120s..86125s", &checkAll);
- test6("azimuth", "86120s..86125s", "elevation", "180deg..190deg", &checkNone);
- test6("azimuth", "86120s..86125s", "elevation", "12730s..12740s", &checkAll);
- test5("lst", "0.154d..0.155d", &checkAll);
- test5("blmin", "145", &checkBLMin);
- test6("blmin", "10", "blmax", "145", &checkBLMinMax);
-}
-
-int main()
-{
- INIT_LOGGER ("tPreFlagger");
- try {
-
- test1(10, 16, 32, 4, false, true, false);
- test1(10, 16, 32, 4, true, true, false);
- test1(10, 16, 32, 4, false, false, false);
- test1(10, 16, 32, 4, true, false, false);
- test1(10, 16, 32, 4, false, true, true);
- test1(10, 16, 32, 4, true, true, true);
- test1(10, 16, 32, 4, false, false, true);
- test1(10, 16, 32, 4, true, false, true);
- test2( 2, 16, 32, 4);
- test2( 2, 36, 16, 2);
- test3( 3, 16, 32, 4, false);
- test3( 4, 16, 4, 2, true);
- test4( 3, 16, 32, 4, false);
- testMany();
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tPreFlagger.sh b/CEP/DP3/DPPP/test/tPreFlagger.sh
deleted file mode 100755
index 9c4ead9986c7e24ae021e8254b1ecfa8021d6f50..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPreFlagger.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tPreFlagger
diff --git a/CEP/DP3/DPPP/test/tPredict.run b/CEP/DP3/DPPP/test/tPredict.run
deleted file mode 100755
index f8f77c514d6965d6c4194ed1e181dc0170662071..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPredict.run
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/bin/bash
-
-# Get the taql executable and srcdir (script created by cmake's CONFIGURE_FILE).
-source findenv.run_script
-echo "srcdirx=$rt_srcdir"
-
-# Set srcdir if not defined (in case run by hand).
-if test "$srcdir" = ""; then
- srcdir="$rt_srcdir"
-fi
-
-if test ! -f ${srcdir}/tNDPPP-generic.in_MS.tgz; then
- exit 3 # untested
-fi
-
-rm -rf tPredict_tmp
-mkdir -p tPredict_tmp
-# Unpack the MS and other files and do the DPPP run.
-cd tPredict_tmp
-tar zxf ${srcdir}/tNDPPP-generic.in_MS.tgz
-tar zxf ${srcdir}/tPredict.tab.tgz
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-
-echo; echo "Test with beam, subtract"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=MODEL_DATA steps=[predict] predict.sourcedb=tNDPPP-generic.MS/sky predict.usebeammodel=true predict.operation=subtract'
-echo $cmd
-$cmd
-# Compare the MODEL_DATA column of the output MS with the original data minus the BBS reference output.
-taqlcmd='select from tNDPPP-generic.MS t1, tPredict.tab t2 where not all(near(t1.MODEL_DATA,t1.DATA-t2.PREDICT_beam,5e-2) || (isnan(t1.DATA) && isnan(t2.PREDICT_beam)))'
-echo $taqlcmd
-$taqlexe $taqlcmd > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test without beam, add"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=MODEL_DATA steps=[predict] predict.sourcedb=tNDPPP-generic.MS/sky predict.usebeammodel=false predict.operation=add'
-echo $cmd
-$cmd
-# Compare the MODEL_DATA column of the output MS with the original data plus the BBS reference output.
-taqlcmd='select from tNDPPP-generic.MS t1, tPredict.tab t2 where not all(near(t1.MODEL_DATA,t1.DATA+t2.PREDICT_nobeam,5e-2) || (isnan(t1.DATA) && isnan(t2.PREDICT_beam)))'
-echo $taqlcmd
-$taqlexe $taqlcmd > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test without beam, with applycal, subtract (like peeling)"; echo
-parmdbm <<EOL
-open table="tPredict.parmdb"
-adddef Gain:0:0:Real values=3.
-adddef Gain:1:1:Real values=3.
-EOL
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. msout.datacolumn=MODEL_DATA steps=[predict] predict.sourcedb=tNDPPP-generic.MS/sky predict.applycal.parmdb=tPredict.parmdb predict.operation=subtract'
-echo $cmd
-$cmd
-# Compare the MODEL_DATA column of the output MS with the original data minus the BBS reference output.
-taqlcmd='select from tNDPPP-generic.MS t1, tPredict.tab t2 where not all(near(t1.MODEL_DATA,t1.DATA-9.0*t2.PREDICT_nobeam,5e-2) || (isnan(t1.DATA) && isnan(t2.PREDICT_nobeam)))'
-echo $taqlcmd
-$taqlexe $taqlcmd > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test without beam"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. steps=[predict] predict.sourcedb=tNDPPP-generic.MS/sky'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the BBS reference output.
-taqlcmd='select from tNDPPP-generic.MS t1, tPredict.tab t2 where not all(near(t1.DATA,t2.PREDICT_nobeam,5e-2) || (isnan(t1.DATA) && isnan(t2.PREDICT_nobeam)))'
-echo $taqlcmd
-$taqlexe $taqlcmd > taql.out
-diff taql.out taql.ref || exit 1
-
-echo; echo "Test with beam"; echo
-cmd='NDPPP msin=tNDPPP-generic.MS msout=. steps=[predict] predict.sourcedb=tNDPPP-generic.MS/sky predict.usebeammodel=true'
-echo $cmd
-$cmd
-# Compare the DATA column of the output MS with the BBS reference output.
-taqlcmd='select from tNDPPP-generic.MS t1, tPredict.tab t2 where not all(near(t1.DATA,t2.PREDICT_beam,5e-2) || (isnan(t1.DATA) && isnan(t2.PREDICT_beam)))' > taql.out
-echo $taqlcmd
-$taqlexe $taqlcmd > taql.out
-diff taql.out taql.ref || exit 1
diff --git a/CEP/DP3/DPPP/test/tPredict.sh b/CEP/DP3/DPPP/test/tPredict.sh
deleted file mode 100755
index 7a13c4cc6f601c334ae2795d9bdc14756f492e50..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPredict.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tPredict
diff --git a/CEP/DP3/DPPP/test/tPredict.tab.tgz b/CEP/DP3/DPPP/test/tPredict.tab.tgz
deleted file mode 100644
index 0fb13910905d98f131ab95030842cdf70d4f06dd..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP/test/tPredict.tab.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP/test/tPredict_ref b/CEP/DP3/DPPP/test/tPredict_ref
deleted file mode 100755
index 5c5cd3d6e998c76dab39927947c4a1cec2908350..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPredict_ref
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/bin/bash
-
-#wfeaw Generate reference data tPredict.tab for tPredict
-# This script uses BBS. Note that casacore data should be up-to-date
-# Run the script in the source directory of CEP/DP3/DPPP/test
-
-tar xf tNDPPP-generic.in_MS.tgz
-#gsm.py tPredict_ref_skymodel 01:37:41.299440 +33.09.35.13240 4
-rm -rf tNDPPP-generic.MS/sky
-makesourcedb in=my.skymodel out=tNDPPP-generic.MS/sky format='<'
-rm -rf tNDPPP-generic.in_MS.tgz
-tar czf tNDPPP-generic.in_MS.tgz tNDPPP-generic.MS
-
-rm -rf tPredict.tab.tgz tPredict.tab
-
-#### Run BBS, do not corrupt data with beam
-
-cat > tPredict-bbs-nobeam.parset <<EOF
-Strategy.ChunkSize=100
-Strategy.Steps=[predict_nobeam]
-
-Step.predict_nobeam.Operation=PREDICT
-Step.predict_nobeam.Model.Beam.Enable=FALSE
-Step.predict_nobeam.Output.Column=PREDICT_nobeam
-EOF
-
-bbs-reducer --sourcedb=tNDPPP-generic.MS/sky tNDPPP-generic.MS tPredict-bbs-nobeam.parset
-
-rm tPredict-bbs-nobeam.parset
-
-#### Run BBS, corrupt data with beam
-
-cat > tPredict-bbs-beam.parset <<EOF
-Strategy.ChunkSize=100
-Strategy.Steps=[predict_beam]
-
-Step.predict_beam.Operation=PREDICT
-Step.predict_beam.Model.Beam.Enable=TRUE
-Step.predict_beam.Model.Beam.UseChannelFreq=TRUE
-Step.predict_beam.Output.Column=PREDICT_beam
-EOF
-
-bbs-reducer --sourcedb=tNDPPP-generic.MS/sky tNDPPP-generic.MS tPredict-bbs-beam.parset
-
-rm tPredict-bbs-beam.parset
-
-#### Run BBS, corrupt data with beam, subtract from data
-
-cat > tPredict-bbs-beam-subtract.parset <<EOF
-Strategy.ChunkSize=100
-Strategy.Steps=[subtract_beam]
-
-Step.subtract_beam.Operation=PREDICT
-Step.subtract_beam.Model.Beam.Enable=TRUE
-Step.subtract_beam.Model.Beam.UseChannelFreq=TRUE
-Step.subtract_beam.Output.Column=SUBTRACT_beam
-EOF
-
-bbs-reducer --sourcedb=tNDPPP-generic.MS/sky tNDPPP-generic.MS tPredict-bbs-beam.parset
-
-#### Store output from BBS in separate table
-
-taql 'select from (select PREDICT_nobeam, PREDICT_beam, SUBTRACT_BEAM from tNDPPP-generic.MS giving tPredict.tab as plain)'
-
-tar czf tPredict.tab.tgz tPredict.tab
diff --git a/CEP/DP3/DPPP/test/tPredict_ref_skymodel b/CEP/DP3/DPPP/test/tPredict_ref_skymodel
deleted file mode 100644
index 864b814969fc876bdc449185b235216498c78719..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tPredict_ref_skymodel
+++ /dev/null
@@ -1,16 +0,0 @@
-FORMAT = Name, Type, Ra, Dec, I, Q, U, V, ReferenceFrequency='60e6', SpectralIndex='[0.0]', MajorAxis, MinorAxis, Orientation
-
-# the next lines define the sources
-0002.2+3139, POINT, 00:02:14.14080000, +31.39.42.01200000, 4.845, , , , , [-0.6711, -0.1114]
-0005.9+3139, POINT, 00:05:55.73040000, +31.39.50.00400000, 5.0769, , , , , [-0.3954, -0.1446]
-0010.1+3045, GAUSSIAN, 00:10:07.75920000, +30.45.24.08400000, 4.7809, , , , , [-0.7518, -0.0461], 35.9, 38.7, 116.3
-0010.4+3329, GAUSSIAN, 00:10:25.44000000, +33.29.39.51600000, 8.5526, , , , , [-0.8742], 93.2, 32.3, 4.6
-0013.7+3441, GAUSSIAN, 00:13:43.97040000, +34.41.34.29600000, 10.94, , , , , [-0.7], 42.9, 27.4, 14.0
-0015.1+3216, POINT, 00:15:06.31920000, +32.16.12.21600000, 12.0884, , , , , [-0.5168, -0.0805]
-0016.2+3238, GAUSSIAN, 00:16:12.48960000, +32.38.56.50800000, 8.1975, , , , , [-0.7735], 56.4, 37.6, 144.7
-0016.5+3259, GAUSSIAN, 00:16:30.97920000, +32.59.53.48400000, 6.1083, , , , , [-0.8603], 39.9, 31.5, 104.2
-0017.0+3209, POINT, 00:17:02.20080000, +32.09.17.78400000, 6.1197, , , , , [-0.6543, -0.1483]
-0018.5+3105, GAUSSIAN, 00:18:31.74960000, +31.05.53.59200000, 5.4645, , , , , [-0.809, -0.02], 52.7, 32.5, 7.7
-2352.3+3303, GAUSSIAN, 23:52:20.92080000, +33.03.59.61600000, 11.3143, , , , , [-0.7843, -0.045], 59.2, 28.0, 160.1
-2354.1+3255, GAUSSIAN, 23:54:11.35920000, +32.55.13.58400000, 22.5335, , , , , [-0.8153, -0.0882], 46.4, 31.5, 44.1
-2358.2+3130, POINT, 23:58:13.49040000, +31.30.00.50400000, 4.9352, , , , , [-0.5184, -0.1986]
diff --git a/CEP/DP3/DPPP/test/tScaleData.cc b/CEP/DP3/DPPP/test/tScaleData.cc
deleted file mode 100644
index 4f6b31fdd14573b7542b2966e7fb3257b96baff7..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tScaleData.cc
+++ /dev/null
@@ -1,263 +0,0 @@
-//# tScaleData.cc: Test program for class ScaleData
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tScaleData.cc 23691 2013-02-12 13:32:33Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/ScaleData.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <Common/StreamUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {
- info().init (ncorr, nchan, ntime, 0., 5., string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanWidth(nchan, 1000000.);
- Vector<double> chanFreqs(nchan);
- indgen (chanFreqs, 10500000., 1000000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- indgen (weights, 0.5f, 0.01f);
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- DPBuffer buf;
- buf.setTime (itsCount*30 + 4472025740.0);
- buf.setData (data);
- buf.setWeights (weights);
- buf.setUVW (uvw);
- Cube<bool> flags(data.shape());
- flags = false;
- buf.setFlags (flags);
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = false;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&) {}
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-// Class to check result of TestInput run by test1.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {}
-private:
- void addData (Cube<Complex>& to, const Cube<Complex>& from, int bl)
- {
- to += from(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl));
- }
- void addConjData (Cube<Complex>& to, const Cube<Complex>& from, int bl)
- {
- to += conj(from(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl)));
- }
- virtual bool process (const DPBuffer& buf)
- {
- // Fill data and scale as needed.
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- Complex* dataPtr = data.data();
- int cnt=0;
- for (int i=0; i<itsNBl; ++i) {
- double freq = 10.5;
- for (int j=0; j<itsNChan; ++j) {
- double sc1 = 3 + 2*freq + freq*freq;
- double sc2 = sc1;
- if ((i%16)/4 == 0) {
- sc1 = 2 + 0.5*freq;
- }
- if (i%4 == 0) {
- sc2 = 2 + 0.5*freq;
- }
- double scale = sqrt(sc1*sc2);
- freq += 1;
- for (int k=0; k<itsNCorr; ++k) {
- *dataPtr++ = Complex(cnt+itsCount*10,cnt-10+itsCount*6) * scale;
- cnt++;
- }
- }
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- indgen (weights, 0.5f, 0.01f);
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- ASSERT (allEQ (buf.getData(), data));
- ASSERT (buf.getFlags().shape() == IPosition(3,itsNCorr,itsNChan,itsNBl));
- ASSERT (allEQ (buf.getFlags(), false));
- ASSERT (allEQ (buf.getWeights(), weights));
- ASSERT (allEQ (buf.getUVW(), uvw));
- ASSERT (buf.getFullResFlags().shape() == IPosition(3,itsNChan,1,itsNBl));
- ASSERT (allEQ (buf.getFullResFlags(), false));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- ASSERT (int(infoIn.nbaselines())==itsNBl);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- step1->getNextStep()->show (cout);
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test scaling.
-void test1(int ntime, int nbl, int nchan, int ncorr)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("stations", "[rs01.s01, *]");
- parset.add ("coeffs", "[[2,0.5],[3,2,1]]");
- parset.add ("scalesize", "false");
- DPStep::ShPtr step2(new ScaleData(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-
-int main()
-{
- INIT_LOGGER ("tScaleData");
- try {
- test1 ( 2, 4, 4, 1);
- test1 (10, 16, 32, 4);
- test1 (10, 12, 16, 2);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tStationAdder.cc b/CEP/DP3/DPPP/test/tStationAdder.cc
deleted file mode 100644
index 82eb1524ed2b4a6182f9261488c43c951a5f5716..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tStationAdder.cc
+++ /dev/null
@@ -1,579 +0,0 @@
-//# tStationAdder.cc: Test program for class StationAdder
-//# Copyright (C) 2012
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/StationAdder.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <Common/StreamUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {
- info().init (ncorr, nchan, ntime, 0., 5., string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanWidth(nchan, 1000000.);
- Vector<double> chanFreqs(nchan);
- indgen (chanFreqs, 10500000., 1000000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- indgen (weights, 0.5f, 0.01f);
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- DPBuffer buf;
- buf.setTime (itsCount*30 + 4472025740.0);
- buf.setData (data);
- buf.setWeights (weights);
- buf.setUVW (uvw);
- Cube<bool> flags(data.shape());
- flags = false;
- buf.setFlags (flags);
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = false;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&) {}
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-// Class to check result of TestInput run by test1.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr, bool sumauto)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsSumAuto(sumauto)
- {}
-private:
- void addData (Cube<Complex>& to, const Cube<Complex>& from, int bl)
- {
- to += from(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl));
- }
- void addConjData (Cube<Complex>& to, const Cube<Complex>& from, int bl)
- {
- to += conj(from(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl)));
- }
- virtual bool process (const DPBuffer& buf)
- {
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- indgen (weights, 0.5f, 0.01f);
- Cube<Complex> databl0 (itsNCorr, itsNChan, 1);
- Cube<Complex> databl1 (itsNCorr, itsNChan, 1);
- // "{ns:[rs01.s01, rs02.s01, cs01.s02]}" was given resulting in 2 new
- // baselines (ns-ns and cs01.s01-ns).
- // Thus adding the baselines below.
- float weight=0;
- if (itsSumAuto) {
- // add autocorr to form new autocorr
- addData (databl0, data, 0);
- addData (databl0, data, 5);
- addData (databl0, data, 15);
- weight = 3;
- } else {
- // add crosscorr to form new autocorr
- addData (databl0, data, 1);
- addData (databl0, data, 3);
- addData (databl0, data, 4);
- addData (databl0, data, 7);
- addData (databl0, data, 12);
- addData (databl0, data, 13);
- weight = 6;
- }
- addData (databl1, data, 8);
- addData (databl1, data, 9);
- addData (databl1, data, 11);
- addConjData (databl1, data, 2);
- addConjData (databl1, data, 6);
- addConjData (databl1, data, 14);
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- IPosition end(3,itsNCorr-1,itsNChan-1,itsNBl-1);
- ASSERT (allEQ (buf.getData()(IPosition(3,0), end), data));
- ASSERT (buf.getFlags().shape() == IPosition(3,itsNCorr,itsNChan,itsNBl+2));
- ASSERT (allEQ (buf.getFlags(), false));
- ASSERT (allEQ (buf.getWeights()(IPosition(3,0), end), weights));
- ASSERT (allEQ (buf.getUVW()(IPosition(2,0),
- IPosition(2,2,itsNBl-1)), uvw));
- ASSERT (buf.getFullResFlags().shape() == IPosition(3,itsNChan,1,itsNBl+2));
- ASSERT (allEQ (buf.getFullResFlags(), false));
- // Now check data of new baselines.
- end[2] = itsNBl;
- ASSERT (allNear (buf.getData()(IPosition(3,0,0,itsNBl), end), databl0/weight, 1e-5));
- ASSERT (allNear (buf.getWeights()(IPosition(3,0,0,itsNBl), end), weight, 1e-5));
- end[2] = itsNBl+1;
- ASSERT (allNear (buf.getData()(IPosition(3,0,0,itsNBl+1), end), databl1/6.f, 1e-5));
- ASSERT (allNear (buf.getWeights()(IPosition(3,0,0,itsNBl+1), end), 6.f, 1e-5));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- ASSERT (int(infoIn.nbaselines())==itsNBl+2);
- ASSERT (int(infoIn.antennaNames().size())==5);
- ASSERT (int(infoIn.antennaDiam().size())==5);
- ASSERT (int(infoIn.antennaPos().size())==5);
- ASSERT (infoIn.antennaNames()[4]=="ns");
- Vector<Double> pos1 (infoIn.antennaPos()[4].getValue().getValue());
- ASSERT (near(pos1[0], (3828763.+3828746.+3828713.)/3));
- ASSERT (near(pos1[1], ( 442449.+ 442592.+ 442878.)/3));
- ASSERT (near(pos1[2], (5064923.+5064924.+5064926.)/3));
- // Check diam.
- double d1 = sqrt ((pos1[0]-3828763) * (pos1[0]-3828763) +
- (pos1[1]- 442449) * (pos1[1]- 442449) +
- (pos1[2]-5064923) * (pos1[2]-5064923));
- double d2 = sqrt ((pos1[0]-3828746) * (pos1[0]-3828746) +
- (pos1[1]- 442592) * (pos1[1]- 442592) +
- (pos1[2]-5064924) * (pos1[2]-5064924));
- double d3 = sqrt ((pos1[0]-3828713) * (pos1[0]-3828713) +
- (pos1[1]- 442878) * (pos1[1]- 442878) +
- (pos1[2]-5064926) * (pos1[2]-5064926));
- ASSERT (near(infoIn.antennaDiam()[4], 70+2*max(d1,max(d2,d3))));
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsSumAuto;
-};
-
-// Class that throws an error when process() is called
-class ThrowStep: public DPStep
-{
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual bool process (const DPBuffer&) {
- cout<<"Previous step should have thrown an error!"<<endl;
- return true;
- }
-};
-
-// Class to check result of flagged, unaveraged TestInput run by test2.
-class TestOutput2: public DPStep
-{
-public:
- TestOutput2(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {}
-private:
- void addData (Cube<Complex>& to, const Cube<Complex>& from,
- Cube<Float>& tow, const Cube<Float>& weights, int bl)
- {
- Cube<Complex> tmp=from.copy();
- Cube<Complex>::iterator tmpit=tmp.begin();
- Cube<Float>::const_iterator weightit=weights.begin();
- for (; tmpit!=tmp.end() && weightit!=weights.end(); tmpit++, weightit++) {
- *tmpit *= *weightit;
- }
- to += tmp(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl));
- tow += weights(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl));
- }
- void addConjData (Cube<Complex>& to, const Cube<Complex>& from,
- Cube<Float>& tow, const Cube<Float>& weights, int bl)
- {
- Cube<Complex> tmp=from.copy();
- Cube<Complex>::iterator tmpit=tmp.begin();
- Cube<Float>::const_iterator weightit=weights.begin();
- for (; tmpit!=tmp.end() && weightit!=weights.end(); tmpit++, weightit++) {
- *tmpit *= *weightit;
- }
- to += conj(tmp(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl)));
- tow += weights(IPosition(3,0,0,bl), IPosition(3,to.nrow()-1,to.ncolumn()-1,bl));
- }
- virtual bool process (const DPBuffer& buf)
- {
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- Cube<Float> weights(itsNCorr, itsNChan, itsNBl);
- indgen (weights, 0.5f, 0.01f);
- Cube<Complex> databl0 (itsNCorr, itsNChan, 1);
- Cube<Complex> databl1 (itsNCorr, itsNChan, 1);
- Cube<Complex> databl2 (itsNCorr, itsNChan, 1);
- Cube<Complex> databl3 (itsNCorr, itsNChan, 1);
- Cube<Complex> databl4 (itsNCorr, itsNChan, 1);
- Cube<Float> weightbl0 (itsNCorr, itsNChan, 1, 0.);
- Cube<Float> weightbl1 (itsNCorr, itsNChan, 1, 0.);
- Cube<Float> weightbl2 (itsNCorr, itsNChan, 1, 0.);
- Cube<Float> weightbl3 (itsNCorr, itsNChan, 1, 0.);
- Cube<Float> weightbl4 (itsNCorr, itsNChan, 1, 0.);
- // "{ns1:[rs01.s01, rs02.s01], ns2:[cs01.s02, cs01.s01]}" was given.
- addData (databl0, data, weightbl0, weights, 8);
- addData (databl0, data, weightbl0, weights, 9);
- addData (databl1, data, weightbl1, weights, 12);
- addData (databl1, data, weightbl1, weights, 13);
- addData (databl2, data, weightbl2, weights, 2);
- addData (databl2, data, weightbl2, weights, 3);
- addData (databl3, data, weightbl3, weights, 6);
- addData (databl3, data, weightbl3, weights, 7);
- addConjData (databl0, data, weightbl0, weights, 2);
- addConjData (databl0, data, weightbl0, weights, 6);
- addConjData (databl1, data, weightbl1, weights, 3);
- addConjData (databl1, data, weightbl1, weights, 7);
- addConjData (databl2, data, weightbl2, weights, 8);
- addConjData (databl2, data, weightbl2, weights, 12);
- addConjData (databl3, data, weightbl3, weights, 9);
- addConjData (databl3, data, weightbl3, weights, 13);
- addConjData (databl4, databl0, weightbl4, weightbl0, 0);
- addConjData (databl4, databl1, weightbl4, weightbl1, 0);
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- IPosition end(3,itsNCorr-1,itsNChan-1,itsNBl-1);
- ASSERT (allEQ (buf.getData()(IPosition(3,0), end), data));
- ASSERT (buf.getFlags().shape() == IPosition(3,itsNCorr,itsNChan,itsNBl+5));
- ASSERT (allEQ (buf.getFlags(), false));
- ASSERT (allEQ (buf.getWeights()(IPosition(3,0), end), weights));
- ASSERT (allEQ (buf.getUVW()(IPosition(2,0),
- IPosition(2,2,itsNBl-1)), uvw));
- ASSERT (buf.getFullResFlags().shape() == IPosition(3,itsNChan,1,itsNBl+5));
- ASSERT (allEQ (buf.getFullResFlags(), false));
- // Now check data of new baselines.
- end[2] = itsNBl;
- cout<< buf.getUVW()(IPosition(2,0,itsNBl-1), IPosition(2,2,itsNBl+4));
- ASSERT (allNear (buf.getData()(IPosition(3,0,0,itsNBl), end), databl0, 1e-5));
- ASSERT (allNear (buf.getWeights()(IPosition(3,0,0,itsNBl), end), weightbl0, 1e-5));
- end[2] = itsNBl+1;
- ASSERT (allNear (buf.getData()(IPosition(3,0,0,itsNBl+1), end), databl1, 1e-5));
- ASSERT (allNear (buf.getWeights()(IPosition(3,0,0,itsNBl+1), end), weightbl1, 1e-5));
- end[2] = itsNBl+2;
- ASSERT (allNear (buf.getData()(IPosition(3,0,0,itsNBl+2), end), databl2, 1e-5));
- ASSERT (allNear (buf.getWeights()(IPosition(3,0,0,itsNBl+2), end), weightbl2, 1e-5));
- end[2] = itsNBl+3;
- ASSERT (allNear (buf.getData()(IPosition(3,0,0,itsNBl+3), end), databl3, 1e-5));
- ASSERT (allNear (buf.getWeights()(IPosition(3,0,0,itsNBl+3), end), weightbl3, 1e-5));
- end[2] = itsNBl+4;
- ASSERT (allNear (buf.getData()(IPosition(3,0,0,itsNBl+4), end), databl4, 1e-5));
- ASSERT (allNear (buf.getWeights()(IPosition(3,0,0,itsNBl+4), end), weightbl4, 1e-5));
- itsCount++;
- return true;
- ///cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), false));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- ASSERT (int(infoIn.nbaselines())==itsNBl+5);
- ASSERT (int(infoIn.antennaNames().size())==6);
- ASSERT (infoIn.antennaNames()[4]=="ns1");
- ASSERT (infoIn.antennaNames()[5]=="ns2");
- Vector<Double> pos1 (infoIn.antennaPos()[4].getValue().getValue());
- ASSERT (near(pos1[0], (3828763.+3828746.)/2));
- ASSERT (near(pos1[1], ( 442449.+ 442592.)/2));
- ASSERT (near(pos1[2], (5064923.+5064924.)/2));
- Vector<Double> pos2 (infoIn.antennaPos()[5].getValue().getValue());
- ASSERT (near(pos2[0], (3828729.+3828713.)/2));
- ASSERT (near(pos2[1], ( 442735.+ 442878.)/2));
- ASSERT (near(pos2[2], (5064925.+5064926.)/2));
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-// Class to check result of TestInput run by test4.
-class TestOutput4: public DPStep
-{
-public:
- TestOutput4(int ntime, int nbl, int nchan, int /*ncorr*/)
- : itsNTime(ntime), itsNBl(nbl), itsNChan(nchan)
- {}
-private:
- virtual bool process (const DPBuffer&)
- {
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- ASSERT (int(infoIn.nbaselines())==itsNBl);
- ASSERT (int(infoIn.antennaNames().size())==4);
- ASSERT (int(infoIn.antennaDiam().size())==4);
- ASSERT (int(infoIn.antennaPos().size())==4);
- }
-
- int itsNTime, itsNBl, itsNChan;
-};
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- step1->getNextStep()->show (cout);
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test adding 3 stations.
-void test1(int ntime, int nbl, int nchan, int ncorr, bool sumauto)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " sumauto=" << sumauto << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("stations",
- "{ns:[rs01.s01, rs02.s01, cs01.s02]}");
- parset.add ("autocorr", "true");
- if (!sumauto) {
- parset.add ("sumauto", "false");
- }
- parset.add ("average", "true");
- parset.add ("useweights", "false");
- DPStep::ShPtr step2(new StationAdder(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr, sumauto));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Test adding two groups of 2 stations.
-void test2(int ntime, int nbl, int nchan, int ncorr)
-{
- cout << "test2: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("stations",
- "{ns1:[rs01.s01, rs02.s01], ns2:[cs01.s02, cs01.s01]}");
- parset.add ("autocorr", "false");
- parset.add ("average", "false");
- DPStep::ShPtr step2(new StationAdder(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput2(ntime, nbl, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
- step2->showCounts (cout);
-}
-
-void test3 (const string& stations)
-{
- // Do some erronous attempts.
- TestInput* in = new TestInput(2, 8, 4, 4);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("stations", stations);
- parset.add ("autocorr", "true");
- parset.add ("average", "false");
- DPStep::ShPtr step2(new StationAdder(in, parset, ""));
- DPStep::ShPtr step3(new ThrowStep());
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- bool ok = true;
- try {
- execute (step1);
- } catch (std::exception& x) {
- cout << "Expected exception: " << x.what() << endl;
- ok = false;
- }
- ASSERT (!ok);
-}
-
-// Test making a superstation out of nonexisting stations (should do nothing)
-void test4(int ntime, int nbl, int nchan, int ncorr)
-{
- cout << "test4: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("stations",
- "{ns1:nonexistingstationpattern}");
- DPStep::ShPtr step2(new StationAdder(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput4(ntime, nbl, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-void testPatterns()
-{
- Vector<String> antNames(10);
- antNames[0] = "CS001HBA0"; antNames[1] = "CS001HBA1";
- antNames[2] = "CS002HBA0"; antNames[3] = "CS002HBA1";
- antNames[4] = "CS003HBA0"; antNames[5] = "CS003HBA1";
- antNames[6] = "CS004HBA0"; antNames[7] = "CS004HBA1";
- antNames[8] = "CS005HBA0"; antNames[9] = "CS005HBA1";
- vector<string> patterns;
- patterns.push_back ("CS00[0-9]*");
- cout << StationAdder::getMatchingStations (antNames, patterns) << endl;
- patterns[0] = "CS00[0-9]*";
- cout << StationAdder::getMatchingStations (antNames, patterns) << endl;
- patterns.push_back ("!CS00[45]*");
- cout << StationAdder::getMatchingStations (antNames, patterns) << endl;
- patterns.push_back ("CS00[124]HBA0");
- cout << StationAdder::getMatchingStations (antNames, patterns) << endl;
-}
-
-
-int main()
-{
- INIT_LOGGER ("tStationAdder");
- try {
- // Test the station selection patterns.
- testPatterns();
- // Test must be done with with 16 baselines.
- test1( 10, 16, 32, 4, true);
- test1( 10, 16, 32, 4, false);
- test2( 10, 16, 32, 4);
- // Unknown station.
- //test3("{ns1:unknown, ns2:[cs01.s02, cs01.s01]}");
- // New station already used.
- test3("{ns1:[rs01.s01, rs02.s01], cs01.s02:[cs01.s02, cs01.s01]}");
- // Old station doubly used.
- test3("{ns1:[rs01.s01, rs02.s01], ns2:[rs01.s01, cs01.s01]}");
- test4( 10, 16, 32, 4);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tUVWFlagger.cc b/CEP/DP3/DPPP/test/tUVWFlagger.cc
deleted file mode 100644
index 56da782026f50e4dfdb935882f926891144c7aa0..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tUVWFlagger.cc
+++ /dev/null
@@ -1,438 +0,0 @@
-//# tUVWFlagger.cc: Test program for class UVWFlagger
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/UVWFlagger.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {
- info().init (ncorr, nchan, ntime, 0., 5., string(), string());
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(nbl);
- Vector<Int> ant2(nbl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<nbl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos(4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanWidth(nchan, 1000000.);
- Vector<double> chanFreqs(nchan);
- indgen (chanFreqs, 10500000., 1000000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-10+itsCount*6);
- }
- Matrix<double> uvw(3, itsNBl);
- for (int i=0; i<itsNBl; ++i) {
- uvw(0,i) = 1 + itsCount + i;
- uvw(1,i) = 2 + itsCount + i;
- uvw(2,i) = 3 + itsCount + i;
- }
- DPBuffer buf;
- buf.setTime (itsCount*30 + 4472025740.0);
- buf.setData (data);
- buf.setUVW (uvw);
- Cube<bool> flags(data.shape());
- flags = false;
- buf.setFlags (flags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&) {}
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-// Class to check result of flagged, unaveraged TestInput run by test1.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // Flag where u,v,w matches intervals given in test1.
- Cube<bool> result(itsNCorr,itsNChan,itsNBl);
- result = false;
- for (int i=0; i<itsNBl; ++i) {
- double u = 1+i+itsCount;
- double v = 2+i+itsCount;
- double w = 3+i+itsCount;
- double uv = sqrt(u*u+v*v);
- if ((uv>5.5 && uv<8.5) || (u>20.5 && u<23.5) || (u>31.5 && u<40.5)
- || (v>11.5 && v<14.5) || w<3.5 || w>44.5) {
- for (int j=0; j<itsNChan; ++j) {
- for (int k=0; k<itsNCorr; ++k) {
- result(k,j,i) = true;
- }
- }
- }
- }
- ///cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), result));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-// Class to check result of flagged, unaveraged TestInput run by test2.
-class TestOutput2: public DPStep
-{
-public:
- TestOutput2(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // Flag where u,v,w matches intervals given in test1.
- Cube<bool> result(itsNCorr,itsNChan,itsNBl);
- result = false;
- for (int i=0; i<itsNBl; ++i) {
- for (int j=0; j<itsNChan; ++j) {
- double wavel = 2.99792458e+08 / (10.5e6 + j*1e6);
- double u = (1+i+itsCount) / wavel;
- double v = (2+i+itsCount) / wavel;
- double w = (3+i+itsCount) / wavel;
- double uv = sqrt(u*u+v*v);
- if ((uv>0.2 && uv<0.31) || (u>1.55 && u<1.485) || (u>0.752 && u<0.862)
- || (v>0.42 && v<0.53) || w<0.12 || w>1.63) {
- for (int k=0; k<itsNCorr; ++k) {
- result(k,j,i) = true;
- }
- }
- }
- }
- ///cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), result));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-
-// Class to check result of flagged, unaveraged TestInput run by test3.
-class TestOutput3: public DPStep
-{
-public:
- TestOutput3(int ntime, int nbl, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr)
- {
- ASSERT (ntime==2 && nbl==16);
- }
-private:
- virtual bool process (const DPBuffer& buf)
- {
- // These are the UVW coordinates as calculated by UVWFlagger for the
- // station positions and times defined in TestInput and phase center
- // defined in test3.
- double uvwvals[] = {
- 0, 0, 0,
- 0.423756, -127.372, 67.1947,
- 0.847513, -254.744, 134.389,
- 0.277918, -382.015, 201.531,
- -0.423756, 127.372, -67.1947,
- 0, 0, 0,
- 0.423756, -127.372, 67.1947,
- -0.145838, -254.642, 134.336,
- -0.847513, 254.744, -134.389,
- -0.423756, 127.372, -67.1947,
- 0, 0, 0,
- -0.569594, -127.27, 67.1417,
- -0.277918, 382.015, -201.531,
- 0.145838, 254.642, -134.336,
- 0.569594, 127.27, -67.1417,
- 0, 0, 0,
- 0, 0, 0,
- 0.738788, -127.371, 67.1942,
- 1.47758, -254.742, 134.388,
- 1.22276, -382.013, 201.53,
- -0.738788, 127.371, -67.1942,
- 0, 0, 0,
- 0.738788, -127.371, 67.1942,
- 0.483976, -254.642, 134.336,
- -1.47758, 254.742, -134.388,
- -0.738788, 127.371, -67.1942,
- 0, 0, 0,
- -0.254812, -127.271, 67.1421,
- -1.22276, 382.013, -201.53,
- -0.483976, 254.642, -134.336,
- 0.254812, 127.271, -67.1421,
- 0, 0, 0
- };
- Cube<double> uvws(IPosition(3,3,16,2), uvwvals, SHARE);
- // Flag where u,v,w matches intervals given in test3.
- Cube<bool> result(itsNCorr,itsNChan,itsNBl);
- result = false;
- for (int i=0; i<itsNBl; ++i) {
- double u = uvws(0,i,itsCount);
- double v = uvws(1,i,itsCount);
- double w = uvws(2,i,itsCount);
- double uv = sqrt(u*u+v*v);
- if ((uv>5.5 && uv<8.5) || (u>20.5 && u<23.5) || (u>31.5 && u<40.5)
- || (v>11.5 && v<14.5) || w<3.5 || w>44.5) {
- for (int j=0; j<itsNChan; ++j) {
- for (int k=0; k<itsNCorr; ++k) {
- result(k,j,i) = true;
- }
- }
- }
- }
- ///cout << buf.getFlags() << endl << result << endl;
- ASSERT (allEQ(buf.getFlags(), result));
- itsCount++;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- ASSERT (int(infoIn.origNChan())==itsNChan);
- ASSERT (int(infoIn.nchan())==itsNChan);
- ASSERT (int(infoIn.ntime())==itsNTime);
- ASSERT (infoIn.timeInterval()==5);
- ASSERT (int(infoIn.nchanAvg())==1);
- ASSERT (int(infoIn.ntimeAvg())==1);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test flagging a few baselines on UV in m.
-void test1(int ntime, int nbl, int nchan, int ncorr)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("uvmrange", "[5.5..8.5]");
- parset.add ("umrange", "[31.5..40.5, 22+-1.5]");
- parset.add ("vmrange", "[11.5..14.5]");
- parset.add ("wmmax", "44.5");
- parset.add ("wmmin", "3.5");
- DPStep::ShPtr step2(new UVWFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Test flagging a few baselines on UV in wavelengths.
-void test2(int ntime, int nbl, int nchan, int ncorr)
-{
- cout << "test2: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("uvlambdarange", "[0.2..0.31]");
- parset.add ("ulambdarange", "[1.55..1.485, 0.807+-0.055]");
- parset.add ("vlambdarange", "[0.42..0.53]");
- parset.add ("wlambdamax", "1.63");
- parset.add ("wlambdamin", "0.12");
- DPStep::ShPtr step2(new UVWFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput2(ntime, nbl, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
- step2->showCounts (cout);
-}
-
-// Test flagging a few baselines on UV in m with a different phase center.
-void test3(int ntime, int nbl, int nchan, int ncorr)
-{
- cout << "test3: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("uvmrange", "[5.5..8.5]");
- parset.add ("umrange", "[31.5..40.5, 22+-1.5]");
- parset.add ("vmrange", "[11.5..14.5]");
- parset.add ("wmmax", "44.5");
- parset.add ("wmmin", "3.5");
- parset.add ("phasecenter", "[-1.92653768rad, 1.09220917rad, j2000]");
- DPStep::ShPtr step2(new UVWFlagger(in, parset, ""));
- DPStep::ShPtr step3(new TestOutput3(ntime, nbl, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Test constructing with the Sun as phase center.
-void test4()
-{
- cout << "test4" << endl;
- // Create the steps.
- TestInput* in = new TestInput(1,1,1,1);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("uvmrange", "[5.5..8.5]");
- parset.add ("phasecenter", "Sun");
- DPStep::ShPtr step2(new UVWFlagger(in, parset, ""));
- step2->show (cout);
-}
-
-
-int main()
-{
- INIT_LOGGER ("tUVWFlagger");
- try {
-
- test1( 10, 16, 32, 4);
- test1(100, 105, 32, 4);
- test2( 2, 16, 32, 4);
- test2( 2, 36, 16, 2);
- test2( 10, 16, 32, 4);
- test2(100, 105, 32, 4);
- test3( 2, 16, 32, 4);
- test4();
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tUVWFlagger.sh b/CEP/DP3/DPPP/test/tUVWFlagger.sh
deleted file mode 100755
index 207563f199ba14d6b8dc36e9265291936a677f6e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tUVWFlagger.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tUVWFlagger
diff --git a/CEP/DP3/DPPP/test/tUpsample.cc b/CEP/DP3/DPPP/test/tUpsample.cc
deleted file mode 100644
index 649f641c364c9fa64dbc05a16fa95af95a01a4a6..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tUpsample.cc
+++ /dev/null
@@ -1,181 +0,0 @@
-//# tAverager.cc: Test program for class Averager
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tAverager.cc 35179 2016-08-25 11:25:17Z dijkema $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/Upsample.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-
-#include <casacore/casa/Quanta/Quantum.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(vector<double> times, vector<bool> flags, double timeInterval)
- : itsTimeStep(0), itsNBl(3), itsNChan(5), itsNCorr(4), itsTimes(times),
- itsFlags(flags), itsTimeInterval(timeInterval)
- {}
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsTimeStep == itsTimes.size()) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsTimeStep*10,i-1000+itsTimeStep*6);
- }
- DPBuffer buf;
- buf.setTime (itsTimes[itsTimeStep]);
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlags[itsTimeStep];
- buf.setFlags (flags);
- buf.setExposure(itsTimeInterval);
-
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, double(itsTimeStep*100));
- buf.setUVW (uvw);
- getNextStep()->process (buf);
- ++itsTimeStep;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
-
- virtual void show (std::ostream&) const {}
-
- virtual void updateInfo (const DPInfo&)
- {
- info().init (itsNCorr, itsNChan, itsTimes.size(), itsTimes[0], itsTimeInterval, string(), string());
- // Define the frequencies.
- Vector<double> chanFreqs(itsNChan);
- Vector<double> chanWidth(itsNChan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
- uint itsTimeStep, itsNBl, itsNChan, itsNCorr;
- vector<double> itsTimes;
- vector<bool> itsFlags;
- double itsTimeInterval;
-};
-
-// Class to check result of upsampling TestInput
-class TestOutput: public DPStep
-{
-public:
- TestOutput(vector<double> times, vector<bool> flags, double timeInterval)
- : itsTimes(times), itsFlags(flags), itsTimeStep(0), itsTimeInterval(timeInterval)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- ASSERT(nearAbs(buf.getTime(), itsTimes[itsTimeStep], itsTimeInterval*0.01));
- ASSERT(allTrue(buf.getFlags()) == itsFlags[itsTimeStep]);
- ++itsTimeStep;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT(near(info.timeInterval(), itsTimeInterval));
- }
-
- vector<double> itsTimes;
- vector<bool> itsFlags;
- uint itsTimeStep;
- double itsTimeInterval;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-void test()
-{
- {
- // Create the steps.
- double times_array[] = {5020763030.74, 5020763032.75, 5020763034.76, 5020763035.77, 5020763037.78, 5020763039.8};
- vector<double> times(times_array, times_array+6);
- bool flags_array[] = {false, false, true, false, false, false};
- vector<bool> flags(flags_array, flags_array+6);
- TestInput* in = new TestInput(times, flags, 2.01327);
- DPStep::ShPtr in_step(in);
- ParameterSet parset;
- parset.add ("timestep", "2");
-
- DPStep::ShPtr upsample(new Upsample(in, parset, ""));
-
- double newtimes_array[] = {5020763030.23, 5020763031.24, 5020763032.25, 5020763033.25, 5020763034.26, 5020763035.27, 5020763036.27, 5020763037.28, 5020763038.29, 5020763039.29, 5020763040.3};
- vector<double> newtimes(newtimes_array, newtimes_array+11);
- bool newflags_array[] = {false, false, false, false, true, false, false, false, false, false, false};
- vector<bool> newflags(newflags_array, newflags_array+11);
-
- DPStep::ShPtr out_step(new TestOutput(newtimes, newflags, 0.5 * 2.01327));
- in_step->setNextStep (upsample);
- upsample->setNextStep (out_step);
- execute (in_step);
- }
-}
-
-
-int main()
-{
- try {
- test();
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP/test/tUpsample.sh b/CEP/DP3/DPPP/test/tUpsample.sh
deleted file mode 100755
index dbdc884b91f52ef1de3ff4e70886242b911e0a8a..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tUpsample.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tUpsample
diff --git a/CEP/DP3/DPPP/test/test_flaggers.sh b/CEP/DP3/DPPP/test/test_flaggers.sh
deleted file mode 100755
index 7cee918c9b154647e5be5f89dc6d950910e6a941..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/test_flaggers.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh test_flaggers
diff --git a/CEP/DP3/DPPP/test/testdemixfilter b/CEP/DP3/DPPP/test/testdemixfilter
deleted file mode 100755
index 19532daaec31a76902f9be848dfbe2d0b0d96bd4..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/testdemixfilter
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/bin/sh
-
-# This script tests if selection in a demix gives the same result as demixing
-# a selection.
-
-# Make a selection of the MS (core stations only)
-msselect in=L29067_SAP000_SB069_uv.MS out=demix.sel.ms baseline='CS*&'
-
-# Do simple averaging of the rest of the MS.
-cat > demix.avg.ps <<EOF
-msin = L29067_SAP000_SB069_uv.MS
-msin.startchan = 2
-msin.nchan=60
-msin.baseline=!CS*&
-msout=demix.avg.ms
-msout.overwrite=true
-steps=[avg]
-avg.type=averager
-avg.freqstep=30
-avg.timestep=5
-EOF
-NDPPP demix.avg.ps
-
-# Do the demixing of the MS selection.
-# Create an empty instrument model.
-parmdbm <<EOF
-create table='demix.inst.pdb1'
-adddef gain values=1
-EOF
-
-cat > demix.sel.demix.ps <<EOF
-msin = demix.sel.ms
-msin.startchan = 2
-msin.nchan=60
-msout=demix.sel.demix.ms
-msout.overwrite=true
-steps=[demix]
-demix.type=demixer
-demix.subtractsources=[CasA,CygA]
-#demix.targetsource=3C196
-demix.skymodel=sky.pdb
-demix.instrumentmodel=demix.inst.pdb1
-demix.demixfreqstep=60
-demix.demixtimestep=10
-demix.freqstep=30
-demix.timestep=5
-demix.ntimechunk=4
-EOF
-NDPPP demix.sel.demix.ps
-
-# Do the demixing of the entire MS using a selection.
-# Create an empty instrument model.
-parmdbm <<EOF
-create table='demix.inst.pdb2'
-adddef gain values=1
-EOF
-
-cat > demix.demixsel.ps <<EOF
-msin = L29067_SAP000_SB069_uv.MS
-msin.startchan = 2
-msin.nchan=60
-msout=demix.demixsel.ms
-msout.overwrite=true
-steps=[demix]
-demix.type=demixer
-demix.subtractsources=[CasA,CygA]
-demix.skymodel=sky.pdb
-demix.instrumentmodel=demix.inst.pdb2
-demix.baseline=CS*&
-demix.demixfreqstep=60
-demix.demixtimestep=10
-demix.freqstep=30
-demix.timestep=5
-demix.ntimechunk=2
-EOF
-NDPPP demix.demixsel.ps
-
-# Select the baselines from the last demix result and check if equal to first demix.
-msselect in=demix.demixsel.ms out=demix.demixsel.sel.ms baseline='CS*&'
-taql 'select t1.DATA,t2.DATA as td from demix.sel.demix.ms t1, demix.demixsel.sel.ms t2 where !all((isnan(t1.DATA) && isnan(t2.DATA)) or t1.DATA=t2.DATA) limit 10'
-taql 'select t1.DATA,t2.DATA as td from demix.sel.demix.ms t1, demix.demixsel.sel.ms t2 where t1.ANTENNA1 != t2.ANTENNA1 or t1.ANTENNA2 != t2.ANTENNA2'
-
-# Select the non-demixed baselines from the last demix result and check if equal
-# to the averaged selection.
-msselect in=demix.demixsel.ms out=demix.demixsel.rest.ms baseline='!CS*&'
-taql 'select t1.DATA,t2.DATA as td from demix.avg.ms t1, demix.demixsel.rest.ms t2 where !all((isnan(t1.DATA) && isnan(t2.DATA)) or t1.DATA=t2.DATA) limit 10'
-taql 'select t1.DATA,t2.DATA as td from demix.avg.ms t1, demix.demixsel.rest.ms t2 where t1.ANTENNA1 != t2.ANTENNA1 or t1.ANTENNA2 != t2.ANTENNA2'
diff --git a/CEP/DP3/DPPP/test/tmwflagger.sh b/CEP/DP3/DPPP/test/tmwflagger.sh
deleted file mode 100755
index 5fd6ce6a98832202cecc074b47fd8099ed06f621..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tmwflagger.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tmwflagger
diff --git a/CEP/DP3/DPPP/test/tparse.cc b/CEP/DP3/DPPP/test/tparse.cc
deleted file mode 100644
index 152c4f4b3488258fcddcc19f5f577ae7c27aa917..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP/test/tparse.cc
+++ /dev/null
@@ -1,69 +0,0 @@
-//# tparse.cc: Test program for function PreFlagger::PSet::exprToRpn
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP/PreFlagger.h>
-#include <Common/StringUtil.h>
-#include <Common/StreamUtil.h>
-#include <Common/LofarLogger.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace std;
-
-namespace LOFAR {
- namespace DPPP {
- // This class name should match the friend in PreFlagger.
- class TestPSet
- {
- public:
- static void testParse (const string&);
- };
- }
-}
-
-void TestPSet::testParse (const string& expr)
-{
- PreFlagger::PSet pset;
- vector<string> names = pset.exprToRpn (expr);
- cout << pset.itsRpn << endl;
- cout << names << endl;
-}
-
-
-int main(int argc, char* argv[])
-{
- INIT_LOGGER ("tPSet");
- try {
- if (argc > 1) {
- TestPSet::testParse (argv[1]);
- } else {
- TestPSet::testParse ("(s1&s_1)|!(!!s2&s2)");
- }
- } catch (std::exception& x) {
- cout << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP_AOFlag/CMakeLists.txt b/CEP/DP3/DPPP_AOFlag/CMakeLists.txt
deleted file mode 100644
index 2cf77e20107edefa59a76f481fd4778717316e8b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/CMakeLists.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-# $Id: CMakeLists.txt 27640 2013-12-04 08:02:49Z diepen $
-
-lofar_package(DPPP_AOFlag 1.0 DEPENDS DPPP)
-
-include(LofarFindPackage)
-lofar_find_package(AOFlagger REQUIRED)
-lofar_find_package(Casacore COMPONENTS casa ms tables REQUIRED)
-lofar_find_package(Boost REQUIRED COMPONENTS date_time thread filesystem system python3)
-# AOFlagger depends on Python 2.7, see aoflagger CMake
-lofar_find_package(Python 3.4 REQUIRED)
-#lofar_find_package(GSL)
-lofar_find_package(LibXml2 REQUIRED)
-lofar_find_package(PNG REQUIRED)
-lofar_find_package(FFTW3 REQUIRED)
-lofar_find_package(CFITSIO REQUIRED)
-#if(CMAKE_SYSTEM_NAME MATCHES "Linux")
-# find_library(RT_LIBRARY rt)
-# list(APPEND LOFAR_EXTRA_LIBRARIES ${RT_LIBRARY})
-#endif(CMAKE_SYSTEM_NAME MATCHES "Linux")
-
-# Copied from AOFlagger CMakeLists to link in GTKMM if AOFlagger was built with it
-find_package(PkgConfig)
-pkg_check_modules(GTKMM gtkmm-3.0>=3.0.0)
-pkg_check_modules(SIGCXX sigc++-2.0)
-
-if(GTKMM_FOUND)
- set(LOFAR_EXTRA_LIBRARIES ${LOFAR_EXTRA_LIBRARIES} ${GTKMM_LIBRARIES} ${GLIBMM_LIBRARIES})
-endif(GTKMM_FOUND)
-# End check for GTKMM
-
-add_subdirectory(include/DPPP_AOFlag)
-add_subdirectory(src)
-add_subdirectory(test)
diff --git a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/AOFlaggerStep.h b/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/AOFlaggerStep.h
deleted file mode 100644
index 3d2800dd8d182a2cf655db954f8d0fd048df8b29..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/AOFlaggerStep.h
+++ /dev/null
@@ -1,161 +0,0 @@
-//# AOFlaggerStep.h: DPPP step class to flag data using rficonsole's functionality
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: AORFlagger.h 26900 2013-10-08 20:12:58Z loose $
-//#
-//# @author Andre Offringa, Ger van Diepen
-
-#ifndef DPPP_AOFLAGGERSTEP_H
-#define DPPP_AOFLAGGERSTEP_H
-
-// @file
-// @brief DPPP step class to flag using aoflagger's functionality
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/FlagCounter.h>
-
-#include <Common/lofar_vector.h>
-#include <Common/lofar_smartptr.h>
-
-#include <aoflagger.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class flagging data points based on the
- // aoflagger library written by Andre Offringa.
- // See the following papers for background information:
- // <ul>
- // <li> Post-correlation radio frequency interference classification
- // methods -- http://arxiv.org/abs/1002.1957
- // <li> A LOFAR RFI detection pipeline and its first results
- // -- http://arxiv.org/abs/1007.2089
- // </ul>
- //
- // When a correlation is flagged, all correlations for that data point
- // are flagged. It is possible to specify which correlations have to be
- // taken into account when flagging. Using, say, only XX may boost
- // performance with a factor 4, but miss points to be flagged.
- // It is also possible to specify the order in which the correlations
- // have to be tested.
- //
- // It is possible to flag specific baselines only using a selection on
- // baseline length.
- // <br>Furthermore it is possible to only flag the autocorrelations and
- // apply the resulting flags to the crosscorrelations, possibly selected
- // on baseline length.
-
- class AOFlaggerStep : public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- AOFlaggerStep(DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~AOFlaggerStep();
-
- // Create an AOFlaggerStep object using the given parset.
- static DPStep::ShPtr makeStep (DPInput*, const ParameterSet&,
- const std::string&);
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Write the statistics into the MS.
- virtual void addToMS (const string& msName);
-
- // Update the general info.
- // It is used to adjust the parms if needed.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the flagger counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- // Flag all baselines in the time window (using OpenMP to parallellize).
- // Process the buffers in the next step.
- void flag (uint rightOverlap);
-
- // Flag a single baseline using the rfistrategy.
- void flagBaseline (uint leftOverlap, uint windowSize,
- uint rightOverlap, uint bl,
- FlagCounter& counter,
- aoflagger::QualityStatistics& rfiStats);
-
- // Add the flags to the statistics.
- void addStats (aoflagger::QualityStatistics& rfiStats,
- const aoflagger::ImageSet& values,
- const aoflagger::FlagMask& rfiMask, const aoflagger::FlagMask& origMask,
- int bl);
-
- // Format a number as kB, MB, etc.
- static void formatBytes(std::ostream&, double);
-
- // Fill the rfi strategy.
- void fillStrategy();
-
- //# Data members.
- string itsName;
- uint itsBufIndex;
- uint itsNTimes;
- string itsStrategyName;
- uint itsWindowSize;
- uint itsOverlap; //# extra time slots on both sides
- double itsOverlapPerc;
- double itsMemory; //# Usable memory in GBytes
- double itsMemoryPerc;
- double itsMemoryNeeded; //# Memory needed for data/flags
- bool itsPulsarMode;
- bool itsPedantic;
- bool itsDoAutoCorr;
- bool itsDoRfiStats;
- vector<DPBuffer> itsBuf;
- FlagCounter itsFlagCounter;
- NSTimer itsTimer;
- NSTimer itsQualityTimer; //# quality writing timer
- NSTimer itsComputeTimer; //# move/flag timer
- double itsMoveTime; //# data move timer (sum all threads)
- double itsFlagTime; //# flag timer (sum of all threads)
- double itsQualTime; //# quality timer (sum of all threads)
- casacore::Vector<double> itsFreqs;
- aoflagger::AOFlagger itsAOFlagger;
- boost::scoped_ptr<aoflagger::Strategy> itsStrategy;
- boost::scoped_ptr<aoflagger::QualityStatistics> itsRfiStats;
- };
-
- } //# end namespace
-}
-
-
-#endif
diff --git a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/CMakeLists.txt b/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/CMakeLists.txt
deleted file mode 100644
index 24c4b41e38db621cd4cf66871791cd9d83e5a0f4..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-# $Id: CMakeLists.txt 30990 2015-02-12 12:27:47Z diepen $
-
-# List of header files that will be installed.
-set(inst_HEADERS AOFlaggerStep.h SlidingFlagger.h Register.h)
-
-# Create symbolic link to include directory.
-execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
- ${CMAKE_CURRENT_SOURCE_DIR}
- ${CMAKE_BINARY_DIR}/include/${PACKAGE_NAME})
-
-# Install header files.
-#install(FILES ${inst_HEADERS} DESTINATION include/${PACKAGE_NAME})
diff --git a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/Register.h b/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/Register.h
deleted file mode 100644
index 4304d347e82a17d92ccfb12dfc3b7640c3b3c981..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/Register.h
+++ /dev/null
@@ -1,37 +0,0 @@
-//# Register.h: Register AOFlag steps in DPPP
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: AORFlagger.h 26900 2013-10-08 20:12:58Z loose $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_AOFLAG_REGISTER_H
-#define DPPP_AOFLAG_REGISTER_H
-
-// @file
-// @brief Register AOFlag steps in DPPP
-
-
-// Define the function (without name mangling) to register the 'constructor'.
-extern "C"
-{
- void register_aoflag();
-}
-
-#endif
diff --git a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/SlidingFlagger.h b/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/SlidingFlagger.h
deleted file mode 100644
index 350ab3aea6130d3ca7f487630ecd6b64a4035652..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/include/DPPP_AOFlag/SlidingFlagger.h
+++ /dev/null
@@ -1,143 +0,0 @@
-//# SlidingFlagger.h: DPPP step class to flag data using rficonsole's functionality
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: SlidingFlagger.h 26900 2013-10-08 20:12:58Z loose $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_AOFLAG_SLIDINGFLAGGER_H
-#define DPPP_AOFLAG_SLIDINGFLAGGER_H
-
-// @file
-// @brief DPPP step class to flag using rficonsole's functionality
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/FlagCounter.h>
-#include <Common/lofar_vector.h>
-#include <Common/lofar_smartptr.h>
-
-#include <aoflagger.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class flagging data points based on the
- // aoflagger library written by Andre Offringa.
- // See the following papers for background information:
- // <ul>
- // <li> Post-correlation radio frequency interference classification
- // methods -- http://arxiv.org/abs/1002.1957
- // <li> A LOFAR RFI detection pipeline and its first results
- // -- http://arxiv.org/abs/1007.2089
- // </ul>
- //
- // When a correlation is flagged, all correlations for that data point
- // are flagged. It is possible to specify which correlations have to be
- // taken into account when flagging. Using, say, only XX may boost
- // performance with a factor 4, but miss points to be flagged.
- // It is also possible to specify the order in which the correlations
- // have to be tested.
- //
- // It is possible to flag specific baselines only using a selection on
- // baseline length.
- // <br>Furthermore it is possible to only flag the autocorrelations and
- // apply the resulting flags to the crosscorrelations, possibly selected
- // on baseline length.
-
- class SlidingFlagger: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- SlidingFlagger (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~SlidingFlagger();
-
- // Create an SlidingFlagger object using the given parset.
- static DPStep::ShPtr makeStep (DPInput*, const ParameterSet&,
- const std::string&);
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- // It is used to adjust the parms if needed.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the flagger counts.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- private:
- struct ThreadData {
- FlagCounter flagCounter;
- NSTimer moveTimer;
- NSTimer flagTimer;
- };
-
- // Flag all baselines in the time window (using OpenMP to parallellize).
- // Process the buffers in the next step.
- void flag();
-
- // Flag a single baseline using the rfistrategy.
- void flagBaseline (uint bl, ThreadData&);
-
- // Fill the rfi strategy.
- void fillStrategy();
-
- //# Data members.
- string itsName;
- uint itsBufIndex;
- uint itsNTimes;
- uint itsNThreads;
- string itsStrategyName;
- uint itsWindowSize;
- uint itsBufferSize;
- double itsMemoryNeeded; //# Memory needed for data/flags
- bool itsPulsarMode;
- bool itsPedantic;
- bool itsDoAutoCorr;
- vector<DPBuffer> itsBuf;
- NSTimer itsTimer;
- NSTimer itsComputeTimer; //# move/flag timer
- mutable FlagCounter itsFlagCounter;
- vector<ThreadData> itsTD;
- casacore::Vector<double> itsFreqs;
- aoflagger::AOFlagger itsAOFlagger;
- boost::scoped_ptr<aoflagger::Strategy> itsStrategy;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP_AOFlag/src/AOFlaggerStep.cc b/CEP/DP3/DPPP_AOFlag/src/AOFlaggerStep.cc
deleted file mode 100644
index 33e711fbc5f85450065af19da5a7643fb2495fb7..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/src/AOFlaggerStep.cc
+++ /dev/null
@@ -1,490 +0,0 @@
-//# AOFlaggerStep.cc: DPPP step class to flag data based on rficonsole
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: AOFlaggerStep.cc 31423 2015-04-03 14:06:21Z dijkema $
-//#
-//# @author Andre Offringa, Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP_AOFlag/AOFlaggerStep.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <Common/StreamUtil.h>
-#include <Common/OpenMP.h>
-
-#include <casacore/casa/OS/HostInfo.h>
-#include <casacore/casa/OS/File.h>
-
-#include <aoflagger.h>
-
-#include <iostream>
-#include <algorithm>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- AOFlaggerStep::AOFlaggerStep (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsName (prefix),
- itsBufIndex (0),
- itsNTimes (0),
- itsMemoryNeeded(0),
- itsFlagCounter (input->msName(), parset, prefix+"count."),
- itsMoveTime (0),
- itsFlagTime (0),
- itsQualTime (0),
- itsRfiStats ()
- {
- itsStrategyName = parset.getString (prefix+"strategy", string());
- itsWindowSize = parset.getUint (prefix+"timewindow", 0);
- itsMemory = parset.getUint (prefix+"memorymax", 0);
- itsMemoryPerc = parset.getUint (prefix+"memoryperc", 0);
- itsOverlap = parset.getUint (prefix+"overlapmax", 0);
- // Also look for keyword overlap for backward compatibility.
- if (itsOverlap == 0) {
- itsOverlap = parset.getUint (prefix+"overlap", 0);
- }
- itsOverlapPerc = parset.getDouble (prefix+"overlapperc", -1);
- itsPulsarMode = parset.getBool (prefix+"pulsar", false);
- itsPedantic = parset.getBool (prefix+"pedantic", false);
- itsDoAutoCorr = parset.getBool (prefix+"autocorr", true);
- itsDoRfiStats = parset.getBool (prefix+"keepstatistics", true);
- }
-
- AOFlaggerStep::~AOFlaggerStep()
- {}
-
- DPStep::ShPtr AOFlaggerStep::makeStep (DPInput* input,
- const ParameterSet& parset,
- const std::string& prefix)
- {
- return DPStep::ShPtr(new AOFlaggerStep(input, parset, prefix));
- }
-
- void AOFlaggerStep::show (std::ostream& os) const
- {
- os << "AOFlaggerStep " << itsName << std::endl;
- os << " strategy: " << itsStrategyName << std::endl;
- os << " timewindow: " << itsWindowSize << std::endl;
- os << " overlap: " << itsOverlap << std::endl;
- os << " pulsar: " << itsPulsarMode << std::endl;
- os << " pedantic: " << itsPedantic << std::endl;
- os << " keepstatistics: " << itsDoRfiStats << std::endl;
- os << " autocorr: " << itsDoAutoCorr << std::endl;
- os << " nthreads (omp) " << OpenMP::maxThreads() << std::endl;
- os << " max memory used ";
- formatBytes(os, itsMemoryNeeded);
- os << std::endl;
- }
-
- void AOFlaggerStep::formatBytes(std::ostream& os, double bytes) {
- int exp=0;
- while (bytes >= 1024 && exp<5) {
- bytes/=1024;
- exp++;
- }
-
- uint origPrec=os.precision();
- os.precision(1);
-
- if (exp==0) {
- os<<fixed<<bytes<<" "<<"B";
- } else {
- os<<fixed<<bytes<<" "<<"KMGTPE"[exp-1]<<"B";
- }
-
- os.precision(origPrec);
- }
-
- void AOFlaggerStep::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteFlags();
- // Get nr of threads.
- uint nthread = OpenMP::maxThreads();
- // Determine available memory.
- double availMemory = HostInfo::memoryTotal() * 1024.;
- // Determine how much memory can be used.
- double memoryMax = itsMemory * 1024*1024*1024;
- double memory = memoryMax;
- if (itsMemoryPerc > 0) {
- memory = itsMemoryPerc * availMemory / 100.;
- if (memoryMax > 0 && memory > memoryMax) {
- memory = memoryMax;
- }
- } else if (itsMemory <= 0) {
- // Nothing given, so use available memory on this machine.
- // Set 50% (max 2 GB) aside for other purposes.
- memory = availMemory - std::min(0.5 * availMemory, 2.*1024*1024*1024);
- }
- // Determine how much buffer space is needed per time slot.
- // The flagger needs 3 extra work buffers (data+flags) per thread.
- double timeSize = (sizeof(Complex) + sizeof(bool)) *
- (infoIn.nbaselines() + 3*nthread) * infoIn.nchan() * infoIn.ncorr();
- // If no overlap percentage is given, set it to 1%.
- if (itsOverlapPerc < 0 && itsOverlap == 0) {
- itsOverlapPerc = 1;
- }
- // If no time window given, determine it from the available memory.
- if (itsWindowSize == 0) {
- double nt = memory / timeSize;
- if (itsOverlapPerc > 0) {
- // Determine the overlap (add 0.5 for rounding).
- // If itsOverLap is also given, it is the maximum.
- double tw = nt / (1 + 2*itsOverlapPerc/100);
- uint overlap = uint(itsOverlapPerc*tw/100 + 0.5);
- if (itsOverlap == 0 || overlap < itsOverlap) {
- itsOverlap = overlap;
- }
- }
- itsWindowSize = uint(std::max(1., nt-2*itsOverlap));
- // Make the window size divide the nr of times nicely (if known).
- // In that way we cannot have a very small last window.
- if (infoIn.ntime() > 0) {
- uint nwindow = 1 + (infoIn.ntime() - 1) / itsWindowSize;
- itsWindowSize = 1 + (infoIn.ntime() - 1) / nwindow;
- if (itsOverlapPerc > 0) {
- uint overlap = uint(itsOverlapPerc*itsWindowSize/100 + 0.5);
- if (overlap < itsOverlap) {
- itsOverlap = overlap;
- }
- }
- }
- }
- if (itsOverlap == 0) {
- itsOverlap = uint(itsOverlapPerc*itsWindowSize/100);
- }
- // Check if it all fits in memory.
- itsMemoryNeeded = (itsWindowSize + 2*itsOverlap) * timeSize;
- ASSERTSTR (itsMemoryNeeded < availMemory,
- "Timewindow " << itsWindowSize
- << " and/or overlap " << itsOverlap
- << ' ' << memory
- << " too large for available memory " << availMemory);
- // Size the buffer (need overlap on both sides).
- itsBuf.resize (itsWindowSize + 2*itsOverlap);
- // Initialize the flag counters.
- itsFlagCounter.init (getInfo());
- itsFreqs = infoIn.chanFreqs();
- // Fill the strategy (used by all threads)
- // (A thread does not need a private strategy; it is
- // safe to share one among different threads.)
- fillStrategy();
- }
-
- void AOFlaggerStep::showCounts (std::ostream& os) const
- {
- os << endl << "Flags set by AOFlaggerStep " << itsName;
- os << endl << "===========================" << endl;
- itsFlagCounter.showBaseline (os, itsNTimes);
- itsFlagCounter.showChannel (os, itsNTimes);
- itsFlagCounter.showCorrelation (os, itsNTimes);
- }
-
- void AOFlaggerStep::showTimings (std::ostream& os, double duration) const
- {
- double flagDur = itsTimer.getElapsed();
- os << " ";
- FlagCounter::showPerc1 (os, flagDur, duration);
- os << " AOFlaggerStep " << itsName << endl;
- os << " ";
- // move time and flag time are sum of all threads.
- // Scale them to a single elapsed time.
- double factor = (itsComputeTimer.getElapsed() /
- (itsMoveTime + itsFlagTime + itsQualTime));
- FlagCounter::showPerc1 (os, itsMoveTime*factor, flagDur);
- os << " of it spent in shuffling data" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, itsFlagTime*factor, flagDur);
- os << " of it spent in calculating flags" << endl;
- if (itsDoRfiStats) {
- os << " ";
- FlagCounter::showPerc1 (os, itsQualTime*factor + itsQualityTimer.getElapsed(),
- flagDur);
- os << " of it spent in making quality statistics" << endl;
- }
- }
-
- // Alternative strategy is to flag in windows
- // 0 .. n+2m
- // n .. 2n+2m
- // 2n .. 3n+2m etc.
- // and also update the flags in the overlaps
- bool AOFlaggerStep::process (const DPBuffer& buf)
- {
- itsTimer.start();
- // Accumulate in the time window until the window and overlap are full.
- itsNTimes++;
- itsBuf[itsBufIndex].copy (buf);
- ++itsBufIndex;
- if (itsBufIndex == itsWindowSize+2*itsOverlap) {
- flag(2*itsOverlap);
- }
- itsTimer.stop();
- return true;
- }
-
- void AOFlaggerStep::finish()
- {
- cerr << " " << itsBufIndex << " time slots to finish in AOFlaggerStep ..."
- << endl;
- itsTimer.start();
- // Set window size to all entries left.
- itsWindowSize = itsBufIndex;
- if (itsWindowSize > 0) {
- // Flag the remaining time slots (without right overlap).
- flag (0);
- }
- itsBuf.clear();
- itsTimer.stop();
- // Let the next step finish its processing.
- getNextStep()->finish();
- }
-
- void AOFlaggerStep::addToMS (const string& msName)
- {
- itsTimer.start();
- if (itsDoRfiStats) {
- itsQualityTimer.start();
- itsAOFlagger.WriteStatistics(*itsRfiStats, msName);
- itsQualityTimer.stop();
- }
- itsTimer.stop();
- getPrevStep()->addToMS (msName);
- }
-
- void AOFlaggerStep::flag (uint rightOverlap)
- {
- // Get the sizes of the axes.
- // Note: OpenMP 2.5 needs signed iteration variables.
- int nrbl = itsBuf[0].getData().shape()[2];
- uint ncorr = itsBuf[0].getData().shape()[0];
- ASSERTSTR (ncorr==4, "AOFlaggerStep can only handle all 4 correlations");
- // Get antenna numbers in case applyautocorr is true.
- const Vector<Int>& ant1 = getInfo().getAnt1();
- const Vector<Int>& ant2 = getInfo().getAnt2();
- itsComputeTimer.start();
- // Now flag each baseline for this time window.
- // The baselines can be processed in parallel.
-#pragma omp parallel
- {
- // Create thread-private counter object.
- FlagCounter counter;
- counter.init (getInfo());
-
- // Create a statistics object for all polarizations.
- std::vector<double> scanTimes(itsBuf.size());
- for (size_t i=0; i<itsBuf.size(); ++i) {
- scanTimes[i] = itsBuf[i].getTime();
- }
- aoflagger::QualityStatistics rfiStats =
- itsAOFlagger.MakeQualityStatistics (scanTimes.data(),
- scanTimes.size(),
- itsFreqs.data(),
- itsFreqs.size(),
- 4, false); // no histograms
-
- // The for loop can be parallellized. This must be done dynamically,
- // because the execution times of iterations can vary.
-#pragma omp for schedule(dynamic)
- // GCC-4.3 only supports OpenMP 2.5 that needs signed iteration
- // variables.
- for (int ib=0; ib<nrbl; ++ib) {
- // Do autocorrelations only if told so.
- if (ant1[ib] == ant2[ib]) {
- if (itsDoAutoCorr) {
- flagBaseline (0, itsWindowSize+rightOverlap, 0, ib,
- counter, rfiStats);
- }
- } else {
- flagBaseline (0, itsWindowSize+rightOverlap, 0, ib,
- counter, rfiStats);
- }
- } // end of OMP for
-#pragma omp critical(aorflagger_updatecounts)
- {
- // Add the counters to the overall object.
- itsFlagCounter.add (counter);
- if (itsDoRfiStats) {
- itsQualityTimer.stop();
- // Add the rfi statistics to the global object.
- if (itsRfiStats == 0) {
- itsRfiStats.reset(new aoflagger::QualityStatistics(rfiStats));
- } else {
- (*itsRfiStats) += rfiStats;
- }
- itsQualityTimer.start();
- }
- }
- } // end of OMP parallel
- itsComputeTimer.stop();
- itsTimer.stop();
- // Let the next step process the buffers.
- // If possible, discard the buffer processed to minimize memory usage.
- for (uint i=0; i<itsWindowSize; ++i) {
- getNextStep()->process (itsBuf[i]);
- /// itsBuf[i] = DPBuffer();
- ///cout << "cleared buffer " << i << endl;
- }
- itsTimer.start();
- // Shift the buffers still needed to the beginning of the vector.
- // This is a bit easier than keeping a wrapped vector.
- // Note it is a cheap operation, because shallow copies are made.
- for (uint i=0; i<rightOverlap; ++i) {
- itsBuf[i].copy (itsBuf[i+itsWindowSize]);
- ///cout << "moved buffer " <<i+itsWindowSize<<" to "<< i << endl;
- }
- itsBufIndex = rightOverlap;
- }
-
- void AOFlaggerStep::flagBaseline (uint leftOverlap, uint windowSize,
- uint rightOverlap, uint bl,
- FlagCounter& counter,
- aoflagger::QualityStatistics& rfiStats)
- {
- NSTimer moveTimer, flagTimer, qualTimer;
- moveTimer.start();
- // Get the sizes of the axes.
- uint ntime = leftOverlap + windowSize + rightOverlap;
- uint nchan = itsBuf[0].getData().shape()[1];
- uint blsize = nchan * itsBuf[0].getData().shape()[0];
- // Fill the rficonsole buffers and flag.
- // Create the objects for the real and imaginary data of all corr.
- aoflagger::ImageSet imageSet =
- itsAOFlagger.MakeImageSet(ntime, nchan, 8);
- aoflagger::FlagMask origFlags =
- itsAOFlagger.MakeFlagMask(ntime, nchan);
- const uint iStride = imageSet.HorizontalStride();
- const uint fStride = origFlags.HorizontalStride();
- for (uint i=0; i<ntime; ++i) {
- const Complex* data = itsBuf[i].getData().data() + bl*blsize;
- const bool* flags = itsBuf[i].getFlags().data() + bl*blsize;
- for (uint j=0; j<nchan; ++j) {
- for (uint p=0; p!=4; ++p) {
- imageSet.ImageBuffer(p*2 )[i + j*iStride] = data->real();
- imageSet.ImageBuffer(p*2+1)[i + j*iStride] = data->imag();
- data++;
- }
- origFlags.Buffer()[i + j*fStride] = *flags;
- flags += 4;
- }
- }
- // Execute the strategy to do the flagging.
- moveTimer.stop();
- flagTimer.start();
- aoflagger::FlagMask rfiMask = itsAOFlagger.Run(*itsStrategy, imageSet);
- flagTimer.stop();
- // Put back the true flags and count newly set flags.
- moveTimer.start();
- for (uint i=leftOverlap; i<windowSize+leftOverlap; ++i) {
- bool* flags = itsBuf[i].getFlags().data() + bl*blsize;
- for (uint j=0; j<nchan; ++j) {
- // Only set if not already set.
- // If any corr is newly set, set all corr.
- if (! flags[0]) {
- bool setFlag = true;
- if (rfiMask.Buffer()[i + j*fStride]) {
- counter.incrCorrelation(0);
- counter.incrCorrelation(1);
- counter.incrCorrelation(2);
- counter.incrCorrelation(3);
- } else {
- setFlag = false;
- }
- if (setFlag) {
- counter.incrBaseline(bl);
- counter.incrChannel(j);
- for (int k=0; k<4; ++k) {
- flags[k] = true;
- }
- }
- }
- flags += 4;
- }
- }
- moveTimer.stop();
- // Update the RFI statistics if needed.
- if (itsDoRfiStats) {
- qualTimer.start();
- addStats (rfiStats, imageSet, rfiMask, origFlags, bl);
- qualTimer.stop();
- }
-#pragma omp critical(aorflagger_updatetimers)
- {
- // Add the timings.
- itsMoveTime += moveTimer.getElapsed();
- itsFlagTime += flagTimer.getElapsed();
- itsQualTime += qualTimer.getElapsed();
- } // end of OMP critical
- }
-
- void AOFlaggerStep::addStats (aoflagger::QualityStatistics& rfiStats,
- const aoflagger::ImageSet& values,
- const aoflagger::FlagMask& rfiMask, const aoflagger::FlagMask& origMask,
- int bl)
- {
- itsAOFlagger.CollectStatistics(rfiStats, values, rfiMask, origMask,
- getInfo().getAnt1()[bl], getInfo().getAnt2()[bl]);
- }
-
- void AOFlaggerStep::fillStrategy ()
- {
- if (! itsStrategyName.empty()) {
- File file(itsStrategyName);
- if (! file.exists()) {
- file = File("$LOFARROOT/share/rfistrategies/" + itsStrategyName);
- if (! file.exists()) {
- THROW (Exception, "Unknown rfistrategy file " << itsStrategyName);
- }
- }
- itsStrategy.reset(new aoflagger::Strategy
- (itsAOFlagger.LoadStrategy(file.path().absoluteName())));
- } else {
- double centralFrequency = 0.5*(itsFreqs[0] + itsFreqs[itsFreqs.size()-1]);
- double timeRes;
- if (itsBuf.size() >=2 ) {
- timeRes = (itsBuf[itsBuf.size()-1].getTime() -
- itsBuf[0].getTime()) / (itsBuf.size()-1);
- } else {
- timeRes = 0.0;
- }
- double frequencyRes;
- if (itsFreqs.size() >= 2) {
- frequencyRes = (itsFreqs[itsFreqs.size()-1] -
- itsFreqs[0]) / (itsFreqs.size()-1);
- } else {
- frequencyRes = 0.0;
- }
- itsStrategy.reset(new aoflagger::Strategy
- (itsAOFlagger.MakeStrategy(aoflagger::LOFAR_TELESCOPE,
- aoflagger::StrategyFlags::NONE,
- centralFrequency,
- timeRes,
- frequencyRes)));
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP_AOFlag/src/CMakeLists.txt b/CEP/DP3/DPPP_AOFlag/src/CMakeLists.txt
deleted file mode 100644
index 3bdb538f6fcabf3b635b118c81ddb2c1386b7111..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/src/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-# $Id: CMakeLists.txt 30439 2014-11-19 15:04:34Z dijkema $
-
-include(LofarPackageVersion)
-
-lofar_add_library(dppp_aoflag
- Package__Version.cc
- AOFlaggerStep.cc
-# SlidingFlagger.cc
- Register.cc
-)
-
-lofar_add_bin_program(versiondppp_aoflag versiondppp_aoflag.cc)
diff --git a/CEP/DP3/DPPP_AOFlag/src/Register.cc b/CEP/DP3/DPPP_AOFlag/src/Register.cc
deleted file mode 100644
index d7ede6b1fad8c21b2d148b1c4f518d4ef7f95d15..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/src/Register.cc
+++ /dev/null
@@ -1,39 +0,0 @@
-//# Register.cc: Register steps in DPPP
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: AOFlaggerStep.cc 31423 2015-04-03 14:06:21Z dijkema $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP_AOFlag/Register.h>
-#include <DPPP_AOFlag/AOFlaggerStep.h>
-//#include <DPPP_AOFlag/SlidingFlagger.h>
-#include <DPPP/DPRun.h>
-
-// Define the function to make the AOFlaggerStep 'constructor' known.
-// Its suffix must be the (lowercase) name of the package (library).
-// Also make the SlidingFlagger known.
-void register_aoflag()
-{
- LOFAR::DPPP::DPRun::registerStepCtor ("aoflag",
- LOFAR::DPPP::AOFlaggerStep::makeStep);
- // LOFAR::DPPP::DPRun::registerStepCtor ("aoflag.sliding",
- // LOFAR::DPPP::SlidingFlagger::makeStep);
-}
diff --git a/CEP/DP3/DPPP_AOFlag/src/SlidingFlagger.cc b/CEP/DP3/DPPP_AOFlag/src/SlidingFlagger.cc
deleted file mode 100644
index 11f20c4c67e6d7c3610a8a37b220146e69006429..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/src/SlidingFlagger.cc
+++ /dev/null
@@ -1,369 +0,0 @@
-//# SlidingFlagger.cc: DPPP step class to flag data using a sliding window
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: SlidingFlagger.cc 30697 2015-01-14 12:17:14Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP_AOFlag/SlidingFlagger.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <Common/StreamUtil.h>
-#include <Common/OpenMP.h>
-
-#include <casacore/casa/OS/File.h>
-#include <casacore/casa/OS/HostInfo.h>
-
-#include <AOFlagger/msio/image2d.h>
-#include <AOFlagger/msio/mask2d.h>
-#include <AOFlagger/msio/timefrequencydata.h>
-#include <AOFlagger/strategy/actions/changeresolutionaction.h>
-#include <AOFlagger/strategy/actions/combineflagresultsaction.h>
-#include <AOFlagger/strategy/actions/foreachcomplexcomponentaction.h>
-#include <AOFlagger/strategy/actions/foreachpolarisationaction.h>
-#include <AOFlagger/strategy/actions/frequencyselectionaction.h>
-#include <AOFlagger/strategy/actions/iterationaction.h>
-#include <AOFlagger/strategy/actions/setflaggingaction.h>
-#include <AOFlagger/strategy/actions/setimageaction.h>
-#include <AOFlagger/strategy/actions/slidingwindowfitaction.h>
-#include <AOFlagger/strategy/actions/statisticalflagaction.h>
-#include <AOFlagger/strategy/actions/strategyaction.h>
-#include <AOFlagger/strategy/actions/sumthresholdaction.h>
-#include <AOFlagger/strategy/actions/timeselectionaction.h>
-#include <AOFlagger/strategy/control/artifactset.h>
-#include <AOFlagger/strategy/control/strategyreader.h>
-
-#include <iostream>
-#include <algorithm>
-
-using namespace casacore;
-using namespace rfiStrategy;
-
-namespace LOFAR {
- namespace DPPP {
-
- SlidingFlagger::SlidingFlagger (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsName (prefix),
- itsBufIndex (0),
- itsNTimes (0),
- itsNThreads (OpenMP::maxThreads()),
- itsMemoryNeeded(0),
- itsFlagCounter (input->msName(), parset, prefix+"count.")
- {
- itsStrategyName = parset.getString (prefix+"strategy", string());
- itsWindowSize = parset.getUint (prefix+"timewindow", 0);
- itsBufferSize = parset.getUint (prefix+"buffersize", 0);
- itsPulsarMode = parset.getBool (prefix+"pulsar", false);
- itsPedantic = parset.getBool (prefix+"pedantic", false);
- itsDoAutoCorr = parset.getBool (prefix+"autocorr", true);
- ASSERT (itsWindowSize > 0 && itsBufferSize > 0);
- if (itsBufferSize > itsWindowSize) {
- THROW (Exception, "timewindow < buffersize, so AOFlagger should "
- "be used instead of SlidingFlagger");
- }
- }
-
- SlidingFlagger::~SlidingFlagger()
- {}
-
- DPStep::ShPtr SlidingFlagger::makeStep (DPInput* input,
- const ParameterSet& parset,
- const std::string& prefix)
- {
- return DPStep::ShPtr(new SlidingFlagger(input, parset, prefix));
- }
-
- void SlidingFlagger::show (std::ostream& os) const
- {
- os << "SlidingFlagger " << itsName << std::endl;
- os << " strategy: " << itsStrategyName << std::endl;
- os << " timewindow: " << itsWindowSize << std::endl;
- os << " buffersize: " << itsBufferSize << std::endl;
- os << " pulsar: " << itsPulsarMode << std::endl;
- os << " pedantic: " << itsPedantic << std::endl;
- os << " autocorr: " << itsDoAutoCorr << std::endl;
- os << " nthreads (omp) " << itsNThreads << std::endl;
- os << " memory used " << itsMemoryNeeded << std::endl;
- }
-
- void SlidingFlagger::updateInfo (const DPInfo& infoIn)
- {
- ASSERTSTR (infoIn.ncorr()==4,
- "SlidingFlagger can only handle all 4 correlations");
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteFlags();
- // Determine available memory.
- double availMemory = HostInfo::memoryTotal() * 1024.;
- // Determine how much buffer space is needed per time slot.
- // The flagger needs 3 extra work buffers (data+flags) per thread.
- double imgSize = 8*sizeof(num_t) * itsWindowSize * infoIn.nchan();
- double bufSize = (infoIn.nbaselines() * infoIn.nchan() * 4 *
- (sizeof(Complex) + sizeof(bool)));
- // Check if it all fits in memory.
- itsMemoryNeeded = (infoIn.nbaselines() * imgSize +
- itsBufferSize * bufSize);
- ASSERTSTR (itsMemoryNeeded < availMemory,
- "timewindow " << itsWindowSize
- << " and buffersize " << itsBufferSize
- << ' ' << itsMemoryNeeded
- << " too large for available memory " << availMemory);
- // Create the AOFlagger objects for each baseline.
- // Fill them with 0 data and True flags.
- ///const Vector<Int>& ant1 = getInfo().getAnt1();
- ///const Vector<Int>& ant2 = getInfo().getAnt2();
- ///int nbl = ant1.size();
- ///int nchan = getInfo().nchan();
- // Create the rficonsole Image2D objects for each baseline to flag.
- // The axes are freq,time (time varies slowest) to make the shifting
- // in flagBaseline easier. Note that AORFlagger uses time,freq, but
- // the order does not matter for rficonsole.
- ///itsBLData.resize (nbl);
- ///for (int i=0; i<nbl; ++i) {
- ///if (itsDoAutoCorr || ant1[i] != ant2[i]) {
- ///}
- ///}
- // Fill the strategy (used by all threads)
- // (A thread does not need a private strategy; it is
- // safe to share one among different threads.)
- fillStrategy();
- itsFreqs = infoIn.chanFreqs();
- // Create the counters for all possible threads.
- itsTD.resize (itsNThreads);
- for (uint i=0; i<itsNThreads; ++i) {
- itsTD[i].flagCounter.init (getInfo());
- }
- itsBuf.resize (itsBufferSize);
- itsFlagCounter.init (getInfo());
- }
-
- void SlidingFlagger::showCounts (std::ostream& os) const
- {
- for (size_t i=0; i<itsTD.size(); ++i) {
- itsFlagCounter.add (itsTD[i].flagCounter);
- }
- os << endl << "Flags set by SlidingFlagger " << itsName;
- os << endl << "============================" << endl;
- itsFlagCounter.showBaseline (os, itsNTimes);
- itsFlagCounter.showChannel (os, itsNTimes);
- itsFlagCounter.showCorrelation (os, itsNTimes);
- }
-
- void SlidingFlagger::showTimings (std::ostream& os, double duration) const
- {
- double moveTime = 0;
- double flagTime = 0;
- for (size_t i=0; i<itsNThreads; ++i) {
- moveTime += itsTD[i].moveTimer.getElapsed();
- flagTime += itsTD[i].flagTimer.getElapsed();
- }
- double flagDur = itsTimer.getElapsed();
- os << " ";
- FlagCounter::showPerc1 (os, flagDur, duration);
- os << " SlidingFlagger " << itsName << endl;
- os << " ";
- // move time and flag time are sum of all threads.
- // Scale them to a single elapsed time.
- double factor = itsComputeTimer.getElapsed() / (moveTime + flagTime);
- FlagCounter::showPerc1 (os, moveTime*factor, flagDur);
- os << " of it spent in shuffling data" << endl;
- os << " ";
- FlagCounter::showPerc1 (os, flagTime*factor, flagDur);
- os << " of it spent in calculating flags" << endl;
- }
-
- bool SlidingFlagger::process (const DPBuffer& buf)
- {
- itsTimer.start();
- itsNTimes++;
- if (itsBufferSize == 1) {
- itsBuf[0].referenceFilled (buf);
- itsBufIndex = 1;
- } else {
- itsBuf[itsBufIndex++].copy (buf);
- }
- if (itsBufIndex == itsBufferSize) {
- flag();
- }
- itsTimer.stop();
- return true;
- }
-
- void SlidingFlagger::finish()
- {
- cerr << " " << itsBufIndex << " time slots to finish in SlidingFlagger ..."
- << endl;
- itsTimer.start();
- if (itsBufIndex > 0) {
- // Flag the remaining time slots.
- flag();
- }
- itsTimer.stop();
- // Let the next step finish its processing.
- getNextStep()->finish();
- }
-
- void SlidingFlagger::flag()
- {
- // Get the sizes of the axes.
- // Note: OpenMP 2.5 needs signed iteration variables.
- int nrbl = itsBuf[0].getData().shape()[2];
- // Get antenna numbers in case applyautocorr is true.
- const Vector<Int>& ant1 = getInfo().getAnt1();
- const Vector<Int>& ant2 = getInfo().getAnt2();
- itsComputeTimer.start();
- // Now flag each baseline for this time window.
- // The baselines can be processed in parallel.
-#pragma omp parallel
- {
- // The for loop can be parallellized. This must be done dynamically,
- // because the execution times of iterations can vary.
-#pragma omp for schedule(dynamic)
- for (int ib=0; ib<nrbl; ++ib) {
- // Flag baseline only if told so.
- // Do autocorrelations only if told so.
- ///if (itsBLData[ib].realXX != 0) {
- if (ant1[ib] == ant2[ib]) {
- if (itsDoAutoCorr) {
- flagBaseline (ib, itsTD[OpenMP::threadNum()]);
- }
- } else {
- flagBaseline (ib, itsTD[OpenMP::threadNum()]);
- }
- } // end of OMP for
- } // end of OMP parallel
- itsComputeTimer.stop();
- itsTimer.stop();
- // Let the next step process the buffers.
- for (uint i=0; i<itsBufIndex; ++i) {
- getNextStep()->process (itsBuf[i]);
- }
- itsTimer.start();
- itsBufIndex = 0;
- }
-
- void SlidingFlagger::flagBaseline (uint bl, ThreadData& td)
- {
- td.moveTimer.start();
- // Get the sizes of the axes.
- uint ntime = itsWindowSize;
- uint nchan = itsBuf[0].getData().shape()[1];
- uint blsize = nchan * itsBuf[0].getData().shape()[0];
- // Fill the rficonsole buffers and flag.
- // Create the objects for the real and imaginary data of all corr.
- aoflagger::ImageSet imageSet =
- itsAOFlagger.MakeImageSet(ntime, nchan, 8);
- aoflagger::FlagMask origFlags =
- itsAOFlagger.MakeFlagMask(ntime, nchan);
- const uint iStride = imageSet.HorizontalStride();
- const uint fStride = origFlags.HorizontalStride();
- for (uint i=0; i<ntime; ++i) {
- const Complex* data = itsBuf[i].getData().data() + bl*blsize;
- const bool* flags = itsBuf[i].getFlags().data() + bl*blsize;
- for (uint j=0; j<nchan; ++j) {
- for (uint p=0; p<4; ++p) {
- imageSet.ImageBuffer(p*2 )[i + j*iStride] = data->real();
- imageSet.ImageBuffer(p*2+1)[i + j*iStride] = data->imag();
- data++;
- }
- origFlags.Buffer()[i + j*fStride] = *flags;
- flags += 4;
- }
- }
- // Execute the strategy to do the flagging.
- td.moveTimer.stop();
- td.flagTimer.start();
- aoflagger::FlagMask rfiMask = itsAOFlagger.Run(*itsStrategy, imageSet);
- td.flagTimer.stop();
- // Put back the true flags and count newly set flags.
- td.moveTimer.start();
- for (uint i=0; i<itsBufIndex; ++i) {
- bool* flags = itsBuf[i].getFlags().data() + bl*blsize;
- for (uint j=0; j<nchan; ++j) {
- // Only set if not already set.
- // Note that if first corr flag is true, all are true.
- // If any corr is newly set, set all corr.
- if (! flags[0]) {
- bool setFlag = true;
- if (rfiMask.Buffer()[i + j*fStride]) {
- td.flagCounter.incrCorrelation(0);
- td.flagCounter.incrCorrelation(1);
- td.flagCounter.incrCorrelation(2);
- td.flagCounter.incrCorrelation(3);
- } else {
- setFlag = false;
- }
- if (setFlag) {
- td.flagCounter.incrBaseline(bl);
- td.flagCounter.incrChannel(j);
- for (int k=0; k<4; ++k) {
- flags[k] = true;
- }
- }
- }
- flags += 4;
- }
- }
- td.moveTimer.stop();
- }
-
- void SlidingFlagger::fillStrategy()
- {
- if (! itsStrategyName.empty()) {
- File file(itsStrategyName);
- if (! file.exists()) {
- file = File("$LOFARROOT/share/rfistrategies/" + itsStrategyName);
- if (! file.exists()) {
- THROW (Exception, "Unknown rfistrategy file " << itsStrategyName);
- }
- }
- itsStrategy.reset(new aoflagger::Strategy
- (itsAOFlagger.LoadStrategy(file.path().absoluteName())));
- } else {
- double centralFrequency = 0.5*(itsFreqs[0] + itsFreqs[itsFreqs.size()-1]);
- double timeRes;
- if (itsBuf.size() >=2 ) {
- timeRes = (itsBuf[itsBuf.size()-1].getTime() -
- itsBuf[0].getTime()) / (itsBuf.size()-1);
- } else {
- timeRes = 0.0;
- }
- double frequencyRes;
- if (itsFreqs.size() >= 2) {
- frequencyRes = (itsFreqs[itsFreqs.size()-1] -
- itsFreqs[0]) / (itsFreqs.size()-1);
- } else {
- frequencyRes = 0.0;
- }
- itsStrategy.reset(new aoflagger::Strategy
- (itsAOFlagger.MakeStrategy(aoflagger::LOFAR_TELESCOPE,
- aoflagger::StrategyFlags::NONE,
- centralFrequency,
- timeRes,
- frequencyRes)));
- }
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP_AOFlag/test/CMakeLists.txt b/CEP/DP3/DPPP_AOFlag/test/CMakeLists.txt
deleted file mode 100644
index 03858a1ffd3e8ce869b15a577708ad7cd8b0554d..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/test/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# $Id: CMakeLists.txt 26355 2013-09-04 11:51:07Z dijkema $
-
-include(LofarCTest)
-
-# Do not use lofar_add_test, because libaoflaggerstep should not be linked in.
-add_executable (tAOFlaggerStep tAOFlaggerStep.cc)
-target_link_libraries (tAOFlaggerStep ${_libs} ${LOFAR_EXTRA_LIBRARIES})
-add_test (tAOFlaggerStep ${CMAKE_SOURCE_DIR}/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.sh)
diff --git a/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.cc b/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.cc
deleted file mode 100644
index 98bbae18d7de84615fa588410cb60da56054ba11..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.cc
+++ /dev/null
@@ -1,263 +0,0 @@
-//# tAOFlaggerStep.cc: Test program for class AOFlaggerStep
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tAOFlaggerStep.cc 24221 2013-03-12 12:24:48Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP_AOFlag/AOFlaggerStep.h>
-#include <DPPP/DPRun.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nant, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nant*(nant+1)/2), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(itsNBl);
- Vector<Int> ant2(itsNBl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<itsNBl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos (4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanFreqs(nchan);
- Vector<double> chanWidth(nchan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- cout << "Input step " << itsCount << ' '<< itsCount*5+2<<endl;
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(1.6, 0.9);
- }
- if (itsCount == 5) {
- data += Complex(10.,10.);
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&)
- // Use startchan=0 and timeInterval=5
- { info().init (itsNCorr, itsNChan, itsNTime, 100, 5, string(), string()); }
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nant, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nant*(nant+1)/2), itsNChan(nchan),
- itsNCorr(ncorr)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- cout << "Output step " << itsCount << ' '<<itsCount*5+2<<endl;
- // Fill expected result in similar way as TestInput.
- Cube<Complex> result(itsNCorr,itsNChan,itsNBl);
- for (int i=0; i<int(result.size()); ++i) {
- result.data()[i] = Complex(1.6, 0.9);
- }
- if (itsCount == 5) {
- result += Complex(10.,10.);
- }
- // Check the result.
- ///cout << buf.getData()<< result;
- ASSERT (allNear(real(buf.getData()), real(result), 1e-10));
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-10));
- ASSERT (near(buf.getTime(), 2+5.*itsCount));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNChan);
- ASSERT (int(info.nchan())==itsNChan);
- ASSERT (int(info.ntime())==itsNTime);
- ASSERT (info.startTime()==100);
- ASSERT (info.timeInterval()==5);
- ASSERT (int(info.nchanAvg())==1);
- ASSERT (int(info.ntimeAvg())==1);
- ASSERT (int(info.chanFreqs().size()) == itsNChan);
- ASSERT (int(info.chanWidths().size()) == itsNChan);
- ASSERT (info.msName().empty());
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set the info in each step.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
- DPStep::ShPtr step = step1;
- while (step) {
- step->showCounts (cout);
- step = step->getNextStep();
- }
-}
-
-// Test simple flagging with or without preflagged points.
-void test1(int ntime, int nant, int nchan, int ncorr, bool flag, int threshold)
-{
- cout << "test1: ntime=" << ntime << " nrant=" << nant << " nchan=" << nchan
- << " ncorr=" << ncorr << " threshold=" << threshold << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nant, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("timewindow", "1");
- DPStep::ShPtr step2 = DPRun::findStepCtor("aoflag")(in, parset, "");
- DPStep::ShPtr step3(new TestOutput(ntime, nant, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- step2->show (cout);
- execute (step1);
-}
-
-// Test applyautocorr flagging with or without preflagged points.
-void test2(int ntime, int nant, int nchan, int ncorr, bool flag, int threshold)
-{
- cout << "test2: ntime=" << ntime << " nrant=" << nant << " nchan=" << nchan
- << " ncorr=" << ncorr << " threshold=" << threshold << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nant, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("timewindow", "4");
- parset.add ("overlapmax", "1");
- DPStep::ShPtr step2 = DPRun::findStepCtor("aoflag")(in, parset, "");
- DPStep::ShPtr step3(new TestOutput(ntime, nant, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-
-int main()
-{
- INIT_LOGGER ("tAOFlaggerStep");
- try {
-
- for (uint i=0; i<2; ++i) {
- test1(10, 2, 32, 4, false, 1);
- test1(10, 5, 32, 4, true, 1);
- test2( 4, 2, 8, 4, false, 100);
- test2(10, 5, 32, 4, true, 1);
- test2( 8, 2, 8, 4, false, 100);
- test2(14, 2, 8, 4, false, 100);
- /// test2(99, 8, 64, 4, false, 100);
- }
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.run b/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.run
deleted file mode 100755
index 1678dd86f1ef83692797986a4d3a9af784c456cd..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.run
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh
-
-# Define the path for finding the library.
-#LD_LIBRARY_PATH=../src:/Users/diepen/external/lib:$LD_LIBRARYPATH
-#DYLD_LIBRARY_PATH=../src:/Users/diepen/external/lib:$DYLD_LIBRARYPATH
-LD_LIBRARY_PATH=../src:$LD_LIBRARY_PATH
-DYLD_LIBRARY_PATH=../src:$DYLD_LIBRARY_PATH
-export LD_LIBRARY_PATH
-export DYLD_LIBRARY_PATH
-echo $LD_LIBRARY_PATH
-echo $DYLD_LIBRARY_PATH
-
-./tAOFlaggerStep
diff --git a/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.sh b/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.sh
deleted file mode 100755
index d016046fd6788350e8076a07970c47e8c2d3d614..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_AOFlag/test/tAOFlaggerStep.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tAOFlaggerStep
diff --git a/CEP/DP3/DPPP_DDECal/CMakeLists.txt b/CEP/DP3/DPPP_DDECal/CMakeLists.txt
deleted file mode 100644
index f31f392f3d03521ed72dcbbd8b12530a57def188..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-# $Id: CMakeLists.txt 27640 2013-12-04 08:02:49Z diepen $
-
-lofar_package(DPPP_DDECal 1.0 DEPENDS DPPP)
-
-include(LofarFindPackage)
-lofar_find_package(Casacore COMPONENTS casa ms tables REQUIRED)
-lofar_find_package(Armadillo REQUIRED)
-lofar_find_package(Boost REQUIRED COMPONENTS date_time)
-
-add_subdirectory(include/DPPP_DDECal)
-add_subdirectory(src)
-add_subdirectory(test)
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/CMakeLists.txt b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/CMakeLists.txt
deleted file mode 100644
index 37345b4307e429f8677085302e23da9476c02738..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-# $Id: CMakeLists.txt 30990 2015-02-12 12:27:47Z diepen $
-
-# List of header files that will be installed.
-set(inst_HEADERS
- Register.h DDECal.h MultiDirSolver.h H5Parm.h
- Constraint.h KernelSmoother.h ScreenConstraint.h TECConstraint.h
- PiercePoint.h Matrix2x2.h PieceWisePhaseFitter.h
- RotationConstraint.h RotationAndDiagonalConstraint.h
- SmoothnessConstraint.h )
-
-# Create symbolic link to include directory.
-execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
- ${CMAKE_CURRENT_SOURCE_DIR}
- ${CMAKE_BINARY_DIR}/include/${PACKAGE_NAME})
-
-# Install header files.
-#install(FILES ${inst_HEADERS} DESTINATION include/${PACKAGE_NAME})
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Constraint.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Constraint.h
deleted file mode 100644
index b9af08e49f4d92dcdd1b208468070e28ffdd687f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Constraint.h
+++ /dev/null
@@ -1,167 +0,0 @@
-#ifndef CONSTRAINT_H
-#define CONSTRAINT_H
-
-#include <complex>
-#include <memory>
-#include <set>
-#include <vector>
-#include <ostream>
-
-/**
- * This class is the base class for classes that implement a constraint on
- * calibration solutions. Constraints are used to increase
- * the converge of calibration by applying these inside the solving step.
- *
- * The MultiDirSolver class uses this class for constrained calibration.
- */
-class Constraint
-{
-public:
- typedef std::complex<double> dcomplex;
- struct Result
- {
- public:
- std::vector<double> vals;
- std::vector<double> weights;
- std::string axes; // Comma-separated string with axis names, fastest varying last
- std::vector<size_t> dims;
- std::string name;
- };
-
- virtual ~Constraint() { }
-
- /**
- * Function that initializes the constraint for the next calibration iteration.
- * It should be called each time all antenna solutions have been calculated,
- * but before the constraint has been applied to all those antenna solutions.
- *
- * Unlike Apply(), this method is not thread safe.
- *
- * @param bool This can be used to specify whether the previous solution "step" is
- * smaller than the requested precision, i.e. calibration with the constrained
- * has converged. This allows a constraint to apply
- * its constraint in steps: apply a better-converging constraint as long as the
- * solutions are far from the correct answer, then switch to a different constraint
- * when hasReachedPrecision=true.
- */
- virtual void PrepareIteration(bool /*hasReachedPrecision*/, size_t /*iteration*/, bool /*finalIter*/) { }
-
- /**
- * Whether the constraint has been satisfied. The calibration process will continue
- * at least as long as Satisfied()=false, and performs at least one more iteration
- * after Satisfied()=true. Together with SetPrecisionReached(), this
- * can make the algorithm change the constraining method based on amount of
- * convergence.
- */
- virtual bool Satisfied() const { return true; }
-
- /**
- * This method applies the constraints to the solutions.
- * @param solutions is an array of array, such that:
- * - solutions[ch] is a pointer for channelblock ch to antenna x directions x pol solutions.
- * - pol is the dimension with the fastest changing index.
- * @param time Central time of interval.
- */
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time, std::ostream* statStream) = 0;
-
- /**
- * Initialize the dimensions for the constraint. Should be overridden when
- * something more than assigning dimensions is needed (e.g. resizing vectors).
- * Weights are initialized to 1. here.
- */
- virtual void InitializeDimensions(size_t nAntennas,
- size_t nDirections,
- size_t nChannelBlocks)
- {
- _nAntennas = nAntennas;
- _nDirections = nDirections;
- _nChannelBlocks = nChannelBlocks;
- }
-
- /**
- * Set weights. The vector should contain an array of size nAntennas * nChannelBlocks,
- * where the channel index varies fastest.
- */
- virtual void SetWeights(const std::vector<double> &) {}
-
- virtual void showTimings (std::ostream&, double) const {}
-
-protected:
- size_t _nAntennas, _nDirections, _nChannelBlocks;
-};
-
-/**
- * This class constraints the amplitudes of the solution to be unity, but
- * keeps the phase.
- */
-class PhaseOnlyConstraint : public Constraint
-{
-public:
- PhaseOnlyConstraint() {};
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time,
- std::ostream* statStream);
-};
-
-/**
- * This class constraints the phases of the solution to be zero, but
- * keeps the amplitude information.
- */
-class AmplitudeOnlyConstraint : public Constraint
-{
-public:
- AmplitudeOnlyConstraint() {};
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time,
- std::ostream* statStream);
-};
-
-class DiagonalConstraint : public Constraint
-{
-public:
- DiagonalConstraint(size_t polsPerSolution) : _polsPerSolution(polsPerSolution) {};
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time,
- std::ostream* statStream);
-private:
- const size_t _polsPerSolution;
-};
-
-/**
- * This constraint averages the solutions of a given list of antennas,
- * so that they have equal solutions.
- *
- * The DDE solver uses this constraint to average the solutions of the core
- * antennas. Core antennas are determined by a given maximum distance from
- * a reference antenna. The reference antenna is by default the first
- * antenna. This constraint is meant to force all core stations to
- * have the same solution, thereby decreasing the noise in their solutions.
- */
-class CoreConstraint : public Constraint
-{
-public:
- CoreConstraint() { }
-
- void initialize(const std::set<size_t>& coreAntennas)
- {
- _coreAntennas = coreAntennas;
- }
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time,
- std::ostream* statStream);
-
-private:
- std::set<size_t> _coreAntennas;
-};
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/DDECal.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/DDECal.h
deleted file mode 100644
index b6185b0cc0f0a0331d8b64e4eac064639491a14d..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/DDECal.h
+++ /dev/null
@@ -1,176 +0,0 @@
-//# DDE.h: DPPP step class to calibrate direction dependent gains
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: DDECal.h 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#ifndef DPPP_DDECAL_H
-#define DPPP_DDECAL_H
-
-// @file
-// @brief DPPP step class to apply a calibration correction to the data
-
-#include <DPPP/DPInput.h>
-#include <DPPP/GainCal.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/H5Parm.h>
-#include <DPPP/BaselineSelection.h>
-#include <DPPP/Patch.h>
-#include <DPPP/UVWFlagger.h>
-#include <DPPP/Predict.h>
-#include <DPPP/SourceDBUtil.h>
-#include <DPPP/ApplyBeam.h>
-#include <DPPP_DDECal/MultiDirSolver.h>
-#include <DPPP_DDECal/Constraint.h>
-#include <StationResponse/Station.h>
-#include <StationResponse/Types.h>
-#include <ParmDB/Parm.h>
-#include <casacore/casa/Arrays/Cube.h>
-#include <casacore/casa/Quanta/MVEpoch.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-
-namespace LOFAR {
-
- class ParameterSet;
-
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a DPStep class to calibrate (direction independent) gains.
-
- typedef vector<Patch::ConstPtr> PatchList;
- typedef std::pair<size_t, size_t> Baseline;
-
- class DDECal: public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- DDECal (DPInput*, const ParameterSet&, const std::string& prefix);
-
- virtual ~DDECal();
-
- // Create an DDECal object using the given parset.
- static DPStep::ShPtr makeStep (DPInput*, const ParameterSet&,
- const std::string&);
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Call the actual solver (called once per solution interval)
- void doSolve();
-
- // Initialize H5parm-file
- void initH5parm();
-
- // Write out the solutions
- void writeSolutions();
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
-
- private:
- // Initialize solutions
- void initializeScalarSolutions();
-
- void initializeFullMatrixSolutions();
-
- // Convert itsDirections to a vector of strings like "[Patch1, Patch2]"
- // Used for setting source names.
- std::vector<std::string> getDirectionNames();
-
- //# Data members.
- DPInput* itsInput;
- std::string itsName;
- vector<DPBuffer> itsBufs;
-
- bool itsUseModelColumn;
- std::vector<casacore::Cube<casacore::Complex>> itsModelData;
-
- // The time of the current buffer (in case of solint, average time)
- double itsAvgTime;
- std::vector<casacore::Complex*> itsDataPtrs;
-
- // For each timeslot, a vector of nDir buffers
- std::vector<std::vector<casacore::Complex*> > itsModelDataPtrs;
-
- // For each time, for each channel block, a vector of size nAntennas * nDirections
- std::vector<std::vector<std::vector<casacore::DComplex> > > itsSols;
- std::vector<uint>
- itsNIter, // Number of iterations taken
- itsNApproxIter;
-
- // For each time, for each constraint, a vector of results (e.g. tec and phase)
- std::vector<std::vector<std::vector<Constraint::Result> > > itsConstraintSols;
-
- std::string itsH5ParmName;
- H5Parm itsH5Parm;
- std::string itsParsetString; // Parset, for logging in H5Parm
-
- GainCal::CalType itsMode;
- bool itsPropagateSolutions;
- uint itsTimeStep;
- uint itsSolInt;
- uint itsStepInSolInt;
- uint itsNChan;
- vector<size_t> itsChanBlockStart; // For each channel block, the index in the channels at which this channel block starts
- vector<double> itsChanBlockFreqs;
- vector<vector<string> > itsDirections; // For each direction, a vector of patches
- vector<casacore::CountedPtr<Constraint> > itsConstraints;
-
- vector<double> itsWeights;
-
- UVWFlagger itsUVWFlagStep;
- ResultStep::ShPtr itsDataResultStep; // Result step for data after UV-flagging
- vector<Predict> itsPredictSteps;
- vector<MultiResultStep::ShPtr> itsResultSteps; // For each directions, a multiresultstep with all times
-
- NSTimer itsTimer;
- NSTimer itsTimerPredict;
- NSTimer itsTimerSolve;
- NSTimer itsTimerWrite;
- double itsCoreConstraint;
- double itsSmoothnessConstraint;
- double itsScreenCoreConstraint;
-
- MultiDirSolver itsMultiDirSolver;
- bool itsFullMatrixMinimalization;
- bool itsApproximateTEC;
- std::string itsStatFilename;
- std::unique_ptr<std::ofstream> itsStatStream;
- };
-
- } //# end namespace
-}
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/KLFitter.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/KLFitter.h
deleted file mode 100644
index 0cd5d46f1da48725ae37ec1825cc4bda7661aefd..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/KLFitter.h
+++ /dev/null
@@ -1,40 +0,0 @@
-#ifndef KLFITTER_H
-#define KLFITTER_H
-#include<vector>
-#include <armadillo>
-#include <DPPP_DDECal/PiercePoint.h>
-
-namespace LOFAR{
-class KLFitter
-{//creates KH base and fits screens from collection of PiercePoints
-public:
- KLFitter(double r0=1000.,double beta=5./3.,int order=3);
- void calculateCorrMatrix(const std::vector<PiercePoint> pp);
- void calculateCorrMatrix(const std::vector<PiercePoint*> pp);
- void doFit();
- size_t getOrder() const {return itsOrder;}
- double* PhaseData() { return _phases.memptr(); }
- double* WData() { return _weights.memptr(); }
- double* ParData() { return itsPar.memptr(); }
- double* TECFitWhiteData() { return itsTECFitWhite.memptr(); }
- double* PPData() { return itsPiercePoints.memptr(); }
- void setR0(double r0) {itsR0=r0;}
- void setBeta(double beta) {itsBeta=beta;}
- void setOrder(double order) {itsOrder=order;}
- size_t getNumberofPP() {return itsPiercePoints.n_rows;}
-
-private:
- size_t itsOrder;
- double itsR0,itsBeta;
- arma::Mat<double> itsPiercePoints;
- arma::Col<double> _phases;
- arma::Mat<double> _weights; //weights of the data points
- arma::Mat<double> itsCorrMatrix; //Correlation Matrix for KL fit
- arma::Mat<double> itsinvC; //for quick interpolation
- arma::Mat<double> itsU;
- arma::Mat<double> itsinvU;
- arma::Mat<double> itsTECFitWhite;
- arma::Col<double> itsPar;
-};
-}
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/KernelSmoother.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/KernelSmoother.h
deleted file mode 100644
index 984e82d173b41faa96324d78545fb134b97a7359..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/KernelSmoother.h
+++ /dev/null
@@ -1,101 +0,0 @@
-#ifndef KERNEL_SMOOTHER_H
-#define KERNEL_SMOOTHER_H
-
-#include <cmath>
-#include <stdexcept>
-#include <vector>
-
-template<typename DataType, typename NumType>
-class KernelSmoother
-{
-public:
- enum KernelType {
- RectangularKernel,
- TriangularKernel,
- /** Gaussian, trimmed off at 3 sigma */
- GaussianKernel,
- /** The Epanechnikov kernel is a quadratic kernel, given by 3/4 (1 - x^2) */
- EpanechnikovKernel
- };
-
- KernelSmoother(const NumType* frequencies, size_t n, KernelType kernelType, NumType kernelBandwidth) :
- _frequencies(frequencies, frequencies+n),
- _scratch(n),
- _kernelType(kernelType),
- _bandwidth(kernelBandwidth)
- {
- }
-
- NumType Kernel(NumType distance) const
- {
- NumType x = distance / _bandwidth;
- if(x < NumType(-1.0) || x > NumType(1.0))
- return NumType(0.0);
- else {
- switch(_kernelType)
- {
- case RectangularKernel:
- default:
- return NumType(0.5);
- case TriangularKernel:
- return x >= NumType(0.0) ? (NumType(1.0) - x) : (NumType(1.0) + x);
- case GaussianKernel:
- // e^(-x^2 / sigma^2), sigma = bandwidth / 3.
- return std::exp(-x*x*NumType(9.0));
- case EpanechnikovKernel:
- // 3/4 * (1-x)^2;
- x = NumType(1.0) - x;
- return (NumType(3.0) / NumType(4.0)) * x * x;
- }
- }
- }
-
- void Smooth(DataType* data, const NumType* weight)
- {
- size_t n = _frequencies.size();
-
- size_t
- bandLeft = 0,
- // find right kernel value for first element
- bandRight = std::lower_bound(_frequencies.begin(), _frequencies.end(), _frequencies[0] + _bandwidth * 0.5) - _frequencies.begin() + 1;
-
- for(size_t i=0; i!=n; ++i)
- {
- // If a boundary is further than half the bandwidth away, move boundary
- while(_frequencies[bandLeft] < _frequencies[i] - _bandwidth * 0.5)
- ++bandLeft;
- while(bandRight!=n && _frequencies[bandRight] < _frequencies[i] + _bandwidth * 0.5)
- ++bandRight;
-
- // A value of 1 is added to make sure we are not skipping a value because of rounding errors
- // (kernel will be zero past boundaries, so including an unnecessary value has no effect)
- size_t start = bandLeft > 0 ? bandLeft-1 : 0;
- size_t end = bandRight < n ? bandRight+1 : n;
-
- DataType sum(0.0);
- NumType weightSum(0.0);
- //std::cout << start << " -> " << end << " (" << _frequencies[start] << " -> " << _frequencies[end] << ")\n";
- for(size_t j=start; j!=end; ++j)
- {
- double distance = _frequencies[i] - _frequencies[j];
- double w = Kernel(distance) * weight[j];
- sum += data[j] * w;
- weightSum += w;
- }
- if(weightSum == 0.0)
- _scratch[i] = 0.0;
- else
- _scratch[i] = sum / weightSum;
- }
- std::copy(_scratch.begin(), _scratch.end(), data);
- }
-
-private:
- std::vector<NumType> _frequencies;
- std::vector<DataType> _scratch;
- enum KernelType _kernelType;
- NumType _bandwidth;
-};
-
-#endif
-
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Matrix2x2.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Matrix2x2.h
deleted file mode 100644
index 69746ed8cab4c955c38f8b860151d96d06f2512b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Matrix2x2.h
+++ /dev/null
@@ -1,445 +0,0 @@
-#ifndef MATRIX_2X2_H
-#define MATRIX_2X2_H
-
-#include <complex>
-#include <limits>
-#include <sstream>
-
-class Matrix2x2
-{
-public:
- template<typename LHS_T, typename RHS_T>
- static void Assign(std::complex<LHS_T>* dest, const std::complex<RHS_T>* source)
- {
- for(size_t p=0; p!=4; ++p)
- dest[p] = source[p];
- }
-
- template<typename LHS_T, typename RHS_T>
- static void Assign(LHS_T* dest, const RHS_T* source)
- {
- for(size_t p=0; p!=4; ++p)
- dest[p] = source[p];
- }
-
- template<typename T, typename RHS_T>
- static void Add(std::complex<T>* dest, const RHS_T * rhs)
- {
- for(size_t p=0; p!=4; ++p)
- dest[p] += rhs[p];
- }
-
- template<typename T>
- static void Subtract(std::complex<T>* dest, const std::complex<T>* rhs)
- {
- for(size_t p=0; p!=4; ++p)
- dest[p] -= rhs[p];
- }
-
- template<typename T>
- static bool IsFinite(const std::complex<T>* matrix)
- {
- return
- std::isfinite(matrix[0].real()) && std::isfinite(matrix[0].imag()) &&
- std::isfinite(matrix[1].real()) && std::isfinite(matrix[1].imag()) &&
- std::isfinite(matrix[2].real()) && std::isfinite(matrix[2].imag()) &&
- std::isfinite(matrix[3].real()) && std::isfinite(matrix[3].imag());
- }
-
- template<typename T>
- static void ScalarMultiply(std::complex<T>* dest, T factor)
- {
- for(size_t p=0; p!=4; ++p)
- dest[p] *= factor;
- }
-
- template<typename T>
- static void ScalarMultiply(T* dest, T factor)
- {
- for(size_t p=0; p!=4; ++p)
- dest[p] *= factor;
- }
-
- template<typename T, typename RHS>
- static void MultiplyAdd(std::complex<T>* dest, const RHS* rhs, T factor)
- {
- for(size_t p=0; p!=4; ++p)
- dest[p] += rhs[p] * factor;
- }
-
- template<typename ComplType, typename LHS_T, typename RHS_T>
- static void ATimesB(std::complex<ComplType>* dest, const LHS_T* lhs, const RHS_T* rhs)
- {
- dest[0] = lhs[0] * rhs[0] + lhs[1] * rhs[2];
- dest[1] = lhs[0] * rhs[1] + lhs[1] * rhs[3];
- dest[2] = lhs[2] * rhs[0] + lhs[3] * rhs[2];
- dest[3] = lhs[2] * rhs[1] + lhs[3] * rhs[3];
- }
-
- static void PlusATimesB(std::complex<double> *dest, const std::complex<double> *lhs, const std::complex<double> *rhs)
- {
- dest[0] += lhs[0] * rhs[0] + lhs[1] * rhs[2];
- dest[1] += lhs[0] * rhs[1] + lhs[1] * rhs[3];
- dest[2] += lhs[2] * rhs[0] + lhs[3] * rhs[2];
- dest[3] += lhs[2] * rhs[1] + lhs[3] * rhs[3];
- }
-
- template<typename ComplType, typename LHS_T, typename RHS_T>
- static void ATimesHermB(std::complex<ComplType> *dest, const LHS_T* lhs, const RHS_T* rhs)
- {
- dest[0] = lhs[0] * std::conj(rhs[0]) + lhs[1] * std::conj(rhs[1]);
- dest[1] = lhs[0] * std::conj(rhs[2]) + lhs[1] * std::conj(rhs[3]);
- dest[2] = lhs[2] * std::conj(rhs[0]) + lhs[3] * std::conj(rhs[1]);
- dest[3] = lhs[2] * std::conj(rhs[2]) + lhs[3] * std::conj(rhs[3]);
- }
-
- template<typename ComplType, typename LHS_T, typename RHS_T>
- static void PlusATimesHermB(std::complex<ComplType> *dest, const LHS_T* lhs, const RHS_T* rhs)
- {
- dest[0] += lhs[0] * std::conj(rhs[0]) + lhs[1] * std::conj(rhs[1]);
- dest[1] += lhs[0] * std::conj(rhs[2]) + lhs[1] * std::conj(rhs[3]);
- dest[2] += lhs[2] * std::conj(rhs[0]) + lhs[3] * std::conj(rhs[1]);
- dest[3] += lhs[2] * std::conj(rhs[2]) + lhs[3] * std::conj(rhs[3]);
- }
-
- template<typename ComplType, typename LHS_T, typename RHS_T>
- static void HermATimesB(std::complex<ComplType> *dest, const LHS_T* lhs, const RHS_T* rhs)
- {
- dest[0] = std::conj(lhs[0]) * rhs[0] + std::conj(lhs[2]) * rhs[2];
- dest[1] = std::conj(lhs[0]) * rhs[1] + std::conj(lhs[2]) * rhs[3];
- dest[2] = std::conj(lhs[1]) * rhs[0] + std::conj(lhs[3]) * rhs[2];
- dest[3] = std::conj(lhs[1]) * rhs[1] + std::conj(lhs[3]) * rhs[3];
- }
-
- static void HermATimesHermB(std::complex<double> *dest, const std::complex<double> *lhs, const std::complex<double> *rhs)
- {
- dest[0] = std::conj(lhs[0]) * std::conj(rhs[0]) + std::conj(lhs[2]) * std::conj(rhs[1]);
- dest[1] = std::conj(lhs[0]) * std::conj(rhs[2]) + std::conj(lhs[2]) * std::conj(rhs[3]);
- dest[2] = std::conj(lhs[1]) * std::conj(rhs[0]) + std::conj(lhs[3]) * std::conj(rhs[1]);
- dest[3] = std::conj(lhs[1]) * std::conj(rhs[2]) + std::conj(lhs[3]) * std::conj(rhs[3]);
- }
-
- template<typename ComplType, typename LHS_T, typename RHS_T>
- static void PlusHermATimesB(std::complex<ComplType> *dest, const LHS_T* lhs, const RHS_T* rhs)
- {
- dest[0] += std::conj(lhs[0]) * rhs[0] + std::conj(lhs[2]) * rhs[2];
- dest[1] += std::conj(lhs[0]) * rhs[1] + std::conj(lhs[2]) * rhs[3];
- dest[2] += std::conj(lhs[1]) * rhs[0] + std::conj(lhs[3]) * rhs[2];
- dest[3] += std::conj(lhs[1]) * rhs[1] + std::conj(lhs[3]) * rhs[3];
- }
-
- template<typename T>
- static bool Invert(T* matrix)
- {
- T d = ((matrix[0]*matrix[3]) - (matrix[1]*matrix[2]));
- if(d == T(0.0))
- return false;
- T oneOverDeterminant = T(1.0) / d;
- T temp;
- temp = matrix[3] * oneOverDeterminant;
- matrix[1] = -matrix[1] * oneOverDeterminant;
- matrix[2] = -matrix[2] * oneOverDeterminant;
- matrix[3] = matrix[0] * oneOverDeterminant;
- matrix[0] = temp;
- return true;
- }
-
- template<typename T>
- static void ConjugateTranspose(T* matrix)
- {
- matrix[0] = std::conj(matrix[0]);
- T temp = matrix[1];
- matrix[1] = std::conj(matrix[2]);
- matrix[2] = std::conj(temp);
- matrix[3] = std::conj(matrix[3]);
- }
-
- static bool MultiplyWithInverse(std::complex<double>* lhs, const std::complex<double>* rhs)
- {
- std::complex<double> d = ((rhs[0]*rhs[3]) - (rhs[1]*rhs[2]));
- if(d == 0.0) return false;
- std::complex<double> oneOverDeterminant = 1.0 / d;
- std::complex<double> temp[4];
- temp[0] = rhs[3] * oneOverDeterminant;
- temp[1] = -rhs[1] * oneOverDeterminant;
- temp[2] = -rhs[2] * oneOverDeterminant;
- temp[3] = rhs[0] * oneOverDeterminant;
-
- std::complex<double> temp2 = lhs[0];
- lhs[0] = lhs[0] * temp[0] + lhs[1] * temp[2];
- lhs[1] = temp2 * temp[1] + lhs[1] * temp[3];
-
- temp2 = lhs[2];
- lhs[2] = lhs[2] * temp[0] + lhs[3] * temp[2];
- lhs[3] = temp2 * temp[1] + lhs[3] * temp[3];
- return true;
- }
-
- static void SingularValues(const std::complex<double>* matrix, double &e1, double &e2)
- {
- // This is not the ultimate fastest method, since we
- // don't need to calculate the imaginary values of b,c at all.
- // Calculate M M^H
- std::complex<double> temp[4] = {
- matrix[0] * std::conj(matrix[0]) + matrix[1] * std::conj(matrix[1]),
- matrix[0] * std::conj(matrix[2]) + matrix[1] * std::conj(matrix[3]),
- matrix[2] * std::conj(matrix[0]) + matrix[3] * std::conj(matrix[1]),
- matrix[2] * std::conj(matrix[2]) + matrix[3] * std::conj(matrix[3])
- };
- // Use quadratic formula, with a=1.
- double
- b = -temp[0].real() - temp[3].real(),
- c = temp[0].real()*temp[3].real() - (temp[1]*temp[2]).real(),
- d = b*b - (4.0*1.0)*c,
- sqrtd = sqrt(d);
-
- e1 = sqrt((-b + sqrtd) * 0.5);
- e2 = sqrt((-b - sqrtd) * 0.5);
- }
-
- static void EigenValues(const double* matrix, double &e1, double &e2)
- {
- double tr = matrix[0] + matrix[3];
- double d = matrix[0]*matrix[3] - matrix[1]*matrix[2];
- double term = sqrt(tr*tr*0.25-d);
- double trHalf = tr*0.5;
- e1 = trHalf + term;
- e2 = trHalf - term;
- }
-
- template<typename ValType>
- static void EigenValues(const std::complex<ValType>* matrix, std::complex<ValType>& e1, std::complex<ValType>& e2)
- {
- std::complex<ValType> tr = matrix[0] + matrix[3];
- std::complex<ValType> d = matrix[0]*matrix[3] - matrix[1]*matrix[2];
- std::complex<ValType> term = sqrt(tr*tr*ValType(0.25)-d);
- std::complex<ValType> trHalf = tr*ValType(0.5);
- e1 = trHalf + term;
- e2 = trHalf - term;
- }
-
- static void EigenValuesAndVectors(const double* matrix, double &e1, double &e2, double* vec1, double* vec2)
- {
- double tr = matrix[0] + matrix[3];
- double d = matrix[0]*matrix[3] - matrix[1]*matrix[2];
- double term = sqrt(tr*tr*0.25-d);
- double trHalf = tr*0.5;
- e1 = trHalf + term;
- e2 = trHalf - term;
- if(matrix[2] != 0.0)
- {
- vec1[0] = matrix[3] - e1;
- vec1[1] = -matrix[2];
- vec2[0] = matrix[3] - e2;
- vec2[1] = -matrix[2];
- }
- else if(matrix[1] != 0.0)
- {
- vec1[0] = -matrix[1];
- vec1[1] = matrix[0] - e1;
- vec2[0] = -matrix[1];
- vec2[1] = matrix[0] - e2;
- }
- else {
- vec1[0] = 1.0;
- vec1[1] = 0.0;
- vec2[0] = 0.0;
- vec2[1] = 1.0;
- }
- }
-
- static void SquareRoot(double* matrix)
- {
- double tr = matrix[0] + matrix[3];
- double d = matrix[0]*matrix[3] - matrix[1]*matrix[2];
- double s = /*+/-*/ sqrt(d);
- double t = /*+/-*/ sqrt(tr + 2.0*s);
- if(t != 0.0)
- {
- matrix[0] = (matrix[0]+s ) / t;
- matrix[1] = (matrix[1] / t);
- matrix[2] = (matrix[2] / t);
- matrix[3] = (matrix[3]+s) / t;
- }
- else {
- if(matrix[0] == 0.0 && matrix[1] == 0.0 &&
- matrix[2] == 0.0 && matrix[3] == 0.0)
- {
- // done: it's the zero matrix
- } else {
- for(size_t i=0; i!=4; ++i)
- matrix[i] = std::numeric_limits<double>::quiet_NaN();
- }
- }
- }
-
- template<typename T>
- static T RotationAngle(const std::complex<T>* matrix)
- {
- return std::atan2((matrix[2].real()-matrix[1].real())*0.5, (matrix[0].real()+matrix[3].real())*0.5);
- }
-
- template<typename T>
- static void RotationMatrix(std::complex<T>* matrix, double alpha)
- {
- T cosAlpha, sinAlpha;
- sincos(alpha, &sinAlpha, &cosAlpha);
- matrix[0] = cosAlpha; matrix[1] = -sinAlpha;
- matrix[2] = sinAlpha; matrix[3] = cosAlpha;
- }
-};
-
-template<typename ValType>
-class MC2x2Base
-{
-public:
- MC2x2Base() { }
- MC2x2Base(const MC2x2Base<ValType>& source) { Matrix2x2::Assign(_values, source._values); }
- template<typename T>
- explicit MC2x2Base(const T source[4]) { Matrix2x2::Assign(_values, source); }
- MC2x2Base(ValType m00, ValType m01, ValType m10, ValType m11) {
- _values[0] = m00; _values[1] = m01;
- _values[2] = m10; _values[3] = m11;
- }
- MC2x2Base(std::complex<ValType> m00, std::complex<ValType> m01, std::complex<ValType> m10, std::complex<ValType> m11) {
- _values[0] = m00; _values[1] = m01;
- _values[2] = m10; _values[3] = m11;
- }
- MC2x2Base<ValType>& operator=(const MC2x2Base<ValType>& source)
- {
- Matrix2x2::Assign(_values, source._values);
- return *this;
- }
- MC2x2Base<ValType>& operator+=(const MC2x2Base<ValType>& rhs)
- {
- Matrix2x2::Add(_values, rhs._values);
- return *this;
- }
- MC2x2Base<ValType>& operator-=(const MC2x2Base<ValType>& rhs)
- {
- Matrix2x2::Subtract(_values, rhs._values);
- return *this;
- }
- MC2x2Base<ValType>& operator*=(const MC2x2Base<ValType>& rhs)
- {
- MC2x2Base<ValType> dest;
- Matrix2x2::ATimesB(dest._values, _values, rhs._values);
- *this = dest;
- return *this;
- }
- MC2x2Base<ValType>& operator*=(ValType rhs)
- {
- Matrix2x2::ScalarMultiply(_values, rhs);
- return *this;
- }
- MC2x2Base<ValType>& operator/=(ValType rhs)
- {
- Matrix2x2::ScalarMultiply(_values, 1.0/rhs);
- return *this;
- }
- const std::complex<ValType>& operator[](size_t index) const { return _values[index]; }
- std::complex<ValType>& operator[](size_t index) { return _values[index]; }
- const ValType& IndexReal(size_t index) const { return reinterpret_cast<const ValType(&)[2]>(_values[index/2])[index%2]; }
- ValType& IndexReal(size_t index) { return reinterpret_cast<ValType(&)[2]>(_values[index/2])[index%2]; }
- static MC2x2Base<ValType> Zero()
- {
- return MC2x2Base<ValType>(0.0, 0.0, 0.0, 0.0);
- }
- static MC2x2Base<ValType> Unity()
- {
- return MC2x2Base(1.0, 0.0, 0.0, 1.0);
- }
- static MC2x2Base<ValType> NaN()
- {
- return MC2x2Base<ValType>(
- std::complex<ValType>(std::numeric_limits<ValType>::quiet_NaN(), std::numeric_limits<ValType>::quiet_NaN()),
- std::complex<ValType>(std::numeric_limits<ValType>::quiet_NaN(), std::numeric_limits<ValType>::quiet_NaN()),
- std::complex<ValType>(std::numeric_limits<ValType>::quiet_NaN(), std::numeric_limits<ValType>::quiet_NaN()),
- std::complex<ValType>(std::numeric_limits<ValType>::quiet_NaN(), std::numeric_limits<ValType>::quiet_NaN()));
- }
- std::complex<ValType>* Data() { return _values; }
- const std::complex<ValType>* Data() const { return _values; }
- MC2x2Base<ValType> Multiply(const MC2x2Base<ValType>& rhs) const
- {
- MC2x2Base<ValType> dest;
- Matrix2x2::ATimesB(dest._values, _values, rhs._values);
- return dest;
- }
- MC2x2Base<ValType> MultiplyHerm(const MC2x2Base<ValType>& rhs) const
- {
- MC2x2Base dest;
- Matrix2x2::ATimesHermB(dest._values, _values, rhs._values);
- return dest;
- }
- MC2x2Base<ValType> HermThenMultiply(const MC2x2Base<ValType>& rhs) const
- {
- MC2x2Base<ValType> dest;
- Matrix2x2::HermATimesB(dest._values, _values, rhs._values);
- return dest;
- }
- MC2x2Base<ValType> HermThenMultiplyHerm(const MC2x2Base<ValType>& rhs) const
- {
- MC2x2Base<ValType> dest;
- Matrix2x2::HermATimesHermB(dest._values, _values, rhs._values);
- return dest;
- }
- void AddWithFactorAndAssign(const MC2x2Base<ValType>& rhs, ValType factor)
- {
- Matrix2x2::MultiplyAdd(_values, rhs._values, factor);
- }
- bool Invert()
- {
- return Matrix2x2::Invert(_values);
- }
- static void ATimesB(MC2x2Base<ValType>& dest, const MC2x2Base<ValType>& lhs, const MC2x2Base<ValType>& rhs)
- {
- Matrix2x2::ATimesB(dest._values, lhs._values, rhs._values);
- }
- static void ATimesB(std::complex<ValType>* dest, const MC2x2Base<ValType>& lhs, const MC2x2Base<ValType>& rhs)
- {
- Matrix2x2::ATimesB(dest, lhs._values, rhs._values);
- }
- static void ATimesHermB(MC2x2Base<ValType>& dest, const MC2x2Base<ValType>& lhs, const MC2x2Base<ValType>& rhs)
- {
- Matrix2x2::ATimesHermB(dest._values, lhs._values, rhs._values);
- }
- static void HermATimesB(MC2x2Base<ValType>& dest, const MC2x2Base<ValType>& lhs, const MC2x2Base<ValType>& rhs)
- {
- Matrix2x2::HermATimesB(dest._values, lhs._values, rhs._values);
- }
- static void HermATimesHermB(MC2x2Base<ValType>& dest, const MC2x2Base<ValType>& lhs, const MC2x2Base<ValType>& rhs)
- {
- Matrix2x2::HermATimesHermB(dest._values, lhs._values, rhs._values);
- }
- std::string ToString() const
- {
- std::stringstream str;
- str << _values[0] << ", " << _values[1] << "; "
- << _values[2] << ", " << _values[3];
- return str.str();
- }
- void CopyValues(std::complex<ValType>* values) const
- {
- Matrix2x2::Assign(values, _values);
- }
- void EigenValues(std::complex<ValType> &e1, std::complex<ValType> &e2) const
- {
- Matrix2x2::EigenValues(_values, e1, e2);
- }
- bool HasNaN() const
- {
- return !(
- std::isfinite(_values[0].real()) && std::isfinite(_values[0].imag()) &&
- std::isfinite(_values[1].real()) && std::isfinite(_values[1].imag()) &&
- std::isfinite(_values[2].real()) && std::isfinite(_values[2].imag()) &&
- std::isfinite(_values[3].real()) && std::isfinite(_values[3].imag())
- );
- }
-private:
- std::complex<ValType> _values[4];
-};
-
-using MC2x2 = MC2x2Base<double>;
-using MC2x2F = MC2x2Base<float>;
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/MultiDirSolver.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/MultiDirSolver.h
deleted file mode 100644
index ddb5b8f397f8b581790a38777a89df98a2570630..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/MultiDirSolver.h
+++ /dev/null
@@ -1,154 +0,0 @@
-#ifndef MULTI_DIR_SOLVER_H
-#define MULTI_DIR_SOLVER_H
-
-#ifdef AOPROJECT
-#include "PhaseFitter.h"
-#include "Constraint.h"
-#include "Stopwatch.h"
-#define UPTR std::unique_ptr
-#else
-#include <DPPP/PhaseFitter.h>
-#include <DPPP_DDECal/Constraint.h>
-#include <DPPP_DDECal/Stopwatch.h>
-#define UPTR std::auto_ptr
-#endif
-
-#include <complex>
-#include <vector>
-#include <memory>
-
-class MultiDirSolver
-{
-public:
- typedef std::complex<double> DComplex;
- typedef std::complex<float> Complex;
-
- class Matrix : public std::vector<DComplex>
- {
- public:
- Matrix() : _m(0) { }
- Matrix(size_t m, size_t n) : std::vector<DComplex>(n * m, 0.0), _m(m)
- { }
- void zeros()
- {
- assign(size(), DComplex(0.0, 0.0));
- }
- DComplex& operator()(size_t i, size_t j)
- {
- return (*this)[i + j*_m];
- }
- private:
- size_t _m;
- };
-
-
- struct SolveResult {
- size_t iterations, constraintIterations;
- std::vector<std::vector<Constraint::Result> > _results;
- };
-
- MultiDirSolver();
-
- void init(size_t nAntennas, size_t nDirections, size_t nChannels,
- const std::vector<int>& ant1, const std::vector<int>& ant2);
-
- // data[i] is een pointer naar de data voor tijdstap i, vanaf die pointer staat het in volgorde als in MS (bl, chan, pol)
- // mdata[i] is een pointer voor tijdstap i naar arrays van ndir model data pointers (elk van die data pointers staat in zelfde volgorde als data)
- // solutions[ch] is een pointer voor channelblock ch naar antenna x directions oplossingen.
- SolveResult processScalar(std::vector<Complex*>& data,
- std::vector<std::vector<Complex* > >& modelData,
- std::vector<std::vector<DComplex> >& solutions, double time,
- std::ostream* statStream);
-
- /**
- * Same as @ref processScalar(), but solves full Jones matrices.
- * @param data als in @ref processScalar()
- * @param modelData als in @ref processScalar()
- * @param solutions An array, where @c solutions[ch] is a pointer to channelblock @c ch, that points to
- * antenna x directions solutions. Each solution consists of 4 complex values forming the full Jones matrix.
- */
- SolveResult processFullMatrix(std::vector<Complex *>& data,
- std::vector<std::vector<Complex *> >& modelData,
- std::vector<std::vector<DComplex> >& solutions, double time,
- std::ostream* statStream);
-
- void set_phase_only(bool phaseOnly) { _phaseOnly = phaseOnly; }
-
- void set_channel_blocks(size_t nChannelBlocks) { _nChannelBlocks = nChannelBlocks; }
-
- size_t max_iterations() const { return _maxIterations; }
- void set_max_iterations(size_t maxIterations) { _maxIterations = maxIterations; }
-
- void set_accuracy(double accuracy) {
- _accuracy = accuracy;
- }
- double get_accuracy() const { return _accuracy; }
- void set_constraint_accuracy(double constraintAccuracy) {
- _constraintAccuracy = constraintAccuracy;
- }
- void set_step_size(double stepSize) { _stepSize = stepSize; }
- double get_step_size() const { return _stepSize; }
-
- void set_detect_stalling(bool detectStalling) { _detectStalling = detectStalling; }
-
- bool get_detect_stalling() const { return _detectStalling; }
-
- void add_constraint(Constraint* constraint) { _constraints.push_back(constraint); }
-
- void showTimings (std::ostream& os, double duration) const;
-
-private:
- void performScalarIteration(size_t channelBlockIndex,
- std::vector<Matrix>& gTimesCs,
- std::vector<Matrix>& vs,
- const std::vector<DComplex>& solutions,
- std::vector<DComplex>& nextSolutions,
- const std::vector<Complex *>& data,
- const std::vector<std::vector<Complex *> >& modelData);
-
- void performFullMatrixIteration(size_t channelBlockIndex,
- std::vector<Matrix>& gTimesCs,
- std::vector<Matrix>& vs,
- const std::vector<DComplex>& solutions,
- std::vector<DComplex>& nextSolutions,
- const std::vector<Complex *>& data,
- const std::vector<std::vector<Complex *> >& modelData);
-
- void makeStep(const std::vector<std::vector<DComplex> >& solutions,
- std::vector<std::vector<DComplex> >& nextSolutions) const;
-
- bool detectStall(size_t iteration, const std::vector<double>& stepMagnitudes) const;
-
- void makeSolutionsFinite(std::vector<std::vector<DComplex> >& solutions, size_t perPol) const;
-
- /**
- * Assign the solutions in nextSolutions to the solutions.
- * @returns whether the solutions have converged. Appends the current step magnitude to step_magnitudes
- */
- template<size_t NPol>
- bool assignSolutions(
- std::vector<std::vector<DComplex> >& solutions,
- std::vector<std::vector<DComplex> >& nextSolutions,
- bool useConstraintAccuracy,
- double& sum,
- std::vector<double>& step_magnitudes
- ) const;
-
- size_t _nAntennas, _nDirections, _nChannels, _nChannelBlocks;
- std::vector<int> _ant1, _ant2;
-
- // Calibration setup
- size_t _maxIterations;
- double _accuracy, _constraintAccuracy;
- double _stepSize;
- bool _detectStalling;
- bool _phaseOnly;
- std::vector<Constraint*> _constraints;
-
- // Timers
- Stopwatch _timerSolve;
- Stopwatch _timerConstrain;
- Stopwatch _timerFillMatrices;
-};
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/PieceWisePhaseFitter.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/PieceWisePhaseFitter.h
deleted file mode 100644
index 2ea7b76c2125ca25542238289da957cb201d2478..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/PieceWisePhaseFitter.h
+++ /dev/null
@@ -1,426 +0,0 @@
-#ifndef PIECE_WISE_PHASE_FITTER_H
-#define PIECE_WISE_PHASE_FITTER_H
-
-#include <algorithm>
-#include <iterator>
-#include <cmath>
-
-class PieceWisePhaseFitter
-{
-public:
- PieceWisePhaseFitter() : _chunkSize(0) { }
-
- /**
- * Constructor.
- * @param chunkSize Size of chunk in number of samples.
- */
- PieceWisePhaseFitter(size_t chunkSize) : _chunkSize(chunkSize) { }
-
- size_t ChunkSize() const { return _chunkSize; }
- void SetChunkSize(size_t chunkSize) { _chunkSize = chunkSize; }
-
- /**
- * Unwrap a range of phase values.
- */
- template<typename Iter>
- static void Unwrap(Iter first, Iter last)
- {
- for(Iter iter = first+1; iter!=last; ++iter)
- {
- while(*iter - (*(iter-1)) > M_PI)
- *iter -= 2.0*M_PI;
- while((*(iter-1) - *iter) > M_PI)
- *iter += 2.0*M_PI;
- }
- }
-
- static size_t CalculateChunkSize(double startFrequencyHz, double endFrequencyHz, size_t channelCount)
- {
- // it seems that 10 chunks per octave seems reasonable
- double nOctaves = (log(endFrequencyHz) - log(startFrequencyHz)) / M_LN2;
- if(nOctaves > 0.0)
- return std::min<size_t>(channelCount, ceil(channelCount / (10.0 * nOctaves)));
- else
- return channelCount;
- }
-
- /**
- * Perform a piece-wise fit of linear phase gradients.
- *
- * The data is divided into chunks with given size, and in each chunk a line is fitted (offset & gradient).
- * The fitted line of each chunk is written into fittedData. The resulting line will have a 'jump' between each
- * chunk.
- * @param nu Frequencies of the channels (in Hz).
- * @param data The values, same size as @c nu.
- * @param fittedData A vector of at least size @c nu.size(), in which the fitted lines are stored.
- */
- void PieceWiseFit(const std::vector<double>& nu, const std::vector<double>& data, std::vector<double>& fittedData)
- {
- size_t chunkSize = std::min(data.size(), _chunkSize);
- if(chunkSize==1)
- fittedData = data;
- else
- {
- for(size_t ch=0; ch<data.size(); ch+=chunkSize)
- {
- size_t pos = ch;
- if(pos > data.size() - chunkSize)
- pos = data.size() - chunkSize;
- fitChunk(chunkSize, pos, nu, data, fittedData);
- }
- }
- }
-
- void PieceWiseFit(const std::vector<double>& nu, const std::vector<double>& data, const double* weights, std::vector<double>& fittedData)
- {
- size_t chunkSize = std::min(data.size(), _chunkSize);
- if(chunkSize==1)
- fittedData = data;
- else
- {
- for(size_t ch=0; ch<data.size(); ch+=chunkSize)
- {
- size_t pos = ch;
- if(pos > data.size() - chunkSize)
- pos = data.size() - chunkSize;
- fitChunk(chunkSize, pos, nu, data, weights, fittedData);
- }
- }
- }
-
- /**
- * Performs a sliding fit of linear phase gradients (without weighting).
- * See other SlidingFit() overload.
- */
- void SlidingFit(const double* nu, const double* data, double* fittedData, size_t n)
- {
- const size_t
- chunkSize = std::min(_chunkSize, n);
- if(chunkSize==1)
- std::copy(data, data+n, fittedData);
- else
- {
- const size_t
- leftEdge = chunkSize/2,
- rightEdge = n - chunkSize + chunkSize/2;
-
- double a, b;
- PieceWisePhaseFitter::fitSlope(&data[0], &nu[0], chunkSize, a, b);
- for(size_t ch=0; ch!=leftEdge; ++ch)
- fittedData[ch] = a + b * nu[ch];
-
- for(size_t ch=0; ch!=n - chunkSize; ++ch)
- {
- PieceWisePhaseFitter::fitSlope(&data[ch], &nu[ch], chunkSize, a, b);
- fittedData[ch + chunkSize/2] = a + b * nu[ch + chunkSize/2];
- }
-
- PieceWisePhaseFitter::fitSlope(&data[n - chunkSize], &nu[n - chunkSize], chunkSize, a, b);
- for(size_t ch=rightEdge; ch!=n; ++ch)
- fittedData[ch] = a + b * nu[ch];
- }
- }
-
- /**
- * Performs a sliding fit of linear phase gradients (with weighting).
- *
- * A window of size @c chunkSize is slid over the data, and a line is fitted at each position. The values near the left
- * and right side of the border are evaluated with the fit to the most left/right window. This is slower than
- * @ref PieceWiseFit(), but it produces a smoother & more accurate fit.
- * The fit is not necessarily entirely smooth, because the line fit tries to guess the right wrapping of the samples.
- * Since that is not a linear process, this can still cause slight jumps.
- *
- * @param nu Frequencies of the channels (in Hz).
- * @param data The values, same size as @c nu.
- * @param weights Inverse-variance weights.
- * @param fittedData A vector of at least size @c nu.size(), in which the fitted lines are stored.
- */
- void SlidingFit(const double* nu, const double* data, const double* weights, double* fittedData, size_t n)
- {
- const size_t
- chunkSize = std::min(_chunkSize, n);
- if(chunkSize==1)
- std::copy(data, data+n, fittedData);
- else
- {
- const size_t
- leftEdge = chunkSize/2,
- rightEdge = n - chunkSize + chunkSize/2;
-
- double a, b;
- PieceWisePhaseFitter::fitSlope(&data[0], &nu[0], &weights[0], chunkSize, a, b);
- for(size_t ch=0; ch!=leftEdge; ++ch)
- fittedData[ch] = a + b * nu[ch];
-
- for(size_t ch=0; ch!=n - chunkSize; ++ch)
- {
- PieceWisePhaseFitter::fitSlope(&data[ch], &nu[ch], &weights[ch], chunkSize, a, b);
- fittedData[ch + chunkSize/2] = a + b * nu[ch + chunkSize/2];
- }
-
- PieceWisePhaseFitter::fitSlope(&data[n - chunkSize], &nu[n - chunkSize], &weights[n - chunkSize], chunkSize, a, b);
- for(size_t ch=rightEdge; ch!=n; ++ch)
- fittedData[ch] = a + b * nu[ch];
- }
- }
-
- size_t SlidingFitWithBreak(const double* nu, const double* data, const double* weights, double* fittedData, size_t n)
- {
- SlidingFit(nu, data, weights, fittedData, n);
- size_t bp = BreakPoint(data, weights, fittedData, n);
- SlidingFit(nu, data, weights, fittedData, bp);
- SlidingFit(nu+bp, data+bp, weights+bp, fittedData+bp, n-bp);
- return bp;
- }
-
- /**
- * Calculates a weighted median. The weighted median is defined as the
- * sample x for which the sum of all values smaller than x is less than half of the
- * total weight, and idem for all values larger than x.
- * For example, given the sequence
- * 1,2,3,4,5 with weights 1,100,1,1,1; the weighted median is '2'.
- * (sum of weight=104, values < 2 a have weight of 1, values > 2 have a weight of 3).
- * This function is internally used by the piece-wise phase fitter.
- * @param values A vector of (value, weight) pairs.
- * @returns The weighted median.
- */
- static double WeightedMedian(std::vector<std::pair<double, double>>& values)
- {
- std::sort(values.begin(), values.end());
- // calculate total weight
- double sum = 0.0;
- for(std::pair<double,double>& v : values)
- sum += v.second;
-
- int index = 0;
- // prefixSum is the weight sum of everything after `index`
- double prefixSum = sum - values[0].second;
- while(prefixSum > sum/2)
- {
- ++index;
- prefixSum -= values[index].second;
- }
- return values[index].first;
- }
-
- size_t BreakPoint(const double* data, const double* weights, const double* fittedData, size_t n)
- {
- double largest = 0.0;
- size_t index = 0;
- for(size_t i=0; i!=n-_chunkSize; ++i)
- {
- double curSum = 0.0;
- for(size_t j=0; j!=_chunkSize; ++j)
- {
- double dist = data[i+j] - fittedData[i+j];
- curSum += fabs(dist) * weights[i+j];
- }
- if(curSum > largest)
- {
- largest = curSum;
- index = i + _chunkSize/2;
- }
- }
- return index;
- }
-
-private:
- size_t _chunkSize;
- /**
- * This is used to avoid having to reallocate data every fit
- */
- std::vector<double> _tempArray;
- std::vector<std::pair<double,double>> _tempWeightedArray;
-
- /**
- * Fit a + bx
- */
- void fitSlope(const double* data, const double* frequencies, size_t n, double& a, double& b)
- {
- double
- sum_x = 0.0,
- sum_xSq = 0.0,
- mean_y = 0.0,
- ss_xy = 0.0;
- std::vector<double>& d = _tempArray;
- d.assign(data, data+n);
- size_t qCounts[4] = {0,0,0,0};
- for(double& v : d)
- {
- if(v > M_PI*1.5)
- ++qCounts[3];
- else if(v > M_PI*1.0)
- ++qCounts[2];
- else if(v > M_PI*0.5)
- ++qCounts[1];
- else
- ++qCounts[0];
- }
- size_t maxQ = 0;
- for(size_t i=1; i!=4; ++i)
- {
- if(qCounts[i] > qCounts[maxQ])
- maxQ = i;
- }
- double wrapTo = (maxQ*0.5+0.25)*M_PI;
- for(double& v : d)
- {
- if(v - wrapTo > M_PI)
- v -= 2.0*M_PI;
- else if(v - wrapTo < -M_PI)
- v += 2.0*M_PI;
- }
- std::nth_element(d.data(), d.data()+n/2, d.data()+n);
- wrapTo = d[n/2];
- for(size_t i=0; i!=n; ++i)
- {
- double x = data[i];
- if(x - wrapTo > M_PI)
- x -= 2.0*M_PI;
- else if(x - wrapTo < -M_PI)
- x += 2.0*M_PI;
- sum_x += frequencies[i];
- sum_xSq += frequencies[i] * frequencies[i];
- mean_y += x;
- ss_xy += frequencies[i] * x;
- }
- mean_y /= n;
- double mean_x = sum_x / n;
- double ss_xx = sum_xSq - mean_x * sum_x;
- ss_xy -= sum_x * mean_y;
- b = ss_xy / ss_xx;
- a = mean_y - b * mean_x;
- }
-
- template<typename Iter1, typename Iter2>
- double weightedMedian(Iter1 valBegin, Iter1 valEnd, Iter2 weightBegin)
- {
- _tempWeightedArray.resize(std::distance(valBegin, valEnd));
- auto wIter = _tempWeightedArray.begin();
- while(valBegin != valEnd)
- {
- *wIter = std::make_pair(*valBegin, *weightBegin);
- ++wIter;
- ++valBegin;
- ++weightBegin;
- }
- return WeightedMedian(_tempWeightedArray);
- }
-
- /**
- * Fit a + bx with weights
- */
- void fitSlope(const double* data, const double* frequencies, const double* weights, size_t n, double& a, double& b)
- {
- size_t qCounts[4] = {0,0,0,0};
- for(size_t i=0; i!=n; ++i)
- {
- if(data[i] > M_PI*1.5)
- qCounts[3] += weights[i];
- else if(data[i] > M_PI*1.0)
- qCounts[2] += weights[i];
- else if(data[i] > M_PI*0.5)
- qCounts[1] += weights[i];
- else
- qCounts[0] += weights[i];
- }
- size_t maxQ = 0;
- for(size_t i=1; i!=4; ++i)
- {
- if(qCounts[i] > qCounts[maxQ])
- maxQ = i;
- }
- double wrapTo = (maxQ*0.5+0.25)*M_PI;
- std::vector<double>& d = _tempArray;
- d.resize(n);
- for(size_t i=n/4; i!=n*3/4; ++i)
- {
- double v = data[i];
- if(v - wrapTo > M_PI)
- v -= 2.0*M_PI;
- else if(v - wrapTo < -M_PI)
- v += 2.0*M_PI;
- d[i] = v;
- }
- wrapTo = weightedMedian(d.data()+n/4, d.data()+n*3/4, weights+n/4);
-
- double
- sum_x = 0.0,
- sum_xSq = 0.0,
- mean_y = 0.0,
- ss_xy = 0.0,
- sum_weight = 0.0;
-
- for(size_t i=0; i!=n; ++i)
- {
- double x = data[i];
- if(x - wrapTo > M_PI)
- x -= 2.0*M_PI;
- else if(x - wrapTo < -M_PI)
- x += 2.0*M_PI;
- sum_x += frequencies[i] * weights[i];
- sum_xSq += frequencies[i] * frequencies[i] * weights[i];
- mean_y += x * weights[i];
- ss_xy += frequencies[i] * x * weights[i];
- sum_weight += weights[i];
- }
- if(sum_weight == 0.0)
- {
- a = 0.0;
- b = 0.0;
- }
- else {
- mean_y /= sum_weight;
- double mean_x = sum_x / sum_weight;
- double ss_xx = sum_xSq - mean_x * sum_x;
- ss_xy -= sum_x * mean_y;
- b = ss_xy / ss_xx;
- a = mean_y - b * mean_x;
- }
- }
-
- void fitChunk(size_t chunkSize, size_t pos, const std::vector<double>& nu, const std::vector<double>& data, std::vector<double>& fittedData)
- {
- double a, b;
- PieceWisePhaseFitter::fitSlope(&data[pos], &nu[pos], chunkSize, a, b);
- for(size_t ch=0; ch!=chunkSize; ++ch)
- fittedData[ch+pos] = a + b * nu[ch+pos];
- }
-
- void fitChunk(size_t chunkSize, size_t pos, const std::vector<double>& nu, const std::vector<double>& data, const double* weights, std::vector<double>& fittedData)
- {
- double a, b;
- PieceWisePhaseFitter::fitSlope(&data[pos], &nu[pos], &weights[pos], chunkSize, a, b);
- for(size_t ch=0; ch!=chunkSize; ++ch)
- fittedData[ch+pos] = a + b * nu[ch+pos];
- }
-
- /**
- * No longer used -- I tried removing samples that deviated far from the fitted line and refit, but this seemed not to help.
- */
- static bool removeDeviations(std::vector<double>& data, std::vector<double>& frequencies, double a, double b, double phaseThreshold)
- {
- if(data.empty())
- return false;
- double maxDev = 0.0;
- size_t index = 0;
- for(size_t i=0; i!=data.size(); ++i)
- {
- double deviation = std::fabs(data[i] - (a + b * frequencies[i]));
- if(deviation >= maxDev)
- {
- maxDev = deviation;
- index = i;
- }
- }
- if(maxDev > phaseThreshold)
- {
- data.erase(data.begin() + index);
- frequencies.erase(frequencies.begin() + index);
- return true;
- }
- else return false;
- }
-};
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/PiercePoint.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/PiercePoint.h
deleted file mode 100644
index 45b7df4656019dde12950c69753e326140cb163c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/PiercePoint.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#ifndef PIERCEPOINT_H
-#define PIERCEPOINT_H
-
-#include <vector>
-#include <casacore/measures/Measures/MDirection.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/measures/Measures/MCPosition.h>
-#include <casacore/measures/Measures/MPosition.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MEpoch.h>
-
-#include <armadillo>
-
-namespace LOFAR{
-
-class PiercePoint
-{
- static const double IONOheight; //= 300000.; //default height in meter
- static const double EarthRadius;// = 6371000.; //default Earth radius in meter
-public:
- PiercePoint(double height=PiercePoint::IONOheight);
- PiercePoint(const casacore::MPosition &ant,const casacore::MDirection &source,const double height);
- PiercePoint(const casacore::MPosition &ant,const casacore::MDirection &source);
- void init(const casacore::MPosition &ant,const casacore::MDirection &source,const double height);
- void evaluate(casacore::MEpoch time);
- arma::Col<double> getValue() const {return itsValue;}
- casacore::MPosition getPos() const {return itsPosition;}
- casacore::MDirection getDir() const {return itsDirection;}
-private:
- //station position
- casacore::MPosition itsPosition;
- //source position
- casacore::MDirection itsDirection;
- // Ionospheric layer height.
- double itsIonoHeight;
- // square of length antenna vector (int ITRF) minus square of vector to piercepoint. This is constant for a assumed spherical Earth
- double itsC;
- arma::Col<double> itsValue; //PiercePoint in ITRF coordinates
-};
-}
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/QRSolver.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/QRSolver.h
deleted file mode 100644
index 2431eeea8ea4353b7a10f91eb40e4c3101ca4134..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/QRSolver.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#ifndef QR_SOLVER_H
-#define QR_SOLVER_H
-
-#include <complex>
-#include <vector>
-#include <cmath>
-
-typedef std::complex<double> dcomplex;
-
-/* ZGELS prototype */
-extern "C" void zgels_(char* trans, int* m, int* n, int* nrhs, dcomplex* a,
- int* lda, dcomplex* b, int* ldb, dcomplex* work, int* lwork, int* info);
-
-class QRSolver
-{
-public:
- QRSolver(int m, int n, int nrhs) :
- _m(m), _n(n), _nrhs(nrhs)
- { }
-
- /**
- * Find X that minimizes || B - A*X || (i.e. best solution to A*X = B ; A (MxN) * X (NxNRHS) = B (MxNRHS)
- * Inputs are ordered column-major.
- * @param a The M-by-N matrix A
- * @param b On entry: input matrix of size M x NRHS, but with leading dimension max(M, N)
- * On succesful exit: the solution vectors, stored columnwise (N, NRHS)
- */
- bool Solve(dcomplex* a, dcomplex* b)
- {
- int info;
- char trans = 'N'; // No transpose
- int ldb = std::max(_m, _n);
- if(_work.empty())
- {
- int lwork = -1;
- dcomplex wkopt;
- zgels_(&trans, &_m, &_n, &_nrhs, a, &_m, b, &ldb, &wkopt, &lwork, &info );
- _work.resize((int) wkopt.real());
- }
- int lwork = _work.size();
- zgels_(&trans, &_m, &_n, &_nrhs, a, &_m, b, &ldb, _work.data(), &lwork, &info);
- // Check for full rank
- return info == 0;
- }
-private:
- /** Number of rows in matrix A */
- int _m;
-
- /** Number of cols in matrix A */
- int _n;
-
- /** Number of columns in matrices B and X */
- int _nrhs;
-
- std::vector<dcomplex> _work;
-};
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Register.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Register.h
deleted file mode 100644
index 8144c2125466b2470e1469160cd2225d005ae92e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Register.h
+++ /dev/null
@@ -1,37 +0,0 @@
-//# Register.h: Register AOFlag steps in DPPP
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: AORFlagger.h 26900 2013-10-08 20:12:58Z loose $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_DDECAL_REGISTER_H
-#define DPPP_DDECAL_REGISTER_H
-
-// @file
-// @brief Register AOFlag steps in DPPP
-
-
-// Define the function (without name mangling) to register the 'constructor'.
-extern "C"
-{
- void register_ddecal();
-}
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/RotationAndDiagonalConstraint.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/RotationAndDiagonalConstraint.h
deleted file mode 100644
index cd4934fdf09ed61b077a5a57791987a5e1da12d2..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/RotationAndDiagonalConstraint.h
+++ /dev/null
@@ -1,37 +0,0 @@
-#ifndef ROTATIONANDDIAGONAL_CONSTRAINT_H
-#define ROTATIONANDDIAGONAL_CONSTRAINT_H
-
-#ifdef AOPROJECT
-#include "Constraint.h"
-#else
-#include <DPPP_DDECal/Constraint.h>
-#endif
-
-#include <vector>
-#include <ostream>
-
-namespace LOFAR {
-
-class RotationAndDiagonalConstraint : public Constraint
-{
-public:
- RotationAndDiagonalConstraint() {};
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time, std::ostream* statStream);
-
- virtual void InitializeDimensions(size_t nAntennas,
- size_t nDirections,
- size_t nChannelBlocks);
-
- virtual void SetWeights(const std::vector<double>& weights);
-
-private:
- std::vector<Constraint::Result> _res;
-};
-
-} // namespace LOFAR
-
-#endif
-
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/RotationConstraint.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/RotationConstraint.h
deleted file mode 100644
index c097b31b8e7e184985da279e4f192184ed5aea9b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/RotationConstraint.h
+++ /dev/null
@@ -1,40 +0,0 @@
-#ifndef ROTATION_CONSTRAINT_H
-#define ROTATION_CONSTRAINT_H
-
-#ifdef AOPROJECT
-#include "Constraint.h"
-#else
-#include <DPPP_DDECal/Constraint.h>
-#endif
-
-#include <vector>
-#include <ostream>
-
-namespace LOFAR {
-
-class RotationConstraint : public Constraint
-{
-public:
- RotationConstraint() {};
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time, std::ostream* statStream);
-
- virtual void InitializeDimensions(size_t nAntennas,
- size_t nDirections,
- size_t nChannelBlocks);
-
- virtual void SetWeights(const std::vector<double>& weights);
-
- /* Compute the rotation from a 2x2 full jones solution */
- static double get_rotation(std::complex<double>* data);
-
-private:
- std::vector<Constraint::Result> _res;
-};
-
-} // namespace LOFAR
-
-#endif
-
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/ScreenConstraint.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/ScreenConstraint.h
deleted file mode 100644
index bc4250fc4919c8662b8fe712da37ab3b514fc1bc..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/ScreenConstraint.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#ifndef SCREEN_CONSTRAINT_H
-#define SCREEN_CONSTRAINT_H
-#include <DPPP/PhaseFitter.h>
-
-#include <DPPP_DDECal/MultiDirSolver.h>
-#include <DPPP_DDECal/PiercePoint.h>
-#include <DPPP_DDECal/KLFitter.h>
-#include <Common/ParameterSet.h>
-
-#include <cmath>
-#include <vector>
-#include <memory>
-#include <ostream>
-
-namespace LOFAR {
-class ParameterSet;
-class ScreenConstraint : public Constraint
-{
- static const double phtoTEC;//=1./8.4479745e9;
- static const double TECtoph;//=8.4479745e9;
- static const size_t maxIter;//number of iterations to store in debug mode
-
-public:
- ScreenConstraint(const ParameterSet& parset,
- const string& prefix);
-
- /** Initialize metadata with frequencies, resize some members.
- * Should be called after InitializeDimensions.
- */
- void initialize(const double* frequencies);
- virtual std::vector<Constraint::Result> Apply(std::vector<std::vector<MultiDirSolver::DComplex> >& solutions,
- double time, std::ostream* statStream);
- virtual void CalculatePiercepoints();
-
- void setAntennaPositions(const std::vector<std::vector<double> > antenna_pos);
- void setDirections(const std::vector<std::pair<double, double> > source_pos);
- void setTime(double time);
- void initPiercePoints();
- void setCoreAntennas(const std::vector<size_t>& coreAntennas)
- {
- _coreAntennas = coreAntennas;
- if (itsMode == "csfull")
- _screenFitters.resize(_nAntennas-_coreAntennas.size()+1);
- }
- void setOtherAntennas(const std::vector<size_t>& otherAntennas)
- {
- _otherAntennas = otherAntennas;
- }
- void getPPValue(std::vector<std::vector<std::complex<double> > >&, size_t, size_t, double&,double&) const;
-private:
- std::vector<std::vector<double> > itsAntennaPos;
- std::vector<std::vector<double> > itsSourcePos;
- std::vector<double> itsFrequencies;
- std::vector<double> itsprevsol;
- std::vector<double> _iterphases;
- // antenna positions
- // source positions
- // measures instance ofzo
- std::vector<std::vector<PiercePoint> > itsPiercePoints; //temporary hold calculated piercepoints per antenna
- std::vector<KLFitter> _screenFitters;
- std::vector<size_t> _coreAntennas;
- std::vector<size_t> _otherAntennas; //has to be a vector for openmp looping
- double itsCurrentTime;
- double itsBeta;
- double itsHeight;
- double itsOrder;
- double itsRdiff;
- string itsMode;
- string itsAVGMode;
- int itsDebugMode;
- size_t itsIter;
-
-};
-}
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/SmoothnessConstraint.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/SmoothnessConstraint.h
deleted file mode 100644
index 270f5eadd8d1127582fa4757f43648833240d0e4..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/SmoothnessConstraint.h
+++ /dev/null
@@ -1,46 +0,0 @@
-#include "Constraint.h"
-#include "KernelSmoother.h"
-
-#ifndef SMOOTHNESS_CONSTRAINT_H
-#define SMOOTHNESS_CONSTRAINT_H
-
-class SmoothnessConstraint : public Constraint
-{
-public:
- typedef std::complex<double> dcomplex;
- typedef KernelSmoother<dcomplex, double> Smoother;
-
- SmoothnessConstraint(double bandwidthHz);
-
- std::vector<Constraint::Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions, double, std::ostream* statStream) final override;
-
- void SetWeights(const std::vector<double> &weights) final override {
- _weights = weights;
- }
-
- void Initialize(const double* frequencies);
-
- virtual void InitializeDimensions(size_t nAntennas,
- size_t nDirections,
- size_t nChannelBlocks) final override;
-
- struct FitData
- {
- FitData(const double* frequencies, size_t n, Smoother::KernelType kernelType, double kernelBandwidth)
- : smoother(frequencies, n, kernelType, kernelBandwidth),
- data(n), weight(n)
- { }
-
- Smoother smoother;
- std::vector<dcomplex> data;
- std::vector<double> weight;
- };
- std::vector<FitData> _fitData;
- std::vector<double> _frequencies, _weights;
- Smoother::KernelType _kernelType;
- double _bandwidth;
-};
-
-#endif
-
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Stopwatch.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Stopwatch.h
deleted file mode 100644
index b93df8b1f0e480a5e2f0675c6e1025efe0643693..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/Stopwatch.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#ifndef STOPWATCH_H
-#define STOPWATCH_H
-
-#include <boost/date_time/posix_time/posix_time_types.hpp>
-
-#include <string>
-
-#ifndef DOXYGEN_SHOULD_SKIP_THIS
-
-/**
- @author A.R. Offringa <offringa@astro.rug.nl>
-*/
-class Stopwatch{
- public:
- Stopwatch();
- explicit Stopwatch(bool start);
- ~Stopwatch();
- void Start();
- void Pause();
- void Reset();
-
- std::string ToString() const;
- std::string ToShortString() const;
- long double Seconds() const;
-
- std::string ToDaysString() const;
- std::string ToHoursString() const;
- std::string ToMinutesString() const;
- std::string ToSecondsString() const;
- std::string ToMilliSecondsString() const;
- std::string ToMicroSecondsString() const;
- std::string ToNanoSecondsString() const;
- private:
- bool _running;
- boost::posix_time::ptime _startTime;
- boost::posix_time::time_duration _sum;
-};
-
-// end of Doxygen skip
-#endif
-
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/TECConstraint.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/TECConstraint.h
deleted file mode 100644
index bee6e9cb745de6d78668282c8c9e77bc8b4dc2d9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/TECConstraint.h
+++ /dev/null
@@ -1,104 +0,0 @@
-#ifndef TEC_CONSTRAINT_H
-#define TEC_CONSTRAINT_H
-
-#include <vector>
-
-#ifdef AOPROJECT
-#include "Constraint.h"
-#include "PhaseFitter.h"
-#include "PieceWisePhaseFitter.h"
-#else
-#include <DPPP_DDECal/Constraint.h>
-#include <DPPP_DDECal/PieceWisePhaseFitter.h>
-#include <DPPP/PhaseFitter.h>
-#endif
-
-#include <vector>
-#include <ostream>
-
-class TECConstraintBase : public Constraint
-{
-public:
- enum Mode {
- /** Solve for both a (differential) TEC and an XX/YY-common scalar */
- TECAndCommonScalarMode,
- /** Solve only for a (differential) TEC value */
- TECOnlyMode
- };
-
- TECConstraintBase(Mode mode);
-
- /** Initialize metadata with frequencies, resize some members.
- * Should be called after InitializeDimensions.
- */
- void initialize(const double* frequencies);
-
- /** Propagate weights to the phase fitters */
- virtual void SetWeights(const std::vector<double>& weights) final override;
-
-protected:
- virtual void initializeChild() { }
-
- void applyReferenceAntenna(std::vector<std::vector<dcomplex> >& solutions) const;
-
- Mode _mode;
- std::vector<PhaseFitter> _phaseFitters;
- std::vector<double> _weights;
-};
-
-class TECConstraint : public TECConstraintBase
-{
-public:
- TECConstraint(Mode mode) : TECConstraintBase(mode) { }
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time,
- std::ostream* statStream) override;
-};
-
-class ApproximateTECConstraint : public TECConstraint
-{
-public:
- ApproximateTECConstraint(Mode mode) :
- TECConstraint(mode),
- _finishedApproximateStage(false),
- _fittingChunkSize(0),
- _maxApproxIters(50)
- { }
-
- virtual void PrepareIteration(bool hasReachedPrecision, size_t iteration, bool finalIter) final override {
- _finishedApproximateStage = hasReachedPrecision || finalIter || iteration >= _maxApproxIters;
- for(size_t thread=0; thread!=_phaseFitters.size(); ++thread) {
- std::fill(_phaseFitters[thread].WeightData(),
- _phaseFitters[thread].WeightData()+_phaseFitters[thread].Size(),
- 1.0);
- }
- }
-
- virtual bool Satisfied() const final override { return _finishedApproximateStage; }
-
- virtual std::vector<Result> Apply(
- std::vector<std::vector<dcomplex> >& solutions,
- double time,
- std::ostream* statStream) final override;
-
- void SetFittingChunkSize(size_t fittingChunkSize)
- { _fittingChunkSize = fittingChunkSize; }
-
- void SetMaxApproximatingIterations(size_t maxApproxIters)
- { _maxApproxIters = maxApproxIters; }
-protected:
- virtual void initializeChild() final override;
-
-private:
- bool _finishedApproximateStage;
- std::vector<PieceWisePhaseFitter> _pwFitters;
- std::vector<std::vector<double> > _threadData;
- std::vector<std::vector<double> > _threadFittedData;
- std::vector<std::vector<double> > _threadWeights;
- size_t _fittingChunkSize, _maxApproxIters;
-};
-
-#endif
-
diff --git a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/screenfitter.h b/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/screenfitter.h
deleted file mode 100644
index c75b2d7fb0738e6385b5b6f90c391ce732c02577..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/include/DPPP_DDECal/screenfitter.h
+++ /dev/null
@@ -1,44 +0,0 @@
-//# screenfitter.h: Class to perform screen fitting
-//# Copyright (C) 2016
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//#
-//# @author Maaijke Mevius
-
-/**
- * @file screenfitter.h Implements TEC model screen filter @ref ScreenFitter.
- * @author Maaijke Mevius
- * @date 2017-02-01
- */
-
-#ifndef SCREEN_FITTER_H
-#define SCREEN_FITTER_H
-#include <armadillo>
-#include <vector>
-
-class ScreenFitter{
- public:
- ScreenFitter();
- double* PhaseData() { return _phases.data(); }
-
- private:
- std::vector<double> _phases,_frequencies, _weights;
- mat _corrmatrix; //correlation matrix
-
-};
-#endif
diff --git a/CEP/DP3/DPPP_DDECal/src/CMakeLists.txt b/CEP/DP3/DPPP_DDECal/src/CMakeLists.txt
deleted file mode 100644
index 480c5c11c2446020cec86c5437db3bf86bd21b72..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-# $Id: CMakeLists.txt 30439 2014-11-19 15:04:34Z dijkema $
-
-include(LofarPackageVersion)
-
-lofar_add_library(dppp_ddecal
- Package__Version.cc
- DDECal.cc Register.cc Stopwatch.cc
- KLFitter.cc DDECal.cc MultiDirSolver.cc Constraint.cc PiercePoint.cc
- ScreenConstraint.cc SmoothnessConstraint.cc
- TECConstraint.cc RotationConstraint.cc
- RotationAndDiagonalConstraint.cc
-)
-
-lofar_add_bin_program(versiondppp_ddecal versiondppp_ddecal.cc)
diff --git a/CEP/DP3/DPPP_DDECal/src/Constraint.cc b/CEP/DP3/DPPP_DDECal/src/Constraint.cc
deleted file mode 100644
index b60f4068bc2eb51a63c5f5085a65b723af3494aa..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/Constraint.cc
+++ /dev/null
@@ -1,77 +0,0 @@
-#ifdef AOPROJECT
-#include "Constraint.h"
-#else
-#include <DPPP_DDECal/Constraint.h>
-#endif
-
-std::vector<Constraint::Result> PhaseOnlyConstraint::Apply(
- std::vector<std::vector<dcomplex> >& solutions, double,
- std::ostream* statStream)
-{
- for (size_t ch=0; ch<solutions.size(); ++ch) {
- for (size_t solIndex=0; solIndex<solutions[ch].size(); ++solIndex) {
- solutions[ch][solIndex] /= std::abs(solutions[ch][solIndex]);
- }
- }
-
- return std::vector<Constraint::Result>();
-}
-
-std::vector<Constraint::Result> AmplitudeOnlyConstraint::Apply(
- std::vector<std::vector<dcomplex> >& solutions, double,
- std::ostream* statStream)
-{
- for (size_t ch=0; ch<solutions.size(); ++ch) {
- for (size_t solIndex=0; solIndex<solutions[ch].size(); ++solIndex) {
- solutions[ch][solIndex] = std::abs(solutions[ch][solIndex]);
- }
- }
-
- return std::vector<Constraint::Result>();
-}
-
-std::vector<Constraint::Result> DiagonalConstraint::Apply(
- std::vector<std::vector<dcomplex> >& solutions, double,
- std::ostream* statStream)
-{
- if(_polsPerSolution == 4)
- {
- for (size_t ch=0; ch<solutions.size(); ++ch) {
- for (size_t solIndex=0; solIndex<solutions[ch].size(); solIndex += 4) {
- solutions[ch][solIndex+1] = 0.0;
- solutions[ch][solIndex+2] = 0.0;
- }
- }
- }
-
- return std::vector<Constraint::Result>();
-}
-
-std::vector<Constraint::Result> CoreConstraint::Apply(
- std::vector<std::vector<dcomplex> >& solutions, double,
- std::ostream* statStream)
-{
- for (uint ch=0; ch<solutions.size(); ++ch) {
- std::vector<dcomplex> coreSolutions(_nDirections, 0.0);
- // Calculate the sum of solutions over the core stations
- for(size_t antennaIndex : _coreAntennas)
- {
- size_t startIndex = antennaIndex * _nDirections;
- for(size_t direction = 0; direction != _nDirections; ++direction)
- coreSolutions[direction] += solutions[ch][startIndex + direction];
- }
-
- // Divide by nr of core stations to get the mean solution
- for(dcomplex& solution : coreSolutions)
- solution /= _coreAntennas.size();
-
- // Assign all core stations to the mean solution
- for(size_t antennaIndex : _coreAntennas)
- {
- size_t startIndex = antennaIndex * _nDirections;
- for(size_t direction = 0; direction != _nDirections; ++direction)
- solutions[ch][startIndex + direction] = coreSolutions[direction];
- }
- }
- return std::vector<Constraint::Result>();
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/DDECal.cc b/CEP/DP3/DPPP_DDECal/src/DDECal.cc
deleted file mode 100644
index a6b1fc64f0d34fddff65ac18248a596b5347188b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/DDECal.cc
+++ /dev/null
@@ -1,980 +0,0 @@
-//# DDECal.cc: DPPP step class to do a direction dependent gain calibration
-//# Copyright (C) 2013
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: DDECal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Tammo Jan Dijkema
-
-#include <lofar_config.h>
-#include <DPPP/Package__Version.h>
-#include <DPPP_DDECal/DDECal.h>
-
-#include <DPPP/ApplyCal.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPLogger.h>
-#include <DPPP/MSReader.h>
-#include <DPPP/Simulate.h>
-#include <DPPP/SourceDBUtil.h>
-
-#include <DPPP_DDECal/ScreenConstraint.h>
-#include <DPPP_DDECal/TECConstraint.h>
-#include <DPPP_DDECal/RotationConstraint.h>
-#include <DPPP_DDECal/RotationAndDiagonalConstraint.h>
-#include <DPPP_DDECal/SmoothnessConstraint.h>
-
-#include <ParmDB/ParmDB.h>
-#include <ParmDB/ParmValue.h>
-#include <ParmDB/SourceDB.h>
-
-#include <Common/LofarLogger.h>
-#include <Common/OpenMP.h>
-#include <Common/ParameterSet.h>
-#include <Common/StreamUtil.h>
-#include <Common/StringUtil.h>
-
-#include <fstream>
-#include <ctime>
-#include <utility>
-
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/MatrixMath.h>
-#include <casacore/measures/Measures/MEpoch.h>
-#include <casacore/measures/Measures/MeasConvert.h>
-#include <casacore/measures/Measures/MCDirection.h>
-#include <casacore/casa/Quanta/Quantum.h>
-#include <casacore/casa/OS/File.h>
-
-#include <vector>
-#include <algorithm>
-
-#include <limits>
-#include <iostream>
-#include <sstream>
-#include <iomanip>
-
-using namespace casacore;
-using namespace LOFAR::BBS;
-
-namespace LOFAR {
- namespace DPPP {
-
- DDECal::DDECal (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsUseModelColumn(parset.getBool (prefix + "usemodelcolumn", false)),
- itsAvgTime (0),
- itsSols (),
- itsH5ParmName (parset.getString (prefix + "h5parm",
- parset.getString("msin")+
- "/instrument.h5")),
- itsH5Parm (itsH5ParmName, true),
- itsPropagateSolutions (parset.getBool (prefix + "propagatesolutions",
- false)),
- itsTimeStep (0),
- itsSolInt (parset.getInt (prefix + "solint", 1)),
- itsStepInSolInt (0),
- itsNChan (parset.getInt (prefix + "nchan", 1)),
- itsUVWFlagStep (input, parset, prefix),
- itsCoreConstraint(parset.getDouble (prefix + "coreconstraint", 0.0)),
- itsSmoothnessConstraint(parset.getDouble (prefix + "smoothnessconstraint", 0.0)),
- itsScreenCoreConstraint(parset.getDouble (prefix + "tecscreen.coreconstraint", 0.0)),
- itsFullMatrixMinimalization(false),
- itsApproximateTEC(false),
- itsStatFilename(parset.getString(prefix + "statfilename", ""))
- {
- stringstream ss;
- ss << parset;
- itsParsetString = ss.str();
-
- itsMultiDirSolver.set_max_iterations(parset.getInt(prefix + "maxiter", 50));
- double tolerance = parset.getDouble(prefix + "tolerance", 1.e-4);
- itsMultiDirSolver.set_accuracy(tolerance);
- itsMultiDirSolver.set_constraint_accuracy(parset.getDouble(prefix + "approxtolerance", tolerance*10.0));
- itsMultiDirSolver.set_step_size(parset.getDouble(prefix + "stepsize", 0.2));
- itsMultiDirSolver.set_detect_stalling(parset.getBool(prefix + "detectstalling", true));
-
- if(!itsStatFilename.empty())
- itsStatStream.reset(new std::ofstream(itsStatFilename));
-
- vector<string> strDirections;
- if (itsUseModelColumn) {
- itsModelData.resize(itsSolInt);
- strDirections.push_back("pointing");
- itsDirections.push_back(vector<string>());
- } else {
- strDirections = parset.getStringVector (prefix + "directions",
- vector<string> ());
- // Default directions are all patches
- if (strDirections.empty()) {
- string sourceDBName = parset.getString(prefix+"sourcedb");
- BBS::SourceDB sourceDB(BBS::ParmDBMeta("", sourceDBName), false);
- vector<string> patchNames = makePatchList(sourceDB, vector<string>());
- itsDirections.resize(patchNames.size());
- for (uint i=0; i<patchNames.size(); ++i) {
- itsDirections[i] = vector<string>(1, patchNames[i]);
- }
- } else {
- itsDirections.resize(strDirections.size());
- for (uint i=0; i<strDirections.size(); ++i) {
- ParameterValue dirStr(strDirections[i]);
- itsDirections[i] = dirStr.getStringVector();
- }
- }
- }
-
- itsMode = GainCal::stringToCalType(
- toLower(parset.getString(prefix + "mode",
- "complexgain")));
- if(itsCoreConstraint != 0.0) {
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new CoreConstraint()));
- }
- if(itsSmoothnessConstraint != 0.0) {
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new SmoothnessConstraint(itsSmoothnessConstraint)));
- }
- switch(itsMode) {
- case GainCal::COMPLEXGAIN:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new DiagonalConstraint(4)));
- itsMultiDirSolver.set_phase_only(false);
- itsFullMatrixMinimalization = true;
- break;
- case GainCal::SCALARCOMPLEXGAIN:
- // no constraints
- itsMultiDirSolver.set_phase_only(false);
- itsFullMatrixMinimalization = false;
- break;
- case GainCal::FULLJONES:
- // no constraints
- itsMultiDirSolver.set_phase_only(false);
- itsFullMatrixMinimalization = true;
- break;
- case GainCal::PHASEONLY:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new PhaseOnlyConstraint()));
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new DiagonalConstraint(4)));
- itsMultiDirSolver.set_phase_only(true);
- itsFullMatrixMinimalization = true;
- break;
- case GainCal::SCALARPHASE:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new PhaseOnlyConstraint()));
- itsMultiDirSolver.set_phase_only(true);
- break;
- case GainCal::AMPLITUDEONLY:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new DiagonalConstraint(4)));
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new AmplitudeOnlyConstraint()));
- itsMultiDirSolver.set_phase_only(false);
- itsFullMatrixMinimalization = true;
- break;
- case GainCal::SCALARAMPLITUDE:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new AmplitudeOnlyConstraint()));
- itsMultiDirSolver.set_phase_only(false);
- itsFullMatrixMinimalization = false;
- break;
- case GainCal::TEC:
- case GainCal::TECANDPHASE:
- itsApproximateTEC = parset.getBool(prefix + "approximatetec", false);
- if(itsApproximateTEC)
- {
- int iters = parset.getInt(prefix + "maxapproxiter", itsMultiDirSolver.max_iterations()/2);
- int chunksize = parset.getInt(prefix + "approxchunksize", 0);
- casacore::CountedPtr<ApproximateTECConstraint> ptr;
- if(itsMode == GainCal::TEC)
- ptr = casacore::CountedPtr<ApproximateTECConstraint>(
- new ApproximateTECConstraint(TECConstraint::TECOnlyMode));
- else
- ptr = casacore::CountedPtr<ApproximateTECConstraint>(
- new ApproximateTECConstraint(TECConstraint::TECAndCommonScalarMode));
- ptr->SetMaxApproximatingIterations(iters);
- ptr->SetFittingChunkSize(chunksize);
- itsConstraints.push_back(ptr);
- }
- else {
- if(itsMode == GainCal::TEC)
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new TECConstraint(TECConstraint::TECOnlyMode)));
- else
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new TECConstraint(TECConstraint::TECAndCommonScalarMode)));
- }
- itsMultiDirSolver.set_phase_only(true);
- itsFullMatrixMinimalization = false;
- break;
- case GainCal::TECSCREEN:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new ScreenConstraint(parset, prefix+"tecscreen.")));
- itsMultiDirSolver.set_phase_only(true);
- itsFullMatrixMinimalization = false;
- break;
- case GainCal::ROTATIONANDDIAGONAL:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new RotationAndDiagonalConstraint()));
- itsFullMatrixMinimalization = true;
- break;
- case GainCal::ROTATION:
- itsConstraints.push_back(casacore::CountedPtr<Constraint>(
- new RotationConstraint()));
- itsFullMatrixMinimalization = true;
- break;
- default:
- THROW (Exception, "Unexpected mode: " <<
- GainCal::calTypeToString(itsMode));
- }
-
- const size_t nDir = itsDirections.size();
- if (itsUseModelColumn) {
- ASSERT(nDir == 1);
- } else {
- itsPredictSteps.resize(nDir);
- for (size_t dir=0; dir<nDir; ++dir) {
- itsPredictSteps[dir]=Predict(input, parset, prefix, itsDirections[dir]);
- }
- }
- }
-
- DDECal::~DDECal()
- {}
-
- DPStep::ShPtr DDECal::makeStep (DPInput* input,
- const ParameterSet& parset,
- const std::string& prefix)
- {
- return DPStep::ShPtr(new DDECal(input, parset, prefix));
- }
-
- void DDECal::updateInfo (const DPInfo& infoIn)
- {
- info() = infoIn;
- info().setNeedVisData();
-
- const size_t nDir = itsDirections.size();
-
- itsUVWFlagStep.updateInfo(infoIn);
- for (size_t dir=0; dir<itsPredictSteps.size(); ++dir) {
- itsPredictSteps[dir].updateInfo(infoIn);
- }
-
- if (itsSolInt==0) {
- itsSolInt=info().ntime();
- }
-
- itsDataPtrs.resize(itsSolInt);
- itsModelDataPtrs.resize(itsSolInt);
- for (uint t=0; t<itsSolInt; ++t) {
- itsModelDataPtrs[t].resize(nDir);
- }
- for (uint i=0;i<itsConstraints.size();i++) {
- itsMultiDirSolver.add_constraint(itsConstraints[i].get());
- }
-
-
- itsBufs.resize(itsSolInt);
-
- itsDataResultStep = ResultStep::ShPtr(new ResultStep());
- itsUVWFlagStep.setNextStep(itsDataResultStep);
-
- itsResultSteps.resize(itsPredictSteps.size());
- for (size_t dir=0; dir<itsPredictSteps.size(); ++dir) {
- itsResultSteps[dir] = MultiResultStep::ShPtr(new MultiResultStep(itsSolInt));
- itsPredictSteps[dir].setNextStep(itsResultSteps[dir]);
- }
-
- if (itsNChan==0 || itsNChan>info().nchan()) {
- itsNChan = info().nchan();
- }
-
- // Convert from casacore::Vector to std::vector
- vector<int> ant1(info().getAnt1().size());
- vector<int> ant2(info().getAnt2().size());
- for (uint i=0; i<ant1.size(); ++i) {
- ant1[i]=info().getAnt1()[i];
- ant2[i]=info().getAnt2()[i];
- }
-
- // Fill antenna info in H5Parm, need to convert from casa types to std types
- std::vector<std::string> antennaNames(info().antennaNames().size());
- std::vector<std::vector<double> > antennaPos(info().antennaPos().size());
- for (uint i=0; i<info().antennaNames().size(); ++i) {
- antennaNames[i]=info().antennaNames()[i];
- casacore::Quantum<casacore::Vector<double> > pos = info().antennaPos()[i].get("m");
- antennaPos[i].resize(3);
- antennaPos[i][0] = pos.getValue()[0];
- antennaPos[i][1] = pos.getValue()[1];
- antennaPos[i][2] = pos.getValue()[2];
- }
-
- itsH5Parm.addAntennas(antennaNames, antennaPos);
-
- std::vector<std::pair<double, double> > sourcePositions(itsDirections.size());
- if (itsUseModelColumn) {
- MDirection dirJ2000(MDirection::Convert(infoIn.phaseCenter(),
- MDirection::J2000)());
- Quantum<Vector<Double> > angles = dirJ2000.getAngle();
- sourcePositions[0] = std::pair<double, double> (
- angles.getBaseValue()[0],
- angles.getBaseValue()[1]);
-
- } else {
- for (uint i=0; i<itsDirections.size(); ++i) {
- sourcePositions[i] = itsPredictSteps[i].getFirstDirection();
- }
- }
- itsH5Parm.addSources(getDirectionNames(), sourcePositions);
-
- uint nSolTimes = (info().ntime()+itsSolInt-1)/itsSolInt;
- itsSols.resize(nSolTimes);
- itsNIter.resize(nSolTimes);
- itsNApproxIter.resize(nSolTimes);
- itsConstraintSols.resize(nSolTimes);
-
- size_t nChannelBlocks = info().nchan()/itsNChan;
- itsChanBlockStart.resize(nChannelBlocks+1);
- itsChanBlockFreqs.resize(nChannelBlocks);
- for(size_t chBlock=0; chBlock!=nChannelBlocks; ++chBlock) {
- const size_t
- channelIndexStart = chBlock * info().nchan() / nChannelBlocks,
- channelIndexEnd = (chBlock+1) * info().nchan() / nChannelBlocks,
- curChannelBlockSize = channelIndexEnd - channelIndexStart;
- double meanfreq = std::accumulate(
- info().chanFreqs().data()+channelIndexStart,
- info().chanFreqs().data()+channelIndexEnd,
- 0.0) / curChannelBlockSize;
- itsChanBlockStart[chBlock] = channelIndexStart;
- itsChanBlockFreqs[chBlock] = meanfreq;
- }
- itsChanBlockStart[itsChanBlockStart.size()-1] = info().nchan();
-
- itsWeights.assign(itsChanBlockFreqs.size()*info().nantenna(), 0.0);
-
- for (uint i=0; i<itsConstraints.size();++i) {
- // Initialize the constraint with some common metadata
- itsConstraints[i]->InitializeDimensions(info().antennaNames().size(),
- itsDirections.size(),
- nChannelBlocks);
-
- // Different constraints need different information. Determine if the constraint is
- // of a type that needs more information, and if so initialize the constraint.
- CoreConstraint* coreConstraint = dynamic_cast<CoreConstraint*>(itsConstraints[i].get());
- if(coreConstraint != 0)
- {
- // Take antenna with index 0 as reference station
- double
- refX = antennaPos[0][0],
- refY = antennaPos[0][1],
- refZ = antennaPos[0][2];
- std::set<size_t> coreAntennaIndices;
- const double coreDistSq = itsCoreConstraint*itsCoreConstraint;
- for(size_t ant=0; ant!=antennaPos.size(); ++ant)
- {
- double
- dx = refX - antennaPos[ant][0],
- dy = refY - antennaPos[ant][1],
- dz = refZ - antennaPos[ant][2],
- distSq = dx*dx + dy*dy + dz*dz;
- if(distSq <= coreDistSq)
- coreAntennaIndices.insert(ant);
- }
- coreConstraint->initialize(coreAntennaIndices);
- }
-
- ScreenConstraint* screenConstraint = dynamic_cast<ScreenConstraint*>(itsConstraints[i].get());
- if(screenConstraint != 0)
- {
- screenConstraint->initialize(&(itsChanBlockFreqs[0]));
- screenConstraint->setAntennaPositions(antennaPos);
- screenConstraint->setDirections(sourcePositions);
- screenConstraint->initPiercePoints();
- double
- refX = antennaPos[i][0],
- refY = antennaPos[i][1],
- refZ = antennaPos[i][2];
- std::vector<size_t> coreAntennaIndices;
- std::vector<size_t> otherAntennaIndices;
- const double coreDistSq = itsScreenCoreConstraint*itsScreenCoreConstraint;
- for(size_t ant=0; ant!=antennaPos.size(); ++ant)
- {
- double
- dx = refX - antennaPos[ant][0],
- dy = refY - antennaPos[ant][1],
- dz = refZ - antennaPos[ant][2],
- distSq = dx*dx + dy*dy + dz*dz;
- if(distSq <= coreDistSq)
- coreAntennaIndices.push_back(ant);
- else
- otherAntennaIndices.push_back(ant);
- }
- screenConstraint->setCoreAntennas(coreAntennaIndices);
- screenConstraint->setOtherAntennas(otherAntennaIndices);
- }
-
- TECConstraintBase* tecConstraint = dynamic_cast<TECConstraintBase*>(itsConstraints[i].get());
- if(tecConstraint != nullptr)
- {
- tecConstraint->initialize(&itsChanBlockFreqs[0]);
- }
- SmoothnessConstraint* sConstraint = dynamic_cast<SmoothnessConstraint*>(itsConstraints[i].get());
- if(sConstraint != nullptr)
- {
- sConstraint->Initialize(&itsChanBlockFreqs[0]);
- }
- }
-
- uint nSt = info().antennaNames().size();
- itsMultiDirSolver.init(nSt, nDir, info().nchan(), ant1, ant2);
- itsMultiDirSolver.set_channel_blocks(nChannelBlocks);
-
- for (uint i=0; i<nSolTimes; ++i) {
- itsSols[i].resize(nChannelBlocks);
- }
- }
-
- void DDECal::show (std::ostream& os) const
- {
- os
- << "DDECal " << itsName << '\n'
- << " H5Parm: " << itsH5ParmName << '\n'
- << " solint: " << itsSolInt << '\n'
- << " nchan: " << itsNChan << '\n'
- << " directions: " << itsDirections << '\n'
- << " use model column: " << boolalpha << itsUseModelColumn << '\n'
- << " tolerance: " << itsMultiDirSolver.get_accuracy() << '\n'
- << " max iter: " << itsMultiDirSolver.max_iterations() << '\n'
- << " detect stalling: " << std::boolalpha << itsMultiDirSolver.get_detect_stalling() << '\n'
- << " step size: " << itsMultiDirSolver.get_step_size() << '\n'
- << " mode (constraints): " << GainCal::calTypeToString(itsMode) << '\n'
- << " coreconstraint: " << itsCoreConstraint << '\n'
- << " smoothnessconstraint:" << itsSmoothnessConstraint << '\n'
- << " approximate fitter: " << itsApproximateTEC << '\n';
- for (uint i=0; i<itsPredictSteps.size(); ++i) {
- itsPredictSteps[i].show(os);
- }
- itsUVWFlagStep.show(os);
- }
-
- void DDECal::showTimings (std::ostream& os, double duration) const
- {
- double totaltime=itsTimer.getElapsed();
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " DDECal " << itsName << endl;
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerPredict.getElapsed(), totaltime);
- os << " of it spent in predict" << endl;
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerSolve.getElapsed(), totaltime);
- os << " of it spent in estimating gains and computing residuals" << endl;
-
- itsMultiDirSolver.showTimings(os, itsTimerSolve.getElapsed());
-
- os << " ";
- FlagCounter::showPerc1 (os, itsTimerWrite.getElapsed(), totaltime);
- os << " of it spent in writing gain solutions to disk" << endl;
-
- os << "Iterations taken: [";
- for (uint i=0; i<itsNIter.size()-1; ++i) {
- os<<itsNIter[i];
- if(itsNApproxIter[i]!=0)
- os << '|' << itsNApproxIter[i];
- os<<",";
- }
- os<<itsNIter[itsNIter.size()-1];
- if(itsNApproxIter[itsNIter.size()-1]!=0)
- os << '|' << itsNApproxIter[itsNIter.size()-1];
- os<<"]"<<endl;
- }
-
- void DDECal::initializeScalarSolutions() {
- if (itsTimeStep/itsSolInt>0 && itsPropagateSolutions) {
- // initialize solutions with those of the previous step
- itsSols[itsTimeStep/itsSolInt] = itsSols[itsTimeStep/itsSolInt-1];
- } else {
- // initialize solutions with 1.
- size_t n = itsDirections.size()*info().antennaNames().size();
- for (vector<DComplex>& solvec : itsSols[itsTimeStep/itsSolInt]) {
- solvec.assign(n, 1.0);
- }
- }
- }
-
- void DDECal::initializeFullMatrixSolutions() {
- if (itsTimeStep/itsSolInt>0 && itsPropagateSolutions) {
- // initialize solutions with those of the previous step
- itsSols[itsTimeStep/itsSolInt] = itsSols[itsTimeStep/itsSolInt-1];
- } else {
- // initialize solutions with unity matrix [1 0 ; 0 1].
- size_t n = itsDirections.size()*info().antennaNames().size();
- for (vector<DComplex>& solvec : itsSols[itsTimeStep/itsSolInt]) {
- solvec.resize(n*4);
- for(size_t i=0; i!=n; ++i)
- {
- solvec[i*4 + 0] = 1.0;
- solvec[i*4 + 1] = 0.0;
- solvec[i*4 + 2] = 0.0;
- solvec[i*4 + 3] = 1.0;
- }
- }
- }
- }
-
- vector<string> DDECal::getDirectionNames() {
- vector<string> res;
-
- for (vector<vector<string> >::iterator dirIter = itsDirections.begin();
- dirIter != itsDirections.end();
- dirIter++) {
- stringstream ss;
- ss << (*dirIter);
- res.push_back(ss.str());
- }
- return res;
- }
-
- void DDECal::doSolve ()
- {
- if(itsFullMatrixMinimalization)
- initializeFullMatrixSolutions();
- else
- initializeScalarSolutions();
-
- itsTimerSolve.start();
- MultiDirSolver::SolveResult solveResult;
- if(itsFullMatrixMinimalization)
- {
- solveResult = itsMultiDirSolver.processFullMatrix(itsDataPtrs, itsModelDataPtrs,
- itsSols[itsTimeStep/itsSolInt],
- itsAvgTime / itsSolInt, itsStatStream.get());
- }
- else {
- solveResult = itsMultiDirSolver.processScalar(itsDataPtrs, itsModelDataPtrs,
- itsSols[itsTimeStep/itsSolInt],
- itsAvgTime / itsSolInt, itsStatStream.get());
- }
- itsTimerSolve.stop();
-
- itsNIter[itsTimeStep/itsSolInt] = solveResult.iterations;
- itsNApproxIter[itsTimeStep/itsSolInt] = solveResult.constraintIterations;
-
- // Store constraint solutions if any constaint has a non-empty result
- bool someConstraintHasResult = false;
- for (uint constraintnum=0; constraintnum<solveResult._results.size(); ++constraintnum) {
- if (!solveResult._results[constraintnum].empty()) {
- someConstraintHasResult = true;
- break;
- }
- }
- if (someConstraintHasResult) {
- itsConstraintSols[itsTimeStep/itsSolInt]=solveResult._results;
- }
- }
-
- bool DDECal::process (const DPBuffer& bufin)
- {
- itsTimer.start();
-
- itsBufs[itsStepInSolInt].copy(bufin);
- itsDataPtrs[itsStepInSolInt] = itsBufs[itsStepInSolInt].getData().data();
-
- // Fetch inputs because parallel PredictSteps should not read it from disk
- itsInput->fetchUVW(bufin, itsBufs[itsStepInSolInt], itsTimer);
- itsInput->fetchWeights(bufin, itsBufs[itsStepInSolInt], itsTimer);
- itsInput->fetchFullResFlags(bufin, itsBufs[itsStepInSolInt], itsTimer);
-
- // UVW flagging happens on a copy of the buffer, so these flags are not written
- itsUVWFlagStep.process(itsBufs[itsStepInSolInt]);
-
- itsTimerPredict.start();
-
-// if(itsPredictSteps.size() < LOFAR::OpenMP::maxThreads())
-// LOFAR::OpenMP::setNested(true);
- if (itsUseModelColumn) {
- itsInput->getModelData (itsBufs[itsStepInSolInt].getRowNrs(),
- itsModelData[itsStepInSolInt]);
- itsModelDataPtrs[itsStepInSolInt][0] = itsModelData[itsStepInSolInt].data();
- } else {
-#pragma omp parallel for schedule(dynamic) if(itsPredictSteps.size()>1)
- for (size_t dir=0; dir<itsPredictSteps.size(); ++dir) {
- itsPredictSteps[dir].process(itsBufs[itsStepInSolInt]);
- itsModelDataPtrs[itsStepInSolInt][dir] =
- itsResultSteps[dir]->get()[itsStepInSolInt].getData().data();
- }
- }
-
- // Handle weights and flags
- const size_t nBl = info().nbaselines();
- const size_t nCh = info().nchan();
- const size_t nCr = 4;
-
- size_t nchanblocks = itsChanBlockFreqs.size();
- size_t chanblock = 0;
-
- double weightFactor = 1./(nCh*(info().nantenna()-1)*nCr*itsSolInt);
-
- for (size_t ch=0; ch<nCh; ++ch) {
- if (ch == itsChanBlockStart[chanblock+1]) {
- chanblock++;
- }
- for (size_t bl=0; bl<nBl; ++bl) {
- for (size_t cr=0; cr<nCr; ++cr) {
- if (itsBufs[itsStepInSolInt].getFlags().data()[bl*nCr*nCh+ch*nCr+cr]) {
- // Flagged points: set data and model to 0
- itsDataPtrs[itsStepInSolInt][bl*nCr*nCh+ch*nCr+cr] = 0;
- for (size_t dir=0; dir<itsModelDataPtrs[0].size(); ++dir) {
- itsModelDataPtrs[itsStepInSolInt][dir][bl*nCr*nCh+ch*nCr+cr] = 0;
- }
- } else {
- // Premultiply non-flagged data with sqrt(weight)
- double weight = itsBufs[itsStepInSolInt].getWeights().data()[bl*nCr*nCh+ch*nCr+cr];
- itsDataPtrs[itsStepInSolInt][bl*nCr*nCh+ch*nCr+cr] *= sqrt(weight);
- itsWeights[info().getAnt1()[bl]*nchanblocks + chanblock] += weight;
- itsWeights[info().getAnt2()[bl]*nchanblocks + chanblock] += weight;
- for (size_t dir=0; dir<itsModelDataPtrs[0].size(); ++dir) {
- itsModelDataPtrs[itsStepInSolInt][dir][bl*nCr*nCh+ch*nCr+cr] *= sqrt(weight);
- }
- }
- }
- }
- }
-
- for (auto& weight: itsWeights) {
- weight *= weightFactor;
- }
-
- itsTimerPredict.stop();
-
- itsAvgTime += itsAvgTime + bufin.getTime();
-
- if (itsStepInSolInt==itsSolInt-1) {
- for (uint constraint_num = 0; constraint_num < itsConstraints.size(); ++constraint_num) {
- itsConstraints[constraint_num]->SetWeights(itsWeights);
- }
-
- doSolve();
-
- // Clean up, prepare for next iteration
- itsStepInSolInt=0;
- itsAvgTime=0;
- itsWeights.assign(itsWeights.size(), 0.);
- for (size_t dir=0; dir<itsResultSteps.size(); ++dir) {
- itsResultSteps[dir]->clear();
- }
- } else {
- itsStepInSolInt++;
- }
-
- itsTimeStep++;
- itsTimer.stop();
-
- getNextStep()->process(bufin);
-
- return false;
- }
-
- void DDECal::writeSolutions() {
- itsTimer.start();
- itsTimerWrite.start();
-
- uint nSolTimes = (info().ntime()+itsSolInt-1)/itsSolInt;
- uint nDir = itsDirections.size();
- ASSERT(nSolTimes==itsSols.size());
- vector<double> solTimes(nSolTimes);
- double starttime=info().startTime();
- for (uint t=0; t<nSolTimes; ++t) {
- solTimes[t] = starttime+(t+0.5)*info().timeInterval()*itsSolInt;
- }
-
- if (itsConstraintSols[0].empty()) {
- // Record the actual iterands of the solver, not constraint results
-
- uint nPol;
-
- vector<string> polarizations;
- if(itsMode == GainCal::COMPLEXGAIN ||
- itsMode == GainCal::PHASEONLY ||
- itsMode == GainCal::AMPLITUDEONLY) {
- nPol = 2;
- polarizations.push_back("XX");
- polarizations.push_back("YY");
- } else if (itsMode == GainCal::FULLJONES) {
- polarizations.push_back("XX");
- polarizations.push_back("XY");
- polarizations.push_back("YX");
- polarizations.push_back("YY");
- nPol = 4;
- } else {
- nPol = 1;
- }
-
- uint nSolChan = itsSols[0].size();
- ASSERT(nSolChan == itsChanBlockFreqs.size());
-
- vector<DComplex> sols(nSolChan*info().nantenna()*nSolTimes*nDir*nPol);
- size_t i=0;
-
- // For nPol=1, loop over pol runs just once
- // For nPol=2, it runs over values 0 and 2 (picking diagonal elements from 4 pols)
- // For nPol=4, it runs over 0, 1, 2, 3
- uint polIncr= (itsMode==GainCal::FULLJONES?1:3);
- uint maxPol = (nPol>1?4:1);
- // Put solutions in a contiguous piece of memory
- for (uint time=0; time<nSolTimes; ++time) {
- for (uint chan=0; chan<nSolChan; ++chan) {
- for (uint ant=0; ant<info().nantenna(); ++ant) {
- for (uint dir=0; dir<nDir; ++dir) {
- for (uint pol=0; pol<maxPol; pol+=polIncr) {
- ASSERT(!itsSols[time].empty());
- ASSERT(!itsSols[time][chan].empty());
- ASSERT(time<itsSols.size());
- ASSERT(chan<itsSols[time].size());
- ASSERT(ant*nDir*maxPol+dir*maxPol+pol<itsSols[time][chan].size());
- ASSERT(i<sols.size());
- sols[i] = itsSols[time][chan][ant*nDir*maxPol+dir*maxPol+pol];
- ++i;
- }
- }
- }
- }
- }
- vector<H5Parm::AxisInfo> axes;
- axes.push_back(H5Parm::AxisInfo("time", itsSols.size()));
- axes.push_back(H5Parm::AxisInfo("freq", nSolChan));
- axes.push_back(H5Parm::AxisInfo("ant", info().nantenna()));
- axes.push_back(H5Parm::AxisInfo("dir", nDir));
- if (nPol>1) {
- axes.push_back(H5Parm::AxisInfo("pol", nPol));
- }
-
- string historyString = "CREATE by DPPP\n" +
- Version::getInfo<DPPPVersion>("DPPP", "top") + "\n" +
- "step " + itsName + " in parset: \n" + itsParsetString;
- uint numsols = 1;
- // For [scalar]complexgain, store two soltabs: phase and amplitude
- if (itsMode == GainCal::COMPLEXGAIN ||
- itsMode == GainCal::SCALARCOMPLEXGAIN || itsMode == GainCal::FULLJONES) {
- numsols = 2;
- }
- for (uint solnum=0; solnum<numsols; ++solnum) {
- string solTabName;
- H5Parm::SolTab soltab;
- switch (itsMode) {
- case GainCal::SCALARPHASE:
- case GainCal::PHASEONLY:
- case GainCal::FULLJONES:
- if (solnum==0) {
- solTabName = "phase000";
- soltab = itsH5Parm.createSolTab(solTabName, "phase", axes);
- soltab.setComplexValues(sols, vector<double>(), false, historyString);
- } else {
- solTabName = "amplitude000";
- soltab = itsH5Parm.createSolTab(solTabName, "amplitude", axes);
- soltab.setComplexValues(sols, vector<double>(), true, historyString);
- }
- break;
- case GainCal::SCALARCOMPLEXGAIN:
- case GainCal::COMPLEXGAIN:
- if (solnum==0) {
- solTabName = "phase000";
- soltab = itsH5Parm.createSolTab(solTabName, "phase", axes);
- soltab.setComplexValues(sols, vector<double>(), false, historyString);
- } else {
- solTabName = "amplitude000";
- soltab = itsH5Parm.createSolTab(solTabName, "amplitude", axes);
- soltab.setComplexValues(sols, vector<double>(), true, historyString);
- }
- break;
- case GainCal::SCALARAMPLITUDE:
- case GainCal::AMPLITUDEONLY:
- solTabName = "amplitude000";
- soltab = itsH5Parm.createSolTab(solTabName, "amplitude", axes);
- soltab.setComplexValues(sols, vector<double>(), true, historyString);
- break;
- default:
- THROW(Exception, "Constraint should have produced output");
- }
-
- // Tell H5Parm that all antennas and directions were used
- std::vector<std::string> antennaNames(info().antennaNames().size());
- for (uint i=0; i<info().antennaNames().size(); ++i) {
- antennaNames[i]=info().antennaNames()[i];
- }
- soltab.setAntennas(antennaNames);
-
- soltab.setSources(getDirectionNames());
-
- if (nPol>1) {
- soltab.setPolarizations(polarizations);
- }
-
- soltab.setFreqs(itsChanBlockFreqs);
-
- soltab.setTimes(solTimes);
- } // solnums loop
- } else {
- // Record the Constraint::Result in the H5Parm
-
- uint nConstraints = itsConstraintSols[0].size();
-
- for (uint constraintNum=0; constraintNum<nConstraints; ++constraintNum) {
- // Number of solution names, e.g. 2 for "TEC" and "ScalarPhase"
- uint nSolNames = itsConstraintSols[0][constraintNum].size();
- for (uint solNameNum=0; solNameNum<nSolNames; ++solNameNum) {
- // Get the result of the constraint solution at first time to get metadata
- Constraint::Result firstResult = itsConstraintSols[0][constraintNum][solNameNum];
-
- vector<hsize_t> dims(firstResult.dims.size()+1); // Add time dimension at beginning
- dims[0]=itsConstraintSols.size(); // Number of times
- size_t numSols=dims[0];
- for (uint i=1; i<dims.size(); ++i) {
- dims[i] = firstResult.dims[i-1];
- numSols *= dims[i];
- }
-
- vector<string> firstaxesnames = StringUtil::tokenize(firstResult.axes,",");
-
- vector<H5Parm::AxisInfo> axes;
- axes.push_back(H5Parm::AxisInfo("time", itsConstraintSols.size()));
- for (size_t axisNum=0; axisNum<firstaxesnames.size(); ++axisNum) {
- axes.push_back(H5Parm::AxisInfo(firstaxesnames[axisNum], firstResult.dims[axisNum]));
- }
-
- // Put solutions in a contiguous piece of memory
- vector<double> sols(numSols);
- vector<double>::iterator nextpos = sols.begin();
- for (uint time=0; time<itsSols.size(); ++time) {
- ASSERTSTR(itsConstraintSols[time].size()==itsConstraintSols[0].size(), "Constraints did not produce enough output at time step "<<time);
- nextpos = std::copy(
- itsConstraintSols[time][constraintNum][solNameNum].vals.begin(),
- itsConstraintSols[time][constraintNum][solNameNum].vals.end(),
- nextpos);
- }
-
- // Put solution weights in a contiguous piece of memory
- vector<double> weights;
- if (!itsConstraintSols[0][constraintNum][solNameNum].weights.empty()) {
- weights.resize(numSols);
- vector<double>::iterator nextpos = weights.begin();
- for (uint time=0; time<itsSols.size(); ++time) {
- nextpos = std::copy(
- itsConstraintSols[time][constraintNum][solNameNum].weights.begin(),
- itsConstraintSols[time][constraintNum][solNameNum].weights.end(),
- nextpos);
- }
- }
-
- string solTabName = firstResult.name+"000";
- H5Parm::SolTab soltab = itsH5Parm.createSolTab(solTabName, firstResult.name, axes);
- soltab.setValues(sols, weights,
- "CREATE by DPPP\n" +
- Version::getInfo<DPPPVersion>("DPPP", "top") + "\n" +
- "step " + itsName + " in parset: \n" +
- itsParsetString);
-
- // Tell H5Parm that all antennas and directions were used
- std::vector<std::string> antennaNames(info().antennaNames().size());
- for (uint i=0; i<info().antennaNames().size(); ++i) {
- antennaNames[i]=info().antennaNames()[i];
- }
- soltab.setAntennas(antennaNames);
-
- soltab.setSources(getDirectionNames());
-
- if (soltab.hasAxis("pol")) {
- vector<string> polarizations;
- switch (soltab.getAxis("pol").size) {
- case 2: polarizations.push_back("XX");
- polarizations.push_back("YY");
- break;
- case 4: polarizations.push_back("XX");
- polarizations.push_back("XY");
- polarizations.push_back("YX");
- polarizations.push_back("YY");
- break;
- default:
- THROW (Exception, "No metadata for numpolarizations = " << soltab.getAxis("pol").size);
- }
-
- soltab.setPolarizations(polarizations);
- }
-
- // Set channel to frequencies
- // Do not use itsChanBlockFreqs, because constraint may have changed size
- uint nChannelBlocks = 1;
- if (soltab.hasAxis("freq")) {
- nChannelBlocks = soltab.getAxis("freq").size;
- }
- vector<double> chanBlockFreqs;
-
- chanBlockFreqs.resize(nChannelBlocks);
- for(size_t chBlock=0; chBlock!=nChannelBlocks; ++chBlock) {
- const size_t
- channelIndexStart = chBlock * info().nchan() / nChannelBlocks,
- channelIndexEnd = (chBlock+1) * info().nchan() / nChannelBlocks,
- curChannelBlockSize = channelIndexEnd - channelIndexStart;
- double meanfreq = std::accumulate(
- info().chanFreqs().data()+channelIndexStart,
- info().chanFreqs().data()+channelIndexEnd,
- 0.0) / curChannelBlockSize;
- chanBlockFreqs[chBlock] = meanfreq;
- }
-
- soltab.setFreqs(chanBlockFreqs);
-
- soltab.setTimes(solTimes);
- }
- }
- }
-
- itsTimerWrite.stop();
- itsTimer.stop();
- }
-
- void DDECal::finish()
- {
- itsTimer.start();
-
- if (itsStepInSolInt!=0) {
- //shrink itsDataPtrs, itsModelDataPtrs
- std::vector<casacore::Complex*>(itsDataPtrs.begin(),
- itsDataPtrs.begin()+itsStepInSolInt).swap(itsDataPtrs);
- std::vector<std::vector<casacore::Complex*> >(itsModelDataPtrs.begin(),
- itsModelDataPtrs.begin()+itsStepInSolInt).swap(itsModelDataPtrs);
-
- doSolve();
- }
-
- writeSolutions();
-
- itsTimer.stop();
-
- // Let the next steps finish.
- getNextStep()->finish();
- }
-
- } //# end namespace
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/KLFitter.cc b/CEP/DP3/DPPP_DDECal/src/KLFitter.cc
deleted file mode 100644
index f8021b25d87529b20d14d9f9f23ef6afa338f533..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/KLFitter.cc
+++ /dev/null
@@ -1,66 +0,0 @@
-#include "DPPP_DDECal/KLFitter.h"
-using namespace arma;
-namespace LOFAR{
-KLFitter::KLFitter(double r0,double beta,int order):
- itsOrder(order),
- itsR0(r0),
- itsBeta(beta)
-{
-
-}
-
-
-
-void KLFitter::calculateCorrMatrix(const std::vector<PiercePoint> pp){
- itsPiercePoints.set_size(pp.size(),3);
- _phases.set_size(pp.size());
- _weights=eye<mat>(pp.size(),pp.size()); //TODO, make weights sensible
- Mat<double> Distance=zeros<mat>(pp.size(),pp.size());
- for(size_t i=0; i<pp.size();i++){
- Mat<double> A(pp[i].getValue().memptr(),1,3);
- itsPiercePoints.row(i)=A;
- }
- for(size_t n=0;n<pp.size();n++)
- for(size_t m=0;m<pp.size();m++)
- for(size_t i=0;i<3;i++)
- Distance(n,m)+=pow(itsPiercePoints.col(i)[n]-itsPiercePoints.col(i)[m],2);
- itsCorrMatrix=-pow((Distance / ( itsR0*itsR0 ) ),( itsBeta / 2.0 ))/2.0;
- itsinvC=pinv(itsCorrMatrix);
- mat V,U;
- Col<double> S;
- svd(U,S,V,itsCorrMatrix);
- itsU=U(span::all,span(0,itsOrder));
- itsinvU=inv(itsU.t()*(_weights*itsU));
-}
-
-void KLFitter::calculateCorrMatrix(const std::vector<PiercePoint*> pp){
- itsPiercePoints.set_size(pp.size(),3);
- _phases.set_size(pp.size());
- _weights=eye<mat>(pp.size(),pp.size()); //TODO, make weights sensible
- Mat<double> Distance=zeros<mat>(pp.size(),pp.size());
- for(size_t i=0; i<pp.size();i++){
- Mat<double> A(pp[i]->getValue().memptr(),1,3);
- itsPiercePoints.row(i)=A;
- }
- for(size_t n=0;n<pp.size();n++)
- for(size_t m=0;m<pp.size();m++)
- for(size_t i=0;i<3;i++)
- Distance(n,m)+=pow(itsPiercePoints.col(i)[n]-itsPiercePoints.col(i)[m],2);
- itsCorrMatrix=-pow((Distance / ( itsR0*itsR0 ) ),( itsBeta / 2.0 ))/2.0;
- itsinvC=pinv(itsCorrMatrix);
- mat V,U;
- Col<double> S;
- svd(U,S,V,itsCorrMatrix);
- itsU=U(span::all,span(0,itsOrder));
- itsinvU=inv(itsU.t()*(_weights*itsU));
-}
-
-
-void KLFitter::doFit(){
- Mat<double> A=itsU.t()*(_weights*_phases);
- itsPar=itsinvU* A;
- itsTECFitWhite=(itsinvC*(itsU*itsPar));
-
- _phases=itsCorrMatrix*itsTECFitWhite;
-}
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/MultiDirSolver.cc b/CEP/DP3/DPPP_DDECal/src/MultiDirSolver.cc
deleted file mode 100644
index 69019b9f2810fd9b90924b1f9466b32274b03988..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/MultiDirSolver.cc
+++ /dev/null
@@ -1,670 +0,0 @@
-
-#ifdef AOPROJECT
-#include "MultiDirSolver.h"
-#include "Matrix2x2.h"
-#include "QRSolver.h"
-#else
-#include <DPPP_DDECal/MultiDirSolver.h>
-#include <DPPP_DDECal/Matrix2x2.h>
-#include <DPPP_DDECal/QRSolver.h>
-#endif
-
-#include <iomanip>
-#include <iostream>
-
-MultiDirSolver::MultiDirSolver() :
- _nAntennas(0),
- _nDirections(0),
- _nChannels(0),
- _nChannelBlocks(0),
- _maxIterations(100),
- _accuracy(1e-5),
- _constraintAccuracy(1e-4),
- _stepSize(0.2),
- _detectStalling(true),
- _phaseOnly(false)
-{ }
-
-void MultiDirSolver::init(size_t nAntennas,
- size_t nDirections,
- size_t nChannels,
- const std::vector<int>& ant1,
- const std::vector<int>& ant2)
-{
- _nAntennas = nAntennas;
- _nDirections = nDirections;
- _nChannels = nChannels;
- _nChannelBlocks = nChannels;
- _ant1 = ant1;
- _ant2 = ant2;
-}
-
-void MultiDirSolver::makeStep(const std::vector<std::vector<DComplex> >& solutions,
- std::vector<std::vector<DComplex> >& nextSolutions) const
-{
- // Move the solutions towards nextSolutions
- // (the moved solutions are stored in 'nextSolutions')
-#pragma omp parallel for
- for(size_t chBlock=0; chBlock<_nChannelBlocks; ++chBlock)
- {
- for(size_t i=0; i!=nextSolutions[chBlock].size(); ++i)
- {
- if(_phaseOnly)
- {
- // In phase only mode, a step is made along the complex circle,
- // towards the shortest direction.
- double phaseFrom = std::arg(solutions[chBlock][i]);
- double distance = std::arg(nextSolutions[chBlock][i]) - phaseFrom;
- if(distance > M_PI) distance = distance - 2.0*M_PI;
- else if(distance < -M_PI) distance = distance + 2.0*M_PI;
- nextSolutions[chBlock][i] = std::polar(1.0, phaseFrom + _stepSize * distance);
- }
- else {
- nextSolutions[chBlock][i] = solutions[chBlock][i]*(1.0-_stepSize) +
- nextSolutions[chBlock][i] * _stepSize;
- }
- }
- }
-}
-
-void MultiDirSolver::makeSolutionsFinite(std::vector<std::vector<DComplex> >& solutions, size_t perPol) const
-{
- for(std::vector<DComplex>& solVector : solutions)
- {
- size_t n = solVector.size() / perPol;
- std::vector<DComplex>::iterator iter = solVector.begin();
- for(size_t i=0; i!=n; ++i)
- {
- bool hasNonFinite = false;
- for(size_t p=0; p!=perPol; ++p)
- {
- hasNonFinite = hasNonFinite || !std::isfinite(iter->real()) || !std::isfinite(iter->imag());
- }
- if(hasNonFinite)
- {
- if(perPol == 4)
- {
- iter[0] = DComplex(1.0, 0.0);
- iter[1] = DComplex(0.0, 0.0);
- iter[2] = DComplex(0.0, 0.0);
- iter[3] = DComplex(1.0, 0.0);
- }
- else {
- for(size_t p=0; p!=perPol; ++p)
- {
- iter[p] = DComplex(1.0, 0.0);
- }
- }
- }
- iter += perPol;
- }
- }
-}
-
-template<size_t NPol>
-bool MultiDirSolver::assignSolutions(std::vector<std::vector<DComplex> >& solutions,
- std::vector<std::vector<DComplex> >& nextSolutions, bool useConstraintAccuracy,
- double& avgAbsDiff, std::vector<double>& stepMagnitudes) const
-{
- avgAbsDiff = 0.0;
- // Calculate the norm of the difference between the old and new solutions
- size_t n = 0;
- for(size_t chBlock=0; chBlock<_nChannelBlocks; ++chBlock)
- {
- for(size_t i=0; i!=solutions[chBlock].size(); i += NPol)
- {
- // A normalized squared difference is calculated between the solutions of this
- // and the previous step:
- // sqrt { 1/n sum over | (t1 - t0) t0^(-1) |_2 }
- // This criterion is scale independent: all solutions can be scaled without
- // affecting the number of iterations. Also, when the polarized version is given
- // scalar matrices, it will use the same number of iterations as the scalar
- // version.
- if(NPol == 1)
- {
- if(solutions[chBlock][i] != 0.0)
- {
- double a = std::abs((nextSolutions[chBlock][i] - solutions[chBlock][i]) / solutions[chBlock][i]);
- if(std::isfinite(a))
- {
- avgAbsDiff += a;
- ++n;
- }
- }
- solutions[chBlock][i] = nextSolutions[chBlock][i];
- }
- else {
- MC2x2 s(&solutions[chBlock][i]), sInv(s);
- if(sInv.Invert())
- {
- MC2x2 ns(&nextSolutions[chBlock][i]);
- ns -= s;
- ns *= sInv;
- double sumabs = 0.0;
- for(size_t p=0; p!=NPol; ++p)
- {
- sumabs += std::abs(ns[p]);
- }
- if(std::isfinite(sumabs))
- {
- avgAbsDiff += sumabs;
- n += 4;
- }
- }
- for(size_t p=0; p!=NPol; ++p)
- {
- solutions[chBlock][i+p] = nextSolutions[chBlock][i+p];
- }
- }
- }
- }
- // The polarized version needs a factor of two normalization to make it work
- // like the scalar version would and when given only scalar matrices.
- //if(NPol == 4)
- // avgSquaredDiff = sqrt(avgSquaredDiff*0.5/n) ;
- //else
- // avgSquaredDiff = sqrt(avgSquaredDiff/n);
-
- // The stepsize is taken out, so that a small stepsize won't cause
- // a premature stopping criterion.
- double stepMagnitude = (n==0 ? 0 : avgAbsDiff/_stepSize/n);
- stepMagnitudes.emplace_back(stepMagnitude);
-
- if(useConstraintAccuracy)
- return stepMagnitude <= _constraintAccuracy;
- else {
- return stepMagnitude <= _accuracy;
- }
-}
-
-MultiDirSolver::SolveResult MultiDirSolver::processScalar(std::vector<Complex *>& data,
- std::vector<std::vector<Complex *> >& modelData,
- std::vector<std::vector<DComplex> >& solutions, double time,
- std::ostream* statStream)
-{
- const size_t nTimes = data.size();
- SolveResult result;
-
- for(size_t i=0; i!=_constraints.size(); ++i)
- _constraints[i]->PrepareIteration(false, 0, false);
-
- std::vector<std::vector<DComplex> > nextSolutions(_nChannelBlocks);
-
-#ifndef NDEBUG
- if (solutions.size() != _nChannelBlocks) {
- std::cout << "Error: 'solutions' parameter does not have the right shape\n";
- result.iterations = 0;
- return result;
- }
-#endif
-
- result._results.resize(_constraints.size());
-
- // Model matrix ant x [N x D] and visibility vector ant x [N x 1],
- // for each channelblock
- // The following loop allocates all structures
- std::vector<std::vector<Matrix> > gTimesCs(_nChannelBlocks);
- std::vector<std::vector<Matrix> > vs(_nChannelBlocks);
- for(size_t chBlock=0; chBlock!=_nChannelBlocks; ++chBlock)
- {
- nextSolutions[chBlock].resize(_nDirections * _nAntennas);
- const size_t
- channelIndexStart = chBlock * _nChannels / _nChannelBlocks,
- channelIndexEnd = (chBlock+1) * _nChannels / _nChannelBlocks,
- curChannelBlockSize = channelIndexEnd - channelIndexStart;
- gTimesCs[chBlock].resize(_nAntennas);
- vs[chBlock].resize(_nAntennas);
-
- for(size_t ant=0; ant!=_nAntennas; ++ant)
- {
- // Model matrix [N x D] and visibility vector [N x 1]
- // Also space for the auto correlation is reserved, but they will be set to 0.
- size_t
- m = _nAntennas * nTimes * curChannelBlockSize * 4,
- n = _nDirections, nrhs = 1;
- gTimesCs[chBlock][ant] = Matrix(m, n);
- vs[chBlock][ant] = Matrix(std::max(m, n), nrhs);
- }
- }
-
- ///
- /// Start iterating
- ///
- size_t iteration = 0, constrainedIterations = 0;
- bool
- hasConverged = false,
- hasPreviouslyConverged = false,
- constraintsSatisfied = false,
- hasStalled = false;
-
- std::vector<double> stepMagnitudes;
- stepMagnitudes.reserve(_maxIterations);
-
- do {
- makeSolutionsFinite(solutions, 1);
-
-#pragma omp parallel for
- for(size_t chBlock=0; chBlock<_nChannelBlocks; ++chBlock)
- {
- performScalarIteration(chBlock, gTimesCs[chBlock], vs[chBlock],
- solutions[chBlock], nextSolutions[chBlock],
- data, modelData);
- }
-
- makeStep(solutions, nextSolutions);
-
- constraintsSatisfied = true;
- _timerConstrain.Start();
-
- if(statStream)
- {
- (*statStream) << iteration << '\t';
- }
-
- for(size_t i=0; i!=_constraints.size(); ++i)
- {
- // PrepareIteration() might change Satisfied(), and since we always want to
- // iterate at least once more when a constraint is not yet satisfied, we
- // evaluate Satisfied() before preparing.
- constraintsSatisfied = _constraints[i]->Satisfied() && constraintsSatisfied;
- _constraints[i]->PrepareIteration(hasPreviouslyConverged, iteration, iteration+1 >= _maxIterations);
- result._results[i] = _constraints[i]->Apply(nextSolutions, time, statStream);
- }
- _timerConstrain.Pause();
-
- if(!constraintsSatisfied)
- constrainedIterations = iteration+1;
-
- double avgSquaredDiff;
- hasConverged = assignSolutions<1>(solutions, nextSolutions, !constraintsSatisfied, avgSquaredDiff, stepMagnitudes);
- if(statStream)
- {
- (*statStream) << stepMagnitudes.back() << '\t' << avgSquaredDiff << '\n';
- }
- iteration++;
-
- hasPreviouslyConverged = hasConverged || hasPreviouslyConverged;
-
- if (_detectStalling && constraintsSatisfied)
- hasStalled = detectStall(iteration, stepMagnitudes);
-
- } while(iteration < _maxIterations && (!hasConverged || !constraintsSatisfied) && !hasStalled);
-
- // When we have not converged yet, we set the nr of iterations to the max+1, so that
- // non-converged iterations can be distinguished from converged ones.
- if((!hasConverged || !constraintsSatisfied) && !hasStalled)
- result.iterations = iteration+1;
- else
- result.iterations = iteration;
- result.constraintIterations = constrainedIterations;
- return result;
-}
-
-void MultiDirSolver::performScalarIteration(size_t channelBlockIndex,
- std::vector<Matrix>& gTimesCs,
- std::vector<Matrix>& vs,
- const std::vector<DComplex>& solutions,
- std::vector<DComplex>& nextSolutions,
- const std::vector<Complex *>& data,
- const std::vector<std::vector<Complex *> >& modelData)
-{
- for(size_t ant=0; ant!=_nAntennas; ++ant)
- {
- gTimesCs[ant].zeros();
- vs[ant].zeros();
- }
-
- const size_t
- channelIndexStart = channelBlockIndex * _nChannels / _nChannelBlocks,
- channelIndexEnd = (channelBlockIndex+1) * _nChannels / _nChannelBlocks,
- curChannelBlockSize = channelIndexEnd - channelIndexStart,
- nTimes = data.size();
-
- // The following loop fills the matrices for all antennas
- for(size_t timeIndex=0; timeIndex!=nTimes; ++timeIndex)
- {
- std::vector<const Complex*> modelPtrs(_nDirections);
- for(size_t baseline=0; baseline!=_ant1.size(); ++baseline)
- {
- size_t antenna1 = _ant1[baseline];
- size_t antenna2 = _ant2[baseline];
- if(antenna1 != antenna2)
- {
- Matrix& gTimesC1 = gTimesCs[antenna1];
- Matrix& v1 = vs[antenna1];
- Matrix& gTimesC2 = gTimesCs[antenna2];
- Matrix& v2 = vs[antenna2];
- for(size_t d=0; d!=_nDirections; ++d)
- modelPtrs[d] = modelData[timeIndex][d] + (channelIndexStart + baseline * _nChannels) * 4;
- const Complex* dataPtr = data[timeIndex] + (channelIndexStart + baseline * _nChannels) * 4;
- const size_t p1top2[4] = {0, 2, 1, 3};
- for(size_t ch=channelIndexStart; ch!=channelIndexEnd; ++ch)
- {
- const size_t
- dataIndex1 = ch-channelIndexStart + (timeIndex + antenna1 * nTimes) * curChannelBlockSize,
- dataIndex2 = ch-channelIndexStart + (timeIndex + antenna2 * nTimes) * curChannelBlockSize;
- for(size_t p1=0; p1!=4; ++p1)
- {
- size_t p2 = p1top2[p1];
- for(size_t d=0; d!=_nDirections; ++d)
- {
- std::complex<double> predicted = *modelPtrs[d];
-
- size_t solIndex1 = antenna1*_nDirections + d;
- size_t solIndex2 = antenna2*_nDirections + d;
- gTimesC2(dataIndex1*4+p1, d) = std::conj(solutions[solIndex1] * predicted); // using a* b* = (ab)*
- gTimesC1(dataIndex2*4+p2, d) = std::conj(solutions[solIndex2]) * predicted;
-
- ++modelPtrs[d]; // Goto the next polarization of this 2x2 matrix.
- }
- v1(dataIndex2*4+p2, 0) = *dataPtr;
- v2(dataIndex1*4+p1, 0) = std::conj(*dataPtr);
- ++dataPtr; // Goto the next polarization of this 2x2 matrix.
- }
- }
- }
- }
- }
-
- // The matrices have been filled; compute the linear solution
- // for each antenna.
- size_t m = _nAntennas * nTimes * curChannelBlockSize * 4;
- size_t n = _nDirections, nrhs = 1;
- QRSolver solver(m, n, nrhs);
- for(size_t ant=0; ant!=_nAntennas; ++ant) {
- // solve x^H in [g C] x^H = v
- bool success = solver.Solve(gTimesCs[ant].data(), vs[ant].data());
- Matrix& x = vs[ant];
- if(success && x(0, 0) != 0.)
- {
- for(size_t d=0; d!=_nDirections; ++d)
- nextSolutions[ant*_nDirections + d] = x(d, 0);
- }
- else {
- for(size_t d=0; d!=_nDirections; ++d)
- nextSolutions[ant*_nDirections + d] = std::numeric_limits<double>::quiet_NaN();
- }
- }
-}
-
-MultiDirSolver::SolveResult MultiDirSolver::processFullMatrix(std::vector<Complex *>& data,
- std::vector<std::vector<Complex *> >& modelData,
- std::vector<std::vector<DComplex> >& solutions, double time,
- std::ostream* statStream)
-{
- // This algorithm is basically the same as the scalar algorithm,
- // but visibility values are extended to 2x2 matrices and concatenated
- // in the matrix equations as block matrices. One difference is that
- // order of operations are important because of the non-commutativity of
- // matrix multiplication, as well as that A^H B^H = [BA]^H.
- //
- // The approach:
- // First we pre-apply the left-hand solutions to the model to make JM. Each
- // 2x2 coherence matrix Ji is matrix-multied by the lh solutions, for all
- // directions, and visibilities (times x channels).
- // JMi = Ji Mi
- // These are stacked in matrix JM :
- // JM0_d0 JM1_d0 ...
- // JM = JM0_d1 JM1_d1
- // ...
- // such that JM is a (2D) rows x (2N) col matrix, N=nvis, D=ndir.
- // The solved 2D x 2 solution matrix is similarly formed with the solution
- // values:
- // ( J0 )
- // J = ( J1 )
- // ( .. )
- // And the 2N x 2 visibility matrix as well:
- // ( V0 )
- // V = ( V1 )
- // ( .. )
- // And we solve the equation:
- // 'JM' J^H = V
- // With dimensions:
- // [ 2N x 2D ] [ 2D x 2 ] = [ 2N x 2 ]
-
- const size_t nTimes = data.size();
- SolveResult result;
-
- for(size_t i=0; i!=_constraints.size(); ++i)
- _constraints[i]->PrepareIteration(false, 0, false);
-
- std::vector<std::vector<DComplex> > nextSolutions(_nChannelBlocks);
-
-#ifndef NDEBUG
- if (solutions.size() != _nChannelBlocks) {
- std::cout << "Error: 'solutions' parameter does not have the right shape\n";
- result.iterations = 0;
- return result;
- }
-#endif
-
- result._results.resize(_constraints.size());
-
- // Dimensions for each channelblock:
- // Model matrix ant x [2N x 2D] and visibility matrix ant x [2N x 2],
- // The following loop allocates all structures
- std::vector<std::vector<Matrix> > gTimesCs(_nChannelBlocks);
- std::vector<std::vector<Matrix> > vs(_nChannelBlocks);
- for(size_t chBlock=0; chBlock!=_nChannelBlocks; ++chBlock)
- {
- nextSolutions[chBlock].resize(_nDirections * _nAntennas * 4);
- const size_t
- channelIndexStart = chBlock * _nChannels / _nChannelBlocks,
- channelIndexEnd = (chBlock+1) * _nChannels / _nChannelBlocks,
- curChannelBlockSize = channelIndexEnd - channelIndexStart;
- gTimesCs[chBlock].resize(_nAntennas);
- vs[chBlock].resize(_nAntennas);
-
- for(size_t ant=0; ant!=_nAntennas; ++ant)
- {
- // Model matrix [2N x 2D] and visibility matrix [2N x 2]
- // Also space for the auto correlation is reserved, but they will be set to 0.
- size_t m = _nAntennas * nTimes * curChannelBlockSize * 2;
- size_t n = _nDirections * 2, nrhs = 2;
- gTimesCs[chBlock][ant] = Matrix(m, n);
- vs[chBlock][ant] = Matrix(std::max(n, m), nrhs);
- }
- }
-
- ///
- /// Start iterating
- ///
- size_t iteration = 0, constrainedIterations = 0;
- bool hasConverged = false,
- hasPreviouslyConverged = false,
- constraintsSatisfied = false,
- hasStalled = false;
-
- std::vector<double> step_magnitudes;
- step_magnitudes.reserve(_maxIterations);
-
- do {
- makeSolutionsFinite(solutions, 4);
-
-#pragma omp parallel for
- for(size_t chBlock=0; chBlock<_nChannelBlocks; ++chBlock)
- {
- performFullMatrixIteration(chBlock, gTimesCs[chBlock], vs[chBlock],
- solutions[chBlock], nextSolutions[chBlock],
- data, modelData);
- }
-
- makeStep(solutions, nextSolutions);
-
- if(statStream)
- {
- (*statStream) << iteration << '\t';
- }
-
- constraintsSatisfied = true;
- for(size_t i=0; i!=_constraints.size(); ++i)
- {
- constraintsSatisfied = _constraints[i]->Satisfied() && constraintsSatisfied;
- _constraints[i]->PrepareIteration(hasPreviouslyConverged, iteration, iteration+1 >= _maxIterations);
- result._results[i] = _constraints[i]->Apply(nextSolutions, time, statStream);
- }
-
- if(!constraintsSatisfied)
- constrainedIterations = iteration+1;
-
- double avgSquaredDiff;
- hasConverged = assignSolutions<4>(solutions, nextSolutions, !constraintsSatisfied, avgSquaredDiff, step_magnitudes);
- if(statStream)
- {
- (*statStream) << step_magnitudes.back() << '\t' << avgSquaredDiff << '\n';
- }
- iteration++;
-
- hasPreviouslyConverged = hasConverged || hasPreviouslyConverged;
-
- if (_detectStalling && constraintsSatisfied)
- hasStalled = detectStall(iteration, step_magnitudes);
-
- } while(iteration < _maxIterations && (!hasConverged || !constraintsSatisfied) && !hasStalled);
-
- // When we have not converged yet, we set the nr of iterations to the max+1, so that
- // non-converged iterations can be distinguished from converged ones.
- if((!hasConverged || !constraintsSatisfied) && !hasStalled)
- result.iterations = iteration+1;
- else
- result.iterations = iteration;
- result.constraintIterations = constrainedIterations;
- return result;
-}
-
-bool MultiDirSolver::detectStall(size_t iteration, const std::vector<double>& step_magnitudes) const
-{
- if (iteration<30) {
- return false;
- } else {
- return std::abs(step_magnitudes[iteration-1]/step_magnitudes[iteration-2]-1) < 1.e-4 &&
- std::abs(step_magnitudes[iteration-2]/step_magnitudes[iteration-3]-1) < 1.e-4;
- }
-}
-
-void MultiDirSolver::performFullMatrixIteration(size_t channelBlockIndex,
- std::vector<Matrix>& gTimesCs,
- std::vector<Matrix>& vs,
- const std::vector<DComplex>& solutions,
- std::vector<DComplex>& nextSolutions,
- const std::vector<Complex *>& data,
- const std::vector<std::vector<Complex *> >& modelData)
-{
- for(size_t ant=0; ant!=_nAntennas; ++ant)
- {
- gTimesCs[ant].zeros();
- vs[ant].zeros();
- }
-
- const size_t
- channelIndexStart = channelBlockIndex * _nChannels / _nChannelBlocks,
- channelIndexEnd = (channelBlockIndex+1) * _nChannels / _nChannelBlocks,
- curChannelBlockSize = channelIndexEnd - channelIndexStart,
- nTimes = data.size();
-
- // The following loop fills the matrices for all antennas
- _timerFillMatrices.Start();
- for(size_t timeIndex=0; timeIndex!=nTimes; ++timeIndex)
- {
- std::vector<const Complex*> modelPtrs(_nDirections);
- for(size_t baseline=0; baseline!=_ant1.size(); ++baseline)
- {
- size_t antenna1 = _ant1[baseline];
- size_t antenna2 = _ant2[baseline];
- if(antenna1 != antenna2)
- {
- // This equation is solved:
- // J_1 M J_2^H = V
- // for visibilities of the 'normal' correlation ant1 x ant2^H.
- // Since in this equation antenna2 is solved, the solve matrices are
- // called gTimesC2 and v2. The index into these matrices is depending
- // on antenna1, hence the index is called dataIndex1.
- //
- // To use visibilities of correlation ant2 x ant1^H to solve ant1, an
- // extra Herm transpose on M and V is required. The equation is:
- // J_2 M^H J_1^H = V^H,
- // and the relevant matrices/index are called gTimesC1, v1 and dataIndex2.
- Matrix
- &gTimesC1 = gTimesCs[antenna1],
- &v1 = vs[antenna1],
- &gTimesC2 = gTimesCs[antenna2],
- &v2 = vs[antenna2];
- for(size_t d=0; d!=_nDirections; ++d)
- modelPtrs[d] = modelData[timeIndex][d] + (channelIndexStart + baseline * _nChannels) * 4;
- const Complex* dataPtr = data[timeIndex] + (channelIndexStart + baseline * _nChannels) * 4;
- for(size_t ch=channelIndexStart; ch!=channelIndexEnd; ++ch)
- {
- const size_t
- dataIndex1 = 2 * (ch-channelIndexStart + (timeIndex + antenna1 * nTimes) * curChannelBlockSize),
- dataIndex2 = 2 * (ch-channelIndexStart + (timeIndex + antenna2 * nTimes) * curChannelBlockSize);
-
- for(size_t d=0; d!=_nDirections; ++d)
- {
- MC2x2
- modelMat(modelPtrs[d]),
- gTimesC1Mat, gTimesC2Mat;
- size_t solIndex1 = (antenna1*_nDirections + d) * 4;
- size_t solIndex2 = (antenna2*_nDirections + d) * 4;
- Matrix2x2::ATimesB(gTimesC2Mat.Data(), &solutions[solIndex1], modelMat.Data());
- Matrix2x2::ATimesHermB(gTimesC1Mat.Data(), &solutions[solIndex2], modelMat.Data());
- for(size_t p=0; p!=4; ++p)
- {
- gTimesC2(dataIndex1+(p/2), d*2+p%2) = gTimesC2Mat[p];
- gTimesC1(dataIndex2+(p/2), d*2+p%2) = gTimesC1Mat[p];
- }
-
- modelPtrs[d] += 4; // Goto the next 2x2 matrix.
- }
- for(size_t p=0; p!=4; ++p)
- {
- v1(dataIndex2+(p%2), p/2) = std::conj(*dataPtr);
- v2(dataIndex1+(p/2), p%2) = *dataPtr; // note that this also performs the Herm transpose
- ++dataPtr; // Goto the next element of the 2x2 matrix.
- }
- }
- }
- }
- }
- _timerFillMatrices.Pause();
-
- // The matrices have been filled; compute the linear solution
- // for each antenna.
- _timerSolve.Start();
-
- size_t m = _nAntennas * nTimes * curChannelBlockSize * 2;
- size_t n = _nDirections * 2, nrhs = 2;
- QRSolver solver(m, n, nrhs);
-
- for(size_t ant=0; ant!=_nAntennas; ++ant) {
- // solve x^H in [g C] x^H = v
- bool success = solver.Solve(gTimesCs[ant].data(), vs[ant].data());
- Matrix& x = vs[ant];
- if(success && x(0, 0) != 0.)
- {
- for(size_t d=0; d!=_nDirections; ++d)
- {
- for(size_t p=0; p!=4; ++p) {
- // The conj transpose is also performed at this point (note swap of % and /)
- nextSolutions[(ant*_nDirections + d)*4 + p] = std::conj(x(d*2+p%2, p/2));
- }
- }
- }
- else {
- for(size_t i=0; i!=_nDirections*4; ++i) {
- nextSolutions[ant*_nDirections*4 + i] = std::numeric_limits<double>::quiet_NaN();
- }
- }
- }
- _timerSolve.Pause();
-}
-
-void MultiDirSolver::showTimings (std::ostream& os, double duration) const {
- //os << " " << std::fixed << std::setprecision(2) << _timerSolve.Seconds()/duration << "% spent in solve" << std::endl;
- //os << " " << std::fixed << std::setprecision(2) << _timerFillMatrices.Seconds()/duration << "% spent in filling matrices" << std::endl;
- if (!_constraints.empty()) {
- os << " " << std::fixed << std::setprecision(2) << _timerConstrain.Seconds()/duration << "% spent in constraints" << std::endl;
- for (auto& constraint: _constraints) {
- constraint->showTimings(os, duration);
- }
- }
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/PiercePoint.cc b/CEP/DP3/DPPP_DDECal/src/PiercePoint.cc
deleted file mode 100644
index 6f2439b5804521bcd469d50fd8e13057224b690f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/PiercePoint.cc
+++ /dev/null
@@ -1,53 +0,0 @@
-#include <DPPP_DDECal/PiercePoint.h>
-using namespace arma;
-
-namespace LOFAR{
-
- const double PiercePoint::IONOheight = 300000.;
- const double PiercePoint::EarthRadius = 6371000.;
-
-PiercePoint::PiercePoint(double height):
- itsValue(3)
-{
- casacore::MPosition ant;//ITRF
- casacore::MDirection source;//J2000 pole
- init(ant,source,height);
-}
-
-PiercePoint::PiercePoint(const casacore::MPosition &ant,const casacore::MDirection &source,const double height):
- itsValue(3)
-{
- init(ant,source,height);
-};
-
-PiercePoint::PiercePoint(const casacore::MPosition &ant,const casacore::MDirection &source):
- itsValue(3)
- {
- init(ant,source,PiercePoint::IONOheight);
-};
-
-void PiercePoint::init(const casacore::MPosition &ant,const casacore::MDirection &source,const double height){
-
- itsPosition=casacore::MPosition::Convert(ant,casacore::MPosition::ITRF)();
- itsDirection=source;
- itsIonoHeight=height;
- const casacore::MVPosition &mPosition = itsPosition.getValue();
- itsC = mPosition(0)*mPosition(0)+mPosition(1)*mPosition(1)+mPosition(2)*mPosition(2)-
- (itsIonoHeight+PiercePoint::EarthRadius)*(itsIonoHeight+PiercePoint::EarthRadius);
-
-}
-
-void PiercePoint::evaluate(casacore::MEpoch time){
- //Convert direction to ITRF vector
- casacore::MeasFrame myframe(itsPosition,time);
- casacore::MDirection::Ref myref(casacore::MDirection::ITRF,myframe);
- const casacore::MDirection dir = casacore::MDirection::Convert(itsDirection,myref)();
- const casacore::MVDirection &mDir = dir.getValue();
- const casacore::MVPosition &mPos = itsPosition.getValue();
- double A = mDir(0)*mDir(0)+mDir(1)*mDir(1)+mDir(2)*mDir(2);
- double B = mDir(0)*mPos(0) + mDir(1)*mPos(1) +mDir(2)*mPos(2);
- double alpha = (-B + sqrt(B*B - A*itsC))/A;
- for(uword i=0;i<3;i++)
- itsValue(i) = mPos(i) + alpha*mDir(i);
-};
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/Register.cc b/CEP/DP3/DPPP_DDECal/src/Register.cc
deleted file mode 100644
index b31ba20ca65c5f904b5dd6881582fb3c7589cee6..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/Register.cc
+++ /dev/null
@@ -1,35 +0,0 @@
-//# Register.cc: Register steps in DPPP
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: AOFlaggerStep.cc 31423 2015-04-03 14:06:21Z dijkema $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP_DDECal/Register.h>
-#include <DPPP_DDECal/DDECal.h>
-#include <DPPP/DPRun.h>
-
-// Define the function to make the DDECal 'constructor' known.
-// Its suffix must be the (lowercase) name of the package (library).
-void register_ddecal()
-{
- LOFAR::DPPP::DPRun::registerStepCtor ("ddecal",
- LOFAR::DPPP::DDECal::makeStep);
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/RotationAndDiagonalConstraint.cc b/CEP/DP3/DPPP_DDECal/src/RotationAndDiagonalConstraint.cc
deleted file mode 100644
index 47368347c677e7bf6081c479f4aef8b5395ac8f3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/RotationAndDiagonalConstraint.cc
+++ /dev/null
@@ -1,112 +0,0 @@
-#include <lofar_config.h>
-
-#include <DPPP_DDECal/RotationAndDiagonalConstraint.h>
-#include <DPPP_DDECal/RotationConstraint.h>
-#include <Common/OpenMP.h>
-#include <cmath>
-#include <assert.h>
-
-using namespace std;
-
-namespace LOFAR {
-
-void RotationAndDiagonalConstraint::InitializeDimensions(size_t nAntennas,
- size_t nDirections,
- size_t nChannelBlocks) {
- Constraint::InitializeDimensions(nAntennas, nDirections, nChannelBlocks);
-
- assert(_nDirections == 1);
-
- _res.resize(3);
- _res[0].vals.resize(_nAntennas*_nChannelBlocks);
- _res[0].weights.resize(_nAntennas*_nChannelBlocks);
- _res[0].axes="ant,freq";
- _res[0].dims.resize(2);
- _res[0].dims[0]=_nAntennas;
- _res[0].dims[1]=_nChannelBlocks;
- _res[0].name="rotation";
-
- _res[1].vals.resize(_nAntennas*_nChannelBlocks*2);
- _res[1].weights.resize(_nAntennas*_nChannelBlocks*2);
- _res[1].axes="ant,freq,pol";
- _res[1].dims.resize(3);
- _res[1].dims[0]=_nAntennas;
- _res[1].dims[1]=_nChannelBlocks;
- _res[1].dims[2]=2;
- _res[1].name="amplitude";
-
- _res[2] = _res[1];
- _res[2].name="phase";
-}
-
-void RotationAndDiagonalConstraint::SetWeights(const vector<double>& weights) {
- _res[0].weights = weights;
-
- // Duplicate weights for two polarizations
- _res[1].weights.resize(_nAntennas*_nChannelBlocks*2);
- size_t indexInWeights = 0;
- for (auto weight: weights) {
- _res[1].weights[indexInWeights++] = weight;
- _res[1].weights[indexInWeights++] = weight;
- }
-
- _res[2].weights = _res[1].weights;
-}
-
-vector<Constraint::Result> RotationAndDiagonalConstraint::Apply(
- vector<vector<dcomplex> >& solutions, double,
- std::ostream* statStream) {
- if (statStream) *statStream<<"["; // begin channel
- double angle0;
- for (uint ch=0; ch<_nChannelBlocks; ++ch) {
- if (statStream) *statStream<<"["; // begin antenna
- for (uint ant=0; ant<_nAntennas; ++ant) {
- // Compute rotation
- complex<double> *data = &(solutions[ch][4*ant]);
-
- double angle = RotationConstraint::get_rotation(data);
- // Restrict angle between -pi/2 and pi/2
- // Add 2pi to make sure that fmod doesn't see negative numbers
- angle = fmod(angle + 3.5*M_PI, M_PI) - 0.5*M_PI;
-
- // Right multiply solution with inverse rotation,
- // save only the diagonal
- // Use sin(-phi) == -sin(phi)
- complex<double> a, b;
- a = data[0]*cos(angle) - data[1]*sin(angle);
- b = data[3]*cos(angle) + data[2]*sin(angle);
-
- // Use station 0 as reference station (for every chanblock), to work
- // around unitary ambiguity
- if (ant==0) {
- angle0 = angle;
- angle = 0.;
- } else {
- angle -= angle0;
- angle = fmod(angle + 3.5*M_PI, M_PI) - 0.5*M_PI;
- }
- _res[0].vals[ant*_nChannelBlocks + ch] = angle;
-
- _res[1].vals[ant*_nChannelBlocks*2 + 2*ch ] = abs(a);
- _res[1].vals[ant*_nChannelBlocks*2 + 2*ch + 1] = abs(b);
- _res[2].vals[ant*_nChannelBlocks*2 + 2*ch ] = arg(a);
- _res[2].vals[ant*_nChannelBlocks*2 + 2*ch +1] = arg(b);
-
- // Do the actual constraining
- data[0] = a * cos(angle);
- data[1] = -a * sin(angle);
- data[2] = b * sin(angle);
- data[3] = b * cos(angle);
- if (statStream) *statStream<<"["<<a.real()<<"+"<<a.imag()<<"j,"<<b.real()<<"+"<<b.imag()<<"j,"<<angle<<"]";
- //if (pd) cout<<angle;
- if (statStream && ant<_nAntennas-1) *statStream<<",";
- }
- if (statStream) *statStream<<"]"; // end antenna
- if (statStream && ch<_nChannelBlocks-1) *statStream<<",";
- }
- if (statStream) *statStream<<"]\t"; //end channel
-
- return _res;
-}
-
-} //namespace LOFAR
diff --git a/CEP/DP3/DPPP_DDECal/src/RotationConstraint.cc b/CEP/DP3/DPPP_DDECal/src/RotationConstraint.cc
deleted file mode 100644
index f71a075e991ae61daa7e84d68e3aca0bb189247b..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/RotationConstraint.cc
+++ /dev/null
@@ -1,69 +0,0 @@
-#include <lofar_config.h>
-
-#include <DPPP_DDECal/RotationConstraint.h>
-#include <Common/OpenMP.h>
-#include <cmath>
-#include <assert.h>
-
-using namespace std;
-
-namespace LOFAR {
-
- void RotationConstraint::InitializeDimensions(size_t nAntennas,
- size_t nDirections,
- size_t nChannelBlocks) {
- Constraint::InitializeDimensions(nAntennas, nDirections, nChannelBlocks);
-
- assert(_nDirections == 1);
-
- _res.resize(1);
- _res[0].vals.resize(_nAntennas*_nChannelBlocks);
- _res[0].axes="ant,freq";
- _res[0].dims.resize(2);
- _res[0].dims[0]=_nAntennas;
- _res[0].dims[1]=_nChannelBlocks;
- _res[0].name="rotation";
-}
-
-void RotationConstraint::SetWeights(const vector<double>& weights) {
- _res[0].weights = weights;
-}
-
-double RotationConstraint::get_rotation(std::complex<double>* data) {
- // Convert to circular
- complex<double> ll, rr;
- complex<double> i(0,1.);
-
- ll = data[0] + data[3] - i*data[1] + i*data[2];
- rr = data[0] + data[3] + i*data[1] - i*data[2];
- double angle = 0.5 * (arg(ll) - arg(rr));
-
- return angle;
-}
-
-vector<Constraint::Result> RotationConstraint::Apply(
- vector<vector<dcomplex> >& solutions, double,
- std::ostream* statStream) {
- // Convert to circular
- complex<double> ll, rr;
- complex<double> i(0,1.);
-
- for (uint ch=0; ch<_nChannelBlocks; ++ch) {
- for (uint ant=0; ant<_nAntennas; ++ant) {
- // Compute rotation
- complex<double> *data= &(solutions[ch][4*ant]);
- double angle = get_rotation(data);
- _res[0].vals[ant*_nChannelBlocks+ch] = angle;
-
- // Constrain the data
- data[0] = cos(angle);
- data[1] = -sin(angle);
- data[2] = -data[1];
- data[3] = data[0];
- }
- }
-
- return _res;
-}
-
-} //namespace LOFAR
diff --git a/CEP/DP3/DPPP_DDECal/src/ScreenConstraint.cc b/CEP/DP3/DPPP_DDECal/src/ScreenConstraint.cc
deleted file mode 100644
index 3e16c70f4669ec4f41c089d771336187dc282428..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/ScreenConstraint.cc
+++ /dev/null
@@ -1,378 +0,0 @@
-#include <DPPP_DDECal/ScreenConstraint.h>
-#include <Common/OpenMP.h>
-
-namespace LOFAR{
-
-const double ScreenConstraint::phtoTEC = 1./8.4479745e9;
-const double ScreenConstraint::TECtoph = 8.4479745e9;
-const size_t ScreenConstraint::maxIter=30;
-
-ScreenConstraint::ScreenConstraint(const ParameterSet& parset,
- const string& prefix)
- :
- itsCurrentTime(0),
- itsIter(0)
-{
- cout<<"=========="<<(prefix + "order")<<"========\n";
- itsBeta=parset.getDouble (prefix + "beta", 5./3.);
- itsHeight=parset.getDouble (prefix + "height", 400e3);
- itsOrder=parset.getInt(prefix + "order", 3);
- itsRdiff=parset.getDouble (prefix +"rdiff",1e3);
- itsMode=toLower(parset.getString(prefix+"mode","station") );
- itsAVGMode=toLower(parset.getString(prefix+"average","tec") );
- itsDebugMode=parset.getInt(prefix + "debug", 0);
-}
-
-void ScreenConstraint::initialize(const double* frequencies) {
- itsFrequencies.resize(_nChannelBlocks);
- itsprevsol.assign(_nDirections*_nAntennas,-999.);
- std::memcpy( itsFrequencies.data(),frequencies, sizeof(double) * _nChannelBlocks);
- itsAntennaPos.resize(_nAntennas);
- itsSourcePos.resize(_nDirections);
- itsPiercePoints.resize(_nAntennas);
- for(uint i=0;i<itsPiercePoints.size();i++)
- itsPiercePoints[i].resize(_nDirections);
-
- if (itsMode=="station")
- _screenFitters.resize(_nAntennas);
- else if (itsMode=="direction")
- _screenFitters.resize(_nDirections);
- else if (itsMode=="full")
- _screenFitters.resize(1);
- else if (itsMode=="csfull")
- _screenFitters.resize(_nAntennas-_coreAntennas.size()+1);
- else
- THROW (Exception, "Unexpected tecscreen mode: " << itsMode);
-
-
- for(size_t i=0; i!=_screenFitters.size(); ++i)
- {
- _screenFitters[i].setR0(itsRdiff);
- _screenFitters[i].setBeta(itsBeta);
- _screenFitters[i].setOrder(itsOrder);
-
- }
- if (itsDebugMode>0)
- {
- _iterphases.resize(_nAntennas*_nDirections*_nChannelBlocks*maxIter);
- }
-}
-
-void ScreenConstraint::setAntennaPositions(const std::vector<std::vector<double> > antenna_pos) {
- itsAntennaPos = antenna_pos;
-}
-
-void ScreenConstraint::setDirections(const std::vector<std::pair<double, double> > source_pos) {
- for(uint i=0;i<source_pos.size();i++){
- itsSourcePos[i].resize(2);
- itsSourcePos[i][0]=source_pos[i].first;
- itsSourcePos[i][1]=source_pos[i].second;
- }
-}
-
-void ScreenConstraint::initPiercePoints(){
- for(uint ipos=0;ipos<itsAntennaPos.size();ipos++){
- casacore::MPosition ant(casacore::MVPosition(itsAntennaPos[ipos][0],itsAntennaPos[ipos][1],itsAntennaPos[ipos][2]),
- casacore::MPosition::ITRF);
- for(uint isrc=0;isrc<itsSourcePos.size();isrc++){
- casacore::MDirection src(casacore::MVDirection(itsSourcePos[isrc][0],itsSourcePos[isrc][1]),
- casacore::MDirection::J2000);
-
- itsPiercePoints[ipos][isrc]=PiercePoint(ant,src,itsHeight);
- }
- }
-}
-
-void ScreenConstraint::setTime(double time){
- if (itsCurrentTime!=time){
-
- itsCurrentTime=time;
- itsIter=0;
-
- CalculatePiercepoints();
-
- if (itsMode=="station")
- {
-#pragma omp parallel for
- for(uint ipos=0;ipos<_nAntennas;ipos++)
- _screenFitters[ipos].calculateCorrMatrix(itsPiercePoints[ipos]);
- }
- else if (itsMode=="direction")
- {
-#pragma omp parallel for
- for(uint idir=0;idir<_nDirections;idir++){
- std::vector<PiercePoint *> tmpV(_nAntennas);
- for(uint ipos=0;ipos<_nAntennas;ipos++)
- tmpV[ipos]=&(itsPiercePoints[ipos][idir]);
- _screenFitters[idir].calculateCorrMatrix(tmpV);
- }
- }
- else if (itsMode=="full")
- {
- std::vector<PiercePoint *> tmpV(_nAntennas*_nDirections);
- size_t i=0;
- for(uint idir=0;idir<_nDirections;idir++)
- {
- for(uint ipos=0;ipos<_nAntennas;ipos++)
- tmpV[i++]=&(itsPiercePoints[ipos][idir]);
- }
- _screenFitters[0].calculateCorrMatrix(tmpV);
- }
- else if (itsMode=="csfull")
- {
- std::vector<PiercePoint *> tmpV(_coreAntennas.size()*itsSourcePos.size());
- for(size_t iant=0; iant<_coreAntennas.size(); iant++){
- size_t ipos=_coreAntennas[iant];
- for(uint idir=0;idir<_nDirections;idir++)
- tmpV[iant*_nDirections+idir]=&(itsPiercePoints[ipos][idir]);
- }
- _screenFitters[0].calculateCorrMatrix(tmpV);
-#pragma omp parallel for
- for(size_t iant=0; iant<_otherAntennas.size(); iant++){
- size_t ipos=_otherAntennas[iant];
- _screenFitters[iant+1].calculateCorrMatrix(itsPiercePoints[ipos]);
- }
- }
- else
- THROW (Exception, "Unexpected tecscreen mode: " << itsMode);
-
- }
- else
- itsIter+=1;
-}
-
-void ScreenConstraint::CalculatePiercepoints(){
- casacore::MEpoch time(casacore::MVEpoch(itsCurrentTime/(24.*3600.))); //convert to MJD
- for (uint i=0;i<itsPiercePoints.size();i++)
- for (uint j=0;j<itsPiercePoints[i].size();j++)
- itsPiercePoints[i][j].evaluate(time);
-}
-
- void ScreenConstraint::getPPValue(std::vector<std::vector<MultiDirSolver::DComplex> >& solutions,size_t solutionIndex,size_t dirIndex,double &avgTEC,double &error) const {
- if (itsAVGMode=="simple"){
- avgTEC=0;
- for(size_t ch=0;ch<_nChannelBlocks; ++ch){
- double refphase=std::arg(solutions[ch][dirIndex]);
- //TODO: more advance frequency averaging...
- avgTEC += std::arg(solutions[ch][solutionIndex]*std::polar<double>(1.0,-1*refphase))*itsFrequencies[ch]*phtoTEC;
- }
- avgTEC/=_nChannelBlocks;
- }
- else{
- PhaseFitter phfit(_nChannelBlocks) ;
- double offset;
- for(size_t ch=0;ch<_nChannelBlocks; ++ch){
- phfit.FrequencyData()[ch]=itsFrequencies.data()[ch];
- phfit.PhaseData()[ch] = std::arg(solutions[ch][solutionIndex]);
- }
- if (itsprevsol[solutionIndex]<-100){
- phfit.FitTEC2ModelParameters(avgTEC,offset);
- error=phfit.TEC2ModelCost(avgTEC,offset);
- }
- else {
- avgTEC=itsprevsol[solutionIndex]*TECtoph;
- phfit.FitTEC2ModelParameters(avgTEC,offset);
- error=phfit.TEC2ModelCost(avgTEC,offset);
- }
-
- avgTEC*=phtoTEC;
- }
-}
-
-
-std::vector<Constraint::Result> ScreenConstraint::Apply(std::vector<std::vector<MultiDirSolver::DComplex> >& solutions,
- double time, std::ostream* statStream) {
- //check if we need to reinitialize piercepoints
- setTime(time);
- size_t nrresults=4;
- if (itsDebugMode>0)
- nrresults=5;
- std::vector<Result> res(nrresults);
- size_t numberofPar=_screenFitters[0].getOrder();
- res[0].vals.resize(_screenFitters.size()*numberofPar);
- res[0].axes="screennr,par";
- res[0].dims.resize(2);
- res[0].dims[0]=_screenFitters.size();
- res[0].dims[1]=numberofPar;
- res[0].name="screenpar";
- res[1].vals.resize(_nAntennas*_nDirections*3);
- res[1].axes="ant,dir,xyz";
- res[1].dims.resize(3);
- res[1].dims[0]=_nAntennas;
- res[1].dims[1]=_nDirections;
- res[1].dims[2]=3;
- res[1].name="piercepoints";
- res[2].vals.resize(_nAntennas*_nDirections);
- res[2].axes="ant,dir";
- res[2].dims.resize(2);
- res[2].dims[0]=_nAntennas;
- res[2].dims[1]=_nDirections;
- res[2].name="TECfitwhite";
- res[3].vals.resize(_nAntennas*_nDirections);
- res[3].axes="ant,dir,freq";
- res[3].dims.resize(3);
- res[3].dims[0]=_nAntennas;
- res[3].dims[1]=_nDirections;
- res[3].dims[2]=1;
- res[3].name="tec";
- if (itsDebugMode>0){
- res[4].vals.resize(_nAntennas*_nDirections*_nChannelBlocks*maxIter);
- res[4].axes="ant,dir,freq,iter";
- res[4].dims.resize(4);
- res[4].dims[0]=_nAntennas;
- res[4].dims[1]=_nDirections;
- res[4].dims[2]=_nChannelBlocks;
- res[4].dims[3]=maxIter;
- res[4].name="phases";
- }
-
- //TODOEstimate Weights
-
-
-#pragma omp parallel for
- for(size_t antIndex = 0; antIndex<_nAntennas; ++antIndex)
- {
- int foundantcs=-999;
- int foundantoth=-999;
- if (itsMode=="csfull"){
- for(size_t iant=0; iant<_coreAntennas.size(); iant++){
- if (_coreAntennas[iant]==antIndex){
- foundantcs=iant;
- break;
- }
- }
- if (foundantcs<0)
- {
- for(size_t iant=0; iant<_otherAntennas.size(); iant++){
- if(_otherAntennas[iant]==antIndex){
- foundantoth=iant;
- break;
- }
- }
- }
- }
- for(size_t dirIndex = 0; dirIndex<_nDirections; ++dirIndex){
- double avgTEC,error;
- size_t solutionIndex=antIndex*_nDirections+dirIndex;
- if (itsDebugMode>0 and itsIter<maxIter)
- {
- for (size_t ch=0; ch<_nChannelBlocks;ch++)
- {
- //cout<<"writing "<<antIndex<<":"<<dirIndex<<":"<<ch<<":"<<itsIter<<":"<<antIndex*_nDirections*30*_nChannelBlocks+dirIndex*30*_nChannelBlocks+ch*30+itsIter<<" "<<res[4].vals.size()<<","<<solutionIndex<<":"<<solutions[ch].size()<<std::arg(solutions[ch][solutionIndex])<<endl;
- _iterphases[antIndex*_nDirections*maxIter*_nChannelBlocks+dirIndex*maxIter*_nChannelBlocks+ch*maxIter+itsIter]= std::arg(solutions[ch][solutionIndex]);
- }
- }
- getPPValue(solutions,solutionIndex,dirIndex,avgTEC,error);
- if(error<=0) error=1;
- if (itsMode=="station"){
- _screenFitters[antIndex].PhaseData()[dirIndex] = avgTEC;
- _screenFitters[antIndex].WData()[dirIndex] = 1./error;
- }
- else if (itsMode=="direction"){
- _screenFitters[dirIndex].PhaseData()[antIndex] = avgTEC;
- _screenFitters[dirIndex].WData()[antIndex] = 1./error;
- }
- else if (itsMode=="full"){
- _screenFitters[0].PhaseData()[dirIndex*_nAntennas+antIndex] = avgTEC;
- _screenFitters[0].WData()[dirIndex*_nAntennas+antIndex] = 1./error;
- }
- else
- {//csfull mode
- if (foundantcs>=0){
- _screenFitters[0].PhaseData()[foundantcs*_nDirections+dirIndex]= avgTEC;
- _screenFitters[0].WData()[foundantcs*_nDirections+dirIndex]= 1./error;
- }
- else if (foundantoth>=0){
- _screenFitters[foundantoth+1].PhaseData()[dirIndex]= avgTEC;
- _screenFitters[foundantoth+1].WData()[dirIndex]= 1./error;
- }
- }
- }
- }
-
-#pragma omp parallel for
- for(size_t isft=0;isft<_screenFitters.size();isft++)
- _screenFitters[isft].doFit();
-
-#pragma omp parallel for
- for(size_t antIndex = 0; antIndex<_nAntennas; ++antIndex)
- {
- int foundantcs=-999;
- int foundantoth=-999;
- if (itsMode=="csfull")
- {
- for(size_t iant=0; iant<_coreAntennas.size(); iant++){
- if (_coreAntennas[iant]==antIndex){
- foundantcs=iant;
- break;
- }
- }
- }
- if (foundantcs<0)
- {
- for(size_t iant=0; iant<_otherAntennas.size(); iant++){
- if(_otherAntennas[iant]==antIndex){
- foundantoth=iant;
- break;
- }
- }
- }
- for(size_t dirIndex = 0; dirIndex<_nDirections; ++dirIndex){
- size_t solutionIndex=antIndex*_nDirections+dirIndex;
- double avgTEC=0;
- if (itsMode=="station")
- avgTEC=_screenFitters[antIndex].PhaseData()[dirIndex];
- else if (itsMode=="direction")
- avgTEC=_screenFitters[dirIndex].PhaseData()[antIndex];
- else if (itsMode=="full")
- avgTEC=_screenFitters[0].PhaseData()[dirIndex*_nAntennas+antIndex];
- else
- {//csfull
- if (foundantcs>=0)
- avgTEC=_screenFitters[0].PhaseData()[foundantcs*_nDirections+dirIndex];
- else if (foundantoth>=0)
- avgTEC=_screenFitters[foundantoth+1].PhaseData()[dirIndex];
- }
-
- for(size_t ch=0;ch<_nChannelBlocks; ++ch)
- solutions[ch][solutionIndex] = std::polar<double>(1.0, avgTEC*TECtoph/itsFrequencies[ch]);
-
- res[3].vals[antIndex*_nDirections+dirIndex]= avgTEC;
- itsprevsol[antIndex*_nDirections+dirIndex]=avgTEC;
- for (size_t i=0;i<3;i++)
- {
- if (itsMode=="station")
- res[1].vals[antIndex*_nDirections*3+dirIndex*3+i]= _screenFitters[antIndex].PPData()[i*_nDirections+dirIndex];
- else if (itsMode=="direction")
- res[1].vals[antIndex*_nDirections*3+dirIndex*3+i]= _screenFitters[dirIndex].PPData()[i*_nAntennas+antIndex];
- else if (itsMode=="full")
- res[1].vals[antIndex*_nDirections*3+dirIndex*3+i]= _screenFitters[0].PPData()[i*_nDirections*_nAntennas+dirIndex*_nAntennas+antIndex];
-
- else
- {//csfull
- if (foundantcs>=0)
- res[1].vals[antIndex*_nDirections*3+dirIndex*3+i]= _screenFitters[0].PPData()[i*_coreAntennas.size()*_nDirections+foundantcs*_nDirections+dirIndex];
- else if (foundantoth>=0)
- res[1].vals[antIndex*_nDirections*3+dirIndex*3+i]= _screenFitters[foundantoth].PPData()[i*_nDirections+dirIndex];
- }
- }
- }
-
- for(size_t dirIndex = 0; dirIndex<_nDirections; ++dirIndex){
- if (itsMode=="station")
- res[2].vals[antIndex*_nDirections+dirIndex]=
- _screenFitters[antIndex].TECFitWhiteData()[dirIndex];
- else //not implemented yet for other modes
- res[2].vals[antIndex*_nDirections+dirIndex]=0;
- }
- }
- for(size_t i=0;i<_screenFitters.size();i++)
- for(size_t j=0;j<numberofPar;j++)
- res[0].vals[i*numberofPar+j]= _screenFitters[i].ParData()[j];
-
- if (itsDebugMode>0)
- res[4].vals=_iterphases;
- return res;
-}
-
-} //namespace LOFAR
diff --git a/CEP/DP3/DPPP_DDECal/src/SmoothnessConstraint.cc b/CEP/DP3/DPPP_DDECal/src/SmoothnessConstraint.cc
deleted file mode 100644
index a37f52305e129e834a162caa0d44e104447585fa..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/SmoothnessConstraint.cc
+++ /dev/null
@@ -1,77 +0,0 @@
-#ifdef AOPROJECT
-#include "KernelSmoother.h"
-#include "SmoothnessConstraint.h"
-#include "omptools.h"
-#else
-#include <DPPP_DDECal/KernelSmoother.h>
-#include <DPPP_DDECal/SmoothnessConstraint.h>
-#include <Common/OpenMP.h>
-#endif
-
-SmoothnessConstraint::SmoothnessConstraint(double bandwidthHz) :
- _kernelType(Smoother::GaussianKernel),
- _bandwidth(bandwidthHz)
-{ }
-
-void SmoothnessConstraint::Initialize(const double* frequencies)
-{
- _frequencies.assign(frequencies, frequencies+_nChannelBlocks);
- size_t nthreads =
-#ifdef AOPROJECT
- omp_get_max_threads();
-#else
- LOFAR::OpenMP::maxThreads();
-#endif
- for(size_t i=0; i!=nthreads; ++i)
- _fitData.emplace_back(_frequencies.data(), _frequencies.size(), _kernelType, _bandwidth);
-}
-
-void SmoothnessConstraint::InitializeDimensions(size_t nAntennas,
- size_t nDirections,
- size_t nChannelBlocks)
-{
- Constraint::InitializeDimensions(nAntennas, nDirections, nChannelBlocks);
-}
-
-std::vector<Constraint::Result> SmoothnessConstraint::Apply(
- std::vector<std::vector<dcomplex> >& solutions, double, std::ostream*)
-{
- const size_t nPol = solutions.front().size() / (_nAntennas*_nDirections);
-#pragma omp parallel for
- for(size_t antDirIndex = 0; antDirIndex<_nAntennas*_nDirections; ++antDirIndex)
- {
-#ifdef AOPROJECT
- const size_t thread = omp_get_thread_num();
-#else
- const size_t thread = LOFAR::OpenMP::threadNum();
-#endif
- size_t antIndex = antDirIndex / _nDirections;
- for(size_t pol = 0; pol!=nPol; ++pol)
- {
- size_t solutionIndex = antDirIndex*nPol + pol;
- for(size_t ch=0; ch!=_nChannelBlocks; ++ch)
- {
- // Flag channels where calibration yielded inf or nan
- if(std::isfinite(solutions[ch][solutionIndex].real()) &&
- std::isfinite(solutions[ch][solutionIndex].imag()))
- {
- _fitData[thread].data[ch] = solutions[ch][solutionIndex];
- _fitData[thread].weight[ch] = _weights[antIndex*_nChannelBlocks + ch];
- }
- else {
- _fitData[thread].data[ch] = 0.0;
- _fitData[thread].weight[ch] = 0.0;
- }
- }
-
- _fitData[thread].smoother.Smooth(_fitData[thread].data.data(), _fitData[thread].weight.data());
-
- for(size_t ch=0; ch!=_nChannelBlocks; ++ch)
- {
- solutions[ch][solutionIndex] = _fitData[thread].data[ch];
- }
- }
- }
-
- return std::vector<Constraint::Result>();
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/Stopwatch.cc b/CEP/DP3/DPPP_DDECal/src/Stopwatch.cc
deleted file mode 100644
index 2e39e06bcf2517240a51c54a8f1a37ec88247c44..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/Stopwatch.cc
+++ /dev/null
@@ -1,159 +0,0 @@
-#include <DPPP_DDECal/Stopwatch.h>
-
-#include <cmath>
-#include <sstream>
-
-#include <boost/date_time/posix_time/posix_time.hpp>
-
-Stopwatch::Stopwatch() : _running(false), _sum(boost::posix_time::seconds(0))
-{
-}
-
-Stopwatch::Stopwatch(bool start) :
- _running(start),
- _startTime(boost::posix_time::microsec_clock::local_time()),
- _sum(boost::posix_time::seconds(0))
-{
-}
-
-Stopwatch::~Stopwatch()
-{
-}
-
-void Stopwatch::Start()
-{
- if(!_running) {
- _startTime = boost::posix_time::microsec_clock::local_time();
- _running = true;
- }
-}
-
-void Stopwatch::Pause()
-{
- if(_running) {
- _sum += (boost::posix_time::microsec_clock::local_time() - _startTime);
- _running = false;
- }
-}
-
-void Stopwatch::Reset()
-{
- _running = false;
- _sum = boost::posix_time::seconds(0);
-}
-
-std::string Stopwatch::ToString() const
-{
- if(_running) {
- boost::posix_time::time_duration current = _sum + (boost::posix_time::microsec_clock::local_time() - _startTime);
- return to_simple_string(current);
- } else {
- return to_simple_string(_sum);
- }
-}
-
-std::string Stopwatch::ToShortString() const
-{
- const long double seconds = Seconds();
- if(seconds >= 60*60*24)
- return ToDaysString();
- else if(seconds >= 60*60)
- return ToHoursString();
- else if(seconds >= 60)
- return ToMinutesString();
- else if(seconds >= 1.0)
- return ToSecondsString();
- else if(seconds >= 0.001)
- return ToMilliSecondsString();
- else if(seconds >= 0.000001)
- return ToMicroSecondsString();
- else
- return ToNanoSecondsString();
-}
-
-std::string Stopwatch::ToDaysString() const
-{
- const long double days = roundl(Seconds() / (60.0*60.0))/24.0;
- std::stringstream str;
- if(days >= 10.0)
- str << roundl(days) << " days";
- else
- str << floorl(days) << 'd' << (days*24.0) << 'h';
- return str.str();
-}
-
-std::string Stopwatch::ToHoursString() const
-{
- const long double hours = roundl(Seconds() / 60.0)/60.0;
- std::stringstream str;
- if(hours >= 10.0)
- str << roundl(hours) << 'h';
- else
- str << floorl(hours) << 'h' << (hours*60.0) << 'm';
- return str.str();
-}
-
-std::string Stopwatch::ToMinutesString() const
-{
- const long double mins = roundl(Seconds())/60.0;
- std::stringstream str;
- if(mins >= 10.0)
- str << roundl(mins) << " min";
- else
- str << floorl(mins) << 'm' << fmod(mins*60.0,60.0) << 's';
- return str.str();
-}
-
-std::string Stopwatch::ToSecondsString() const
-{
- const long double seconds = roundl(Seconds()*10.0)/10.0;
- std::stringstream str;
- if(seconds >= 10.0)
- str << roundl(Seconds()) << 's';
- else
- str << seconds << 's';
- return str.str();
-}
-
-std::string Stopwatch::ToMilliSecondsString() const
-{
- const long double msec = roundl(Seconds()*10000.0)/10.0;
- std::stringstream str;
- if(msec >= 10.0)
- str << roundl(Seconds()*1000.0) << "ms";
- else
- str << msec << "ms";
- return str.str();
-}
-
-std::string Stopwatch::ToMicroSecondsString() const
-{
- const long double usec = roundl(Seconds()*10000000.0)/10.0;
- std::stringstream str;
- if(usec >= 10.0)
- str << roundl(Seconds()*1000000.0) << "µs";
- else
- str << usec << "µs";
- return str.str();
-}
-
-std::string Stopwatch::ToNanoSecondsString() const
-{
- const long double nsec = roundl(Seconds()*10000000000.0)/10.0;
- std::stringstream str;
- if(nsec >= 10.0)
- str << roundl(Seconds()*1000000000.0) << "ns";
- else
- str << nsec << "ns";
- return str.str();
-}
-
-long double Stopwatch::Seconds() const
-{
- if(_running) {
- boost::posix_time::time_duration current = _sum + (boost::posix_time::microsec_clock::local_time() - _startTime);
- return (long double) current.total_microseconds()/1000000.0;
- } else {
- return (long double) _sum.total_microseconds()/1000000.0;
- }
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/TECConstraint.cc b/CEP/DP3/DPPP_DDECal/src/TECConstraint.cc
deleted file mode 100644
index c19df7ee953f3dc49aed71921030910325aacaad..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/TECConstraint.cc
+++ /dev/null
@@ -1,223 +0,0 @@
-#ifdef AOPROJECT
-#include "TECConstraint.h"
-#include "omptools.h"
-#else
-#include <DPPP_DDECal/TECConstraint.h>
-#include <Common/OpenMP.h>
-#endif
-
-TECConstraintBase::TECConstraintBase(Mode mode) :
- _mode(mode),
- _phaseFitters()
-{
-}
-
-void TECConstraintBase::initialize(const double* frequencies) {
- _phaseFitters.resize(
-#ifdef AOPROJECT
- omp_get_max_threads()
-#else
- LOFAR::OpenMP::maxThreads()
-#endif
- );
-
- for(size_t i=0; i!=_phaseFitters.size(); ++i)
- {
- _phaseFitters[i].SetChannelCount(_nChannelBlocks);
- std::memcpy(_phaseFitters[i].FrequencyData(), frequencies,
- sizeof(double) * _nChannelBlocks);
- }
- _weights.assign(_nChannelBlocks*_nAntennas, 1.0);
- initializeChild();
-}
-
-void TECConstraintBase::SetWeights(const std::vector<double>& weights) {
- _weights = weights;
-}
-
-void ApproximateTECConstraint::initializeChild()
-{
- _pwFitters.resize(
-#ifdef AOPROJECT
- omp_get_max_threads()
-#else
- LOFAR::OpenMP::maxThreads()
-#endif
- );
- _threadData.resize(_pwFitters.size());
- _threadFittedData.resize(_pwFitters.size());
- _threadWeights.resize(_pwFitters.size());
- for(size_t threadId=0; threadId!=_pwFitters.size(); ++threadId)
- {
- _threadData[threadId].resize(_nChannelBlocks);
- _threadFittedData[threadId].resize(_nChannelBlocks);
- _threadWeights[threadId].resize(_nChannelBlocks);
- }
-
- if(_fittingChunkSize == 0)
- {
- size_t
- n = _phaseFitters.front().Size();
- const double
- startFreq = _phaseFitters.front().FrequencyData()[0],
- endFreq = _phaseFitters.front().FrequencyData()[n-1];
- _fittingChunkSize = PieceWisePhaseFitter::CalculateChunkSize(startFreq, endFreq, n);
- }
- for(size_t i=0; i!=_pwFitters.size(); ++i)
- _pwFitters[i].SetChunkSize(_fittingChunkSize);
-}
-
-void TECConstraintBase::applyReferenceAntenna(std::vector<std::vector<dcomplex> >& solutions) const
-{
- // TODO chose this more cleverly?
- size_t refAntenna = 0;
-
- for(size_t ch=0; ch!=_nChannelBlocks; ++ch)
- {
- for(size_t antennaIndex=0; antennaIndex!=_nAntennas; ++antennaIndex)
- {
- for(size_t d=0; d!=_nDirections; ++d)
- {
- size_t solutionIndex = antennaIndex*_nDirections + d;
- size_t refAntennaIndex = d + refAntenna*_nDirections;
- if(antennaIndex != refAntenna)
- {
- solutions[ch][solutionIndex] = solutions[ch][solutionIndex] / solutions[ch][refAntennaIndex];
- }
- }
- }
- for(size_t d=0; d!=_nDirections; ++d)
- solutions[ch][refAntenna*_nDirections + d] = 1.0;
- }
-}
-
-std::vector<Constraint::Result> TECConstraint::Apply(
- std::vector<std::vector<dcomplex> >& solutions, double,
- std::ostream* /*statStream*/)
-{
- size_t nRes = 3;
- if(_mode == TECOnlyMode) {
- nRes = 2; // TEC and error
- }
- else {
- nRes = 3; // TEC, phase and error
- }
-
- std::vector<Constraint::Result> res(nRes);
- res[0].vals.resize(_nAntennas*_nDirections);
- res[0].weights.resize(_nAntennas*_nDirections);
- res[0].axes="ant,dir,freq";
- res[0].name="tec";
- res[0].dims.resize(3);
- res[0].dims[0]=_nAntennas;
- res[0].dims[1]=_nDirections;
- res[0].dims[2]=1;
- if(_mode == TECAndCommonScalarMode) {
- res[1]=res[0];
- res[1].name="phase";
- }
- res.back()=res[0];
- res.back().name="error";
-
- // Divide out the reference antenna
- applyReferenceAntenna(solutions);
-
-#pragma omp parallel for
- for(size_t solutionIndex = 0; solutionIndex<_nAntennas*_nDirections; ++solutionIndex)
- {
- size_t antennaIndex = solutionIndex/_nDirections;
- size_t thread =
-#ifdef AOPROJECT
- omp_get_thread_num();
-#else
- LOFAR::OpenMP::threadNum();
-#endif
-
- // Flag channels where calibration yielded inf or nan
- double weightSum = 0.0;
- for(size_t ch=0; ch!=_nChannelBlocks; ++ch) {
- if(std::isfinite(solutions[ch][solutionIndex].real()) &&
- std::isfinite(solutions[ch][solutionIndex].imag()))
- {
- _phaseFitters[thread].PhaseData()[ch] = std::arg(solutions[ch][solutionIndex]);
- _phaseFitters[thread].WeightData()[ch] = _weights[antennaIndex*_nChannelBlocks + ch];
- weightSum += _weights[antennaIndex*_nChannelBlocks + ch];
- }
- else {
- _phaseFitters[thread].PhaseData()[ch] = 0.0;
- _phaseFitters[thread].WeightData()[ch] = 0.0;
- }
- }
-
- double alpha, beta=0.0;
- if(_mode == TECOnlyMode) {
- res.back().vals[solutionIndex]=_phaseFitters[thread].FitDataToTEC1Model(alpha);
- } else {
- res.back().vals[solutionIndex]=_phaseFitters[thread].FitDataToTEC2Model(alpha, beta);
- }
- res.back().weights[solutionIndex] = weightSum;
-
- res[0].vals[solutionIndex] = alpha / -8.44797245e9;
- res[0].weights[solutionIndex] = weightSum;
- if(_mode == TECAndCommonScalarMode) {
- res[1].vals[solutionIndex] = beta;
- res[1].weights[solutionIndex] = weightSum;
- }
-
- for(size_t ch=0; ch!=_nChannelBlocks; ++ch)
- {
- solutions[ch][solutionIndex] = std::polar<double>(1.0, _phaseFitters[thread].PhaseData()[ch]);
- }
- }
-
- return res;
-}
-
-std::vector<Constraint::Result> ApproximateTECConstraint::Apply(
- std::vector<std::vector<dcomplex> >& solutions, double time,
- std::ostream* statStream)
-{
- if(_finishedApproximateStage)
- return TECConstraint::Apply(solutions, time, statStream);
- else {
- applyReferenceAntenna(solutions);
-
-#pragma omp parallel for
- for(size_t solutionIndex = 0; solutionIndex<_nAntennas*_nDirections; ++solutionIndex)
- {
- size_t antennaIndex = solutionIndex/_nDirections;
-#ifdef AOPROJECT
- size_t thread = omp_get_thread_num();
-#else
- size_t thread = LOFAR::OpenMP::threadNum();
-#endif
- std::vector<double>& data = _threadData[thread];
- std::vector<double>& fittedData = _threadFittedData[thread];
- std::vector<double>& weights = _threadWeights[thread];
-
- // Flag channels where calibration yielded inf or nan
- for(size_t ch=0; ch!=_nChannelBlocks; ++ch) {
- if(std::isfinite(solutions[ch][solutionIndex].real()) &&
- std::isfinite(solutions[ch][solutionIndex].imag()))
- {
- data[ch] = std::arg(solutions[ch][solutionIndex]);
- weights[ch] = _weights[antennaIndex*_nChannelBlocks + ch];
- }
- else {
- data[ch] = 0.0;
- weights[ch] = 0.0;
- }
- }
-
- // TODO might be nice to make it a user option whether to break or not
- _pwFitters[thread].SlidingFitWithBreak(_phaseFitters[thread].FrequencyData(), data.data(), weights.data(), fittedData.data(), data.size());
-
- for(size_t ch=0; ch!=_nChannelBlocks; ++ch)
- {
- solutions[ch][solutionIndex] = std::polar<double>(1.0, fittedData[ch]);
- }
- }
-
- return std::vector<Constraint::Result>();
- }
-}
diff --git a/CEP/DP3/DPPP_DDECal/src/screenfitter.cc b/CEP/DP3/DPPP_DDECal/src/screenfitter.cc
deleted file mode 100644
index 18563530647b9db9b0169e479a1bfb90f8aed967..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/src/screenfitter.cc
+++ /dev/null
@@ -1,8 +0,0 @@
-
-#include <DPPP/screenfitter.h>
-
-ScreenFitter::ScreenFitter():
- _phases(),
- _frequencies(),
- _weights()
-{ }
diff --git a/CEP/DP3/DPPP_DDECal/test/CMakeLists.txt b/CEP/DP3/DPPP_DDECal/test/CMakeLists.txt
deleted file mode 100644
index deaf0ee440dcafff28e0192c0ea72e63ca0b9c56..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/test/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# $Id: CMakeLists.txt 38047 2017-07-26 12:49:53Z dijkema $
-
-# Run script to find current directory (needed for getting a tarball from
-# the source directory).
-configure_file(findenv.run_tmpl findenv.run_script)
-
-include(LofarCTest)
-
-lofar_add_test(tDDECal)
-lofar_add_test(tRotationConstraint tRotationConstraint.cc)
diff --git a/CEP/DP3/DPPP_DDECal/test/findenv.run_tmpl b/CEP/DP3/DPPP_DDECal/test/findenv.run_tmpl
deleted file mode 100644
index ca6b5eaf16fdc7f764abd8c53447fa495523cdb9..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/test/findenv.run_tmpl
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# Find taql program.
-taqlexe=@TAQL_EXECUTABLE@
-# Find srcdir in the runctest file.
-if test -f runctest.sh; then
- rt_srcdir=`grep 'srcdir=' runctest.sh | sed -e 's/srcdir="//' -e 's/";.*//'`
-fi
diff --git a/CEP/DP3/DPPP_DDECal/test/tDDECal.in_MS.tgz b/CEP/DP3/DPPP_DDECal/test/tDDECal.in_MS.tgz
deleted file mode 100644
index bbaf4898f3f8483129e2be6a59d00b67e1194f3e..0000000000000000000000000000000000000000
Binary files a/CEP/DP3/DPPP_DDECal/test/tDDECal.in_MS.tgz and /dev/null differ
diff --git a/CEP/DP3/DPPP_DDECal/test/tDDECal.run b/CEP/DP3/DPPP_DDECal/test/tDDECal.run
deleted file mode 100755
index 36569aa3d6143fe6ec0479e59236ac4bc261fde3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/test/tDDECal.run
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/bin/bash
-
-# Get the taql executable and srcdir (script created by cmake's CONFIGURE_FILE).
-set -e
-source findenv.run_script
-echo "srcdirx=$rt_srcdir"
-
-# The lofar cmake logic for finding shared libraries at test time does not
-# work well on Darwin, work around this
-if [[ $(uname) == 'Darwin' ]]; then
- DYLD_LIBRARY_PATH=$(realpath ../../../../lib):$DYLD_LIBRARY_PATH
- export DYLD_LIBRARY_PATH
- echo $DYLD_LIBRARY_PATH
-fi
-
-if test ! -f ${srcdir}/tDDECal.in_MS.tgz; then
- exit 3 # untested
-fi
-
-mkdir -p tDDECal_tmp
-# Unpack the MS and other files and do the DPPP run.
-cd tDDECal_tmp
-tar zxf ${srcdir}/tDDECal.in_MS.tgz
-
-echo "Running tDDECal"
-
-$taqlexe 'update tDDECal.MS set WEIGHT_SPECTRUM=1, FLAG=False'
-
-# Create expected taql output.
-echo " select result of 0 rows" > taql.ref
-
-echo "Predict corrupted visibilities"
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. msout.datacolumn=DATA\
- steps=[predict] predict.sourcedb=tDDECal.MS/sky_corrupted"
-echo $cmd
-$cmd >& /dev/null
-
-echo "Predict model data column"
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. msout.datacolumn=MODEL_DATA\
- steps=[]"
-echo $cmd
-$cmd >& /dev/null
-
-for caltype in complexgain scalarcomplexgain amplitudeonly scalaramplitude
-do
- for solint in 0 1 2 4
- do
- for nchan in 1 2 5
- do
- echo "Calibrate on the original sources, caltype=$caltype"
- cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. steps=[ddecal]\
- ddecal.sourcedb=tDDECal.MS/sky ddecal.solint=$solint ddecal.nchan=$nchan \
- ddecal.directions=[[center,dec_off],[ra_off],[radec_off]] \
- ddecal.h5parm=instrument.h5 ddecal.mode=$caltype"
- echo $cmd
- $cmd
-
- echo "Apply solutions with multiple predict steps, caltype=$caltype"
- cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. msout.datacolumn=SUBTRACTED_DATA\
- steps=[predict1,predict2,predict3]\
- predict1.sourcedb=tDDECal.MS/sky\
- predict1.applycal.parmdb=instrument.h5 predict1.sources=[center,dec_off]\
- predict1.operation=subtract predict1.applycal.correction=amplitude000 \
- predict2.sourcedb=tDDECal.MS/sky\
- predict2.applycal.parmdb=instrument.h5 predict2.sources=[radec_off]\
- predict2.operation=subtract predict2.applycal.correction=amplitude000 \
- predict3.sourcedb=tDDECal.MS/sky\
- predict3.applycal.parmdb=instrument.h5 predict3.sources=[ra_off]\
- predict3.operation=subtract predict3.applycal.correction=amplitude000"
- echo $cmd
- $cmd
-
- #h5sols.py instrument.h5
-
- echo "Check that residual is small, caltype=$caltype, nchan=$nchan, solint=$solint"
- cmd="$taqlexe 'select norm_residual/norm_data FROM (select sqrt(abs(gsumsqr(WEIGHT_SPECTRUM*DATA))) as norm_data, sqrt(abs(gsumsqr(WEIGHT_SPECTRUM*SUBTRACTED_DATA))) as norm_residual from tDDECal.MS)'"
- echo $cmd
- eval $cmd
-
- cmd="$taqlexe 'select FROM (select sqrt(abs(gsumsqr(WEIGHT_SPECTRUM*DATA))) as norm_data, sqrt(abs(gsumsqr(WEIGHT_SPECTRUM*SUBTRACTED_DATA))) as norm_residual from tDDECal.MS) where norm_residual/norm_data > 0.015 or isinf(norm_residual/norm_data) or isnan(norm_residual/norm_data)' > taql.out"
- echo $cmd
- eval $cmd
-
- diff taql.out taql.ref || exit 1
- done # Loop over nchan
- done # Loop over solint
-done # Loop over caltype
-
-echo "Apply solutions with h5parmpredict"
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. msout.datacolumn=SUBTRACTED_DATA_H5PARM\
- steps=[predict]\
- predict.type=h5parmpredict\
- predict.sourcedb=tDDECal.MS/sky\
- predict.applycal.parmdb=instrument.h5\
- predict.operation=subtract predict.applycal.correction=amplitude000"
-echo $cmd
-$cmd
-
-echo "Check that h5parmpredict creates the same output as multiple predict steps"
-cmd="$taqlexe 'select abs(gsumsqr(SUBTRACTED_DATA-SUBTRACTED_DATA)) as diff from tDDECal.MS'"
-echo $cmd
-eval $cmd
-cmd="$taqlexe 'select from (select abs(gsumsqr(SUBTRACTED_DATA-SUBTRACTED_DATA)) as diff from tDDECal.MS) where diff>1.e-6' > taql.out"
-echo $cmd
-eval $cmd
-diff taql.out taql.ref || exit 1
-
-echo "Check pre-apply"
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. numthreads=4\
- steps=[ddecal]\
- ddecal.sourcedb=tDDECal.MS/sky\
- ddecal.directions=[[center,dec_off],[ra_off],[radec_off]]\
- ddecal.applycal.parmdb=instrument.h5 ddecal.applycal.steps=applyampl\
- ddecal.applycal.applyampl.correction=amplitude000\
- ddecal.h5parm=instrument2.h5 ddecal.mode=scalarcomplexgain"
-echo $cmd
-$cmd
-
-echo "Check tec"
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. numthreads=4\
- steps=[ddecal]\
- ddecal.sourcedb=tDDECal.MS/sky\
- ddecal.directions=[[center,dec_off],[ra_off],[radec_off]]\
- ddecal.h5parm=instrument-tec.h5 ddecal.mode=tec"
-echo $cmd
-$cmd
-
-echo "Check tec and phase"
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. numthreads=4\
- steps=[ddecal]\
- ddecal.sourcedb=tDDECal.MS/sky\
- ddecal.directions=[[center,dec_off],[ra_off],[radec_off]]\
- ddecal.h5parm=instrument-tecandphase.h5 ddecal.mode=tecandphase"
-echo $cmd
-$cmd
-
-echo "Create MODEL_DATA"
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. msout.datacolumn=MODEL_DATA\
- steps=[]"
-echo $cmd
-$cmd >& /dev/null
-cmd="NDPPP checkparset=1 msin=tDDECal.MS msout=. steps=[ddecal]\
- ddecal.usemodelcolumn=true ddecal.h5parm=instrument-modeldata \
- ddecal.solint=2 ddecal.nchan=3"
-echo $cmd
-$cmd
diff --git a/CEP/DP3/DPPP_DDECal/test/tDDECal.sh b/CEP/DP3/DPPP_DDECal/test/tDDECal.sh
deleted file mode 100755
index c6f56dae9703623be7973f13a501161a47198a57..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/test/tDDECal.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tDDECal
diff --git a/CEP/DP3/DPPP_DDECal/test/tDDECal_ref b/CEP/DP3/DPPP_DDECal/test/tDDECal_ref
deleted file mode 100755
index b052afc7813889d9e06e14c6cec30b489e21636d..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/test/tDDECal_ref
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-cat << EOF > tDDECal_tmp.skymodel
-FORMAT = Name, Type, Ra, Dec, I
-
-radec_off, POINT, 01:33:40, +32.09.0, 10
-ra_off, POINT, 01:29:40, +33.09.35.13240, 20
-dec_off, POINT, 01:37:41.299440, +32.09.0, 20
-center, POINT, 01:37:41.299440, +33.09.35.13240, 10
-EOF
-
-# Corruptions:
-# [center, dec_off] : 5
-# [radec_off] : 13
-# [ra_off] : 9
-
-cat << EOF > tDDECal_tmp_corrupted.skymodel
-FORMAT = Name, Type, Ra, Dec, I
-
-radec_off, POINT, 01:33:40, +32.09.0, 130
-ra_off, POINT, 01:29:40, +33.09.35.13240, 180
-dec_off, POINT, 01:37:41.299440, +32.09.0, 100
-center, POINT, 01:37:41.299440, +33.09.35.13240, 50
-EOF
-
-rm -rf tDDECal.MS/sky
-rm -rf tDDECal.MS/sky_corrupted
-makesourcedb in=tDDECal_tmp.skymodel out=tDDECal.MS/sky format='<'
-makesourcedb in=tDDECal_tmp_corrupted.skymodel out=tDDECal.MS/sky_corrupted format='<'
-rm tDDECal_tmp.skymodel
-rm tDDECal_tmp_corrupted.skymodel
-
-tar czf tDDECal.in_MS.tgz tDDECal.MS
diff --git a/CEP/DP3/DPPP_DDECal/test/tRotationConstraint.cc b/CEP/DP3/DPPP_DDECal/test/tRotationConstraint.cc
deleted file mode 100644
index 5d629d526467425b8e1fd273806fbe2578e34601..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/test/tRotationConstraint.cc
+++ /dev/null
@@ -1,101 +0,0 @@
-#include <lofar_config.h>
-#include <Common/LofarLogger.h> // ASSERT
-#include <casacore/casa/BasicMath/Math.h> // near
-
-#include <vector>
-#include <iostream>
-#include <complex>
-
-#include <DPPP_DDECal/RotationConstraint.h>
-#include <DPPP_DDECal/RotationAndDiagonalConstraint.h>
-
-using namespace std;
-using namespace casacore;
-using namespace LOFAR;
-
-void test_rotation() {
- RotationConstraint constraint;
- constraint.InitializeDimensions(1, 1, 1);
-
- vector<vector<complex<double> > > onesolution(1);
- onesolution[0].resize(4);
- double pi = 3.1415;
- for (double phi=-pi; phi<pi; phi+=pi/6) {
- cout<<"test phi = "<<phi<<endl;
- /* Solution is of the form ((a,0),(0,b))*rot(phi)
- with rot(phi) = ((cos(phi),-sin(phi)),(sin(phi),cos(phi)))
- */
- onesolution[0][0] = cos(phi);
- onesolution[0][1] = -sin(phi);
- onesolution[0][2] = sin(phi);
- onesolution[0][3] = cos(phi);
-
- vector<Constraint::Result> constraint_result;
- constraint_result = constraint.Apply(onesolution, 0., nullptr);
-
- ASSERT( constraint_result.size() == 1 );
- ASSERT( constraint_result[0].axes == "ant,freq" );
- cout<<" got phi = "<<constraint_result[0].vals[0] <<endl;
- ASSERT( near(constraint_result[0].vals[0], phi) );
- ASSERT( constraint_result[0].name == "rotation" );
- ASSERT( constraint_result[0].dims.size() == 2 );
- ASSERT( constraint_result[0].dims[0] == 1 );
- ASSERT( constraint_result[0].dims[1] == 1 );
- }
-}
-
-void test_rotation_and_diagonal() {
- RotationAndDiagonalConstraint constraint;
- constraint.InitializeDimensions(1, 1, 1);
-
- vector<vector<complex<double> > > onesolution(1);
- onesolution[0].resize(4);
- double pi = 3.1415;
- double phi= pi/6;
- const complex<double> i(0,1.);
- complex<double> a=2.*exp(i*0.3), b=3.*exp(i*-0.2);
-
- /* Solution is of the form ((a,0),(0,b))*rot(phi)
- with rot(phi) = ((cos(phi),-sin(phi)),(sin(phi),cos(phi)))
- */
- onesolution[0][0] = a * cos(phi);
- onesolution[0][1] = a *-sin(phi);
- onesolution[0][2] = b * sin(phi);
- onesolution[0][3] = b * cos(phi);
-
- vector<Constraint::Result> constraint_result;
- constraint_result = constraint.Apply(onesolution, 0., nullptr);
-
- ASSERT( constraint_result.size() == 3 );
- ASSERT( constraint_result[0].name == "rotation" );
- ASSERT( constraint_result[0].axes == "ant,freq" );
- ASSERT( near(constraint_result[0].vals[0], 0.) );
- ASSERT( constraint_result[0].dims.size() == 2 );
- ASSERT( constraint_result[0].dims[0] == 1 );
- ASSERT( constraint_result[0].dims[1] == 1 );
-
- ASSERT( constraint_result[1].name == "amplitude" );
- ASSERT( constraint_result[1].axes == "ant,freq,pol" );
- ASSERT( near(constraint_result[1].vals[0], abs(a)) );
- ASSERT( near(constraint_result[1].vals[1], abs(b)) );
- ASSERT( constraint_result[1].dims.size() == 3 );
- ASSERT( constraint_result[1].dims[0] == 1 );
- ASSERT( constraint_result[1].dims[1] == 1 );
- ASSERT( constraint_result[1].dims[2] == 2 );
-
- ASSERT( constraint_result[2].name == "phase" );
- ASSERT( constraint_result[2].axes == "ant,freq,pol" );
- ASSERT( near(constraint_result[2].vals[0], arg(a)) );
- ASSERT( near(constraint_result[2].vals[1], arg(b)) );
- ASSERT( constraint_result[2].dims.size() == 3 );
- ASSERT( constraint_result[2].dims[0] == 1 );
- ASSERT( constraint_result[2].dims[1] == 1 );
- ASSERT( constraint_result[2].dims[2] == 2 );
-}
-
-int main(int, char**) {
- test_rotation();
- test_rotation_and_diagonal();
-
- return 0;
-}
diff --git a/CEP/DP3/DPPP_DDECal/test/tRotationConstraint.sh b/CEP/DP3/DPPP_DDECal/test/tRotationConstraint.sh
deleted file mode 100755
index b0777e425ca64e4eaeb7674acfe90633f9c61833..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_DDECal/test/tRotationConstraint.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tRotationConstraint
diff --git a/CEP/DP3/DPPP_Interpolate/CMake/CheckCXXSymbolExists.cmake b/CEP/DP3/DPPP_Interpolate/CMake/CheckCXXSymbolExists.cmake
deleted file mode 100644
index 084fbb422f5a607b4083e5d7108aefbf90d9aa3c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/CMake/CheckCXXSymbolExists.cmake
+++ /dev/null
@@ -1,49 +0,0 @@
-#.rst:
-# CheckCXXSymbolExists
-# --------------------
-#
-# Check if a symbol exists as a function, variable, or macro in C++
-#
-# CHECK_CXX_SYMBOL_EXISTS(<symbol> <files> <variable>)
-#
-# Check that the <symbol> is available after including given header
-# <files> and store the result in a <variable>. Specify the list of
-# files in one argument as a semicolon-separated list.
-# CHECK_CXX_SYMBOL_EXISTS() can be used to check in C++ files, as
-# opposed to CHECK_SYMBOL_EXISTS(), which works only for C.
-#
-# If the header files define the symbol as a macro it is considered
-# available and assumed to work. If the header files declare the symbol
-# as a function or variable then the symbol must also be available for
-# linking. If the symbol is a type or enum value it will not be
-# recognized (consider using CheckTypeSize or CheckCSourceCompiles).
-#
-# The following variables may be set before calling this macro to modify
-# the way the check is run:
-#
-# ::
-#
-# CMAKE_REQUIRED_FLAGS = string of compile command line flags
-# CMAKE_REQUIRED_DEFINITIONS = list of macros to define (-DFOO=bar)
-# CMAKE_REQUIRED_INCLUDES = list of include directories
-# CMAKE_REQUIRED_LIBRARIES = list of libraries to link
-# CMAKE_REQUIRED_QUIET = execute quietly without messages
-
-#=============================================================================
-# Copyright 2003-2011 Kitware, Inc.
-#
-# Distributed under the OSI-approved BSD License (the "License");
-# see accompanying file Copyright.txt for details.
-#
-# This software is distributed WITHOUT ANY WARRANTY; without even the
-# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the License for more information.
-#=============================================================================
-# (To distribute this file outside of CMake, substitute the full
-# License text for the above reference.)
-
-include(CheckSymbolExists)
-
-macro(CHECK_CXX_SYMBOL_EXISTS SYMBOL FILES VARIABLE)
- _CHECK_SYMBOL_EXISTS("${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/CheckSymbolExists.cxx" "${SYMBOL}" "${FILES}" "${VARIABLE}" )
-endmacro()
diff --git a/CEP/DP3/DPPP_Interpolate/CMake/FindCFITSIO.cmake b/CEP/DP3/DPPP_Interpolate/CMake/FindCFITSIO.cmake
deleted file mode 100644
index 6501f7762ca300f4ec488a0adf66e013723cfc5c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/CMake/FindCFITSIO.cmake
+++ /dev/null
@@ -1,49 +0,0 @@
-# - Try to find CFITSIO.
-# Variables used by this module:
-# CFITSIO_ROOT_DIR - CFITSIO root directory
-# Variables defined by this module:
-# CFITSIO_FOUND - system has CFITSIO
-# CFITSIO_INCLUDE_DIR - the CFITSIO include directory (cached)
-# CFITSIO_INCLUDE_DIRS - the CFITSIO include directories
-# (identical to CFITSIO_INCLUDE_DIR)
-# CFITSIO_LIBRARY - the CFITSIO library (cached)
-# CFITSIO_LIBRARIES - the CFITSIO libraries
-# (identical to CFITSIO_LIBRARY)
-
-# Copyright (C) 2009
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: FindCFITSIO.cmake 22498 2012-10-23 10:51:12Z loose $
-
-if(NOT CFITSIO_FOUND)
-
- find_path(CFITSIO_INCLUDE_DIR fitsio.h
- HINTS ${CFITSIO_ROOT_DIR} PATH_SUFFIXES include include/cfitsio include/libcfitsio0)
- find_library(CFITSIO_LIBRARY cfitsio
- HINTS ${CFITSIO_ROOT_DIR} PATH_SUFFIXES lib)
- find_library(M_LIBRARY m)
- mark_as_advanced(CFITSIO_INCLUDE_DIR CFITSIO_LIBRARY M_LIBRARY)
-
- include(FindPackageHandleStandardArgs)
- find_package_handle_standard_args(CFITSIO DEFAULT_MSG
- CFITSIO_LIBRARY M_LIBRARY CFITSIO_INCLUDE_DIR)
-
- set(CFITSIO_INCLUDE_DIRS ${CFITSIO_INCLUDE_DIR})
- set(CFITSIO_LIBRARIES ${CFITSIO_LIBRARY} ${M_LIBRARY})
-
-endif(NOT CFITSIO_FOUND)
diff --git a/CEP/DP3/DPPP_Interpolate/CMake/FindCasacore.cmake b/CEP/DP3/DPPP_Interpolate/CMake/FindCasacore.cmake
deleted file mode 100644
index e4689f75102cc9e72847ed03ac2e71bc7ded82c7..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/CMake/FindCasacore.cmake
+++ /dev/null
@@ -1,267 +0,0 @@
-# - Try to find Casacore include dirs and libraries
-# Usage:
-# find_package(Casacore [REQUIRED] [COMPONENTS components...])
-# Valid components are:
-# casa, coordinates, derivedmscal, fits, images, lattices,
-# meas, measures, mirlib, ms, msfits, python, scimath, scimath_f, tables
-#
-# Note that most components are dependent on other (more basic) components.
-# In that case, it suffices to specify the "top-level" components; dependent
-# components will be searched for automatically.
-#
-# The dependency tree can be generated using the script get_casacore_deps.sh.
-# For this, you need to have a complete casacore installation, built with shared
-# libraries, at your disposal.
-#
-# The dependencies in this macro were generated against casacore release 1.7.0.
-#
-# Variables used by this module:
-# CASACORE_ROOT_DIR - Casacore root directory.
-#
-# Variables defined by this module:
-# CASACORE_FOUND - System has Casacore, which means that the
-# include dir was found, as well as all
-# libraries specified (not cached)
-# CASACORE_INCLUDE_DIR - Casacore include directory (cached)
-# CASACORE_INCLUDE_DIRS - Casacore include directories (not cached)
-# identical to CASACORE_INCLUDE_DIR
-# CASACORE_LIBRARIES - The Casacore libraries (not cached)
-# CASA_${COMPONENT}_LIBRARY - The absolute path of Casacore library
-# "component" (cached)
-# HAVE_AIPSPP - True if system has Casacore (cached)
-# for backward compatibility with AIPS++
-# HAVE_CASACORE - True if system has Casacore (cached)
-# identical to CASACORE_FOUND
-# TAQL_EXECUTABLE - The absolute path of the TaQL executable
-# (cached)
-#
-# ATTENTION: The component names need to be in lower case, just as the
-# casacore library names. However, the CMake variables use all upper case.
-
-# Copyright (C) 2009
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: FindCasacore.cmake 31487 2015-04-16 11:28:17Z dijkema $
-
-# - casacore_resolve_dependencies(_result)
-#
-# Resolve the Casacore library dependencies for the given components.
-# The list of dependent libraries will be returned in the variable result.
-# It is sorted from least dependent to most dependent library, so it can be
-# directly fed to the linker.
-#
-# Usage: casacore_resolve_dependencies(result components...)
-#
-macro(casacore_resolve_dependencies _result)
- set(${_result} ${ARGN})
- set(_index 0)
- # Do a breadth-first search through the dependency graph; append to the
- # result list the dependent components for each item in that list.
- # Duplicates will be removed later.
- while(1)
- list(LENGTH ${_result} _length)
- if(NOT _index LESS _length)
- break()
- endif(NOT _index LESS _length)
- list(GET ${_result} ${_index} item)
- list(APPEND ${_result} ${Casacore_${item}_DEPENDENCIES})
- math(EXPR _index "${_index}+1")
- endwhile(1)
- # Remove all duplicates in the current result list, while retaining only the
- # last of each duplicate.
- list(REVERSE ${_result})
- list(REMOVE_DUPLICATES ${_result})
- list(REVERSE ${_result})
-endmacro(casacore_resolve_dependencies _result)
-
-
-# - casacore_find_library(_name)
-#
-# Search for the library ${_name}.
-# If library is found, add it to CASACORE_LIBRARIES; if not, add ${_name}
-# to CASACORE_MISSING_COMPONENTS and set CASACORE_FOUND to false.
-#
-# Usage: casacore_find_library(name)
-#
-macro(casacore_find_library _name)
- string(TOUPPER ${_name} _NAME)
- find_library(${_NAME}_LIBRARY ${_name}
- HINTS ${CASACORE_ROOT_DIR} PATH_SUFFIXES lib)
- mark_as_advanced(${_NAME}_LIBRARY)
- if(${_NAME}_LIBRARY)
- list(APPEND CASACORE_LIBRARIES ${${_NAME}_LIBRARY})
- else(${_NAME}_LIBRARY)
- set(CASACORE_FOUND FALSE)
- list(APPEND CASACORE_MISSING_COMPONENTS ${_name})
- endif(${_NAME}_LIBRARY)
-endmacro(casacore_find_library _name)
-
-
-# - casacore_find_package(_name)
-#
-# Search for the package ${_name}.
-# If the package is found, add the contents of ${_name}_INCLUDE_DIRS to
-# CASACORE_INCLUDE_DIRS and ${_name}_LIBRARIES to CASACORE_LIBRARIES.
-#
-# If Casacore itself is required, then, strictly speaking, the packages it
-# requires must be present. However, when linking against static libraries
-# they may not be needed. One can override the REQUIRED setting by switching
-# CASACORE_MAKE_REQUIRED_EXTERNALS_OPTIONAL to ON. Beware that this might cause
-# compile and/or link errors.
-#
-# Usage: casacore_find_package(name [REQUIRED])
-#
-macro(casacore_find_package _name)
- if("${ARGN}" MATCHES "^REQUIRED$" AND
- Casacore_FIND_REQUIRED AND
- NOT CASACORE_MAKE_REQUIRED_EXTERNALS_OPTIONAL)
- find_package(${_name} REQUIRED)
- else()
- find_package(${_name})
- endif()
- if(${_name}_FOUND)
- list(APPEND CASACORE_INCLUDE_DIRS ${${_name}_INCLUDE_DIRS})
- list(APPEND CASACORE_LIBRARIES ${${_name}_LIBRARIES})
- endif(${_name}_FOUND)
-endmacro(casacore_find_package _name)
-
-# Define the Casacore components.
-set(Casacore_components
- casa
- coordinates
- derivedmscal
- fits
- images
- lattices
- meas
- measures
- mirlib
- ms
- msfits
- python
- scimath
- scimath_f
- tables
-)
-
-# Define the Casacore components' inter-dependencies.
-set(Casacore_casa_DEPENDENCIES)
-set(Casacore_coordinates_DEPENDENCIES fits measures casa)
-set(Casacore_derivedmscal_DEPENDENCIES ms measures tables casa)
-set(Casacore_fits_DEPENDENCIES measures tables casa)
-set(Casacore_images_DEPENDENCIES mirlib lattices coordinates fits measures scimath tables casa)
-set(Casacore_lattices_DEPENDENCIES tables scimath casa)
-set(Casacore_meas_DEPENDENCIES measures tables casa)
-set(Casacore_measures_DEPENDENCIES tables casa)
-set(Casacore_mirlib_DEPENDENCIES)
-set(Casacore_ms_DEPENDENCIES measures scimath tables casa)
-set(Casacore_msfits_DEPENDENCIES ms fits measures tables casa)
-set(Casacore_python_DEPENDENCIES casa)
-set(Casacore_scimath_DEPENDENCIES scimath_f casa)
-set(Casacore_scimath_f_DEPENDENCIES)
-set(Casacore_tables_DEPENDENCIES casa)
-
-# Initialize variables.
-set(CASACORE_FOUND FALSE)
-set(CASACORE_DEFINITIONS)
-set(CASACORE_LIBRARIES)
-set(CASACORE_MISSING_COMPONENTS)
-
-# Search for the header file first.
-if(NOT CASACORE_INCLUDE_DIR)
- find_path(CASACORE_INCLUDE_DIR casacore/casa/aips.h
- HINTS ${CASACORE_ROOT_DIR} PATH_SUFFIXES include)
- mark_as_advanced(CASACORE_INCLUDE_DIR)
-endif(NOT CASACORE_INCLUDE_DIR)
-
-# Fallback for systems that have old casacore installed in directory not called 'casacore'
-# This fallback can be removed once we move to casacore 2.0 which always puts headers in 'casacore'
-if(NOT CASACORE_INCLUDE_DIR)
- find_path(CASACORE_INCLUDE_DIR casa/aips.h
- HINTS ${CASACORE_ROOT_DIR} PATH_SUFFIXES include)
- mark_as_advanced(CASACORE_INCLUDE_DIR)
-endif(NOT CASACORE_INCLUDE_DIR)
-
-if(NOT CASACORE_INCLUDE_DIR)
- set(CASACORE_ERROR_MESSAGE "Casacore: unable to find the header file casa/aips.h.\nPlease set CASACORE_ROOT_DIR to the root directory containing Casacore.")
-else(NOT CASACORE_INCLUDE_DIR)
- # We've found the header file; let's continue.
- set(CASACORE_FOUND TRUE)
- # Note that new Casacore uses #include<casacore/casa/...>, while
- # LOFAR still uses #include<casa/...>. Hence use both in -I path.
- set(CASACORE_INCLUDE_DIRS ${CASACORE_INCLUDE_DIR} ${CASACORE_INCLUDE_DIR}/casacore)
-
- # Search for some often used binaries.
- find_program(TAQL_EXECUTABLE taql
- HINTS ${CASACORE_ROOT_DIR}/bin)
- mark_as_advanced(TAQL_EXECUTABLE)
-
- # If the user specified components explicity, use that list; otherwise we'll
- # assume that the user wants to use all components.
- if(NOT Casacore_FIND_COMPONENTS)
- set(Casacore_FIND_COMPONENTS ${Casacore_components})
- endif(NOT Casacore_FIND_COMPONENTS)
-
- # Get a list of all dependent Casacore libraries that need to be found.
- casacore_resolve_dependencies(_find_components ${Casacore_FIND_COMPONENTS})
-
- # Find the library for each component, and handle external dependencies
- foreach(_comp ${_find_components})
- casacore_find_library(casa_${_comp})
- if(${_comp} STREQUAL casa)
- casacore_find_package(HDF5)
- casacore_find_library(m)
- list(APPEND CASACORE_LIBRARIES ${CMAKE_DL_LIBS})
- elseif(${_comp} STREQUAL coordinates)
- casacore_find_package(WCSLIB REQUIRED)
- elseif(${_comp} STREQUAL fits)
- casacore_find_package(CFITSIO REQUIRED)
- elseif(${_comp} STREQUAL scimath_f)
- casacore_find_package(LAPACK REQUIRED)
- endif(${_comp} STREQUAL casa)
- endforeach(_comp ${_find_components})
-endif(NOT CASACORE_INCLUDE_DIR)
-
-# Set HAVE_CASACORE; and HAVE_AIPSPP (for backward compatibility with AIPS++).
-if(CASACORE_FOUND)
- set(HAVE_CASACORE TRUE CACHE INTERNAL "Define if Casacore is installed")
- set(HAVE_AIPSPP TRUE CACHE INTERNAL "Define if AIPS++/Casacore is installed")
-endif(CASACORE_FOUND)
-
-# Compose diagnostic message if not all necessary components were found.
-if(CASACORE_MISSING_COMPONENTS)
- set(CASACORE_ERROR_MESSAGE "Casacore: the following components could not be found:\n ${CASACORE_MISSING_COMPONENTS}")
-endif(CASACORE_MISSING_COMPONENTS)
-
-# Print diagnostics.
-if(CASACORE_FOUND)
- if(NOT Casacore_FIND_QUIETLY)
- message(STATUS "Found the following Casacore components: ")
- foreach(_comp ${_find_components})
- string(TOUPPER casa_${_comp} _COMP)
- message(STATUS " ${_comp}: ${${_COMP}_LIBRARY}")
- endforeach(_comp ${_find_components})
- endif(NOT Casacore_FIND_QUIETLY)
-else(CASACORE_FOUND)
- if(Casacore_FIND_REQUIRED)
- message(FATAL_ERROR "${CASACORE_ERROR_MESSAGE}")
- else(Casacore_FIND_REQUIRED)
- message(STATUS "${CASACORE_ERROR_MESSAGE}")
- endif(Casacore_FIND_REQUIRED)
-endif(CASACORE_FOUND)
-
diff --git a/CEP/DP3/DPPP_Interpolate/CMakeLists-standalone.txt b/CEP/DP3/DPPP_Interpolate/CMakeLists-standalone.txt
deleted file mode 100644
index 798f11ee87b4e503c317dc8e8d280c36025c39cf..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/CMakeLists-standalone.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-cmake_minimum_required(VERSION 2.8)
-
-project(DPPP_Interpolate)
-
-# Casacore has a separate CMake file in this directory
-set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMake)
-
-SET(CASACORE_MAKE_REQUIRED_EXTERNALS_OPTIONAL TRUE)
-find_package(Casacore REQUIRED COMPONENTS casa ms tables measures)
-
-find_package(Threads REQUIRED)
-
-include_directories(${CASACORE_INCLUDE_DIRS} ../../../build/gnucxx11_opt/include/)
-
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -Wall -DNDEBUG --std=c++11 -ggdb")
-
-add_library(interpolate-object OBJECT
- src/Interpolate.cc
- ../../../LCS/Common/src/LofarLogCout.cc)
-set_property(TARGET interpolate-object PROPERTY POSITION_INDEPENDENT_CODE 1)
-
-# Note: casapy fails if Casa is linked in the storage manager, so we have to trust that
-# casapy's version of casacore is binary compatible with this storage manager's casacore.
-add_library(dppp_interpolate SHARED $<TARGET_OBJECTS:interpolate-object>)
-target_link_libraries(dppp_interpolate ${CASACORE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
-
-install (TARGETS dppp_interpolate DESTINATION lib)
diff --git a/CEP/DP3/DPPP_Interpolate/CMakeLists.txt b/CEP/DP3/DPPP_Interpolate/CMakeLists.txt
deleted file mode 100644
index a45bd5adc12c880e93b85c0dda08e8c0df827277..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# $Id: CMakeLists.txt 27640 2013-12-04 08:02:49Z diepen $
-
-lofar_package(DPPP_Interpolate 1.0 DEPENDS DPPP)
-
-include(LofarFindPackage)
-lofar_find_package(Casacore COMPONENTS casa ms tables REQUIRED)
-
-add_subdirectory(include/DPPP_Interpolate)
-add_subdirectory(src)
-add_subdirectory(test)
diff --git a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/CMakeLists.txt b/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/CMakeLists.txt
deleted file mode 100644
index 3d9cb5dee52325f9070cf7377adc73744106910e..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-# $Id: CMakeLists.txt 39071 2018-02-08 15:50:02Z dijkema $
-
-# List of header files that will be installed.
-set(inst_HEADERS
- buffered_lane.h Interpolate.h lane.h
- )
-
-# Create symbolic link to include directory.
-execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
- ${CMAKE_CURRENT_SOURCE_DIR}
- ${CMAKE_BINARY_DIR}/include/${PACKAGE_NAME})
-
-# Install header files.
-install(FILES ${inst_HEADERS} DESTINATION include/${PACKAGE_NAME})
diff --git a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/Interpolate.h b/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/Interpolate.h
deleted file mode 100644
index 97570460a720a504820db81f30831d21cf5be37f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/Interpolate.h
+++ /dev/null
@@ -1,77 +0,0 @@
-#ifndef INTERPOLATE_H
-#define INTERPOLATE_H
-
-#include <deque>
-#include <string>
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-
-#include <Common/ParameterSet.h>
-
-#include <casacore/casa/Arrays/Cube.h>
-
-#include "lane.h"
-
-extern "C" void register_interpolate();
-
-namespace LOFAR { namespace DPPP {
-
- class Interpolate : public DPStep
- {
- public:
- // Construct the object.
- // Parameters are obtained from the parset using the given prefix.
- Interpolate (DPInput*, const ParameterSet&, const string& prefix);
-
- virtual ~Interpolate() = default;
-
- // Process the data.
- // It keeps the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- static DPStep::ShPtr makeStep(DPInput* input, const ParameterSet& parset, const std::string& prefix);
-
- private:
- void interpolateTimestep(size_t index);
- void interpolateSample(size_t timestep, size_t baseline, size_t channel, size_t pol);
- void sendFrontBufferToNextStep();
- void interpolationThread();
-
- struct Sample {
- Sample() = default;
- Sample(size_t timestep_, size_t baseline_, size_t channel_, size_t pol_) :
- timestep(timestep_), baseline(baseline_), channel(channel_), pol(pol_)
- { }
- size_t timestep;
- size_t baseline;
- size_t channel;
- size_t pol;
- };
-
- //# Data members.
- std::string _name;
- size_t _interpolatedPos;
- std::deque<DPBuffer> _buffers;
- size_t _windowSize;
- NSTimer _timer;
- ao::lane<Sample> _lane;
- std::vector<float> _kernelLookup;
- };
-
-} } //# end namespace
-
-#endif
diff --git a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/buffered_lane.h b/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/buffered_lane.h
deleted file mode 100644
index d9c0f9ec07316b11d42c73c8661a425fce426ae3..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/buffered_lane.h
+++ /dev/null
@@ -1,122 +0,0 @@
-#ifndef BUFFER_LANE_H
-#define BUFFER_LANE_H
-
-#include <vector>
-
-#include "lane.h"
-
-template<typename Tp>
-class lane_write_buffer
-{
-public:
- typedef typename ao::lane<Tp>::size_type size_type;
- typedef typename ao::lane<Tp>::value_type value_type;
-
- lane_write_buffer() : _buffer_size(0), _lane(0)
- { }
-
- lane_write_buffer(ao::lane<Tp>* lane, size_type buffer_size) : _buffer_size(buffer_size), _lane(lane)
- {
- _buffer.reserve(buffer_size);
- }
-
- ~lane_write_buffer()
- {
- flush();
- }
-
- void reset(ao::lane<Tp>* lane, size_type buffer_size)
- {
- _buffer.clear();
- _buffer.reserve(buffer_size);
- _buffer_size = buffer_size;
- _lane = lane;
- }
-
- void clear()
- {
- _lane->clear();
- _buffer.clear();
- }
-
- void write(const value_type& element)
- {
- _buffer.push_back(element);
- if(_buffer.size() == _buffer_size)
- flush();
- }
-
- void write(value_type&& element)
- {
- _buffer.push_back(std::move(element));
- if(_buffer.size() == _buffer_size)
- flush();
- }
-
- template<typename... Args>
- void emplace(Args&&... args)
- {
- _buffer.emplace_back(args...);
- if(_buffer.size() == _buffer_size)
- flush();
- }
-
- void write_end()
- {
- flush();
- _lane->write_end();
- }
-
- void flush()
- {
- _lane->move_write(&_buffer[0], _buffer.size());
- _buffer.clear();
- }
-private:
- size_type _buffer_size;
- std::vector<value_type> _buffer;
- ao::lane<Tp>* _lane;
-};
-
-template<typename Tp>
-class lane_read_buffer
-{
-public:
- lane_read_buffer(ao::lane<Tp>* lane, size_t buffer_size) :
- _buffer(new Tp[buffer_size]),
- _buffer_size(buffer_size),
- _buffer_pos(0),
- _buffer_fill_count(0),
- _lane(lane)
- {
- }
-
- ~lane_read_buffer()
- {
- delete[] _buffer;
- }
-
- bool read(Tp& element)
- {
- if(_buffer_pos == _buffer_fill_count)
- {
- _buffer_fill_count = _lane->read(_buffer, _buffer_size);
- _buffer_pos = 0;
- if(_buffer_fill_count == 0)
- return false;
- }
- element = std::move(_buffer[_buffer_pos]);
- ++_buffer_pos;
- return true;
- }
-
-private:
- lane_read_buffer(const lane_read_buffer&) = delete;
- lane_read_buffer& operator=(const lane_read_buffer&) = delete;
-
- Tp* _buffer;
- size_t _buffer_size, _buffer_pos, _buffer_fill_count;
- ao::lane<Tp>* _lane;
-};
-
-#endif
diff --git a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/lane.h b/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/lane.h
deleted file mode 100644
index 4be42a6e061dfb3acc26c7d25f75d611040b1bfe..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/include/DPPP_Interpolate/lane.h
+++ /dev/null
@@ -1,576 +0,0 @@
-#ifndef AO_LANE_11_H
-#define AO_LANE_11_H
-
-#include <cstring>
-#include <deque>
-#include <mutex>
-#include <condition_variable>
-
-/**
- * @file
- * Internal header file for the lane.
- * @headername{lane.h}
- */
-
-//#define LANE_DEBUG_MODE
-
-#ifdef LANE_DEBUG_MODE
-#include <string>
-#include <iostream>
-#include <sstream>
-#include <cmath>
-#endif
-
-namespace ao
-{
-
-#ifdef LANE_DEBUG_MODE
-#define set_lane_debug_name(lane, str) (lane).setDebugName(str)
-#define LANE_REGISTER_DEBUG_INFO registerDebugInfo()
-#define LANE_REGISTER_DEBUG_WRITE_WAIT registerDebugWriteWait()
-#define LANE_REGISTER_DEBUG_READ_WAIT registerDebugReadWait()
-#define LANE_REPORT_DEBUG_INFO reportDebugInfo()
-
-#else
-
-#define set_lane_debug_name(lane, str)
-#define LANE_REGISTER_DEBUG_INFO
-#define LANE_REGISTER_DEBUG_WRITE_WAIT
-#define LANE_REGISTER_DEBUG_READ_WAIT
-#define LANE_REPORT_DEBUG_INFO
-
-#endif
-
-/**
- * @brief The lane is an efficient cyclic buffer that is synchronized.
- * @details
- * A lane can typically be used in a multi-threaded producer-consumer
- * situation. The lane also holds a state which allows for
- * an ellegant way of communicating from producer(s) to
- * consumer(s) that all data has been produced.
- *
- * A simple example:
- * @code
- * void producer(lane<Task>* taskLane)
- * {
- * while(moreTasks)
- * taskLane->write(nextTask());
- * taskLane->write_end();
- * }
- *
- * void consumer(lane<Task>* taskLane)
- * {
- * Task task;
- * while(taskLane->read(task))
- * processTask(task);
- * }
- *
- * void run()
- * {
- * lane<Task> taskLane;
- * std::thread consumerThread(&consumer(), &taskLane);
- * producer(&taskLane);
- * consumerThread.join();
- * }
- * @endcode
- *
- * The various read and write methods, as well as the empty(),
- * capacity() and size() methods are always thread safe. The other
- * methods are not: assignment, swap(), clear() and resize() can not
- * be called from a different thread while another thread is also
- * accessing the lane. The same holds obviously for the constructors
- * and destructor. This is chosen because these methods should almost never
- * be called in parallel with other methods, and hence it is not worth
- * to increase every call with extra locks to make this possible.
- *
- * With one reader and one writer, the order is guaranteed to be consistent.
- * With multiple readers or writers in combination with multi-element
- * write or read functions, a sequence of symbols might be interrupted. For
- * example, if a multi-element write() won't fit completely in the buffer,
- * the thread will wait for free space. Another thread might get now write
- * access first, causing the single call to the multi-element write to be
- * "split up".
- *
- * @author Andre Offringa
- * @tparam Tp Type of elements to be stored in the lane.
- */
-template<typename Tp>
-class lane
-{
- public:
- /** @brief Integer type used to store size types. */
- typedef std::size_t size_type;
-
- /** @brief Type of elements stored in the lane. */
- typedef Tp value_type;
-
- /** @brief Construct a lane with zero elements.
- * @details A lane with zero elements can not be written to or read to
- * (both operations will wait forever).
- *
- * This constructor makes it easy to construct e.g. a container
- * of lanes. After the container is created, the lanes can be
- * resized with @ref resize().
- */
- lane() noexcept :
- _buffer(0),
- _capacity(0),
- _write_position(0),
- _free_write_space(0),
- _status(status_normal)
- {
- }
-
- /** @brief Construct a lane with the given capacity.
- * @details After construction, the lane is ready for writing to and reading from.
- * @param capacity Number of elements that the lane can hold at once.
- */
- explicit lane(size_t capacity) :
- _buffer(new Tp[capacity]),
- _capacity(capacity),
- _write_position(0),
- _free_write_space(_capacity),
- _status(status_normal)
- {
- }
-
- lane(const lane<Tp>& source) = delete;
-
- /** @brief Move construct a lane.
- * @details This operation is not thread safe: the behaviour is undefined when
- * other threads access the source lane.
- * @param source Original lane to be moved from.
- */
- lane(lane<Tp>&& source) noexcept :
- _buffer(0),
- _capacity(0),
- _write_position(0),
- _free_write_space(0),
- _status(status_normal)
- {
- swap(source);
- }
-
- /** @brief Destructor.
- * @details The destructor is not synchronized.
- */
- ~lane()
- {
- LANE_REPORT_DEBUG_INFO;
- delete[] _buffer;
- }
-
- lane<Tp>& operator=(const lane<Tp>& source) = delete;
-
- /** @brief Move assignment.
- * @details This operation is not thread safe: the behaviour is undefined when
- * other threads access the source lane.
- * @param source Original lane to be moved from.
- * @returns This lane.
- */
- lane<Tp>& operator=(lane<Tp>&& source) noexcept
- {
- swap(source);
- return *this;
- }
-
- /** @brief Swap the contents of this lane with another.
- * @details This operation is not thread safe: the behaviour is undefined when
- * other threads access either lane.
- */
- void swap(lane<Tp>& other) noexcept
- {
- std::swap(_buffer, other._buffer);
- std::swap(_capacity, other._capacity);
- std::swap(_write_position, other._write_position);
- std::swap(_free_write_space, other._free_write_space);
- std::swap(_status, other._status);
- }
-
- /** @brief Clear the contents and reset the state of the lane.
- * @details After calling clear(), the lane is in the same state as after
- * construction. This also means that after clearing the lane, it
- * is as if write_end() has not been called yet.
- *
- * This method is not thread safe.
- */
- void clear() noexcept
- {
- _write_position = 0;
- _free_write_space = _capacity;
- _status = status_normal;
- }
-
- /** @brief Write a single element.
- * @details This method is thread safe, and can be called together with
- * other write and read methods from different threads.
- *
- * If this call comes after a call to write_end(), the call
- * will be ignored.
- * @param element Object to be copied into the cyclic buffer.
- */
- void write(const value_type& element)
- {
- std::unique_lock<std::mutex> lock(_mutex);
- LANE_REGISTER_DEBUG_INFO;
-
- if(_status == status_normal)
- {
- while(_free_write_space == 0)
- {
- LANE_REGISTER_DEBUG_WRITE_WAIT;
- _writing_possible_condition.wait(lock);
- }
-
- _buffer[_write_position] = element;
- _write_position = (_write_position+1) % _capacity;
- --_free_write_space;
- // Now that there is less free write space, there is more free read
- // space and thus readers can possibly continue.
- _reading_possible_condition.notify_all();
- }
- }
-
- /** @brief Write a single element by constructing it.
- * @details This method is thread safe, and can be called together with
- * other write and read methods from different threads.
- *
- * If this call comes after a call to write_end(), the call
- * will be ignored. The implementation does not construct the value
- * in place, but rather constructs the value and then move assigns it.
- * This is because the value that it is moved into has already been
- * constructed (in the current implementation).
- * @param element Object to be moved into the cyclic buffer.
- */
- template<typename... Args>
- void emplace(Args&&... args)
- {
- std::unique_lock<std::mutex> lock(_mutex);
- LANE_REGISTER_DEBUG_INFO;
-
- if(_status == status_normal)
- {
- while(_free_write_space == 0)
- {
- LANE_REGISTER_DEBUG_WRITE_WAIT;
- _writing_possible_condition.wait(lock);
- }
-
- _buffer[_write_position] = value_type(args...);
- _write_position = (_write_position+1) % _capacity;
- --_free_write_space;
- // Now that there is less free write space, there is more free read
- // space and thus readers can possibly continue.
- _reading_possible_condition.notify_all();
- }
- }
-
- /** @brief Write a single element by moving it in.
- * @details This method is thread safe, and can be called together with
- * other write and read methods from different threads.
- *
- * If this call comes after a call to write_end(), the call
- * will be ignored.
- * @param element Object to be moved into the cyclic buffer.
- */
- void write(value_type&& element)
- {
- std::unique_lock<std::mutex> lock(_mutex);
- LANE_REGISTER_DEBUG_INFO;
-
- if(_status == status_normal)
- {
- while(_free_write_space == 0)
- {
- LANE_REGISTER_DEBUG_WRITE_WAIT;
- _writing_possible_condition.wait(lock);
- }
-
- _buffer[_write_position] = std::move(element);
- _write_position = (_write_position+1) % _capacity;
- --_free_write_space;
- // Now that there is less free write space, there is more free read
- // space and thus readers can possibly continue.
- _reading_possible_condition.notify_all();
- }
- }
-
- void write(const value_type* elements, size_t n)
- {
- write_generic(elements, n);
- }
-
- void move_write(value_type* elements, size_t n)
- {
- write_generic(elements, n);
- }
-
- bool read(value_type& destination)
- {
- std::unique_lock<std::mutex> lock(_mutex);
- LANE_REGISTER_DEBUG_INFO;
- while(free_read_space() == 0 && _status == status_normal)
- {
- LANE_REGISTER_DEBUG_READ_WAIT;
- _reading_possible_condition.wait(lock);
- }
- if(free_read_space() == 0)
- return false;
- else
- {
- destination = std::move(_buffer[read_position()]);
- ++_free_write_space;
- // Now that there is more free write space, writers can possibly continue.
- _writing_possible_condition.notify_all();
- return true;
- }
- }
-
- size_t read(value_type* destinations, size_t n)
- {
- size_t n_left = n;
-
- std::unique_lock<std::mutex> lock(_mutex);
- LANE_REGISTER_DEBUG_INFO;
-
- size_t free_space = free_read_space();
- size_t read_size = free_space > n ? n : free_space;
- immediate_read(destinations, read_size);
- n_left -= read_size;
-
- while(n_left != 0 && _status == status_normal)
- {
- destinations += read_size;
-
- do {
- LANE_REGISTER_DEBUG_READ_WAIT;
- _reading_possible_condition.wait(lock);
- } while(free_read_space() == 0 && _status == status_normal);
-
- free_space = free_read_space();
- read_size = free_space > n_left ? n_left : free_space;
- immediate_read(destinations, read_size);
- n_left -= read_size;
- }
- return n - n_left;
- }
-
- void write_end()
- {
- std::lock_guard<std::mutex> lock(_mutex);
- LANE_REGISTER_DEBUG_INFO;
- _status = status_end;
- _writing_possible_condition.notify_all();
- _reading_possible_condition.notify_all();
- }
-
- size_t capacity() const noexcept
- {
- return _capacity;
- }
-
- size_t size() const
- {
- std::lock_guard<std::mutex> lock(_mutex);
- return _capacity - _free_write_space;
- }
-
- bool empty() const
- {
- std::lock_guard<std::mutex> lock(_mutex);
- return _capacity == _free_write_space;
- }
-
- /**
- * Change the capacity of the lane. This will erase all data in the lane.
- */
- void resize(size_t new_capacity)
- {
- Tp *new_buffer = new Tp[new_capacity];
- delete[] _buffer;
- _buffer = new_buffer;
- _capacity = new_capacity;
- _write_position = 0;
- _free_write_space = new_capacity;
- _status = status_normal;
- }
-
-#ifdef LANE_DEBUG_MODE
- /**
- * Change the name of this lane to make it appear in the output along
- * with statistics. Do not use this function directly; use the
- * set_lane_debug_name() macro instead.
- * @param nameStr New debug description of this lane.
- */
- void setDebugName(const std::string& nameStr)
- {
- _debugName = nameStr;
- }
-#endif
- private:
- Tp* _buffer;
-
- size_t _capacity;
-
- size_t _write_position;
-
- size_t _free_write_space;
-
- enum { status_normal, status_end } _status;
-
- mutable std::mutex _mutex;
-
- std::condition_variable _writing_possible_condition, _reading_possible_condition;
-
- size_t read_position() const noexcept
- {
- return (_write_position + _free_write_space) % _capacity;
- }
-
- size_t free_read_space() const noexcept
- {
- return _capacity - _free_write_space;
- }
-
- // This is a template to allow const and non-const (to be able to move)
- template<typename T>
- void write_generic(T* elements, size_t n)
- {
- std::unique_lock<std::mutex> lock(_mutex);
- LANE_REGISTER_DEBUG_INFO;
-
- if(_status == status_normal)
- {
- size_t write_size = _free_write_space > n ? n : _free_write_space;
- immediate_write(elements, write_size);
- n -= write_size;
-
- while(n != 0) {
- elements += write_size;
-
- do {
- LANE_REGISTER_DEBUG_WRITE_WAIT;
- _writing_possible_condition.wait(lock);
- } while(_free_write_space == 0 && _status == status_normal);
-
- write_size = _free_write_space > n ? n : _free_write_space;
- immediate_write(elements, write_size);
- n -= write_size;
- } while(n != 0);
- }
- }
-
- // This is a template to allow const and non-const (to be able to move)
- template<typename T>
- void immediate_write(T *elements, size_t n) noexcept
- {
- // Split the writing in two ranges if needed. The first range fits in
- // [_write_position, _capacity), the second range in [0, end). By doing
- // so, we only have to calculate the modulo in the write position once.
- if(n > 0)
- {
- size_t nPart;
- if(_write_position + n > _capacity)
- {
- nPart = _capacity - _write_position;
- } else {
- nPart = n;
- }
- for(size_t i = 0; i < nPart ; ++i, ++_write_position)
- {
- _buffer[_write_position] = std::move(elements[i]);
- }
-
- _write_position = _write_position % _capacity;
-
- for(size_t i = nPart; i < n ; ++i, ++_write_position)
- {
- _buffer[_write_position] = std::move(elements[i]);
- }
-
- _free_write_space -= n;
-
- // Now that there is less free write space, there is more free read
- // space and thus readers can possibly continue.
- _reading_possible_condition.notify_all();
- }
- }
-
- void immediate_read(value_type *elements, size_t n) noexcept
- {
- // As with write, split in two ranges if needed. The first range fits in
- // [read_position(), _capacity), the second range in [0, end).
- if(n > 0)
- {
- size_t nPart;
- size_t position = read_position();
- if(position + n > _capacity)
- {
- nPart = _capacity - position;
- } else {
- nPart = n;
- }
- for(size_t i = 0; i < nPart ; ++i, ++position)
- {
- elements[i] = std::move(_buffer[position]);
- }
-
- position = position % _capacity;
-
- for(size_t i = nPart; i < n ; ++i, ++position)
- {
- elements[i] = std::move(_buffer[position]);
- }
-
- _free_write_space += n;
-
- // Now that there is more free write space, writers can possibly continue.
- _writing_possible_condition.notify_all();
- }
- }
-#ifdef LANE_DEBUG_MODE
- void registerDebugInfo() noexcept
- {
- _debugSummedSize += _capacity - _free_write_space;
- _debugMeasureCount++;
- }
- void registerDebugReadWait() noexcept
- {
- ++_debugReadWaitCount;
- }
- void registerDebugWriteWait() noexcept
- {
- ++_debugWriteWaitCount;
- }
- void reportDebugInfo()
- {
- if(!_debugName.empty())
- {
- std::stringstream str;
- str
- << "*** Debug report for the following lane: ***\n"
- << "\"" << _debugName << "\"\n"
- << "Capacity: " << _capacity << '\n'
- << "Total read/write ops: " << _debugMeasureCount << '\n'
- << "Average size of buffer, measured per read/write op.: " << round(double(_debugSummedSize)*100.0/_debugMeasureCount)/100.0 << '\n'
- << "Number of wait events during reading: " << _debugReadWaitCount << '\n'
- << "Number of wait events during writing: " << _debugWriteWaitCount << '\n';
- std::cout << str.str();
- }
- }
- std::string _debugName;
- size_t
- _debugSummedSize = 0, _debugMeasureCount = 0,
- _debugReadWaitCount = 0, _debugWriteWaitCount = 0;
-#endif
-};
-
-template<typename Tp>
-void swap(ao::lane<Tp>& first, ao::lane<Tp>& second) noexcept
-{
- first.swap(second);
-}
-
-} // end of namespace
-
-#endif // AO_LANE11_H
diff --git a/CEP/DP3/DPPP_Interpolate/src/CMakeLists.txt b/CEP/DP3/DPPP_Interpolate/src/CMakeLists.txt
deleted file mode 100644
index 52d938d9014f9bdb933302d19c1184f257dddece..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/src/CMakeLists.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-include(LofarPackageVersion)
-
-lofar_add_library(dppp_interpolate
- Interpolate.cc ../../../../LCS/Common/src/LofarLogCout.cc
-)
diff --git a/CEP/DP3/DPPP_Interpolate/src/Interpolate.cc b/CEP/DP3/DPPP_Interpolate/src/Interpolate.cc
deleted file mode 100644
index 79ace62ae8ddece0d3ca6d0efb1b4d4b263afa2c..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/src/Interpolate.cc
+++ /dev/null
@@ -1,245 +0,0 @@
-#include <DPPP_Interpolate/Interpolate.h>
-#include <DPPP_Interpolate/buffered_lane.h>
-
-#include <lofar_config.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPRun.h>
-#include <Common/ParameterSet.h>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <Common/StringUtil.h>
-
-#include <iostream>
-#include <iomanip>
-#include <thread>
-
-using namespace casacore;
-
-void register_interpolate()
-{
- LOFAR::DPPP::DPRun::registerStepCtor("interpolate", LOFAR::DPPP::Interpolate::makeStep);
-}
-
-namespace LOFAR { namespace DPPP {
-
-Interpolate::Interpolate(DPInput* /*input*/, const ParameterSet& parset, const string& prefix) :
- _name(prefix),
- _interpolatedPos(0),
- _windowSize(parset.getUint(prefix+"windowsize", 15))
-{
- if(_windowSize%2 != 1)
- throw std::runtime_error("Window size of Interpolate action should be an odd number");
-
- _kernelLookup.reserve(_windowSize*_windowSize);
- for(int t=0; t!=int(_windowSize); ++t)
- {
- int y = t - int(_windowSize/2);
- for(int ch=0; ch!=int(_windowSize); ++ch)
- {
- int x = ch - int(_windowSize/2);
- double windowDist = double(x*x + y*y);
- // Gaussian function with sigma = 1
- // (evaluated with double prec, then converted to floats)
- double w = std::exp(windowDist * -0.5);
- _kernelLookup.emplace_back(w);
- }
- }
-}
-
-DPStep::ShPtr Interpolate::makeStep(DPInput* input, const ParameterSet& parset, const std::string& prefix)
-{
- return DPStep::ShPtr(new Interpolate(input, parset, prefix));
-}
-
-void Interpolate::updateInfo(const DPInfo& infoIn)
-{
- info() = infoIn;
- info().setNeedVisData();
- info().setWriteData();
- info().setWriteFlags();
-}
-
-void Interpolate::show(std::ostream& os) const
-{
- os << "Interpolate " << _name << '\n';
- os << " windowsize: " << _windowSize << '\n';
-}
-
-void Interpolate::showTimings(std::ostream& os, double duration) const
-{
- os << " ";
- FlagCounter::showPerc1 (os, _timer.getElapsed(), duration);
- os << " Interpolate " << _name << endl;
-}
-
-bool Interpolate::process(const DPBuffer& buf)
-{
- _timer.start();
- // Collect the data in buffers.
- _buffers.emplace_back();
- _buffers.back().copy(buf);
- // If we have a full window of data, interpolate everything
- // up to the middle of the window
- if(_buffers.size() >= _windowSize)
- {
- size_t mid = _windowSize/2;
- while(_interpolatedPos <= mid)
- {
- interpolateTimestep(_interpolatedPos);
- ++_interpolatedPos;
- }
- // Buffers are only pushed to the next step when they are completely
- // out of the window. This is because flags need to be set to false,
- // however the flag information of the entire window is needed during
- // interpolation, so these can only be set to false after processing.
- sendFrontBufferToNextStep();
- }
- _timer.stop();
- return true;
-}
-
-void Interpolate::sendFrontBufferToNextStep()
-{
- IPosition shp = _buffers.front().getData().shape();
- size_t
- nPol = shp[0],
- nChan = shp[1],
- nBl = shp[2],
- n = nPol * nChan * nBl;
- // Set all flags to false
- bool* flags = _buffers.front().getFlags().data();
- Complex* data = _buffers.front().getData().data();
- std::fill(flags, flags+n, false);
- // Flag NaN values (values for which the entire window was flagged on input)
- for(size_t i=0; i!=n; ++i)
- {
- if(!std::isfinite(data[i].real()) || !std::isfinite(data[i].imag()))
- {
- // The datum value is also set to 0, because NaNs sometimes give problems in
- // certain software, even when they are flagged (e.g. in Sagecal).
- data[i] = 0.0;
- flags[i] = true;
- }
- }
-
- _timer.stop();
- getNextStep()->process(_buffers.front());
- _timer.start();
-
- _buffers.pop_front();
- --_interpolatedPos;
-}
-
-void Interpolate::finish()
-{
- _timer.start();
-
- // Interpolate everything up to the end of the window
- while(_interpolatedPos<_buffers.size()) {
- interpolateTimestep(_interpolatedPos);
- ++_interpolatedPos;
- }
- while(!_buffers.empty())
- {
- sendFrontBufferToNextStep();
- }
-
- _timer.stop();
-
- getNextStep()->finish();
-}
-
-#define BUFFER_SIZE 1024
-
-void Interpolate::interpolateTimestep(size_t index)
-{
- const IPosition shp = _buffers.front().getData().shape();
- const size_t
- nPol = shp[0],
- nChan = shp[1],
- nPerBl = nPol*nChan,
- nBl = shp[2];
-
- std::vector<std::thread> threads;
- size_t nthreads = std::min<size_t>(sysconf(_SC_NPROCESSORS_ONLN), 8);
- _lane.resize(nthreads*BUFFER_SIZE);
- lane_write_buffer<Sample> buflane(&_lane, BUFFER_SIZE);
- threads.reserve(nthreads);
- for(size_t i=0; i!=nthreads; ++i)
- threads.emplace_back(&Interpolate::interpolationThread, this);
-
- std::vector<Complex> dataBlock;
- for (size_t bl=0; bl<nBl; ++bl)
- {
- bool* flags = _buffers[index].getFlags().data() + bl*nPerBl;
- for(size_t ch=0; ch!=nChan; ++ch)
- {
- for(size_t p=0; p!=nPol; ++p)
- {
- if(*flags) {
- buflane.emplace(index, bl, ch, p);
- }
- ++flags;
- }
- }
- }
- buflane.write_end();
-
- for(std::thread& t : threads)
- t.join();
-}
-
-void Interpolate::interpolationThread()
-{
- lane_read_buffer<Sample> buflane(&_lane, BUFFER_SIZE);
- Sample sample;
- while(buflane.read(sample))
- {
- interpolateSample(sample.timestep, sample.baseline, sample.channel, sample.pol);
- }
-}
-
-void Interpolate::interpolateSample(size_t timestep, size_t baseline, size_t channel, size_t pol)
-{
- const IPosition shp = _buffers.front().getData().shape();
- const size_t
- nPol = shp[0],
- nChan = shp[1],
- timestepBegin = (timestep > _windowSize/2) ? (timestep - _windowSize/2) : 0,
- timestepEnd = std::min(timestep + _windowSize/2 + 1, _buffers.size()),
- channelBegin = (channel > _windowSize/2) ? (channel - _windowSize/2) : 0,
- channelEnd = std::min(channel + _windowSize/2 + 1, nChan);
-
- std::complex<float> valueSum = 0.0;
- float windowSum = 0.0;
-
- for(size_t t=timestepBegin; t!=timestepEnd; ++t)
- {
- Complex* data = _buffers[t].getData().data() + (baseline*nChan + channelBegin)*nPol + pol;
- const bool* flags = _buffers[t].getFlags().data() + (baseline*nChan + channelBegin)*nPol + pol;
- const float* row = &_kernelLookup[_windowSize * (t + int(_windowSize/2) - timestep)];
- for(size_t ch=channelBegin; ch!=channelEnd; ++ch)
- {
- if(!*flags)
- {
- int x = ch + int(_windowSize/2) - channel;
- float w = row[x];
- valueSum += *data * w;
- windowSum += w;
- }
-
- data += nPol;
- flags += nPol;
- }
- }
- // This write is multithreaded, but is allowed because this value is never read from in
- // the loops above (because flagged values are skipped).
- Complex& value = _buffers[timestep].getData().data()[(baseline*nChan + channel)*nPol + pol];
- if(windowSum != 0.0)
- value = valueSum / windowSum;
- else
- value = Complex(std::numeric_limits<float>::quiet_NaN(), std::numeric_limits<float>::quiet_NaN());
-}
-
-} } //# end namespace
diff --git a/CEP/DP3/DPPP_Interpolate/test/CMakeLists.txt b/CEP/DP3/DPPP_Interpolate/test/CMakeLists.txt
deleted file mode 100644
index d7d8fbd417438b518d304edfee883f15727efa62..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/test/CMakeLists.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# $Id: CMakeLists.txt 26355 2013-09-04 11:51:07Z dijkema $
-
-include(LofarCTest)
-
-lofar_add_test(tInterpolateStep tInterpolateStep.cc)
diff --git a/CEP/DP3/DPPP_Interpolate/test/tInterpolateStep.cc b/CEP/DP3/DPPP_Interpolate/test/tInterpolateStep.cc
deleted file mode 100644
index 1dd558c5621ba0f6bf700e0d568cc05160a0d703..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/test/tInterpolateStep.cc
+++ /dev/null
@@ -1,233 +0,0 @@
-//# tInterpolateStep.cc: Test program for class InterpolateStep
-//# Copyright (C) 2010
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tInterpolateStep.cc 24221 2013-03-12 12:24:48Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <DPPP_Interpolate/Interpolate.h>
-#include <DPPP/DPRun.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all to false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nant, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nant*(nant+1)/2), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {
- // Fill the baseline stations; use 4 stations.
- // So they are called 00 01 02 03 10 11 12 13 20, etc.
- Vector<Int> ant1(itsNBl);
- Vector<Int> ant2(itsNBl);
- int st1 = 0;
- int st2 = 0;
- for (int i=0; i<itsNBl; ++i) {
- ant1[i] = st1;
- ant2[i] = st2;
- if (++st2 == 4) {
- st2 = 0;
- if (++st1 == 4) {
- st1 = 0;
- }
- }
- }
- Vector<String> antNames(4);
- antNames[0] = "rs01.s01";
- antNames[1] = "rs02.s01";
- antNames[2] = "cs01.s01";
- antNames[3] = "cs01.s02";
- // Define their positions (more or less WSRT RT0-3).
- vector<MPosition> antPos (4);
- Vector<double> vals(3);
- vals[0] = 3828763; vals[1] = 442449; vals[2] = 5064923;
- antPos[0] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828746; vals[1] = 442592; vals[2] = 5064924;
- antPos[1] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828729; vals[1] = 442735; vals[2] = 5064925;
- antPos[2] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- vals[0] = 3828713; vals[1] = 442878; vals[2] = 5064926;
- antPos[3] = MPosition(Quantum<Vector<double> >(vals,"m"),
- MPosition::ITRF);
- Vector<double> antDiam(4, 70.);
- info().set (antNames, antDiam, antPos, ant1, ant2);
- // Define the frequencies.
- Vector<double> chanFreqs(nchan);
- Vector<double> chanWidth(nchan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- cout << "Input step " << itsCount << ' '<< itsCount*5+2<<endl;
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(1.6, 0.9);
- }
- if (itsCount == 5) {
- data += Complex(10.,10.);
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&)
- // Use startchan=0 and timeInterval=5
- { info().init (itsNCorr, itsNChan, itsNTime, 100, 5, string(), string()); }
-
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nant, int nchan, int ncorr)
- : itsCount(0), itsNTime(ntime), itsNBl(nant*(nant+1)/2), itsNChan(nchan),
- itsNCorr(ncorr)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- cout << "Output step " << itsCount << ' '<<itsCount*5+2<<endl;
- // Fill expected result in similar way as TestInput.
- Cube<Complex> result(itsNCorr,itsNChan,itsNBl);
- for (int i=0; i<int(result.size()); ++i) {
- result.data()[i] = Complex(1.6, 0.9);
- }
- if (itsCount == 5) {
- result += Complex(10.,10.);
- }
- // Check the result.
- ///cout << buf.getData()<< result;
- ASSERT (allNear(real(buf.getData()), real(result), 1e-10));
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-10));
- ASSERT (near(buf.getTime(), 2+5.*itsCount));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNChan);
- ASSERT (int(info.nchan())==itsNChan);
- ASSERT (int(info.ntime())==itsNTime);
- ASSERT (info.startTime()==100);
- ASSERT (info.timeInterval()==5);
- ASSERT (int(info.nchanAvg())==1);
- ASSERT (int(info.ntimeAvg())==1);
- ASSERT (int(info.chanFreqs().size()) == itsNChan);
- ASSERT (int(info.chanWidths().size()) == itsNChan);
- ASSERT (info.msName().empty());
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set the info in each step.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
- DPStep::ShPtr step = step1;
- while (step) {
- step->showCounts (cout);
- step = step->getNextStep();
- }
-}
-
-void test1(int ntime, int nant, int nchan, int ncorr, bool flag, int threshold)
-{
- cout << "test1: ntime=" << ntime << " nrant=" << nant << " nchan=" << nchan
- << " ncorr=" << ncorr << " threshold=" << threshold << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nant, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("windowsize", "9");
- DPStep::ShPtr step2 = DPRun::findStepCtor("interpolate")(in, parset, "");
- DPStep::ShPtr step3(new TestOutput(ntime, nant, nchan, ncorr));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- step2->show (cout);
- execute (step1);
-}
-
-int main()
-{
- try {
- test1(10, 2, 32, 4, false, 1);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/DPPP_Interpolate/test/tInterpolateStep.sh b/CEP/DP3/DPPP_Interpolate/test/tInterpolateStep.sh
deleted file mode 100755
index 7cd507992e304dd04023b576c26a8c3b948cd87f..0000000000000000000000000000000000000000
--- a/CEP/DP3/DPPP_Interpolate/test/tInterpolateStep.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tInterpolateStep
diff --git a/CEP/DP3/PythonDPPP/CMakeLists.txt b/CEP/DP3/PythonDPPP/CMakeLists.txt
deleted file mode 100644
index fb329585b2dbcf193bfaf51f70c99a43d60235ce..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-# $Id: CMakeLists.txt 27640 2013-12-04 08:02:49Z diepen $
-
-lofar_package(PythonDPPP 0.1 DEPENDS DPPP pyparameterset)
-
-# Try to find Boost-Python.
-# If found, build this package. Otherwise give a warning.
-FIND_PATH(BOOST_PYTHON_FOUND "boost/python.hpp")
-
-if(BOOST_PYTHON_FOUND)
- include(LofarFindPackage)
- lofar_find_package(Python 3.4 REQUIRED)
- lofar_find_package(Boost REQUIRED COMPONENTS python3)
- lofar_find_package(Casacore COMPONENTS casa ms tables python REQUIRED)
-
- add_subdirectory(include/PythonDPPP)
- add_subdirectory(src)
- add_subdirectory(test)
-else()
- message (WARNING, " Boost-Python not found; PythonDPPP will not be built.")
-endif()
diff --git a/CEP/DP3/PythonDPPP/include/PythonDPPP/CMakeLists.txt b/CEP/DP3/PythonDPPP/include/PythonDPPP/CMakeLists.txt
deleted file mode 100644
index 9e7e4ca3d6ea6c83199f2f02e59604c2f5207a64..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/include/PythonDPPP/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-# $Id: CMakeLists.txt 30166 2014-10-07 12:00:40Z dijkema $
-
-# List of header files that will be installed.
-set(inst_HEADERS
- PythonStep.h
-)
-
-# Create symbolic link to include directory.
-execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
- ${CMAKE_CURRENT_SOURCE_DIR}
- ${CMAKE_BINARY_DIR}/include/${PACKAGE_NAME})
-
-# Install header files.
-install(FILES ${inst_HEADERS} DESTINATION include/${PACKAGE_NAME})
diff --git a/CEP/DP3/PythonDPPP/include/PythonDPPP/DPStepBase.h b/CEP/DP3/PythonDPPP/include/PythonDPPP/DPStepBase.h
deleted file mode 100644
index ebb7b50cdd96822eca287d81766cba720ac4337f..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/include/PythonDPPP/DPStepBase.h
+++ /dev/null
@@ -1,113 +0,0 @@
-//# DPStepBase.cc: Python base class for a DPStep in python
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: pyparameterset.cc 23074 2012-12-03 07:51:29Z diepen $
-
-#ifndef DPPP_DPSTEPBASE_H
-#define DPPP_DPSTEPBASE_H
-
-#include <PythonDPPP/PythonStep.h>
-#include <DPPP/DPInfo.h>
-
-
-namespace LOFAR {
- namespace DPPP {
-
- // DPStepBase is the interface for callbacks from the DPPP PythonStep
- // class (and its derivations) to C++. Note it is not used for calls
- // from C++ to Python, which are done in PythonStep.cc using the
- // Boost::Python functionality.
- // DPStepBase is bound to the Python interface by PythonDPPP.cc.
- // See class PythonStep and __init.py__ for more information.
-
- class DPStepBase
- {
- public:
- // Keep the this pointer in a static. It is used by PythonStep.cc
- // to call setStep to have a pointer from the python interface
- // to the C++ code.
- DPStepBase()
- : itsStep(0)
- {
- theirPtr = this;
- }
-
- // Create the link for callbacks from Python to C++.
- // It is used by the PythonStep constructor.
- void setStep (PythonStep* step)
- {
- itsStep = step;
- }
-
- // Get the visibility data into the ValueHolder array.
- // The array must have the correct type and shape.
- void _getData (const casacore::ValueHolder& vh)
- {
- itsStep->getData (vh);
- }
-
- // Get the flags into the ValueHolder array.
- // The array must have the correct type and shape.
- void _getFlags (const casacore::ValueHolder& vh)
- {
- itsStep->getFlags (vh);
- }
-
- // Get the weights into the ValueHolder array.
- // The array must have the correct type and shape.
- void _getWeights (const casacore::ValueHolder& vh)
- {
- itsStep->getWeights (vh);
- }
-
- // Get the UVW coordinates data into the ValueHolder array.
- // The array must have the correct type and shape.
- void _getUVW (const casacore::ValueHolder& vh)
- {
- itsStep->getUVW (vh);
- }
-
- // Get the model data into the ValueHolder array.
- // The array must have the correct type and shape.
- void _getModelData (const casacore::ValueHolder& vh)
- {
- itsStep->getModelData (vh);
- }
-
- // Call the process function in the next DPPP step.
- // The record must contain the changed data fields which will be
- // stored in the output buffer before calling the next step.
- bool _processNext (const casacore::Record& rec)
- {
- return itsStep->processNext (rec);
- }
-
- // Keep the pointer to this object.
- // It is filled by the constructor and used shortly thereafter
- // in the PythonStep constructor.
- static DPStepBase* theirPtr;
-
- private:
- PythonStep* itsStep;
- };
-
- }
-}
-
-#endif
diff --git a/CEP/DP3/PythonDPPP/include/PythonDPPP/PythonStep.h b/CEP/DP3/PythonDPPP/include/PythonDPPP/PythonStep.h
deleted file mode 100644
index 665802bed7408d8e9d0920cc7f71763046aba687..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/include/PythonDPPP/PythonStep.h
+++ /dev/null
@@ -1,162 +0,0 @@
-//# PythonStep.h: A DPStep executed in some python module
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: DPStep.h 30718 2015-01-19 15:31:51Z diepen $
-//#
-//# @author Ger van Diepen
-
-#ifndef DPPP_PYTHONSTEP_H
-#define DPPP_PYTHONSTEP_H
-
-// @file
-// @brief Class to execute a DPStep in some Python module
-
-#include <DPPP/DPInput.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <Common/ParameterSet.h>
-#include <Common/Timer.h>
-
-#include <casacore/casa/Containers/ValueHolder.h>
-#include <casacore/casa/Containers/Record.h>
-#include <boost/python.hpp>
-
-namespace LOFAR {
- namespace DPPP {
-
- // @ingroup NDPPP
-
- // This class defines a step in the DPPP pipeline to be executed in
- // Python.
- // the PythonStep functionality is not directly part of the DPPP program,
- // but is automatically loaded on demand from a shared library.
-
- // The control flow of a python step is as follows:
- // <ol>
- // <li> PythonStep derives from DPStep as all other DPPP steps do.
- // Its constructor initializes Python, opens the module given in
- // the parset key 'python.module' and creates an instance of the
- // class given in 'python.class'.
- // That class must derive from the Python class DPStepBase defined
- // in __init.py__. The C++ class DPStepBase is the interface for
- // callbacks from Python to C++.
- // <li> The updateInfo function calls its Python counterpart. The
- // DPInfo object is encoded in a dict. Currently, Measure objects
- // in DPInfo are not handled yet.
- // <li> The process function calls its Python counterpart with the
- // time and exposure in the DPBuffer object. The Python class can
- // obtain the other dats (visibilities, flags, weights, UVW, modeldata)
- // by means of explicit callbacks. This was done for 2 reasons:
- // <br>- it makes it possible to directly fill a numpy array.
- // <br>- only data really needed is sent.
- // <li> When the Python step has output ready, it needs to call the
- // processNext function with the data that has changed. Note that
- // (as in e.g. class Averager) it is possible that only every N
- // input buffers result in an output buffer.
- // <li> The finish, show, showCounts, showTimings, and addToMS
- // functions call their Python counterparts. The Python base class
- // offers default implementations, so they do not need to be
- // implemented in a derived Python step class.
- // </ol>
- // The class also contains several functions that are basically the
- // callback function for Python.
-
- class PythonStep: public DPStep
- {
- public:
- PythonStep (DPInput* input,
- const ParameterSet& parset,
- const string& prefix);
-
- virtual ~PythonStep();
-
- // The 'constructor' for dynamically loaded steps.
- // It is registered when the shared library is loaded.
- static DPStep::ShPtr makeStep (DPInput*, const ParameterSet&,
- const std::string&);
-
- // Process the data.
- // When processed, it invokes the process function of the next step.
- virtual bool process (const DPBuffer&);
-
- // Finish the processing of this step and subsequent steps.
- virtual void finish();
-
- // Update the general info.
- virtual void updateInfo (const DPInfo&);
-
- // Add some data to the MeasurementSet written/updated.
- virtual void addToMS (const string& msName);
-
- // Show the step parameters.
- virtual void show (std::ostream&) const;
-
- // Show the flag counts if needed.
- virtual void showCounts (std::ostream&) const;
-
- // Show the timings.
- virtual void showTimings (std::ostream&, double duration) const;
-
- // Callback functions for Python.
- // <group>
- // Tell that the Python step needs the visibility data.
- void setNeedVisData();
- // Tell that the Python step needs data to be written.
- void setNeedWrite();
- // Get the data into the given Complex array.
- void getData (const casacore::ValueHolder&);
- // Get the flags into the given bool array.
- void getFlags (const casacore::ValueHolder&);
- // Get the weights into the given float array.
- void getWeights (const casacore::ValueHolder&);
- // Get the UVWs into the given double array.
- void getUVW (const casacore::ValueHolder&);
- // Get the model data into the given Complex array.
- void getModelData (const casacore::ValueHolder&);
- // Execute the process function of the next step.
- // The record should contain the changed buffer fields.
- bool processNext (const casacore::Record&);
- // </group>
-
- private:
- //# Data members.
- DPInput* itsInput;
- std::string itsName;
- ParameterSet itsParset;
- DPBuffer itsBufIn;
- DPBuffer itsBufTmp;
- DPBuffer itsBufOut;
- bool itsNChanChg;
- bool itsNBlChg;
- std::string itsPythonClass;
- std::string itsPythonModule;
- NSTimer itsTimer;
- boost::python::object itsPyObject;
- };
-
- } //# end namespace
-}
-
-// Define the function (without name mangling) to register the 'constructor'.
-extern "C"
-{
- void register_pythondppp();
-}
-
-#endif
diff --git a/CEP/DP3/PythonDPPP/src/CMakeLists.txt b/CEP/DP3/PythonDPPP/src/CMakeLists.txt
deleted file mode 100644
index 1fff2d9aa4443d7ef6b05bfc27b1418a217d127a..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/src/CMakeLists.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-# $Id: CMakeLists.txt 30439 2014-11-19 15:04:34Z dijkema $
-
-include(LofarPackageVersion)
-include(PythonInstall)
-
-lofar_add_library(_pythondppp MODULE PythonDPPP.cc)
-set_target_properties(_pythondppp PROPERTIES
- PREFIX ""
- LIBRARY_OUTPUT_DIRECTORY ${PYTHON_BUILD_DIR}/lofar/pythondppp)
-
-# This is a quick-and-dirty fix to install the Python binding module in the
-# right place. It will now be installed twice, because lofar_add_library()
-# will install it in $prefix/$libdir
-install(TARGETS _pythondppp
- DESTINATION ${PYTHON_INSTALL_DIR}/lofar/pythondppp)
-
-# We only need this library because dependent packages need to link against
-# this package's version info.
-lofar_add_library(dppp_pythondppp
- Package__Version.cc
- PythonStep.cc
- DPStepBase.cc
-)
-
-lofar_add_bin_program(versionpythondppp versionpythondppp.cc)
-
-# Install Python modules
-python_install(__init__.py DESTINATION lofar/pythondppp)
diff --git a/CEP/DP3/PythonDPPP/src/DPStepBase.cc b/CEP/DP3/PythonDPPP/src/DPStepBase.cc
deleted file mode 100644
index ffa1b9595c4a006c99721c7069097a5c5e8ca51e..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/src/DPStepBase.cc
+++ /dev/null
@@ -1,40 +0,0 @@
-//# DPStepBase.cc: Python base class for a DPStep in python
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: pyparameterset.cc 23074 2012-12-03 07:51:29Z diepen $
-
-#include <lofar_config.h>
-#include <PythonDPPP/DPStepBase.h>
-
-// The C++ PythonStep must be able to call functions in Python.
-// But the Python functions must be able to call the C++ functions
-// (e.g., to get data, flags, etc.)
-// All communication goes through DPStepBase
-// Let a python step be created by a static function creating a
-// PythonWorker doing the actual work. Its pointer is kept in a static
-// and thereafter used to create a PythonStep.
-
-namespace LOFAR {
- namespace DPPP {
-
- // Define the static.
- DPStepBase* DPStepBase::theirPtr=0;
-
- }
-}
diff --git a/CEP/DP3/PythonDPPP/src/PythonDPPP.cc b/CEP/DP3/PythonDPPP/src/PythonDPPP.cc
deleted file mode 100644
index feee455096d5101adf385ff825c1f9f48e765c72..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/src/PythonDPPP.cc
+++ /dev/null
@@ -1,95 +0,0 @@
-//# PythonDPPP.cc: Python base class for a DPStep in python
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: pyparameterset.cc 23074 2012-12-03 07:51:29Z diepen $
-
-#include <lofar_config.h>
-#include <PythonDPPP/DPStepBase.h>
-
-#if defined(HAVE_CASACORE)
-#include <casacore/python/Converters/PycExcp.h>
-#include <casacore/python/Converters/PycBasicData.h>
-#include <casacore/python/Converters/PycValueHolder.h>
-#include <casacore/python/Converters/PycRecord.h>
-#define pyrap python
-#else
-#include <pyrap/Converters/PycExcp.h>
-#include <pyrap/Converters/PycBasicData.h>
-#include <pyrap/Converters/PycValueHolder.h>
-#include <pyrap/Converters/PycRecord.h>
-#endif
-#include <boost/python.hpp>
-#include <boost/python/args.hpp>
-
-using namespace boost::python;
-
-// The C++ PythonStep must be able to call functions in Python.
-// But the Python functions must be able to call the C++ functions
-// (e.g., to get data, flags, etc.)
-// All communication goes through DPStepBase
-// Let a python step be created by a static function creating a
-// PythonWorker doing the actual work. Its pointer is kept in a static
-// and thereafter used to create a PythonStep.
-
-namespace LOFAR {
- namespace DPPP {
-
- // Define the interface for the PythonStep C++ functions callable
- // from python.
- // Note that the python functions called from C++ are done using
- // the boost::python attr function.
- void dpstepbase()
- {
- class_<DPStepBase> ("_DPStepBase")
- .def (init<>())
- .def ("_getData", &DPStepBase::_getData,
- "Get the visibility data into the given array",
- (boost::python::arg("value")))
- .def ("_getFlags", &DPStepBase::_getFlags,
- "Get the flags into the given array",
- (boost::python::arg("value")))
- .def ("_getWeights", &DPStepBase::_getWeights,
- "Get the weights into the given array",
- (boost::python::arg("value")))
- .def ("_getUVW", &DPStepBase::_getUVW,
- "Get the UVW coordinates into the given array",
- (boost::python::arg("value")))
- .def ("_getModelData", &DPStepBase::_getModelData,
- "Get the model data into the given array",
- (boost::python::arg("value")))
- .def ("_processNext", &DPStepBase::_processNext,
- "Process the next step in the DPPP run",
- (boost::python::arg("values")))
- ;
- }
-
- }
-}
-
-
-// Define the python module itself.
-BOOST_PYTHON_MODULE(_pythondppp)
-{
- casacore::pyrap::register_convert_excp();
- casacore::pyrap::register_convert_basicdata();
- casacore::pyrap::register_convert_casa_valueholder();
- casacore::pyrap::register_convert_casa_record();
-
- LOFAR::DPPP::dpstepbase();
-}
diff --git a/CEP/DP3/PythonDPPP/src/PythonStep.cc b/CEP/DP3/PythonDPPP/src/PythonStep.cc
deleted file mode 100644
index bdc50fcbc1b91e0c019c476a5d934b03cd689a49..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/src/PythonStep.cc
+++ /dev/null
@@ -1,369 +0,0 @@
-//# PythonStep.cc: A DPStep executed in some python module
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: ApplyCal.cc 21598 2012-07-16 08:07:34Z diepen $
-//#
-//# @author Ger van Diepen
-
-#include <lofar_config.h>
-#include <PythonDPPP/PythonStep.h>
-#include <PythonDPPP/DPStepBase.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPRun.h>
-#include <Common/ParameterSet.h>
-
-#if defined(HAVE_CASACORE)
-#include <casacore/python/Converters/PycExcp.h>
-#include <casacore/python/Converters/PycBasicData.h>
-#include <casacore/python/Converters/PycValueHolder.h>
-#include <casacore/python/Converters/PycRecord.h>
-#include <casacore/python/Converters/PycArray.h>
-#define pyrap python
-#else
-#include <pyrap/Converters/PycExcp.h>
-#include <pyrap/Converters/PycBasicData.h>
-#include <pyrap/Converters/PycValueHolder.h>
-#include <pyrap/Converters/PycRecord.h>
-#include <pyrap/Converters/PycArray.h>
-#endif
-
-#include <casacore/casa/OS/Path.h>
-#include <unistd.h>
-
-using namespace casacore;
-
-namespace LOFAR {
- namespace DPPP {
-
- PythonStep::PythonStep (DPInput* input,
- const ParameterSet& parset,
- const string& prefix)
- : itsInput (input),
- itsName (prefix),
- itsParset (parset.makeSubset (prefix)),
- itsNChanChg (false),
- itsNBlChg (false),
- itsPythonClass (itsParset.getString ("python.class")),
- itsPythonModule (itsParset.getString ("python.module", itsPythonClass))
- {
- // Initialize Python interpreter.
- // Note: a second call is a no-op.
- Py_Initialize();
- // Insert the current working directory into the python path.
- // (from http://stackoverflow.com/questions/9285384/
- // how-does-import-work-with-boost-python-from-inside-python-files)
- string workingDir = Path(".").absoluteName();
- char path[] = "path"; // needed to avoid warning if "path" used below
- PyObject* sysPath = PySys_GetObject(path);
-#if PYTHON_VERSION_MAJOR < 3
- PyList_Insert (sysPath, 0, PyString_FromString(workingDir.c_str()));
-#else
- PyList_Insert (sysPath, 0, PyUnicode_FromString(workingDir.c_str()));
-#endif
- // Register converters for casa types from/to python types
- casacore::pyrap::register_convert_excp();
- casacore::pyrap::register_convert_basicdata();
- casacore::pyrap::register_convert_casa_valueholder();
- casacore::pyrap::register_convert_casa_record();
- try {
- // First import main
- boost::python::object mainModule = boost::python::import
- ("__main__");
- // Import the given module
- boost::python::object dpppModule = boost::python::import
- (itsPythonModule.c_str());
- // Get the python class object from the imported module
- boost::python::object dpppAttr = dpppModule.attr
- (itsPythonClass.c_str());
-
- // Convert the ParameterSet to a Record (using its string values).
- Record rec;
- for (ParameterSet::const_iterator iter=itsParset.begin();
- iter!=itsParset.end(); ++iter) {
- rec.define (iter->first, iter->second.get());
- }
- // Create an instance of the python class passing the record.
- itsPyObject = dpppAttr(rec);
- // Set the pointer to this object in the DPStepBase object.
- DPStepBase::theirPtr->setStep (this);
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- PythonStep::~PythonStep()
- {
- }
-
- DPStep::ShPtr PythonStep::makeStep (DPInput* input,
- const ParameterSet& pset,
- const std::string& prefix)
- {
- return DPStep::ShPtr(new PythonStep(input, pset, prefix));
- }
-
- void PythonStep::updateInfo (const DPInfo& infoIn)
- {
- try {
- boost::python::object result =
- itsPyObject.attr("_updateInfo")(infoIn.toRecord());
- Record rec = boost::python::extract<Record>(result);
- info() = infoIn;
- // Merge possible result back in DPInfo object.
- info().fromRecord (rec);
- // Check if nr of channels has changed.
- // Note: currently a change in antennae/baselines is not supported
- // because the antenna positions are not passed yet.
- if (infoIn.nchan() != info().nchan()) {
- itsNChanChg = true;
- }
- ASSERT (infoIn.getAnt1().size() == info().getAnt1().size() &&
- infoIn.getAnt2().size() == info().getAnt2().size() &&
- allEQ(infoIn.getAnt1(), info().getAnt1()) &&
- allEQ(infoIn.getAnt2(), info().getAnt2()));
- boost::python::object res1 = itsPyObject.attr("needVisData")();
- if (boost::python::extract<bool>(res1)) {
- info().setNeedVisData();
- }
- boost::python::object res2 = itsPyObject.attr("needWrite")();
- if (boost::python::extract<bool>(res2)) {
- info().setWriteData();
- }
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- bool PythonStep::process (const DPBuffer& buf)
- {
- try {
- itsTimer.start();
- itsBufIn.referenceFilled (buf);
- boost::python::object result =
- itsPyObject.attr("process")(itsBufIn.getTime(),
- itsBufIn.getExposure());
- bool res = boost::python::extract<bool> (result);
- itsTimer.stop();
- return res;
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- void PythonStep::finish()
- {
- try {
- itsPyObject.attr("finish")();
- getNextStep()->finish();
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- void PythonStep::addToMS (const string& msName)
- {
- try {
- itsPyObject.attr("addToMS")(msName);
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- void PythonStep::show (std::ostream& os) const
- {
- try {
- boost::python::object result = itsPyObject.attr("show")();
- string str = boost::python::extract<string>(result);
- os << "PythonStep " << itsName << " class=" << itsPythonClass << endl;
- if (! str.empty()) {
- os << str;
- }
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- void PythonStep::showCounts (std::ostream& os) const
- {
- try {
- boost::python::object result = itsPyObject.attr("showCounts")();
- string str = boost::python::extract<string>(result);
- if (! str.empty()) {
- os << str;
- }
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- void PythonStep::showTimings (std::ostream& os, double duration) const
- {
- try {
- os << " ";
- FlagCounter::showPerc1 (os, itsTimer.getElapsed(), duration);
- os << " PythonStep " << itsName << " class=" << itsPythonClass << endl;
- boost::python::object result =
- itsPyObject.attr("showTimings")(itsTimer.getElapsed());
- string str = boost::python::extract<string>(result);
- if (! str.empty()) {
- os << str;
- }
- } catch (boost::python::error_already_set const &) {
- // handle the exception in some way
- PyErr_Print();
- throw;
- }
- }
-
- // Implement all functions to communicate with python.
- void PythonStep::setNeedVisData()
- {
- info().setNeedVisData();
- }
- void PythonStep::setNeedWrite()
- {
- info().setWriteData();
- }
-
- void PythonStep::getData (const ValueHolder& vh)
- {
- ASSERT (vh.dataType() == TpArrayComplex);
- Array<Complex> arr(vh.asArrayComplex());
- arr = itsBufIn.getData(); // operator= checks if shape matches
- }
-
- void PythonStep::getFlags (const ValueHolder& vh)
- {
- ASSERT (vh.dataType() == TpArrayBool);
- Array<Bool> arr(vh.asArrayBool());
- arr = itsBufIn.getFlags();
- }
-
- void PythonStep::getWeights (const ValueHolder& vh)
- {
- ASSERT (vh.dataType() == TpArrayFloat);
- Array<Float> arr(vh.asArrayFloat());
- arr = itsInput->fetchWeights (itsBufIn, itsBufTmp, itsTimer);
- }
-
- void PythonStep::getUVW (const ValueHolder& vh)
- {
- ASSERT (vh.dataType() == TpArrayDouble);
- Array<Double> arr(vh.asArrayDouble());
- arr = itsInput->fetchUVW (itsBufIn, itsBufTmp, itsTimer);
- }
-
- void PythonStep::getModelData (const ValueHolder& vh)
- {
- ASSERT (vh.dataType() == TpArrayComplex);
- Cube<Complex> arr(vh.asArrayComplex());
- itsInput->getModelData (RefRows(itsBufIn.getRowNrs()), arr);
- }
-
- bool PythonStep::processNext (const Record& rec)
- {
- itsTimer.stop();
- uint nproc = 0;
- uint narr = 0;
- if (rec.isDefined("TIME")) {
- itsBufOut.setTime (rec.asDouble("TIME"));
- nproc++;
- } else {
- itsBufOut.setTime (itsBufIn.getTime());
- }
- if (rec.isDefined("EXPOSURE")) {
- itsBufOut.setExposure (rec.asDouble("EXPOSURE"));
- nproc++;
- } else {
- itsBufOut.setExposure (itsBufIn.getExposure());
- }
- itsBufOut.setRowNrs (itsBufIn.getRowNrs());
- if (rec.isDefined("DATA")) {
- itsBufOut.getData().assign (rec.toArrayComplex("DATA"));
- narr++;
- } else if (! itsNChanChg && ! itsNBlChg) {
- itsBufOut.getData().assign (itsBufIn.getData());
- }
- if (rec.isDefined("FLAGS")) {
- itsBufOut.getFlags().assign (rec.toArrayBool("FLAGS"));
- narr++;
- } else if (! itsNChanChg && ! itsNBlChg) {
- itsBufOut.getFlags().assign (itsBufIn.getFlags());
- }
- if (rec.isDefined("FULLRESFLAGS")) {
- itsBufOut.getFullResFlags().assign (rec.toArrayBool("FULLRESFLAGS"));
- narr++;
- } else if (! itsNChanChg && ! itsNBlChg) {
- itsBufOut.getFullResFlags().assign (itsBufIn.getFullResFlags());
- }
- if (rec.isDefined("WEIGHTS")) {
- itsBufOut.getWeights().assign (rec.toArrayFloat("WEIGHTS"));
- narr++;
- } else if (! itsNChanChg && ! itsNBlChg) {
- if (! itsBufIn.getWeights().empty()) {
- itsBufOut.getWeights().assign (itsBufIn.getWeights());
- }
- }
- if (rec.isDefined("UVW")) {
- itsBufOut.getUVW().assign (rec.toArrayDouble("UVW"));
- narr++;
- } else if (! itsNChanChg && ! itsNBlChg) {
- if (! itsBufIn.getUVW().empty()) {
- itsBufOut.getUVW().assign (itsBufIn.getUVW());
- }
- }
- nproc += narr;
- if (nproc != rec.nfields()) {
- THROW (Exception,
- "Record/dict given to processNext() contains unknown fields");
- }
- if ((itsNChanChg || itsNBlChg) && narr != 4) {
- THROW (Exception,
- "Record/dict given to processNext() must contain DATA, FLAGS, "
- "WEIGHTS, and UVW if the nr of channels or baselines changes");
- }
- bool res = getNextStep()->process (itsBufOut);
- itsTimer.start();
- return res;
- }
-
- } //# end namespace
-}
-
-// Define the function to make the PythonStep 'constructor' known.
-void register_pythondppp()
-{
- LOFAR::DPPP::DPRun::registerStepCtor ("pythondppp",
- LOFAR::DPPP::PythonStep::makeStep);
-}
diff --git a/CEP/DP3/PythonDPPP/src/__init__.py b/CEP/DP3/PythonDPPP/src/__init__.py
deleted file mode 100644
index afbd75d7c0422479bec042f59fd1b32831d1fd0c..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/src/__init__.py
+++ /dev/null
@@ -1,272 +0,0 @@
-# __init__.py: Top level .py file for python DPPP step
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: __init__.py 23074 2012-12-03 07:51:29Z diepen $
-
-from lofar.pythondppp._pythondppp import _DPStepBase
-import numpy as np
-
-
-class DPStep(_DPStepBase):
- """
- The superclass for a python DPPP step.
- An DPPP step implemented in Python must derive from this class.
- An example can be seen in CEP/DP3/PythonDPPP/test/tPythonStep.py.
- """
-
- def __init__(self, parset):
- """ This constructor must be called by the subclass """
- _DPStepBase.__init__(self)
- self.itsParset = parset
-
- def _updateInfo(self, dpinfo):
- """ Private function (called by C++ layer) to update the info.
-
- The dpinfo argument is a dict containing all fields in the
- C++ DPInfo object. The names of the keys in the dict are the
- DPInfo member names without their 'its' prefix.
-
- This function extracts the information telling the buffer sizes.
- Thereafter it calls updateInfo to let a subclass extract and
- optionally set the info.
- Finally it extracts the information again to get the output sizes.
-
- """
- # Get the input sizes.
- self.itsNCorrIn = dpinfo['NCorr']
- self.itsNChanIn = dpinfo['NChan']
- self.itsNBlIn = len(dpinfo['Ant1'])
- # Default output size is the input size.
- self.itsNCorrOut = self.itsNCorrIn
- self.itsNChanOut = self.itsNChanIn
- self.itsNBlOut = self.itsNBlIn
- # Let the subclass extract and set info.
- infoOut = self.updateInfo(dpinfo)
- # Extract the output sizes (if defined).
- if 'NCorr' in infoOut:
- self.itsNCorrOut = infoOut['NCorr']
- if 'NChan' in infoOut:
- self.itsNChanOut = infoOut['NChan']
- if 'Ant1' in infoOut:
- self.itsNBlOut = len(infoOut['Ant1'])
- return infoOut
-
- # The following functions can be overwritten in a subclass.
- def updateInfo(self, dpinfo):
- """ Extract and optionally set the DPInfo fields.
-
- This function must be implemented in a subclass.
-
- The dpinfo argument is a dict containing all fields in the
- C++ DPInfo object. The names of the keys in the dict are the
- DPInfo member names without their 'its' prefix.
-
- It must return a (possibly empty) dict containing values of the
- same keys. Only the keys that have changed must be part of the
- dict; other keys can be part of it.
-
- """
- raise ValueError("A class derived from DPStep must implement updateInfo")
-
- def needVisData(self):
- """ Does the subclass need the visibility data?
-
- This function only needs to be implemented in a subclass
- if it does not need the visibility data.
-
- """
- return True
-
- def needWrite(self):
- """ Does the subclass change data to be written by DPPP?
-
- This function needs to be implemented in a subclass
- if it changes data (visibility data, flags, weights, and/or UVW)
- that needs to be written (or possibly updated) in the MS.
-
- """
- return False
-
- def process(self, time, exposure):
- """ Process the data of the given time slot.
-
- This function must be implemented in a subclass.
- it will need the functions getData, etc. to get the visibility data,
- flags, etc..
- It should call processNext to process the next DPPP step on the
- output of this step.
-
- """
- raise ValueError("A class derived from DPStep must implement process")
-
- def finish(self):
- """ Finish the processing.
-
- This function must be implemented in a subclass if data were
- buffered and the last buffers have not been (fully) processed yet.
- In that case finish also has to call processNext.
-
- """
- pass
-
- def show(self):
- """ Show the parset parameters.
-
- It should return a string (with newlines) that will be
- printed by the C++ layer.
- An empty string is not printed.
-
- This function does not need to be implemented.
- The default implementation shows all parset keys.
- """
- s = ''
- for k,v in list(self.itsParset.items()):
- if k not in ['type', 'python.class', 'python.module']:
- s += ' %-15s %s\n' % (k+':', v)
- return s
-
- def showCounts(self):
- """ Show possible counts (e.g., nr of flags set).
-
- It should return a string (with newlines) that will be
- printed by the C++ layer.
- An empty string is not printed.
-
- This function does not need to be implemented.
- """
- return ''
-
- def showTimings(self, elapsedDuration):
- """ Show possible timings.
-
- It should return a string (with newlines) that will be
- printed by the C++ layer.
- An empty string is not printed.
-
- This function does not need to be implemented.
- """
- return ''
-
- def addToMS(self, msname):
- """ Add information to the output MeasurementSet.
-
- This function will only be needed in very special cases
- where dedicated info needs to be added to the MS.
-
- """
- pass
-
-
- # The following functions are to be called from Python.
- def processNext(self, arraysDict):
- """ Let the next step in the DPPP pipeline execute its process.
-
- If data (time, exposure, visibility, flags, weights, and/or UVW)
- have been changed by the process function, they should be passed
- using the arraysDict argument.
- This is a dict that can contain 6 fields: TIME, EXPOSURE, DATA,
- FLAGS, WEIGHTS, and UVW.
-
- If process does not buffer data, the dict only needs to contain
- the changed data items. But if buffered and pocessNext is called
- later, all data items have to be part of the dict.
- """
- return self._processNext(arraysDict)
-
- def getData(self, nparray):
- """ Get the visibility data into the given numpy array.
-
- The array must be a contiguous array of the complex64 data type and
- the correct shape.
- The array can be created with the makeDataArrayIn function.
-
- """
- if not nparray.flags.c_contiguous or nparray.size == 0:
- raise ValueError("getData argument 'nparray' has to be a contiguous numpy a\
-rray")
- return self._getData (nparray)
-
- def getFlags(self, nparray):
- """ Get the flags into the given numpy array.
-
- The array must be a contiguous array of the boolean data type and
- the correct shape.
- The array can be created with the makeFlagsArrayIn function.
-
- """
- if not nparray.flags.c_contiguous or nparray.size == 0:
- raise ValueError("getFlags argument 'nparray' has to be a contiguous numpy a\
-rray")
- return self._getFlags (nparray)
-
- def getWeights(self, nparray):
- """ Get the weights into the given numpy array.
-
- The array must be a contiguous array of the float32 data type and
- the correct shape.
- The array can be created with the makeWeightsArrayIn function.
-
- """
- if not nparray.flags.c_contiguous or nparray.size == 0:
- raise ValueError("getWeights argument 'nparray' has to be a contiguous numpy a\
-rray")
- return self._getWeights (nparray)
-
- def getUVW(self, nparray):
- """ Get the UVW coordinates into the given numpy array.
-
- The array must be a contiguous array of the float64 data type and
- the correct shape.
- The array can be created with the makeUVWArrayIn function.
-
- """
- if not nparray.flags.c_contiguous or nparray.size == 0:
- raise ValueError("getUVW argument 'nparray' has to be a contiguous numpy a\
-rray")
- return self._getUVW (nparray)
-
- def getModelData(self, nparray):
- """ Get the model data into the given numpy array.
-
- The array must be a contiguous array of the complex64 data type and
- the correct shape.
- The array can be created with the makeDataArrayIn function.
-
- """
- if not nparray.flags.c_contiguous or nparray.size == 0:
- raise ValueError("getModelData argument 'nparray' has to be a contiguous numpy a\
-rray")
- return self._getModelData (nparray)
-
- def makeArrayDataIn(self):
- """ Make a numpy array for the visibility input data. """
- return np.empty([self.itsNBlIn, self.itsNChanIn, self.itsNCorrIn], dtype='complex64')
-
- def makeArrayFlagsIn(self):
- """ Make a numpy array for the input flags. """
- return np.empty([self.itsNBlIn, self.itsNChanIn, self.itsNCorrIn], dtype='bool')
-
- def makeArrayWeightsIn(self):
- """ Make a numpy array for the input weights. """
- return np.empty([self.itsNBlIn, self.itsNChanIn, self.itsNCorrIn], dtype='float32')
-
- def makeArrayUVWIn(self):
- """ Make a numpy array for the input UVW coordinates. """
- return np.empty([self.itsNBlIn, 3], dtype='float64')
-
diff --git a/CEP/DP3/PythonDPPP/test/CMakeLists.txt b/CEP/DP3/PythonDPPP/test/CMakeLists.txt
deleted file mode 100644
index e5af0b34e2f6598516fed409fe1078463c37bcde..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/test/CMakeLists.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# $Id: CMakeLists.txt 26355 2013-09-04 11:51:07Z dijkema $
-
-include(LofarCTest)
-
-lofar_add_test(tPythonStep)
diff --git a/CEP/DP3/PythonDPPP/test/tPythonStep.py b/CEP/DP3/PythonDPPP/test/tPythonStep.py
deleted file mode 100644
index 4bb03c8beed85bb9224264bb5f6e5117578d5d46..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/test/tPythonStep.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# tPythonStep.py: Test python DPPP class
-# Copyright (C) 2015
-# ASTRON (Netherlands Institute for Radio Astronomy)
-# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-#
-# This file is part of the LOFAR software suite.
-# The LOFAR software suite is free software: you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# The LOFAR software suite is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-#
-# $Id: __init__.py 23074 2012-12-03 07:51:29Z diepen $
-
-
-
-from lofar.pythondppp import DPStep
-from lofar.parameterset import parameterset
-
-class tPythonStep(DPStep):
- def __init__(self, parsetDict):
- # The constructor gets the subset of the NDPPP parset containing
- # all keys-value pairs for this step.
- # Note: the superclass constructor MUST be called.
- DPStep.__init__(self, parsetDict)
- parset = parameterset(parsetDict)
- self.itsIncr = parset.getDouble('incr', 1)
-
- def updateInfo(self, dpinfo):
- # This function must be implemented.
- self.itsInfo = dpinfo
- # Make the arrays that will get the input buffer data from
- # the getData, etc. calls in the process function.
- self.itsData = self.makeArrayDataIn()
- self.itsFlags = self.makeArrayFlagsIn()
- self.itsWeights = self.makeArrayWeightsIn()
- self.itsUVW = self.makeArrayUVWIn()
- # Return the dict with info fields that change in this step.
- return {};
-
- def process(self, time, exposure):
- # This function must be implemented.
- # First get the data arrays needed by this step.
- self.getData (self.itsData);
- self.getFlags (self.itsFlags);
- self.getWeights (self.itsWeights);
- self.getUVW (self.itsUVW);
- # Process the data.
- print("process tPythonStep", time-4.47203e9, exposure, self.itsData.sum(), self.itsFlags.sum(), self.itsWeights.sum(), self.itsUVW.sum())
- # Execute the next step in the DPPP pipeline. TIME,UVW are changed.
- return self.processNext ({'TIME': time+self.itsIncr, 'UVW': self.itsUVW+self.itsIncr})
-
- def finish(self):
- # Finish the step as needed.
- # This function does not need to be implemented.
- # Note: finish of the next step is called by the C++ layer.
- print("finish tPythonStep")
-
- def showCounts(self):
- # Show the counts of this test.
- # This function does not need to be implemented.
- return " **showcounttest**"
-
- def addToMS(self, msname):
- # Add some info the the output MeasurementSet.
- # This function does not need to be implemented.
- print("addToMS tPythonStep", msname)
diff --git a/CEP/DP3/PythonDPPP/test/tPythonStep.run b/CEP/DP3/PythonDPPP/test/tPythonStep.run
deleted file mode 100755
index 3d7416429f9567f36e4acab05f74843784fe7536..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/test/tPythonStep.run
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/sh
-
-# Set srcdir if not defined (in case run by hand).
-if test "$srcdir" = ""; then
- srcdir=`find ../../../../../.. -name PythonDPPP | grep -v build/ | grep -v /include/`/test
-fi
-
-if test ! -f $srcdir/../../DPPP/test/tNDPPP.in_MS.tgz; then
- return 3 # untested
-fi
-
-# Get the test MS and rename it to be deleted automatically.
-tar zxf $srcdir/../../DPPP/test/tNDPPP.in_MS.tgz
-mv tNDPPP_tmp.MS tPythonStep_tmp.MS
-
-# Define the path for finding the library.
-LD_LIBRARY_PATH=../src:$LD_LIBRARY_PATH
-DYLD_LIBRARY_PATH=../src:$DYLD_LIBRARY_PATH
-export LD_LIBRARY_PATH
-export DYLD_LIBRARY_PATH
-
-# Run NDPPP with the test python step.
-NDPPP msin=tPythonStep_tmp.MS msout=tPythonStep_tmp.msout msout.overwrite=T steps='[pystep,pystep2]' pystep.type=PythonDPPP pystep.python.module=tPythonStep pystep.python.class=tPythonStep pystep.somekey=somevalue pystep2.type=pythondppp pystep2.python.class=tPythonStep pystep2.incr=3.5
-
diff --git a/CEP/DP3/PythonDPPP/test/tPythonStep.sh b/CEP/DP3/PythonDPPP/test/tPythonStep.sh
deleted file mode 100755
index 4b2a0f3d7d05ab6b31b572d39c3dc8957768185b..0000000000000000000000000000000000000000
--- a/CEP/DP3/PythonDPPP/test/tPythonStep.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tPythonStep
diff --git a/CEP/DP3/SPWCombine/CMakeLists.txt b/CEP/DP3/SPWCombine/CMakeLists.txt
deleted file mode 100644
index dd56a319e4d25a975e6821fbb1ba26e9e553f6e2..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/CMakeLists.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# $Id$
-
-lofar_package(SPW_Combine 1.0 DEPENDS Common PLC)
-
-include(LofarFindPackage)
-lofar_find_package(Casacore COMPONENTS casa ms tables REQUIRED)
-
-add_subdirectory(include/SPWCombine)
-add_subdirectory(src)
-#add_subdirectory(test)
diff --git a/CEP/DP3/SPWCombine/include/SPWCombine/CMakeLists.txt b/CEP/DP3/SPWCombine/include/SPWCombine/CMakeLists.txt
deleted file mode 100644
index 04c3c85ff94a95e0aef01ba9bc9e1f3a3df528d0..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/include/SPWCombine/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-# $Id$
-
-# List of header files that will be installed.
-set(inst_HEADERS
- CombinerProcessControl.h
- SPWCombine.h)
-
-# Create symbolic link to include directory.
-execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
- ${CMAKE_CURRENT_SOURCE_DIR}
- ${CMAKE_BINARY_DIR}/include/SPWCombine)
-
-# Install header files.
-install(FILES ${inst_HEADERS} DESTINATION include/${PACKAGE_NAME})
diff --git a/CEP/DP3/SPWCombine/include/SPWCombine/CombinerProcessControl.h b/CEP/DP3/SPWCombine/include/SPWCombine/CombinerProcessControl.h
deleted file mode 100644
index 3b258afba99e43dfb0a750e5a9ac9406a50231b9..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/include/SPWCombine/CombinerProcessControl.h
+++ /dev/null
@@ -1,91 +0,0 @@
-//# Copyright (C) 2007
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Adriaan Renting
-
-#ifndef LOFARCOMBINERPROCESSCONTROL_H
-#define LOFARCOMBINERPROCESSCONTROL_H
-
-#include <PLC/ProcessControl.h>
-#include <Common/ParameterSet.h>
-#include <vector>
-
-/**
-@author Adriaan Renting
-*/
-namespace casacore
-{
- class MeasurementSet; //foreward declaration
-}
-
-namespace LOFAR
-{
- namespace CS1
- {
- class SPWCombine; //foreward declaration
-
- class CombinerProcessControl : public LOFAR::ACC::PLC::ProcessControl
- {
- private:
- vector<std::string> itsInMS;
- std::string itsOutMS;
- vector<casacore::MeasurementSet*> inMS;
- SPWCombine* itsCombiner;
- public:
- CombinerProcessControl(void);
-
- ~CombinerProcessControl(void);
- // \name Command to control the processes.
- // There are a dozen commands that can be sent to a application process
- // to control its flow. The return values for these command are:<br>
- // - True - Command executed succesfully.
- // - False - Command could not be executed.
- //
- // @{
-
- // During the \c define state the process check the contents of the
- // ParameterSet it received during start-up. When everthing seems ok the
- // process constructs the communication channels for exchanging data
- // with the other processes. The connection are NOT made in the stage.
- tribool define (void);
-
- // When a process receives an \c init command it allocates the buffers it
- // needs an makes the connections with the other processes. When the
- // process succeeds in this it is ready for dataprocessing (or whatever
- // task the process has).
- tribool init (void);
-
- // During the \c run phase the process does the work it is designed for.
- // The run phase stays active until another command is send.
- tribool run (void);
-
- tribool pause(const std::string&);
- tribool release(void);
- tribool quit(void);
- tribool recover(const std::string&);
- tribool reinit(const std::string&);
- tribool snapshot(const std::string&);
- std::string askInfo(const std::string&);
-
- }; //class CombinerProcessControl
- } //namespace CS1
-}; //namespace LOFAR
-
-#endif
diff --git a/CEP/DP3/SPWCombine/include/SPWCombine/SPWCombine.h b/CEP/DP3/SPWCombine/include/SPWCombine/SPWCombine.h
deleted file mode 100644
index b4bc0aaef1307c339233c598512751491ad1fff0..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/include/SPWCombine/SPWCombine.h
+++ /dev/null
@@ -1,64 +0,0 @@
-//# Copyright (C) 2007
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Adriaan Renting
-
-#ifndef LOFARSPWCOMBINE_H
-#define LOFARSPWCOMBINE_H
-
-/**
-@author Adriaan Renting
-*/
-#include <iostream>
-#include <cstdlib>
-#include <string>
-#include <casacore/ms/MeasurementSets.h>
-#include <casacore/casa/Arrays.h>
-#include <casacore/tables/Tables.h>
-#include <casacore/tables/Tables/TableIter.h>
-
-namespace LOFAR
-{
- namespace CS1
- {
- using namespace casacore;
-
- class SPWCombine
- {
- private:
- int itsNumAntennae;
- int itsNumPairs;
-
- TableIterator CreateDataIterator(MeasurementSet& myMS);
-
- public:
- SPWCombine(void);
- ~SPWCombine(void);
- int itsNumPolarizations;
- int itsNumChannels;
- int itsNumBands;
-
- void TableResize(TableDesc tdesc, IPosition ipos, std::string name, Table& table);
- void GetMSInfo(MeasurementSet& myMS);
- void Combine(vector<MeasurementSet*> inMS, MeasurementSet& myMS, std::string Data);
- };
- } //namespace CS1
-}; //namespace LOFAR
-#endif
diff --git a/CEP/DP3/SPWCombine/src/CMakeLists.txt b/CEP/DP3/SPWCombine/src/CMakeLists.txt
deleted file mode 100644
index 64a1f8d276f6545fe3383b8974fdc730e3d7d919..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/src/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# $Id$
-
-include(LofarPackageVersion)
-
-lofar_add_library(spwcombine
- Package__Version.cc)
-
-lofar_add_bin_program(combinespw
- SPWCombine.cc
- Combiner.cc
- CombinerProcessControl.cc)
-
-lofar_add_bin_program(versionspw_combine versionspw_combine.cc)
diff --git a/CEP/DP3/SPWCombine/src/Combiner.cc b/CEP/DP3/SPWCombine/src/Combiner.cc
deleted file mode 100644
index 44c542b2b509f452f96ff992315dd766b72c8456..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/src/Combiner.cc
+++ /dev/null
@@ -1,46 +0,0 @@
-//# Copyright (C) 2007
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Adriaan Renting
-
-#include <lofar_config.h>
-#include <Common/lofar_iostream.h>
-#include <Common/SystemUtil.h>
-#include <Common/Exceptions.h>
-#include <PLC/ACCmain.h>
-#include <SPWCombine/CombinerProcessControl.h>
-
-using namespace LOFAR;
-
-// Use a terminate handler that can produce a backtrace.
-Exception::TerminateHandler t(Exception::terminate);
-
-int main(int argc, char *argv[])
-{
- try {
- INIT_LOGGER(LOFAR::basename(argv[0]));
- LOFAR::CS1::CombinerProcessControl myProcess;
- return LOFAR::ACC::PLC::ACCmain(argc, argv, &myProcess);
- } catch(Exception& ex) {
- cerr << ex << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/SPWCombine/src/CombinerProcessControl.cc b/CEP/DP3/SPWCombine/src/CombinerProcessControl.cc
deleted file mode 100644
index 308f1b3be841f154a924cd837a936ca68e839dcb..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/src/CombinerProcessControl.cc
+++ /dev/null
@@ -1,271 +0,0 @@
-//# Copyright (C) 2007
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Adriaan Renting
-
-#include <lofar_config.h>
-#include <iostream>
-#include <cstdlib>
-#include <string>
-#include <limits>
-#include <casacore/tables/Tables.h>
-#include <casacore/tables/TaQL/TableParse.h>
-#include <casacore/ms/MeasurementSets.h>
-#include <casacore/casa/Exceptions.h>
-
-#include <SPWCombine/CombinerProcessControl.h>
-#include <SPWCombine/SPWCombine.h>
-
-#define COMBINER_VERSION "0.22"
-// 0.10 Initial version based on DataSquasher
-// 0.20 Ported additions and updates from DataSquasher
-// 0.21 Fixed calculation of REF_FREQUENCY
-// 0.22 Added handing of Measurementsets with different numbers of timesamples
-
-namespace LOFAR
-{
- namespace CS1
- {
- using namespace casacore;
- //===============>>> CombinerProcessControl::CombinerProcessControl <<<===============
- CombinerProcessControl::CombinerProcessControl()
- : ProcessControl()
- {
- itsCombiner = NULL;
- }
-
- //===============>>> CombinerProcessControl::~CombinerProcessControl <<<==============
- CombinerProcessControl::~CombinerProcessControl()
- {
- }
-
- //===============>>> CombinerProcessControl::define <<<==============================
- tribool CombinerProcessControl::define()
- {
- ParameterSet* ParamSet = globalParameterSet();
- itsInMS = ParamSet->getStringVector("inms");
- itsOutMS = ParamSet->getString("outms");
- return true;
- }
-
- //===============>>> CombinerProcessControl::run <<<=================================
- tribool CombinerProcessControl::run()
- {
- try{
- std::cout << "Creating " << itsOutMS << ", please wait..." << std::endl;
- unsigned int min_nrow = std::numeric_limits<unsigned int>::max();
- int sourceMS = 0;
- for (unsigned int i = 0; i < itsInMS.size(); i++) // to search the shortest MS
- {
- if (inMS[i]->nrow() < min_nrow)
- {
- min_nrow = inMS[i]->nrow();
- sourceMS = i;
- }
- }
- if (sourceMS > 0)
- { cout << "Not all sources are the same lenght, using te shortest one." << endl;
- }
-
- Table TempTable = tableCommand(string("SELECT UVW,FLAG_CATEGORY,WEIGHT,SIGMA,ANTENNA1,ANTENNA2,ARRAY_ID,DATA_DESC_ID,") +
- string("EXPOSURE,FEED1,FEED2,FIELD_ID,FLAG_ROW,INTERVAL,OBSERVATION_ID,PROCESSOR_ID,") +
- string("SCAN_NUMBER,STATE_ID,TIME,TIME_CENTROID,WEIGHT_SPECTRUM,FLAG FROM ")
- + itsInMS[sourceMS] + string(" WHERE DATA_DESC_ID = 0"));
- // Need FLAG to make it a valid MS
- TempTable.deepCopy(itsOutMS, Table::NewNoReplace, true);
- tableCommand(string("DELETE FROM ") + itsOutMS + string("/DATA_DESCRIPTION WHERE rownumber() > 1"));
- tableCommand(string("DELETE FROM ") + itsOutMS + string("/SPECTRAL_WINDOW WHERE rownumber() > 1"));
-
- MeasurementSet outMS = MeasurementSet(itsOutMS, Table::Update);
- int nchan = 0;
- for (unsigned int i = 0; i < itsInMS.size(); i++)
- {
- itsCombiner->GetMSInfo(*(inMS[i]));
- for (int j = 0; j < itsCombiner->itsNumBands; j++)
- { nchan += itsCombiner->itsNumChannels;
- }
- }
- TableDesc tdesc = inMS[sourceMS]->tableDesc();
- Vector<Int> temp(2);
- temp(0) = itsCombiner->itsNumPolarizations;
- temp(1) = nchan;
- std::cout << "New number of channels: " << nchan << std::endl;
- IPosition data_ipos(temp);
-
- itsCombiner->TableResize(tdesc, data_ipos, "DATA", outMS);
-
- //fix the FLAGS column
- itsCombiner->TableResize(tdesc, data_ipos, "FLAG", outMS);
-
- //Fix the SpectralWindow values
- IPosition spw_ipos(1, nchan);
- //ugly workaround MSSpectral window does no allow deleting and then recreating columns
- Table outSPW = Table(itsOutMS + "/SPECTRAL_WINDOW", Table::Update);
- ScalarColumn<Int> channum(outSPW, "NUM_CHAN");
- channum.fillColumn(nchan);
-
- TableDesc SPWtdesc = outSPW.tableDesc();
- itsCombiner->TableResize(SPWtdesc, spw_ipos, "CHAN_FREQ", outSPW);
- itsCombiner->TableResize(SPWtdesc, spw_ipos, "CHAN_WIDTH", outSPW);
- itsCombiner->TableResize(SPWtdesc, spw_ipos, "EFFECTIVE_BW", outSPW);
- itsCombiner->TableResize(SPWtdesc, spw_ipos, "RESOLUTION", outSPW);
-
- ArrayColumn<Double> outFREQ(outSPW, "CHAN_FREQ");
- ArrayColumn<Double> outWIDTH(outSPW, "CHAN_WIDTH");
- ArrayColumn<Double> outBW(outSPW, "EFFECTIVE_BW");
- ArrayColumn<Double> outRESOLUTION(outSPW, "RESOLUTION");
- ScalarColumn<Double> outREF_FREQUENCY(outSPW, "REF_FREQUENCY");
-
- Vector<Double> new_FREQ(nchan, 0.0);
- Vector<Double> new_WIDTH(nchan, 0.0);
- Vector<Double> new_BW(nchan, 0.0);
- Vector<Double> new_RESOLUTION(nchan, 0.0);
- int total_channels = 0;
- int total_bands = 0;
- double ref_frequency = 0.0;
-
- for (unsigned int i = 0; i < itsInMS.size(); i++)
- {
- itsCombiner->GetMSInfo(*(inMS[i]));
- int old_nchan = itsCombiner->itsNumChannels;
- Vector<Double> old_temp(old_nchan, 0.0);
-
- MSSpectralWindow inSPW = inMS[i]->spectralWindow();
-
- ROArrayColumn<Double> inFREQ(inSPW, "CHAN_FREQ");
- ROArrayColumn<Double> inWIDTH(inSPW, "CHAN_WIDTH");
- ROArrayColumn<Double> inBW(inSPW, "EFFECTIVE_BW");
- ROArrayColumn<Double> inRESOLUTION(inSPW, "RESOLUTION");
- ROScalarColumn<Double> inREF_FREQUENCY(inSPW, "REF_FREQUENCY");
-
- for (unsigned int n = 0; n < inSPW.nrow(); n++)
- {
- for (int m = 0; m < old_nchan; m++)
- {
- inFREQ.get(n, old_temp); // could be outsid this loop
- new_FREQ(total_channels + m) = old_temp(m);
-
- inWIDTH.get(n, old_temp);
- new_WIDTH(total_channels + m) = old_temp(m);
-
- inBW.get(n, old_temp);
- new_WIDTH(total_channels + m) = old_temp(m);
-
- inRESOLUTION.get(n, old_temp);
- new_RESOLUTION(total_channels + m) = old_temp(m);
- }
- total_channels += old_nchan;
-
- double temp_freq;
- inREF_FREQUENCY.get(n, temp_freq);
- ref_frequency += temp_freq;
- total_bands++;
- }
- outFREQ.put(0, new_FREQ);
- outWIDTH.put(0, new_WIDTH);
- outBW.put(0, new_BW);
- outRESOLUTION.put(0, new_RESOLUTION);
- outREF_FREQUENCY.put(0, ref_frequency/total_bands);
- }
-
- //Do the real stuff
- itsCombiner->Combine(inMS, outMS, "DATA");
- }
- catch(casacore::AipsError& err)
- {
- std::cerr << "Aips++ error detected: " << err.getMesg() << std::endl;
- return false;
- }
- return true;
- }
-
- //===============>>> CombinerProcessControl::init <<<================================
- tribool CombinerProcessControl::init()
- {
- try {
- using std::cout;
- using std::cerr;
- using std::endl;
-
- cout << string(COMBINER_VERSION) + string(" spw combine by Adriaan Renting for LOFAR CS1\n") +
- string("This is experimental software, please report errors or requests to renting@astron.nl\n") +
- string("Documentation can be found at: www.lofar.org/operations/doku.php?id=engineering:software:postprocessing_software\n");
- cout << string("Combining ");
- for (unsigned int i = 0; i < itsInMS.size(); i++)
- {
- cout << itsInMS[i] << ", ";
- }
- cout << string(" into ") << itsOutMS << endl;
- if (itsInMS.size() == 0 || itsOutMS == "")
- {
- cerr << " Error missing input" << endl;
- return false;
- }
- inMS.resize(itsInMS.size());
- for (unsigned int i = 0; i < itsInMS.size(); i++)
- {
- inMS[i] = new MeasurementSet(itsInMS[i]);
- }
- itsCombiner = new SPWCombine();
- }
- catch(casacore::AipsError& err)
- {
- std::cerr << "Aips++ error detected: " << err.getMesg() << std::endl;
- return false;
- }
- return true;
- }
-
- //===============>>> CombinerProcessControl::pause <<<===============================
- tribool CombinerProcessControl::pause(const std::string&)
- { return false;
- }
-
- //===============>>> CombinerProcessControl::quit <<<================================
- tribool CombinerProcessControl::quit()
- {
- return true;
- }
-
- //===============>>> CombinerProcessControl::release <<<=============================
- tribool CombinerProcessControl::release()
- { return false;
- }
- //===============>>> CombinerProcessControl::recover <<<=============================
- tribool CombinerProcessControl::recover(const std::string&)
- { return false;
- }
-
- //===============>>> CombinerProcessControl::reinit <<<==============================
- tribool CombinerProcessControl::reinit(const std::string&)
- { return false;
- }
-
- //===============>>> CombinerProcessControl::askInfo <<<=============================
- std::string CombinerProcessControl::askInfo(const std::string&)
- { return std::string("");
- }
-
- //===============>>> CombinerProcessControl::snapshot <<<============================
- tribool CombinerProcessControl::snapshot(const std::string&)
- { return false;
- }
- } //namespace CS1
-}; //namespace LOFAR
diff --git a/CEP/DP3/SPWCombine/src/SPWCombine.cc b/CEP/DP3/SPWCombine/src/SPWCombine.cc
deleted file mode 100644
index 3f946f6d04cb92c33529c8a27f319f4af0a7cdc2..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/src/SPWCombine.cc
+++ /dev/null
@@ -1,166 +0,0 @@
-//# Copyright (C) 2007
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-//#
-//# @author Adriaan Renting
-
-#include <lofar_config.h>
-#include <SPWCombine/SPWCombine.h>
-
-namespace LOFAR
-{
- namespace CS1
- {
- using namespace casacore;
-
- //===============>>> SPWCombine::SPWCombine <<<===============
-
- SPWCombine::SPWCombine(void)
- {
- }
-
- //===============>>> SPWCombine::~SPWCombine <<<===============
-
- SPWCombine::~SPWCombine(void)
- {
- }
-
- //===============>>> SPWCombine::GetMSInfo <<<===============
-
- void SPWCombine::GetMSInfo(MeasurementSet& myMS)
- {
- //Number of channels in the Band
- MSSpectralWindow spectral_window = myMS.spectralWindow();
- ROScalarColumn<Int> NUM_CHAN_col(spectral_window, "NUM_CHAN");
- itsNumChannels = NUM_CHAN_col(0);
-
- //Number of polarizations
- MSPolarization polarization = myMS.polarization();
- ROScalarColumn<Int> NUM_CORR_col(polarization, "NUM_CORR");
- itsNumPolarizations = NUM_CORR_col(0);
-
- //Number of Bands
- itsNumBands = NUM_CHAN_col.nrow();
-
- //Number of Antennae
- MSAntenna antennae = myMS.antenna();
- itsNumAntennae = antennae.nrow();
-
- //calculate number of baselines.
- itsNumPairs = (itsNumAntennae) * (itsNumAntennae + 1) / 2; //Triangular numbers formula
- }
-
- //===============>>> SPWCombine::CreateDataIterator <<<===============
-
- TableIterator SPWCombine::CreateDataIterator(MeasurementSet& myMS)
- {
- Block<String> ms_iteration_variables(4);
- ms_iteration_variables[0] = "TIME_CENTROID";
- ms_iteration_variables[1] = "DATA_DESC_ID";
- ms_iteration_variables[2] = "ANTENNA1";
- ms_iteration_variables[3] = "ANTENNA2";
-
- return TableIterator(myMS, ms_iteration_variables);
- }
-
- //===============>>> SPWCombine::TableResize <<<===============
-
- void SPWCombine::TableResize(TableDesc tdesc, IPosition ipos, string name, Table& table)
- {
- ColumnDesc desc = tdesc.rwColumnDesc(name);
- desc.setOptions(0);
- desc.setShape(ipos);
- desc.setOptions(4);
- if (table.tableDesc().isColumn(name))
- { table.removeColumn(name);
- }
- table.addColumn(desc);
- }
-
- //===============>>> SPWCombine::Combine <<<===============
-
- void SPWCombine::Combine(vector<MeasurementSet*> inMS, MeasurementSet& myMS, std::string Data)
- {
- int nrMS = inMS.size();
- vector<int> nrBands(nrMS);
- vector<int> nrChannels(nrMS);
- vector<TableIterator> myIters(nrMS);
-
- for (int i = 0; i < nrMS; i++)
- {
- GetMSInfo(*(inMS[i]));
- nrBands[i] = itsNumBands;
- nrChannels[i] = itsNumChannels;
- myIters[i] = CreateDataIterator(*(inMS[i]));
- }
-
- TableIterator iter = CreateDataIterator(myMS);
- GetMSInfo(myMS);
- int step = myMS.nrow() / 10 + 1; //not exact but it'll do
- int row = 0;
- while (!iter.pastEnd())
- {
- Cube<Bool> myFlags(itsNumPolarizations, itsNumChannels, itsNumPairs);
- Cube<Complex> myData(itsNumPolarizations, itsNumChannels, itsNumPairs);
-
- int bandMScounter = 0;
- for (int i = 0; i < nrMS; i++)
- {
- for (int j = 0; j < nrBands[i]; j++)
- {
- for (int k = 0; k < itsNumPairs; k++)
- {
- Table oldTable = myIters[i].table();
- ROArrayColumn<Complex> Old(oldTable, Data);
- Matrix<Complex> oldData(itsNumPolarizations, nrChannels[i]);
- Old.get(0, oldData);
- ROArrayColumn<Bool> OldFlags(oldTable, "FLAG");
- Matrix<Bool> oldFlags(itsNumPolarizations, nrChannels[i]);
- OldFlags.get(0, oldFlags);
- (myIters[i])++;
- for (int m = 0; m < itsNumPolarizations; m++)
- {
- for (int n = 0; n < nrChannels[i]; n++)
- {
- myData(m, bandMScounter + n, k) = oldData(m, n);
- myFlags(m, bandMScounter + n, k) = oldFlags(m, n);
- }
- }
- }
- bandMScounter += nrChannels[i];
- }
- }
-
- for (int i = 0; i < itsNumPairs; i ++)
- {
- if (row++ % step == 0) // to tell the user how much % we have processed,
- { std::cout << 10*(row/step) << "%" << std::endl; //not very accurate for low numbers of timeslots
- }
-
- Table DataTable = iter.table();
- ArrayColumn<Bool> Flags(DataTable, "FLAG");
- ArrayColumn<Complex> New(DataTable, Data);
- Flags.put(0, myFlags.xyPlane(i));
- New.put(0, myData.xyPlane(i));
- iter++;
- }
- }
- }
- } //namespace CS1
-}; //namespace LOFAR
diff --git a/CEP/DP3/SPWCombine/test/SPWCombine.debug b/CEP/DP3/SPWCombine/test/SPWCombine.debug
deleted file mode 100644
index 37f812a6cc65cdd131b6f2334852546aed8a419f..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/test/SPWCombine.debug
+++ /dev/null
@@ -1,2 +0,0 @@
-Global 20
-Everything 20
diff --git a/CEP/DP3/SPWCombine/test/SPWCombine.parset b/CEP/DP3/SPWCombine/test/SPWCombine.parset
deleted file mode 100644
index 37f812a6cc65cdd131b6f2334852546aed8a419f..0000000000000000000000000000000000000000
--- a/CEP/DP3/SPWCombine/test/SPWCombine.parset
+++ /dev/null
@@ -1,2 +0,0 @@
-Global 20
-Everything 20
diff --git a/CEP/DP3/TestDynDPPP/CMakeLists.txt b/CEP/DP3/TestDynDPPP/CMakeLists.txt
deleted file mode 100644
index 06f1f9e3b978f7b944567204945580bd730ddb99..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# $Id: CMakeLists.txt 27640 2013-12-04 08:02:49Z diepen $
-
-lofar_package(TestDynDPPP 0.1 DEPENDS DPPP)
-
-include(LofarFindPackage)
-lofar_find_package(Casacore COMPONENTS casa ms tables REQUIRED)
-
-add_subdirectory(src)
-add_subdirectory(test)
diff --git a/CEP/DP3/TestDynDPPP/src/CMakeLists.txt b/CEP/DP3/TestDynDPPP/src/CMakeLists.txt
deleted file mode 100644
index 01b4952ecf97972f070544f2890468675c698adf..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/src/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# $Id: CMakeLists.txt 30439 2014-11-19 15:04:34Z dijkema $
-
-add_library(dppp_testdyndppp
- TestDynStep.cc
-)
-
-target_link_libraries (dppp_testdyndppp dppp)
diff --git a/CEP/DP3/TestDynDPPP/src/TestDynStep.cc b/CEP/DP3/TestDynDPPP/src/TestDynStep.cc
deleted file mode 100644
index ab532fb346d0a9c5be960d6ea50b6a322f6d0823..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/src/TestDynStep.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-//# TestDyn.cc: Test of a dynamically loaded DPPP step
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: Averager.h 30711 2015-01-15 14:36:28Z diepen $
-//#
-//# @author Ger van Diepen
-
-// @file
-// @brief Test of a dynamically loaded DPPP step
-
-#include <lofar_config.h>
-#include "TestDynStep.h"
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPRun.h>
-
-namespace LOFAR {
- namespace DPPP {
-
- TestDynStep::TestDynStep (DPInput* input, const ParameterSet& pset,
- const std::string& prefix)
- : Averager (input, pset, prefix)
- {}
-
- TestDynStep::~TestDynStep()
- {}
-
- DPStep::ShPtr TestDynStep::makeStep (DPInput* input,
- const ParameterSet& pset,
- const std::string& prefix)
- { return DPStep::ShPtr(new TestDynStep(input, pset, prefix)); }
-
- }
-}
-
-// Define the function to make the TestDynStep 'constructor' known.
-// Its suffix must be the (lowercase) name of the package (library).
-void register_testdyndppp()
-{
- LOFAR::DPPP::DPRun::registerStepCtor ("TestDynDPPP",
- LOFAR::DPPP::TestDynStep::makeStep);
-}
diff --git a/CEP/DP3/TestDynDPPP/src/TestDynStep.h b/CEP/DP3/TestDynDPPP/src/TestDynStep.h
deleted file mode 100644
index 60ea8ab5de9f4f7f553641389dd7e10b74fc42db..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/src/TestDynStep.h
+++ /dev/null
@@ -1,62 +0,0 @@
-//# TestDynStep.h: Test of a dynamically loaded DPPP step
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: Averager.h 30711 2015-01-15 14:36:28Z diepen $
-//#
-//# @author Ger van Diepen
-
-#ifndef TESTDYNDPPP_TESTDYNSTEP_H
-#define TESTDYNDPPP_TESTDYNSTEP_H
-
-// @file
-// @brief Test of a dynamically loaded DPPP step
-
-#include <DPPP/DPStep.h>
-#include <DPPP/Averager.h>
-#include <DPPP/DPInput.h>
-#include <Common/ParameterSet.h>
-
-namespace LOFAR {
- namespace DPPP {
- // @ingroup NDPPP
-
- // This class is a test (and an example) of a DPStep loaded
- // dynamically from a shared library.
- // To make test life easy it uses the Averager class underneath.
-
- class TestDynStep: public Averager
- {
- public:
- TestDynStep (DPInput*, const ParameterSet&, const std::string&);
- virtual ~TestDynStep();
- static DPStep::ShPtr makeStep (DPInput*, const ParameterSet&,
- const std::string&);
- };
-
-
- }
-}
-
-// Define the function (without name mangling) to register the 'constructor'.
-extern "C"
-{
- void register_testdyndppp();
-}
-
-#endif
diff --git a/CEP/DP3/TestDynDPPP/test/CMakeLists.txt b/CEP/DP3/TestDynDPPP/test/CMakeLists.txt
deleted file mode 100644
index f0c471a411b1a91f31524b66a3a01c0b21aa76b5..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/test/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# $Id: CMakeLists.txt 26355 2013-09-04 11:51:07Z dijkema $
-
-include(LofarCTest)
-
-# Do not use lofar_add_test, because libtestdyndppp should not be linked in.
-add_executable (tTestDynStep tTestDynStep.cc)
-target_link_libraries (tTestDynStep ${_libs} ${LOFAR_EXTRA_LIBRARIES})
-add_test (tTestDynStep ${CMAKE_SOURCE_DIR}/CEP/DP3/TestDynDPPP/test/tTestDynStep.sh)
diff --git a/CEP/DP3/TestDynDPPP/test/tTestDynStep.cc b/CEP/DP3/TestDynDPPP/test/tTestDynStep.cc
deleted file mode 100644
index 71ce22b953d9891bdcacf1b64839c34021106522..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/test/tTestDynStep.cc
+++ /dev/null
@@ -1,592 +0,0 @@
-//# tTestDynStep.cc: Test program for class TestDynStep
-//# Copyright (C) 2015
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id: tAverager.cc 30718 2015-01-19 15:31:51Z diepen $
-//#
-//# @author Ger van Diepen
-
-// This file is mostly a copy of DPPP/test/tAverager.cc.
-// Only the way the average steps are created is different.
-// Average is used underneath TestDynStep.
-
-#include <lofar_config.h>
-#include <DPPP/DPBuffer.h>
-#include <DPPP/DPInfo.h>
-#include <DPPP/DPInput.h>
-#include <DPPP/DPRun.h>
-#include <Common/ParameterSet.h>
-#include <Common/StringUtil.h>
-#include <Common/LofarLogger.h>
-#include <casacore/casa/Arrays/ArrayMath.h>
-#include <casacore/casa/Arrays/ArrayLogical.h>
-#include <casacore/casa/Arrays/ArrayIO.h>
-#include <iostream>
-
-using namespace LOFAR;
-using namespace LOFAR::DPPP;
-using namespace casacore;
-using namespace std;
-
-
-// Simple class to generate input arrays.
-// It can only set all flags to true or all false.
-// Weights are always 1.
-// It can be used with different nr of times, channels, etc.
-class TestInput: public DPInput
-{
-public:
- TestInput(int ntime, int nbl, int nchan, int ncorr, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNTime) {
- return false;
- }
- Cube<Complex> data(itsNCorr, itsNChan, itsNBl);
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- Cube<float> weights(data.shape());
- weights = 1.;
- buf.setWeights (weights);
- Cube<bool> flags(data.shape());
- flags = itsFlag;
- buf.setFlags (flags);
- // The fullRes flags are a copy of the XX flags, but differently shaped.
- // They are not averaged, thus only 1 time per row.
- Cube<bool> fullResFlags(itsNChan, 1, itsNBl);
- fullResFlags = itsFlag;
- buf.setFullResFlags (fullResFlags);
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, double(itsCount*100));
- buf.setUVW (uvw);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&)
- {
- // Use timeInterval=5
- info().init (itsNCorr, itsNChan, itsNTime, 100, 5, string(), string());
- // Define the frequencies.
- Vector<double> chanFreqs(itsNChan);
- Vector<double> chanWidth(itsNChan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
- int itsCount, itsNTime, itsNBl, itsNChan, itsNCorr;
- bool itsFlag;
-};
-
-// Class to check result of averaging TestInput.
-class TestOutput: public DPStep
-{
-public:
- TestOutput(int ntime, int nbl, int nchan, int ncorr,
- int navgtime, int navgchan, bool flag)
- : itsCount(0), itsNTime(ntime), itsNBl(nbl), itsNChan(nchan),
- itsNCorr(ncorr), itsNAvgTime(navgtime), itsNAvgChan(navgchan),
- itsFlag(flag)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- int nchan = 1+(itsNChan-1)/itsNAvgChan;
- int navgtime = std::min(itsNAvgTime, itsNTime-itsCount*itsNAvgTime);
- // Fill expected result in similar way as TestInput.
- Cube<Complex> data(itsNCorr,itsNChan,itsNBl);
- Cube<float> weights(itsNCorr,itsNChan,itsNBl);
- Cube<bool> fullResFlags(itsNChan,itsNAvgTime,itsNBl);
- fullResFlags = true; // takes care of missing times at the end
- weights = 0;
- for (int j=itsCount*itsNAvgTime; j<itsCount*itsNAvgTime+navgtime; ++j) {
- for (int i=0; i<int(data.size()); ++i) {
- data.data()[i] += Complex(i+j*10,i-1000+j*6);
- weights.data()[i] += float(1);
- }
- fullResFlags(Slicer(IPosition(3,0,0,0),
- IPosition(3,itsNChan,navgtime,itsNBl))) = itsFlag;
- }
- Cube<Complex> result(itsNCorr,nchan,itsNBl);
- Cube<float> resultw(itsNCorr,nchan,itsNBl);
- resultw = 0;
- // Average to get the true expected result.
- for (int k=0; k<itsNBl; ++k) {
- for (int i=0; i<itsNCorr; ++i) {
- for (int j=0; j<nchan; ++j) {
- int jc;
- for (jc=j*itsNAvgChan;
- jc<std::min((j+1)*itsNAvgChan, itsNChan); ++jc) {
- result(i,j,k) += data(i,jc,k);
- resultw(i,j,k) += weights(i,jc,k);
- }
- result(i,j,k) /= float(navgtime*(jc-j*itsNAvgChan));
- }
- }
- }
- // Check the averaged result.
- ASSERT (allNear(real(buf.getData()), real(result), 1e-5));
- ///cout << imag(buf.getData()) << endl<<imag(result);
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (allEQ(buf.getFlags(), itsFlag));
- ASSERT (near(buf.getTime(),
- 2+5*(itsCount*itsNAvgTime + (itsNAvgTime-1)/2.)));
- ASSERT (allNear(buf.getWeights(), resultw, 1e-5));
- if (navgtime == itsNAvgTime) {
- Matrix<double> uvw(3,itsNBl);
- indgen (uvw, 100*(itsCount*itsNAvgTime + 0.5*(itsNAvgTime-1)));
- ASSERT (allNear(buf.getUVW(), uvw, 1e-5));
- }
- cout <<buf.getFullResFlags()<< fullResFlags;
- ASSERT (allEQ(buf.getFullResFlags(), fullResFlags));
- ++itsCount;
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNChan);
- ASSERT (int(info.nchan())==1+(itsNChan-1)/itsNAvgChan);
- ASSERT (int(info.ntime())==1+(itsNTime-1)/itsNAvgTime);
- ASSERT (info.timeInterval()==5*itsNAvgTime);
- ASSERT (int(info.nchanAvg())==itsNAvgChan);
- ASSERT (int(info.ntimeAvg())==itsNAvgTime);
- }
-
- int itsCount;
- int itsNTime, itsNBl, itsNChan, itsNCorr, itsNAvgTime, itsNAvgChan;
- bool itsFlag;
-};
-
-
-// More elaborate class which can set different flags and weights.
-class TestInput3: public DPInput
-{
-public:
- TestInput3(int nrtime, int nrbl, int nrchan, int nrcorr)
- : itsCount(0),
- itsNrTime(nrtime), itsNrBl(nrbl), itsNrChan(nrchan), itsNrCorr(nrcorr)
- {
- itsFullResFlags.resize (itsNrChan,1,nrbl);
- }
-private:
- virtual bool process (const DPBuffer&)
- {
- // Stop when all times are done.
- if (itsCount == itsNrTime) {
- return false;
- }
- Cube<Complex> data(itsNrCorr,itsNrChan,itsNrBl);
- Cube<float> weights(itsNrCorr,itsNrChan,itsNrBl);
- Cube<bool> flags(itsNrCorr,itsNrChan,itsNrBl);
- int i = 0;
- for (int ib=0; ib<itsNrBl; ++ib) {
- for (int ic=0; ic<itsNrChan; ++ic) {
- for (int ip=0; ip<itsNrCorr; ++ip) {
- data.data()[i] = Complex(i+itsCount*10,i-1000+itsCount*6);
- weights.data()[i] = (1 + (itsCount+ib+ic)%5) / 5.;
- flags.data()[i] = ((itsCount+2*ib+3*ic) % 7 == 0);
- i++;
- }
- itsFullResFlags(ic,0,ib) = ((itsCount+2*ib+3*ic) % 7 == 0);
- }
- }
- DPBuffer buf;
- buf.setTime (itsCount*5 + 2); //same interval as in updateAveragInfo
- buf.setData (data);
- buf.setWeights (weights);
- buf.setFlags (flags);
- Vector<uint> rownrs(1,itsCount);
- buf.setRowNrs (rownrs);
- getNextStep()->process (buf);
- ++itsCount;
- return true;
- }
-
- virtual void getUVW (const casacore::RefRows&, double, DPBuffer& buf)
- {
- buf.getUVW().resize (3, itsNrBl);
- indgen (buf.getUVW());
- }
- virtual bool getFullResFlags (const casacore::RefRows&, DPBuffer& buf)
- {
- buf.getFullResFlags().assign (itsFullResFlags);
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo&)
- {
- // Use timeInterval=5
- info().init (itsNrCorr, itsNrChan, itsNrTime, 100, 5, string(), string());
- // Define the frequencies.
- Vector<double> chanFreqs(itsNrChan);
- Vector<double> chanWidth(itsNrChan, 100000.);
- indgen (chanFreqs, 1050000., 100000.);
- info().set (chanFreqs, chanWidth);
- }
- int itsCount, itsNrTime, itsNrBl, itsNrChan, itsNrCorr;
- Cube<bool> itsFullResFlags;
-};
-
-// Class to check result of averaging TestInput3.
-// All input must be averaged (in one or more steps) to a single value
-// per corr/baseline.
-class TestOutput3: public DPStep
-{
-public:
- TestOutput3(int nrtime, int nrbl, int nrchan, int nrcorr)
- : itsNrTime(nrtime), itsNrBl(nrbl), itsNrChan(nrchan), itsNrCorr(nrcorr)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- Cube<Complex> result(itsNrCorr,1,itsNrBl);
- Cube<float> weights(itsNrCorr,1,itsNrBl);
- Cube<bool> flags(itsNrCorr,1,itsNrBl);
- Cube<bool> fullResFlags(itsNrChan,itsNrTime,itsNrBl);
- weights = float(0);
- flags = true;
- fullResFlags = true;
- // Create data in the same way as in TestInput3.
- for (int it=0; it<itsNrTime; ++it) {
- int i = 0;
- for (int ib=0; ib<itsNrBl; ++ib) {
- for (int ic=0; ic<itsNrChan; ++ic) {
- for (int ip=0; ip<itsNrCorr; ++ip) {
- if ((it+2*ib+3*ic) % 7 != 0) {
- float weight = (1 + (it+ib+ic)%5) / 5.;
- result(ip,0,ib) += weight * Complex(i+it*10,i-1000+it*6);
- weights(ip,0,ib) += weight;
- /// cout << result(ip,0,ib) << weight << endl;
- flags(ip,0,ib) = false;
- fullResFlags(ic,it,ib) = false;
- }
- i++;
- }
- }
- }
- }
- ASSERT (allNE(weights, float(0.)));
- for (uint i=0; i<result.size(); ++i) {
- result.data()[i] /= weights.data()[i];
- }
- // Check the averaged result.
- ///cout << real(buf.getData()) << endl<<real(result);
- ASSERT (allNear(real(buf.getData()), real(result), 1e-5));
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (allEQ(buf.getFlags(), flags));
- ASSERT (near(buf.getTime(), 2.+5*(itsNrTime-1)/2.));
- ASSERT (allNear(buf.getWeights(), weights, 1e-5));
- Matrix<double> uvw(3,itsNrBl);
- indgen (uvw);
- ASSERT (allNear(buf.getUVW(), uvw, 1e-5));
- ///cout <<buf.getFullResFlags()<< fullResFlags;
- ASSERT (allEQ(buf.getFullResFlags(), fullResFlags));
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNrChan);
- ASSERT (info.nchan()==1);
- ASSERT (info.ntime()==1);
- ASSERT (info.timeInterval()==5*itsNrTime);
- ASSERT (int(info.nchanAvg())==itsNrChan);
- ASSERT (int(info.ntimeAvg())==itsNrTime);
- }
-
- int itsNrTime, itsNrBl, itsNrChan, itsNrCorr;
-};
-
-// Simple class to flag every step-th XX point.
-class TestFlagger: public DPStep
-{
-public:
- TestFlagger(int step)
- : itsCount(0), itsStep(step)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- DPBuffer buf2(buf);
- int ncorr = buf2.getFlags().shape()[0];
- int np = buf2.getFlags().size() / ncorr;
- bool* flagPtr = buf2.getFlags().data();
- for (int i=0; i<np; ++i) {
- if ((i+itsCount)%itsStep == 0) {
- ///cout << "flagged " <<itsCount <<' '<< i << endl;
- for (int j=0; j<ncorr; ++j) {
- flagPtr[i*ncorr + j] = true;
- }
- }
- }
- getNextStep()->process (buf2);
- ++itsCount;
- return true;
- }
-
- virtual void finish() {getNextStep()->finish();}
- virtual void show (std::ostream&) const {}
-
- int itsCount, itsStep;
-};
-
-// Class to check result of averaging and flagging TestInput3.
-// First the data are averaged from 8,4 to 4,2, then every step-th point
-// is flagged, and finally it is averaged to 1,1.
-class TestOutput4: public DPStep
-{
-public:
- TestOutput4(int nrtime, int nrbl, int nrchan, int nrcorr, int step)
- : itsNrTime(nrtime), itsNrBl(nrbl), itsNrChan(nrchan), itsNrCorr(nrcorr),
- itsStep(step)
- {}
-private:
- virtual bool process (const DPBuffer& buf)
- {
- Cube<Complex> result(itsNrCorr,1,itsNrBl);
- Cube<float> weights(itsNrCorr,1,itsNrBl);
- Cube<bool> flags(itsNrCorr,1,itsNrBl);
- Cube<bool> fullResFlags(itsNrChan,itsNrTime,itsNrBl);
- weights = float(0);
- flags = true;
- fullResFlags = true;
- // Create data in the same way as in TestInput3.
- for (int it=0; it<itsNrTime; ++it) {
- int i = 0;
- for (int ib=0; ib<itsNrBl; ++ib) {
- for (int ic=0; ic<itsNrChan; ++ic) {
- // TestFlagger flags every step-th point of 2x2 averaged data.
- int tf = it/2; // same as itsCount in testFlagger
- if (((ib*itsNrChan + ic)/2 + tf) % itsStep == 0) {
- ///cout << "out4 flagged "<< tf<<' '<< i/itsNrCorr<<' ' <<ib<<' '<<ic/2 << endl;
- i += itsNrCorr;
- } else {
- for (int ip=0; ip<itsNrCorr; ++ip) {
- if ((it+2*ib+3*ic) % 7 != 0) {
- float weight = (1 + (it+ib+ic)%5) / 5.;
- result(ip,0,ib) += weight * Complex(i+it*10,i-1000+it*6);
- weights(ip,0,ib) += weight;
- /// cout << result(ip,0,ib) << weight << endl;
- flags(ip,0,ib) = false;
- fullResFlags(ic,it,ib) = false;
- }
- i++;
- }
- }
- }
- }
- }
- for (uint i=0; i<result.size(); ++i) {
- if (!flags.data()[i]) {
- result.data()[i] /= weights.data()[i];
- }
- }
- // Check the averaged result.
- ///cout << real(buf.getData()) << endl<<real(result);
- ASSERT (allNear(real(buf.getData()), real(result), 1e-5));
- ASSERT (allNear(imag(buf.getData()), imag(result), 1e-5));
- ASSERT (allEQ(buf.getFlags(), flags));
- ASSERT (near(buf.getTime(), 2.+5*(itsNrTime-1)/2.));
- ASSERT (allNear(buf.getWeights(), weights, 1e-5));
- Matrix<double> uvw(3,itsNrBl);
- indgen (uvw);
- ASSERT (allNear(buf.getUVW(), uvw, 1e-5));
- ///cout <<buf.getFullResFlags()<< fullResFlags;
- ASSERT (allEQ(buf.getFullResFlags(), fullResFlags));
- return true;
- }
-
- virtual void finish() {}
- virtual void show (std::ostream&) const {}
- virtual void updateInfo (const DPInfo& info)
- {
- ASSERT (int(info.origNChan())==itsNrChan);
- ASSERT (info.nchan()==1);
- ASSERT (info.ntime()==1);
- ASSERT (info.timeInterval()==5*itsNrTime);
- ASSERT (int(info.nchanAvg())==itsNrChan);
- ASSERT (int(info.ntimeAvg())==itsNrTime);
- }
-
- int itsNrTime, itsNrBl, itsNrChan, itsNrCorr, itsStep;
-};
-
-
-// Execute steps.
-void execute (const DPStep::ShPtr& step1)
-{
- // Set DPInfo.
- step1->setInfo (DPInfo());
- // Execute the steps.
- DPBuffer buf;
- while (step1->process(buf));
- step1->finish();
-}
-
-// Test simple averaging without flagged points.
-void test1(int ntime, int nbl, int nchan, int ncorr,
- int navgtime, int navgchan, bool flag)
-{
- cout << "test1: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " navgtime=" << navgtime
- << " navgchan=" << navgchan << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset;
- parset.add ("freqstep", toString(navgchan));
- parset.add ("timestep", toString(navgtime));
- DPStep::ShPtr step2 = DPRun::findStepCtor("TestDynDPPP")(in, parset, "");
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr,
- navgtime, navgchan, flag));
- step1->setNextStep (step2);
- step2->setNextStep (step3);
- execute (step1);
-}
-
-// Like test1, but the averaging is done in two steps.
-void test2(int ntime, int nbl, int nchan, int ncorr, bool flag)
-{
- cout << "test2: ntime=" << ntime << " nrbl=" << nbl << " nchan=" << nchan
- << " ncorr=" << ncorr << " navgtime=2"
- << " navgchan=4" << endl;
- // Create the steps.
- TestInput* in = new TestInput(ntime, nbl, nchan, ncorr, flag);
- DPStep::ShPtr step1(in);
- ParameterSet parset1, parset2;
- parset1.add ("freqstep", "4");
- parset2.add ("timestep", "2");
- DPStep::ShPtr step2a = DPRun::findStepCtor("TestDynDPPP")(in, parset1, "");
- DPStep::ShPtr step2b = DPRun::findStepCtor("TestDynDPPP")(in, parset2, "");
- DPStep::ShPtr step3(new TestOutput(ntime, nbl, nchan, ncorr, 2, 4, flag));
- step1->setNextStep (step2a);
- step2a->setNextStep (step2b);
- step2b->setNextStep (step3);
- execute (step1);
-}
-
-// Do tests with weighting and some flagged points.
-void test3(int nrbl, int nrcorr)
-{
- {
- cout << "test3: ntime=2 nrbl=" << nrbl << " nchan=2 ncorr=" << nrcorr
- << endl;
- cout << " navgtime=2 navgchan=2" << endl;
- // Create the steps.
- TestInput3* in = new TestInput3(2, nrbl, 2, nrcorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset1;
- parset1.add ("freqstep", "2");
- parset1.add ("timestep", "2");
- DPStep::ShPtr step2a = DPRun::findStepCtor("TestDynDPPP")(in, parset1, "");
- DPStep::ShPtr step3(new TestOutput3(2, nrbl, 2, nrcorr));
- step1->setNextStep (step2a);
- step2a->setNextStep (step3);
- execute (step1);
- }
- {
- cout << "test3: ntime=4 nrbl=" << nrbl << " nchan=8 ncorr=" << nrcorr
- << endl;
- cout << " [navgtime=2 navgchan=4], [navgtime=2 navgchan=2]" << endl;
- // Create the steps.
- TestInput3* in = new TestInput3(4, nrbl, 8, nrcorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset1, parset2;
- parset1.add ("freqstep", "4");
- parset1.add ("timestep", "2");
- parset2.add ("freqstep", "2");
- parset2.add ("timestep", "2");
- DPStep::ShPtr step2a = DPRun::findStepCtor("TestDynDPPP")(in, parset1, "");
- DPStep::ShPtr step2b = DPRun::findStepCtor("TestDynDPPP")(in, parset2, "");
- DPStep::ShPtr step3(new TestOutput3(4, nrbl, 8, nrcorr));
- step1->setNextStep (step2a);
- step2a->setNextStep (step2b);
- step2b->setNextStep (step3);
- execute (step1);
- }
-}
-
-// Do tests with averaging and flagging steps to see if the flags are
-// promoted to the FULLRES flags.
-void test4(int nrbl, int nrcorr, int flagstep)
-{
- {
- cout << "test4: ntime=4 nrbl=" << nrbl << " nchan=8 ncorr=" << nrcorr
- << endl;
- cout << " [navgtime=2 navgchan=2], [flagstep=" << flagstep
- << "] [navgtime=2 navgchan=4]" << endl;
- // Create the steps.
- TestInput3* in = new TestInput3(4, nrbl, 8, nrcorr);
- DPStep::ShPtr step1(in);
- ParameterSet parset1, parset2;
- parset1.add ("freqstep", "2");
- parset1.add ("timestep", "2");
- parset2.add ("freqstep", "4");
- parset2.add ("timestep", "2");
- DPStep::ShPtr step2a = DPRun::findStepCtor("TestDynDPPP")(in, parset1, "");
- DPStep::ShPtr step2b(new TestFlagger(flagstep));
- DPStep::ShPtr step2c = DPRun::findStepCtor("TestDynDPPP")(in, parset2, "");
- DPStep::ShPtr step3(new TestOutput4(4, nrbl, 8, nrcorr, flagstep));
- step1->setNextStep (step2a);
- step2a->setNextStep (step2b);
- step2b->setNextStep (step2c);
- step2c->setNextStep (step3);
- execute (step1);
- }
-}
-
-
-int main()
-{
- try {
- test1(10, 3, 32, 4, 2, 4, false);
- test1(10, 3, 30, 1, 3, 3, true);
- test1(10, 3, 30, 1, 3, 3, false);
- test1(11, 3, 30, 2, 3, 3, false);
- test1(10, 3, 32, 4, 1, 32, false);
- test1(10, 3, 32, 1, 1, 1, false);
- test2(10, 3, 32, 2, true);
- test2(10, 3, 32, 2, false);
- test3(1, 1);
- test3(10, 4);
- test4(1, 4, 3);
- test4(20, 4, 5);
- } catch (std::exception& x) {
- cout << "Unexpected exception: " << x.what() << endl;
- return 1;
- }
- return 0;
-}
diff --git a/CEP/DP3/TestDynDPPP/test/tTestDynStep.run b/CEP/DP3/TestDynDPPP/test/tTestDynStep.run
deleted file mode 100755
index 2987402091ea6f4e39ffab738e382cd9cad22d98..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/test/tTestDynStep.run
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-
-# Define the path for finding the library.
-LD_LIBRARY_PATH=../src:$LD_LIBRARY_PATH
-DYLD_LIBRARY_PATH=../src:$DYLD_LIBRARY_PATH
-export LD_LIBRARY_PATH
-export DYLD_LIBRARY_PATH
-
-./tTestDynStep
diff --git a/CEP/DP3/TestDynDPPP/test/tTestDynStep.sh b/CEP/DP3/TestDynDPPP/test/tTestDynStep.sh
deleted file mode 100755
index 6f59e84345ddc1a5a2a56ef6483b4bdb08fecbb2..0000000000000000000000000000000000000000
--- a/CEP/DP3/TestDynDPPP/test/tTestDynStep.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-./runctest.sh tTestDynStep
diff --git a/CEP/DP3/doc/package.dox b/CEP/DP3/doc/package.dox
deleted file mode 100644
index 3dd5595671b586c8742a55ef51acf87087898314..0000000000000000000000000000000000000000
--- a/CEP/DP3/doc/package.dox
+++ /dev/null
@@ -1,7 +0,0 @@
-// \ingroup CEP
-
-// \defgroup DP3 DP3
-// DP3 contains the packages for the pre-processing pipeline. It consists of
-// a collection of flaggers, and a data squasher.
-// Furthermore it contains a program to combine multiple spectral
-// windows and MeasurementSets into a single one.