diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7f96632c36e9b1caa7887e0fb323adc3fcf69678..05e0d45428a39cce21aab87aefd63f4cb0c6945b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -227,6 +227,7 @@ unit_test_MCU_MAC:
dockerize_TMSS:
stage: dockerize
+ allow_failure: true
script:
- cd build/gnucxx11_opt
- ls *
@@ -405,4 +406,4 @@ deploy-MCU_MAC-test:
- ssh lofarsys@mcu199.control.lofar "MAC_install -b ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA} -v ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
needs:
- unit_test_MCU_MAC
- when: manual
\ No newline at end of file
+ when: manual
diff --git a/CMake/FindCUDAnvToolsExt.cmake b/CMake/FindCUDAnvToolsExt.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..aa92a5b99a9d825955908a0f6e2cf07327d65477
--- /dev/null
+++ b/CMake/FindCUDAnvToolsExt.cmake
@@ -0,0 +1,37 @@
+# - Try to find the CUDA nvToolsExt library
+# Variables used by this module:
+# CUDA_TOOLKIT_ROOT_DIR - CUDA toolkit root directory
+# Variables defined by this module:
+# CUDA_nvToolsExt_FOUND - system has CUDA_nvToolsExt_LIBRARY
+# CUDA_nvToolsExt_LIBRARY - the CUDA_nvToolsExt_LIBRARY library
+
+# Copyright (C) 2020
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# Oude Hoogeveensedijk 4, 7991 PD Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+if(NOT CUDA_nvToolsExt_LIBRARY_FOUND)
+
+find_library(
+ CUDA_nvToolsExt_LIBRARY
+ NAMES nvToolsExt
+ HINTS ${CUDA_TOOLKIT_ROOT_DIR}
+ PATH_SUFFIXES lib64)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(CUDA_nvToolsExt_LIBRARY DEFAULT_MSG CUDA_nvToolsExt_LIBRARY)
+
+endif(NOT CUDA_nvToolsExt_LIBRARY_FOUND)
diff --git a/CMake/FindCUDAnvrtc.cmake b/CMake/FindCUDAnvrtc.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..6ae942843ed042dd6e0204d602673c43f0c3abdc
--- /dev/null
+++ b/CMake/FindCUDAnvrtc.cmake
@@ -0,0 +1,38 @@
+# - Try to find the CUDA nvToolsExt library
+# Variables used by this module:
+# CUDA_TOOLKIT_ROOT_DIR - CUDA toolkit root directory
+# Variables defined by this module:
+# CUDA_nvToolsExt_FOUND - system has CUDA_nvToolsExt_LIBRARY
+# CUDA_nvToolsExt_LIBRARY - the CUDA_nvToolsExt_LIBRARY library
+
+# Copyright (C) 2020
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# Oude Hoogeveensedijk 4, 7991 PD Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+if(NOT CUDA_nvrtc_LIBRARY_FOUND)
+
+find_library(
+ CUDA_nvrtc_LIBRARY
+ NAMES nvrtc
+ HINTS ${CUDA_TOOLKIT_ROOT_DIR}
+ PATH_SUFFIXES lib64)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(CUDA_nvrtc_LIBRARY DEFAULT_MSG CUDA_nvrtc_LIBRARY)
+
+
+endif(NOT CUDA_nvrtc_LIBRARY_FOUND)
diff --git a/LCS/PyCommon/json_utils.py b/LCS/PyCommon/json_utils.py
index 232ba7841c4d588378ab01fb192d8d25b59577bc..4b84ad903e6e0510240743901ba4b9ec7397fb49 100644
--- a/LCS/PyCommon/json_utils.py
+++ b/LCS/PyCommon/json_utils.py
@@ -103,6 +103,12 @@ def get_default_json_object_for_schema(schema: str) -> dict:
def add_defaults_to_json_object_for_schema(json_object: dict, schema: str) -> dict:
'''return a copy of the json object with defaults filled in according to the schema for all the missing properties'''
copy_of_json_object = deepcopy(json_object)
+
+ # add a $schema to the json doc if needed
+ if '$schema' not in copy_of_json_object and '$id' in schema:
+ copy_of_json_object['$schema'] = schema['$id']
+
+ # run validator, which populates the properties with defaults.
get_validator_for_schema(schema, add_defaults=True).validate(copy_of_json_object)
return copy_of_json_object
diff --git a/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc b/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc
index b0e3845b4132fb994d9a9c9f4692a62ccf3b50f8..d30b734cb3c25711ed214f4e6945fa5dc047c6d5 100644
--- a/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc
+++ b/RTCP/Cobalt/CoInterface/src/LTAFeedback.cc
@@ -98,15 +98,15 @@ namespace LOFAR
const string prefix = beamFormedPrefix(fileno);
const string locationHost = f.location.cluster != "" ? f.location.cluster : f.location.host;
- const struct ObservationSettings::BeamFormer::StokesSettings &stokesSet =
- f.coherent ? settings.beamFormer.coherentSettings
- : settings.beamFormer.incoherentSettings;
-
const struct ObservationSettings::SAP &sap =
settings.SAPs.at(f.sapNr);
const struct ObservationSettings::BeamFormer::TAB &tab =
settings.beamFormer.SAPs.at(f.sapNr).TABs.at(f.tabNr);
+ const struct ObservationSettings::BeamFormer::StokesSettings &stokesSet =
+ f.coherent ? settings.beamFormer.pipelines[tab.pipelineNr].coherentSettings
+ : settings.beamFormer.pipelines[tab.pipelineNr].incoherentSettings;
+
vector<string> stokesVars;
switch (stokesSet.type) {
@@ -141,8 +141,8 @@ namespace LOFAR
ps.add(prefix + "storageWriter", "HDF5DEFAULT");
ps.add(prefix + "storageWriterVersion", "UNKNOWN");
- const string type =
- settings.beamFormer.doFlysEye ? "FlysEyeBeam" :
+ const string type =
+ settings.beamFormer.pipelines[tab.pipelineNr].doFlysEye ? "FlysEyeBeam" :
(f.coherent ? "CoherentStokesBeam" : "IncoherentStokesBeam");
ps.add(prefix + "nrOfCoherentStokesBeams", "0");
@@ -195,8 +195,9 @@ namespace LOFAR
ps.add(beamPrefix + "Offset.angle1", str(format("%f") % (tab.direction.angle1 - sap.direction.angle1)));
ps.add(beamPrefix + "Offset.angle2", str(format("%f") % (tab.direction.angle2 - sap.direction.angle2)));
} else if (type == "FlysEyeBeam") {
- ps.add(beamPrefix + "stationName", settings.beamFormer.antennaFieldNames.at(f.tabNr).station);
- ps.add(beamPrefix + "antennaFieldName", settings.beamFormer.antennaFieldNames.at(f.tabNr).antennaField);
+ auto& antennaFieldName = settings.beamFormer.pipelines.at(tab.pipelineNr).antennaFieldNames.at(tab.localTabNr);
+ ps.add(beamPrefix + "stationName", antennaFieldName.station);
+ ps.add(beamPrefix + "antennaFieldName", antennaFieldName.antennaField);
}
return ps;
@@ -265,31 +266,40 @@ namespace LOFAR
settings.rawStationList);
}
- if (settings.beamFormer.enabled && settings.beamFormer.anyCoherentTABs() && settings.beamFormer.doFlysEye) {
- /* Fly's Eye */
- const ObservationSettings::BeamFormer::StokesSettings&
- coherentStokes = settings.beamFormer.coherentSettings;
-
- // The 'rawSamplingTime' is the duration of a sample right after the PPF
- ps.add("Observation.FlysEye.rawSamplingTime",
- str(format("%.16g") %
- (settings.sampleDuration() * coherentStokes.nrChannels)));
-
- // The 'samplingTime' is the duration of a sample in the output
- ps.add("Observation.FlysEye.samplingTime",
- str(format("%.16g") %
- (settings.sampleDuration() * coherentStokes.nrChannels * coherentStokes.timeIntegrationFactor)));
-
- ps.add("Observation.FlysEye.timeDownsamplingFactor",
- str(format("%.16g") % coherentStokes.timeIntegrationFactor));
-
- ps.add("Observation.FlysEye.channelsPerSubband",
- str(format("%u") % coherentStokes.nrChannels));
- ps.add("Observation.FlysEye.channelWidth",
- str(format("%.16g") % (settings.subbandWidth() / coherentStokes.nrChannels)));
-
- ps.add("Observation.FlysEye.stokes",
- stokesType(coherentStokes.type));
+ for (auto& pipeline : settings.beamFormer.pipelines)
+ {
+ if (settings.beamFormer.enabled && pipeline.anyCoherentTABs() && pipeline.doFlysEye) {
+ /* Fly's Eye */
+ const ObservationSettings::BeamFormer::StokesSettings&
+ coherentStokes = pipeline.coherentSettings;
+
+ // The 'rawSamplingTime' is the duration of a sample right after the PPF
+ ps.add("Observation.FlysEye.rawSamplingTime",
+ str(format("%.16g") %
+ (settings.sampleDuration() * coherentStokes.nrChannels)));
+
+ // The 'samplingTime' is the duration of a sample in the output
+ ps.add("Observation.FlysEye.samplingTime",
+ str(format("%.16g") %
+ (settings.sampleDuration() * coherentStokes.nrChannels * coherentStokes.timeIntegrationFactor)));
+
+ ps.add("Observation.FlysEye.timeDownsamplingFactor",
+ str(format("%.16g") % coherentStokes.timeIntegrationFactor));
+
+ ps.add("Observation.FlysEye.channelsPerSubband",
+ str(format("%u") % coherentStokes.nrChannels));
+ ps.add("Observation.FlysEye.channelWidth",
+ str(format("%.16g") % (settings.subbandWidth() / coherentStokes.nrChannels)));
+
+ ps.add("Observation.FlysEye.stokes",
+ stokesType(coherentStokes.type));
+
+ if (settings.beamFormer.pipelines.size() > 1)
+ {
+ LOG_INFO_STR("The Observation.FlysEye.* settings are written only once, for the first BF pipeline in FE mode.");
+ break;
+ }
+ }
}
if (settings.beamFormer.enabled && settings.beamFormer.anyIncoherentTABs()) {
diff --git a/RTCP/Cobalt/CoInterface/src/OMPThread.h b/RTCP/Cobalt/CoInterface/src/OMPThread.h
index 7134ed33ce21f8c97c32f3a329453af63d84b10c..572392f63912376c924471dfb6b166851e12c667 100644
--- a/RTCP/Cobalt/CoInterface/src/OMPThread.h
+++ b/RTCP/Cobalt/CoInterface/src/OMPThread.h
@@ -34,7 +34,6 @@
#include <Common/Thread/Condition.h>
#include <Common/Thread/Thread.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/TimeFuncs.h>
#include <CoInterface/Exceptions.h>
#include <CoInterface/cpu_utils.h>
@@ -275,7 +274,7 @@ namespace LOFAR
private:
Mutex mutex;
- std::vector< SmartPtr<OMPThread> > threads;
+ std::vector<std::unique_ptr<OMPThread>> threads;
bool stopped;
const std::string name;
@@ -286,9 +285,9 @@ namespace LOFAR
if (stopped)
THROW(CannotStartException, "ThreadSet " << name << " was ordered to stop before this thread started");
- SmartPtr<OMPThread> t = new OMPThread;
+ std::unique_ptr<OMPThread> t(new OMPThread());
- threads.push_back(t);
+ threads.push_back(std::move(t));
return **threads.rbegin();
}
diff --git a/RTCP/Cobalt/CoInterface/src/Parset.cc b/RTCP/Cobalt/CoInterface/src/Parset.cc
index 6473cded1f5a09b560116131f00512718748bc19..d1a8c613b642cbb9953e99adfa55a7eeea768da1 100644
--- a/RTCP/Cobalt/CoInterface/src/Parset.cc
+++ b/RTCP/Cobalt/CoInterface/src/Parset.cc
@@ -312,24 +312,11 @@ namespace LOFAR
return a < b; // at least 1 short name
}
-
- struct ObservationSettings Parset::observationSettings() const
+ void Parset::addGenericInformation(
+ struct ObservationSettings& settings) const
{
- struct ObservationSettings settings;
-
- // the set of hosts on which outputProc has to run, which will
- // be constructed during the parsing of the parset
- set<string> outputProcHosts;
-
- // NOTE: Make sure that all keys have defaults, to make test parsets
- // a lot shorter.
- // --Update: No, instead use tParsetDefault in tests and reject parsets with missing required values.
-
- const vector<string> emptyVectorString;
- const vector<unsigned> emptyVectorUnsigned;
const vector<double> emptyVectorDouble;
- // Generic information
settings.realTime = getBool("Cobalt.realTime", false);
settings.observationID = getUint32("Observation.ObsID", 0);
settings.momID = getUint32("Observation.momID", 0);
@@ -345,7 +332,7 @@ namespace LOFAR
settings.corrections.bandPass = getBool("Cobalt.correctBandPass", true);
settings.corrections.clock = getBool("Cobalt.correctClocks", true);
settings.corrections.dedisperse = getBool("Cobalt.BeamFormer.coherentDedisperseChannels", true);
-
+
settings.writeToDisk = getBool("Cobalt.writeToDisk", true);
settings.delayCompensation.enabled = getBool("Cobalt.delayCompensation", true);
@@ -353,24 +340,49 @@ namespace LOFAR
if (settings.delayCompensation.referencePhaseCenter == emptyVectorDouble)
LOG_WARN("Parset: Observation.referencePhaseCenter is missing (or (0.0, 0.0, 0.0)).");
- // Station information (required by pointing information)
- settings.antennaSet = getString("Observation.antennaSet", "LBA_INNER");
- settings.bandFilter = getString("Observation.bandFilter", "LBA_30_70");
+ settings.blockSize = getUint32("Cobalt.blockSize", 196608);
+ }
+
+ void Parset::addAnaBeamInformation(
+ struct ObservationSettings& settings) const
+ {
+ settings.anaBeam.enabled = settings.antennaSet.substr(0,3) == "HBA";
+ if (settings.anaBeam.enabled) {
+ settings.anaBeam.direction.type = getString("Observation.AnaBeam[0].directionType", "J2000");
+ settings.anaBeam.direction.angle1 = getDouble("Observation.AnaBeam[0].angle1", 0.0);
+ settings.anaBeam.direction.angle2 = getDouble("Observation.AnaBeam[0].angle2", 0.0);
+ }
+ }
+
+ void Parset::addSAPs(
+ struct ObservationSettings& settings) const
+ {
+ const vector<unsigned> emptyVectorUnsigned;
+ const vector<double> emptyVectorDouble;
- // Pointing information
size_t nrSAPs = getUint32("Observation.nrBeams", 1);
+
unsigned subbandOffset = 512 * (settings.nyquistZone() - 1);
settings.SAPs.resize(nrSAPs);
settings.subbands.clear();
- for (unsigned sapNr = 0; sapNr < nrSAPs; ++sapNr)
+
+ for (unsigned sapNr = 0; sapNr < nrSAPs; ++sapNr)
{
- struct ObservationSettings::SAP &sap = settings.SAPs[sapNr];
+
+ auto& sap = settings.SAPs[sapNr];
sap.direction.type = getString(str(format("Observation.Beam[%u].directionType") % sapNr), "J2000");
sap.direction.angle1 = getDouble(str(format("Observation.Beam[%u].angle1") % sapNr), 0.0);
sap.direction.angle2 = getDouble(str(format("Observation.Beam[%u].angle2") % sapNr), 0.0);
sap.target = getString(str(format("Observation.Beam[%u].target") % sapNr), "");
+ sap.correlatorEnabled = getBool(str(format("Observation.Beam[%u].CorrelatorEnabled") % sapNr), settings.correlator.enabled);
+ if (sap.correlatorEnabled)
+ {
+ ASSERTSTR(settings.correlator.enabled,
+ (format("Observation.Beam[%u].CorrelatorEnabled=true") % sapNr) <<
+ " is invalid with Observation.DataProducts.Output_Correlated.enable=false");
+ }
// Process the subbands of this SAP
vector<unsigned> subbandList = getUint32Vector(str(format("Observation.Beam[%u].subbandList") % sapNr), emptyVectorUnsigned, true);
@@ -394,20 +406,15 @@ namespace LOFAR
settings.SAPs[sapNr].subbands.push_back(subband);
}
}
+ }
- settings.anaBeam.enabled = settings.antennaSet.substr(0,3) == "HBA";
- if (settings.anaBeam.enabled) {
- settings.anaBeam.direction.type = getString("Observation.AnaBeam[0].directionType", "J2000");
- settings.anaBeam.direction.angle1 = getDouble("Observation.AnaBeam[0].angle1", 0.0);
- settings.anaBeam.direction.angle2 = getDouble("Observation.AnaBeam[0].angle2", 0.0);
- }
-
- settings.blockSize = getUint32("Cobalt.blockSize", 196608);
-
- // Station information (used pointing information to verify settings)
- vector<string> stations = getStringVector("Observation.VirtualInstrument.stationList", emptyVectorString, true);
- ASSERTSTR(!stations.empty(), "station list (Observation.VirtualInstrument.stationList) must be non-empty");
- settings.rawStationList = getString("Observation.VirtualInstrument.stationList", "[]");
+ void Parset::addStationInformation(
+ vector<string>& stations,
+ struct ObservationSettings& settings) const
+ {
+ const vector<string> emptyVectorString;
+ const vector<unsigned> emptyVectorUnsigned;
+ const vector<double> emptyVectorDouble;
// Sort stations (CS, RS, int'l), to get a consistent and predictable
// order in the MeasurementSets.
@@ -484,9 +491,14 @@ namespace LOFAR
" subbands. Please correct either Observation.rspSlotList or Observation.Dataslots." <<
antennaField.name << ".DataslotList" );
}
+ }
+ void Parset::addResourceInformation(
+ struct ObservationSettings& settings) const
+ {
+ const vector<string> emptyVectorString;
+ const vector<unsigned> emptyVectorUnsigned;
- // Resource information
settings.outputCluster = getString("Observation.Cluster.ProcessingCluster.clusterName", "");
vector<string> nodes = getStringVector("Cobalt.Nodes", emptyVectorString, true);
@@ -525,96 +537,296 @@ namespace LOFAR
settings.nodes.push_back(node);
}
+ }
- /* ===============================
- * Correlator pipeline information
- * ===============================
- */
+ void Parset::addCorrelatorInformation(
+ set<string>& outputProcHosts,
+ struct ObservationSettings& settings) const
+ {
+ const vector<string> emptyVectorString;
+ const vector<unsigned> emptyVectorUnsigned;
- settings.correlator.enabled = getBool("Observation.DataProducts.Output_Correlated.enabled", false);
- if (settings.correlator.enabled) {
- settings.correlator.nrChannels = getUint32("Cobalt.Correlator.nrChannelsPerSubband", 64);
- //settings.correlator.nrChannels = getUint32("Observation.channelsPerSubband", 64);
- settings.correlator.channelWidth = settings.subbandWidth() / settings.correlator.nrChannels;
- settings.correlator.nrSamplesPerBlock = settings.blockSize / settings.correlator.nrChannels;
- settings.correlator.nrBlocksPerIntegration = getUint32("Cobalt.Correlator.nrBlocksPerIntegration", 1);
- settings.correlator.nrIntegrationsPerBlock = getUint32("Cobalt.Correlator.nrIntegrationsPerBlock", 1);
-
- // We either have the integration time spanning multiple blocks, or the integration time being a part
- // of a block, but never both.
- ASSERT(settings.correlator.nrBlocksPerIntegration == 1 || settings.correlator.nrIntegrationsPerBlock == 1);
-
- settings.correlator.nrIntegrations = settings.nrBlocks()
- * settings.correlator.nrIntegrationsPerBlock
- / settings.correlator.nrBlocksPerIntegration;
-
- // super-station beam former
- //
- // TODO: Super-station beam former is unused, so will likely be
- // implemented differently. The code below is only there to show how
- // the OLAP.* keys used to be interpreted.
- //
- // Note: then, also adapt TODO in writeCommonLofarAttributes()
-
- // OLAP.CNProc.tabList[i] = j <=> superstation j contains (input) station i
- vector<unsigned> tabList = getUint32Vector("OLAP.CNProc.tabList", emptyVectorUnsigned, true);
-
- // Names for all superstations, including those that are simple copies
- // of (input) antenna fields.
- vector<string> tabNames = getStringVector("OLAP.tiedArrayStationNames", emptyVectorString, true);
-
- if (tabList.empty()) {
- // default: input station list = output station list
- settings.correlator.stations.resize(settings.antennaFields.size());
- for (size_t i = 0; i < settings.correlator.stations.size(); ++i) {
- struct ObservationSettings::Correlator::Station &station = settings.correlator.stations[i];
-
- station.name = settings.antennaFields[i].name;
- station.inputStations = vector<size_t>(1, i);
- }
- } else {
- // process super-station beam former list
- settings.correlator.stations.resize(tabList.size());
- for (size_t i = 0; i < settings.correlator.stations.size(); ++i) {
- struct ObservationSettings::Correlator::Station &station = settings.correlator.stations[i];
+ settings.correlator.nrChannels = getUint32("Cobalt.Correlator.nrChannelsPerSubband", 64);
+ //settings.correlator.nrChannels = getUint32("Observation.channelsPerSubband", 64);
+ settings.correlator.channelWidth = settings.subbandWidth() / settings.correlator.nrChannels;
+ settings.correlator.nrSamplesPerBlock = settings.blockSize / settings.correlator.nrChannels;
+ settings.correlator.nrBlocksPerIntegration = getUint32("Cobalt.Correlator.nrBlocksPerIntegration", 1);
+ settings.correlator.nrIntegrationsPerBlock = getUint32("Cobalt.Correlator.nrIntegrationsPerBlock", 1);
+
+ // We either have the integration time spanning multiple blocks, or the integration time being a part
+ // of a block, but never both.
+ ASSERT(settings.correlator.nrBlocksPerIntegration == 1 || settings.correlator.nrIntegrationsPerBlock == 1);
+
+ settings.correlator.nrIntegrations = settings.nrBlocks()
+ * settings.correlator.nrIntegrationsPerBlock
+ / settings.correlator.nrBlocksPerIntegration;
+
+ // super-station beam former
+ //
+ // TODO: Super-station beam former is unused, so will likely be
+ // implemented differently. The code below is only there to show how
+ // the OLAP.* keys used to be interpreted.
+ //
+ // Note: then, also adapt TODO in writeCommonLofarAttributes()
+
+ // OLAP.CNProc.tabList[i] = j <=> superstation j contains (input) station i
+ vector<unsigned> tabList = getUint32Vector("OLAP.CNProc.tabList", emptyVectorUnsigned, true);
+
+ // Names for all superstations, including those that are simple copies
+ // of (input) antenna fields.
+ vector<string> tabNames = getStringVector("OLAP.tiedArrayStationNames", emptyVectorString, true);
+
+ if (tabList.empty()) {
+ // default: input station list = output station list
+ settings.correlator.stations.resize(settings.antennaFields.size());
+ for (size_t i = 0; i < settings.correlator.stations.size(); ++i) {
+ struct ObservationSettings::Correlator::Station &station = settings.correlator.stations[i];
+
+ station.name = settings.antennaFields[i].name;
+ station.inputStations = vector<size_t>(1, i);
+ }
+ } else {
+ // process super-station beam former list
+ settings.correlator.stations.resize(tabList.size());
+ for (size_t i = 0; i < settings.correlator.stations.size(); ++i) {
+ struct ObservationSettings::Correlator::Station &station = settings.correlator.stations[i];
- station.name = tabNames[i];
- }
- for (size_t i = 0; i < tabList.size(); ++i) {
- settings.correlator.stations[tabList[i]].inputStations.push_back(i);
+ station.name = tabNames[i];
+ }
+ for (size_t i = 0; i < tabList.size(); ++i) {
+ settings.correlator.stations[tabList[i]].inputStations.push_back(i);
+ }
+ }
+
+ // Files to output
+ const vector<ObservationSettings::FileLocation> locations = getFileLocations("Correlated");
+
+ settings.correlator.files.resize(settings.subbands.size());
+ for (size_t i = 0; i < settings.correlator.files.size(); ++i) {
+ if (i >= locations.size())
+ THROW(CoInterfaceException, "No correlator filename or location specified for subband " << i);
+
+ settings.correlator.files[i].streamNr = i;
+ settings.correlator.files[i].location = locations[i];
+
+ settings.correlator.files[i].dataStagingDirTargetFS =
+ getString(str(format("PIC.Core.%s.dataStagingDirTargetFS") % locations[i].cluster), "");
+ settings.correlator.files[i].metaDataStagingDirCreateFS =
+ getString(str(format("PIC.Core.%s.metaDataStagingDirCreateFS") % locations[i].cluster), "");
+ settings.correlator.files[i].metaDataStagingDirTargetFS =
+ getString(str(format("PIC.Core.%s.metaDataStagingDirTargetFS") % locations[i].cluster), "");
+
+ outputProcHosts.insert(settings.correlator.files[i].location.host);
+ }
+ }
+
+ void Parset::readTABs(
+ const ObservationSettings& settings,
+ const string& beamPrefix,
+ std::vector<ObservationSettings::SAP>& SAPs,
+ bool doFlysEye) const
+ {
+ const vector<unsigned> emptyVectorUnsigned;
+
+ unsigned nrSAPs = SAPs.size();
+
+ // Strategy:
+ // * If Fly's Eye is enabled, we have 1 TAB/station. Do NOT process any other TABs.
+ // * If Fly's Eye is disabled, process TABs in this order:
+ // 1. Manually specified TABs (Observation.Beam[x].TiedArrayBeam[y]
+ // 2. TAB rings (Observation.Beam[x].nrTabRings)
+ for (unsigned i = 0; i < nrSAPs; ++i)
+ {
+ auto& sap = SAPs[i];
+
+ // Read remaining parameters for SAP
+ size_t nrTABs = getUint32(beamPrefix + str(format("[%u].nrTiedArrayBeams") % i), 0);
+ size_t nrTABSParset = nrTABs;
+ size_t nrRings = getUint32(beamPrefix + str(format("[%u].nrTabRings") % i), 0);
+ double ringWidth = getDouble(beamPrefix + str(format("[%u].tabRingSize") % i), 0.0);
+
+ // Throw if we encounter an unsupported configuration
+ if (doFlysEye) {
+ if (nrRings > 0 || nrTABSParset > 0) {
+ THROW(CoInterfaceException, "Cannot produce (in)coherent TABs in Fly's Eye mode. Error detected in SAP " << i << ".");
}
}
- // Files to output
- const vector<ObservationSettings::FileLocation> locations = getFileLocations("Correlated");
+ // Create a ptr to RingCoordinates object
+ // If there are tab rings the object will be actuall constructed
+ // The actual tabs will be extracted after we added all manual tabs
+ // But we need the number of tabs from rings at this location
+ std::unique_ptr<RingCoordinates> ptrRingCoords;
+ if (doFlysEye) {
+ // For Fly's Eye mode we have exactly one TAB per antenna field.
+ nrTABs = settings.antennaFields.size();
+ } else if (nrRings > 0) {
+ const string prefix = beamPrefix + str(format("[%u]") % i);
+ string directionType = getString(prefix + ".directionType", "J2000");
+
+ // Convert to COORDTYPES
+ RingCoordinates::COORDTYPES type;
+ if (directionType == "J2000")
+ type = RingCoordinates::J2000;
+ else if (directionType == "B1950")
+ type = RingCoordinates::B1950;
+ else
+ type = RingCoordinates::OTHER;
+
+ // Create coords object
+ ptrRingCoords = std::unique_ptr<RingCoordinates>(
+ new RingCoordinates(nrRings, ringWidth,
+ RingCoordinates::Coordinate(sap.direction.angle1, sap.direction.angle2), type));
+
+ // Increase the amount of tabs with the number from the coords object
+ // this might be zero
+ nrTABs = nrTABSParset + ptrRingCoords->nCoordinates();
+ }
+
+ sap.TABs.resize(nrTABs);
+
+ for (unsigned j = 0; j < nrTABs; ++j)
+ {
+ auto& tab = sap.TABs[j];
+
+ // Add flys eye tabs
+ if (doFlysEye)
+ {
+ // Copy direction from SAP
+ const string prefix = beamPrefix + str(format("[%u]") % i);
- settings.correlator.files.resize(settings.subbands.size());
- for (size_t i = 0; i < settings.correlator.files.size(); ++i) {
- if (i >= locations.size())
- THROW(CoInterfaceException, "No correlator filename or location specified for subband " << i);
+ tab.direction.type = getString(prefix + ".directionType", "J2000");
+ tab.direction.angle1 = getDouble(prefix + ".angle1", 0.0);
+ tab.direction.angle2 = getDouble(prefix + ".angle2", 0.0);
- settings.correlator.files[i].streamNr = i;
- settings.correlator.files[i].location = locations[i];
+ tab.dispersionMeasure = 0.0;
+ tab.coherent = true;
+ }
+ // Add manual tabs and then the tab rings.
+ else
+ {
+ if (j < nrTABSParset) // If we are working on manual tabs
+ {
+ const string prefix = beamPrefix + str(format("[%u].TiedArrayBeam[%u]") % i % j);
+ tab.direction.type = getString(prefix + ".directionType", "J2000");
- settings.correlator.files[i].dataStagingDirTargetFS =
- getString(str(format("PIC.Core.%s.dataStagingDirTargetFS") % locations[i].cluster), "");
- settings.correlator.files[i].metaDataStagingDirCreateFS =
- getString(str(format("PIC.Core.%s.metaDataStagingDirCreateFS") % locations[i].cluster), "");
- settings.correlator.files[i].metaDataStagingDirTargetFS =
- getString(str(format("PIC.Core.%s.metaDataStagingDirTargetFS") % locations[i].cluster), "");
+ tab.dispersionMeasure = getDouble(prefix + ".dispersionMeasure", 0.0);
+ tab.coherent = getBool(prefix + ".coherent", true);
- outputProcHosts.insert(settings.correlator.files[i].location.host);
+ // Incoherent TABs point in the same direction as the SAP by definition. The processing
+ // pipelines do not use the angles, but the data writer does as part of its annotation.
+ tab.direction.angle1 = tab.coherent ? getDouble(prefix + ".angle1", 0.0) : sap.direction.angle1;
+ tab.direction.angle2 = tab.coherent ? getDouble(prefix + ".angle2", 0.0) : sap.direction.angle2;
+ }
+ else
+ {
+ // Get the pointing for the tabrings.
+ // Subtract the number of manual to get index in the ringCoords
+ RingCoordinates::Coordinate pointing =
+ ptrRingCoords->coordinates().at(j - nrTABSParset);
+
+ // Note that RingCoordinates provide *relative* coordinates, and
+ // we need absolute ones.
+ tab.direction.type = ptrRingCoords->coordTypeAsString();
+ tab.direction.angle1 = sap.direction.angle1 + pointing.first; // TODO: missing projection bug (also below for angle2)
+ tab.direction.angle2 = sap.direction.angle2 + pointing.second;
+ // One dispersion measure for all TABs in rings is inconvenient,
+ // but not used anyway. Unclear if setting to 0.0 is better/worse.
+ const string prefix = beamPrefix + str(format("[%u]") % i);
+ tab.dispersionMeasure = getDouble(prefix + ".tabRingDispersionMeasure", 0.0);
+ tab.coherent = true; // rings cannot be incoherent, since we use non-(0,0) pointings
+ }
+ }
+
+ // Add this TAB to the global list of TABs
+ sap.TABs[j] = tab;
}
}
+ }
- /* ===============================
- * Beamformer pipeline information
- * ===============================
- */
+ void Parset::readQuantizerSettings(
+ std::string prefix,
+ ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings& qtSettings,
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings& qtDefaultSettings) const
+ {
+ // Enable quantizer if it is defined
+ qtSettings.enabled = getBool(prefix + ".quantize", qtDefaultSettings.enabled);
+ if (qtSettings.enabled)
+ {
+ qtSettings.nrBits = getUint32(prefix + ".quantizeBits", qtDefaultSettings.nrBits);
+ ASSERT(qtSettings.nrBits == 8 || qtSettings.nrBits == 16);
+ qtSettings.scaleMax = getDouble(prefix + ".quantizeScaleMax", qtDefaultSettings.scaleMax);
+ qtSettings.scaleMin = getDouble(prefix + ".quantizeScaleMin", qtDefaultSettings.scaleMin);
+ ASSERT(qtSettings.scaleMax > qtSettings.scaleMin);
+ qtSettings.sIpositive = getBool(prefix+".quantizeIpositive", qtDefaultSettings.sIpositive);
+ }
+ }
- // SAP/TAB-crossing counter for the files we generate
- size_t bfStreamNr = 0;
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings Parset::defaultQuantizerSettings() const
+ {
+ ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings
+ qtDefaultSettings =
+ {
+ false, // by default, no quantization of output
+ 8, // nrBits
+ 5, // scaleMax
+ -5, // scaleMin
+ false // sIpositive
+ };
+
+ return qtDefaultSettings;
+ }
+ const ObservationSettings::BeamFormer::StokesSettings Parset::defaultStokesSettings() const
+ {
+ ObservationSettings::BeamFormer::StokesSettings
+ stokesDefaultSettings =
+ {
+ true, // coherent
+ StokesType::STOKES_I, // type
+ (unsigned int) nrStokes(StokesType::STOKES_I), // nrStokes
+ 1, // nrChannels
+ 1, // timeIntegrationFactor
+ 0, // nrSamples
+ 0, // nrSubbandsPerFile
+ defaultQuantizerSettings()
+ };
+
+ return stokesDefaultSettings;
+ }
+
+ void Parset::readStokesSettings(
+ std::string bfPrefix,
+ bool coherent,
+ ObservationSettings::BeamFormer::StokesSettings& stSettings,
+ const ObservationSettings::BeamFormer::StokesSettings& stDefaultSettings,
+ const struct ObservationSettings& settings) const
+ {
+ stSettings.coherent = coherent;
+ std::string prefix = bfPrefix + (coherent ? ".CoherentStokes" : ".IncoherentStokes");
+
+ // Obtain settings of selected stokes
+ stSettings.type = stokesType(getString(prefix + ".which", stokesType(stDefaultSettings.type)));
+ stSettings.nrStokes = nrStokes(stSettings.type);
+ stSettings.nrChannels = getUint32(prefix + ".nrChannelsPerSubband", stDefaultSettings.nrChannels);
+ ASSERT(stSettings.nrChannels > 0);
+
+ stSettings.timeIntegrationFactor = getUint32(prefix + ".timeIntegrationFactor", stDefaultSettings.timeIntegrationFactor);
+ ASSERT(stSettings.timeIntegrationFactor > 0);
+ stSettings.nrSubbandsPerFile = getUint32(prefix + ".subbandsPerFile", stDefaultSettings.nrSubbandsPerFile); // 0 or a large nr is interpreted below
+ stSettings.nrSamples = settings.blockSize / stSettings.timeIntegrationFactor / stSettings.nrChannels;
+
+ // Enable quantizer if it is defined
+ auto qtDefaultSettings = stDefaultSettings.quantizerSettings.enabled ?
+ stDefaultSettings.quantizerSettings : defaultQuantizerSettings();
+ readQuantizerSettings(prefix, stSettings.quantizerSettings, qtDefaultSettings);
+ }
+
+ void Parset::addBeamFormer(
+ set<string>& outputProcHosts,
+ const vector<string>& stations,
+ struct ObservationSettings& settings) const
+ {
bool doCoherentStokes = getBool(
"Observation.DataProducts.Output_CoherentStokes.enabled", false);
bool doIncoherentStokes = getBool(
@@ -622,235 +834,262 @@ namespace LOFAR
settings.beamFormer.enabled = doCoherentStokes || doIncoherentStokes;
- if (settings.beamFormer.enabled) {
- // Parse global settings
- settings.beamFormer.doFlysEye = getBool("Cobalt.BeamFormer.flysEye", false);
+ if (!settings.beamFormer.enabled)
+ {
+ return;
+ }
- unsigned nrDelayCompCh;
- if (!isDefined("Cobalt.BeamFormer.nrDelayCompensationChannels")) {
- nrDelayCompCh = calcNrDelayCompensationChannels(settings);
- } else {
- nrDelayCompCh = getUint32("Cobalt.BeamFormer.nrDelayCompensationChannels");
- }
- settings.beamFormer.nrDelayCompensationChannels = nrDelayCompCh;
+ auto& beamFormer = settings.beamFormer;
- // Derive antennaFields to use for beam forming
- settings.beamFormer.antennaFieldNames = getOutputTypeAntennaFieldNames("Cobalt.BeamFormer.stationList",
- stations, settings.antennaSet);
- LOG_DEBUG_STR("Beamforming " << settings.beamFormer.antennaFieldNames.size() << " fields: " << settings.beamFormer.antennaFieldNames);
+ // Parse global settings
+ beamFormer.doFlysEye = getBool("Cobalt.BeamFormer.flysEye", false);
- ObservationSettings::BeamFormer::StokesSettings
- defaultSettings =
- {
- true, // coherent stokes?
- STOKES_I, // StokesType
- 1, // nrStokes
- 1, // nrChannels
- 1, // timeIntegrationFactor
- 0, // nrSamples
- 0 // nrSubbandsPerFile
- };
-
- settings.beamFormer.coherentSettings = defaultSettings;
- settings.beamFormer.incoherentSettings = defaultSettings;
-
- for (unsigned i = 0; i < 2; ++i) {
- // Set coherent and incoherent Stokes settings by
- // iterating twice.
- // TODO: This is an ugly way to do this.
-
- string prefix = "";
- struct ObservationSettings::BeamFormer::StokesSettings *stSettings = 0;
-
- // Select coherent or incoherent for this iteration
- switch(i) {
- case 0:
- prefix = "Cobalt.BeamFormer.CoherentStokes";
- stSettings = &settings.beamFormer.coherentSettings;
- stSettings->coherent = true;
- break;
-
- case 1:
- prefix = "Cobalt.BeamFormer.IncoherentStokes";
- stSettings = &settings.beamFormer.incoherentSettings;
- stSettings->coherent = false;
- break;
-
- default:
- ASSERT(false);
- break;
- }
+ unsigned nrDelayCompCh;
+ if (!isDefined("Cobalt.BeamFormer.nrDelayCompensationChannels")) {
+ nrDelayCompCh = calcNrDelayCompensationChannels(settings);
+ } else {
+ nrDelayCompCh = getUint32("Cobalt.BeamFormer.nrDelayCompensationChannels");
+ }
+ beamFormer.nrDelayCompensationChannels = nrDelayCompCh;
- // Coherent Stokes
- if (i == 0 && !doCoherentStokes)
- continue;
+ // Derive antennaFields to use for beam forming
+ beamFormer.antennaFieldNames = getOutputTypeAntennaFieldNames("Cobalt.BeamFormer.stationList",
+ stations, settings.antennaSet);
+ LOG_DEBUG_STR("Beamforming " << beamFormer.antennaFieldNames.size() << " fields: " << beamFormer.antennaFieldNames);
- // Incoherent Stokes
- if (i == 1 && !doIncoherentStokes)
- continue;
+ // Read default stokes settings
+ if (doCoherentStokes)
+ {
+ auto stokesDefaultSettings = defaultStokesSettings();
+ stokesDefaultSettings.coherent = true;
+ readStokesSettings("Cobalt.BeamFormer", true, beamFormer.coherentSettings, stokesDefaultSettings, settings);
+ }
+
+ if (doIncoherentStokes)
+ {
+ auto stokesDefaultSettings = defaultStokesSettings();
+ stokesDefaultSettings.coherent = false;
+ readStokesSettings("Cobalt.BeamFormer", false, beamFormer.incoherentSettings, stokesDefaultSettings, settings);
+ }
+
+ beamFormer.dedispersionFFTsize = getUint32("Cobalt.BeamFormer.dedispersionFFTsize", settings.blockSize);
- // Obtain settings of selected stokes
- stSettings->type = stokesType(getString(prefix + ".which", "I"));
- stSettings->nrStokes = nrStokes(stSettings->type);
- stSettings->nrChannels = getUint32(prefix + ".nrChannelsPerSubband", 1);
- ASSERT(stSettings->nrChannels > 0);
+ unsigned int nrSAPs = settings.SAPs.size();
- stSettings->timeIntegrationFactor = getUint32(prefix + ".timeIntegrationFactor", 1);
- ASSERT(stSettings->timeIntegrationFactor > 0);
- stSettings->nrSubbandsPerFile = getUint32(prefix + ".subbandsPerFile", 0); // 0 or a large nr is interpreted below
- stSettings->nrSamples = settings.blockSize / stSettings->timeIntegrationFactor / stSettings->nrChannels;
+ readTABs(settings, "Observation.Beam", settings.SAPs, beamFormer.doFlysEye);
+
+ // Read settings for all pipelines
+ unsigned nrPipelines = getUint32("Cobalt.BeamFormer.nrPipelines", 1);
+ beamFormer.pipelines.resize(nrPipelines);
+
+ // Read beamFormer pipeline settings
+ for (unsigned int pipelineNr = 0; pipelineNr < nrPipelines; pipelineNr++)
+ {
+ // Initialize the current pipeline with the default stokes settings
+ auto& pipeline = beamFormer.pipelines[pipelineNr];
+ pipeline.antennaFieldNames = beamFormer.antennaFieldNames;
+ pipeline.coherentSettings = beamFormer.coherentSettings;
+ pipeline.incoherentSettings = beamFormer.incoherentSettings;
+ string prefix = str(format("Cobalt.BeamFormer.Pipeline[%u]") % pipelineNr);
+
+ // Read stokes settings, use default settings when a key is not specified
+ if (doCoherentStokes)
+ {
+ readStokesSettings(prefix, true, pipeline.coherentSettings, beamFormer.coherentSettings, settings);
}
- const vector<ObservationSettings::FileLocation> coherent_locations =
- getFileLocations("CoherentStokes");
- const vector<ObservationSettings::FileLocation> incoherent_locations =
- getFileLocations("IncoherentStokes");
+ if (doIncoherentStokes)
+ {
+ readStokesSettings(prefix, false, pipeline.incoherentSettings, beamFormer.incoherentSettings, settings);
+ }
- size_t coherent_idx = 0;
- size_t incoherent_idx = 0;
+ // Derive antennaFields to use for beam forming
+ if (isDefined(prefix + ".stationList"))
+ {
+ pipeline.antennaFieldNames = getOutputTypeAntennaFieldNames(prefix + ".stationList",
+ stations, settings.antennaSet);
+ LOG_DEBUG_STR("Pipeline " << pipelineNr << " is beamforming " << beamFormer.antennaFieldNames.size() << " fields: " << beamFormer.antennaFieldNames);
+ } else {
+ pipeline.antennaFieldNames = beamFormer.antennaFieldNames;
+ }
+
+ // Read the flysEye setting
+ pipeline.doFlysEye = getBool(prefix + ".flysEye", beamFormer.doFlysEye);
+ } // end for pipelineNr
- // Parse all TABs
- //
- // Strategy:
- // * If Fly's Eye is enabled, we have 1 TAB/station. Do NOT process any other TABs.
- // * If Fly's Eye is disabled, process TABs in this order:
- // 1. Manually specified TABs (Observation.Beam[x].TiedArrayBeam[y]
- // 2. TAB rings (Observation.Beam[x].nrTabRings)
- settings.beamFormer.SAPs.resize(nrSAPs);
+ // Read beamFormer pipeline TABs
+ for (unsigned int pipelineNr = 0; pipelineNr < nrPipelines; pipelineNr++)
+ {
+ auto& pipeline = beamFormer.pipelines[pipelineNr];
- for (unsigned i = 0; i < nrSAPs; ++i)
+ // Make copy of observation beams
+ auto obsSAPs = settings.SAPs;
+
+ // Clear all TABs (if any)
+ for (auto& obsSAP : obsSAPs)
{
- struct ObservationSettings::BeamFormer::SAP &sap = settings.beamFormer.SAPs[i];
- struct ObservationSettings::SAP &obsSap = settings.SAPs[i];
+ obsSAP.TABs.resize(0);
+ }
- // Clear counters
- sap.nrCoherent = 0;
- sap.nrIncoherent = 0;
+ // Initialize beamFormer beams
+ pipeline.SAPs.resize(nrSAPs);
- size_t nrTABs = getUint32(str(format("Observation.Beam[%u].nrTiedArrayBeams") % i), 0);
- size_t nrTABSParset = nrTABs;
- size_t nrRings = getUint32(str(format("Observation.Beam[%u].nrTabRings") % i), 0);
- double ringWidth = getDouble(str(format("Observation.Beam[%u].tabRingSize") % i), 0.0);
+ // Try to read TABs for this beamFormer pipeline
+ string prefix = str(format("Cobalt.BeamFormer.Pipeline[%u]") % pipelineNr);
+ readTABs(settings, prefix + ".Beam", obsSAPs, pipeline.doFlysEye);
- // Throw if we encounter an unsupported configuration
- if (settings.beamFormer.doFlysEye) {
- if (nrRings > 0 || nrTABSParset > 0) {
- THROW(CoInterfaceException, "Cannot produce (in)coherent TABs in Fly's Eye mode. Error detected in SAP " << i << ".");
- }
- }
+ for (unsigned int sapNr = 0; sapNr < settings.SAPs.size(); sapNr++)
+ {
+ // This is a copy of the observation SAP, with the TABs for the current pipeline
+ auto& obsSAP = obsSAPs[sapNr];
- // Create a ptr to RingCoordinates object
- // If there are tab rings the object will be actuall constructed
- // The actual tabs will be extracted after we added all manual tabs
- // But we need the number of tabs from rings at this location
- std::unique_ptr<RingCoordinates> ptrRingCoords;
- if (settings.beamFormer.doFlysEye) {
- // For Fly's Eye mode we have exactly one TAB per antenna field.
- nrTABs = settings.antennaFields.size();
- } else if (nrRings > 0) {
- const string prefix = str(format("Observation.Beam[%u]") % i);
- string directionType = getString(prefix + ".directionType", "J2000");
-
- // Convert to COORDTYPES
- RingCoordinates::COORDTYPES type;
- if (directionType == "J2000")
- type = RingCoordinates::J2000;
- else if (directionType == "B1950")
- type = RingCoordinates::B1950;
- else
- type = RingCoordinates::OTHER;
-
- // Create coords object
- ptrRingCoords = std::unique_ptr<RingCoordinates>(
- new RingCoordinates(nrRings, ringWidth,
- RingCoordinates::Coordinate(obsSap.direction.angle1, obsSap.direction.angle2), type));
-
- // Increase the amount of tabs with the number from the coords object
- // this might be zero
- nrTABs = nrTABSParset + ptrRingCoords->nCoordinates();
- }
+ // This is the SAP for the current beamFormer pipeline
+ auto& bfSAP = pipeline.SAPs[sapNr];
- sap.TABs.resize(nrTABs);
- for (unsigned j = 0; j < nrTABs; ++j)
+ // In case no TABs were specified for the current SAP, fall back
+ // to the settings read from "Observation.Beam[sapNr]"
+ if (obsSAP.TABs.size() == 0)
{
- struct ObservationSettings::BeamFormer::TAB &tab = sap.TABs[j];
- // Add flys eye tabs
- if (settings.beamFormer.doFlysEye)
- {
- // Copy direction from SAP
- const string prefix = str(format("Observation.Beam[%u]") % i);
+ obsSAP.TABs = settings.SAPs[sapNr].TABs;
+ }
- tab.direction.type = getString(prefix + ".directionType", "J2000");
- tab.direction.angle1 = getDouble(prefix + ".angle1", 0.0);
- tab.direction.angle2 = getDouble(prefix + ".angle2", 0.0);
-
- tab.dispersionMeasure = 0.0;
- tab.coherent = true;
- }
- // Add manual tabs and then the tab rings.
- else
+ // Initialize the list of TABs for the current pipeline/SAP
+ unsigned nrTABs = obsSAP.TABs.size();
+ bfSAP.TABs.resize(nrTABs);
+
+ // Reset nrCoherent and nrIncoherent
+ bfSAP.nrCoherent = 0;
+ bfSAP.nrIncoherent = 0;
+
+ // Read subband numbers for the current beamFormer pipeline
+ const vector<unsigned> emptyVectorUnsigned;
+ auto bfSubbandNrs = getUint32Vector(prefix + str(format(".Beam[%u].subbandList") % sapNr), emptyVectorUnsigned, true);
+
+ // Try to match the subband numbers with the global subband numbers,
+ // if a match is found, add the corresponding subband index to the
+ // local list of subband indices
+ bfSAP.subbandIndices.resize(0);
+ for (unsigned bfSubbandNr : bfSubbandNrs)
+ {
+ for (auto obsSubband : obsSAP.subbands)
{
- if (j < nrTABSParset) // If we are working on manual tabs
- {
- const string prefix = str(format("Observation.Beam[%u].TiedArrayBeam[%u]") % i % j);
- tab.direction.type = getString(prefix + ".directionType", "J2000");
-
- tab.dispersionMeasure = getDouble(prefix + ".dispersionMeasure", 0.0);
- tab.coherent = getBool(prefix + ".coherent", true);
-
- // Incoherent TABs point in the same direction as the SAP by definition. The processing
- // pipelines do not use the angles, but the data writer does as part of its annotation.
- tab.direction.angle1 = tab.coherent ? getDouble(prefix + ".angle1", 0.0) : obsSap.direction.angle1;
- tab.direction.angle2 = tab.coherent ? getDouble(prefix + ".angle2", 0.0) : obsSap.direction.angle2;
- }
- else
+ if (bfSubbandNr == obsSubband.stationIdx)
{
- // Get the pointing for the tabrings.
- // Subtract the number of manual to get index in the ringCoords
- RingCoordinates::Coordinate pointing =
- ptrRingCoords->coordinates().at(j - nrTABSParset);
-
- // Note that RingCoordinates provide *relative* coordinates, and
- // we need absolute ones.
- tab.direction.type = ptrRingCoords->coordTypeAsString();
- tab.direction.angle1 = obsSap.direction.angle1 + pointing.first; // TODO: missing projection bug (also below for angle2)
- tab.direction.angle2 = obsSap.direction.angle2 + pointing.second;
- // One dispersion measure for all TABs in rings is inconvenient,
- // but not used anyway. Unclear if setting to 0.0 is better/worse.
- const string prefix = str(format("Cobalt.Observation.Beam[%u]") % i);
- tab.dispersionMeasure = getDouble(prefix + ".tabRingDispersionMeasure", 0.0);
- tab.coherent = true; // rings cannot be incoherent, since we use non-(0,0) pointings
+ bfSAP.subbandIndices.push_back(obsSubband.idx);
}
}
+ }
+
+ ASSERTSTR(bfSAP.subbandIndices.size() == bfSubbandNrs.size(),
+ "Could not map all subbands for beamFormer " << pipelineNr <<
+ ", selected " << bfSubbandNrs.size() << " subbands, but matched " <<
+ bfSAP.subbandIndices.size() << " subbands.");
+
+ // In case no subband numbers were matched,
+ // fall back to using all global subband indices
+ if (bfSAP.subbandIndices.size() == 0)
+ {
+ bfSAP.subbandIndices = settings.SAPs[sapNr].subbandIndices();
+ }
+
+ // Fill in the TABs for the current pipeline/SAP
+ for (unsigned localTabNr = 0; localTabNr < nrTABs; localTabNr++) {
+ auto& bfTAB = bfSAP.TABs[localTabNr];
+ auto& obsTAB = obsSAP.TABs[localTabNr];
+
+ bfSAP.globalSapIdx = sapNr;
+ bfTAB.localTabNr = localTabNr;
+ bfTAB.direction = obsTAB.direction;
+ bfTAB.coherent = obsTAB.coherent;
+ bfTAB.dispersionMeasure = obsTAB.dispersionMeasure;
+ // The tab.files entry is filled below
+ bfTAB.pipelineNr = pipelineNr;
+ bfSAP.nrCoherent += obsTAB.coherent;
+ bfSAP.nrIncoherent += !obsTAB.coherent;
+ }
+ } // end for sapNr
+ } // end for pipelineNr
+
+ // Register all TABs in the global list of TABs
+ beamFormer.SAPs.resize(nrSAPs);
+ for (unsigned sapNr = 0; sapNr < nrSAPs; sapNr++)
+ {
+ auto& sap = beamFormer.SAPs[sapNr];
+ sap.globalSapIdx = sapNr;
+ sap.nrCoherent = 0;
+ sap.nrIncoherent = 0;
+ sap.subbandIndices = settings.SAPs[sapNr].subbandIndices();
+
+ for (auto& pipeline : beamFormer.pipelines)
+ {
+ for (auto& tab : pipeline.SAPs[sapNr].TABs)
+ {
+ sap.nrCoherent += tab.coherent;
+ sap.nrIncoherent += !tab.coherent;
+ beamFormer.SAPs[sapNr].TABs.push_back(tab);
+ }
+ } // end for pipeline
+ } // end for sapNr
+
+ const vector<ObservationSettings::FileLocation> coherent_locations =
+ getFileLocations("CoherentStokes");
+ const vector<ObservationSettings::FileLocation> incoherent_locations =
+ getFileLocations("IncoherentStokes");
+
+ size_t coherent_idx = 0;
+ size_t incoherent_idx = 0;
+
+ size_t bfStreamNr = 0;
+
+ // Set the files for all TABs
+ for (unsigned sapNr = 0; sapNr < nrSAPs; ++sapNr)
+ {
+ struct ObservationSettings::SAP &obsSAP = settings.SAPs[sapNr];
+
+ size_t tabNr = 0;
+ size_t coherentTabNr = 0;
+ for (auto& pipeline : settings.beamFormer.pipelines)
+ {
+ struct ObservationSettings::BeamFormer::SAP &sap = pipeline.SAPs[sapNr];
+
+ // Clear counters
+ sap.nrCoherent = 0;
+ sap.nrIncoherent = 0;
+
+ for (auto& tab : sap.TABs)
+ {
if (tab.coherent)
sap.nrCoherent++;
else
sap.nrIncoherent++;
struct ObservationSettings::BeamFormer::StokesSettings &stSettings =
- tab.coherent ? settings.beamFormer.coherentSettings
- : settings.beamFormer.incoherentSettings;
+ tab.coherent ? pipeline.coherentSettings
+ : pipeline.incoherentSettings;
+
+ // This is the index in the tabDelays
+ if (tab.coherent)
+ tab.coherentIdxInSAP = coherentTabNr;
// If needed, limit to / apply default: the #subbands in this SAP.
size_t nrSubbandsPerFile = stSettings.nrSubbandsPerFile;
if (nrSubbandsPerFile == 0 ||
- nrSubbandsPerFile > settings.SAPs[i].subbands.size()) {
- nrSubbandsPerFile = settings.SAPs[i].subbands.size();
+ nrSubbandsPerFile > obsSAP.subbands.size()) {
+ nrSubbandsPerFile = obsSAP.subbands.size();
}
// Generate file list
- unsigned nrParts = max(1UL, ceilDiv(settings.SAPs[i].subbands.size(), nrSubbandsPerFile));
- tab.files.resize(stSettings.nrStokes * nrParts);
- for (size_t s = 0; s < stSettings.nrStokes; ++s)
+ unsigned nrParts = max(1UL, ceilDiv(obsSAP.subbands.size(), nrSubbandsPerFile));
+ for (size_t s = 0; s < stSettings.nrStokes; ++s)
{
for (unsigned part = 0; part < nrParts; ++part)
{
struct ObservationSettings::BeamFormer::File file;
file.streamNr = bfStreamNr++;
- file.sapNr = i;
- file.tabNr = j;
+ file.sapNr = sapNr;
+ file.tabNr = tabNr;
file.stokesNr = s;
file.partNr = part;
file.coherent = tab.coherent;
@@ -867,148 +1106,181 @@ namespace LOFAR
file.location = incoherent_locations[incoherent_idx++];
}
- file.firstSubbandIdx = settings.SAPs[i].subbands[0].idx +
+ file.firstSubbandIdx = pipeline.SAPs[sapNr].subbandIndices[0] +
part * nrSubbandsPerFile;
file.lastSubbandIdx = min(file.firstSubbandIdx + nrSubbandsPerFile,
// last file(s) in part series can have fewer subbands
- settings.SAPs[i].subbands[0].idx +
- settings.SAPs[i].subbands.size());
+ pipeline.SAPs[sapNr].subbandIndices[0] +
+ pipeline.SAPs[sapNr].subbandIndices.size());
ASSERTSTR(file.firstSubbandIdx < file.lastSubbandIdx,
"strmNr=" << file.streamNr << " 1stIdx=" << file.firstSubbandIdx << " lstIdx=" << file.lastSubbandIdx);
ASSERTSTR(file.lastSubbandIdx <= settings.subbands.size(),
"strmNr=" << file.streamNr << " lstIdx=" << file.lastSubbandIdx << " nSb=" << settings.subbands.size());
- tab.files[s * nrParts + part] = file;
settings.beamFormer.files.push_back(file);
outputProcHosts.insert(file.location.host);
- }
- }
- }
- }
+ } // end for nrParts
+ } // end for nrStokes
+
+ // Increment the tabNr
+ tabNr++;
+
+ // Increment the coherentTabNr
+ if (tab.coherent)
+ coherentTabNr++;
+ } // end for TABs
+ } // end for pipelines
+ } // end for sapNr
+ }
- settings.beamFormer.dedispersionFFTsize = getUint32("Cobalt.BeamFormer.dedispersionFFTsize", settings.blockSize);
+ void Parset::addRSPRawInformation(
+ set<string>& outputProcHosts,
+ const vector<string>& stations,
+ struct ObservationSettings& settings) const
+ {
+ const vector<unsigned> emptyVectorUnsigned;
+
+ if (isDefined("Cobalt.RSPRaw.startTime")) {
+ settings.rspRaw.startTime = getTime("Cobalt.RSPRaw.startTime", "");
+ LOG_INFO_STR("RSP raw startTime specified as " << getString("Cobalt.RSPRaw.startTime"));
+ } else {
+ settings.rspRaw.startTime = (time_t)settings.startTime;
+ }
+ if (isDefined("Cobalt.RSPRaw.stopTime")) {
+ settings.rspRaw.stopTime = getTime("Cobalt.RSPRaw.stopTime", "");
+ LOG_INFO_STR("RSP raw stopTime specified as " << getString("Cobalt.RSPRaw.stopTime"));
+ } else {
+ settings.rspRaw.stopTime = (time_t)settings.stopTime;
}
- /* ===============================
- * RSPRaw collection information
- * ===============================
- */
+ // Read antenna field names (via stationList key) to use for RSP raw output
+ settings.rspRaw.antennaFieldNames = getOutputTypeAntennaFieldNames("Cobalt.RSPRaw.stationList",
+ stations, settings.antennaSet);
+ if (settings.rspRaw.antennaFieldNames.empty()) {
+ settings.rspRaw.antennaFieldNames = settings.antennaFieldNames; // default
+ }
+ LOG_INFO_STR("RSP raw: " << settings.rspRaw.antennaFieldNames.size() << " fields: " << settings.rspRaw.antennaFieldNames);
- settings.rspRaw.enabled = getBool("Observation.DataProducts.Output_RSPRaw.enabled", false);
- if (settings.rspRaw.enabled) {
- if (isDefined("Cobalt.RSPRaw.startTime")) {
- settings.rspRaw.startTime = getTime("Cobalt.RSPRaw.startTime", "");
- LOG_INFO_STR("RSP raw startTime specified as " << getString("Cobalt.RSPRaw.startTime"));
- } else {
- settings.rspRaw.startTime = (time_t)settings.startTime;
- }
- if (isDefined("Cobalt.RSPRaw.stopTime")) {
- settings.rspRaw.stopTime = getTime("Cobalt.RSPRaw.stopTime", "");
- LOG_INFO_STR("RSP raw stopTime specified as " << getString("Cobalt.RSPRaw.stopTime"));
- } else {
- settings.rspRaw.stopTime = (time_t)settings.stopTime;
+ /*
+ * RSPRaw is not a supported output type in the rest of the system (it is COBALT-only).
+ * The Observatory specifies a correlation or beamforming observation, then copies a
+ * parset override in place. Hence, all RSPRaw settings must be auto-detectable (sane defaults),
+ * possibly from correlation or beamforming settings, even if those are not enabled (anymore)!
+ */
+ vector<ObservationSettings::FileLocation> locations = getFileLocations("RSPRaw");
+ const bool locationsAutoDetected = locations.empty();
+ if (locationsAutoDetected) {
+ LOG_INFO("RSP raw: trying to auto-detect storage locations from correlated and/or coherent stokes and/or incoherent stokes storage locations");
+ const vector<ObservationSettings::FileLocation> correlated_locations = getFileLocations("Correlated");
+ const vector<ObservationSettings::FileLocation> coherent_locations = getFileLocations("CoherentStokes");
+ const vector<ObservationSettings::FileLocation> incoherent_locations = getFileLocations("IncoherentStokes");
+
+ locations.insert(locations.end(), correlated_locations.begin(), correlated_locations.end());
+ locations.insert(locations.end(), coherent_locations.begin(), coherent_locations.end());
+ locations.insert(locations.end(), incoherent_locations.begin(), incoherent_locations.end());
+ if (locations.empty()) {
+ THROW(CoInterfaceException, "No RSP raw locations specified and could not derive any location(s) from correlated, coherent stokes, or incoherent stokes file locations (even if not enabled).");
}
- // Read antenna field names (via stationList key) to use for RSP raw output
- settings.rspRaw.antennaFieldNames = getOutputTypeAntennaFieldNames("Cobalt.RSPRaw.stationList",
- stations, settings.antennaSet);
- if (settings.rspRaw.antennaFieldNames.empty()) {
- settings.rspRaw.antennaFieldNames = settings.antennaFieldNames; // default
+ // Purge duplicate hostname locations. By clearing filenames first, we can simply do sort() and unique().
+ for (size_t i = 0; i < locations.size(); ++i) {
+ locations[i].filename.clear();
}
- LOG_INFO_STR("RSP raw: " << settings.rspRaw.antennaFieldNames.size() << " fields: " << settings.rspRaw.antennaFieldNames);
-
- /*
- * RSPRaw is not a supported output type in the rest of the system (it is COBALT-only).
- * The Observatory specifies a correlation or beamforming observation, then copies a
- * parset override in place. Hence, all RSPRaw settings must be auto-detectable (sane defaults),
- * possibly from correlation or beamforming settings, even if those are not enabled (anymore)!
- */
- vector<ObservationSettings::FileLocation> locations = getFileLocations("RSPRaw");
- const bool locationsAutoDetected = locations.empty();
- if (locationsAutoDetected) {
- LOG_INFO("RSP raw: trying to auto-detect storage locations from correlated and/or coherent stokes and/or incoherent stokes storage locations");
- const vector<ObservationSettings::FileLocation> correlated_locations = getFileLocations("Correlated");
- const vector<ObservationSettings::FileLocation> coherent_locations = getFileLocations("CoherentStokes");
- const vector<ObservationSettings::FileLocation> incoherent_locations = getFileLocations("IncoherentStokes");
-
- locations.insert(locations.end(), correlated_locations.begin(), correlated_locations.end());
- locations.insert(locations.end(), coherent_locations.begin(), coherent_locations.end());
- locations.insert(locations.end(), incoherent_locations.begin(), incoherent_locations.end());
- if (locations.empty()) {
- THROW(CoInterfaceException, "No RSP raw locations specified and could not derive any location(s) from correlated, coherent stokes, or incoherent stokes file locations (even if not enabled).");
- }
+ std::sort(locations.begin(), locations.end());
+ std::unique(locations.begin(), locations.end());
+ }
- // Purge duplicate hostname locations. By clearing filenames first, we can simply do sort() and unique().
- for (size_t i = 0; i < locations.size(); ++i) {
- locations[i].filename.clear();
- }
- std::sort(locations.begin(), locations.end());
- std::unique(locations.begin(), locations.end());
+ // Assign output file locations
+ vector<unsigned> maxNrBeamletsPerBoard; // normally we end up with 4 sending RSP boards per ant field
+ unsigned locationIdx = 0;
+ for (size_t i = 0; i < settings.rspRaw.antennaFieldNames.size(); ++i) {
+ const ObservationSettings::AntennaFieldName& afName = settings.rspRaw.antennaFieldNames[i];
+ vector<ObservationSettings::AntennaFieldName>::iterator nameIt = std::find(settings.antennaFieldNames.begin(),
+ settings.antennaFieldNames.end(), afName);
+ ASSERTSTR(nameIt != settings.antennaFieldNames.end(), "RSP raw antenna field name " << afName.fullName() <<
+ " missing in observation antenna field list"); // getOutputTypeAntennaFieldNames() must have avoided this
+ vector<ObservationSettings::AntennaField>::iterator afIt = settings.antennaFields.begin() +
+ std::distance(settings.antennaFieldNames.begin(), nameIt);
+ size_t nrStreams = afIt->inputStreams.size();
+ if (maxNrBeamletsPerBoard.size() < nrStreams) {
+ maxNrBeamletsPerBoard.resize(nrStreams);
}
- // Assign output file locations
- vector<unsigned> maxNrBeamletsPerBoard; // normally we end up with 4 sending RSP boards per ant field
- unsigned locationIdx = 0;
- for (size_t i = 0; i < settings.rspRaw.antennaFieldNames.size(); ++i) {
- const ObservationSettings::AntennaFieldName& afName = settings.rspRaw.antennaFieldNames[i];
- vector<ObservationSettings::AntennaFieldName>::iterator nameIt = std::find(settings.antennaFieldNames.begin(),
- settings.antennaFieldNames.end(), afName);
- ASSERTSTR(nameIt != settings.antennaFieldNames.end(), "RSP raw antenna field name " << afName.fullName() <<
- " missing in observation antenna field list"); // getOutputTypeAntennaFieldNames() must have avoided this
- vector<ObservationSettings::AntennaField>::iterator afIt = settings.antennaFields.begin() +
- std::distance(settings.antennaFieldNames.begin(), nameIt);
- size_t nrStreams = afIt->inputStreams.size();
- if (maxNrBeamletsPerBoard.size() < nrStreams) {
- maxNrBeamletsPerBoard.resize(nrStreams);
- }
-
- for (unsigned s = 0; s < nrStreams; ++s) {
- if (locationIdx >= locations.size()) {
- if (locationsAutoDetected) {
- locationIdx = 0; // if auto-detected, assign locations in round-robin order
- } else { // Either you specify no locations and we do it all, or you specify a complete list. No half-baked RSP raw settings!
- THROW(CoInterfaceException, "No RSP raw filename or location specified for antenna field " <<
- afName.fullName() << " stream " << s);
- }
- }
-
- ObservationSettings::RSPRaw::File file;
- file.antennaFieldNameIdx = i;
- file.streamNr = s;
+ for (unsigned s = 0; s < nrStreams; ++s) {
+ if (locationIdx >= locations.size()) {
if (locationsAutoDetected) {
- locations[locationIdx].filename = str(format("L%u_%s_%u_rsp.raw") % settings.observationID % afName.fullName().c_str() % s);
+ locationIdx = 0; // if auto-detected, assign locations in round-robin order
+ } else { // Either you specify no locations and we do it all, or you specify a complete list. No half-baked RSP raw settings!
+ THROW(CoInterfaceException, "No RSP raw filename or location specified for antenna field " <<
+ afName.fullName() << " stream " << s);
}
- file.location = locations[locationIdx];
- settings.rspRaw.files.push_back(file);
+ }
- outputProcHosts.insert(file.location.host);
- LOG_INFO_STR("RSP raw: file " << locations[locationIdx].filename <<
- " is written to " << file.location.toString());
+ ObservationSettings::RSPRaw::File file;
+ file.antennaFieldNameIdx = i;
+ file.streamNr = s;
+ if (locationsAutoDetected) {
+ locations[locationIdx].filename = str(format("L%u_%s_%u_rsp.raw") % settings.observationID % afName.fullName().c_str() % s);
+ }
+ file.location = locations[locationIdx];
+ settings.rspRaw.files.push_back(file);
- locationIdx += 1;
+ outputProcHosts.insert(file.location.host);
+ LOG_INFO_STR("RSP raw: file " << locations[locationIdx].filename <<
+ " is written to " << file.location.toString());
- maxNrBeamletsPerBoard[s] = std::max(maxNrBeamletsPerBoard[s],
- (unsigned)std::count(afIt->rspBoardMap.begin(),
- afIt->rspBoardMap.end(), s));
- }
- }
+ locationIdx += 1;
- // Read nrBeamletsPerBoardList and apply sane defaults wrt max nr beamlets per board in the obs
- settings.rspRaw.nrBeamletsPerBoardList = getUint32Vector("Cobalt.RSPRaw.nrBeamletsPerBoardList",
- emptyVectorUnsigned, true);
- if (settings.rspRaw.nrBeamletsPerBoardList.size() < maxNrBeamletsPerBoard.size()) {
- settings.rspRaw.nrBeamletsPerBoardList.resize(maxNrBeamletsPerBoard.size(), UINT_MAX);
+ maxNrBeamletsPerBoard[s] = std::max(maxNrBeamletsPerBoard[s],
+ (unsigned)std::count(afIt->rspBoardMap.begin(),
+ afIt->rspBoardMap.end(), s));
}
- for (size_t i = 0; i < settings.rspRaw.nrBeamletsPerBoardList.size(); ++i) {
- settings.rspRaw.nrBeamletsPerBoardList[i] = std::min(settings.rspRaw.nrBeamletsPerBoardList[i],
- maxNrBeamletsPerBoard[i]);
- }
- LOG_INFO_STR("RSP raw: max nr beamlets per RSP board to forward: " << settings.rspRaw.nrBeamletsPerBoardList);
}
+ // Read nrBeamletsPerBoardList and apply sane defaults wrt max nr beamlets per board in the obs
+ settings.rspRaw.nrBeamletsPerBoardList = getUint32Vector("Cobalt.RSPRaw.nrBeamletsPerBoardList",
+ emptyVectorUnsigned, true);
+ if (settings.rspRaw.nrBeamletsPerBoardList.size() < maxNrBeamletsPerBoard.size()) {
+ settings.rspRaw.nrBeamletsPerBoardList.resize(maxNrBeamletsPerBoard.size(), UINT_MAX);
+ }
+ for (size_t i = 0; i < settings.rspRaw.nrBeamletsPerBoardList.size(); ++i) {
+ settings.rspRaw.nrBeamletsPerBoardList[i] = std::min(settings.rspRaw.nrBeamletsPerBoardList[i],
+ maxNrBeamletsPerBoard[i]);
+ }
+ LOG_INFO_STR("RSP raw: max nr beamlets per RSP board to forward: " << settings.rspRaw.nrBeamletsPerBoardList);
+ }
- // set output hosts
+ void Parset::addCobaltInformation(
+ struct ObservationSettings& settings) const
+ {
+ settings.cobalt.kernel.dumpBandPassCorrectionKernel = getBool("Cobalt.Kernels.BandPassCorrectionKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpBeamFormerKernel = getBool("Cobalt.Kernels.BeamFormerKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpBeamFormerTransposeKernel = getBool("Cobalt.Kernels.BeamFormerTransposeKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpCoherentStokesKernel = getBool("Cobalt.Kernels.CoherentStokesKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpCoherentStokesTransposeKernel = getBool("Cobalt.Kernels.CoherentStokesTransposeKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpCorrelatorKernel = getBool("Cobalt.Kernels.CorrelatorKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpDelayAndBandPassKernel = getBool("Cobalt.Kernels.DelayAndBandPassKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpFFTShiftKernel = getBool("Cobalt.Kernels.FFTShiftKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpFFT_Kernel = getBool("Cobalt.Kernels.FFT_Kernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpFIR_FilterKernel = getBool("Cobalt.Kernels.FIR_FilterKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpIncoherentStokesKernel = getBool("Cobalt.Kernels.IncoherentStokesKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpIncoherentStokesTransposeKernel = getBool("Cobalt.Kernels.IncoherentStokesTransposeKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpIntToFloatKernel = getBool("Cobalt.Kernels.IntToFloatKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpIntToFloatKernel = getBool("Cobalt.Kernels.IntToFloatKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpQuantizeOutputKernel = getBool("Cobalt.Kernels.QuantizeOutputKernel.dumpOutput", false);
+ settings.cobalt.kernel.dumpZeroingKernel = getBool("Cobalt.Kernels.ZeroingKernel.dumpOutput", false);
+
+ const std::string emptyString = "";
+ settings.cobalt.benchmark.enabled = getBool("Cobalt.Benchmark.enabled", false);
+ settings.cobalt.benchmark.file = getString("Cobalt.Benchmark.file", emptyString);
+ }
+
+ void Parset::setOutputHosts(
+ set<string>& outputProcHosts,
+ struct ObservationSettings& settings) const
+ {
settings.outputProcHosts.clear();
+
for (set<string>::const_iterator i = outputProcHosts.begin(); i != outputProcHosts.end(); ++i) {
if (i->empty()) {
continue;
@@ -1016,6 +1288,83 @@ namespace LOFAR
settings.outputProcHosts.push_back(*i);
}
+ }
+
+ struct ObservationSettings Parset::observationSettings() const
+ {
+ struct ObservationSettings settings;
+
+ // the set of hosts on which outputProc has to run, which will
+ // be constructed during the parsing of the parset
+ set<string> outputProcHosts;
+
+ // NOTE: Make sure that all keys have defaults, to make test parsets
+ // a lot shorter.
+ // --Update: No, instead use tParsetDefault in tests and reject parsets with missing required values.
+
+ const vector<string> emptyVectorString;
+ const vector<unsigned> emptyVectorUnsigned;
+ const vector<double> emptyVectorDouble;
+
+ // Generic information
+ addGenericInformation(settings);
+
+ // Station information (required by pointing information)
+ settings.antennaSet = getString("Observation.antennaSet", "LBA_INNER");
+ settings.bandFilter = getString("Observation.bandFilter", "LBA_30_70");
+
+ // Station list
+ vector<string> stations = getStringVector("Observation.VirtualInstrument.stationList", emptyVectorString, true);
+ ASSERTSTR(!stations.empty(), "station list (Observation.VirtualInstrument.stationList) must be non-empty");
+ settings.rawStationList = getString("Observation.VirtualInstrument.stationList", "[]");
+
+ // The correlator.enabled setting is used as default value for sap.correlatorEnabled
+ // and thus has thus be set before addSAPs is called.
+ settings.correlator.enabled = getBool("Observation.DataProducts.Output_Correlated.enabled", false);
+
+ // Pointing information
+ addSAPs(settings);
+ addAnaBeamInformation(settings);
+
+ // Station information (used pointing information to verify settings)
+ addStationInformation(stations, settings);
+
+ // Resource information
+ addResourceInformation(settings);
+
+ /* ===============================
+ * Correlator pipeline information
+ * ===============================
+ */
+ if (settings.correlator.enabled) {
+ addCorrelatorInformation(outputProcHosts, settings);
+ }
+
+ /* ===============================
+ * Beamformer pipeline information
+ * ===============================
+ */
+ addBeamFormer(
+ outputProcHosts,
+ stations, settings);
+
+ /* ===============================
+ * RSPRaw collection information
+ * ===============================
+ */
+ settings.rspRaw.enabled = getBool("Observation.DataProducts.Output_RSPRaw.enabled", false);
+ if (settings.rspRaw.enabled) {
+ addRSPRawInformation(outputProcHosts, stations, settings);
+ }
+
+ /* ===============================
+ * RSPRaw collection information
+ * ===============================
+ */
+ addCobaltInformation(settings);
+
+ // set output hosts
+ setOutputHosts(outputProcHosts, settings);
return settings;
}
@@ -1176,6 +1525,16 @@ namespace LOFAR
return blockSize * sampleDuration();
}
+ vector<unsigned> ObservationSettings::subbandIndices() const {
+ vector<unsigned> indices;
+
+ for (size_t i = 0; i < subbands.size(); ++i) {
+ indices.push_back(subbands[i].idx);
+ }
+
+ return indices;
+ }
+
vector<unsigned> ObservationSettings::SAP::subbandIndices() const {
vector<unsigned> indices;
@@ -1194,12 +1553,15 @@ namespace LOFAR
return nrSamplesPerBlock / nrIntegrationsPerBlock * nrBlocksPerIntegration;
}
- std::vector<struct ObservationSettings::FileLocation> Parset::getFileLocations(const std::string outputType) const {
- const string prefix = "Observation.DataProducts.Output_" + outputType;
+ std::vector<struct ObservationSettings::FileLocation> Parset::getFileLocations(
+ const std::string outputType,
+ const std::string prefix) const
+ {
+ const string output_prefix = prefix + ".DataProducts.Output_" + outputType;
vector<string> empty;
- vector<string> filenames = getStringVector(prefix + ".filenames", empty, true);
- vector<string> locations = getStringVector(prefix + ".locations", empty, true);
+ vector<string> filenames = getStringVector(output_prefix + ".filenames", empty, true);
+ vector<string> locations = getStringVector(output_prefix + ".locations", empty, true);
size_t numValidEntries = std::min(filenames.size(), locations.size());
@@ -1219,7 +1581,7 @@ namespace LOFAR
location.host = host_dir[1];
location.directory = host_dir[2];
} else {
- THROW(CoInterfaceException, "Location must adhere to '[cluster:]host:directory' in " << prefix << ".locations: " << locations[i]);
+ THROW(CoInterfaceException, "Location must adhere to '[cluster:]host:directory' in " << output_prefix << ".locations: " << locations[i]);
}
}
@@ -1261,12 +1623,33 @@ namespace LOFAR
}
+ bool ObservationSettings::BeamFormer::Pipeline::anyCoherentTABs() const
+ {
+ return maxNrCoherentTABsPerSAP() > 0;
+ }
+
+
+ bool ObservationSettings::BeamFormer::Pipeline::anyIncoherentTABs() const
+ {
+ return maxNrIncoherentTABsPerSAP() > 0;
+ }
+
+
size_t ObservationSettings::BeamFormer::maxNrTABsPerSAP() const
{
size_t max = 0;
- for (size_t sapNr = 0; sapNr < SAPs.size(); ++sapNr)
- max = std::max(max, SAPs[sapNr].TABs.size());
+ for (auto& pipeline : pipelines)
+ {
+ unsigned nrTABs = 0;
+
+ for (auto& sap : pipeline.SAPs)
+ {
+ nrTABs += sap.TABs.size();
+ }
+
+ max = std::max(max, (size_t) nrTABs);
+ }
return max;
}
@@ -1293,6 +1676,38 @@ namespace LOFAR
return max;
}
+ size_t ObservationSettings::BeamFormer::Pipeline::maxNrTABsPerSAP() const
+ {
+ size_t max = 0;
+
+ for (size_t sapNr = 0; sapNr < SAPs.size(); ++sapNr)
+ max = std::max(max, SAPs[sapNr].TABs.size());
+
+ return max;
+ }
+
+
+ size_t ObservationSettings::BeamFormer::Pipeline::maxNrCoherentTABsPerSAP() const
+ {
+ size_t max = 0;
+
+ for (size_t sapNr = 0; sapNr < SAPs.size(); ++sapNr)
+ max = std::max(max, SAPs[sapNr].nrCoherent);
+
+ return max;
+ }
+
+
+ size_t ObservationSettings::BeamFormer::Pipeline::maxNrIncoherentTABsPerSAP() const
+ {
+ size_t max = 0;
+
+ for (size_t sapNr = 0; sapNr < SAPs.size(); ++sapNr)
+ max = std::max(max, SAPs[sapNr].nrIncoherent);
+
+ return max;
+ }
+
void Parset::updateSettings()
{
diff --git a/RTCP/Cobalt/CoInterface/src/Parset.h b/RTCP/Cobalt/CoInterface/src/Parset.h
index 507b9c51d5bbcb9a49eb0bd881651ea388c3de15..b780569a615f67dc03aff1c6e62144bdda04debc 100644
--- a/RTCP/Cobalt/CoInterface/src/Parset.h
+++ b/RTCP/Cobalt/CoInterface/src/Parset.h
@@ -354,6 +354,9 @@ namespace LOFAR
double centralFrequency;
};
+ // Return the list of indices of all subbands
+ vector<unsigned> subbandIndices() const;
+
// The list of subbands
//
// length: len(Observation.subbandList)
@@ -376,6 +379,28 @@ namespace LOFAR
double angle2;
};
+ struct TAB {
+ // The (absolute) direction where the TAB points to.
+ //
+ // key: Observation.Beam[sap].TiedArrayBeam[tab].*
+ struct Direction direction;
+
+ // Whether the beam is coherent (or incoherent)
+ //
+ // key: Observation.Beam[sap].TiedArrayBeam[tab].coherent
+ bool coherent;
+
+ // The DM with which to dedisperse this beam, or
+ // 0.0 for no dedispersion.
+ //
+ // key: Observation.Beam[sap].TiedArrayBeam[tab].dispersionMeasure
+ double dispersionMeasure;
+
+ // The list of files to write, one file
+ // per part/stokes.
+ //std::vector<struct File> files;
+ };
+
struct SAP {
// Direction in which the SAP points
//
@@ -395,6 +420,14 @@ namespace LOFAR
// Return the list of indices of our subbands
// within the global settings.subbands list.
vector<unsigned> subbandIndices() const;
+
+ // The TABs to form in this SAP
+ //
+ // size: Observation.Beam[sap].nrTiedArrayBeams
+ std::vector<struct TAB> TABs;
+
+ // Whether the correlator should be applied to this SAP
+ bool correlatorEnabled;
};
// All station beams
@@ -615,9 +648,14 @@ namespace LOFAR
// key: Observation.Beam[sap].TiedArrayBeam[tab].dispersionMeasure
double dispersionMeasure;
- // The list of files to write, one file
- // per part/stokes.
- std::vector<struct File> files;
+ // this TAB is the ....th coherent TAB in this SAP
+ size_t coherentIdxInSAP;
+
+ // this TAB is the ....th TAB for the beamFormer pipeline processing this TAB
+ size_t localTabNr;
+
+ // The index of the beamFormer pipeline processing this TAB
+ unsigned pipelineNr;
};
struct SAP {
@@ -630,8 +668,13 @@ namespace LOFAR
size_t nrCoherent;
size_t nrIncoherent;
- // list of subbands in this sap
- vector<unsigned> subbandIndices;
+ // Index in the global list of SAPs
+ unsigned globalSapIdx;
+
+ // The list of subbands in this SAP
+ //
+ // key: Observation.Beam[idx].subbandList
+ std::vector<unsigned> subbandIndices;
};
// Antenna fields to use for beam forming.
@@ -651,6 +694,7 @@ namespace LOFAR
bool anyCoherentTABs() const;
bool anyIncoherentTABs() const;
+ // Return the maximum number of TABs for any SAP
size_t maxNrTABsPerSAP() const;
size_t maxNrCoherentTABsPerSAP() const;
size_t maxNrIncoherentTABsPerSAP() const;
@@ -689,8 +733,65 @@ namespace LOFAR
//
// key: *.subbandsPerFile
size_t nrSubbandsPerFile;
+
+ // Quantizer settings
+ struct QuantizerSettings {
+ // Enable quantization
+ //
+ // key: Cobalt.BeamFormer.{Coherent|Incoherent}Stokes.quantize
+ bool enabled;
+
+ // How many bits to use (8 right now)
+ //
+ // key: Cobalt.BeamFormer.{Coherent|Incoherent}Stokes.quantizeBits
+ unsigned nrBits;
+
+ // Pre-scaling of data before quntization
+ //
+ // key: Cobalt.BeamFormer.{Coherent|Incoherent}Stokes.quantizeScaleMax
+ // key: Cobalt.BeamFormer.{Coherent|Incoherent}Stokes.quantizeScaleMin
+ double scaleMax;
+ double scaleMin;
+ // Keeping range of Stokes I positive if true
+ // key: Cobalt.BeamFormer.{Coherent|Incoherent}Stokes.quantizeIpositive
+ bool sIpositive;
+ };
+
+ struct QuantizerSettings quantizerSettings;
+ };
+
+ struct Pipeline
+ {
+ bool doFlysEye;
+
+ struct StokesSettings coherentSettings;
+ struct StokesSettings incoherentSettings;
+
+ // All antenna fields that this pipeline processes
+ //
+ // key: Cobalt.BeamFormer.Pipeline[idx].stationList
+ std::vector<AntennaFieldName> antennaFieldNames;
+
+ // All SAPs, with information about the TABs to form.
+ //
+ // size: len(Observation.nrBeams)
+ std::vector<struct SAP> SAPs;
+
+ // Return the maximum number of TABs for any SAP
+ size_t maxNrTABsPerSAP() const;
+ size_t maxNrCoherentTABsPerSAP() const;
+ size_t maxNrIncoherentTABsPerSAP() const;
+
+ bool anyCoherentTABs() const;
+ bool anyIncoherentTABs() const;
};
+ // The beamFormer pipelines
+ //
+ // key: Cobalt.BeamFormer.nrPipelines
+ // Default: 1
+ std::vector<Pipeline> pipelines;
+
// Settings for Coherent Stokes output
//
// key: Cobalt.BeamFormer.CoherentStokes.*
@@ -708,6 +809,7 @@ namespace LOFAR
size_t dedispersionFFTsize;
};
+ // In case there is only one beamFormer, it is defined here
struct BeamFormer beamFormer;
/* ===============================
@@ -779,6 +881,53 @@ namespace LOFAR
// List of host names to start outputProc on
std::vector<std::string> outputProcHosts;
+
+ /* ===============================
+ * Cobalt information
+ * ===============================
+ */
+ struct Cobalt
+ {
+ struct Kernel
+ {
+ // Whether dumping of kernel output was request.
+ //
+ // key: Cobalt.kernels.KernelName.dumpOutput
+ // Default: false
+ bool dumpBandPassCorrectionKernel;
+ bool dumpBeamFormerKernel;
+ bool dumpBeamFormerTransposeKernel;
+ bool dumpCoherentStokesKernel;
+ bool dumpCoherentStokesTransposeKernel;
+ bool dumpCorrelatorKernel;
+ bool dumpDelayAndBandPassKernel;
+ bool dumpFFTShiftKernel;
+ bool dumpFFT_Kernel;
+ bool dumpFIR_FilterKernel;
+ bool dumpIncoherentStokesKernel;
+ bool dumpIncoherentStokesTransposeKernel;
+ bool dumpIntToFloatKernel;
+ bool dumpQuantizeOutputKernel;
+ bool dumpZeroingKernel;
+ };
+
+ struct Kernel kernel;
+
+ struct Benchmark
+ {
+ // Whether logging of benchmark data was enabled (handled by gpu_load test)
+ // key: Cobalt.Benchmark.file; Cobalt.Benchmark.enabled;
+ // Default: empty; false
+ bool enabled;
+ std::string file;
+ };
+
+ struct Benchmark benchmark;
+
+ };
+
+ struct Cobalt cobalt;
+
}; // struct ObservationSettings
// Reads a ParameterSet from a Stream
@@ -800,6 +949,66 @@ namespace LOFAR
// Transform the parset into an ObservationSettings object
struct ObservationSettings observationSettings() const;
+ // Helper methods for the observationSettings() method
+ void addGenericInformation(
+ struct ObservationSettings& settings) const;
+
+ void addAnaBeamInformation(
+ struct ObservationSettings& settings) const;
+
+ void addSAPs(
+ struct ObservationSettings& settings) const;
+
+ void addStationInformation(
+ vector<string>& stations,
+ struct ObservationSettings& settings) const;
+
+ void addResourceInformation(
+ struct ObservationSettings& settings) const;
+
+ void addCorrelatorInformation(
+ set<string>& outputProcHosts,
+ struct ObservationSettings& settings) const;
+
+ void readTABs(
+ const ObservationSettings& settings,
+ const string& prefix,
+ std::vector<ObservationSettings::SAP>& SAPs,
+ bool doFlysEye) const;
+
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings defaultQuantizerSettings() const;
+
+ void readQuantizerSettings(
+ std::string prefix,
+ ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings& qtSettings,
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings& qtDefaultSettings) const;
+
+ const ObservationSettings::BeamFormer::StokesSettings defaultStokesSettings() const;
+
+ void readStokesSettings(
+ std::string bfPrefix,
+ bool coherent,
+ ObservationSettings::BeamFormer::StokesSettings& stSettings,
+ const ObservationSettings::BeamFormer::StokesSettings& stDefaultSettings,
+ const struct ObservationSettings& settings) const;
+
+ void addBeamFormer(
+ set<string>& outputProcHosts,
+ const vector<string>& stations,
+ struct ObservationSettings& settings) const;
+
+ void addRSPRawInformation(
+ set<string>& outputProcHosts,
+ const vector<string>& stations,
+ struct ObservationSettings& settings) const;
+
+ void addCobaltInformation(
+ struct ObservationSettings& settings) const;
+
+ void setOutputHosts(
+ set<string>& outputProcHosts,
+ struct ObservationSettings& settings) const;
+
// Fill the settings based on the ParameterSet keys.
// Call this if keys are added or changed.
void updateSettings();
@@ -856,7 +1065,9 @@ namespace LOFAR
const std::string& antennaSet) const;
std::vector<struct ObservationSettings::FileLocation>
- getFileLocations(const std::string outputType) const;
+ getFileLocations(
+ const std::string outputType,
+ const std::string prefix = "Observation") const;
// Returns whether nodeName has to participate in the observation
// given antenna fields, antenna mode, and configured antenna field streams.
diff --git a/RTCP/Cobalt/CoInterface/src/Pool.h b/RTCP/Cobalt/CoInterface/src/Pool.h
index af2ef91bbb2bf521043186909737139d00122493..a1eb40b662f3c4ec100a7919311b8813cd936136 100644
--- a/RTCP/Cobalt/CoInterface/src/Pool.h
+++ b/RTCP/Cobalt/CoInterface/src/Pool.h
@@ -22,7 +22,6 @@
#define LOFAR_COINTERFACE_POOL_H
#include <CoInterface/Queue.h>
-#include <CoInterface/SmartPtr.h>
namespace LOFAR
{
@@ -30,7 +29,7 @@ namespace LOFAR
{
// The pool operates using a free and a filled queue to cycle through buffers. Producers
// move elements free->filled, and consumers move elements filled->free. By
- // wrapping the elements in a SmartPtr, memory leaks are prevented.
+ // wrapping the elements in an shared_ptr, memory leaks are prevented.
//
// If warn_on_bad_performance == True, the queues will log warnings if they are not filled/freed
// fast enough (see Queue.h for more details).
@@ -42,8 +41,8 @@ namespace LOFAR
{
typedef T element_type;
- Queue< SmartPtr<element_type> > free;
- Queue< SmartPtr<element_type> > filled;
+ Queue<std::shared_ptr<element_type>> free;
+ Queue<std::shared_ptr<element_type>> filled;
Pool(const std::string &name, bool warn_on_bad_performance, int batch_size = 1)
:
diff --git a/RTCP/Cobalt/CoInterface/src/RunningStatistics.cc b/RTCP/Cobalt/CoInterface/src/RunningStatistics.cc
index 36f3cf2b0aa416990a6e81a55ad6a51c1e14767e..462f50cbc2d8c377b07d4c400c2f9b9602f4a2a4 100644
--- a/RTCP/Cobalt/CoInterface/src/RunningStatistics.cc
+++ b/RTCP/Cobalt/CoInterface/src/RunningStatistics.cc
@@ -23,6 +23,7 @@
#include <limits>
#include <ostream>
#include <iomanip>
+#include <fstream>
namespace LOFAR
{
namespace Cobalt
@@ -154,5 +155,33 @@ namespace LOFAR
rs.print(os);
return os;
}
+
+ void RunningStatistics::printCSV(std::ofstream& ofs) const
+ {
+ if (count() == 0)
+ ofs << "0,x,x,x,x," << unit();
+ else
+ {
+ // generate floating-point output in fixed-point format (fix digit position)
+ ofs.setf(std::ios::fixed);
+ // without above flags setting the precision takes the total amount of numbers
+ // before and after the decimal point and ignores leading zeros
+ ofs.precision(5); // set precision to 5 decimals
+
+ ofs << "; " << count()
+ << "; " << mean()
+ << "; " << stDev()
+ << "; " << min()
+ << "; " << max()
+ << "; " << unit();
+ }
+ }
+
+ std::ofstream& operator<<(std::ofstream& ofs, RunningStatistics const &rs)
+ {
+ rs.printCSV(ofs);
+ return ofs;
+ }
+
}
}
diff --git a/RTCP/Cobalt/CoInterface/src/RunningStatistics.h b/RTCP/Cobalt/CoInterface/src/RunningStatistics.h
index d46f9f63e54b7022e3c60e6cb008c24501b988df..8653c35bdd10353e935601386d61aff77718dc7d 100644
--- a/RTCP/Cobalt/CoInterface/src/RunningStatistics.h
+++ b/RTCP/Cobalt/CoInterface/src/RunningStatistics.h
@@ -76,6 +76,9 @@ namespace LOFAR
void print(std::ostream& os) const;
+ // for logging data in CSV format, return ofstream with CSV
+ void printCSV(std::ofstream& ofs) const;
+
private:
std::string _unit;
@@ -89,6 +92,7 @@ namespace LOFAR
RunningStatistics operator+(const RunningStatistics a, const RunningStatistics b);
std::ostream& operator<<(std::ostream& os, RunningStatistics const & rs);
+ std::ofstream& operator<<(std::ofstream& ofs, RunningStatistics const &rs);
}
}
#endif
diff --git a/RTCP/Cobalt/CoInterface/src/SmartPtr.h b/RTCP/Cobalt/CoInterface/src/SmartPtr.h
deleted file mode 100644
index d88f8f93e1ef4244c412cee985dc45851971d126..0000000000000000000000000000000000000000
--- a/RTCP/Cobalt/CoInterface/src/SmartPtr.h
+++ /dev/null
@@ -1,203 +0,0 @@
-//# SmartPtr.h: unsafe pre-C++11 version of the C++11 unique_ptr
-//# Copyright (C) 2011-2013, 2017
-//# ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef LOFAR_INTERFACE_SMART_PTR_H
-#define LOFAR_INTERFACE_SMART_PTR_H
-
-//# Never #include <config.h> or #include <lofar_config.h> in a header file!
-
-#include <cstdlib>
-#include <utility> // std::swap()
-
-namespace LOFAR
-{
- namespace Cobalt
- {
-
- template <typename T>
- class SmartPtrDelete;
-
- // T is the type of pointer (such as SmartPtr<int> to emulate int*)
- // D is the deletion strategy (to choose between delete/delete[]/free)
- template <typename T, class D = SmartPtrDelete<T> >
- class SmartPtr
- {
- public:
- SmartPtr(T * = NULL);
- SmartPtr(const SmartPtr<T,D> &orig); // WARNING: move semantics; orig no longer contains pointer
-
- ~SmartPtr();
-
- operator T * () const;
- T & operator * () const;
- T * operator -> () const;
-
- bool operator ! () const;
-
- SmartPtr<T,D> & operator = (T *);
- SmartPtr<T,D> & operator = (const SmartPtr<T,D> &);
-
- T *get();
- T *release();
-
- void swap(SmartPtr<T,D>& other) /*noexcept*/;
-
- private:
- T *ptr;
- };
-
- // Deletion strategies
- template <typename T>
- class SmartPtrDelete
- {
- public:
- static void free( T *ptr )
- {
- delete ptr;
- }
- };
-
- template <typename T>
- class SmartPtrDeleteArray
- {
- public:
- static void free( T *ptr )
- {
- delete[] ptr;
- }
- };
-
- template <typename T>
- class SmartPtrFree
- {
- public:
- static void free( T *ptr )
- {
- ::free(ptr);
- }
- };
-
- template <typename T, void (*F) (T*) >
- class SmartPtrFreeFunc
- {
- public:
- static void free( T *ptr )
- {
- F(ptr);
- }
- };
-
- template <typename T, class D>
- inline SmartPtr<T,D>::SmartPtr(T *orig)
- :
- ptr(orig)
- {
- }
-
- template <typename T, class D>
- inline SmartPtr<T,D>::SmartPtr(const SmartPtr<T,D> &orig)
- :
- ptr(orig.ptr)
- {
- const_cast<T *&>(orig.ptr) = 0;
- }
-
-
- template <typename T, class D>
- inline SmartPtr<T,D>::~SmartPtr()
- {
- D::free(ptr);
- }
-
-
- template <typename T, class D>
- inline SmartPtr<T,D>::operator T * () const
- {
- return ptr;
- }
-
-
- template <typename T, class D>
- inline T &SmartPtr<T,D>::operator * () const
- {
- return *ptr;
- }
-
-
- template <typename T, class D>
- inline T *SmartPtr<T,D>::operator -> () const
- {
- return ptr;
- }
-
-
- template <typename T, class D>
- inline bool SmartPtr<T,D>::operator ! () const
- {
- return ptr == 0;
- }
-
-
- template <typename T, class D>
- inline SmartPtr<T,D> &SmartPtr<T,D>::operator = (T *orig)
- {
- D::free(ptr);
- ptr = orig;
- return *this;
- }
-
-
- template <typename T, class D>
- inline SmartPtr<T,D> &SmartPtr<T,D>::operator = (const SmartPtr<T,D> &orig)
- {
- D::free(ptr);
- ptr = orig;
- const_cast<T *&>(orig.ptr) = 0;
- return *this;
- }
-
-
- template <typename T, class D>
- inline T *SmartPtr<T,D>::get()
- {
- return ptr;
- }
-
-
- template <typename T, class D>
- inline T *SmartPtr<T,D>::release()
- {
- T *tmp = ptr;
- ptr = 0;
- return tmp;
- }
-
-
- template <typename T, class D>
- inline void SmartPtr<T,D>::swap(SmartPtr<T,D>& other) /*noexcept*/
- {
- std::swap(ptr, other.ptr);
- }
- } // namespace Cobalt
-} // namespace LOFAR
-
-#endif
-
diff --git a/RTCP/Cobalt/CoInterface/src/StreamableData.h b/RTCP/Cobalt/CoInterface/src/StreamableData.h
index 46fb070c1bdbf06a48fd179fb38b4e05e381a36c..8320b2c15b299b1d08b203c13b2f1860898f8871 100644
--- a/RTCP/Cobalt/CoInterface/src/StreamableData.h
+++ b/RTCP/Cobalt/CoInterface/src/StreamableData.h
@@ -123,18 +123,23 @@ namespace LOFAR
// A typical data set contains a MultiDimArray of tuples and a set of flags.
- template <typename T = fcomplex, unsigned DIM = 4, unsigned FLAGS_DIM = 2>
+ // QD,QF are types for quantized data/scale+offset
+ template <typename T = fcomplex, unsigned DIM = 4, unsigned FLAGS_DIM = 2, typename QD = int8_t, typename QF = float>
class SampleData : public StreamableData
{
public:
typedef typename MultiDimArray<T,DIM>::ExtentList ExtentList;
typedef typename MultiDimArray<SparseSet<unsigned>,FLAGS_DIM>::ExtentList FlagsExtentList;
- SampleData(const ExtentList &extents, const FlagsExtentList &flagsExtents, Allocator & = heapAllocator);
+ SampleData(const ExtentList &extents, const FlagsExtentList &flagsExtents, bool quantized=false, Allocator & = heapAllocator);
+ SampleData(size_t nrSubbands, size_t nrChannels, size_t nrSamples, bool quantized=false, Allocator & = heapAllocator);
MultiDimArray<T,DIM> samples;
MultiDimArray<SparseSet<unsigned>,FLAGS_DIM> flags;
-
+ const bool quantized;
+ MultiDimArray<QD,DIM> qsamples;
+ MultiDimArray<QF, DIM-1> qoffsets;
+ MultiDimArray<QF, DIM-1> qscales;
protected:
virtual void readData(Stream *, unsigned);
virtual void writeData(Stream *, unsigned);
@@ -181,33 +186,59 @@ namespace LOFAR
}
- template <typename T, unsigned DIM, unsigned FLAGS_DIM>
- inline SampleData<T,DIM,FLAGS_DIM>::SampleData(const ExtentList &extents, const FlagsExtentList &flagsExtents, Allocator &allocator)
+ template <typename T, unsigned DIM, unsigned FLAGS_DIM, typename QD, typename QF>
+ inline SampleData<T,DIM,FLAGS_DIM,QD,QF>::SampleData(const ExtentList &extents, const FlagsExtentList &flagsExtents, bool quantized, Allocator &allocator)
:
// This clarifies seq nr handling for beamforming, however, beamformed output is *not*
// written via StreamableData->write(), but in a custom way in MSWriterDAL::write().
StreamableData(true, false),
- samples(extents, alignment, allocator),
- flags(flagsExtents) // e.g., for FilteredData [nrChannels][nrStations], sparse dimension [nrSamplesPerIntegration]
+ samples((!quantized ? extents : boost::extents[0][0][0]), alignment, allocator),
+ flags(flagsExtents), // e.g., for FilteredData [nrChannels][nrStations], sparse dimension [nrSamplesPerIntegration]
+ quantized(quantized)
{
}
- template <typename T, unsigned DIM, unsigned FLAGS_DIM>
- inline void SampleData<T,DIM,FLAGS_DIM>::readData(Stream *str, unsigned alignment)
+ template <typename T, unsigned DIM, unsigned FLAGS_DIM, typename QD, typename QF>
+ inline SampleData<T,DIM,FLAGS_DIM,QD,QF>::SampleData(size_t nrSubbands, size_t nrChannels, size_t nrSamples, bool quantized, Allocator & allocator)
+ :
+ StreamableData(true, false),
+ samples((!quantized ? boost::extents[nrSamples][nrSubbands][nrChannels] : boost::extents[0][0][0]), alignment, allocator),
+ flags(boost::extents[nrSubbands]),
+ quantized(quantized),
+ qsamples((quantized ? boost::extents[nrSamples][nrSubbands][nrChannels] : boost::extents[0][0][0]), alignment, allocator),
+ qoffsets((quantized ? boost::extents[nrSubbands][nrChannels] : boost::extents[0][0]), alignment, allocator),
+ qscales((quantized ? boost::extents[nrSubbands][nrChannels] : boost::extents[0][0]), alignment, allocator)
+ {
+ }
+
+ template <typename T, unsigned DIM, unsigned FLAGS_DIM, typename QD, typename QF>
+ inline void SampleData<T,DIM,FLAGS_DIM,QD,QF>::readData(Stream *str, unsigned alignment)
{
(void)alignment;
- str->read(samples.origin(), samples.num_elements() * sizeof(T));
+ if (!quantized) {
+ str->read(samples.origin(), samples.num_elements() * sizeof(T));
+ } else {
+ str->read(qsamples.origin(), qsamples.num_elements() * sizeof(QD));
+ str->read(qoffsets.origin(), qoffsets.num_elements() * sizeof(QF));
+ str->read(qscales.origin(), qscales.num_elements() * sizeof(QF));
+ }
}
- template <typename T, unsigned DIM, unsigned FLAGS_DIM>
- inline void SampleData<T,DIM,FLAGS_DIM>::writeData(Stream *str, unsigned alignment)
+ template <typename T, unsigned DIM, unsigned FLAGS_DIM, typename QD, typename QF>
+ inline void SampleData<T,DIM,FLAGS_DIM,QD,QF>::writeData(Stream *str, unsigned alignment)
{
(void)alignment;
- str->write(samples.origin(), samples.num_elements() * sizeof(T));
+ if (!quantized) {
+ str->write(samples.origin(), samples.num_elements() * sizeof(T));
+ } else {
+ str->write(qsamples.origin(), qsamples.num_elements() * sizeof(QD));
+ str->write(qoffsets.origin(), qoffsets.num_elements() * sizeof(QF));
+ str->write(qscales.origin(), qscales.num_elements() * sizeof(QF));
+ }
}
diff --git a/RTCP/Cobalt/CoInterface/src/TABTranspose.cc b/RTCP/Cobalt/CoInterface/src/TABTranspose.cc
index 533bf2f7a2218c51f12cbf92f8e96d88c090af39..01cb2bb317371105fd90280fcbb2196aa6d14f21 100644
--- a/RTCP/Cobalt/CoInterface/src/TABTranspose.cc
+++ b/RTCP/Cobalt/CoInterface/src/TABTranspose.cc
@@ -42,9 +42,13 @@ namespace TABTranspose {
// A Subband here is the part of a Block (see below) that contains the data
// values in a subband during a block duration (e.g. 1 second).
-Subband::Subband( size_t nrSamples, size_t nrChannels )
+Subband::Subband( size_t nrSamples, size_t nrChannels, bool _quantized )
:
- data(boost::extents[nrSamples][nrChannels])
+ data(!_quantized ? boost::extents[nrSamples][nrChannels]: boost::extents[0][0]),
+ qdata(_quantized ? boost::extents[nrSamples][nrChannels]: boost::extents[0][0]),
+ qoffsets(_quantized ? boost::extents[nrChannels]: boost::extents[0]),
+ qscales(_quantized ? boost::extents[nrChannels]: boost::extents[0]),
+ quantized(_quantized)
{
id.fileIdx = 0;
id.subband = 0;
@@ -57,12 +61,23 @@ Subband::Subband( size_t nrSamples, size_t nrChannels )
void Subband::write(Stream &stream) const {
stream.write(&id, sizeof id);
- size_t dim1 = data.shape()[0];
- size_t dim2 = data.shape()[1];
- stream.write(&dim1, sizeof dim1);
- stream.write(&dim2, sizeof dim2);
- stream.write(data.origin(), data.num_elements() * sizeof *data.origin());
-
+ // write flag to signal data is quantized
+ stream.write(&quantized, sizeof quantized);
+ if (!quantized) {
+ size_t dim1 = data.shape()[0];
+ size_t dim2 = data.shape()[1];
+ stream.write(&dim1, sizeof dim1);
+ stream.write(&dim2, sizeof dim2);
+ stream.write(data.origin(), data.num_elements() * sizeof *data.origin());
+ } else {
+ size_t dim1 = qdata.shape()[0];
+ size_t dim2 = qdata.shape()[1];
+ stream.write(&dim1, sizeof dim1);
+ stream.write(&dim2, sizeof dim2);
+ stream.write(qdata.origin(), qdata.num_elements() * sizeof *qdata.origin());
+ stream.write(qoffsets.origin(), qoffsets.num_elements() * sizeof *qoffsets.origin());
+ stream.write(qscales.origin(), qscales.num_elements() * sizeof *qscales.origin());
+ }
LOG_DEBUG_STR("Written block " << id);
}
@@ -74,11 +89,24 @@ void Subband::read(Stream &stream) {
stream.read(&id, sizeof id);
LOG_DEBUG_STR("Read block id " << id);
+ // read flag to signal data is quantized
+ stream.read(&quantized, sizeof quantized);
+
size_t dim1, dim2;
stream.read(&dim1, sizeof dim1);
stream.read(&dim2, sizeof dim2);
- data.resize(boost::extents[dim1][dim2]);
- stream.read(data.origin(), data.num_elements() * sizeof *data.origin());
+
+ if (!quantized) {
+ data.resize(boost::extents[dim1][dim2]);
+ stream.read(data.origin(), data.num_elements() * sizeof *data.origin());
+ } else {
+ qdata.resize(boost::extents[dim1][dim2]);
+ qoffsets.resize(boost::extents[dim2]);
+ qscales.resize(boost::extents[dim2]);
+ stream.read(qdata.origin(), qdata.num_elements() * sizeof *qdata.origin());
+ stream.read(qoffsets.origin(), qoffsets.num_elements() * sizeof *qoffsets.origin());
+ stream.read(qscales.origin(), qscales.num_elements() * sizeof *qscales.origin());
+ }
LOG_DEBUG_STR("Read block " << id);
}
@@ -106,7 +134,7 @@ Block::Block( size_t fileIdx, size_t blockIdx, size_t nrSubbands, size_t nrSampl
// Used by the BlockCollector to add a received Subband to a Block.
// Runs in outputProc.
-void Block::addSubband( SmartPtr<Subband> &subband ) {
+void Block::addSubband(std::shared_ptr<Subband> &subband) {
ASSERT(nrSubbandsLeft > 0);
const Subband::BlockID id = subband->id;
@@ -117,13 +145,21 @@ void Block::addSubband( SmartPtr<Subband> &subband ) {
ASSERTSTR(id.block == blockIdx, "Got block " << id.block << ", expected " << blockIdx);
ASSERT(id.subband < nrSubbands);
- ASSERT(subband->data.shape()[0] == nrSamples);
- ASSERT(subband->data.shape()[1] == nrChannels);
+ if (!subband->quantized) {
+ ASSERT(subband->data.shape()[0] == nrSamples);
+ ASSERT(subband->data.shape()[1] == nrChannels);
+ } else {
+ ASSERT(subband->qdata.shape()[0] == nrSamples);
+ ASSERT(subband->qdata.shape()[1] == nrChannels);
+ ASSERT(subband->qoffsets.shape()[0] == nrChannels);
+ ASSERT(subband->qscales.shape()[0] == nrChannels);
+ }
// Subbands should not arrive twice
ASSERT(subbandCache[id.subband] == NULL);
subbandCache[id.subband] = subband;
+ subband.reset();
ASSERT(subband == NULL);
nrSubbandsLeft--;
@@ -134,10 +170,19 @@ void Block::addSubband( SmartPtr<Subband> &subband ) {
void Block::write( BeamformedData &output ) {
// Check dimensions
- ASSERT( output.samples.shape()[0] == nrSamples );
- ASSERT( output.samples.shape()[1] == nrSubbands );
- ASSERT( output.samples.shape()[2] == nrChannels );
-
+ if (!output.quantized) {
+ ASSERT( output.samples.shape()[0] == nrSamples );
+ ASSERT( output.samples.shape()[1] == nrSubbands );
+ ASSERT( output.samples.shape()[2] == nrChannels );
+ } else {
+ ASSERT( output.qsamples.shape()[0] == nrSamples );
+ ASSERT( output.qsamples.shape()[1] == nrSubbands );
+ ASSERT( output.qsamples.shape()[2] == nrChannels );
+ ASSERT( output.qoffsets.shape()[0] == nrSubbands );
+ ASSERT( output.qoffsets.shape()[1] == nrChannels );
+ ASSERT( output.qscales.shape()[0] == nrSubbands );
+ ASSERT( output.qscales.shape()[1] == nrChannels );
+ }
// Set annotation
output.setSequenceNumber(blockIdx);
@@ -166,53 +211,127 @@ void Block::write( BeamformedData &output ) {
// is more efficient.
const size_t MAXBATCHSIZE = 488;
- // Stride between samples in output
- const ptrdiff_t dst_sample_stride = output.samples.strides()[0];
- // Stride between subbands in output
- const ptrdiff_t dst_sb_stride = output.samples.strides()[1];
+ if (!output.quantized) {
+ // Stride between samples in output
+ const ptrdiff_t dst_sample_stride = output.samples.strides()[0];
+ // Stride between subbands in output
+ const ptrdiff_t dst_sb_stride = output.samples.strides()[1];
- const vector<float> zeroes(nrChannels * nrSamples, 0.0f);
+ const vector<float> zeroes(nrChannels * nrSamples, 0.0f);
- for (size_t subbandBase = 0; subbandBase < subbandCache.size(); subbandBase += MAXBATCHSIZE) {
- // Determine actual batch size
- const size_t BATCHSIZE = std::min(MAXBATCHSIZE, subbandCache.size() - subbandBase);
+ for (size_t subbandBase = 0; subbandBase < subbandCache.size(); subbandBase += MAXBATCHSIZE) {
+ // Determine actual batch size
+ const size_t BATCHSIZE = std::min(MAXBATCHSIZE, subbandCache.size() - subbandBase);
- // Collect source pointers for all our subbands, or to our zeroes array if
- // a subband is not available. Each element points to an array of dimensions
- // [nrSamples][nrChannels]
- const float *src[MAXBATCHSIZE];
+ // Collect source pointers for all our subbands, or to our zeroes array if
+ // a subband is not available. Each element points to an array of dimensions
+ // [nrSamples][nrChannels]
+ const float *src[MAXBATCHSIZE];
- for (size_t i = 0; i < BATCHSIZE; ++i) {
- size_t subbandIdx = subbandBase + i;
+ for (size_t i = 0; i < BATCHSIZE; ++i) {
+ size_t subbandIdx = subbandBase + i;
- src[i] = subbandCache[subbandIdx] ? subbandCache[subbandIdx]->data.origin() : &zeroes[0];
- }
+ src[i] = subbandCache[subbandIdx] ? subbandCache[subbandIdx]->data.origin() : &zeroes[0];
+ }
+
+ // Pointer to walk over all samples
+ float *dst = &output.samples[0][subbandBase][0];
- // Pointer to walk over all samples
- float *dst = &output.samples[0][subbandBase][0];
+ if (nrChannels == 1) {
+ /* Use assignment to copy data */
+ for (size_t t = 0; t < nrSamples; ++t) {
+ float *sample = dst;
+ dst += dst_sample_stride;
- if (nrChannels == 1) {
- /* Use assignment to copy data */
- for (size_t t = 0; t < nrSamples; ++t) {
- float *sample = dst;
- dst += dst_sample_stride;
+ for (size_t i = 0; i < BATCHSIZE; ++i) {
+ *sample = src[i][t];
+
+ sample += dst_sb_stride;
+ }
+ }
+ } else {
+ /* Use memcpy to copy data */
+ for (size_t t = 0; t < nrSamples; ++t) {
+ float *sample = dst;
+ dst += dst_sample_stride;
- for (size_t i = 0; i < BATCHSIZE; ++i) {
- *sample = src[i][t];
+ for (size_t i = 0; i < BATCHSIZE; ++i) {
+ memcpy(sample, &src[i][t * nrChannels], nrChannels * sizeof(float));
- sample += dst_sb_stride;
+ sample += dst_sb_stride;
+ }
}
}
- } else {
- /* Use memcpy to copy data */
- for (size_t t = 0; t < nrSamples; ++t) {
- float *sample = dst;
- dst += dst_sample_stride;
+ }
+ } else { // quantized data
+ // Stride between samples in output
+ const ptrdiff_t dst_sample_stride = output.qsamples.strides()[0];
+ // Stride between subbands in output
+ const ptrdiff_t dst_sb_stride = output.qsamples.strides()[1];
+
+ const vector<int8_t> zeroes(nrChannels * nrSamples, 0);
+
+ for (size_t subbandBase = 0; subbandBase < subbandCache.size(); subbandBase += MAXBATCHSIZE) {
+ // Determine actual batch size
+ const size_t BATCHSIZE = std::min(MAXBATCHSIZE, subbandCache.size() - subbandBase);
+
+ // Collect source pointers for all our subbands, or to our zeroes array if
+ // a subband is not available. Each element points to an array of dimensions
+ // [nrSamples][nrChannels]
+ const int8_t *src[MAXBATCHSIZE];
+
+ for (size_t i = 0; i < BATCHSIZE; ++i) {
+ size_t subbandIdx = subbandBase + i;
+
+ src[i] = subbandCache[subbandIdx] ? subbandCache[subbandIdx]->qdata.origin() : &zeroes[0];
+ }
- for (size_t i = 0; i < BATCHSIZE; ++i) {
- memcpy(sample, &src[i][t * nrChannels], nrChannels * sizeof(float));
+ // Pointer to walk over all samples
+ int8_t *dst = &output.qsamples[0][subbandBase][0];
- sample += dst_sb_stride;
+ if (nrChannels == 1) {
+ /* Use assignment to copy data */
+ for (size_t t = 0; t < nrSamples; ++t) {
+ int8_t *sample = dst;
+ dst += dst_sample_stride;
+
+ for (size_t i = 0; i < BATCHSIZE; ++i) {
+ *sample = src[i][t];
+
+ sample += dst_sb_stride;
+ }
+ }
+ } else {
+ /* Use memcpy to copy data */
+ for (size_t t = 0; t < nrSamples; ++t) {
+ int8_t *sample = dst;
+ dst += dst_sample_stride;
+
+ for (size_t i = 0; i < BATCHSIZE; ++i) {
+ memcpy(sample, &src[i][t * nrChannels], nrChannels * sizeof(int8_t));
+
+ sample += dst_sb_stride;
+ }
+ }
+ }
+ }
+ // write offsets, scales : nrSubbands x nrChannels
+ // input: 'nrSubbands' number of vectors, each 'nrChannels' long
+ // output 2D array [nrSubbands][nrChannels]
+ // We do this the naive way here,
+ // hoping compiler loop unrolling will improve performance
+ for (size_t sb = 0; sb < subbandCache.size(); sb++) {
+ if (subbandCache[sb]) {
+ float *srcqo = subbandCache[sb]->qoffsets.origin();
+ float *srcqs = subbandCache[sb]->qscales.origin();
+ float *dstqo = &output.qoffsets[sb][0];
+ float *dstqs = &output.qscales[sb][0];
+ // copy
+ size_t incy=0;
+ for (size_t incx = 0; incx < nrChannels; ++incx) {
+ dstqo[incy]=srcqo[incx];
+ dstqs[incy]=srcqs[incx];
+ incy+=1;
}
}
}
@@ -286,13 +405,13 @@ BlockCollector::~BlockCollector()
}
-void BlockCollector::addSubband( SmartPtr<Subband> &subband ) {
+void BlockCollector::addSubband(std::shared_ptr<Subband> &subband ) {
inputQueue.append(subband);
}
void BlockCollector::inputLoop() {
- SmartPtr<Subband> subband;
+ std::shared_ptr<Subband> subband;
BudgetTimer processTimer(
"BlockCollector::inputLoop: weaving subbands",
@@ -308,7 +427,7 @@ void BlockCollector::inputLoop() {
void BlockCollector::outputLoop() {
- SmartPtr<Block> block;
+ std::shared_ptr<Block> block;
BudgetTimer writeTimer(
"BlockCollector::outputLoop: data transpose/zeroing",
@@ -316,7 +435,7 @@ void BlockCollector::outputLoop() {
true, true);
while ((block = outputQueue.remove()) != NULL) {
- SmartPtr<BeamformedData> output = outputPool.free.remove();
+ std::shared_ptr<BeamformedData> output = outputPool.free.remove();
{
BudgetTimer::StartStop ss(writeTimer);
@@ -334,7 +453,7 @@ void BlockCollector::outputLoop() {
// If this completes a Block (all Subbands for this Block received) and no
// subsequent Blocks are missing something, send it (or them) off into the
// outputQueue for write-back to storage.
-void BlockCollector::processSubband( SmartPtr<Subband> &subband ) {
+void BlockCollector::processSubband(std::shared_ptr<Subband> &subband ) {
++nrBlockSubbandsReceived;
LOG_DEBUG_STR(logPrefix << "Add " << subband->id);
@@ -356,7 +475,7 @@ void BlockCollector::processSubband( SmartPtr<Subband> &subband ) {
}
}
- SmartPtr<Block> &block = blocks.at(blockIdx);
+ std::shared_ptr<Block> &block = blocks.at(blockIdx);
block->addSubband(subband);
@@ -445,7 +564,7 @@ void BlockCollector::emit(size_t blockIdx) {
}
// fetch data
- SmartPtr<Block> &block = blocks.at(blockIdx);
+ std::shared_ptr<Block> &block = blocks.at(blockIdx);
LOG_DEBUG_STR(logPrefix << "Emitting block " << blockIdx << " (last emit: " << lastEmitted << ")");
// update loss statistics
@@ -497,7 +616,7 @@ bool BlockCollector::fetch(size_t block) {
// Add and annotate
ASSERT(!have(block));
- blocks[block] = new Block(fileIdx, block, nrSubbands, nrSamples, nrChannels);
+ blocks[block] = std::make_shared<Block>(fileIdx, block, nrSubbands, nrSamples, nrChannels);
return true;
}
@@ -540,7 +659,7 @@ void Receiver::receiveLoop()
{
try {
for(;;) {
- SmartPtr<Subband> subband = new Subband;
+ std::shared_ptr<Subband> subband(new Subband);
subband->read(stream);
@@ -610,10 +729,10 @@ void MultiReceiver::listenLoop()
{
for(;;) {
// Accept a new client
- SmartPtr<PortBroker::ServerStream> stream;
+ std::unique_ptr<PortBroker::ServerStream> stream;
try {
- stream = new PortBroker::ServerStream(servicePrefix, true);
+ stream.reset(new PortBroker::ServerStream(servicePrefix, true));
} catch(TimeOutException &) {
// fail silently if no client connected
LOG_DEBUG_STR("TABTranspose::MultiReceiver: Timed out");
@@ -636,8 +755,8 @@ void MultiReceiver::dispatch( PortBroker::ServerStream *stream )
struct Client client;
- client.stream = stream;
- client.receiver = new Receiver(*stream, collectors);
+ client.stream = std::make_shared<PortBroker::ServerStream>(*stream);
+ client.receiver = std::make_shared<Receiver>(*stream, collectors);
clients.push_back(client);
@@ -668,7 +787,7 @@ MultiSender::MultiSender( const HostMap &hostMap, const Parset &parset,
}
for (vector<struct Host>::const_iterator i = hosts.begin(); i != hosts.end(); ++i) {
- queues[*i] = new Queue< SmartPtr<struct Subband> >(str(format("MultiSender::queue [to %s]") % i->hostName));
+ queues[*i].reset(new Queue<std::shared_ptr<struct Subband>>(str(format("MultiSender::queue [to %s]") % i->hostName)));
}
}
@@ -730,14 +849,14 @@ void MultiSender::process( OMPThreadSet *threadSet )
QueuePtr &queue = queues.at(host);
- SmartPtr<struct Subband> subband;
+ std::shared_ptr<struct Subband> subband;
NSTimer sendTimer(str(format("Send Subband to %s") % host.hostName), true, true);
while ((subband = queue->remove()) != NULL) {
NSTimer::StartStop ss(sendTimer);
// Note: this can take any amount of time, even hours, even in real-time mode
- subband->write(stream);
+ subband->write(stream); //FIXME
}
LOG_DEBUG_STR(logPrefix << "Done");
@@ -751,7 +870,7 @@ void MultiSender::process( OMPThreadSet *threadSet )
// The pipeline calls here to write a block for a single file (part).
-bool MultiSender::append( SmartPtr<struct Subband> &subband )
+bool MultiSender::append(std::shared_ptr<struct Subband> &subband )
{
using namespace TimeSpec;
@@ -768,7 +887,7 @@ bool MultiSender::append( SmartPtr<struct Subband> &subband )
drop_rates.at(host).push(100.0);
// remove oldest item
- SmartPtr<struct Subband> subband = queue->remove();
+ std::shared_ptr<struct Subband> subband = queue->remove();
// would be weird to have NULL in here while we're appending elements
ASSERT(subband);
diff --git a/RTCP/Cobalt/CoInterface/src/TABTranspose.h b/RTCP/Cobalt/CoInterface/src/TABTranspose.h
index 0204787a68a5068139859623463b7695ecabbae5..0cb163331f1ccf6bf7cd0665048f7eb82d18ad0a 100644
--- a/RTCP/Cobalt/CoInterface/src/TABTranspose.h
+++ b/RTCP/Cobalt/CoInterface/src/TABTranspose.h
@@ -32,7 +32,6 @@
#include "RunningStatistics.h"
#include "BestEffortQueue.h"
#include "MultiDimArray.h"
-#include "SmartPtr.h"
#include "Pool.h"
#include "StreamableData.h"
#include "OMPThread.h"
@@ -52,8 +51,11 @@ namespace LOFAR
struct Subband {
MultiDimArray<float, 2> data; // [samples][channels]
+ MultiDimArray<int8_t, 2> qdata; // [samples][channels]
+ MultiDimArray<float, 1> qoffsets; // [channels]
+ MultiDimArray<float, 1> qscales; // [channels]
- Subband( size_t nrSamples = 0, size_t nrChannels = 0 );
+ Subband( size_t nrSamples = 0, size_t nrChannels = 0, bool quantized=false );
struct BlockID {
size_t fileIdx;
@@ -63,18 +65,19 @@ namespace LOFAR
void write(Stream &stream) const;
void read(Stream &stream);
+
+ bool quantized;
};
std::ostream &operator<<(std::ostream &str, const Subband::BlockID &id);
class BeamformedData: public SampleData<float, 3, 1> {
public:
- BeamformedData(size_t nrSubbands, size_t nrChannels, size_t nrSamples, Allocator &allocator = heapAllocator):
+ BeamformedData(size_t nrSubbands, size_t nrChannels, size_t nrSamples, bool quantized=false, Allocator &allocator = heapAllocator):
// data: [nrSubbands][nrChannels][nrSamples]
// flags: [nrSubbands], and encodes *output data loss*, not flagged input
- SampleData<float, 3, 1>(boost::extents[nrSamples][nrSubbands][nrChannels],
- boost::extents[nrSubbands],
- allocator),
+ SampleData<float, 3, 1>(nrSubbands, nrChannels, nrSamples, quantized, allocator),
+ quantized(quantized),
nrSubbands(nrSubbands),
nrChannels(nrChannels),
nrSamples(nrSamples)
@@ -91,6 +94,7 @@ namespace LOFAR
return 1.0 * nrFlaggedSamples / (nrSubbands * nrSamples);
}
+ const bool quantized;
private:
const size_t nrSubbands, nrChannels, nrSamples;
};
@@ -118,7 +122,7 @@ namespace LOFAR
/*
* Add data for a single subband to the cache.
*/
- void addSubband( SmartPtr<Subband> &subband );
+ void addSubband(std::shared_ptr<Subband> &subband);
/*
* Flush the subband cache to a SampleData array,
@@ -143,7 +147,7 @@ namespace LOFAR
const size_t nrChannels;
// Cache of subband data for this block
- std::vector< SmartPtr<Subband> > subbandCache;
+ std::vector<std::shared_ptr<Subband>> subbandCache;
public:
// The number of subbands left to receive.
@@ -189,7 +193,7 @@ namespace LOFAR
/*
* Add a subband of any block.
*/
- void addSubband( SmartPtr<Subband> &subband );
+ void addSubband(std::shared_ptr<Subband> &subband );
/*
* Send all remaining blocks downstream,
@@ -214,10 +218,10 @@ namespace LOFAR
* outputThread: outputQueue -> outputPool.filled
*/
- std::map<size_t, SmartPtr<Block> > blocks;
+ std::map<size_t, std::shared_ptr<Block> > blocks;
- BestEffortQueue< SmartPtr<Subband> > inputQueue;
- BestEffortQueue< SmartPtr<Block> > outputQueue;
+ BestEffortQueue<std::shared_ptr<Subband> > inputQueue;
+ BestEffortQueue<std::shared_ptr<Block> > outputQueue;
Pool<BeamformedData> &outputPool;
const size_t fileIdx;
@@ -274,7 +278,7 @@ namespace LOFAR
* Processes input elements from inputQueue.
*/
void inputLoop();
- void processSubband( SmartPtr<Subband> &subband );
+ void processSubband(std::shared_ptr<Subband> &subband );
/*
* Processes output elements from outputQueue.
@@ -290,7 +294,7 @@ namespace LOFAR
class Receiver {
public:
// [fileIdx] -> BlockCollector
- typedef std::map<size_t, SmartPtr<BlockCollector> > CollectorMap;
+ typedef std::map<size_t, std::unique_ptr<BlockCollector> > CollectorMap;
/*
* Start receiving from `stream', into `collectors'.
@@ -341,8 +345,8 @@ namespace LOFAR
Condition newClient;
struct Client {
- SmartPtr<PortBroker::ServerStream> stream;
- SmartPtr<Receiver> receiver;
+ std::shared_ptr<PortBroker::ServerStream> stream;
+ std::shared_ptr<Receiver> receiver;
};
const std::string servicePrefix;
@@ -404,7 +408,7 @@ namespace LOFAR
//
// Returns `true' if the subband was appended without dropping data.
// Returns `false' if a subband was dropped to make space for this one.
- bool append( SmartPtr<struct Subband> &subband );
+ bool append(std::shared_ptr<struct Subband> &subband);
// Flush the queues.
void finish();
@@ -434,7 +438,7 @@ namespace LOFAR
std::vector<struct Host> hosts;
// A queue for data to be sent to each host
- typedef SmartPtr< Queue< SmartPtr<struct Subband> > > QueuePtr;
+ typedef std::unique_ptr<Queue<std::shared_ptr<struct Subband>>> QueuePtr;
std::map<struct Host, QueuePtr> queues;
};
diff --git a/RTCP/Cobalt/CoInterface/test/tParset.cc b/RTCP/Cobalt/CoInterface/test/tParset.cc
index f169bc96e1159b4cf2463b1d535e54f271b6b91b..4dd971a71fba85e44aa0b388265a827722afbff3 100644
--- a/RTCP/Cobalt/CoInterface/test/tParset.cc
+++ b/RTCP/Cobalt/CoInterface/test/tParset.cc
@@ -81,8 +81,9 @@ TEST(defaultTestParset) {
/*
* Test generic information.
*/
-
TEST(realTime) {
+ LOG_INFO("Test realTime");
+
TESTBOOL {
Parset ps = makeDefaultTestParset("Cobalt.realTime", valstr);
@@ -91,24 +92,33 @@ TEST(realTime) {
}
TEST(observationID) {
+ LOG_INFO("Test observationID");
+
Parset ps = makeDefaultTestParset("Observation.ObsID", "12345");
CHECK_EQUAL(12345U, ps.settings.observationID);
}
TEST(momID) {
+ LOG_INFO("Test momID");
+
Parset ps = makeDefaultTestParset("Observation.momID", "12345");
CHECK_EQUAL(12345U, ps.settings.momID);
}
TEST(startTime) {
+ LOG_INFO("Test startTime");
+
Parset ps = makeDefaultTestParset("Observation.startTime", "2013-03-17 10:55:08");
CHECK_CLOSE(1363517708.0, ps.settings.startTime, 0.1);
}
TEST(stopTime) {
+
+ LOG_INFO("Test stopTime");
+
Parset ps = makeDefaultTestParset("Observation.stopTime", "2013-03-17 10:55:08");
CHECK_CLOSE(1363517708.0, ps.settings.stopTime, 0.1);
@@ -116,6 +126,8 @@ TEST(stopTime) {
SUITE(clockMHz) {
TEST(200) {
+ LOG_INFO("Test clockMHz 200"); \
+
Parset ps = makeDefaultTestParset("Observation.sampleClock", "200");
CHECK_EQUAL(200U, ps.settings.clockMHz);
@@ -126,6 +138,8 @@ SUITE(clockMHz) {
}
TEST(160) {
+ LOG_INFO("Test clockMHz 160"); \
+
Parset ps = makeDefaultTestParset("Observation.sampleClock", "160");
CHECK_EQUAL(160U, ps.settings.clockMHz);
@@ -138,6 +152,8 @@ SUITE(clockMHz) {
SUITE(nrBitsPerSample) {
TEST(16) {
+ LOG_INFO("Test nrBitsPerSample 16");
+
Parset ps = makeDefaultTestParset("Observation.nrBitsPerSample", "16");
CHECK_EQUAL(16U, ps.settings.nrBitsPerSample);
@@ -146,6 +162,8 @@ SUITE(nrBitsPerSample) {
}
TEST(8) {
+ LOG_INFO("Test nrBitsPerSample 8");
+
Parset ps = makeDefaultTestParset("Observation.nrBitsPerSample", "8");
CHECK_EQUAL(8U, ps.settings.nrBitsPerSample);
@@ -154,6 +172,8 @@ SUITE(nrBitsPerSample) {
}
TEST(4) {
+ LOG_INFO("Test nrBitsPerSample 4");
+
Parset ps = makeDefaultTestParset("Observation.nrBitsPerSample", "4");
CHECK_EQUAL(4U, ps.settings.nrBitsPerSample);
@@ -173,6 +193,8 @@ TEST(nrPolarisations) {
SUITE(corrections) {
TEST(bandPass) {
+ LOG_INFO("Test corrections bandPass");
+
TESTBOOL {
Parset ps = makeDefaultTestParset("Cobalt.correctBandPass", valstr);
@@ -181,6 +203,8 @@ SUITE(corrections) {
}
TEST(clock) {
+ LOG_INFO("Test corrections clock");
+
TESTBOOL {
Parset ps = makeDefaultTestParset("Cobalt.correctClocks", valstr);
@@ -189,6 +213,8 @@ SUITE(corrections) {
}
TEST(dedisperse) {
+ LOG_INFO("Test corrections dedisperse");
+
TESTBOOL {
Parset ps = makeDefaultTestParset("Cobalt.BeamFormer.coherentDedisperseChannels", valstr);
@@ -199,6 +225,8 @@ SUITE(corrections) {
SUITE(delayCompensation) {
TEST(enabled) {
+ LOG_INFO("Test delayCompensation enabled");
+
TESTBOOL {
Parset ps = makeDefaultTestParset("Cobalt.delayCompensation", valstr);
@@ -207,6 +235,8 @@ SUITE(delayCompensation) {
}
TEST(referencePhaseCenter) {
+ LOG_INFO("Test delayCompensation referencePhaseCenter");
+
Parset ps = makeDefaultTestParset("Observation.referencePhaseCenter", "[1,2,3]");
vector<double> refPhaseCenter(3);
@@ -223,6 +253,8 @@ SUITE(delayCompensation) {
*/
TEST(antennaSetLBA) {
+ LOG_INFO("Test antennaSetLBA");
+
vector<string> antennaSets;
antennaSets.push_back("LBA_INNER");
antennaSets.push_back("LBA_OUTER");
@@ -235,6 +267,8 @@ TEST(antennaSetLBA) {
}
TEST(antennaSetHBA) {
+ LOG_INFO("Test antennaSetHBA");
+
vector<string> antennaSets;
antennaSets.push_back("HBA_ZERO");
antennaSets.push_back("HBA_ONE");
@@ -254,6 +288,8 @@ TEST(antennaSetHBA) {
}
TEST(bandFilter) {
+ LOG_INFO("Test bandFilter");
+
// bandFilter[filter] = nyquistZone
map<string, unsigned> bandFilters;
bandFilters["LBA_10_90"] = 1;
@@ -272,6 +308,8 @@ TEST(bandFilter) {
SUITE(antennaFieldNames) {
TEST(LBA) {
+ LOG_INFO("Test antennaFieldNames LBA");
+
vector<string> stations, expectedFields;
stations.push_back("CS002");
expectedFields.push_back("CS002LBA");
@@ -292,6 +330,8 @@ SUITE(antennaFieldNames) {
}
TEST(HBA0) {
+ LOG_INFO("Test antennaFieldNames HBA0");
+
vector<string> stations, expectedFields;
stations.push_back("CS002");
expectedFields.push_back("CS002HBA0");
@@ -312,6 +352,8 @@ SUITE(antennaFieldNames) {
}
TEST(HBA1) {
+ LOG_INFO("Test antennaFieldNames HBA1");
+
vector<string> stations, expectedFields;
stations.push_back("CS002");
expectedFields.push_back("CS002HBA1");
@@ -332,6 +374,8 @@ SUITE(antennaFieldNames) {
}
TEST(HBA_DUAL) {
+ LOG_INFO("Test antennaFieldNames HBA_DUAL");
+
vector<string> stations, expectedFields;
stations.push_back("CS002");
expectedFields.push_back("CS002HBA0");
@@ -354,6 +398,8 @@ SUITE(antennaFieldNames) {
}
TEST(HBA_JOINED) {
+ LOG_INFO("Test antennaFieldNames HBA_JOINED");
+
vector<string> stations, expectedFields;
stations.push_back("CS002");
expectedFields.push_back("CS002HBA");
@@ -375,12 +421,14 @@ SUITE(antennaFieldNames) {
SUITE(indices) {
TEST(empty) {
+ LOG_INFO("Test indices empty");
vector<ObservationSettings::AntennaFieldName> full, sub;
vector<unsigned> correct_result;
CHECK_EQUAL(ObservationSettings::AntennaFieldName::indices(sub, full), correct_result);
}
TEST(one_present) {
+ LOG_INFO("Test indices one_present");
vector<ObservationSettings::AntennaFieldName> full, sub;
vector<unsigned> correct_result;
@@ -392,6 +440,7 @@ SUITE(antennaFieldNames) {
}
TEST(one_not_present) {
+ LOG_INFO("Test indices one_not_present");
vector<ObservationSettings::AntennaFieldName> full, sub;
full.push_back(ObservationSettings::AntennaFieldName("CS001","LBA"));
@@ -401,6 +450,7 @@ SUITE(antennaFieldNames) {
}
TEST(multi_present) {
+ LOG_INFO("Test indices multi_present");
vector<ObservationSettings::AntennaFieldName> full, sub;
vector<unsigned> correct_result;
@@ -418,6 +468,7 @@ SUITE(antennaFieldNames) {
}
TEST(multi_filtered) {
+ LOG_INFO("Test indices multi_filtered");
vector<ObservationSettings::AntennaFieldName> full, sub;
vector<unsigned> correct_result;
@@ -435,7 +486,10 @@ SUITE(antennaFieldNames) {
}
SUITE(stations) {
+
TEST(phaseCenter) {
+ LOG_INFO("Test stations phaseCenter");
+
Parset ps = makeDefaultTestParset();
// set
@@ -451,6 +505,8 @@ SUITE(stations) {
}
TEST(empty_map) {
+ LOG_INFO("Test stations empty_map");
+
Parset ps = makeDefaultTestParset();
const size_t nrSubbands = 488;
@@ -488,6 +544,8 @@ SUITE(stations) {
}
TEST(identity_map) {
+ LOG_INFO("Test stations identity_map");
+
Parset ps = makeDefaultTestParset();
// add stations and default board/slot lists
@@ -510,6 +568,8 @@ SUITE(stations) {
}
TEST(station_map) {
+ LOG_INFO("Test stations station_map");
+
Parset ps = makeDefaultTestParset();
// add stations and station-specific board/slot lists
@@ -542,7 +602,10 @@ SUITE(stations) {
}
SUITE(OutputCluster) {
+
TEST(clusterName) {
+ LOG_INFO("Test OutputCluster clusterName");
+
Parset ps = makeDefaultTestParset();
ps.replace("Observation.Cluster.ProcessingCluster.clusterName", "foo");
@@ -552,6 +615,8 @@ SUITE(OutputCluster) {
}
TEST(NIC_binding) {
+ LOG_INFO("Test OutputCluster NIC_binding");
+
Parset ps = makeDefaultTestParset();
ps.replace("Observation.Cluster.ProcessingCluster.clusterName", "CEP4");
@@ -569,8 +634,9 @@ SUITE(OutputCluster) {
SUITE(StationStreams) {
-
TEST(restrictNodes) {
+ LOG_INFO("Test StationStreams restrictNodes");
+
// optional key Cobalt.restrictNodesToStationStreams
// By default, all (half-)nodes must be used, because we may need them for
@@ -662,6 +728,8 @@ SUITE(StationStreams) {
SUITE(SAPs) {
TEST(nr) {
+ LOG_INFO("Test SAPs nr");
+
Parset ps = makeDefaultTestParset();
for (size_t nrSAPs = 1; nrSAPs < 244; ++nrSAPs) {
ps.replace("Observation.nrBeams", str(format("%u") % nrSAPs));
@@ -677,6 +745,8 @@ SUITE(SAPs) {
}
TEST(target) {
+ LOG_INFO("Test SAPs target");
+
Parset ps = makeDefaultTestParset();
// set
@@ -697,6 +767,8 @@ SUITE(SAPs) {
}
TEST(direction) {
+ LOG_INFO("Test SAPs direction");
+
Parset ps = makeDefaultTestParset();
// set
@@ -716,6 +788,8 @@ SUITE(SAPs) {
SUITE(anaBeam) {
TEST(enabled) {
+ LOG_INFO("Test anaBeam enabled");
+
TESTBOOL {
Parset ps = makeDefaultTestParset("Observation.antennaSet", val ? "HBA_ZERO" : "LBA_INNER");
@@ -724,6 +798,8 @@ SUITE(anaBeam) {
}
TEST(direction) {
+ LOG_INFO("Test anaBeam direction");
+
Parset ps = makeDefaultTestParset();
// set
@@ -742,7 +818,10 @@ SUITE(anaBeam) {
}
SUITE(subbands) {
+
TEST(nr) {
+ LOG_INFO("Test subbands nr");
+
for (size_t nrSubbands = 1; nrSubbands <= 244; ++nrSubbands) {
Parset ps = makeDefaultTestParset();
@@ -761,6 +840,8 @@ SUITE(subbands) {
}
TEST(idx_stationIdx) {
+ LOG_INFO("Test subbands stationIdx");
+
Parset ps = makeDefaultTestParset();
// set
@@ -774,6 +855,8 @@ SUITE(subbands) {
}
TEST(SAP) {
+ LOG_INFO("Test SAP");
+
Parset ps = makeDefaultTestParset();
// set -- note: for now, omitting actual SAP specifications is allowed
@@ -791,6 +874,8 @@ SUITE(subbands) {
}
TEST(centralFrequency) {
+ LOG_INFO("Test centralFrequency");
+
// test for both 200 and 160 MHz clocks,
// and for all three Nyquist zones.
@@ -842,6 +927,8 @@ SUITE(subbands) {
SUITE(correlator) {
TEST(enabled) {
+ LOG_INFO("Test correlator enabled");
+
TESTBOOL {
Parset ps = makeDefaultTestParset("Observation.DataProducts.Output_Correlated.enabled", valstr);
@@ -850,6 +937,8 @@ SUITE(correlator) {
}
TEST(nrChannels) {
+ LOG_INFO("Test correlator nrChannels");
+
Parset ps = makeDefaultTestParset();
ps.replace("Observation.DataProducts.Output_Correlated.enabled", "true");
@@ -860,6 +949,8 @@ SUITE(correlator) {
}
TEST(channelWidth) {
+ LOG_INFO("Test correlator channelWidth");
+
// validate all powers of 2 in [1, 4096]
for (size_t nrChannels = 1; nrChannels <= 4096; nrChannels <<= 1) {
Parset ps = makeDefaultTestParset();
@@ -873,6 +964,8 @@ SUITE(correlator) {
}
TEST(nrSamplesPerChannel) {
+ LOG_INFO("Test correlator nrSamplesPerChannel");
+
Parset ps = makeDefaultTestParset();
// set
@@ -886,6 +979,8 @@ SUITE(correlator) {
}
TEST(nrBlocksPerIntegration) {
+ LOG_INFO("Test correlator nrBlocksPerIntegration");
+
Parset ps = makeDefaultTestParset();
// set
@@ -901,6 +996,8 @@ SUITE(correlator) {
SUITE(files) {
TEST(filenames_mandatory) {
+ LOG_INFO("Test files filenames_mandatory");
+
Parset ps = makeDefaultTestParset();
// set
@@ -921,6 +1018,8 @@ SUITE(correlator) {
}
TEST(locations_mandatory) {
+ LOG_INFO("Test files locations_mandatory");
+
Parset ps = makeDefaultTestParset();
// set
@@ -941,6 +1040,8 @@ SUITE(correlator) {
}
TEST(nr) {
+ LOG_INFO("Test files nr");
+
// this test is expensive, so select a few values to test
vector<size_t> testNrSubbands;
testNrSubbands.push_back(1);
@@ -970,6 +1071,8 @@ SUITE(correlator) {
}
TEST(location) {
+ LOG_INFO("Test files location");
+
Parset ps = makeDefaultTestParset();
// set
@@ -988,6 +1091,8 @@ SUITE(correlator) {
}
TEST(cluster) {
+ LOG_INFO("Test files cluster");
+
Parset ps = makeDefaultTestParset();
// set
@@ -1015,6 +1120,8 @@ SUITE(correlator) {
SUITE(beamformer) {
SUITE(files) {
TEST(coherentLocation) {
+ LOG_INFO("Test beamformer/files coherentLocation");
+
Parset ps = makeDefaultTestParset();
// set
@@ -1034,6 +1141,8 @@ SUITE(beamformer) {
}
TEST(manyLocations) {
+ LOG_INFO("Test beamformer/files manyLocations");
+
Parset ps = makeDefaultTestParset();
// set
@@ -1055,6 +1164,8 @@ SUITE(beamformer) {
}
TEST(calcInternalNrChannels) {
+ LOG_INFO("Test beamformer calcInternalNrChannels");
+
// Validate that we compute the (max) nr of channels for delay compensation
// correctly.
Parset ps = makeDefaultTestParset();
@@ -1114,6 +1225,8 @@ SUITE(beamformer) {
* ===============================================
*/
TEST(testRing) {
+ LOG_INFO("Test testRing");
+
string prefix = "Observation.Beam[0]";
Parset ps = makeDefaultTestParset();
@@ -1164,10 +1277,11 @@ TEST(testRing) {
// The manual tabs should be added before the ring tabs.
// test values are for the 3rd value in the ring
struct ObservationSettings::Direction sap = ps.settings.SAPs[0].direction;
+ auto& SAPs = ps.settings.beamFormer.pipelines[0].SAPs;
CHECK_CLOSE(sap.angle1 + 6.10603350157654,
- ps.settings.beamFormer.SAPs[0].TABs[3].direction.angle1, 0.00000001);
+ SAPs[0].TABs[3].direction.angle1, 0.00000001);
CHECK_CLOSE(sap.angle2 + 1.0,
- ps.settings.beamFormer.SAPs[0].TABs[3].direction.angle2, 0.00000001);
+ SAPs[0].TABs[3].direction.angle2, 0.00000001);
// Full list of value for 1 circle:
//[(0.0, 0.0), (0.0, 2.0), (6.10603350157654, 1.0), (sqrt(3.0), -1.0), (0.0, -2.0), (-sqrt(3.0), 0), (-6.10603350157654, 1.0)]
}
@@ -1181,31 +1295,36 @@ TEST(testRing) {
SUITE(integration) {
TEST(228591) {
+ LOG_INFO("Test integration 228591");
+
// ===== read parset of observation L228591
Parset ps("tParset.parset_obs228591");
// check basic assumptions without which the subsequent
// checks will return random crap
CHECK(ps.settings.beamFormer.enabled);
- CHECK_EQUAL(1U, ps.settings.beamFormer.SAPs.size());
+ auto& SAPs = ps.settings.beamFormer.pipelines[0].SAPs;
+ CHECK_EQUAL(1U, SAPs.size());
// check the TAB rings: 4 rings = 61 TABs
- CHECK_EQUAL(61U, ps.settings.beamFormer.SAPs[0].TABs.size());
+ CHECK_EQUAL(61U, SAPs[0].TABs.size());
// first TAB is equal to SAP
struct ObservationSettings::Direction sap = ps.settings.SAPs[0].direction;
- CHECK_EQUAL(sap.angle1, ps.settings.beamFormer.SAPs[0].TABs[0].direction.angle1);
- CHECK_EQUAL(sap.angle2, ps.settings.beamFormer.SAPs[0].TABs[0].direction.angle2);
+ CHECK_EQUAL(sap.angle1, SAPs[0].TABs[0].direction.angle1);
+ CHECK_EQUAL(sap.angle2, SAPs[0].TABs[0].direction.angle2);
// subsequent TABs are NOT (0,0)
- for (size_t tab = 1; tab < ps.settings.beamFormer.SAPs[0].TABs.size(); tab++) {
- struct ObservationSettings::Direction dir = ps.settings.beamFormer.SAPs[0].TABs[tab].direction;
+ for (size_t tab = 1; tab < SAPs[0].TABs.size(); tab++) {
+ struct ObservationSettings::Direction dir = SAPs[0].TABs[tab].direction;
CHECK(dir.angle1 != sap.angle1 || dir.angle2 != sap.angle2);
}
}
TEST(99275) {
+ LOG_INFO("Test integration 99275");
+
// ===== read parset of observation L99275
Parset ps("tParset.parset_obs99275");
@@ -1294,19 +1413,21 @@ SUITE(integration) {
}
CHECK_EQUAL(nrSubbands, ps.settings.correlator.files.size());
+ auto& SAPs = ps.settings.beamFormer.pipelines[0].SAPs;
+
// ===== test beam-former settings
CHECK_EQUAL(true, ps.settings.beamFormer.enabled);
- CHECK_EQUAL(nrSAPs, ps.settings.beamFormer.SAPs.size());
+ CHECK_EQUAL(nrSAPs, SAPs.size());
// check SAP 0
- CHECK_EQUAL(2U, ps.settings.beamFormer.SAPs[0].TABs.size());
- CHECK_EQUAL(true, ps.settings.beamFormer.SAPs[0].TABs[0].coherent);
- CHECK_EQUAL(false, ps.settings.beamFormer.SAPs[0].TABs[1].coherent);
+ CHECK_EQUAL(2U, SAPs[0].TABs.size());
+ CHECK_EQUAL(true, SAPs[0].TABs[0].coherent);
+ CHECK_EQUAL(false, SAPs[0].TABs[1].coherent);
// check SAP 1
- CHECK_EQUAL(2U, ps.settings.beamFormer.SAPs[1].TABs.size());
- CHECK_EQUAL(true, ps.settings.beamFormer.SAPs[1].TABs[0].coherent);
- CHECK_EQUAL(false, ps.settings.beamFormer.SAPs[1].TABs[1].coherent);
+ CHECK_EQUAL(2U, SAPs[1].TABs.size());
+ CHECK_EQUAL(true, SAPs[1].TABs[0].coherent);
+ CHECK_EQUAL(false, SAPs[1].TABs[1].coherent);
// check coherent settings
CHECK_EQUAL(STOKES_I, ps.settings.beamFormer.coherentSettings.type);
@@ -1322,6 +1443,72 @@ SUITE(integration) {
}
}
+/*
+ * ===============================================
+ * Test interaction with full Parsets for coherency
+ * between fields.
+ * ===============================================
+ */
+
+SUITE(lmm) {
+ TEST(beamformers) {
+ LOG_INFO("Test lmm multiple beamformers");
+
+ Parset ps = makeDefaultTestParset();
+
+ // Specify base beamFormer
+ ps.replace("Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband", "16");
+ ps.replace("Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor", "1");
+ ps.replace("Cobalt.BeamFormer.CoherentStokes.which", "IQUV");
+
+ // Disable correlator
+ ps.replace("Observation.DataProducts.Output_Correlated.enabled", "false");
+
+ // Set the number of beamFormer pipelines
+ ps.add("Cobalt.BeamFormer.nrPipelines", "2");
+
+ // Don't specify the first beamFormer pipeline
+ // pass
+
+ // Specify second beamFormer pipeline
+ ps.add("Cobalt.BeamFormer.Pipeline[1].CoherentStokes.nrChannelsPerSubband", "64");
+ ps.add("Cobalt.BeamFormer.Pipeline[1].CoherentStokes.timeIntegrationFactor", "4");
+ ps.add("Cobalt.BeamFormer.Pipeline[1].CoherentStokes.which", "I");
+
+ // Specify output files
+ ps.replace("Observation.DataProducts.Output_CoherentStokes.filenames",
+ "[L12345_SAP000_B000_S000_P000_bf.h5, \
+ L12345_SAP000_B000_S000_P001_bf.h5, \
+ L12345_SAP000_B000_S002_P000_bf.h5, \
+ L12345_SAP000_B000_S003_P000_bf.h5, \
+ L12345_SAP000_B001_S000_P000_bf.h5, \
+ L12345_SAP000_B002_S000_P000_bf.h5, \
+ L12345_SAP000_B002_S000_P001_bf.h5, \
+ L12345_SAP000_B002_S002_P000_bf.h5, \
+ L12345_SAP000_B002_S003_P000_bf.h5, \
+ L12345_SAP000_B003_S000_P000_bf.h5]");
+ ps.replace("Observation.DataProducts.Output_CoherentStokes.locations",
+ "[10*localhost:tParset-data/]");
+
+ ps.updateSettings();
+
+ // Check settings for the base beamFormer
+ CHECK_EQUAL(16, ps.settings.beamFormer.coherentSettings.nrChannels);
+ CHECK_EQUAL(1, ps.settings.beamFormer.coherentSettings.timeIntegrationFactor);
+ CHECK_EQUAL(STOKES_IQUV, ps.settings.beamFormer.coherentSettings.type);
+
+ // The first beamFormer should have the same settings as the base beamFormer
+ CHECK_EQUAL(16, ps.settings.beamFormer.pipelines[0].coherentSettings.nrChannels);
+ CHECK_EQUAL(1, ps.settings.beamFormer.pipelines[0].coherentSettings.timeIntegrationFactor);
+ CHECK_EQUAL(STOKES_IQUV, ps.settings.beamFormer.pipelines[0].coherentSettings.type);
+
+ // Check settings of the second beamFormer
+ CHECK_EQUAL(64, ps.settings.beamFormer.pipelines[1].coherentSettings.nrChannels);
+ CHECK_EQUAL(4, ps.settings.beamFormer.pipelines[1].coherentSettings.timeIntegrationFactor);
+ CHECK_EQUAL(STOKES_I, ps.settings.beamFormer.pipelines[1].coherentSettings.type);
+ }
+}
+
int main(void)
{
INIT_LOGGER("tParset");
diff --git a/RTCP/Cobalt/CoInterface/test/tTABTranspose.cc b/RTCP/Cobalt/CoInterface/test/tTABTranspose.cc
index 170110c90c13db8629bb2e35c0cdec50a3e67a77..945ec87e18c8e5335e1677fd644d5af6b527c5a8 100644
--- a/RTCP/Cobalt/CoInterface/test/tTABTranspose.cc
+++ b/RTCP/Cobalt/CoInterface/test/tTABTranspose.cc
@@ -50,7 +50,7 @@ SUITE(Block) {
Block block(0, 0, nrSubbands, nrSamples, nrChannels);
for (size_t subbandIdx = 0; subbandIdx < nrSubbands; ++subbandIdx) {
- SmartPtr<Subband> subband = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> subband(new Subband(nrSamples, nrChannels));
subband->id.subband = subbandIdx;
CHECK(!block.complete());
@@ -78,7 +78,7 @@ SUITE(Block) {
// avoid starting at 0, because the ordered test already does
size_t subbandIdx = (3 + n * subbandIncrement) % nrSubbands;
- SmartPtr<Subband> subband = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> subband(new Subband(nrSamples, nrChannels));
subband->id.subband = subbandIdx;
CHECK(!block.complete());
@@ -108,7 +108,7 @@ SUITE(Block) {
for (size_t n = 0; n < nrSubbands; ++n) {
size_t subbandIdx = (3 + n * subbandIncrement) % nrSubbands;
- SmartPtr<Subband> subband = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> subband(new Subband(nrSamples, nrChannels));
subband->id.subband = subbandIdx;
block.addSubband(subband);
@@ -141,7 +141,8 @@ struct Fixture {
ctr(outputPool, 0, nrSubbands, nrChannels, nrSamples)
{
for (size_t i = 0; i < nrBlocks; ++i) {
- outputPool.free.append(new BeamformedData(nrSubbands, nrChannels, nrSamples), false);
+ std::shared_ptr<BeamformedData> data(new BeamformedData(nrSubbands, nrChannels, nrSamples));
+ outputPool.free.append(data, false);
}
}
};
@@ -165,7 +166,7 @@ struct Fixture_Loss: public Fixture {
if (subbandIdx == 3)
continue;
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = blockIdx;
sb->id.subband = subbandIdx;
@@ -187,7 +188,7 @@ SUITE(BlockCollector) {
TEST_FIXTURE(Fixture, OneBlock) {
// add all subbands for block 0
for (size_t subbandIdx = 0; subbandIdx < nrSubbands; ++subbandIdx) {
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 0;
sb->id.subband = subbandIdx;
@@ -203,7 +204,7 @@ SUITE(BlockCollector) {
// add all subbands for all blocks
for (size_t blockIdx = 0; blockIdx < nrBlocks; ++blockIdx) {
for (size_t subbandIdx = 0; subbandIdx < nrSubbands; ++subbandIdx) {
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = blockIdx;
sb->id.subband = subbandIdx;
@@ -225,13 +226,13 @@ SUITE(BlockCollector) {
// we add blocks [1,3], enough to keep in flight
{
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 1;
sb->id.subband = 0;
ctr_loss.addSubband(sb);
}
{
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 3;
sb->id.subband = 1;
ctr_loss.addSubband(sb);
@@ -240,7 +241,7 @@ SUITE(BlockCollector) {
// we let block 0 arrive, which could (erroneously) emit block 1
// in favour of block 0.
{
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 0;
sb->id.subband = 2;
ctr_loss.addSubband(sb);
@@ -249,7 +250,7 @@ SUITE(BlockCollector) {
// if ok, we now have [1,3] still, and should be able to add block 2,
// causing block 1 to be emitted.
{
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 2;
sb->id.subband = 3;
ctr_loss.addSubband(sb);
@@ -269,7 +270,7 @@ SUITE(BlockCollector) {
if (subbandIdx == 3)
continue;
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 0;
sb->id.subband = subbandIdx;
@@ -281,7 +282,7 @@ SUITE(BlockCollector) {
/*
// add all subbands for block 1
for (size_t subbandIdx = 0; subbandIdx < nrSubbands; ++subbandIdx) {
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 1;
sb->id.subband = subbandIdx;
@@ -300,7 +301,7 @@ SUITE(BlockCollector) {
// add one subband for a new block, causing block
// 0 to spill.
{
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = maxInFlight;
sb->id.subband = 0;
@@ -315,7 +316,7 @@ SUITE(BlockCollector) {
/*
// let subband 3 arrive late
{
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = 0;
sb->id.subband = 3;
@@ -338,7 +339,7 @@ SUITE(BlockCollector) {
if (subbandIdx == 3)
continue;
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.block = blockIdx;
sb->id.subband = subbandIdx;
@@ -395,16 +396,17 @@ SUITE(SendReceive) {
TEST_FIXTURE(Fixture, OneToOne) {
const size_t nrTABs = nrBlocks; // we know we have enough outputPool.free for nrBlocks TABs
- map<size_t, SmartPtr< Pool<BeamformedData> > > outputPools;
+ map<size_t, std::shared_ptr<Pool<BeamformedData>>> outputPools;
Receiver::CollectorMap collectors;
for (size_t i = 0; i < nrTABs; ++i) {
- outputPools[i] = new Pool<BeamformedData>(str(format("OneToOne::outputPool[%u]") % i), true);
+ outputPools[i].reset(new Pool<BeamformedData>(str(format("OneToOne::outputPool[%u]") % i), true));
for (size_t b = 0; b < nrBlocks; ++b) {
- outputPools[i]->free.append(new BeamformedData(nrSubbands, nrChannels, nrSamples), false);
+ std::shared_ptr<BeamformedData> data(new BeamformedData(nrSubbands, nrChannels, nrSamples));
+ outputPools[i]->free.append(data, false);
}
- collectors[i] = new BlockCollector(*outputPools[i], i, nrSubbands, nrChannels, nrSamples);
+ collectors[i].reset(new BlockCollector(*outputPools[i], i, nrSubbands, nrChannels, nrSamples));
}
StringStream str;
@@ -412,7 +414,7 @@ SUITE(SendReceive) {
Receiver receiver(str, collectors);
for (size_t i = 0; i < nrTABs * nrSubbands; ++i) {
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.fileIdx = i % nrTABs;
sb->id.block = 0;
sb->id.subband = i / nrTABs;
@@ -435,7 +437,7 @@ SUITE(SendReceive) {
// Should have one complete block, plus NULL
CHECK_EQUAL(2UL, outputPools[i]->filled.size());
- SmartPtr<BeamformedData> block = outputPools[i]->filled.remove();
+ std::shared_ptr<BeamformedData> block = outputPools[i]->filled.remove();
CHECK(block != NULL);
@@ -485,7 +487,7 @@ SUITE(MultiReceiver) {
// Set up receiver
Receiver::CollectorMap collectors;
- collectors[0] = new BlockCollector(outputPool, 0, nrSubbands, nrChannels, nrSamples);
+ collectors[0].reset(new BlockCollector(outputPool, 0, nrSubbands, nrChannels, nrSamples));
MultiReceiver mr("foo-", collectors);
@@ -495,7 +497,7 @@ SUITE(MultiReceiver) {
// Send one block
{
- SmartPtr<Subband> sb = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> sb(new Subband(nrSamples, nrChannels));
sb->id.fileIdx = 0;
sb->id.block = 0;
sb->id.subband = 0;
@@ -525,13 +527,19 @@ SUITE(MultiReceiver) {
MultiSender msender(hostMap, ps);
}
- TEST(Transpose) {
+ struct Fixture_Transpose {
+ Fixture_Transpose() { }
+
+ void run(bool quantized) {
// We use the even fileIdx to simulate a sparse set.
// Do create enough filenames. (Ab)use the subbandsPerFile (multiple parts) feature for that.
#define SPARSITY 2
#define FILEIDX(tabNr) ((tabNr)*SPARSITY)
-
- LOG_DEBUG_STR("Transpose test started");
+ if (quantized) {
+ LOG_DEBUG_STR("Transpose test started with quantization");
+ } else {
+ LOG_DEBUG_STR("Transpose test started");
+ }
const int nrSubbands = 4;
const size_t nrBlocks = 2;
@@ -553,6 +561,12 @@ SUITE(MultiReceiver) {
ps.replace("Observation.DataProducts.Output_CoherentStokes.locations", "[" + lexical_cast<string>(SPARSITY * nrTABs) + "*localhost:tParset-data/]");
ps.replace("Cobalt.blockSize", lexical_cast<string>(nrSamples * nrChannels));
ps.replace("Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband", lexical_cast<string>(nrSamples));
+ if (quantized) {
+ ps.replace("Cobalt.BeamFormer.CoherentStokes.quantize", "true");
+ ps.replace("Cobalt.BeamFormer.CoherentStokes.quantizeBits", "8");
+ ps.replace("Cobalt.BeamFormer.CoherentStokes.quantizeScaleMax", "5.0");
+ ps.replace("Cobalt.BeamFormer.CoherentStokes.quantizeScaleMin", "-5.0");
+ }
ps.updateSettings();
// Give both senders and receivers multiple tasks,
@@ -560,6 +574,8 @@ SUITE(MultiReceiver) {
const int nrSenders = 4;
const int nrReceivers = 2;
+ const bool quantize=ps.settings.beamFormer.coherentSettings.quantizerSettings.enabled;
+
Semaphore sendersDone;
LOG_DEBUG_STR("Spawning threads");
@@ -577,19 +593,20 @@ SUITE(MultiReceiver) {
LOG_DEBUG_STR("Populating outputPools");
// collect our TABs
- std::map<size_t, SmartPtr< Pool<BeamformedData> > > outputPools;
+ std::map<size_t, std::shared_ptr<Pool<BeamformedData>>> outputPools;
Receiver::CollectorMap collectors;
for (int t = 0; t < nrTABs; ++t) {
if (t % nrReceivers != r)
continue;
- outputPools[t] = new Pool<BeamformedData>(str(format("MultiReceiver::Transpose::outputPool[%u]") % t), true);
+ outputPools[t].reset(new Pool<BeamformedData>(str(format("MultiReceiver::Transpose::outputPool[%u]") % t), true));
for (size_t i = 0; i < nrBlocks; ++i) {
- outputPools[t]->free.append(new BeamformedData(nrSubbands, nrChannels, nrSamples), false);
+ std::shared_ptr<BeamformedData> data(new BeamformedData(nrSubbands, nrChannels, nrSamples, quantize));
+ outputPools[t]->free.append(data, false);
}
- collectors[FILEIDX(t)] = new BlockCollector(*outputPools[t], FILEIDX(t), nrSubbands, nrChannels, nrSamples, nrBlocks);
+ collectors[FILEIDX(t)].reset(new BlockCollector(*outputPools[t], FILEIDX(t), nrSubbands, nrChannels, nrSamples, nrBlocks));
}
LOG_DEBUG_STR("Starting receiver " << r);
@@ -611,7 +628,7 @@ SUITE(MultiReceiver) {
CHECK_EQUAL(nrBlocks + 1UL, outputPools[t]->filled.size());
for (size_t b = 0; b < nrBlocks; ++b) {
- SmartPtr<BeamformedData> block = outputPools[t]->filled.remove();
+ std::shared_ptr<BeamformedData> block = outputPools[t]->filled.remove();
CHECK(block != NULL);
@@ -663,7 +680,7 @@ SUITE(MultiReceiver) {
// Send all TABs
for (int t = 0; t < nrTABs; ++t) {
- SmartPtr<Subband> subband = new Subband(nrSamples, nrChannels);
+ std::shared_ptr<Subband> subband(new Subband(nrSamples, nrChannels, quantize));
subband->id.fileIdx = FILEIDX(t);
subband->id.block = b;
subband->id.subband = sb;
@@ -685,6 +702,16 @@ SUITE(MultiReceiver) {
}
}
+ }
+
+ };
+
+ TEST_FIXTURE(Fixture_Transpose, Transpose) {
+ run(0); //test with no quantization
+ }
+
+ TEST_FIXTURE(Fixture_Transpose, QuantizedTranspose) {
+ run(1); //test with quantization
}
}
diff --git a/RTCP/Cobalt/CobaltTest/test/tManyPartTABOutput.cc b/RTCP/Cobalt/CobaltTest/test/tManyPartTABOutput.cc
index 3cd86fb66398a2b84eb93fb5363593cf548b2472..d6238e3028faf831f53be1bc876a90fca069aed2 100644
--- a/RTCP/Cobalt/CobaltTest/test/tManyPartTABOutput.cc
+++ b/RTCP/Cobalt/CobaltTest/test/tManyPartTABOutput.cc
@@ -41,7 +41,7 @@ using boost::format;
using boost::str;
// Fill sb with all values equal to its station sb nr (see .parset).
-SmartPtr<SubbandProcOutputData> getTestSbCohData(const Parset& ps, gpu::Context& ctx,
+std::shared_ptr<SubbandProcOutputData> getTestSbCohData(const Parset& ps, gpu::Context& ctx,
unsigned blockIdx, unsigned sbIdx)
{
SubbandProcOutputData *bfData = new SubbandProcOutputData(ps, ctx);
@@ -51,11 +51,24 @@ SmartPtr<SubbandProcOutputData> getTestSbCohData(const Parset& ps, gpu::Context&
bfData->blockID.localSubbandIdx = sbIdx;
bfData->blockID.subbandProcSubbandIdx = sbIdx;
const unsigned sbFreqNr = ps.settings.subbands[sbIdx].stationIdx;
- for (size_t v = 0; v < bfData->coherentData.num_elements(); v++) {
- bfData->coherentData.origin()[v] = (float)sbFreqNr;
+ auto& coherentData = bfData->coherentData();
+ if (!coherentData.is_quantized()) {
+ for (size_t v = 0; v < coherentData.data_num_elements(); v++) {
+ coherentData.get_data_origin()[v] = (float)sbFreqNr;
+ }
+ } else {
+ for (size_t v = 0; v < coherentData.qdata_num_elements(); v++) {
+ coherentData.get_qdata_origin()[v] = (int8_t)sbFreqNr;
+ }
+ for (size_t v = 0; v < coherentData.qoffsets_num_elements(); v++) {
+ coherentData.get_qoffsets_origin()[v] = (float)-1.0;
+ }
+ for (size_t v = 0; v < coherentData.qscales_num_elements(); v++) {
+ coherentData.get_qscales_origin()[v] = (float)1.0;
+ }
}
- SmartPtr<SubbandProcOutputData> data = bfData;
+ std::shared_ptr<SubbandProcOutputData> data(bfData);
return data;
}
@@ -114,10 +127,10 @@ int main()
// Insert 2 blocks of test data per sb.
std::vector<struct Pipeline::Output> writePool(nrSubbands); // [localSubbandIdx]
for (unsigned i = 0; i < writePool.size(); i++) {
- SmartPtr<SubbandProcOutputData> data;
+ std::shared_ptr<SubbandProcOutputData> data;
unsigned blockIdx;
- writePool[i].queue = new Queue< SmartPtr<SubbandProcOutputData> >(str(format("writePool [file %u]") % i));
+ writePool[i].queue.reset(new Queue<std::shared_ptr<SubbandProcOutputData>>(str(format("writePool [file %u]") % i)));
blockIdx = 0;
data = getTestSbCohData(ps, ctx, blockIdx, i);
diff --git a/RTCP/Cobalt/CobaltTest/test/tMultiPartTABOutput.cc b/RTCP/Cobalt/CobaltTest/test/tMultiPartTABOutput.cc
index 9177c52d59d47d1def460125f4579308e0be6eac..1a286438415bb2944ac6321cc4b7d5ab7fb54c60 100644
--- a/RTCP/Cobalt/CobaltTest/test/tMultiPartTABOutput.cc
+++ b/RTCP/Cobalt/CobaltTest/test/tMultiPartTABOutput.cc
@@ -41,7 +41,7 @@ using boost::format;
using boost::str;
// Fill sb with all values equal to its station sb nr (see .parset).
-SmartPtr<SubbandProcOutputData> getTestSbIncohData(const Parset& ps, gpu::Context& ctx,
+std::shared_ptr<SubbandProcOutputData> getTestSbIncohData(const Parset& ps, gpu::Context& ctx,
unsigned blockIdx, unsigned sbIdx)
{
SubbandProcOutputData *bfData = new SubbandProcOutputData(ps, ctx);
@@ -51,11 +51,24 @@ SmartPtr<SubbandProcOutputData> getTestSbIncohData(const Parset& ps, gpu::Contex
bfData->blockID.localSubbandIdx = sbIdx;
bfData->blockID.subbandProcSubbandIdx = sbIdx;
const unsigned sbFreqNr = ps.settings.subbands[sbIdx].stationIdx;
- for (size_t v = 0; v < bfData->incoherentData.num_elements(); v++) {
- bfData->incoherentData.origin()[v] = (float)sbFreqNr;
+ auto& incoherentData = bfData->incoherentData();
+ if (!incoherentData.is_quantized()) {
+ for (size_t v = 0; v < incoherentData.data_num_elements(); v++) {
+ incoherentData.get_data_origin()[v] = (float)sbFreqNr;
+ }
+ } else {
+ for (size_t v = 0; v < incoherentData.qdata_num_elements(); v++) {
+ incoherentData.get_qdata_origin()[v] = (int8_t)sbFreqNr;
+ }
+ for (size_t v = 0; v < incoherentData.qoffsets_num_elements(); v++) {
+ incoherentData.get_qoffsets_origin()[v] = (float)1.0;
+ }
+ for (size_t v = 0; v < incoherentData.qscales_num_elements(); v++) {
+ incoherentData.get_qscales_origin()[v] = (float)1.0;
+ }
}
- SmartPtr<SubbandProcOutputData> data = bfData;
+ std::shared_ptr<SubbandProcOutputData> data(bfData);
return data;
}
@@ -114,10 +127,10 @@ int main()
// Insert 1 block of test data per sb.
std::vector<struct Pipeline::Output> writePool(nrSubbands); // [localSubbandIdx]
for (unsigned i = 0; i < writePool.size(); i++) {
- SmartPtr<SubbandProcOutputData> data;
+ std::shared_ptr<SubbandProcOutputData> data;
unsigned blockIdx;
- writePool[i].queue = new Queue< SmartPtr<SubbandProcOutputData> >(str(format("writePool [file %u]") % i));
+ writePool[i].queue.reset(new Queue<std::shared_ptr<SubbandProcOutputData>>(str(format("writePool [file %u]") % i)));
blockIdx = 0;
data = getTestSbIncohData(ps, ctx, blockIdx, i);
diff --git a/RTCP/Cobalt/GPUProc/CMakeLists.txt b/RTCP/Cobalt/GPUProc/CMakeLists.txt
index 17b1eedac03de781dc64538db5cd40234033d615..234ae15aecf4bf3c9392e654d8131c8ad8c77d74 100644
--- a/RTCP/Cobalt/GPUProc/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/CMakeLists.txt
@@ -14,6 +14,8 @@ set(_gpuproc_deps "")
set(CUDA_PROPAGATE_HOST_FLAGS OFF)
lofar_find_package(CUDA 4.1 REQUIRED)
lofar_find_package(CUDADriver REQUIRED)
+lofar_find_package(CUDAnvToolsExt REQUIRED)
+lofar_find_package(CUDAnvrtc REQUIRED)
if(LOFAR_BUILD_VARIANT MATCHES "^DEBUG$")
list(APPEND CUDA_NVCC_FLAGS -g -G)
diff --git a/RTCP/Cobalt/GPUProc/doc/opencl-cuda-types.txt b/RTCP/Cobalt/GPUProc/doc/opencl-cuda-types.txt
index a5873336437ba0c17608b7cc18fa3433d8dc741e..ab8621b0bbb3221c41dcd3cf17bf6e9f6810a26a 100644
--- a/RTCP/Cobalt/GPUProc/doc/opencl-cuda-types.txt
+++ b/RTCP/Cobalt/GPUProc/doc/opencl-cuda-types.txt
@@ -22,7 +22,7 @@ cl::Platform (none) gpu::Platform
For the GPUProc types, we use the CUDA names as much as possible.
To match the ref counting OpenCL types, all resource managing (meaningful
-destructor) CUDA types have a boost::shared_ptr around Impl classes.
+destructor) CUDA types have a std::shared_ptr around Impl classes.
Mapping of OpenCL Dimensions and Indices to CUDA
diff --git a/RTCP/Cobalt/GPUProc/doc/performanceTest/PerformanceTest.md b/RTCP/Cobalt/GPUProc/doc/performanceTest/PerformanceTest.md
new file mode 100644
index 0000000000000000000000000000000000000000..170927b904117018e95ec4872c038a43eac7c0ac
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/doc/performanceTest/PerformanceTest.md
@@ -0,0 +1,73 @@
+# Goal
+We run a set of representative benchmarks to test performance regression of the kernels for the different pipelines. As we only aim to test a representative parameter space for the kernels it seems best to run a set of pipeline tests with representative use cases for LOFAR and LOFAR Mega Mode. Tests are specified through the LOFAR parsets and are passed to the gpu_load test that sets up a relative simple pipeline with performance benchmarking enabled.
+
+# Run benchmarks
+```
+INPUT: (required) location of gpu_load binary,
+ (required) location of source parsets,
+ (required) root directory for results,
+ (optional) specify [int] number of iterations for gpu_load, default is 1000 iterations
+ (optional) enable storing of parsets,
+ (optional) select specific parset(s)
+ (optional) disable processing (e.g. dry run)
+
+Take as test suites a list of: <parset-name><observation ID>
+
+FOR EACH test suite P in a list of test suites (P1 through Pn):
+ Take the base parset
+ SET Cobalt.Benchmark.file=<benchmark file path>/<P.name>.csv
+ IF storing of parsets is enabled store modified parset to <path>/<P.name>.parset
+ IF processing is enabled
+ blocking call gpu_load with parset as argument, results are written (by gpu_load) to <benchmark file path>/<P.name>.csv
+ ENDIF
+END
+
+Optionally later add test cases (variation of parset parameters)
+FOR EACH test suite P in a list of test suites (P1 through Pn):
+ Take its base parset (observation ID) or a minimal version of that (TBD)
+ FOR EACH test case C in the test suite
+ Modify the parset with use case specific parameters
+ SET Cobalt.Benchmark.file=<benchmark file path>/<P.name><C.name>.csv
+ IF storing of parsets is enabled store modified parset to <path>/<P.name><C.name>.parset
+ IF processing is enabled
+ blocking call gpu_load with parset as argument, results are written (by gpu_load) to <benchmark file path>/<P.name><C.name>.csv
+ ENDIF
+ END
+END
+```
+
+# Analyze benchmarks
+```
+INPUT: (required) root directory with benchmark results
+
+FOR EACH file in <benchmark file path> (list of files <P.name>.csv)
+ Read the file
+ FOR EACH KernelName in the file
+ Filter for "PerformanceCounter" and list the performance statistics
+ <File name (test ID) > ; <kernel name> ; <statistics>
+ Construct kernelIndex from test ID and kernel name
+ Write in a separate file: <kernelIndex> ; <mean>
+ END
+END
+```
+Example of input that we are filtering for:
+```
+format; kernelName; count; mean; stDev; min; max; unit
+PerformanceCounter; output (incoherent); 10; 0.28483; 0.01499; 0.24310; 0.29318; ms
+```
+# Compare analysis
+```
+INPUT: (required) file with reference mean values (output format from analysis script)
+ (required) file with current mean values (output format from analysis script)
+
+Construct a dict with kernelIndex and mean values for both files
+FOR EACH key in the reference dict
+ IF key also exists in the current dict
+ Calculate diff and write output as <kernelIndex> ; <referece mean> ; <current mean> ; <difference of mean values>
+ END
+END
+```
+
+Optionally extend with:
+* Calculation of % real-time
+* Compare with performance (golden) reference file, pass fail output to top script for CI pipeline
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/doc/performanceTest/gpu_load.md b/RTCP/Cobalt/GPUProc/doc/performanceTest/gpu_load.md
new file mode 100644
index 0000000000000000000000000000000000000000..7ce9513a105c298825ceb4437b4e72bbc91b7f38
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/doc/performanceTest/gpu_load.md
@@ -0,0 +1,55 @@
+This document describes the usage of the *gpu_load* test for performance measurements.
+
+The *gpu_load* test source is in: `lofar/RTCP/Cobalt/GPUProc/src`
+
+After build and installation the *gpu_load* binary is in: `<install directory>/lofar/bin`
+
+Usage: call *gpu_load* with a parset as parameter, e.g. `~/opt/lofar/bin/gpu_load ~/parsets/rtcp-698421-COBALT2-ppf-stokes-coh-incoh.parset`
+
+Optional: add an [int] nr of iterations for the tests, the default nr of iterations is set to 10. e.g. `~/opt/lofar/bin/gpu_load ~/parsets/rtcp-698421-COBALT2-ppf-stokes-coh-incoh.parset 1000`
+
+The *gpu_load* test is intended for performance measurements of a GPU kernels. The test sets up a processing pipeline on a single node. It takes the configuration as specified in the parset. It does not use actual data and does not produce functional output.
+
+By default the *gpu_load* test reports kernel and pipeline performance numbers to stdout. In order to store the performance numbers to file the parset should contain the following two keys:
+* `Cobalt.Benchmark.enabled=true`
+* `Cobalt.Benchmark.file=<benchmark file path>/<file name>.csv`
+
+If logging to file is not required then these keys can be left out of the parset. By default `Cobalt.Benchmark.enabled` is set to *false*. If `Cobalt.Benchmark.enabled` is set to *true* then individual kernel statistics will be logged to file upon completion of the kernel (handled in the destructor).
+If a path is specified and it is not accessible (directory does not exist or rights issue) then `Cobalt.Benchmark.enabled` is set to *false* again and no performance numbers will be stored.
+If the specified file does not exist yet it will be created, if it already exists then it will be overwritten.
+
+The test reports extrapolated performance numbers on stdout. In order to calculate these numbers the following parset keys will need to be specified:
+* `Observation.Beam[0].subbandList` How many subbands are in this SAP. Work load scales with total number of subbands across all SAPs. Typical: 400, 480, 488. e.g. *[0..487]*
+* `Cobalt.Nodes` Determines how many GPUs will be available for load distribution. Typical: 22*localhost.
+
+Performance numbers are controlled by `LOFAR::Cobalt::PerformanceCounter` (part of *GPUProc*) and collected and printed by `LOFAR::Cobalt::RunningStatistics` (part of *CoInterface*) which are both used through `LOFAR::Cobalt::Kernel`.
+
+Alternativley the *gpu_load* test can be used with NVVP to visualize and inspect the processing pipline and individual kernel performance.
+
+
+Example output to file:
+
+```
+Tue Jun 2 08:36:21 2020
+info; Test gpu_load: /home/.../opt/lofar/bin/gpu_load with parset: /home/.../parsets/rtcp-698421-COBALT2-ppf-stokes-coh-incoh.parset
+format; kernelName; count; mean; stDev; min; max; unit
+PerformanceCounter; output (incoherent); 10; 0.28483; 0.01499; 0.24310; 0.29318; ms
+PerformanceCounter; incoherentStokes; 10; 1.05907; 0.10314; 0.85702; 1.20778; ms
+PerformanceCounter; FFT (incoherent, final); 10; 0.40620; 0.00172; 0.40406; 0.40963; ms
+PerformanceCounter; FIR (incoherent, final); 10; 1.65774; 0.02571; 1.59002; 1.67645; ms
+PerformanceCounter; FFT-shift (incoherent, inverse); 10; 0.36252; 0.00027; 0.36224; 0.36307; ms
+PerformanceCounter; FFT (incoherent, inverse); 10; 0.40509; 0.00212; 0.40211; 0.40813; ms
+PerformanceCounter; incoherentStokesTranspose; 10; 0.42096; 0.00722; 0.41098; 0.43024; ms
+PerformanceCounter; quantizeOutput; 10; 0.04149; 0.00192; 0.04010; 0.04682; ms
+PerformanceCounter; output (coherent); 10; 0.00794; 0.00076; 0.00698; 0.00954; ms
+PerformanceCounter; coherentStokes; 10; 0.03905; 0.00132; 0.03814; 0.04266; ms
+PerformanceCounter; FFT (coherent, final); 10; 0.01025; 0.00239; 0.00941; 0.01706; ms
+PerformanceCounter; FIR (coherent, final); 10; 0.77494; 0.01318; 0.74938; 0.78496; ms
+PerformanceCounter; FFT-shift (coherent, inverse); 10; 0.00765; 0.00114; 0.00723; 0.01088; ms
+PerformanceCounter; FFT (coherent, inverse); 10; 0.00963; 0.00090; 0.00915; 0.01219; ms
+PerformanceCounter; coherentStokesTranspose; 10; 0.02697; 0.00137; 0.02643; 0.03085; ms
+PerformanceCounter; beamFormer; 10; 1.20297; 0.00429; 1.19654; 1.21264; ms
+PerformanceCounter; bandPassCorrection; 10; 0.63590; 0.00828; 0.62384; 0.64678; ms
+PerformanceCounter; delayAndBandPass; 10; 0.35521; 0.00149; 0.35408; 0.35910; ms
+PerformanceCounter; Zeroing (beamformer); 10; 0.06737; 0.00092; 0.06682; 0.06989; ms
+```
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/doc/pipeline-buffers.txt b/RTCP/Cobalt/GPUProc/doc/pipeline-buffers.txt
index 3f6e6283fa0767349eeeb11f8453881f2f4eee25..63dec18b739cb432777e686ead387eb0db493150 100644
--- a/RTCP/Cobalt/GPUProc/doc/pipeline-buffers.txt
+++ b/RTCP/Cobalt/GPUProc/doc/pipeline-buffers.txt
@@ -83,7 +83,7 @@ BandPass + Transpose {I/O: weights}
V = [station][channel][sample][pol]
X
-Complex Voltages/Coherent Stokes: [B -> C, trashes A]
+Complex Voltages/Coherent Stokes: [B -> O, trashes D, quantization: trashes E]
-----------------------------------
X [station][channel][sample][pol] [48][4096][48][2] = 144 MiB B
|
@@ -92,32 +92,40 @@ BeamFormer {I/O: weights}
| [channel][sample][tab][pol] [4096][48][tab][2] = 3 MiB/TAB D
V
Transpose
- | [tab][pol][sample][channel] [tab][2][48][4096] = 3 MiB/TAB C
+ | [tab][pol][sample][channel] [tab][2][48][4096] = 3 MiB/TAB O
|
V
-iFFT-64 {inplace}
- | [tab][pol][sample] [tab][2][196608] = 3 MiB/TAB C
+iFFT-64
+ | [tab][pol][sample] [tab][2][196608] = 3 MiB/TAB D
|
V
FFT-shift {inplace}
- | [tab][pol][sample] [tab][2][196608] = 3 MiB/TAB C
+ | [tab][pol][sample] [tab][2][196608] = 3 MiB/TAB D
|
V
FIR-16 (if >1ch) {I/O: history samples}
- | [tab][pol][sample] [tab][2][196608] = 3 MiB/TAB Nch: D
+ | [tab][pol][sample] [tab][2][196608] = 3 MiB/TAB Nch: O
|
V
FFT-16 {inplace} (if >1ch)
- | [tab][pol][sample][channel] [tab][2][12288][16] = 3 MiB/TAB Nch: C
+ | [tab][pol][sample][channel] [tab][2][12288][16] = 3 MiB/TAB Nch: D
|
V
Coherent Stokes
- | [tab][stokes][sample][channel] [tab][4][12288][16] = 0.75 MiB/TAB/Stokes D
+ | [tab][stokes][sample][channel] [tab][4][12288][16] = 0.75 MiB/TAB/Stokes O
+ | (float) Quantization: E
+ V
+Rebitting (quantization)
+ | [tab][stokes][sample][channel] [tab][4][12288][16] = 3/16 MiB/TAB/Stokes O
+ | (short)
+ | [tab][stokes][channel] [tab][4][16] = 1/16 KiB/TAB/Stokes O
+ | (float)
+ | [tab][stokes][channel] [tab][4][16] = 1/16 KiB/TAB/Stokes O
| (float)
V
(output)
-Incoherent Stokes: [B -> B, trashes A]
+Incoherent Stokes: [B -> O, trashes D, with quantization: trashes E]
-----------------------------------
X [station][channel][sample][pol] [48][4096][48][2] = 144 MiB B
|
@@ -132,39 +140,49 @@ FFT-shift {inplace}
| [station][pol][sample] [48][2][196608] = 144 MiB A
V
FIR-16 (if >1ch) {I/O: history samples}
- | [station][pol][sample] [48][2][196608] = 144 MiB Nch: B
+ | [station][pol][sample] [48][2][196608] = 144 MiB Nch: O
|
V
FFT-16 {inplace} (if >1ch)
| [station][pol][sample][channel] [48][2][12288][16] = 144 MiB Nch: A
V
Incoherent Stokes
- | [stokes][sample][channel] [4][12288][16] = 3 MiB B
- V (float)
+ | [stokes][sample][channel] [4][12288][16] = 3 MiB O
+ | (float) Quantization: E
+ V
+Rebitting (quantization)
+ | [stokes][sample][channel] [4][12288][16] = 3/16 MiB/Stokes O
+ | (short)
+ | [stokes][channel] [4][16] = 1/16 KiB/Stokes O
+ | (float)
+ | [stokes][channel] [4][16] = 1/16 KiB/Stokes O
+ | (float)
+ V
(output)
The buffers thus have the following sizes (in MiB) in the various observational modes:
-buffer | bf | corr | commensal
--------+----------+------+--------------
-A | 144 | 120 | 144
-B | 144 | 240 | 240
-C | 3 * #TAB | - | 3 * #TAB
-D | 3 * #TAB | - | 3 * #TAB
-E | - | 240 | 240
+buffer | bf | corr | commensal
+-------+----------------+------+----------------
+A | 144 | 120 | 144
+B | 144 | 240 | 240
+D | 3 * #TAB | - | 3 * #TAB
+E | - | 240 | 240
+O | 3 * #TAB * #BF | - | 3 * #TAB * #BF
#TAB = number of coherent TABs (for one subband)
+#BF = number of beam former pipelines, each pipeline has its own O buffer
The WQ has 1791 MiB GPU memory available, because we have 2 WQ's per GPU core,
and (according to nvidia-smi) we have 3583 MiB GPU memory per core.
BF only:
---------
-Since we can form more than 48 TABs, the size of buffers C and D are determined by the number of TABs.
+Since we can form more than 48 TABs, the size of buffers D and O are determined by the number of TABs.
So, the maximum number of TABs that we can form is: (1791 - 288) MiB / (2 * 3 MiB/TAB) >= 250 TABs.
Corr+BF:
---------
-Since we can form more than 48 TABs, the size of buffers C and D are determined by the number of TABs.
+Since we can form more than 48 TABs, the size of buffers D and O are determined by the number of TABs.
So, the maximum number of TABs that we can form is: (1791 - 624) MiB / (2 * 3 MiB/TAB) >= 194 TABs.
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/BandPassCorrection.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/BandPassCorrection.cu
index d6a27c2ac122568b87344d2945c5743824379f1b..67bd6e57ccbd3b00202d78ce112192c2a02e987c 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/BandPassCorrection.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/BandPassCorrection.cu
@@ -30,8 +30,7 @@
* into account:
* - @c NR_POLARIZATIONS: 2
* - @c NR_STATIONS: > 0
- * - @c NR_CHANNELS_1: a multiple of 16
- * - @c NR_CHANNELS_2: > 0
+ * - @c NR_CHANNELS: > 0
* - @c NR_SAMPLES_PER_CHANNEL: > a multiple of 16
* - @c DO_BANDPASS_CORRECTION: if defined, perform bandpass correction
*/
@@ -46,21 +45,17 @@
#error Precondition violated: NR_STATIONS > 0
#endif
-#if !(NR_CHANNELS_1 > 0 && NR_CHANNELS_1 % 16 == 0)
-#error Precondition violated: NR_CHANNELS_1 > 0 && NR_CHANNELS_1 % 16 == 0
-#endif
-
-#if !(NR_CHANNELS_2 > 0)
-#error Precondition violated: NR_CHANNELS_2 > 0
+#if !(NR_CHANNELS > 0)
+#error Precondition violated: NR_CHANNELS > 0
#endif
#if !(NR_SAMPLES_PER_CHANNEL > 0)
#error Precondition violated: NR_SAMPLES_PER_CHANNEL > 0
#endif
-typedef fcomplex (* OutputDataType)[NR_STATIONS][NR_CHANNELS_1 * NR_CHANNELS_2][NR_SAMPLES_PER_CHANNEL][NR_POLARIZATIONS];
-typedef fcomplex (* InputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_CHANNELS_1][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS_2];
-typedef const float (* BandPassFactorsType)[NR_CHANNELS_1 * NR_CHANNELS_2];
+typedef fcomplex (* OutputDataType)[NR_STATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL][NR_POLARIZATIONS];
+typedef fcomplex (* InputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL];
+typedef const float (* BandPassFactorsType)[NR_CHANNELS];
/**
* This kernel performs on the input data:
@@ -70,17 +65,16 @@ typedef const float (* BandPassFactorsType)[NR_CHANNELS_1 * NR_CHANNELS_2];
* hence it can be fully compensated for.
* - Transpose the data so that the samples for each channel are placed
* consecutively in memory with both polarization next to each other.
- * - Note: This kernel is optimized for performance in dims samples and channel_1
- * Previous version was optimized for channel_2 (still supported)
+ * - Note: This kernel is optimized for performance in dims samples and channel
*
*
* @param[out] correctedDataPtr pointer to output data of ::OutputDataType,
- * a 4D array [station][channels1 * channels2][samples][pol]
+ * a 4D array [station][channels][samples][pol]
* of ::complex (2 complex polarizations)
* @param[in] intputDataPtr pointer to input data;
- * 5D array [station][pol][channels1][samples][channels2]
+ * 4D array [station][pol][channels][samples]
* @param[in] bandPassFactorsPtr pointer to bandpass correction data of
- * ::BandPassFactorsType, a 1D array [channels1 * channels2] of
+ * ::BandPassFactorsType, a 1D array [channels] of
* float, containing bandpass correction factors
*/
extern "C" {
@@ -98,26 +92,24 @@ __global__ void bandPassCorrection( fcomplex * outputDataPtr,
// fasted dims
unsigned sample = blockIdx.x * blockDim.x + threadIdx.x;
- unsigned idx_channel1 = blockIdx.y * blockDim.y + threadIdx.y;
- unsigned chan2 = blockIdx.z * blockDim.z + threadIdx.z;
+ unsigned channel = blockIdx.y * blockDim.y + threadIdx.y;
for (unsigned station = 0; station < NR_STATIONS; ++station)
{
// Read from global memory in the quickest dimension (optimal)
- fcomplex sampleX = (*inputData)[station][0][idx_channel1][sample][chan2];
- fcomplex sampleY = (*inputData)[station][1][idx_channel1][sample][chan2];
- unsigned chan_index = idx_channel1 * NR_CHANNELS_2 + chan2;
+ fcomplex sampleX = (*inputData)[station][0][channel][sample];
+ fcomplex sampleY = (*inputData)[station][1][channel][sample];
#if defined DO_BANDPASS_CORRECTION
- float weight((*bandPassFactors)[chan_index]);
+ float weight((*bandPassFactors)[channel]);
sampleX.x *= weight;
sampleX.y *= weight;
sampleY.x *= weight;
sampleY.y *= weight;
#endif
- (*outputData)[station][chan_index][sample][0] = sampleX;
- (*outputData)[station][chan_index][sample][1] = sampleY;
+ (*outputData)[station][channel][sample][0] = sampleX;
+ (*outputData)[station][channel][sample][1] = sampleY;
}
}
}
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/BeamFormer.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/BeamFormer.cu
index 88a1a4a5ed8356e92a4f451b62549e2a53052594..b4b8137e69be4b4b3e25e4e70d1c9696879013e6 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/BeamFormer.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/BeamFormer.cu
@@ -23,29 +23,30 @@
//# Some defines used to determine the correct way the process the data
#define MAX(A,B) ((A)>(B) ? (A) : (B))
-#define NR_PASSES MAX((NR_STATIONS + 6) / 16, 1) // gives best results on GTX 680
+#define NR_PASSES MAX((NR_OUTPUT_STATIONS + 6) / 16, 1) // gives best results on GTX 680
#ifndef NR_STATIONS_PER_PASS // Allow overriding for testing optimalizations
-#define NR_STATIONS_PER_PASS ((NR_STATIONS + NR_PASSES - 1) / NR_PASSES)
+#define NR_STATIONS_PER_PASS ((NR_OUTPUT_STATIONS + NR_PASSES - 1) / NR_PASSES)
#endif
#if NR_STATIONS_PER_PASS > 32
#error "need more passes to beamform this number of stations"
#endif
#ifdef FLYS_EYE
-# if !(NR_STATIONS == NR_TABS)
-# error Precondition violated: NR_STATIONS == NR_TABS
+# if !(NR_INPUT_STATIONS == NR_TABS)
+# error Precondition violated: NR_INPUT_STATIONS == NR_TABS
# endif
#endif
+#define STATION_INDEX(s) (stationIndices[s])
#define DELAY_INDEX(s) (delayIndices[s])
//# Typedefs used to map input data on arrays
-typedef double (*DelaysType)[NR_SAPS][NR_DELAYS][NR_TABS];
+typedef double (*DelaysType)[1][NR_DELAYS][NR_TABS];
#ifdef FLYS_EYE
-typedef float2 (*BandPassCorrectedType)[NR_STATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL][NR_POLARIZATIONS];
+typedef float2 (*BandPassCorrectedType)[NR_INPUT_STATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL][NR_POLARIZATIONS];
#else
-typedef float4 (*BandPassCorrectedType)[NR_STATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL];
+typedef float4 (*BandPassCorrectedType)[NR_INPUT_STATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL];
#endif
typedef float2 (*ComplexVoltagesType)[NR_CHANNELS][NR_SAMPLES_PER_CHANNEL][NR_TABS][NR_POLARIZATIONS];
@@ -64,11 +65,11 @@ typedef float2 (*ComplexVoltagesType)[NR_CHANNELS][NR_SAMPLES_PER_CHANNEL][NR_T
* Pre-processor input symbols (some are tied to the execution configuration)
* Symbol | Valid Values | Description
* ----------------------- | ----------------------- | -----------
- * NR_STATIONS | >= 1 | number of antenna fields
+ * NR_INPUT_STATIONS | >= 1 | number of antenna fields in the input
+ * NR_OUTPUT_STATIONS | >= 1 | number of antenna fields selected for beamforming
* NR_DELAYS | >= 1 | number of delays = number of antenna fields in the observation
* NR_SAMPLES_PER_CHANNEL | >= 1 | number of input samples per channel
* NR_CHANNELS | >= 1 | number of frequency channels per subband
- * NR_SAPS | >= 1 && > sap | number of Sub-Array Pointings
* NR_TABS | >= 1 | number of Tied Array Beams (old name: pencil beams) to create
* SUBBAND_BANDWIDTH | double, multiple of NR_CHANNELS | Bandwidth of a subband in Hz
* NR_STATIONS_PER_PASS | 1 >= && <= 32 | Set to overide default: Parallelization parameter, controls the number stations to beamform in a single pass over the input data.
@@ -80,6 +81,7 @@ typedef float2 (*ComplexVoltagesType)[NR_CHANNELS][NR_SAMPLES_PER_CHANNEL][NR_T
*/
extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
const void *samplesPtr,
+ const unsigned *stationIndices, // lookup index for stations to use in samplesPtr
const void *delaysPtr,
const unsigned *delayIndices, // lookup index for stations to use in delaysPtr
double subbandFrequency,
@@ -96,7 +98,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
if (tab < NR_TABS) {
unsigned t = blockIdx.x;
- (*complexVoltages)[channel][t][tab][pol] = (*samples)[tab][channel][t][pol];
+ (*complexVoltages)[channel][t][tab][pol] = (*samples)[STATION_INDEX(tab)][channel][t][pol];
}
#else
@@ -116,7 +118,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#pragma unroll
for (unsigned first_station = 0; // Step over data with NR_STATIONS_PER_PASS stride
- first_station < NR_STATIONS;
+ first_station < NR_OUTPUT_STATIONS;
first_station += NR_STATIONS_PER_PASS)
{ // this for loop spans the whole file
@@ -127,9 +129,9 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
unsigned tab_or_zero = tab < NR_TABS ? tab : 0;
#if NR_STATIONS_PER_PASS >= 1
- fcomplex weight_00; // assign the weights to register variables
- if (first_station + 0 < NR_STATIONS) { // Number of station might be larger then 32:
- // We then do multiple passes to span all stations
+ fcomplex weight_00; // assign the weights to register variables
+ if (first_station + 0 < NR_OUTPUT_STATIONS) { // Number of station might be larger then 32:
+ // We then do multiple passes to span all stations
double delay = (*delays)[sap][DELAY_INDEX(first_station + 0)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_00 = make_float2(weight.x, weight.y);
@@ -138,7 +140,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
// Loop unrolling allows usage of registers for weights
#if NR_STATIONS_PER_PASS >= 2
fcomplex weight_01;
- if (first_station + 1 < NR_STATIONS) {
+ if (first_station + 1 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 1)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_01 = make_float2(weight.x, weight.y);
@@ -147,7 +149,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 3
fcomplex weight_02;
- if (first_station + 2 < NR_STATIONS) {
+ if (first_station + 2 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 2)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_02 = make_float2(weight.x, weight.y);
@@ -156,7 +158,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 4
fcomplex weight_03;
- if (first_station + 3 < NR_STATIONS) {
+ if (first_station + 3 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 3)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_03 = make_float2(weight.x, weight.y);
@@ -165,7 +167,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 5
fcomplex weight_04;
- if (first_station + 4 < NR_STATIONS) {
+ if (first_station + 4 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 4)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_04 = make_float2(weight.x, weight.y);
@@ -174,7 +176,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 6
fcomplex weight_05;
- if (first_station + 5 < NR_STATIONS) {
+ if (first_station + 5 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 5)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_05 = make_float2(weight.x, weight.y);
@@ -183,7 +185,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 7
fcomplex weight_06;
- if (first_station + 6 < NR_STATIONS) {
+ if (first_station + 6 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 6)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_06 = make_float2(weight.x, weight.y);
@@ -192,7 +194,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 8
fcomplex weight_07;
- if (first_station + 7 < NR_STATIONS) {
+ if (first_station + 7 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 7)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_07 = make_float2(weight.x, weight.y);
@@ -201,7 +203,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 9
fcomplex weight_08;
- if (first_station + 8 < NR_STATIONS) {
+ if (first_station + 8 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 8)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_08 = make_float2(weight.x, weight.y);
@@ -210,7 +212,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 10
fcomplex weight_09;
- if (first_station + 9 < NR_STATIONS) {
+ if (first_station + 9 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 9)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_09 = make_float2(weight.x, weight.y);
@@ -219,7 +221,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 11
fcomplex weight_10;
- if (first_station + 10 < NR_STATIONS) {
+ if (first_station + 10 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 10)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_10 = make_float2(weight.x, weight.y);
@@ -228,7 +230,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 12
fcomplex weight_11;
- if (first_station + 11 < NR_STATIONS) {
+ if (first_station + 11 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 11)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_11 = make_float2(weight.x, weight.y);
@@ -237,7 +239,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 13
fcomplex weight_12;
- if (first_station + 12 < NR_STATIONS) {
+ if (first_station + 12 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 12)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_12 = make_float2(weight.x, weight.y);
@@ -246,7 +248,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 14
fcomplex weight_13;
- if (first_station + 13 < NR_STATIONS) {
+ if (first_station + 13 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 13)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_13 = make_float2(weight.x, weight.y);
@@ -255,7 +257,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 15
fcomplex weight_14;
- if (first_station + 14 < NR_STATIONS) {
+ if (first_station + 14 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 14)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_14 = make_float2(weight.x, weight.y);
@@ -264,7 +266,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 16
fcomplex weight_15;
- if (first_station + 15 < NR_STATIONS) {
+ if (first_station + 15 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 15)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_15 = make_float2(weight.x, weight.y);
@@ -273,7 +275,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 17
fcomplex weight_16;
- if (first_station + 16 < NR_STATIONS) {
+ if (first_station + 16 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 16)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_16 = make_float2(weight.x, weight.y);
@@ -282,7 +284,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 18
fcomplex weight_17;
- if (first_station + 17 < NR_STATIONS) {
+ if (first_station + 17 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 17)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_17 = make_float2(weight.x, weight.y);
@@ -291,7 +293,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 19
fcomplex weight_18;
- if (first_station + 18 < NR_STATIONS) {
+ if (first_station + 18 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 18)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_18 = make_float2(weight.x, weight.y);
@@ -300,7 +302,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 20
fcomplex weight_19;
- if (first_station + 19 < NR_STATIONS) {
+ if (first_station + 19 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 19)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_19 = make_float2(weight.x, weight.y);
@@ -309,7 +311,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 21
fcomplex weight_20;
- if (first_station + 20 < NR_STATIONS) {
+ if (first_station + 20 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 20)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_20 = make_float2(weight.x, weight.y);
@@ -318,7 +320,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 22
fcomplex weight_21;
- if (first_station + 21 < NR_STATIONS) {
+ if (first_station + 21 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 21)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_21 = make_float2(weight.x, weight.y);
@@ -327,7 +329,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 23
fcomplex weight_22;
- if (first_station + 22 < NR_STATIONS) {
+ if (first_station + 22 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 22)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_22 = make_float2(weight.x, weight.y);
@@ -336,7 +338,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 24
fcomplex weight_23;
- if (first_station + 23 < NR_STATIONS) {
+ if (first_station + 23 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 23)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_23 = make_float2(weight.x, weight.y);
@@ -345,7 +347,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 25
fcomplex weight_24;
- if (first_station + 24 < NR_STATIONS) {
+ if (first_station + 24 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 24)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_24 = make_float2(weight.x, weight.y);
@@ -354,7 +356,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 26
fcomplex weight_25;
- if (first_station + 25 < NR_STATIONS) {
+ if (first_station + 25 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 25)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_25 = make_float2(weight.x, weight.y);
@@ -363,7 +365,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 27
fcomplex weight_26;
- if (first_station + 26 < NR_STATIONS) {
+ if (first_station + 26 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 26)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_26 = make_float2(weight.x, weight.y);
@@ -372,7 +374,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 28
fcomplex weight_27;
- if (first_station + 27 < NR_STATIONS) {
+ if (first_station + 27 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 27)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_27 = make_float2(weight.x, weight.y);
@@ -381,7 +383,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 29
fcomplex weight_28;
- if (first_station + 28 < NR_STATIONS) {
+ if (first_station + 28 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 28)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_28 = make_float2(weight.x, weight.y);
@@ -390,7 +392,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 30
fcomplex weight_29;
- if (first_station + 29 < NR_STATIONS) {
+ if (first_station + 29 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 29)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_29 = make_float2(weight.x, weight.y);
@@ -399,7 +401,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 31
fcomplex weight_30;
- if (first_station + 30 < NR_STATIONS) {
+ if (first_station + 30 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 30)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_30 = make_float2(weight.x, weight.y);
@@ -408,7 +410,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 32
fcomplex weight_31;
- if (first_station + 31 < NR_STATIONS) {
+ if (first_station + 31 < NR_OUTPUT_STATIONS) {
double delay = (*delays)[sap][DELAY_INDEX(first_station + 31)][tab_or_zero];
dcomplex weight = dphaseShift(frequency, delay);
weight_31 = make_float2(weight.x, weight.y);
@@ -419,7 +421,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
for (unsigned time = 0; time < NR_SAMPLES_PER_CHANNEL; time += 16)
{
// precalculate the upper limit and only do the loop for a valid range
- unsigned laststation = min(NR_STATIONS_PER_PASS, NR_STATIONS - first_station);
+ unsigned laststation = min(NR_STATIONS_PER_PASS, NR_OUTPUT_STATIONS - first_station);
// Optimized memory transfer: Threads load from memory in parallel
for (unsigned i = threadIdx.x + NR_POLARIZATIONS * threadIdx.y;
i < laststation * 16;
@@ -429,7 +431,7 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
unsigned s = i / 16;
if (NR_SAMPLES_PER_CHANNEL % 16 == 0 || time + t < NR_SAMPLES_PER_CHANNEL)
- _local.samples4[0][i] = (*samples)[first_station + s][channel][time + t];
+ _local.samples4[0][i] = (*samples)[STATION_INDEX(first_station + s)][channel][time + t];
}
__syncthreads();
@@ -448,14 +450,14 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
// Calculate the weighted complex sum of the samples
#if NR_STATIONS_PER_PASS >= 1
- if (first_station + 1 <= NR_STATIONS) { // Remember that the number of stations might not be a multiple of 32. Skip if station does not exist
+ if (first_station + 1 <= NR_OUTPUT_STATIONS) { // Remember that the number of stations might not be a multiple of 32. Skip if station does not exist
fcomplex sample = _local.samples[ 0][t][pol];
sum = sum + weight_00 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 2
- if (first_station + 2 <= NR_STATIONS) {
+ if (first_station + 2 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 1][t][pol];
sum = sum + weight_01 * sample;
}
@@ -463,210 +465,210 @@ extern "C" __global__ void beamFormer( void *complexVoltagesPtr,
#if NR_STATIONS_PER_PASS >= 3
- if (first_station + 3 <= NR_STATIONS) {
+ if (first_station + 3 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 2][t][pol];
sum = sum + weight_02 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 4
- if (first_station + 4 <= NR_STATIONS) {
+ if (first_station + 4 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 3][t][pol];
sum = sum + weight_03 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 5
- if (first_station + 5 <= NR_STATIONS) {
+ if (first_station + 5 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 4][t][pol];
sum = sum + weight_04 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 6
- if (first_station + 6 <= NR_STATIONS) {
+ if (first_station + 6 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 5][t][pol];
sum = sum + weight_05 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 7
- if (first_station + 7 <= NR_STATIONS) {
+ if (first_station + 7 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 6][t][pol];
sum = sum + weight_06 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 8
- if (first_station + 8 <= NR_STATIONS) {
+ if (first_station + 8 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 7][t][pol];
sum = sum + weight_07 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 9
- if (first_station + 9 <= NR_STATIONS) {
+ if (first_station + 9 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 8][t][pol];
sum = sum + weight_08 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 10
- if (first_station + 10 <= NR_STATIONS) {
+ if (first_station + 10 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[ 9][t][pol];
sum = sum + weight_09 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 11
- if (first_station + 11 <= NR_STATIONS) {
+ if (first_station + 11 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[10][t][pol];
sum = sum + weight_10 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 12
- if (first_station + 12 <= NR_STATIONS) {
+ if (first_station + 12 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[11][t][pol];
sum = sum + weight_11 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 13
- if (first_station + 13 <= NR_STATIONS) {
+ if (first_station + 13 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[12][t][pol];
sum = sum + weight_12 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 14
- if (first_station + 14 <= NR_STATIONS) {
+ if (first_station + 14 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[13][t][pol];
sum = sum + weight_13 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 15
- if (first_station + 15 <= NR_STATIONS) {
+ if (first_station + 15 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[14][t][pol];
sum = sum + weight_14 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 16
- if (first_station + 16 <= NR_STATIONS) {
+ if (first_station + 16 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[15][t][pol];
sum = sum + weight_15 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 17
- if (first_station + 17 <= NR_STATIONS) {
+ if (first_station + 17 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[16][t][pol];
sum = sum + weight_16 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 18
- if (first_station + 18 <= NR_STATIONS) {
+ if (first_station + 18 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[17][t][pol];
sum = sum + weight_17 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 19
- if (first_station + 19 <= NR_STATIONS) {
+ if (first_station + 19 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[18][t][pol];
sum = sum + weight_18 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 20
- if (first_station + 20 <= NR_STATIONS) {
+ if (first_station + 20 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[19][t][pol];
sum = sum + weight_19 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 21
- if (first_station + 21 <= NR_STATIONS) {
+ if (first_station + 21 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[20][t][pol];
sum = sum + weight_20 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 22
- if (first_station + 22 <= NR_STATIONS) {
+ if (first_station + 22 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[21][t][pol];
sum = sum + weight_21 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 23
- if (first_station + 23 <= NR_STATIONS) {
+ if (first_station + 23 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[22][t][pol];
sum = sum + weight_22 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 24
- if (first_station + 24 <= NR_STATIONS) {
+ if (first_station + 24 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[23][t][pol];
sum = sum + weight_23 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 25
- if (first_station + 25 <= NR_STATIONS) {
+ if (first_station + 25 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[24][t][pol];
sum = sum + weight_24 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 26
- if (first_station + 26 <= NR_STATIONS) {
+ if (first_station + 26 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[25][t][pol];
sum = sum + weight_25 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 27
- if (first_station + 27 <= NR_STATIONS) {
+ if (first_station + 27 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[26][t][pol];
sum = sum + weight_26 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 28
- if (first_station + 28 <= NR_STATIONS) {
+ if (first_station + 28 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[27][t][pol];
sum = sum + weight_27 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 29
- if (first_station + 29 <= NR_STATIONS) {
+ if (first_station + 29 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[28][t][pol];
sum = sum + weight_28 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 30
- if (first_station + 30 <= NR_STATIONS) {
+ if (first_station + 30 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[29][t][pol];
sum = sum + weight_29 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 31
- if (first_station + 31 <= NR_STATIONS) {
+ if (first_station + 31 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[30][t][pol];
sum = sum + weight_30 * sample;
}
#endif
#if NR_STATIONS_PER_PASS >= 32
- if (first_station + 32 <= NR_STATIONS) {
+ if (first_station + 32 <= NR_OUTPUT_STATIONS) {
fcomplex sample = _local.samples[31][t][pol];
sum = sum + weight_31 * sample;
}
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/CoherentStokes.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/CoherentStokes.cu
index ce60c3dc1a9e08cd39ffe4cd88254c62c1937353..36671dda52b99795637c0f9c4946a0ae79c55cd1 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/CoherentStokes.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/CoherentStokes.cu
@@ -51,8 +51,65 @@
typedef float2 (*InputDataType)[NR_TABS][NR_POLARIZATIONS][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS];
//4D output array of stokes values. Each sample contains 1 or 4 stokes
-//paramters. For each tab, there are NR_COHERENT_STOKES timeseries of channels
+//parameters. For each tab, there are NR_COHERENT_STOKES timeseries of channels
+#if defined(OUTPUT_ORDER_SAMPLES_CHANNELS)
typedef float (*OutputDataType)[NR_TABS][NR_COHERENT_STOKES][NR_SAMPLES_PER_CHANNEL / TIME_INTEGRATION_FACTOR][NR_CHANNELS];
+#elif defined(OUTPUT_ORDER_CHANNELS_SAMPLES)
+typedef float (*OutputDataType)[NR_TABS][NR_COHERENT_STOKES][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL / TIME_INTEGRATION_FACTOR];
+#else
+#error Unknown layout for OutputDataType
+#endif
+
+inline __device__ void write_complex_voltages(
+ OutputDataType output,
+ unsigned tab_idx,
+ unsigned sample_idx,
+ unsigned channel_idx,
+ float4 stokes)
+{
+# if OUTPUT_ORDER_SAMPLES_CHANNELS
+ (*output)[tab_idx][0][sample_idx][channel_idx] = stokes.x;
+ (*output)[tab_idx][1][sample_idx][channel_idx] = stokes.y;
+ (*output)[tab_idx][2][sample_idx][channel_idx] = stokes.z;
+ (*output)[tab_idx][3][sample_idx][channel_idx] = stokes.w;
+# elif OUTPUT_ORDER_CHANNELS_SAMPLES
+ (*output)[tab_idx][0][channel_idx][sample_idx] = stokes.x;
+ (*output)[tab_idx][1][channel_idx][sample_idx] = stokes.y;
+ (*output)[tab_idx][2][channel_idx][sample_idx] = stokes.z;
+ (*output)[tab_idx][3][channel_idx][sample_idx] = stokes.w;
+# endif
+}
+
+inline __device__ void write_stokes(
+ OutputDataType output,
+ unsigned tab_idx,
+ unsigned sample_idx,
+ unsigned channel_idx,
+ float stokesI
+ #if NR_COHERENT_STOKES == 4
+ ,
+ float stokesQ,
+ float stokesU,
+ float stokesV
+ #endif
+ )
+{
+# if OUTPUT_ORDER_SAMPLES_CHANNELS
+ (*output)[tab_idx][0][sample_idx][channel_idx] = stokesI;
+# if NR_COHERENT_STOKES == 4
+ (*output)[tab_idx][1][sample_idx][channel_idx] = stokesQ;
+ (*output)[tab_idx][2][sample_idx][channel_idx] = stokesU;
+ (*output)[tab_idx][3][sample_idx][channel_idx] = stokesV;
+# endif
+# elif OUTPUT_ORDER_CHANNELS_SAMPLES
+ (*output)[tab_idx][0][channel_idx][sample_idx] = stokesI;
+# if NR_COHERENT_STOKES == 4
+ (*output)[tab_idx][1][channel_idx][sample_idx] = stokesQ;
+ (*output)[tab_idx][2][channel_idx][sample_idx] = stokesU;
+ (*output)[tab_idx][3][channel_idx][sample_idx] = stokesV;
+# endif
+# endif
+}
/*!
* Computes the Stokes I or IQUV, or outputs the 4 complex voltages (XrXiYrYi).
@@ -181,17 +238,15 @@ extern "C" __global__ void coherentStokes(OutputDataType output,
}
# if COMPLEX_VOLTAGES == 1
- (*output)[tab_idx][0][write_idx][channel_idx] = stokes.x;
- (*output)[tab_idx][1][write_idx][channel_idx] = stokes.y;
- (*output)[tab_idx][2][write_idx][channel_idx] = stokes.z;
- (*output)[tab_idx][3][write_idx][channel_idx] = stokes.w;
+ write_complex_voltages(output, tab_idx, write_idx, channel_idx, stokes);
# else
- (*output)[tab_idx][0][write_idx][channel_idx] = stokesI;
# if NR_COHERENT_STOKES == 4
- (*output)[tab_idx][1][write_idx][channel_idx] = stokesQ;
- (*output)[tab_idx][2][write_idx][channel_idx] = 2.0f * halfStokesU;
- (*output)[tab_idx][3][write_idx][channel_idx] = 2.0f * halfStokesV;
-# endif
+ float stokesU = 2.0f * halfStokesU;
+ float stokesV = 2.0f * halfStokesV;
+ write_stokes(output, tab_idx, write_idx, channel_idx, stokesI, stokesQ, stokesU, stokesV);
+# else
+ write_stokes(output, tab_idx, write_idx, channel_idx, stokesI);
+# endif
# endif
}
}
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/Correlator.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/Correlator.cu
index 8ce03145e646d9ca9763fb6e964a6b681d8fdb93..efc1c9e88ee341662ce6b0f7ea0e8ce4556fdb1e 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/Correlator.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/Correlator.cu
@@ -42,6 +42,8 @@
#error Precondition violated: NR_SAMPLES_PER_INTEGRATION % BLOCK_SIZE == 0
#endif
+typedef unsigned int uint;
+
typedef float2 fcomplex;
typedef float4 fcomplex2;
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/DelayAndBandPass.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/DelayAndBandPass.cu
index 3ae289a1258685063d07fcc77493c790ae049ee9..abbaa1d8ce3b6ea602275403a3578cb514f7c5a8 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/DelayAndBandPass.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/DelayAndBandPass.cu
@@ -39,7 +39,6 @@
* - @c NR_SAMPLES_PER_SUBBAND: a multiple of 16
* - if @c NR_CHANNELS > 1 (input data is in floating point format):
* - @c NR_SAMPLES_PER_CHANNEL: a multiple of 16
-* - @c NR_SAPS: > 0
* - @c NR_POLARIZATIONS: 2
* - @c SUBBAND_BANDWIDTH: a multiple of @c NR_CHANNELS
*
@@ -52,6 +51,10 @@
#include "IntToFloat.cuh"
+// M_PI is needed for NVRTC compilation
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
#if NR_CHANNELS == 1
//# #chnl==1 && BANDPASS_CORRECTION is rejected on the CPU early, (TODO)
@@ -66,7 +69,7 @@ typedef fcomplex(*OutputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_CHANNELS][N
#endif
typedef fcomplex(*InputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS];
-typedef const double(*DelaysType)[NR_SAPS][NR_DELAYS][NR_POLARIZATIONS]; // 2 Polarizations; in seconds
+typedef const double(*DelaysType)[1][NR_DELAYS][NR_POLARIZATIONS]; // 2 Polarizations; in seconds
typedef const double2(*Phase0sType)[NR_STATIONS]; // 2 Polarizations; in radians
typedef const float(*BandPassFactorsType)[NR_CHANNELS];
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/FIR_Filter.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/FIR_Filter.cu
index 7e849e38f6fa3d2b7e74c135cb3980a2ed462614..a0cea1c64a1028625f966f18fbd5faa33faa9ca5 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/FIR_Filter.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/FIR_Filter.cu
@@ -28,10 +28,6 @@
#error Precondition violated: NR_TAPS == 16
#endif
-#if !(NR_SUBBANDS > 0)
-#error Precondition violated: NR_SUBBANDS > 0
-#endif
-
#if !(NR_SAMPLES_PER_CHANNEL > 0 && NR_SAMPLES_PER_CHANNEL % NR_TAPS == 0)
#error Precondition violated: NR_SAMPLES_PER_CHANNEL > 0 && NR_SAMPLES_PER_CHANNEL % NR_TAPS == 0
#endif
@@ -92,7 +88,7 @@ inline __device__ float convertIntToFloat(float x)
}
#endif
-typedef SampleType (*HistoryDataType)[NR_SUBBANDS][NR_STABS][NR_TAPS - 1][NR_CHANNELS][NR_POLARIZATIONS * COMPLEX];
+typedef SampleType (*HistoryDataType)[1][NR_STABS][NR_TAPS - 1][NR_CHANNELS][NR_POLARIZATIONS * COMPLEX];
typedef float (*FilteredDataType)[NR_STABS][NR_POLARIZATIONS][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS][COMPLEX];
typedef const float (*WeightsType)[NR_CHANNELS][NR_TAPS];
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokes.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokes.cu
index 312972ec970549ed0bcdad30833aa9560ef95e29..5798bca1155f07a52f986505cec07a3f4e389746 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokes.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokes.cu
@@ -51,6 +51,8 @@
#error Precondition violated: NR_STATIONS >= 1
#endif
+typedef unsigned int uint;
+
// 5-D input array of complex samples. Note that, actually, the data is 4-D
// (<tt>[station][pol][time][channel]</tt>). The 5th dimension is just a
// convenience to make striding through the array in the time domain (used for
@@ -65,10 +67,54 @@ typedef float2 (*InputDataType)
// 3-D output array of incoherent stokes values. Its dimensions are
// <tt>[stokes][time][channels]</tt>, where <tt>[stokes]</tt> can be either 1
// (Stokes \e I), or 4 (Stokes \e I, \e Q, \e U, and \e V).
+#if defined(OUTPUT_ORDER_SAMPLES_CHANNELS)
typedef float (*OutputDataType)
[NR_INCOHERENT_STOKES]
[NR_SAMPLES_PER_CHANNEL / TIME_INTEGRATION_FACTOR]
[NR_CHANNELS];
+#elif defined(OUTPUT_ORDER_CHANNELS_SAMPLES)
+typedef float (*OutputDataType)
+[NR_INCOHERENT_STOKES]
+[NR_CHANNELS]
+[NR_SAMPLES_PER_CHANNEL / TIME_INTEGRATION_FACTOR];
+#else
+#error Unknown layout for OutputDataType
+#endif
+
+inline __device__ void write_stokes(
+ OutputDataType output,
+ unsigned sample_idx,
+ unsigned channel_idx,
+ float stokesI)
+{
+# if defined(OUTPUT_ORDER_SAMPLES_CHANNELS)
+ (*output)[0][sample_idx][channel_idx] = stokesI;
+# elif defined(OUTPUT_ORDER_CHANNELS_SAMPLES)
+ (*output)[0][channel_idx][sample_idx] = stokesI;
+# endif
+}
+
+inline __device__ void write_stokes(
+ OutputDataType output,
+ unsigned sample_idx,
+ unsigned channel_idx,
+ float stokesI,
+ float stokesQ,
+ float stokesU,
+ float stokesV)
+{
+# if defined(OUTPUT_ORDER_SAMPLES_CHANNELS)
+ (*output)[0][sample_idx][channel_idx] = stokesI;
+ (*output)[1][sample_idx][channel_idx] = stokesQ;
+ (*output)[2][sample_idx][channel_idx] = stokesU;
+ (*output)[3][sample_idx][channel_idx] = stokesV;
+# elif defined(OUTPUT_ORDER_CHANNELS_SAMPLES)
+ (*output)[0][channel_idx][sample_idx] = stokesI;
+ (*output)[1][channel_idx][sample_idx] = stokesQ;
+ (*output)[2][channel_idx][sample_idx] = stokesU;
+ (*output)[3][channel_idx][sample_idx] = stokesV;
+# endif
+}
// Compute the \e incoherent Stokes parameters. The incoherent Stokes
// parameters are calculated by adding the Stokes parameters of the station
@@ -119,11 +165,12 @@ extern "C" __global__ void incoherentStokes(OutputDataType output,
}
}
- (*output)[0][time][channel] = stokesI;
-#if NR_INCOHERENT_STOKES == 4
- (*output)[1][time][channel] = stokesQ;
- (*output)[2][time][channel] = 2.0f * halfStokesU;
- (*output)[3][time][channel] = 2.0f * halfStokesV;
-#endif
+# if NR_INCOHERENT_STOKES == 4
+ float stokesU = 2.0f * halfStokesU;
+ float stokesV = 2.0f * halfStokesV;
+ write_stokes(output, time, channel, stokesI, stokesQ, stokesU, stokesV);
+# else
+ write_stokes(output, time, channel, stokesI);
+# endif
}
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokesTranspose.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokesTranspose.cu
index 683958951c09690aa058ef64a961253426559096..8e31201a0b710a34af37841490003bb9feee3921 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokesTranspose.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/IncoherentStokesTranspose.cu
@@ -30,8 +30,16 @@
#error Precondition violated: NR_SAMPLES_PER_CHANNEL > 0
#endif
-#if !(NR_STATIONS > 0)
-#error Precondition violated: NR_STATIONS > 0
+#if !(NR_INPUT_STATIONS > 0)
+#error Precondition violated: NR_INPUT_STATIONS > 0
+#endif
+
+#if !(NR_OUTPUT_STATIONS > 0)
+#error Precondition violated: NR_OUTPUT_STATIONS > 0
+#endif
+
+#if !(NR_INPUT_STATIONS >= NR_OUTPUT_STATIONS)
+#error Precondition violated: NR_INPUT_STATIONS >= NR_OUTPUT_STATIONS
#endif
#if !(TILE_SIZE > 0 && TILE_SIZE <= 32)
@@ -41,11 +49,11 @@
// 3-D input data array of band-pass corrected data. Note that, actually, the
// data is 4-D (<tt>[station][channel][time][pol]</tt>), but the 4th dimension
// has been squashed into a single float4 (i.e., two complex polarizations).
-typedef float4 (*InputDataType)[NR_STATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL];
+typedef float4 (*InputDataType)[NR_INPUT_STATIONS][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL];
// 4-D output data array of band-pass corrected data. The output data
// (<tt>[station][pol][time][channel]</tt>) can be fed into an inverse FFT.
-typedef float2 (*OutputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS];
+typedef float2 (*OutputDataType)[NR_OUTPUT_STATIONS][NR_POLARIZATIONS][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS];
// Transpose the output of the band-pass correction kernel into a format
@@ -66,6 +74,8 @@ typedef float2 (*OutputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_SAMPLES_PER_C
// writing outside of the bounds of the global memory arrays.
//
// \param[in] input 4-D input array of band-pass corrected data.
+// \param[in] stationIndices 1-D input array of input stations to use for
+// each output station
// \param[out] output 4-D output array of band-pass corrected data that can be
// fed into an inverse FFT.
//
@@ -75,8 +85,10 @@ typedef float2 (*OutputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_SAMPLES_PER_C
// NR_CHANNELS | > 0 | number of channels per subband
// NR_POLARIZATIONS | 2 | number of polarizations
// NR_SAMPLES_PER_CHANNEL | > 0 | number of input samples per channel
-// NR_STATIONS | > 0 | number of stations
+// NR_INPUT_STATIONS | > 0 | number of input stations
+// NR_OUTPUT_STATIONS | > 0 | number of output stations
// TILE_SIZE | > 0 and <= 32 | size of shared memory tile
+// DO_STATIONSUBSET | 0 or 1 | use the stationIndices input array
//
// Execution configuration:
//
@@ -88,17 +100,27 @@ typedef float2 (*OutputDataType)[NR_STATIONS][NR_POLARIZATIONS][NR_SAMPLES_PER_C
// \c TILE_SIZE
extern "C"
__global__ void transpose(OutputDataType output,
- const InputDataType input)
+ const InputDataType input,
+ const unsigned *stationIndices)
{
unsigned time, channel;
__shared__ float4 tmp[TILE_SIZE][TILE_SIZE + 1];
- for (int station = 0; station < NR_STATIONS; station++) {
+ for (int station = 0; station < NR_OUTPUT_STATIONS; station++) {
+
+#ifdef DO_STATIONSUBSET
+ uint station_in = stationIndices[station];
+ uint station_out = station;
+#else
+ uint station_in = station;
+ uint station_out = station;
+#endif
+
time = blockIdx.x * blockDim.x + threadIdx.x;
channel = blockIdx.y * blockDim.y + threadIdx.y;
if (channel < NR_CHANNELS && time < NR_SAMPLES_PER_CHANNEL) {
- tmp[threadIdx.y][threadIdx.x] = (*input)[station][channel][time];
+ tmp[threadIdx.y][threadIdx.x] = (*input)[station_in][channel][time];
}
__syncthreads();
@@ -106,8 +128,8 @@ __global__ void transpose(OutputDataType output,
channel = blockIdx.y * blockDim.y + threadIdx.x;
if (channel < NR_CHANNELS && time < NR_SAMPLES_PER_CHANNEL) {
float4 sample = tmp[threadIdx.x][threadIdx.y];
- (*output)[station][0][time][channel] = make_float2(sample.x, sample.y);
- (*output)[station][1][time][channel] = make_float2(sample.z, sample.w);
+ (*output)[station_out][0][time][channel] = make_float2(sample.x, sample.y);
+ (*output)[station_out][1][time][channel] = make_float2(sample.z, sample.w);
}
__syncthreads();
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/IntToFloat.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/IntToFloat.cu
index f0a8fb035c35727cad3470df3cdd702eb4dd2af4..372143f1ab396afc764a1cd4612fd0bed2e6aa34 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/IntToFloat.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/IntToFloat.cu
@@ -31,6 +31,8 @@ typedef short2 SampleType;
#error unsupported NR_BITS_PER_SAMPLE: must be 4, 8, or 16
#endif
+typedef unsigned int uint;
+
#if NR_BITS_PER_SAMPLE == 4
#define REAL(sample) extractRI(sample, false)
#define IMAG(sample) extractRI(sample, true)
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/QuantizeOutput.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/QuantizeOutput.cu
new file mode 100644
index 0000000000000000000000000000000000000000..b8064dc8574078fe75a7507754fbdf8fa5b9e470
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/QuantizeOutput.cu
@@ -0,0 +1,365 @@
+//# QuantizeOutput.cu:
+//# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
+//# Oude Hoogeveensedijk 4, 7991 PD Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id: QuantizeOutput.cu 29617 2014-06-23 08:08:41Z mol $
+
+#include <stdint.h>
+
+#include <cooperative_groups.h>
+
+namespace cg = cooperative_groups;
+
+#if !(NR_CHANNELS >= 1)
+#error Precondition violated: NR_CHANNELS >= 1
+#endif
+
+#if !(NR_COHERENT_STOKES == 1 || NR_COHERENT_STOKES == 4)
+#error Precondition violated: NR_COHERENT_STOKES == 1 || NR_COHERENT_STOKES == 4
+#endif
+
+#if !(NR_SAMPLES_PER_CHANNEL > 0)
+#error Precondition violated: NR_SAMPLES_PER_CHANNEL > 0
+#endif
+
+#if !(NR_TABS >= 1)
+#error Precondition violated: NR_TABS >= 1
+#endif
+
+#if (NR_QUANTIZE_BITS == 16)
+#define UINT uint16_t
+#define INT int16_t
+#elif (NR_QUANTIZE_BITS == 8)
+#define UINT uint8_t
+#define INT int8_t
+#else
+#error Precondition violated: invalid value for NR_QUANTIZE_BITS
+#endif
+
+// A shared memory buffer is used to cache the input data
+// for a single tab, stokes parameter and channel:
+// size = 12288 * sizeof(float) / 1024 = 48 Kb
+// This size matches the amount of shared memory available
+// on CUDA 3.0 capable devices onwards and should therefore
+// not be exceeded.
+// In case NR_SAMPLES_PER_CHANNEL > BATCH_SIZE, the input
+// is processed in batches.
+#define MAX_BATCH_SIZE 6144
+#define BATCH_SIZE NR_SAMPLES_PER_CHANNEL < MAX_BATCH_SIZE ? \
+ NR_SAMPLES_PER_CHANNEL : MAX_BATCH_SIZE
+
+// 4D input array of stokes values. Each sample contains 1 or 4 stokes
+// parameters. For each tab, there are NR_COHERENT_STOKES timeseries of channels
+typedef float (*InputDataType)[NR_TABS][NR_COHERENT_STOKES][NR_CHANNELS][NR_SAMPLES_PER_CHANNEL];
+
+// The output consists of three arrays stored in one buffer:
+// - scales[NR_TABS][NR_COHERENT_STOKES][NR_CHANNELS]
+// - offsets[NR_TABS][NR_COHERENT_STOKES][NR_CHANNELS]
+// - values[NR_TABS][NR_COHERENT_STOKES][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS]
+// The scales and offsets are floating-point numbers:
+typedef float FloatArray[NR_TABS][NR_COHERENT_STOKES][NR_CHANNELS];
+// The values are signed integer or unsigned integers with NR_QUANTIZE_BITS bits:
+typedef INT IntArray[NR_TABS][NR_COHERENT_STOKES][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS];
+// when writing unsigned integers, cast the array to an unsigned integer pointer.
+
+/*!
+ * Copy n elements from src to dst, with the possibility
+ * to specify input and output strides.
+ */
+ template<typename T>
+inline __device__ void copy_data(
+ cg::thread_group group,
+ T *dst,
+ T *src,
+ size_t n,
+ size_t istride = 1,
+ size_t ostride = 1)
+{
+ unsigned lane = group.thread_rank();
+
+ for (unsigned i = lane; i < n; i += group.size())
+ {
+ if (i < n) {
+ dst[i * (ostride - 1) + i] = src[i * (istride - 1) + i];
+ }
+ }
+
+ group.sync();
+}
+
+/*!
+ * Compute and return the sum of n elements of data.
+ * This is implemented in the form of an parallel reduction
+ * which overwrites the input. The result (the sum) is written
+ * to data[0] and returned.
+ */
+__device__ float reduce_sum(
+ cg::thread_group group,
+ float *data,
+ size_t n)
+{
+ // Each iteration halves the number of active threads
+ // Each thread adds its partial data[i] to data[lane+i]
+ for (unsigned i = (n+1) / 2; i > 0; i /= 2)
+ {
+ for (unsigned j = 0; j < i; j += group.size())
+ {
+ unsigned lane = group.thread_rank() + j;
+ group.sync();
+ if (lane < i)
+ {
+ data[lane] += data[lane + i];
+ }
+ group.sync();
+ }
+ }
+
+ return data[0];
+}
+
+__device__ void compute_stats(
+ cg::thread_group group,
+ float *d_data,
+ float *s_data,
+ size_t batch_size,
+ size_t n,
+ float *mean,
+ float *std)
+{
+ unsigned current_nr_samples = batch_size;
+
+ float sum = 0.0f;
+ float sq_sum = 0.0f;
+
+ // Compute partial sum and squared sum for batch_size samples at once
+ for (unsigned i = 0; i < n; i += batch_size)
+ {
+ current_nr_samples = n - i < batch_size ? n - i : batch_size;
+
+ group.sync();
+
+ // Load samples from device memory to shared memory
+ copy_data<float>(group, s_data, d_data + i, current_nr_samples);
+
+ // Determine middle of the s_data buffer
+ unsigned middle = (current_nr_samples + 1) / 2;
+
+ // Perform first step in reduction:
+ // - store partial sums in s_data[0:middle]
+ // - store partial squared sums in s_data[middle:n]
+ for (unsigned lane = group.thread_rank(); lane < middle; lane += group.size())
+ {
+ float sum = s_data[lane] + s_data[lane + middle];
+ s_data[lane] = sum;
+ s_data[lane + middle] = sum * sum;
+ }
+
+ group.sync();
+
+ // Perform remainder of reduction:
+ // - compute sum in s_data[0]
+ // - computes squared sum in s_data[middle]
+ sum += reduce_sum(group, s_data, middle);
+ sq_sum += reduce_sum(group, s_data + middle, middle);
+ }
+
+ // Compute mean
+ *mean = sum / n;
+
+ // Compute variance
+ float variance = sq_sum / n - *mean * *mean;
+
+ // Compute standard deviation
+ *std = sqrtf(variance);
+}
+
+/*!
+ * Bring the value of x in the domain [a:b]
+ */
+__device__ int clamp(int x, int a, int b)
+{
+ return max(a, min(b, x));
+}
+
+/*!
+ * Digitizes the 32-bit floating-point input data to scaled and offset integers (i.e. 16-bit or 8-bit)
+ * Read one block of samples (one TAB, one channel) for either Complex Voltages (XXYY) or
+ * Coherent/Incoherent Stokes.
+ *
+ * For Coherent and Incoherent Stokes the Stokes I values (sum of two polarizations) will not
+ * have a zero mean. Hence, the scale and offset is determined from the mean and the standard deviation.
+ * Stokes Q, U and V are differences between two polarizations and will have a zero mean. Still, the
+ * scale and offset are also determined from the mean and standard deviation.
+ * For Complex Voltages, each of the four values of XXYY (real and imaginary of X and Y) will also have zero mean.
+ *
+ * The user can specify (smin, smax) as scale = (smax - smin) * std(block)
+ * For non-zero mean: offset = mean(block) + smin * std(block)
+ * For zero mean: offset = smin * std(block)
+ *
+ * The kernel maps blocks of data to thread blocks in round-robin fashion.
+ *
+ * Threads in a thread block are mapped to samples based on their rank within the thread block.
+ *
+ * \param[out] output
+ * pointer to a buffer for output data. This buffer will be populated with the digitized
+ * data (signed integers for zero-mean, unsigned integers for non-zero mean), scales
+ * and offsets. The number of bits for the signed and unsigned integers is compile-time
+ * constant (\c NR_QUANTIZE_BITS). The scales and offsets are 32-bit floating-point numbers.
+ * \param[in] input
+ * 4D output array of stokes values. Each sample contains 1 or 4
+ * stokes paramters. For each tab, there are \c NR_COHERENT_STOKES
+ * time series of channels. The dimensions are: \c NR_TABS, \c
+ * NR_COHERENT_STOKES,
+ * <tt>(NR_SAMPLES_PER_CHANNEL)</tt>, \c NR_CHANNELS.
+ * \param[in] smax
+ * digitize/map raw data from [-smin*std(block), smax*std(block]
+ * outliers are clipped
+ * \param[in] smin
+ * digitize/map raw data from [-smin*std(block), smax*std(block]
+ * outliers are clipped
+ *
+ * Pre-processor input symbols (some are tied to the execution configuration)
+ * Symbol | Valid Values | Description
+ * ----------------------- | ------------- | -----------
+ * NR_CHANNELS | >= 1 | number of frequency channels per subband
+ * NR_COHERENT_STOKES | 1 or 4 | number of stokes paramters to create
+ * COMPLEX_VOLTAGES | 1 or 0 | whether we compute complex voltages or coherent stokes
+ * NR_SAMPLES_PER_CHANNEL | >= 4 | number of input samples per channel
+ * NR_TABS | >= 1 | number of tabs to create
+ * NR_QUANTIZE_BITS | 8, 16 | number of bits used for the quantization
+ * STOKES_I_POSITIVE | 1 or 0 | if 1, and if smin<0, override and set smin=0 for stokes I
+ *
+ * Every block of data will require sizeof(INT) + 2*sizeof(float),
+ * therefore NR_SAMPLES_PER_CHANNEL should be at least 5 (in case of 16-bit ouput) to
+ * make sure that the size of the digitized output is smaller than the size of the input.
+ *
+ */
+extern "C"
+__launch_bounds__(NR_THREADS_PER_BLOCK, 2)
+__global__ void quantizeOutput(
+ size_t output,
+ const InputDataType input,
+ float smax,
+ float smin)
+{
+
+ // Pointers to output
+ FloatArray* scale_ptr = (FloatArray *) output;
+ FloatArray* offset_ptr = (FloatArray *) ((size_t) scale_ptr + sizeof(FloatArray));
+ IntArray* value_ptr = (IntArray *) ((size_t) offset_ptr + sizeof(FloatArray));
+
+ __shared__ float s_data[BATCH_SIZE];
+
+ // Setup thread group
+ cg::thread_block block = cg::this_thread_block();
+ int lane = block.thread_rank();
+
+ for (unsigned i = blockIdx.x; i < (NR_TABS * NR_COHERENT_STOKES * NR_CHANNELS); i += gridDim.x)
+ {
+ unsigned tab_idx = i / (NR_COHERENT_STOKES * NR_CHANNELS);
+ unsigned block_idx = i % (NR_COHERENT_STOKES * NR_CHANNELS);
+ unsigned stokes_idx = block_idx / NR_CHANNELS;
+ unsigned channel_idx = block_idx % NR_CHANNELS;
+
+ if (tab_idx >= NR_TABS ||
+ stokes_idx >= NR_COHERENT_STOKES)
+ {
+ return;
+ }
+
+ // Determine the type of quantization to apply
+ bool stokes_quv = (stokes_idx > 0);
+ bool zero_mean = (COMPLEX_VOLTAGES || stokes_quv);
+
+ // Get pointer to first sample for current tab, stokes and channel
+ float* d_input = &((*input)[tab_idx][stokes_idx][channel_idx][0]);
+
+ // Compute mean and standard deviation
+ float mean = 0;
+ float std = 0;
+ compute_stats(block, d_input, s_data, BATCH_SIZE, NR_SAMPLES_PER_CHANNEL, &mean, &std);
+
+ // Set mean to zero for zero-mean data
+ if (zero_mean)
+ {
+ mean = 0.0f;
+ }
+
+ // In Stokes IQUV mode, I (unsigned, 0..inf) and QUV (signed, -inf..inf)
+ // Make sure that the full range is used, if smin<0 therefore:
+ // - for Stokes I, map the values to (0:std*smax)
+ // - for Stokes QUV use the range (std*smin:std:smax)
+#if (STOKES_I_POSITIVE == 1)
+ bool unsigned_stokes_i=(!COMPLEX_VOLTAGES && !stokes_quv && smin<0.0f);
+ if (unsigned_stokes_i)
+ {
+ smin=0.0f;
+ }
+#endif
+
+ for (unsigned sample_idx = lane; sample_idx < NR_SAMPLES_PER_CHANNEL; sample_idx += block.size())
+ {
+ // Compute scale and offset
+ int xmin = 0;
+ int xmax = (1 << NR_QUANTIZE_BITS) - 1;
+#if (STOKES_I_POSITIVE == 1)
+ float xoffset = (unsigned_stokes_i ? 0.0f: mean + smin * std);
+#else
+ float xoffset = mean + smin * std;
+#endif
+ float xscale = ((smax - smin) * std) / (xmax + 1);
+
+ if (sample_idx < NR_SAMPLES_PER_CHANNEL)
+ {
+ // Get adress of sample (cast to either unsigned or signed integer pointer below)
+ void* d_output = &((*value_ptr)[tab_idx][stokes_idx][sample_idx][channel_idx]);
+
+ // Convert to int
+ float x = (*input)[tab_idx][stokes_idx][channel_idx][sample_idx];
+ int x_int = clamp(__int2float_rd(((x - xoffset) / xscale) + 0.5f), xmin, xmax);
+
+ // Shift value
+ if (zero_mean)
+ {
+ xmax = 1 << (NR_QUANTIZE_BITS - 1);
+ x_int -= xmax;
+ xoffset += xmax * xscale;
+ }
+
+ // Store value
+ if (zero_mean)
+ {
+ *((INT *) d_output) = (INT) x_int;
+ } else
+ {
+ *((UINT *) d_output) = (UINT) x_int;
+ }
+ }
+
+ // Store scale and offset for current block
+ if (sample_idx == 0)
+ {
+ (*scale_ptr)[tab_idx][stokes_idx][channel_idx] = xscale;
+ (*offset_ptr)[tab_idx][stokes_idx][channel_idx] = xoffset;
+ }
+
+ } // end for sample_idx
+
+ block.sync();
+
+ } // end for i
+}
diff --git a/RTCP/Cobalt/GPUProc/share/gpu/kernels/Zeroing.cu b/RTCP/Cobalt/GPUProc/share/gpu/kernels/Zeroing.cu
index 1da7cecb61f54367cbbebfb3da227f83d39f36ad..304c68faa943b878c685b7a5934443551cf0c99d 100644
--- a/RTCP/Cobalt/GPUProc/share/gpu/kernels/Zeroing.cu
+++ b/RTCP/Cobalt/GPUProc/share/gpu/kernels/Zeroing.cu
@@ -19,7 +19,6 @@
//# $Id$
#include "gpu_math.cuh"
-#include <stdio.h>
typedef float2 FilteredDataType[NR_STABS][NR_POLARIZATIONS][NR_SAMPLES_PER_CHANNEL][NR_CHANNELS];
diff --git a/RTCP/Cobalt/GPUProc/src/CMakeLists.txt b/RTCP/Cobalt/GPUProc/src/CMakeLists.txt
index a73efffb36a9b3f0fae24baf54e5208776b5a68a..c22bf074a2f12701bbcdeb467d4d2667d3bcdf00 100644
--- a/RTCP/Cobalt/GPUProc/src/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/src/CMakeLists.txt
@@ -33,6 +33,7 @@ set(_gpuproc_sources
list(APPEND _gpuproc_sources
gpu_wrapper.cc
gpu_utils.cc
+ KernelParameters.cc
KernelFactory.cc
PerformanceCounter.cc
Kernels/Kernel.cc
@@ -54,6 +55,7 @@ list(APPEND _gpuproc_sources
Kernels/IntToFloatKernel.cc
Kernels/FFTShiftKernel.cc
Kernels/ZeroingKernel.cc
+ Kernels/QuantizeOutputKernel.cc
#Kernels/UHEP_BeamFormerKernel.cc
#Kernels/UHEP_InvFFT_Kernel.cc
#Kernels/UHEP_InvFIR_Kernel.cc
@@ -64,7 +66,6 @@ list(APPEND _gpuproc_sources
SubbandProcs/SubbandProc.cc
SubbandProcs/SubbandProcInputData.cc
SubbandProcs/SubbandProcOutputData.cc
- SubbandProcs/KernelFactories.cc
SubbandProcs/CorrelatorStep.cc
SubbandProcs/BeamFormerPreprocessingStep.cc
SubbandProcs/BeamFormerCoherentStep.cc
@@ -77,11 +78,19 @@ lofar_add_library(gpuproc ${_gpuproc_sources})
if(CUDA_cufft_LIBRARY)
target_link_libraries(gpuproc ${CUDA_cufft_LIBRARY})
endif()
+if(CUDA_nvToolsExt_LIBRARY)
+ target_link_libraries(gpuproc ${CUDA_nvToolsExt_LIBRARY})
+endif()
+if(CUDA_nvrtc_LIBRARY)
+ target_link_libraries(gpuproc ${CUDA_nvrtc_LIBRARY})
+endif()
+
lofar_add_bin_program(rtcp rtcp.cc)
lofar_add_bin_program(station_stream Station/station_stream.cc)
lofar_add_bin_program(gpu_load gpu_load.cc)
+lofar_add_bin_program(gpu_load_lmm gpu_load_lmm.cc)
# install scripts used to run an observation under bin
lofar_add_bin_scripts(
diff --git a/RTCP/Cobalt/GPUProc/src/CommandThread.cc b/RTCP/Cobalt/GPUProc/src/CommandThread.cc
index 5c3f3f4ab1d5babb09d36db909b433f7c59aa2e7..86c8eeca62b82777367058d2c83c90db421618de 100644
--- a/RTCP/Cobalt/GPUProc/src/CommandThread.cc
+++ b/RTCP/Cobalt/GPUProc/src/CommandThread.cc
@@ -53,7 +53,7 @@ namespace LOFAR {
LOG_INFO_STR("[CommandThread] Reading from " << streamdesc);
- SmartPtr<Stream> commandStream = createStream(streamdesc, true);
+ std::unique_ptr<Stream> commandStream(createStream(streamdesc, true));
const string command = commandStream->readLine();
diff --git a/RTCP/Cobalt/GPUProc/src/Flagger.cc b/RTCP/Cobalt/GPUProc/src/Flagger.cc
index 57272e2e3800f9d55dd055e8b1083a34ed4fc012..1bbd63bc108ba13e281f8eb6d51b8f18d202b40f 100644
--- a/RTCP/Cobalt/GPUProc/src/Flagger.cc
+++ b/RTCP/Cobalt/GPUProc/src/Flagger.cc
@@ -34,7 +34,29 @@ namespace LOFAR
const unsigned nrChannels,
const ssize_t nrPrefixedSamples)
{
- ASSERT(inputFlags.num_elements() == flagsPerChannel.num_elements());
+ // Use identity mapping for station indices
+ unsigned nrStations = inputFlags.num_elements();
+ std::vector<unsigned> stationIndices(nrStations);
+ for (unsigned station = 0; station < nrStations; station++)
+ {
+ stationIndices[station] = station;
+ }
+
+ convertFlagsToChannelFlags(
+ stationIndices, inputFlags, flagsPerChannel,
+ nrSamples, nrChannels, nrPrefixedSamples);
+ }
+
+ void Flagger::convertFlagsToChannelFlags(
+ const std::vector<unsigned> stationIndices,
+ MultiDimArray<LOFAR::SparseSet<unsigned>, 1>const &inputFlags,
+ MultiDimArray<SparseSet<unsigned>, 1>& flagsPerChannel,
+ const unsigned nrSamples,
+ const unsigned nrChannels,
+ const ssize_t nrPrefixedSamples)
+ {
+ ASSERT(stationIndices.size() == flagsPerChannel.num_elements());
+ ASSERT(inputFlags.num_elements() >= stationIndices.size());
// If nrChannels == 1, we do not expect nrPrefixedSamples
ASSERT(nrChannels > 1 || nrPrefixedSamples == 0);
@@ -43,13 +65,16 @@ namespace LOFAR
unsigned log2NrChannels = log2(nrChannels);
// Convert the flags per sample to flags per channel
- for (unsigned station = 0; station < inputFlags.num_elements(); station ++)
+ for (unsigned station = 0; station < stationIndices.size(); station++)
{
+ unsigned station_in = stationIndices[station];
+ unsigned station_out = station;
+
// reset the channel flags for this station
- flagsPerChannel[station].reset();
+ flagsPerChannel[station_out].reset();
// get the flag ranges
- const SparseSet<unsigned>::Ranges &ranges = inputFlags[station].getRanges();
+ const SparseSet<unsigned>::Ranges &ranges = inputFlags[station_in].getRanges();
for (SparseSet<unsigned>::const_iterator it = ranges.begin();
it != ranges.end(); it ++)
{
@@ -97,7 +122,7 @@ namespace LOFAR
}
// Now copy the transformed ranges to the channelflags
- flagsPerChannel[station].include(begin_idx, end_idx);
+ flagsPerChannel[station_out].include(begin_idx, end_idx);
}
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/Flagger.h b/RTCP/Cobalt/GPUProc/src/Flagger.h
index b56b629bf423c4030b55243752f8b988096392d1..f76d94df56f8a67d9f93ff2aa0bb284f1a8c4167 100644
--- a/RTCP/Cobalt/GPUProc/src/Flagger.h
+++ b/RTCP/Cobalt/GPUProc/src/Flagger.h
@@ -41,6 +41,14 @@ namespace LOFAR
const unsigned nrSamplesPerChannel,
const unsigned nrChannels,
const ssize_t nrPrefixedSamples);
+
+ static void convertFlagsToChannelFlags(
+ const std::vector<unsigned> stationIndices,
+ MultiDimArray<SparseSet<unsigned>, 1>const &inputFlags,
+ MultiDimArray<SparseSet<unsigned>, 1>& flagsPerChannel,
+ const unsigned nrSamplesPerChannel,
+ const unsigned nrChannels,
+ const ssize_t nrPrefixedSamples);
};
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/KernelParameters.cc b/RTCP/Cobalt/GPUProc/src/KernelParameters.cc
new file mode 100644
index 0000000000000000000000000000000000000000..54db7cdf0154d8af21fed77da1bee0b24376cd1e
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/KernelParameters.cc
@@ -0,0 +1,72 @@
+//# KernelParameters.cc
+//#
+//# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include "KernelParameters.h"
+
+namespace LOFAR
+{
+ namespace Cobalt
+ {
+
+ KernelParameters::Observation::Observation(
+ const ObservationSettings& obsSettings) :
+ observationID(obsSettings.observationID),
+ nrSAPs(obsSettings.SAPs.size()),
+ nrStations(obsSettings.antennaFieldNames.size()),
+ nrBitsPerSample(obsSettings.nrBitsPerSample),
+ blockSize(obsSettings.blockSize),
+ subbandWidth(obsSettings.subbandWidth()),
+ subbands(obsSettings.subbands)
+ {
+ ASSERT(obsSettings.antennaFieldNames.size() == obsSettings.antennaFields.size());
+ }
+
+ KernelParameters::Preprocessor::Preprocessor(
+ const std::vector<ObservationSettings::AntennaFieldName>& obsAntennaFieldNames,
+ const std::vector<ObservationSettings::AntennaFieldName>& bfAntennaFieldNames,
+ bool delayCompensationEnabled,
+ bool bandPassCorrectionEnabled,
+ unsigned nrDelayCompensationChannels) :
+ obsStationIndices(ObservationSettings::AntennaFieldName::indices(bfAntennaFieldNames, obsAntennaFieldNames)),
+ delayCompensationEnabled(delayCompensationEnabled),
+ bandPassCorrectionEnabled(bandPassCorrectionEnabled),
+ nrDelayCompensationChannels(nrDelayCompensationChannels)
+ {
+ }
+
+ KernelParameters::BeamFormer::BeamFormer(
+ const std::vector<ObservationSettings::AntennaFieldName>& obsAntennaFieldNames,
+ const std::vector<ObservationSettings::AntennaFieldName>& preAntennaFieldNames,
+ const ObservationSettings::BeamFormer::Pipeline& bfSettings) :
+ preStationIndices(ObservationSettings::AntennaFieldName::indices(bfSettings.antennaFieldNames, preAntennaFieldNames)),
+ obsStationIndices(ObservationSettings::AntennaFieldName::indices(bfSettings.antennaFieldNames, obsAntennaFieldNames)),
+ doFlysEye(bfSettings.doFlysEye),
+ nrSAPs(bfSettings.SAPs.size()),
+ maxNrCoherentTABsPerSAP(bfSettings.maxNrCoherentTABsPerSAP()),
+ maxNrIncoherentTABsPerSAP(bfSettings.maxNrIncoherentTABsPerSAP()),
+ coherentSettings(bfSettings.coherentSettings),
+ incoherentSettings(bfSettings.incoherentSettings)
+ {
+ ASSERTSTR(preStationIndices.size() <= obsStationIndices.size(), "preStationIndices.size() <= ObsStationIndices.size()");
+ }
+
+ } // end namespace Cobalt
+} // end namespace LOFAR
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/KernelParameters.h b/RTCP/Cobalt/GPUProc/src/KernelParameters.h
new file mode 100644
index 0000000000000000000000000000000000000000..e6045d8625069592456f8953e1818fd0b938764c
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/KernelParameters.h
@@ -0,0 +1,123 @@
+//# KernelParameters.h
+//#
+//# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#ifndef LOFAR_GPUPROC_KERNEL_PARAMETERS
+#define LOFAR_GPUPROC_KERNEL_PARAMETERS
+
+#include <vector>
+
+#include <CoInterface/Parset.h>
+
+namespace LOFAR
+{
+ namespace Cobalt
+ {
+ struct KernelParameters
+ {
+ public:
+ // Components
+ bool correlatorEnabled;
+ bool preprocessingEnabled;
+ bool beamFormerCoherentStokesEnabled;
+ bool beamFormerIncoherentStokesEnabled;
+
+ // Observation settings
+ struct Observation
+ {
+ // Constructor
+ Observation(
+ const ObservationSettings& obsSettings);
+
+ // Settings
+ unsigned observationID;
+ unsigned nrSAPs;
+ unsigned nrStations;
+ unsigned nrBitsPerSample;
+ unsigned blockSize;
+ double subbandWidth;
+ std::vector<ObservationSettings::Subband> subbands;
+ };
+
+ Observation observation;
+
+ // Correlator settings
+ using Correlator = ObservationSettings::Correlator;
+ Correlator correlator;
+
+ // Preprocessing settings
+ struct Preprocessor
+ {
+ // Constructor
+ Preprocessor(
+ const std::vector<ObservationSettings::AntennaFieldName>& obsAntennaFieldNames,
+ const std::vector<ObservationSettings::AntennaFieldName>& bfAntennaFieldNames,
+ bool delayCompensationEnabled,
+ bool bandPassCorrectionEnabled,
+ unsigned nrDelayCompensationSettings);
+
+ // Settings
+ std::vector<unsigned> obsStationIndices;
+ bool delayCompensationEnabled;
+ bool bandPassCorrectionEnabled;
+ unsigned nrDelayCompensationChannels;
+ };
+
+ Preprocessor preprocessing;
+
+ // BeamFormer settings
+ struct BeamFormer
+ {
+ // Constructor
+ BeamFormer(
+ const std::vector<ObservationSettings::AntennaFieldName>& obsAntennaFieldNames,
+ const std::vector<ObservationSettings::AntennaFieldName>& preAntennaFieldNames,
+ const ObservationSettings::BeamFormer::Pipeline& bfSettings);
+
+ // The observation has a set of stations specified, ordered by name and numbered
+ // from 0 to ps.settings.obsAntennaFieldNames.size()
+ // For the beamFormer, there are two sets of indices for the (subset of) stations being processed:
+ // obsStationIndices: the indices of the stations in the global list of stations
+ // use these indices when indexing the tabDelays
+ // preStationIndices: the indices of the stations in the list of stations in the preprocessor
+ // use these indices when indexing the preprocssed input data
+ // Note that stationIndices must be a subset of delayIndices
+ std::vector<unsigned> preStationIndices;
+ std::vector<unsigned> obsStationIndices;
+
+ // Settings
+ bool doFlysEye;
+ unsigned int nrSAPs;
+ unsigned int maxNrCoherentTABsPerSAP;
+ unsigned int maxNrIncoherentTABsPerSAP;
+ ObservationSettings::BeamFormer::StokesSettings coherentSettings;
+ ObservationSettings::BeamFormer::StokesSettings incoherentSettings;
+ };
+
+ BeamFormer beamFormer;
+
+ // Cobalt settings
+ using Cobalt = ObservationSettings::Cobalt;
+ Cobalt cobalt;
+ };
+ } // end namespace Cobalt
+} // end namespace LOFAR
+
+#endif
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.cc
index 32bf4da8a3350d84144d21954e43871b399dec2d..e0f061d7f987e219cda50a4cc16775c6cff4acf9 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.cc
@@ -44,21 +44,27 @@ namespace LOFAR
string BandPassCorrectionKernel::theirSourceFile = "BandPassCorrection.cu";
string BandPassCorrectionKernel::theirFunction = "bandPassCorrection";
- BandPassCorrectionKernel::Parameters::Parameters(const Parset& ps) :
- Kernel::Parameters("bandpassCorrection"),
- nrStations(ps.settings.beamFormer.antennaFieldNames.size()),
-
- nrDelayCompensationChannels(ps.settings.beamFormer.nrDelayCompensationChannels),
- nrHighResolutionChannels(ps.settings.beamFormer.nrDelayCompensationChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrHighResolutionChannels),
-
- correctBandPass(ps.settings.corrections.bandPass)
+ BandPassCorrectionKernel::Parameters::Parameters() :
+ Kernel::Parameters(theirFunction)
+ {}
+
+ BandPassCorrectionKernel::Parameters::Parameters(
+ unsigned nrStations_,
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ bool correctBandPass_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
+ Kernel::Parameters(theirFunction),
+ nrStations(nrStations_),
+
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
+
+ correctBandPass(correctBandPass_)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.BandPassCorrectionKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_BandPassCorrectionKernel.dat") %
- ps.settings.observationID );
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
size_t BandPassCorrectionKernel::Parameters::bufferSize(BandPassCorrectionKernel::BufferType bufferType) const
@@ -68,17 +74,17 @@ namespace LOFAR
return
(size_t) nrStations * NR_POLARIZATIONS *
nrSamplesPerChannel *
- nrHighResolutionChannels *
+ nrChannels *
sizeof(std::complex<float>);
case BandPassCorrectionKernel::OUTPUT_DATA:
return
(size_t) nrStations * NR_POLARIZATIONS *
nrSamplesPerChannel *
- nrHighResolutionChannels *
+ nrChannels *
sizeof(std::complex<float>);
case BandPassCorrectionKernel::BAND_PASS_CORRECTION_WEIGHTS:
return
- (size_t) nrHighResolutionChannels * sizeof(float);
+ (size_t) nrChannels * sizeof(float);
default:
THROW(GPUProcException, "Invalid bufferType (" << bufferType << ")");
}
@@ -95,24 +101,22 @@ namespace LOFAR
setArg(1, buffers.input);
setArg(2, bandPassCorrectionWeights);
- const unsigned nrChannels2 = params.nrHighResolutionChannels / params.nrDelayCompensationChannels;
-
// The cu kernel requires a square for the (x,y) block dimensions.
-
+
setEnqueueWorkSizes( gpu::Grid(params.nrSamplesPerChannel,
- params.nrDelayCompensationChannels,
- nrChannels2),
+ params.nrChannels,
+ 1),
gpu::Block(16, 16, 1));
size_t nrSamples = params.nrStations * params.nrSamplesPerChannel *
- params.nrHighResolutionChannels * NR_POLARIZATIONS;
+ params.nrChannels * NR_POLARIZATIONS;
nrOperations = nrSamples ;
nrBytesRead = nrBytesWritten = nrSamples * sizeof(std::complex<float>);
gpu::HostMemory bpWeights(stream.getContext(), bandPassCorrectionWeights.size());
BandPass::computeCorrectionFactors(bpWeights.get<float>(),
- params.nrHighResolutionChannels,
- 1.0 / params.nrHighResolutionChannels);
+ params.nrChannels,
+ 1.0 / params.nrChannels);
stream.writeBuffer(bandPassCorrectionWeights, bpWeights, true);
}
@@ -127,12 +131,9 @@ namespace LOFAR
defs["NR_STATIONS"] = lexical_cast<string>(itsParameters.nrStations);
- defs["NR_CHANNELS_1"] =
- lexical_cast<string>(itsParameters.nrDelayCompensationChannels);
- defs["NR_CHANNELS_2"] =
- lexical_cast<string>(itsParameters.nrHighResolutionChannels /
- itsParameters.nrDelayCompensationChannels);
- defs["NR_SAMPLES_PER_CHANNEL"] =
+ defs["NR_CHANNELS"] =
+ lexical_cast<string>(itsParameters.nrChannels);
+ defs["NR_SAMPLES_PER_CHANNEL"] =
lexical_cast<string>(itsParameters.nrSamplesPerChannel);
if (itsParameters.correctBandPass)
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.h
index 607a3f1cd90d9a73e70ecb380f37f4d191de485e..e50f048f4be9cd69ad94e08b67e327b27881b03d 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/BandPassCorrectionKernel.h
@@ -21,8 +21,6 @@
#ifndef LOFAR_GPUPROC_CUDA_BAND_PASS_CORRECTION_KERNEL_H
#define LOFAR_GPUPROC_CUDA_BAND_PASS_CORRECTION_KERNEL_H
-#include <CoInterface/Parset.h>
-
#include <GPUProc/Kernels/Kernel.h>
#include <GPUProc/KernelFactory.h>
#include <GPUProc/gpu_wrapper.h>
@@ -49,11 +47,19 @@ namespace LOFAR
// BandPassCorrectionKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters();
+
+ Parameters(
+ unsigned nrStations,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ bool correctBandPass,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
unsigned nrStations;
- unsigned nrDelayCompensationChannels;
- unsigned nrHighResolutionChannels;
+ unsigned nrChannels;
unsigned nrSamplesPerChannel;
bool correctBandPass;
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.cc
index ef83105acd97cc22dac1070ddc017ea776869fdd..907d3265b782051508726d3ad74ff0298c5dd8a3 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.cc
@@ -44,27 +44,39 @@ namespace LOFAR
string BeamFormerKernel::theirSourceFile = "BeamFormer.cu";
string BeamFormerKernel::theirFunction = "beamFormer";
- BeamFormerKernel::Parameters::Parameters(const Parset& ps) :
+ BeamFormerKernel::Parameters::Parameters(
+ std::vector<unsigned> stationIndices_,
+ std::vector<unsigned> delayIndices_,
+ unsigned nrDelays_,
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ unsigned nrSAPs_,
+ unsigned nrTABs_,
+ double subbandBandWidth_,
+ bool doFlysEye_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
Kernel::Parameters("beamFormer"),
+ // mapping to translate station numbers in input array (which
+ // are the station selected in the beamformer preprocessor)
+ stationIndices(stationIndices_),
+
// mapping to translate station numbers in delay arrays (which
// are the observation stations) to the beamformer stations
- delayIndices(ObservationSettings::AntennaFieldName::indices(ps.settings.beamFormer.antennaFieldNames, ps.settings.antennaFieldNames)),
- nrDelays(ps.settings.antennaFieldNames.size()),
+ delayIndices(delayIndices_),
+ nrDelays(nrDelays_),
- nrChannels(ps.settings.beamFormer.nrDelayCompensationChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
- nrSAPs(ps.settings.beamFormer.SAPs.size()),
- nrTABs(ps.settings.beamFormer.maxNrCoherentTABsPerSAP()),
- subbandBandwidth(ps.settings.subbandWidth()),
- doFlysEye(ps.settings.beamFormer.doFlysEye)
+ nrSAPs(nrSAPs_),
+ nrTABs(nrTABs_),
+ subbandBandwidth(subbandBandWidth_),
+ doFlysEye(doFlysEye_)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.BeamFormerKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_BeamFormerKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
@@ -74,12 +86,15 @@ namespace LOFAR
return
(size_t) nrChannels *
nrSamplesPerChannel * NR_POLARIZATIONS *
- nrStations() * sizeof(std::complex<float>);
+ nrInputStations() * sizeof(std::complex<float>);
case BeamFormerKernel::OUTPUT_DATA:
return
(size_t) nrChannels *
nrSamplesPerChannel * NR_POLARIZATIONS *
nrTABs * sizeof(std::complex<float>);
+ case BeamFormerKernel::STATION_INDICES:
+ return
+ (size_t) stationIndices.size() * sizeof stationIndices[0];
case BeamFormerKernel::DELAY_INDICES:
return
(size_t) delayIndices.size() * sizeof delayIndices[0];
@@ -99,13 +114,15 @@ namespace LOFAR
const Buffers& buffers,
const Parameters& params) :
CompiledKernel(stream, gpu::Function(module, theirFunction), buffers, params),
+ stationIndices(stream.getContext(), params.bufferSize(STATION_INDICES)),
delayIndices(stream.getContext(), params.bufferSize(DELAY_INDICES)),
beamFormerDelays(stream.getContext(), params.bufferSize(BEAM_FORMER_DELAYS))
{
setArg(0, buffers.output);
setArg(1, buffers.input);
- setArg(2, beamFormerDelays);
- setArg(3, delayIndices);
+ setArg(2, stationIndices);
+ setArg(3, beamFormerDelays);
+ setArg(4, delayIndices);
// Beamformer kernel requires 1 channel in the blockDim.z dimension
// FlysEye parallellises over time in blockIdx.x
@@ -120,6 +137,12 @@ namespace LOFAR
// in the kernel file. Additional threads are used to optimize
// memory access
+ // upload stationIndices, as they are static across the observation
+ gpu::HostMemory stationIndicesHost(stream.getContext(), params.bufferSize(STATION_INDICES));
+ std::memcpy(stationIndicesHost.get<void>(), ¶ms.stationIndices.front(),
+ stationIndicesHost.size());
+ stream.writeBuffer(stationIndices, stationIndicesHost, false);
+
// upload delayIndices, as they are static across the observation
gpu::HostMemory delayIndicesHost(stream.getContext(), params.bufferSize(DELAY_INDICES));
std::memcpy(delayIndicesHost.get<void>(), ¶ms.delayIndices.front(),
@@ -130,8 +153,8 @@ namespace LOFAR
void BeamFormerKernel::enqueue(const BlockID &blockId,
double subbandFrequency, unsigned SAP)
{
- setArg(4, subbandFrequency);
- setArg(5, SAP);
+ setArg(5, subbandFrequency);
+ setArg(6, SAP);
Kernel::enqueue(blockId);
}
@@ -151,15 +174,14 @@ namespace LOFAR
CompileDefinitions defs =
KernelFactoryBase::compileDefinitions(itsParameters);
- defs["NR_STATIONS"] = lexical_cast<string>(itsParameters.nrStations());
+ defs["NR_INPUT_STATIONS"] = lexical_cast<string>(itsParameters.nrInputStations());
+ defs["NR_OUTPUT_STATIONS"] = lexical_cast<string>(itsParameters.nrSelectedStations());
defs["NR_DELAYS"] = lexical_cast<string>(itsParameters.nrDelays);
defs["NR_CHANNELS"] = lexical_cast<string>(itsParameters.nrChannels);
defs["NR_SAMPLES_PER_CHANNEL"] =
lexical_cast<string>(itsParameters.nrSamplesPerChannel);
- defs["NR_SAPS"] =
- lexical_cast<string>(itsParameters.nrSAPs);
defs["NR_TABS"] =
lexical_cast<string>(itsParameters.nrTABs);
defs["SUBBAND_BANDWIDTH"] =
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.h
index 6ba5995049b8697b87c8fe4835e3c29f57cfb882..b9a91c3966f88ae989c92531ae5f5f238856c7f2 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerKernel.h
@@ -21,8 +21,6 @@
#ifndef LOFAR_GPUPROC_CUDA_BEAM_FORMER_KERNEL_H
#define LOFAR_GPUPROC_CUDA_BEAM_FORMER_KERNEL_H
-#include <CoInterface/Parset.h>
-
#include <GPUProc/Kernels/Kernel.h>
#include <GPUProc/KernelFactory.h>
#include <GPUProc/gpu_wrapper.h>
@@ -44,6 +42,7 @@ namespace LOFAR
{
INPUT_DATA,
OUTPUT_DATA,
+ STATION_INDICES,
DELAY_INDICES,
BEAM_FORMER_DELAYS
};
@@ -52,7 +51,23 @@ namespace LOFAR
// BeamFormerKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters(
+ std::vector<unsigned> stationIndices,
+ std::vector<unsigned> delayIndices,
+ unsigned nrDelays,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ unsigned nrSAPs,
+ unsigned nrTABs,
+ double subbandWidth,
+ bool doFlysEye,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
+ // The input data is indexed by its station number
+ // in the beamformer preprocessor, so we need to map
+ // them to beam-former station numbers.
+ std::vector<unsigned> stationIndices;
// The beam-former delays are indexed by their station number
// in the observation, so we need to map them to beam-former
@@ -72,7 +87,8 @@ namespace LOFAR
size_t bufferSize(BufferType bufferType) const;
- size_t nrStations() const { return delayIndices.size(); }
+ size_t nrInputStations() const { return delayIndices.size(); }
+ size_t nrSelectedStations() const { return stationIndices.size(); }
};
BeamFormerKernel(const gpu::Stream &stream,
@@ -83,6 +99,7 @@ namespace LOFAR
void enqueue(const BlockID &blockId,
double subbandFrequency, unsigned SAP);
+ gpu::DeviceMemory stationIndices;
gpu::DeviceMemory delayIndices;
gpu::DeviceMemory beamFormerDelays;
};
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.cc
index 994c4eff04219de9c230ea906ff6d37367d6f20a..0e201cef02962ca2036582b958682ecc2affd13a 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.cc
@@ -44,21 +44,21 @@ namespace LOFAR
string BeamFormerTransposeKernel::theirSourceFile = "Transpose.cu";
string BeamFormerTransposeKernel::theirFunction = "transpose";
- BeamFormerTransposeKernel::Parameters::Parameters(const Parset& ps) :
+ BeamFormerTransposeKernel::Parameters::Parameters(
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ unsigned nrTABs_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
Kernel::Parameters("beamFormerTranspose"),
- nrChannels(ps.settings.beamFormer.nrDelayCompensationChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
- nrTABs(ps.settings.beamFormer.maxNrCoherentTABsPerSAP())
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
+ nrTABs(nrTABs_)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.BeamFormerTransposeKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_BeamFormerTransposeKernel.dat") %
- ps.settings.observationID);
-
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
-
size_t BeamFormerTransposeKernel::Parameters::bufferSize(BufferType bufferType) const
{
switch (bufferType) {
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.h
index dc65338ba69a0b3209b56a95d0068bfd6b79bdf7..620722ad8a539204e8071a077d47f24177122c65 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/BeamFormerTransposeKernel.h
@@ -47,7 +47,13 @@ namespace LOFAR
// BeamFormerKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters(
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ unsigned nrTABs,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
unsigned nrChannels;
unsigned nrSamplesPerChannel;
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.cc
index 2960e87739cf5985cf9e96c1190d67920ce063fc..69947d4c9d101a9d8ec24d1ecf9b77692f37d67e 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.cc
@@ -45,24 +45,33 @@ namespace LOFAR
string CoherentStokesKernel::theirSourceFile = "CoherentStokes.cu";
string CoherentStokesKernel::theirFunction = "coherentStokes";
- CoherentStokesKernel::Parameters::Parameters(const Parset& ps) :
- Kernel::Parameters("coherentStokes"),
- nrChannels(ps.settings.beamFormer.coherentSettings.nrChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
-
- nrTABs(ps.settings.beamFormer.maxNrCoherentTABsPerSAP()),
- nrStokes(ps.settings.beamFormer.coherentSettings.nrStokes),
- outputComplexVoltages(ps.settings.beamFormer.coherentSettings.type == STOKES_XXYY),
- timeIntegrationFactor(ps.settings.beamFormer.coherentSettings.timeIntegrationFactor)
- {
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.CoherentStokesKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_CoherentStokesKernel.dat") %
- ps.settings.observationID);
+ CoherentStokesKernel::Parameters::Parameters() :
+ Kernel::Parameters(theirFunction)
+ {}
+
+ CoherentStokesKernel::Parameters::Parameters(
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ unsigned nrTABs_,
+ unsigned nrStokes_,
+ bool outputComplexVoltages_,
+ unsigned timeIntegrationFactor_,
+ bool quantizeOutput_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
+ Kernel::Parameters(theirFunction),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
+ nrTABs(nrTABs_),
+ nrStokes(nrStokes_),
+ outputComplexVoltages(outputComplexVoltages_),
+ timeIntegrationFactor(timeIntegrationFactor_),
+ outputOrder(quantizeOutput_)
+ {
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
-
size_t CoherentStokesKernel::Parameters::bufferSize(BufferType bufferType) const
{
@@ -288,6 +297,14 @@ namespace LOFAR
defs["TIME_INTEGRATION_FACTOR"] =
lexical_cast<string>(itsParameters.timeIntegrationFactor);
+ if (!itsParameters.outputOrder)
+ {
+ defs["OUTPUT_ORDER_SAMPLES_CHANNELS"] = "1";
+ } else
+ {
+ defs["OUTPUT_ORDER_CHANNELS_SAMPLES"] = "1";
+ }
+
return defs;
}
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.h
index 6b12782d2ed8172f1953f6be7420a3093bf5b38c..f2150d61629b5cd7b46fbf5722b9c27481373edc 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesKernel.h
@@ -48,7 +48,18 @@ namespace LOFAR
// CoherentStokesKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters();
+ Parameters(
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ unsigned nrTABs,
+ unsigned nrStokes,
+ bool outputComplexVoltages,
+ unsigned timeIntegrationFactor,
+ bool quantizeOutput, //needed to set output ordering
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
unsigned nrChannels;
unsigned nrSamplesPerChannel;
@@ -57,6 +68,8 @@ namespace LOFAR
bool outputComplexVoltages;
unsigned timeIntegrationFactor;
+ bool outputOrder; // 1: channels/samples 0: samples/channels
+
size_t bufferSize(BufferType bufferType) const;
};
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.cc
index c06839adbf82e7560cf2366d864eaa72ddafece4..1f5afb15d0891b32b023d61ece886c254fa7ed1f 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.cc
@@ -44,18 +44,20 @@ namespace LOFAR
string CoherentStokesTransposeKernel::theirSourceFile = "CoherentStokesTranspose.cu";
string CoherentStokesTransposeKernel::theirFunction = "coherentStokesTranspose";
- CoherentStokesTransposeKernel::Parameters::Parameters(const Parset& ps) :
- Kernel::Parameters("coherentStokesTranspose"),
- nrChannels(ps.settings.beamFormer.nrDelayCompensationChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
-
- nrTABs(ps.settings.beamFormer.maxNrCoherentTABsPerSAP())
+ CoherentStokesTransposeKernel::Parameters::Parameters(
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ unsigned nrTABs_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_
+ ) :
+ Kernel::Parameters(theirFunction),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
+ nrTABs(nrTABs_)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.CoherentStokesTransposeKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_CoherentStokesTransposeKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.h
index 78c6905bfdd5919e9daf57392a09cd76f0f5dc9c..930858eebd90617c8da2226e72e7c1348acfcb0d 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/CoherentStokesTransposeKernel.h
@@ -47,7 +47,13 @@ namespace LOFAR
// BeamFormerKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters(
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ unsigned nrTABs,
+ bool dumpOutput = false,
+ std::string dumpFilePattern = "");
+
unsigned nrChannels;
unsigned nrSamplesPerChannel;
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.cc
index 754e30499895c5611fa5cac932721a423bbee550..a7b35d5859dcbfd085246c617b89ec0a5b0fd56c 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.cc
@@ -43,22 +43,25 @@ namespace LOFAR
string CorrelatorKernel::theirSourceFile = "Correlator.cu";
string CorrelatorKernel::theirFunction = "correlate";
- CorrelatorKernel::Parameters::Parameters(const Parset& ps) :
- Kernel::Parameters("correlator"),
- nrStations(ps.settings.antennaFields.size()),
+ CorrelatorKernel::Parameters::Parameters(
+ unsigned nrStations_,
+ unsigned nrChannels_,
+ unsigned nrSamplesPerIntegration_,
+ unsigned nrIntegrationsPerBlock_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
+ Kernel::Parameters(theirFunction),
+ nrStations(nrStations_),
// For Cobalt (= up to 80 antenna fields), the 2x2 kernel gives the best
// performance.
nrStationsPerThread(2),
- nrChannels(ps.settings.correlator.nrChannels),
- nrSamplesPerIntegration(ps.settings.correlator.nrSamplesPerBlock / ps.settings.correlator.nrIntegrationsPerBlock),
- nrIntegrationsPerBlock(ps.settings.correlator.nrIntegrationsPerBlock)
+ nrChannels(nrChannels_),
+ nrSamplesPerIntegration(nrSamplesPerIntegration_),
+ nrIntegrationsPerBlock(nrIntegrationsPerBlock_)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.CorrelatorKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_CorrelatorKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
unsigned CorrelatorKernel::Parameters::nrBaselines() const {
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.h
index c4343b9d9b9556ac5e20ba657f5b55e3dcc5b61c..55f15e3035a607ffb09dc2249e801b046e9780fd 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/CorrelatorKernel.h
@@ -43,7 +43,14 @@ namespace LOFAR
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters(
+ unsigned nrStations,
+ unsigned nrChannels,
+ unsigned nrSamplesPerIntegration,
+ unsigned nrIntegrationsPerBlock,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
unsigned nrStations;
unsigned nrStationsPerThread;
unsigned nrBaselines() const;
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.cc
index 6205b45ad6c2e935adaf8f71e52e622bacac7103..3ad35b6fdb9789810664c95a0c8763ebc587fd4e 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.cc
@@ -44,43 +44,44 @@ namespace LOFAR
string DelayAndBandPassKernel::theirSourceFile = "DelayAndBandPass.cu";
string DelayAndBandPassKernel::theirFunction = "applyDelaysAndCorrectBandPass";
- DelayAndBandPassKernel::Parameters::Parameters(const Parset& ps, bool correlator) :
- Kernel::Parameters(correlator ? "delayAndBandPass" : "delayCompensation"),
- nrStations(correlator ? ps.settings.antennaFieldNames.size() : ps.settings.beamFormer.antennaFieldNames.size()),
- delayIndices(nrStations),
- nrDelays(ps.settings.antennaFieldNames.size()),
- nrBitsPerSample(ps.settings.nrBitsPerSample),
- nrChannels(correlator ? ps.settings.correlator.nrChannels
- : ps.settings.beamFormer.nrDelayCompensationChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
- subbandBandwidth(ps.settings.subbandWidth()),
-
- nrSAPs(ps.settings.SAPs.size()),
-
- delayCompensation(ps.settings.delayCompensation.enabled),
- correctBandPass(correlator ? ps.settings.corrections.bandPass
- : false),
- transpose(correlator ? true
- : false)
+ DelayAndBandPassKernel::Parameters::Parameters(
+ unsigned nrStations_,
+ std::vector<unsigned> delayIndices_,
+ unsigned nrDelays_,
+ unsigned nrBitsPerSample_,
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ double subbandBandwidth_,
+ unsigned nrSAPs_,
+ bool correlator_,
+ bool delayCompensation_,
+ bool correctBandPass_,
+ bool transpose_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
+ Kernel::Parameters(correlator_ ? "delayAndBandPass" : "delayCompensation"),
+ nrStations(nrStations_),
+ delayIndices(nrStations_),
+ nrDelays(nrDelays_),
+ nrBitsPerSample(nrBitsPerSample_),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
+ subbandBandwidth(subbandBandwidth_),
+ nrSAPs(nrSAPs_),
+ delayCompensation(delayCompensation_),
+ correctBandPass(correctBandPass_),
+ transpose(transpose_)
{
- if (correlator) {
+ if (correlator_) {
// Use identity mappnig for station indices
for (unsigned i = 0; i < nrStations; i++)
delayIndices[i] = i;
} else {
- delayIndices = ObservationSettings::AntennaFieldName::indices(
- ps.settings.beamFormer.antennaFieldNames,
- ps.settings.antennaFieldNames);
+ delayIndices = delayIndices_;
}
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.DelayAndBandPassKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_DelayAndBandPassKernel_%c%c%c.dat") %
- ps.settings.observationID %
- (correctBandPass ? "B" : "b") %
- (delayCompensation ? "D" : "d") %
- (transpose ? "T" : "t"));
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
@@ -215,9 +216,6 @@ namespace LOFAR
defs["SUBBAND_BANDWIDTH"] =
str(format("%.7f") % itsParameters.subbandBandwidth);
- defs["NR_SAPS"] =
- lexical_cast<string>(itsParameters.nrSAPs);
-
if (itsParameters.delayCompensation)
defs["DELAY_COMPENSATION"] = "1";
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.h
index a2f051c6f0aef458ac27e3d96d3b58e9d50fb314..214e5723bf8ca556acd3d81dd269d0109c7f2d62 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/DelayAndBandPassKernel.h
@@ -55,7 +55,22 @@ namespace LOFAR
// DelayAndBandPassKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps, bool correlator);
+ Parameters(
+ unsigned nrStations,
+ std::vector<unsigned> delayIndices,
+ unsigned nrDelays,
+ unsigned nrBitsPerSample,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ double subbandBandwidth,
+ unsigned nrSAPs,
+ bool correlator,
+ bool delayCompensation,
+ bool correctBandPass,
+ bool transpose,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
unsigned nrStations;
std::vector<unsigned> delayIndices;
unsigned nrDelays;
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.cc
index e2fab771ab225b953ffca43a68457d4922d0ca4f..e9f9b2b6ab732fe3d2ef0ab7e324c745727f1574 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.cc
@@ -42,18 +42,20 @@ namespace LOFAR
string FFTShiftKernel::theirSourceFile = "FFTShift.cu";
string FFTShiftKernel::theirFunction = "FFTShift";
- FFTShiftKernel::Parameters::Parameters(const Parset& ps, unsigned nrSTABs, unsigned nrChannels, const std::string &name):
+ FFTShiftKernel::Parameters::Parameters(
+ unsigned nrSTABs_,
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ const std::string& name,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
Kernel::Parameters(name),
- nrSTABs(nrSTABs),
-
- nrChannels(nrChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels)
+ nrSTABs(nrSTABs_),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.FFTShiftKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_FFTShiftKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
@@ -79,6 +81,8 @@ namespace LOFAR
{
setArg(0, buffers.input);
+ ASSERT(buffers.input.get() == buffers.output.get());
+
// Number of samples per channel must be even
ASSERT(params.nrSamplesPerChannel % 2 == 0);
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.h
index edfa3c6d867cd9fbb6b8e839afaa33c49e312288..735e24b4a440e01c1ee7ff5a63f0a9bb6f2fdd5a 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/FFTShiftKernel.h
@@ -47,7 +47,14 @@ namespace LOFAR
// IntToFloatKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps, unsigned nrSTABs, unsigned nrChannels, const std::string &name = "FFT-shift");
+ Parameters(
+ unsigned nrSTABs,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ const std::string &name = "FFT-shift",
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
unsigned nrSTABs;
unsigned nrChannels;
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.cc
index 34049e89aae0441a3f9d0fd208131d6206199601..9e3f153f4b4628643fc14b2d9bd10f2db39afb06 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.cc
@@ -43,24 +43,30 @@ namespace LOFAR
string FIR_FilterKernel::theirSourceFile = "FIR_Filter.cu";
string FIR_FilterKernel::theirFunction = "FIR_filter";
- FIR_FilterKernel::Parameters::Parameters(const Parset& ps, unsigned nrSTABs, bool inputIsStationData, unsigned nrSubbands, unsigned nrChannels, float scaleFactor, const std::string &name) :
+ FIR_FilterKernel::Parameters::Parameters(
+ unsigned nrSTABs,
+ unsigned nrBitsPerSample,
+ bool inputIsStationData,
+ unsigned nrSubbands,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ float scaleFactor,
+ const std::string &name,
+ const bool dumpBuffers_,
+ std::string dumpFilePattern_) :
Kernel::Parameters(name),
nrSTABs(nrSTABs),
- nrBitsPerSample(ps.settings.nrBitsPerSample),
+ nrBitsPerSample(nrBitsPerSample),
nrChannels(nrChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
+ nrSamplesPerChannel(nrSamplesPerChannel),
nrSubbands(nrSubbands),
scaleFactor(scaleFactor),
inputIsStationData(inputIsStationData)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.FIR_FilterKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_FIR_FilterKernel.dat") %
- ps.settings.observationID);
-
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
const unsigned FIR_FilterKernel::Parameters::nrTaps;
@@ -115,13 +121,14 @@ namespace LOFAR
const Parameters& params) :
CompiledKernel(stream, gpu::Function(module, theirFunction), buffers, params),
params(params),
- filterWeights(stream.getContext(), params.bufferSize(FILTER_WEIGHTS)),
+ h_filterWeights(stream.getContext(), params.bufferSize(FILTER_WEIGHTS)),
+ d_filterWeights(stream.getContext(), params.bufferSize(FILTER_WEIGHTS)),
historySamples(stream.getContext(), params.bufferSize(HISTORY_DATA)),
historyFlags(boost::extents[params.nrSubbands][params.nrSTABs])
{
setArg(0, buffers.output);
setArg(1, buffers.input);
- setArg(2, filterWeights);
+ setArg(2, d_filterWeights);
setArg(3, historySamples);
unsigned totalNrThreads = params.nrChannels * NR_POLARIZATIONS * 2;
@@ -152,10 +159,9 @@ namespace LOFAR
filterBank.negateWeights();
filterBank.scaleWeights(params.scaleFactor);
- gpu::HostMemory firWeights(stream.getContext(), filterWeights.size());
- std::memcpy(firWeights.get<void>(), filterBank.getWeights().origin(),
- firWeights.size());
- stream.writeBuffer(filterWeights, firWeights, true);
+ std::memcpy(h_filterWeights.get<void>(), filterBank.getWeights().origin(),
+ h_filterWeights.size());
+ stream.writeBuffer(d_filterWeights, h_filterWeights, true);
// start with all history samples flagged
for (size_t n = 0; n < historyFlags.num_elements(); ++n)
@@ -212,8 +218,6 @@ namespace LOFAR
defs["NR_TAPS"] =
lexical_cast<string>(itsParameters.nrTaps);
- defs["NR_SUBBANDS"] =
- lexical_cast<string>(itsParameters.nrSubbands);
if (itsParameters.inputIsStationData)
defs["INPUT_IS_STATIONDATA"] = "1";
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.h
index 8d7d8df7825fc2651038a6938f1d888568bbac4f..e57b4527c48b328146c8f1c25a254a5292df8e73 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/FIR_FilterKernel.h
@@ -51,7 +51,17 @@ namespace LOFAR
// FIR_FilterKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps, unsigned nrSTABs, bool inputIsStationData, unsigned nrSubbands, unsigned nrChannels, float scaleFactor, const std::string &name = "FIR");
+ Parameters(
+ unsigned nrSTABs,
+ unsigned nrBitsPerSample,
+ bool inputIsStationData,
+ unsigned nrSubbands,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ float scaleFactor,
+ const std::string &name = "FIR",
+ const bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
// The number of stations or TABs to filter. The FIR filter will
// deal with either in the same way.
@@ -108,7 +118,8 @@ namespace LOFAR
const Parameters params;
// The FIR filter weights
- gpu::DeviceMemory filterWeights;
+ gpu::HostMemory h_filterWeights;
+ gpu::DeviceMemory d_filterWeights;
// The history samples
gpu::DeviceMemory historySamples;
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.cc
index d89ca9940486bb5a1053e4c71b5c6c1ae21e4b1b..3ff436a8837a4a6f66251de6c54608e5c6571d84 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.cc
@@ -39,23 +39,32 @@ namespace LOFAR
string IncoherentStokesKernel::theirSourceFile = "IncoherentStokes.cu";
string IncoherentStokesKernel::theirFunction = "incoherentStokes";
- IncoherentStokesKernel::Parameters::Parameters(const Parset& ps) :
- Kernel::Parameters("incoherentStokes"),
- nrStations(ps.settings.beamFormer.antennaFieldNames.size()),
- nrChannels(ps.settings.beamFormer.incoherentSettings.nrChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
-
- nrStokes(ps.settings.beamFormer.incoherentSettings.nrStokes),
- timeIntegrationFactor(ps.settings.beamFormer.incoherentSettings.timeIntegrationFactor)
+ IncoherentStokesKernel::Parameters::Parameters() :
+ Kernel::Parameters(theirFunction)
+ {}
+
+ IncoherentStokesKernel::Parameters::Parameters(
+ unsigned nrStations_,
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ unsigned nrStokes_,
+ unsigned timeIntegrationFactor_,
+ bool quantizeOutput_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_
+ ) :
+ Kernel::Parameters(theirFunction),
+ nrStations(nrStations_),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
+ nrStokes(nrStokes_),
+ timeIntegrationFactor(timeIntegrationFactor_),
+ outputOrder(quantizeOutput_)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.IncoherentStokesKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_IncoherentStokesKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
-
size_t IncoherentStokesKernel::Parameters::bufferSize(BufferType bufferType) const
{
switch (bufferType) {
@@ -120,6 +129,15 @@ namespace LOFAR
lexical_cast<string>(itsParameters.nrStokes);
defs["TIME_INTEGRATION_FACTOR"] =
lexical_cast<string>(itsParameters.timeIntegrationFactor);
+
+ if (!itsParameters.outputOrder)
+ {
+ defs["OUTPUT_ORDER_SAMPLES_CHANNELS"] = "1";
+ } else
+ {
+ defs["OUTPUT_ORDER_CHANNELS_SAMPLES"] = "1";
+ }
+
return defs;
}
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.h
index a50cf7ed173677ff65257cd5f1bffd432f5788ea..0c11dbb14b894414b8fb03f4fd648c2da72e4fe5 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesKernel.h
@@ -48,7 +48,18 @@ namespace LOFAR
// IncoherentStokesKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters();
+ Parameters(
+ unsigned nrStations,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ unsigned nrStokes,
+ unsigned timeIntegrationFactor,
+ bool quantizeOutput,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = ""
+ );
+
unsigned nrStations;
unsigned nrChannels;
@@ -57,6 +68,8 @@ namespace LOFAR
unsigned nrStokes;
unsigned timeIntegrationFactor;
+ bool outputOrder; // 1: channels/samples 0: samples/channels
+
size_t bufferSize(BufferType bufferType) const;
};
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.cc
index 7ab683197f763edf7f3822d66291be86d4de8223..d0a53d0db80c39637ba1d933597e8ddecd40decd 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.cc
@@ -41,31 +41,43 @@ namespace LOFAR
const string IncoherentStokesTransposeKernel::theirFunction =
"transpose";
- IncoherentStokesTransposeKernel::Parameters::Parameters(const Parset& ps) :
+ IncoherentStokesTransposeKernel::Parameters::Parameters(
+ unsigned nrInputStations_,
+ std::vector<unsigned> stationIndices_,
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ bool beamFormerStationSubset,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_
+ ) :
Kernel::Parameters("incoherentStokesTranspose"),
- nrStations(ps.settings.beamFormer.antennaFieldNames.size()),
- nrChannels(ps.settings.beamFormer.nrDelayCompensationChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels),
-
+ nrInputStations(nrInputStations_),
+ stationIndices(stationIndices_),
+ doStationSubset(beamFormerStationSubset),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
tileSize(16)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.IncoherentStokesTransposeKernel.dumpOutput",
- false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_IncoherentStokesTransposeKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
size_t IncoherentStokesTransposeKernel::Parameters::bufferSize(BufferType bufferType) const
{
switch (bufferType) {
case IncoherentStokesTransposeKernel::INPUT_DATA:
+ return
+ (size_t) nrInputStations *
+ nrChannels * nrSamplesPerChannel *
+ NR_POLARIZATIONS * sizeof(std::complex<float>);
case IncoherentStokesTransposeKernel::OUTPUT_DATA:
return
- (size_t) nrStations *
+ (size_t) nrOutputStations() *
nrChannels * nrSamplesPerChannel *
NR_POLARIZATIONS * sizeof(std::complex<float>);
+ case IncoherentStokesTransposeKernel::STATION_INDICES:
+ return
+ (size_t) stationIndices.size() * sizeof stationIndices[0];
default:
THROW(GPUProcException, "Invalid bufferType (" << bufferType << ")");
}
@@ -76,10 +88,18 @@ namespace LOFAR
const gpu::Module& module,
const Buffers& buffers,
const Parameters& params) :
- CompiledKernel(stream, gpu::Function(module, theirFunction), buffers, params)
+ CompiledKernel(stream, gpu::Function(module, theirFunction), buffers, params),
+ stationIndices(stream.getContext(), params.bufferSize(STATION_INDICES))
{
setArg(0, buffers.output);
setArg(1, buffers.input);
+ setArg(2, stationIndices);
+
+ // upload stationIndices, as they are static across the observation
+ gpu::HostMemory stationIndicesHost(stream.getContext(), params.bufferSize(STATION_INDICES));
+ std::memcpy(stationIndicesHost.get<void>(), ¶ms.stationIndices.front(),
+ stationIndicesHost.size());
+ stream.writeBuffer(stationIndices, stationIndicesHost, true);
// LOG_DEBUG_STR("align(params.nrSamplesPerChannel, params.tileSize) = "
// << "align(" << params.nrSamplesPerChannel
@@ -104,14 +124,20 @@ namespace LOFAR
CompileDefinitions defs =
KernelFactoryBase::compileDefinitions(itsParameters);
- defs["NR_STATIONS"] =
- lexical_cast<string>(itsParameters.nrStations);
+ defs["NR_INPUT_STATIONS"] =
+ lexical_cast<string>(itsParameters.nrInputStations);
+ defs["NR_OUTPUT_STATIONS"] =
+ lexical_cast<string>(itsParameters.nrOutputStations());
defs["NR_CHANNELS"] =
lexical_cast<string>(itsParameters.nrChannels);
defs["NR_SAMPLES_PER_CHANNEL"] =
lexical_cast<string>(itsParameters.nrSamplesPerChannel);
defs["TILE_SIZE"] = lexical_cast<string>(itsParameters.tileSize);
+
+ if (itsParameters.doStationSubset)
+ defs["DO_STATIONSUBSET"] = "1";
+
return defs;
}
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.h
index b3e38f1d3f55c02872579b4d659d6adf46afbd06..5f15fe8bac300d441308305bac858d44326ca127 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/IncoherentStokesTransposeKernel.h
@@ -45,16 +45,27 @@ namespace LOFAR
enum BufferType
{
INPUT_DATA,
- OUTPUT_DATA
+ OUTPUT_DATA,
+ STATION_INDICES
};
// Parameters that must be passed to the constructor of the
// IncoherentStokesTransposeKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps);
+ Parameters(
+ unsigned nrInputStations,
+ std::vector<unsigned> stationIndices,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ bool beamFormerStationSubset,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
+ unsigned nrInputStations;
+ std::vector<unsigned> stationIndices; // input station nr for each output station
+ bool doStationSubset;
- unsigned nrStations;
unsigned nrChannels;
unsigned nrSamplesPerChannel;
@@ -62,6 +73,8 @@ namespace LOFAR
const unsigned tileSize;
size_t bufferSize(BufferType bufferType) const;
+
+ unsigned nrOutputStations() const { return doStationSubset ? stationIndices.size() : nrInputStations; }
};
IncoherentStokesTransposeKernel(const gpu::Stream &stream,
@@ -69,6 +82,7 @@ namespace LOFAR
const Buffers &buffers,
const Parameters ¶m);
+ gpu::DeviceMemory stationIndices;
};
//# -------- Template specializations for KernelFactory -------- #//
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.cc
index 57717698695df1aa0c066546f7b9f642ee10204b..3860cae9fb54cd50c0d3459bdc620c817c62a956 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.cc
@@ -45,21 +45,27 @@ namespace LOFAR
string IntToFloatKernel::theirSourceFile = "IntToFloat.cu";
string IntToFloatKernel::theirFunction = "intToFloat";
- IntToFloatKernel::Parameters::Parameters(const Parset& ps, bool fftShift, bool beamFormerStationSubset) :
- Kernel::Parameters("intToFloat"),
- nrInputStations(ps.settings.antennaFields.size()),
- stationIndices(beamFormerStationSubset ? ObservationSettings::AntennaFieldName::indices(ps.settings.beamFormer.antennaFieldNames, ps.settings.antennaFieldNames) : vector<unsigned>()),
- nrBitsPerSample(ps.settings.nrBitsPerSample),
-
- nrSamplesPerSubband(ps.settings.blockSize),
+ IntToFloatKernel::Parameters::Parameters(
+ unsigned nrInputStations,
+ std::vector<unsigned> stationIndices,
+ unsigned nrBitsPerSample,
+ unsigned nrSamplesPerSubband,
+ bool fftShift,
+ bool beamFormerStationSubset,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
+ Kernel::Parameters(theirFunction),
+
+ nrInputStations(nrInputStations),
+ stationIndices(stationIndices),
+ nrBitsPerSample(nrBitsPerSample),
+
+ nrSamplesPerSubband(nrSamplesPerSubband),
fftShift(fftShift),
doStationSubset(beamFormerStationSubset)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.IntToFloatKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_IntToFloatKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.h
index 434d6f31b097ee17b94aa685120b47f61e76e5a6..bcd4b71706496a57c414282091fd8e768b7fa9fe 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/IntToFloatKernel.h
@@ -51,7 +51,16 @@ namespace LOFAR
// IntToFloatKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps, bool fftShift, bool beamFormerStationSubset);
+ Parameters(
+ unsigned nrInputStations,
+ std::vector<unsigned> stationIndices,
+ unsigned nrBitsPerSample,
+ unsigned nrSamplesPerSubband,
+ bool fftShift,
+ bool beamFormerStationSubset,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
unsigned nrInputStations;
std::vector<unsigned> stationIndices; // input station nr for ewch output station
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/QuantizeOutputKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/QuantizeOutputKernel.cc
new file mode 100644
index 0000000000000000000000000000000000000000..b6a2937ebd16bee844134154ae0ca71b84c26798
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/QuantizeOutputKernel.cc
@@ -0,0 +1,258 @@
+//# QuantizeOutputKernel.cc
+//# Copyright (C) 2012-2014 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id: QuantizeOutputKernel.cc 43504 2019-07-04 12:07:45Z klazema $
+
+#include <lofar_config.h>
+
+#include "QuantizeOutputKernel.h"
+
+#include <utility>
+#include <fstream>
+#include <boost/lexical_cast.hpp>
+#include <boost/format.hpp>
+
+#include <Common/lofar_complex.h>
+#include <Common/LofarLogger.h>
+#include <CoInterface/BlockID.h>
+#include <CoInterface/Align.h>
+#include <CoInterface/Config.h>
+#include <CoInterface/fpequals.h>
+#include <GPUProc/gpu_utils.h>
+
+namespace LOFAR
+{
+ namespace Cobalt
+ {
+ using boost::lexical_cast;
+ using boost::format;
+
+ string QuantizeOutputKernel::theirSourceFile = "QuantizeOutput.cu";
+ string QuantizeOutputKernel::theirFunction = "quantizeOutput";
+
+ QuantizeOutputKernel::Parameters::Parameters() :
+ Kernel::Parameters(theirFunction)
+ {}
+
+ QuantizeOutputKernel::Parameters::Parameters(
+ unsigned nrChannels_,
+ unsigned nrSamplesPerChannel_,
+ unsigned nrTABs_,
+ unsigned nrStokes_,
+ bool outputComplexVoltages_,
+ unsigned nrQuantizeBits_,
+ float quantizeScaleMax_,
+ float quantizeScaleMin_,
+ bool sIpositive_,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_
+ ) :
+ Kernel::Parameters(theirFunction),
+ nrChannels(nrChannels_),
+ nrSamplesPerChannel(nrSamplesPerChannel_),
+ nrTABs(nrTABs_),
+ nrStokes(nrStokes_),
+ outputComplexVoltages(outputComplexVoltages_),
+ nrQuantizeBits(nrQuantizeBits_),
+ quantizeScaleMax(quantizeScaleMax_),
+ quantizeScaleMin(quantizeScaleMin_),
+ sIpositive(sIpositive_)
+ {
+ nrThreadsPerBlock = 512;
+
+ // The QuantizeOutputKernel reuses the buffers of CoherentStokesKernel in reverse order:
+ // CoherentStokesKernel: C -> D, QuantizeOutputKernel: D -> C'
+ // While D < C due to integration of the data, there is no guarantee that
+ // C' <= C, thus need to make sure that C is large enough.
+ ASSERT(bufferSize(INPUT_DATA) > bufferSize(OUTPUT_DATA));
+
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
+ }
+
+
+ size_t QuantizeOutputKernel::Parameters::bufferSize(BufferType bufferType) const
+ {
+ switch (bufferType) {
+ case QuantizeOutputKernel::INPUT_DATA:
+ return
+ (size_t) nrTABs * nrStokes * nrSamplesPerChannel * nrChannels *
+ sizeof(float); // values
+ case QuantizeOutputKernel::OUTPUT_VALUES:
+ return
+ (size_t) nrTABs * nrStokes * nrSamplesPerChannel * nrChannels *
+ ((float) nrQuantizeBits / 8);
+ case QuantizeOutputKernel::OUTPUT_METADATA:
+ return
+ (size_t) nrTABs * nrStokes * nrChannels * sizeof(float);
+ case QuantizeOutputKernel::OUTPUT_DATA:
+ return
+ bufferSize(OUTPUT_VALUES) + 2 * bufferSize(OUTPUT_METADATA);
+
+ default:
+ THROW(GPUProcException, "Invalid bufferType (" << bufferType << ")");
+ }
+ }
+
+
+ QuantizeOutputKernel::QuantizeOutputKernel(const gpu::Stream& stream,
+ const gpu::Module& module,
+ const Buffers& buffers,
+ const Parameters& params) :
+ CompiledKernel(stream, gpu::Function(module, theirFunction), buffers, params),
+ parameters(params)
+ {
+ ASSERT(params.nrSamplesPerChannel > 0);
+ ASSERT(params.nrStokes == 1 || params.nrStokes == 4);
+
+ setArg(0, buffers.output);
+ setArg(1, buffers.input);
+ setArg(2, params.quantizeScaleMax);
+ setArg(3, params.quantizeScaleMin);
+
+ const gpu::Device device(_context.getDevice());
+ unsigned int nrMPs = device.getAttribute(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT);
+ gpu::Grid grid(params.nrStokes * nrMPs * params.nrThreadsPerBlock);
+ gpu::Block block(params.nrThreadsPerBlock);
+
+ setEnqueueWorkSizes(grid, block);
+ }
+
+ struct QuantizeOutputKernel::GPUBufferOffsets QuantizeOutputKernel::bufferOffsets() const
+ {
+ unsigned nrTABs = parameters.nrTABs;
+ unsigned nrStokes = parameters.nrStokes;
+ unsigned nrChannels = parameters.nrChannels;
+ size_t size = nrTABs * nrStokes * nrChannels * sizeof(float);
+ // use a struct for default buffer offset values
+ // offset for scale, offset, data
+ struct GPUBufferOffsets defBoff = { 0, size, 2*size };
+ return defBoff;
+ }
+
+ #define DEBUG_CONVERT_I2F 0
+ template<typename INT, typename UINT>
+ void convertI2F_(
+ MultiDimArray<float, 4>& data,
+ const MultiDimArray<INT, 4>& qdata,
+ const MultiDimArray<float, 3>& qscales,
+ const MultiDimArray<float, 3>& qoffsets,
+ bool outputComplexVoltages = false)
+ {
+ ASSERTSTR(sizeof(INT)==sizeof(UINT), "INT/UINT sizes differ.");
+
+ // Get parameters
+ unsigned nrTABs = data.shape()[0];
+ unsigned nrStokes = data.shape()[1];
+ unsigned nrSamples = data.shape()[2];
+ unsigned nrChannels = data.shape()[3];
+
+ #if DEBUG_CONVERT_I2F
+ unsigned i = 0;
+ #endif
+
+ // Convert quantized output to floating-point output
+ for (size_t tab = 0; tab < nrTABs; tab++)
+ {
+ for (size_t stokes = 0; stokes < nrStokes; stokes++)
+ {
+ // Determine the type of quantization to apply
+ bool stokes_quv = (stokes > 0);
+ bool zero_mean = (outputComplexVoltages || stokes_quv);
+
+ for (size_t sample = 0; sample < nrSamples; sample++)
+ {
+ for (size_t channel = 0; channel < nrChannels; channel++)
+ {
+ // Reconstruct sample
+ auto x_ptr = &(qdata[tab][stokes][sample][channel]);
+ auto x_int = zero_mean ? *((INT *) x_ptr) : *((UINT *) x_ptr);
+ auto x_scale = qscales[tab][stokes][channel];
+ auto x_offset = qoffsets[tab][stokes][channel];
+ auto x_float = x_int * x_scale + x_offset;
+ data[tab][stokes][sample][channel] = x_float;
+
+ #if DEBUG_CONVERT_I2F
+ std::cout << "[" << i++ << "] ";
+ std::cout << "xint: " << x_int << ", ";
+ std::cout << "xscale: " << x_scale << ", ";
+ std::cout << "xoffset: " << x_offset << ", ";
+ std::cout << "xfloat: " << x_float << std::endl;
+ #endif
+ }
+ }
+ }
+ }
+ }
+
+ void QuantizeOutputKernel::convertI2F(
+ MultiDimArrayHostBuffer<float, 4>& dst,
+ MultiDimArrayHostBuffer<fcomplex, 4>& src)
+ {
+ auto nrQuantizeBits = parameters.nrQuantizeBits;
+ auto outputComplexVoltages = parameters.outputComplexVoltages;
+
+ // Get multi-dimensional array references
+ MultiDimArray<float, 3> qscales = getQuantizeScales(src);
+ MultiDimArray<float, 3> qoffsets = getQuantizeOffsets(src);
+
+ if (nrQuantizeBits == 8)
+ {
+ MultiDimArray<int8_t, 4> qdata = getQuantizedData<int8_t>(src);
+ convertI2F_<int8_t, uint8_t>(dst, qdata, qscales, qoffsets, outputComplexVoltages);
+ } else if (nrQuantizeBits == 16)
+ {
+ MultiDimArray<int16_t, 4> qdata = getQuantizedData<int16_t>(src);
+ convertI2F_<int16_t, uint16_t>(dst, qdata, qscales, qoffsets, outputComplexVoltages);
+ } else {
+ throw std::runtime_error("Invalid value for nrQuantizeBits");
+ }
+ };
+ //-------- Template specializations for KernelFactory --------//
+
+ template<> CompileDefinitions
+ KernelFactory<QuantizeOutputKernel>::compileDefinitions() const
+ {
+ CompileDefinitions defs =
+ KernelFactoryBase::compileDefinitions(itsParameters);
+
+ defs["NR_CHANNELS"] = lexical_cast<string>(itsParameters.nrChannels);
+ defs["NR_SAMPLES_PER_CHANNEL"] =
+ lexical_cast<string>(itsParameters.nrSamplesPerChannel);
+
+ defs["NR_TABS"] =
+ lexical_cast<string>(itsParameters.nrTABs);
+ defs["COMPLEX_VOLTAGES"] =
+ itsParameters.outputComplexVoltages ? "1" : "0";
+ defs["STOKES_I_POSITIVE"] =
+ itsParameters.sIpositive ? "1" : "0";
+ defs["NR_COHERENT_STOKES"] =
+ lexical_cast<string>(itsParameters.nrStokes);
+
+ defs["NR_QUANTIZE_BITS"] =
+ lexical_cast<string>(itsParameters.nrQuantizeBits);
+
+ defs["NR_THREADS_PER_BLOCK"] =
+ lexical_cast<string>(itsParameters.nrThreadsPerBlock);
+
+ return defs;
+ }
+
+ }
+}
+
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/QuantizeOutputKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/QuantizeOutputKernel.h
new file mode 100644
index 0000000000000000000000000000000000000000..45ab76246176524114e07e9ecd581dc4061a0d09
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/QuantizeOutputKernel.h
@@ -0,0 +1,140 @@
+//# QuantizeOutputKernel.h
+//# Copyright (C) 2012-2014 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id: QuantizeOutputKernel.h 43504 2019-07-04 12:07:45Z klazema $
+
+#ifndef LOFAR_GPUPROC_CUDA_QUANTIZE_OUTPUT_KERNEL_H
+#define LOFAR_GPUPROC_CUDA_QUANTIZE_OUTPUT_KERNEL_H
+
+#include <CoInterface/Parset.h>
+
+#include <GPUProc/Kernels/Kernel.h>
+#include <GPUProc/KernelFactory.h>
+#include <GPUProc/gpu_wrapper.h>
+#include <GPUProc/MultiDimArrayHostBuffer.h>
+
+namespace LOFAR
+{
+ namespace Cobalt
+ {
+
+ class QuantizeOutputKernel : public CompiledKernel
+ {
+ public:
+ static std::string theirSourceFile;
+ static std::string theirFunction;
+
+ enum BufferType
+ {
+ INPUT_DATA,
+ OUTPUT_DATA, // total size of output, quantized values, scales and offsets
+ OUTPUT_VALUES, // size of quantized values
+ OUTPUT_METADATA // size of metadata (same for scales and offsets)
+ };
+
+ // Parameters that must be passed to the constructor of the
+ struct Parameters : Kernel::Parameters
+ {
+ Parameters();
+ Parameters(
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ unsigned nrTABs,
+ unsigned nrStokes,
+ bool outputComplexVoltages,
+ unsigned nrQuantizeBits,
+ float quantizeScaleMax,
+ float quantizeScaleMin,
+ bool sIpositive,
+ bool dumpBuffers = false,
+ std::string dumpFilePattern = "");
+
+ unsigned nrChannels;
+ unsigned nrSamplesPerChannel;
+
+ unsigned nrTABs;
+ unsigned nrStokes;
+ bool outputComplexVoltages;
+ unsigned nrQuantizeBits;
+ float quantizeScaleMax;
+ float quantizeScaleMin;
+ bool sIpositive;
+
+ unsigned int nrThreadsPerBlock;
+
+ size_t bufferSize(BufferType bufferType) const;
+ };
+
+ // offsets in GPU buffer during quantization
+ struct GPUBufferOffsets { size_t scale,offset,data; };
+
+ QuantizeOutputKernel(const gpu::Stream &stream,
+ const gpu::Module &module,
+ const Buffers &buffers,
+ const Parameters ¶m);
+
+ MultiDimArray<float, 3> getQuantizeScales(
+ MultiDimArrayHostBuffer<fcomplex, 4>& src)
+ {
+ size_t ptr = (size_t) src.data() + bufferOffsets().scale;
+ boost::multi_array_types::extent_gen extent_gen;
+ auto& p = parameters;
+ auto extents(extent_gen[p.nrTABs][p.nrStokes][p.nrChannels]);
+ return MultiDimArray<float, 3>(extents, (float *) ptr, false);
+ }
+
+ MultiDimArray<float, 3> getQuantizeOffsets(
+ MultiDimArrayHostBuffer<fcomplex, 4>& src)
+ {
+ size_t ptr = (size_t) src.data() + bufferOffsets().offset;
+ boost::multi_array_types::extent_gen extent_gen;
+ auto& p = parameters;
+ auto extents(extent_gen[p.nrTABs][p.nrStokes][p.nrChannels]);
+ return MultiDimArray<float, 3>(extents, (float *) ptr, false);
+ }
+
+ template<typename INT>
+ MultiDimArray<INT, 4> getQuantizedData(
+ MultiDimArrayHostBuffer<fcomplex, 4>& src)
+ {
+ size_t ptr = (size_t) src.data() + bufferOffsets().data;
+ boost::multi_array_types::extent_gen extent_gen;
+ auto& p = parameters;
+ auto extents(extent_gen[p.nrTABs][p.nrStokes][p.nrSamplesPerChannel][p.nrChannels]);
+ MultiDimArray<INT, 4> qdata(extents, (INT *) ptr, false);
+ return qdata;
+ }
+
+ // Reconstuct floating point data from quantized (integer) data
+ void convertI2F(
+ MultiDimArrayHostBuffer<float, 4>& dst,
+ MultiDimArrayHostBuffer<fcomplex, 4>& src);
+
+ struct GPUBufferOffsets bufferOffsets() const;
+
+ struct Parameters parameters;
+
+ };
+ //# -------- Template specializations for KernelFactory -------- #//
+ template<> CompileDefinitions
+ KernelFactory<QuantizeOutputKernel>::compileDefinitions() const;
+ }
+}
+
+#endif
+
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.cc b/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.cc
index 9b3175eeceebfcb927043b61adfe50fec24360b1..5b113badf2085f8f6aab2d72620c87d15041a4b8 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.cc
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.cc
@@ -45,18 +45,21 @@ namespace LOFAR
string ZeroingKernel::theirSourceFile = "Zeroing.cu";
string ZeroingKernel::theirFunction = "Zeroing";
- ZeroingKernel::Parameters::Parameters(const Parset& ps, unsigned nrSTABs, unsigned nrChannels, const std::string &name):
+ ZeroingKernel::Parameters::Parameters(
+ unsigned nrSTABs,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ const std::string &name,
+ bool dumpBuffers_,
+ std::string dumpFilePattern_) :
Kernel::Parameters(name),
nrSTABs(nrSTABs),
nrChannels(nrChannels),
- nrSamplesPerChannel(ps.settings.blockSize / nrChannels)
+ nrSamplesPerChannel(nrSamplesPerChannel)
{
- dumpBuffers =
- ps.getBool("Cobalt.Kernels.ZeroingKernel.dumpOutput", false);
- dumpFilePattern =
- str(format("L%d_SB%%03d_BL%%03d_ZeroingKernel.dat") %
- ps.settings.observationID);
+ dumpBuffers = dumpBuffers_;
+ dumpFilePattern = dumpFilePattern_;
}
diff --git a/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.h b/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.h
index edf360f86f1e4d1cd0d6f83eba65a15b4d770bb4..43613b763192511ea764a2811568d6bf0ca1e74d 100644
--- a/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.h
+++ b/RTCP/Cobalt/GPUProc/src/Kernels/ZeroingKernel.h
@@ -52,7 +52,14 @@ namespace LOFAR
// IntToFloatKernel class.
struct Parameters : Kernel::Parameters
{
- Parameters(const Parset& ps, unsigned nrSTABs, unsigned nrChannels, const std::string &name = "Zeroing");
+ Parameters(
+ unsigned nrSTABs,
+ unsigned nrChannels,
+ unsigned nrSamplesPerChannel,
+ const std::string &name = theirFunction,
+ bool dumpBuffers_ = false,
+ std::string dumpFilePattern_ = "");
+
unsigned nrSTABs;
unsigned nrChannels;
diff --git a/RTCP/Cobalt/GPUProc/src/MPIReceiver.cc b/RTCP/Cobalt/GPUProc/src/MPIReceiver.cc
index 36d154df37c3824cace13e80c0ef727e0ccfb908..070ae30541c50a73185e24fb4dc272e39b9042b6 100644
--- a/RTCP/Cobalt/GPUProc/src/MPIReceiver.cc
+++ b/RTCP/Cobalt/GPUProc/src/MPIReceiver.cc
@@ -31,10 +31,10 @@ namespace LOFAR
template<typename SampleT> void MPIRecvData::allocate(
size_t nrStations, size_t nrBeamlets, size_t nrSamples)
{
- data = (char*)mpiAllocator.allocate(
- nrStations * nrBeamlets * nrSamples * sizeof(SampleT));
- metaData = (char*)mpiAllocator.allocate(
- nrStations * nrBeamlets * sizeof(struct MPIProtocol::MetaData));
+ data.reset((char*)mpiAllocator.allocate(
+ nrStations * nrBeamlets * nrSamples * sizeof(SampleT)));
+ metaData.reset((char*)mpiAllocator.allocate(
+ nrStations * nrBeamlets * sizeof(struct MPIProtocol::MetaData)));
}
template void MPIRecvData::allocate< SampleType<i16complex> >(
@@ -72,7 +72,7 @@ namespace LOFAR
for (size_t i = 0; i < N_PoolItems; i++)
{
// Create a raw + meta datablock
- SmartPtr<struct MPIRecvData> mpiData = new MPIRecvData;
+ std::shared_ptr<struct MPIRecvData> mpiData(new MPIRecvData);
// allocate the data (using mpi allocate)
mpiData->allocate<SampleT>(nrStations, subbandIndices.size(),
@@ -94,7 +94,7 @@ namespace LOFAR
LOG_INFO_STR("[block " << block << "] Collecting input buffers");
// Get a free data item
- SmartPtr<struct MPIRecvData> mpiData = mpiPool.free.remove();
+ std::shared_ptr<struct MPIRecvData> mpiData = mpiPool.free.remove();
mpiData->block = block;
diff --git a/RTCP/Cobalt/GPUProc/src/MPIReceiver.h b/RTCP/Cobalt/GPUProc/src/MPIReceiver.h
index eeccb9516c7602f42c0b8183376b48cfb883f819..e13c1783e8f3d0ceae576fe5064ca97881d9e51f 100644
--- a/RTCP/Cobalt/GPUProc/src/MPIReceiver.h
+++ b/RTCP/Cobalt/GPUProc/src/MPIReceiver.h
@@ -26,7 +26,6 @@
#include <InputProc/Transpose/MPIUtil.h>
#include <InputProc/SampleType.h>
-#include <CoInterface/SmartPtr.h>
#include <Common/ComplexStdInt.h>
// From here MPIInput
@@ -51,8 +50,8 @@ namespace LOFAR
// Index of the block current stored in this object
size_t block;
- SmartPtr<char, SmartPtrMPI<char> > data;
- SmartPtr<char, SmartPtrMPI<char> > metaData;
+ std::shared_ptr<char> data;
+ std::shared_ptr<char> metaData;
template<typename SampleT>
void allocate(size_t nrStations, size_t nrBeamlets, size_t nrSamples);
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceCounter.cc b/RTCP/Cobalt/GPUProc/src/PerformanceCounter.cc
index 1128de26aba970537f47d5c02fc78454ccb494ac..823a83fe33ac412316725969b5ea2aba1a603941 100644
--- a/RTCP/Cobalt/GPUProc/src/PerformanceCounter.cc
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceCounter.cc
@@ -21,6 +21,7 @@
#include <lofar_config.h>
#include <iomanip>
+#include <fstream>
#include "PerformanceCounter.h"
#include <Common/LofarLogger.h>
@@ -48,6 +49,12 @@ namespace LOFAR
recording(false)
{}
+ // initialization of static members
+ // set to false by default
+ bool PerformanceCounter::csvLogEnabled = false;
+ // empty by default
+ std::string PerformanceCounter::csvLogFilePath = "";
+
PerformanceCounter::~PerformanceCounter()
{
if (!gpuProfiling)
@@ -57,8 +64,22 @@ namespace LOFAR
logTime();
LOG_INFO_STR("(" << std::setw(30) << name << "): " << stats);
+
+ // if enabled also log to benchmark file in csv (';') format
+ // we use ';' because std::string name can have a ',' character
+ if(csvLogEnabled)
+ {
+ std::ofstream ofs (csvLogFilePath, std::ofstream::app);
+ ofs << "PerformanceCounter; " << name;
+ (ofstream&) ofs << stats << std::endl; // not the neatest implementation, but
+ // without the typecast, overloading of the IO operator will result in
+ // calling the ostream overloaded function for both ostream and ofstream
+ // as both in the end have the same base class (ostream).
+ // This additional typecast makes sure that the ofstream overloaded function is called
+ ofs.close();
}
+ }
void PerformanceCounter::recordStart(const gpu::Stream &stream)
{
@@ -98,6 +119,23 @@ namespace LOFAR
// has no easy way to check beforehand.
}
}
+
+ void PerformanceCounter::enableCSVLog(const std::string &filePath)
+ {
+ csvLogFilePath = filePath;
+ // check if the file is accessible, should be done on top level already
+ // but lets be sure for future usage
+ std::ofstream ofs(csvLogFilePath, std::ofstream::app);
+ if(ofs.good())
+ {
+ csvLogEnabled = true;
+ LOG_INFO_STR("Logging GPU kernel performance counters to file: " << csvLogFilePath);
+ }
+ else
+ {
+ LOG_WARN_STR("PerformanceCounter: Failed to open file: " << csvLogFilePath);
+ }
+ }
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceCounter.h b/RTCP/Cobalt/GPUProc/src/PerformanceCounter.h
index 5748b5f4d199dadcc121f4a363d99823ae6fcd42..2b7875f0e896c168e0d3f31c706cce9d1ccd321a 100644
--- a/RTCP/Cobalt/GPUProc/src/PerformanceCounter.h
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceCounter.h
@@ -38,12 +38,20 @@ namespace LOFAR
void recordStart(const gpu::Stream &stream);
void recordStop(const gpu::Stream &stream);
+ // Enable logging of benchmark info to a csv file and set the file path
+ void static enableCSVLog(const std::string &filePath);
+
// Warning: user must make sure that the counter is not running!
RunningStatistics getStats() { logTime(); return stats; }
private:
const std::string name;
+ // Whether we are logging benchmark info to a csv file
+ // static because common for all objects of PerformanceCounter
+ static bool csvLogEnabled;
+ static std::string csvLogFilePath;
+
// Public event: it needs to be inserted into a stream.
// @{
gpu::Event start;
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-763847.parset b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-763847.parset
new file mode 100644
index 0000000000000000000000000000000000000000..f5cf12eaf03b48f9b49f3e7acbbc4ca731105b65
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-763847.parset
@@ -0,0 +1,354 @@
+#
+# ----- Info -----
+#
+# Purpose: LOFAR reference test
+# LOTAAS pulsar survey observation
+# Intended to be used with gpu_load test for GPU kernel performance benchmarking
+
+# Enable performance benchmarking
+Cobalt.Benchmark.enabled=true
+Cobalt.Benchmark.file=Benchmarks/763847.csv
+
+# These figures are used by gpu_load to predict the load across the whole cluster.
+# Set available processing resources for the full COBALT 2.0 system (2 GPUs/sockets x 11 nodes)
+Cobalt.Nodes = [ 22*localhost ]
+
+#
+# ----- Beamformer settings -----
+#
+Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=16
+Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=6
+Cobalt.BeamFormer.CoherentStokes.which=I
+Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=16
+Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=6
+Cobalt.BeamFormer.IncoherentStokes.which=I
+Cobalt.BeamFormer.coherentDedisperseChannels=false
+Cobalt.BeamFormer.flysEye=false
+Cobalt.BeamFormer.stationList=[]
+
+#
+# ----- Correlator settings -----
+#
+Cobalt.Correlator.integrationTime=1.00663
+Cobalt.Correlator.nrBlocksPerIntegration=1
+Cobalt.Correlator.nrChannelsPerSubband=16
+Cobalt.Correlator.nrIntegrationsPerBlock=1
+
+#
+# ----- Misc settings -----
+#
+Cobalt.blockSize=196608
+Cobalt.correctBandPass=true
+Cobalt.correctClocks=true
+Cobalt.delayCompensation=true
+Cobalt.realTime=true
+
+#
+# ----- Pointings -----
+#
+Observation.Beam[0].TiedArrayBeam[0].angle1=2.04736882885
+Observation.Beam[0].TiedArrayBeam[0].angle2=0.435171843579
+Observation.Beam[0].TiedArrayBeam[0].coherent=true
+Observation.Beam[0].TiedArrayBeam[0].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[0].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[10].angle1=2.0142057899
+Observation.Beam[0].TiedArrayBeam[10].angle2=0.417671078103
+Observation.Beam[0].TiedArrayBeam[10].coherent=true
+Observation.Beam[0].TiedArrayBeam[10].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[10].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[11].angle1=2.02811107217
+Observation.Beam[0].TiedArrayBeam[11].angle2=0.430482527603
+Observation.Beam[0].TiedArrayBeam[11].coherent=true
+Observation.Beam[0].TiedArrayBeam[11].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[11].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[12].angle1=0
+Observation.Beam[0].TiedArrayBeam[12].angle2=0
+Observation.Beam[0].TiedArrayBeam[12].coherent=false
+Observation.Beam[0].TiedArrayBeam[12].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[12].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[1].angle1=2.06662658554
+Observation.Beam[0].TiedArrayBeam[1].angle2=0.430482527603
+Observation.Beam[0].TiedArrayBeam[1].coherent=true
+Observation.Beam[0].TiedArrayBeam[1].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[1].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[2].angle1=2.08053186781
+Observation.Beam[0].TiedArrayBeam[2].angle2=0.417671078103
+Observation.Beam[0].TiedArrayBeam[2].coherent=true
+Observation.Beam[0].TiedArrayBeam[2].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[2].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[3].angle1=2.08537288312
+Observation.Beam[0].TiedArrayBeam[3].angle2=0.400170312628
+Observation.Beam[0].TiedArrayBeam[3].coherent=true
+Observation.Beam[0].TiedArrayBeam[3].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[3].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[4].angle1=2.0800444331
+Observation.Beam[0].TiedArrayBeam[4].angle2=0.382669547152
+Observation.Beam[0].TiedArrayBeam[4].coherent=true
+Observation.Beam[0].TiedArrayBeam[4].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[4].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[5].angle1=2.06613884838
+Observation.Beam[0].TiedArrayBeam[5].angle2=0.369858097653
+Observation.Beam[0].TiedArrayBeam[5].coherent=true
+Observation.Beam[0].TiedArrayBeam[5].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[5].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[6].angle1=2.04736882885
+Observation.Beam[0].TiedArrayBeam[6].angle2=0.365168781677
+Observation.Beam[0].TiedArrayBeam[6].coherent=true
+Observation.Beam[0].TiedArrayBeam[6].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[6].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[7].angle1=2.02859880932
+Observation.Beam[0].TiedArrayBeam[7].angle2=0.369858097653
+Observation.Beam[0].TiedArrayBeam[7].coherent=true
+Observation.Beam[0].TiedArrayBeam[7].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[7].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[8].angle1=2.01469322461
+Observation.Beam[0].TiedArrayBeam[8].angle2=0.382669547152
+Observation.Beam[0].TiedArrayBeam[8].coherent=true
+Observation.Beam[0].TiedArrayBeam[8].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[8].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[9].angle1=2.00936477459
+Observation.Beam[0].TiedArrayBeam[9].angle2=0.400170312628
+Observation.Beam[0].TiedArrayBeam[9].coherent=true
+Observation.Beam[0].TiedArrayBeam[9].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[9].dispersionMeasure=0
+Observation.Beam[0].angle1=2.0473688288544127
+Observation.Beam[0].angle2=0.4001703126277363
+Observation.Beam[0].directionType=J2000
+Observation.Beam[0].duration=3600
+Observation.Beam[0].momID=1000092
+Observation.Beam[0].nrTabRings=4
+Observation.Beam[0].nrTiedArrayBeams=13
+Observation.Beam[0].startTime=
+Observation.Beam[0].subbandList=[100..261]
+Observation.Beam[0].tabRingSize=0.0042799
+Observation.Beam[0].target=LOTAAS-P1562C-SAP0
+Observation.Beam[1].TiedArrayBeam[0].angle1=2.10423944005
+Observation.Beam[1].TiedArrayBeam[0].angle2=0.468519360905
+Observation.Beam[1].TiedArrayBeam[0].coherent=true
+Observation.Beam[1].TiedArrayBeam[0].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[0].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[10].angle1=2.07055933522
+Observation.Beam[1].TiedArrayBeam[10].angle2=0.451018595429
+Observation.Beam[1].TiedArrayBeam[10].coherent=true
+Observation.Beam[1].TiedArrayBeam[10].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[10].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[11].angle1=2.08467120412
+Observation.Beam[1].TiedArrayBeam[11].angle2=0.463830044928
+Observation.Beam[1].TiedArrayBeam[11].coherent=true
+Observation.Beam[1].TiedArrayBeam[11].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[11].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[12].angle1=0
+Observation.Beam[1].TiedArrayBeam[12].angle2=0
+Observation.Beam[1].TiedArrayBeam[12].coherent=false
+Observation.Beam[1].TiedArrayBeam[12].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[12].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[1].angle1=2.12380767598
+Observation.Beam[1].TiedArrayBeam[1].angle2=0.463830044928
+Observation.Beam[1].TiedArrayBeam[1].coherent=true
+Observation.Beam[1].TiedArrayBeam[1].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[1].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[2].angle1=2.13791954489
+Observation.Beam[1].TiedArrayBeam[2].angle2=0.451018595429
+Observation.Beam[1].TiedArrayBeam[2].coherent=true
+Observation.Beam[1].TiedArrayBeam[2].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[2].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[3].angle1=2.14280889014
+Observation.Beam[1].TiedArrayBeam[3].angle2=0.433517829953
+Observation.Beam[1].TiedArrayBeam[3].coherent=true
+Observation.Beam[1].TiedArrayBeam[3].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[3].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[4].angle1=2.13737819979
+Observation.Beam[1].TiedArrayBeam[4].angle2=0.416017064478
+Observation.Beam[1].TiedArrayBeam[4].coherent=true
+Observation.Beam[1].TiedArrayBeam[4].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[4].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[5].angle1=2.12326598325
+Observation.Beam[1].TiedArrayBeam[5].angle2=0.403205614978
+Observation.Beam[1].TiedArrayBeam[5].coherent=true
+Observation.Beam[1].TiedArrayBeam[5].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[5].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[6].angle1=2.10423944005
+Observation.Beam[1].TiedArrayBeam[6].angle2=0.398516299002
+Observation.Beam[1].TiedArrayBeam[6].coherent=true
+Observation.Beam[1].TiedArrayBeam[6].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[6].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[7].angle1=2.08521289685
+Observation.Beam[1].TiedArrayBeam[7].angle2=0.403205614978
+Observation.Beam[1].TiedArrayBeam[7].coherent=true
+Observation.Beam[1].TiedArrayBeam[7].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[7].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[8].angle1=2.07110068032
+Observation.Beam[1].TiedArrayBeam[8].angle2=0.416017064478
+Observation.Beam[1].TiedArrayBeam[8].coherent=true
+Observation.Beam[1].TiedArrayBeam[8].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[8].dispersionMeasure=0
+Observation.Beam[1].TiedArrayBeam[9].angle1=2.06566998997
+Observation.Beam[1].TiedArrayBeam[9].angle2=0.433517829953
+Observation.Beam[1].TiedArrayBeam[9].coherent=true
+Observation.Beam[1].TiedArrayBeam[9].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[9].dispersionMeasure=0
+Observation.Beam[1].angle1=2.1042394400528517
+Observation.Beam[1].angle2=0.4335178299534014
+Observation.Beam[1].directionType=J2000
+Observation.Beam[1].duration=3600
+Observation.Beam[1].momID=1000093
+Observation.Beam[1].nrTabRings=4
+Observation.Beam[1].nrTiedArrayBeams=13
+Observation.Beam[1].startTime=
+Observation.Beam[1].subbandList=[100..261]
+Observation.Beam[1].tabRingSize=0.0042799
+Observation.Beam[1].target=LOTAAS-P1562C-SAP1
+Observation.Beam[2].TiedArrayBeam[0].angle1=2.1021394599
+Observation.Beam[2].TiedArrayBeam[0].angle2=0.401824326253
+Observation.Beam[2].TiedArrayBeam[0].coherent=true
+Observation.Beam[2].TiedArrayBeam[0].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[0].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[10].angle1=2.06944204232
+Observation.Beam[2].TiedArrayBeam[10].angle2=0.384323560778
+Observation.Beam[2].TiedArrayBeam[10].coherent=true
+Observation.Beam[2].TiedArrayBeam[10].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[10].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[11].angle1=2.08316171112
+Observation.Beam[2].TiedArrayBeam[11].angle2=0.397135010277
+Observation.Beam[2].TiedArrayBeam[11].coherent=true
+Observation.Beam[2].TiedArrayBeam[11].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[11].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[12].angle1=0
+Observation.Beam[2].TiedArrayBeam[12].angle2=0
+Observation.Beam[2].TiedArrayBeam[12].coherent=false
+Observation.Beam[2].TiedArrayBeam[12].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[12].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[1].angle1=2.12111720868
+Observation.Beam[2].TiedArrayBeam[1].angle2=0.397135010277
+Observation.Beam[2].TiedArrayBeam[1].coherent=true
+Observation.Beam[2].TiedArrayBeam[1].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[1].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[2].angle1=2.13483687747
+Observation.Beam[2].TiedArrayBeam[2].angle2=0.384323560778
+Observation.Beam[2].TiedArrayBeam[2].coherent=true
+Observation.Beam[2].TiedArrayBeam[2].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[2].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[3].angle1=2.13963554686
+Observation.Beam[2].TiedArrayBeam[3].angle2=0.366822795302
+Observation.Beam[2].TiedArrayBeam[3].coherent=true
+Observation.Beam[2].TiedArrayBeam[3].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[3].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[4].angle1=2.1344000544
+Observation.Beam[2].TiedArrayBeam[4].angle2=0.349322029826
+Observation.Beam[2].TiedArrayBeam[4].coherent=true
+Observation.Beam[2].TiedArrayBeam[4].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[4].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[5].angle1=2.12068012296
+Observation.Beam[2].TiedArrayBeam[5].angle2=0.336510580327
+Observation.Beam[2].TiedArrayBeam[5].coherent=true
+Observation.Beam[2].TiedArrayBeam[5].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[5].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[6].angle1=2.1021394599
+Observation.Beam[2].TiedArrayBeam[6].angle2=0.331821264351
+Observation.Beam[2].TiedArrayBeam[6].coherent=true
+Observation.Beam[2].TiedArrayBeam[6].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[6].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[7].angle1=2.08359879684
+Observation.Beam[2].TiedArrayBeam[7].angle2=0.336510580327
+Observation.Beam[2].TiedArrayBeam[7].coherent=true
+Observation.Beam[2].TiedArrayBeam[7].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[7].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[8].angle1=2.0698788654
+Observation.Beam[2].TiedArrayBeam[8].angle2=0.349322029826
+Observation.Beam[2].TiedArrayBeam[8].coherent=true
+Observation.Beam[2].TiedArrayBeam[8].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[8].dispersionMeasure=0
+Observation.Beam[2].TiedArrayBeam[9].angle1=2.06464337293
+Observation.Beam[2].TiedArrayBeam[9].angle2=0.366822795302
+Observation.Beam[2].TiedArrayBeam[9].coherent=true
+Observation.Beam[2].TiedArrayBeam[9].directionType=J2000
+Observation.Beam[2].TiedArrayBeam[9].dispersionMeasure=0
+Observation.Beam[2].angle1=2.102139459896852
+Observation.Beam[2].angle2=0.3668227953020712
+Observation.Beam[2].directionType=J2000
+Observation.Beam[2].duration=3600
+Observation.Beam[2].momID=1000094
+Observation.Beam[2].nrTabRings=4
+Observation.Beam[2].nrTiedArrayBeams=13
+Observation.Beam[2].startTime=
+Observation.Beam[2].subbandList=[100..261]
+Observation.Beam[2].tabRingSize=0.0042799
+Observation.Beam[2].target=LOTAAS-P1562C-SAP2
+
+#
+# ----- Output settings -----
+#
+Observation.DataProducts.Output_CoherentStokes.enabled=true
+Observation.DataProducts.Output_CoherentStokes.filenames=[L763847_SAP000_B000_S0_P000_bf.h5,L763847_SAP000_B001_S0_P000_bf.h5,L763847_SAP000_B002_S0_P000_bf.h5,L763847_SAP000_B003_S0_P000_bf.h5,L763847_SAP000_B004_S0_P000_bf.h5,L763847_SAP000_B005_S0_P000_bf.h5,L763847_SAP000_B006_S0_P000_bf.h5,L763847_SAP000_B007_S0_P000_bf.h5,L763847_SAP000_B008_S0_P000_bf.h5,L763847_SAP000_B009_S0_P000_bf.h5,L763847_SAP000_B010_S0_P000_bf.h5,L763847_SAP000_B011_S0_P000_bf.h5,L763847_SAP000_B013_S0_P000_bf.h5,L763847_SAP000_B014_S0_P000_bf.h5,L763847_SAP000_B015_S0_P000_bf.h5,L763847_SAP000_B016_S0_P000_bf.h5,L763847_SAP000_B017_S0_P000_bf.h5,L763847_SAP000_B018_S0_P000_bf.h5,L763847_SAP000_B019_S0_P000_bf.h5,L763847_SAP000_B020_S0_P000_bf.h5,L763847_SAP000_B021_S0_P000_bf.h5,L763847_SAP000_B022_S0_P000_bf.h5,L763847_SAP000_B023_S0_P000_bf.h5,L763847_SAP000_B024_S0_P000_bf.h5,L763847_SAP000_B025_S0_P000_bf.h5,L763847_SAP000_B026_S0_P000_bf.h5,L763847_SAP000_B027_S0_P000_bf.h5,L763847_SAP000_B028_S0_P000_bf.h5,L763847_SAP000_B029_S0_P000_bf.h5,L763847_SAP000_B030_S0_P000_bf.h5,L763847_SAP000_B031_S0_P000_bf.h5,L763847_SAP000_B032_S0_P000_bf.h5,L763847_SAP000_B033_S0_P000_bf.h5,L763847_SAP000_B034_S0_P000_bf.h5,L763847_SAP000_B035_S0_P000_bf.h5,L763847_SAP000_B036_S0_P000_bf.h5,L763847_SAP000_B037_S0_P000_bf.h5,L763847_SAP000_B038_S0_P000_bf.h5,L763847_SAP000_B039_S0_P000_bf.h5,L763847_SAP000_B040_S0_P000_bf.h5,L763847_SAP000_B041_S0_P000_bf.h5,L763847_SAP000_B042_S0_P000_bf.h5,L763847_SAP000_B043_S0_P000_bf.h5,L763847_SAP000_B044_S0_P000_bf.h5,L763847_SAP000_B045_S0_P000_bf.h5,L763847_SAP000_B046_S0_P000_bf.h5,L763847_SAP000_B047_S0_P000_bf.h5,L763847_SAP000_B048_S0_P000_bf.h5,L763847_SAP000_B049_S0_P000_bf.h5,L763847_SAP000_B050_S0_P000_bf.h5,L763847_SAP000_B051_S0_P000_bf.h5,L763847_SAP000_B052_S0_P000_bf.h5,L763847_SAP000_B053_S0_P000_bf.h5,L763847_SAP000_B054_S0_P000_bf.h5,L763847_SAP000_B055_S0_P000_bf.h5,L763847_SAP000_B056_S0_P000_bf.h5,L763847_SAP000_B057_S0_P000_bf.h5,L763847_SAP000_B058_S0_P000_bf.h5,L763847_SAP000_B059_S0_P000_bf.h5,L763847_SAP000_B060_S0_P000_bf.h5,L763847_SAP000_B061_S0_P000_bf.h5,L763847_SAP000_B062_S0_P000_bf.h5,L763847_SAP000_B063_S0_P000_bf.h5,L763847_SAP000_B064_S0_P000_bf.h5,L763847_SAP000_B065_S0_P000_bf.h5,L763847_SAP000_B066_S0_P000_bf.h5,L763847_SAP000_B067_S0_P000_bf.h5,L763847_SAP000_B068_S0_P000_bf.h5,L763847_SAP000_B069_S0_P000_bf.h5,L763847_SAP000_B070_S0_P000_bf.h5,L763847_SAP000_B071_S0_P000_bf.h5,L763847_SAP000_B072_S0_P000_bf.h5,L763847_SAP000_B073_S0_P000_bf.h5,L763847_SAP001_B000_S0_P000_bf.h5,L763847_SAP001_B001_S0_P000_bf.h5,L763847_SAP001_B002_S0_P000_bf.h5,L763847_SAP001_B003_S0_P000_bf.h5,L763847_SAP001_B004_S0_P000_bf.h5,L763847_SAP001_B005_S0_P000_bf.h5,L763847_SAP001_B006_S0_P000_bf.h5,L763847_SAP001_B007_S0_P000_bf.h5,L763847_SAP001_B008_S0_P000_bf.h5,L763847_SAP001_B009_S0_P000_bf.h5,L763847_SAP001_B010_S0_P000_bf.h5,L763847_SAP001_B011_S0_P000_bf.h5,L763847_SAP001_B013_S0_P000_bf.h5,L763847_SAP001_B014_S0_P000_bf.h5,L763847_SAP001_B015_S0_P000_bf.h5,L763847_SAP001_B016_S0_P000_bf.h5,L763847_SAP001_B017_S0_P000_bf.h5,L763847_SAP001_B018_S0_P000_bf.h5,L763847_SAP001_B019_S0_P000_bf.h5,L763847_SAP001_B020_S0_P000_bf.h5,L763847_SAP001_B021_S0_P000_bf.h5,L763847_SAP001_B022_S0_P000_bf.h5,L763847_SAP001_B023_S0_P000_bf.h5,L763847_SAP001_B024_S0_P000_bf.h5,L763847_SAP001_B025_S0_P000_bf.h5,L763847_SAP001_B026_S0_P000_bf.h5,L763847_SAP001_B027_S0_P000_bf.h5,L763847_SAP001_B028_S0_P000_bf.h5,L763847_SAP001_B029_S0_P000_bf.h5,L763847_SAP001_B030_S0_P000_bf.h5,L763847_SAP001_B031_S0_P000_bf.h5,L763847_SAP001_B032_S0_P000_bf.h5,L763847_SAP001_B033_S0_P000_bf.h5,L763847_SAP001_B034_S0_P000_bf.h5,L763847_SAP001_B035_S0_P000_bf.h5,L763847_SAP001_B036_S0_P000_bf.h5,L763847_SAP001_B037_S0_P000_bf.h5,L763847_SAP001_B038_S0_P000_bf.h5,L763847_SAP001_B039_S0_P000_bf.h5,L763847_SAP001_B040_S0_P000_bf.h5,L763847_SAP001_B041_S0_P000_bf.h5,L763847_SAP001_B042_S0_P000_bf.h5,L763847_SAP001_B043_S0_P000_bf.h5,L763847_SAP001_B044_S0_P000_bf.h5,L763847_SAP001_B045_S0_P000_bf.h5,L763847_SAP001_B046_S0_P000_bf.h5,L763847_SAP001_B047_S0_P000_bf.h5,L763847_SAP001_B048_S0_P000_bf.h5,L763847_SAP001_B049_S0_P000_bf.h5,L763847_SAP001_B050_S0_P000_bf.h5,L763847_SAP001_B051_S0_P000_bf.h5,L763847_SAP001_B052_S0_P000_bf.h5,L763847_SAP001_B053_S0_P000_bf.h5,L763847_SAP001_B054_S0_P000_bf.h5,L763847_SAP001_B055_S0_P000_bf.h5,L763847_SAP001_B056_S0_P000_bf.h5,L763847_SAP001_B057_S0_P000_bf.h5,L763847_SAP001_B058_S0_P000_bf.h5,L763847_SAP001_B059_S0_P000_bf.h5,L763847_SAP001_B060_S0_P000_bf.h5,L763847_SAP001_B061_S0_P000_bf.h5,L763847_SAP001_B062_S0_P000_bf.h5,L763847_SAP001_B063_S0_P000_bf.h5,L763847_SAP001_B064_S0_P000_bf.h5,L763847_SAP001_B065_S0_P000_bf.h5,L763847_SAP001_B066_S0_P000_bf.h5,L763847_SAP001_B067_S0_P000_bf.h5,L763847_SAP001_B068_S0_P000_bf.h5,L763847_SAP001_B069_S0_P000_bf.h5,L763847_SAP001_B070_S0_P000_bf.h5,L763847_SAP001_B071_S0_P000_bf.h5,L763847_SAP001_B072_S0_P000_bf.h5,L763847_SAP001_B073_S0_P000_bf.h5,L763847_SAP002_B000_S0_P000_bf.h5,L763847_SAP002_B001_S0_P000_bf.h5,L763847_SAP002_B002_S0_P000_bf.h5,L763847_SAP002_B003_S0_P000_bf.h5,L763847_SAP002_B004_S0_P000_bf.h5,L763847_SAP002_B005_S0_P000_bf.h5,L763847_SAP002_B006_S0_P000_bf.h5,L763847_SAP002_B007_S0_P000_bf.h5,L763847_SAP002_B008_S0_P000_bf.h5,L763847_SAP002_B009_S0_P000_bf.h5,L763847_SAP002_B010_S0_P000_bf.h5,L763847_SAP002_B011_S0_P000_bf.h5,L763847_SAP002_B013_S0_P000_bf.h5,L763847_SAP002_B014_S0_P000_bf.h5,L763847_SAP002_B015_S0_P000_bf.h5,L763847_SAP002_B016_S0_P000_bf.h5,L763847_SAP002_B017_S0_P000_bf.h5,L763847_SAP002_B018_S0_P000_bf.h5,L763847_SAP002_B019_S0_P000_bf.h5,L763847_SAP002_B020_S0_P000_bf.h5,L763847_SAP002_B021_S0_P000_bf.h5,L763847_SAP002_B022_S0_P000_bf.h5,L763847_SAP002_B023_S0_P000_bf.h5,L763847_SAP002_B024_S0_P000_bf.h5,L763847_SAP002_B025_S0_P000_bf.h5,L763847_SAP002_B026_S0_P000_bf.h5,L763847_SAP002_B027_S0_P000_bf.h5,L763847_SAP002_B028_S0_P000_bf.h5,L763847_SAP002_B029_S0_P000_bf.h5,L763847_SAP002_B030_S0_P000_bf.h5,L763847_SAP002_B031_S0_P000_bf.h5,L763847_SAP002_B032_S0_P000_bf.h5,L763847_SAP002_B033_S0_P000_bf.h5,L763847_SAP002_B034_S0_P000_bf.h5,L763847_SAP002_B035_S0_P000_bf.h5,L763847_SAP002_B036_S0_P000_bf.h5,L763847_SAP002_B037_S0_P000_bf.h5,L763847_SAP002_B038_S0_P000_bf.h5,L763847_SAP002_B039_S0_P000_bf.h5,L763847_SAP002_B040_S0_P000_bf.h5,L763847_SAP002_B041_S0_P000_bf.h5,L763847_SAP002_B042_S0_P000_bf.h5,L763847_SAP002_B043_S0_P000_bf.h5,L763847_SAP002_B044_S0_P000_bf.h5,L763847_SAP002_B045_S0_P000_bf.h5,L763847_SAP002_B046_S0_P000_bf.h5,L763847_SAP002_B047_S0_P000_bf.h5,L763847_SAP002_B048_S0_P000_bf.h5,L763847_SAP002_B049_S0_P000_bf.h5,L763847_SAP002_B050_S0_P000_bf.h5,L763847_SAP002_B051_S0_P000_bf.h5,L763847_SAP002_B052_S0_P000_bf.h5,L763847_SAP002_B053_S0_P000_bf.h5,L763847_SAP002_B054_S0_P000_bf.h5,L763847_SAP002_B055_S0_P000_bf.h5,L763847_SAP002_B056_S0_P000_bf.h5,L763847_SAP002_B057_S0_P000_bf.h5,L763847_SAP002_B058_S0_P000_bf.h5,L763847_SAP002_B059_S0_P000_bf.h5,L763847_SAP002_B060_S0_P000_bf.h5,L763847_SAP002_B061_S0_P000_bf.h5,L763847_SAP002_B062_S0_P000_bf.h5,L763847_SAP002_B063_S0_P000_bf.h5,L763847_SAP002_B064_S0_P000_bf.h5,L763847_SAP002_B065_S0_P000_bf.h5,L763847_SAP002_B066_S0_P000_bf.h5,L763847_SAP002_B067_S0_P000_bf.h5,L763847_SAP002_B068_S0_P000_bf.h5,L763847_SAP002_B069_S0_P000_bf.h5,L763847_SAP002_B070_S0_P000_bf.h5,L763847_SAP002_B071_S0_P000_bf.h5,L763847_SAP002_B072_S0_P000_bf.h5,L763847_SAP002_B073_S0_P000_bf.h5]
+Observation.DataProducts.Output_CoherentStokes.locations=[220*localhost:output/763847]
+Observation.DataProducts.Output_CoherentStokes.storageClusterName="localhost"
+
+Observation.DataProducts.Output_Correlated.enabled=false
+
+Observation.DataProducts.Output_IncoherentStokes.enabled=true
+Observation.DataProducts.Output_IncoherentStokes.filenames=[L763847_SAP000_B012_S0_P000_bf.h5,L763847_SAP001_B012_S0_P000_bf.h5,L763847_SAP002_B012_S0_P000_bf.h5]
+Observation.DataProducts.Output_IncoherentStokes.locations=[218*localhost:output/763847]
+Observation.DataProducts.Output_IncoherentStokes.storageClusterName="localhost"
+
+Observation.DataProducts.Output_InstrumentModel.enabled=false
+Observation.DataProducts.Output_Pulsar.enabled=false
+Observation.DataProducts.Output_SkyImage.enabled=false
+
+#
+# ----- Dataslot settings, remove? -----
+#
+Observation.Dataslots.CS002HBA0.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS002HBA0.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS002HBA1.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS002HBA1.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS003HBA0.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS003HBA0.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS003HBA1.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS003HBA1.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS004HBA0.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS004HBA0.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS004HBA1.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS004HBA1.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS005HBA0.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS005HBA0.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS005HBA1.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS005HBA1.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS006HBA0.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS006HBA0.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS006HBA1.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS006HBA1.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS007HBA0.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS007HBA0.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.CS007HBA1.DataslotList=[0..121,0..121,0..121,0..119]
+Observation.Dataslots.CS007HBA1.RSPBoardList=[122*0,122*1,122*2,120*3]
+Observation.Dataslots.DataslotInfo.DataslotList=[]
+Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+
+#
+# ----- Observation settings -----
+#
+Observation.ObsID=763847
+Observation.VirtualInstrument.minimalNrStations=1
+Observation.VirtualInstrument.stationList=[CS007,CS006,CS003,CS002,CS005,CS004]
+Observation.VirtualInstrument.stationSet=Custom
+Observation.antennaArray=HBA
+Observation.antennaSet=HBA_DUAL
+Observation.bandFilter=HBA_110_190
+Observation.claimPeriod=35
+Observation.clockMode=<<Clock200
+Observation.momID=1000091
+Observation.nrAnaBeams=1
+Observation.nrBeams=3
+Observation.nrBitsPerSample=8
+Observation.nrTBBSettings=0
+Observation.originID=650576
+Observation.otdbID=763847
+Observation.preparePeriod=20
+Observation.processSubtype=Beam Observation
+Observation.processType=Observation
+Observation.sampleClock=200
+Observation.startTime=2020-01-04 00:01:00
+Observation.stopTime=2020-01-04 01:01:00
+Observation.strategy=default
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-784441.parset b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-784441.parset
new file mode 100644
index 0000000000000000000000000000000000000000..c94cc68015ba0ba4285365719995e13d60733382
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-784441.parset
@@ -0,0 +1,196 @@
+#
+# ----- Info -----
+#
+# Purpose: LOFAR reference test
+# Solar observing
+# Intended to be used with gpu_load test for GPU kernel performance benchmarking
+
+# Enable performance benchmarking
+Cobalt.Benchmark.enabled=true
+Cobalt.Benchmark.file=Benchmarks/784441.csv
+
+# These figures are used by gpu_load to predict the load across the whole cluster.
+# Set available processing resources for the full COBALT 2.0 system (2 GPUs/sockets x 11 nodes)
+Cobalt.Nodes = [ 22*localhost ]
+
+#
+# ----- Beamformer settings -----
+#
+Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=16
+Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=128
+Cobalt.BeamFormer.CoherentStokes.which=IQUV
+Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=1
+Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=1
+Cobalt.BeamFormer.IncoherentStokes.which=I
+Cobalt.BeamFormer.coherentDedisperseChannels=false
+Cobalt.BeamFormer.flysEye=false
+Cobalt.BeamFormer.stationList=[]
+
+#
+# ----- Correlator settings -----
+#
+Cobalt.Correlator.integrationTime=0.671089
+Cobalt.Correlator.nrBlocksPerIntegration=1
+Cobalt.Correlator.nrChannelsPerSubband=16
+Cobalt.Correlator.nrIntegrationsPerBlock=1
+
+#
+# ----- Misc settings -----
+#
+Cobalt.blockSize=131072
+Cobalt.correctBandPass=true
+Cobalt.correctClocks=true
+Cobalt.delayCompensation=true
+Cobalt.realTime=true
+
+#
+# ----- Pointings -----
+#
+Observation.Beam[0].angle1=5.166310285252558
+Observation.Beam[0].angle2=-0.3888216873213144
+Observation.Beam[0].directionType=J2000
+Observation.Beam[0].duration=0
+Observation.Beam[0].momID=1021616
+Observation.Beam[0].nrTabRings=6
+Observation.Beam[0].nrTiedArrayBeams=0
+Observation.Beam[0].startTime=
+Observation.Beam[0].subbandList=[98,100,106,110,117,121,126,131,136,141,146,150,156,161,172,178,183,189,194,201,206,211,216,221,232,237,244,249,255,263,269,274,279,283,295,300,302,312,316,320,328,330,333,338,340,347,351,356,360,366,370,376,380,385,389,391,394,400,404,410]
+Observation.Beam[0].tabRingSize=0.00174533
+Observation.Beam[0].target=Sun
+Observation.Beam[1].TiedArrayBeam[0].angle1=1.4596726677614609
+Observation.Beam[1].TiedArrayBeam[0].angle2=0.3842255081802917
+Observation.Beam[1].TiedArrayBeam[0].coherent=true
+Observation.Beam[1].TiedArrayBeam[0].directionType=J2000
+Observation.Beam[1].TiedArrayBeam[0].dispersionMeasure=0
+Observation.Beam[1].angle1=5.233686568453055
+Observation.Beam[1].angle2=0.7109409436737796
+Observation.Beam[1].directionType=J2000
+Observation.Beam[1].duration=0
+Observation.Beam[1].momID=1021617
+Observation.Beam[1].nrTabRings=0
+Observation.Beam[1].nrTiedArrayBeams=1
+Observation.Beam[1].startTime=
+Observation.Beam[1].subbandList=[98,100,106,110,117,121,126,131,136,141,146,150,156,161,172,178,183,189,194,201,206,211,216,221,232,237,244,249,255,263,269,274,279,283,295,300,302,312,316,320,328,330,333,338,340,347,351,356,360,366,370,376,380,385,389,391,394,400,404,410]
+Observation.Beam[1].tabRingSize=0
+Observation.Beam[1].target=CygA
+
+#
+# ----- Output settings -----
+#
+Observation.DataProducts.Output_CoherentStokes.enabled=true
+Observation.DataProducts.Output_CoherentStokes.filenames=[L784441_SAP000_B000_S0_P000_bf.h5,L784441_SAP000_B000_S1_P000_bf.h5,L784441_SAP000_B000_S2_P000_bf.h5,L784441_SAP000_B000_S3_P000_bf.h5,L784441_SAP000_B001_S0_P000_bf.h5,L784441_SAP000_B001_S1_P000_bf.h5,L784441_SAP000_B001_S2_P000_bf.h5,L784441_SAP000_B001_S3_P000_bf.h5,L784441_SAP000_B002_S0_P000_bf.h5,L784441_SAP000_B002_S1_P000_bf.h5,L784441_SAP000_B002_S2_P000_bf.h5,L784441_SAP000_B002_S3_P000_bf.h5,L784441_SAP000_B003_S0_P000_bf.h5,L784441_SAP000_B003_S1_P000_bf.h5,L784441_SAP000_B003_S2_P000_bf.h5,L784441_SAP000_B003_S3_P000_bf.h5,L784441_SAP000_B004_S0_P000_bf.h5,L784441_SAP000_B004_S1_P000_bf.h5,L784441_SAP000_B004_S2_P000_bf.h5,L784441_SAP000_B004_S3_P000_bf.h5,L784441_SAP000_B005_S0_P000_bf.h5,L784441_SAP000_B005_S1_P000_bf.h5,L784441_SAP000_B005_S2_P000_bf.h5,L784441_SAP000_B005_S3_P000_bf.h5,L784441_SAP000_B006_S0_P000_bf.h5,L784441_SAP000_B006_S1_P000_bf.h5,L784441_SAP000_B006_S2_P000_bf.h5,L784441_SAP000_B006_S3_P000_bf.h5,L784441_SAP000_B007_S0_P000_bf.h5,L784441_SAP000_B007_S1_P000_bf.h5,L784441_SAP000_B007_S2_P000_bf.h5,L784441_SAP000_B007_S3_P000_bf.h5,L784441_SAP000_B008_S0_P000_bf.h5,L784441_SAP000_B008_S1_P000_bf.h5,L784441_SAP000_B008_S2_P000_bf.h5,L784441_SAP000_B008_S3_P000_bf.h5,L784441_SAP000_B009_S0_P000_bf.h5,L784441_SAP000_B009_S1_P000_bf.h5,L784441_SAP000_B009_S2_P000_bf.h5,L784441_SAP000_B009_S3_P000_bf.h5,L784441_SAP000_B010_S0_P000_bf.h5,L784441_SAP000_B010_S1_P000_bf.h5,L784441_SAP000_B010_S2_P000_bf.h5,L784441_SAP000_B010_S3_P000_bf.h5,L784441_SAP000_B011_S0_P000_bf.h5,L784441_SAP000_B011_S1_P000_bf.h5,L784441_SAP000_B011_S2_P000_bf.h5,L784441_SAP000_B011_S3_P000_bf.h5,L784441_SAP000_B012_S0_P000_bf.h5,L784441_SAP000_B012_S1_P000_bf.h5,L784441_SAP000_B012_S2_P000_bf.h5,L784441_SAP000_B012_S3_P000_bf.h5,L784441_SAP000_B013_S0_P000_bf.h5,L784441_SAP000_B013_S1_P000_bf.h5,L784441_SAP000_B013_S2_P000_bf.h5,L784441_SAP000_B013_S3_P000_bf.h5,L784441_SAP000_B014_S0_P000_bf.h5,L784441_SAP000_B014_S1_P000_bf.h5,L784441_SAP000_B014_S2_P000_bf.h5,L784441_SAP000_B014_S3_P000_bf.h5,L784441_SAP000_B015_S0_P000_bf.h5,L784441_SAP000_B015_S1_P000_bf.h5,L784441_SAP000_B015_S2_P000_bf.h5,L784441_SAP000_B015_S3_P000_bf.h5,L784441_SAP000_B016_S0_P000_bf.h5,L784441_SAP000_B016_S1_P000_bf.h5,L784441_SAP000_B016_S2_P000_bf.h5,L784441_SAP000_B016_S3_P000_bf.h5,L784441_SAP000_B017_S0_P000_bf.h5,L784441_SAP000_B017_S1_P000_bf.h5,L784441_SAP000_B017_S2_P000_bf.h5,L784441_SAP000_B017_S3_P000_bf.h5,L784441_SAP000_B018_S0_P000_bf.h5,L784441_SAP000_B018_S1_P000_bf.h5,L784441_SAP000_B018_S2_P000_bf.h5,L784441_SAP000_B018_S3_P000_bf.h5,L784441_SAP000_B019_S0_P000_bf.h5,L784441_SAP000_B019_S1_P000_bf.h5,L784441_SAP000_B019_S2_P000_bf.h5,L784441_SAP000_B019_S3_P000_bf.h5,L784441_SAP000_B020_S0_P000_bf.h5,L784441_SAP000_B020_S1_P000_bf.h5,L784441_SAP000_B020_S2_P000_bf.h5,L784441_SAP000_B020_S3_P000_bf.h5,L784441_SAP000_B021_S0_P000_bf.h5,L784441_SAP000_B021_S1_P000_bf.h5,L784441_SAP000_B021_S2_P000_bf.h5,L784441_SAP000_B021_S3_P000_bf.h5,L784441_SAP000_B022_S0_P000_bf.h5,L784441_SAP000_B022_S1_P000_bf.h5,L784441_SAP000_B022_S2_P000_bf.h5,L784441_SAP000_B022_S3_P000_bf.h5,L784441_SAP000_B023_S0_P000_bf.h5,L784441_SAP000_B023_S1_P000_bf.h5,L784441_SAP000_B023_S2_P000_bf.h5,L784441_SAP000_B023_S3_P000_bf.h5,L784441_SAP000_B024_S0_P000_bf.h5,L784441_SAP000_B024_S1_P000_bf.h5,L784441_SAP000_B024_S2_P000_bf.h5,L784441_SAP000_B024_S3_P000_bf.h5,L784441_SAP000_B025_S0_P000_bf.h5,L784441_SAP000_B025_S1_P000_bf.h5,L784441_SAP000_B025_S2_P000_bf.h5,L784441_SAP000_B025_S3_P000_bf.h5,L784441_SAP000_B026_S0_P000_bf.h5,L784441_SAP000_B026_S1_P000_bf.h5,L784441_SAP000_B026_S2_P000_bf.h5,L784441_SAP000_B026_S3_P000_bf.h5,L784441_SAP000_B027_S0_P000_bf.h5,L784441_SAP000_B027_S1_P000_bf.h5,L784441_SAP000_B027_S2_P000_bf.h5,L784441_SAP000_B027_S3_P000_bf.h5,L784441_SAP000_B028_S0_P000_bf.h5,L784441_SAP000_B028_S1_P000_bf.h5,L784441_SAP000_B028_S2_P000_bf.h5,L784441_SAP000_B028_S3_P000_bf.h5,L784441_SAP000_B029_S0_P000_bf.h5,L784441_SAP000_B029_S1_P000_bf.h5,L784441_SAP000_B029_S2_P000_bf.h5,L784441_SAP000_B029_S3_P000_bf.h5,L784441_SAP000_B030_S0_P000_bf.h5,L784441_SAP000_B030_S1_P000_bf.h5,L784441_SAP000_B030_S2_P000_bf.h5,L784441_SAP000_B030_S3_P000_bf.h5,L784441_SAP000_B031_S0_P000_bf.h5,L784441_SAP000_B031_S1_P000_bf.h5,L784441_SAP000_B031_S2_P000_bf.h5,L784441_SAP000_B031_S3_P000_bf.h5,L784441_SAP000_B032_S0_P000_bf.h5,L784441_SAP000_B032_S1_P000_bf.h5,L784441_SAP000_B032_S2_P000_bf.h5,L784441_SAP000_B032_S3_P000_bf.h5,L784441_SAP000_B033_S0_P000_bf.h5,L784441_SAP000_B033_S1_P000_bf.h5,L784441_SAP000_B033_S2_P000_bf.h5,L784441_SAP000_B033_S3_P000_bf.h5,L784441_SAP000_B034_S0_P000_bf.h5,L784441_SAP000_B034_S1_P000_bf.h5,L784441_SAP000_B034_S2_P000_bf.h5,L784441_SAP000_B034_S3_P000_bf.h5,L784441_SAP000_B035_S0_P000_bf.h5,L784441_SAP000_B035_S1_P000_bf.h5,L784441_SAP000_B035_S2_P000_bf.h5,L784441_SAP000_B035_S3_P000_bf.h5,L784441_SAP000_B036_S0_P000_bf.h5,L784441_SAP000_B036_S1_P000_bf.h5,L784441_SAP000_B036_S2_P000_bf.h5,L784441_SAP000_B036_S3_P000_bf.h5,L784441_SAP000_B037_S0_P000_bf.h5,L784441_SAP000_B037_S1_P000_bf.h5,L784441_SAP000_B037_S2_P000_bf.h5,L784441_SAP000_B037_S3_P000_bf.h5,L784441_SAP000_B038_S0_P000_bf.h5,L784441_SAP000_B038_S1_P000_bf.h5,L784441_SAP000_B038_S2_P000_bf.h5,L784441_SAP000_B038_S3_P000_bf.h5,L784441_SAP000_B039_S0_P000_bf.h5,L784441_SAP000_B039_S1_P000_bf.h5,L784441_SAP000_B039_S2_P000_bf.h5,L784441_SAP000_B039_S3_P000_bf.h5,L784441_SAP000_B040_S0_P000_bf.h5,L784441_SAP000_B040_S1_P000_bf.h5,L784441_SAP000_B040_S2_P000_bf.h5,L784441_SAP000_B040_S3_P000_bf.h5,L784441_SAP000_B041_S0_P000_bf.h5,L784441_SAP000_B041_S1_P000_bf.h5,L784441_SAP000_B041_S2_P000_bf.h5,L784441_SAP000_B041_S3_P000_bf.h5,L784441_SAP000_B042_S0_P000_bf.h5,L784441_SAP000_B042_S1_P000_bf.h5,L784441_SAP000_B042_S2_P000_bf.h5,L784441_SAP000_B042_S3_P000_bf.h5,L784441_SAP000_B043_S0_P000_bf.h5,L784441_SAP000_B043_S1_P000_bf.h5,L784441_SAP000_B043_S2_P000_bf.h5,L784441_SAP000_B043_S3_P000_bf.h5,L784441_SAP000_B044_S0_P000_bf.h5,L784441_SAP000_B044_S1_P000_bf.h5,L784441_SAP000_B044_S2_P000_bf.h5,L784441_SAP000_B044_S3_P000_bf.h5,L784441_SAP000_B045_S0_P000_bf.h5,L784441_SAP000_B045_S1_P000_bf.h5,L784441_SAP000_B045_S2_P000_bf.h5,L784441_SAP000_B045_S3_P000_bf.h5,L784441_SAP000_B046_S0_P000_bf.h5,L784441_SAP000_B046_S1_P000_bf.h5,L784441_SAP000_B046_S2_P000_bf.h5,L784441_SAP000_B046_S3_P000_bf.h5,L784441_SAP000_B047_S0_P000_bf.h5,L784441_SAP000_B047_S1_P000_bf.h5,L784441_SAP000_B047_S2_P000_bf.h5,L784441_SAP000_B047_S3_P000_bf.h5,L784441_SAP000_B048_S0_P000_bf.h5,L784441_SAP000_B048_S1_P000_bf.h5,L784441_SAP000_B048_S2_P000_bf.h5,L784441_SAP000_B048_S3_P000_bf.h5,L784441_SAP000_B049_S0_P000_bf.h5,L784441_SAP000_B049_S1_P000_bf.h5,L784441_SAP000_B049_S2_P000_bf.h5,L784441_SAP000_B049_S3_P000_bf.h5,L784441_SAP000_B050_S0_P000_bf.h5,L784441_SAP000_B050_S1_P000_bf.h5,L784441_SAP000_B050_S2_P000_bf.h5,L784441_SAP000_B050_S3_P000_bf.h5,L784441_SAP000_B051_S0_P000_bf.h5,L784441_SAP000_B051_S1_P000_bf.h5,L784441_SAP000_B051_S2_P000_bf.h5,L784441_SAP000_B051_S3_P000_bf.h5,L784441_SAP000_B052_S0_P000_bf.h5,L784441_SAP000_B052_S1_P000_bf.h5,L784441_SAP000_B052_S2_P000_bf.h5,L784441_SAP000_B052_S3_P000_bf.h5,L784441_SAP000_B053_S0_P000_bf.h5,L784441_SAP000_B053_S1_P000_bf.h5,L784441_SAP000_B053_S2_P000_bf.h5,L784441_SAP000_B053_S3_P000_bf.h5,L784441_SAP000_B054_S0_P000_bf.h5,L784441_SAP000_B054_S1_P000_bf.h5,L784441_SAP000_B054_S2_P000_bf.h5,L784441_SAP000_B054_S3_P000_bf.h5,L784441_SAP000_B055_S0_P000_bf.h5,L784441_SAP000_B055_S1_P000_bf.h5,L784441_SAP000_B055_S2_P000_bf.h5,L784441_SAP000_B055_S3_P000_bf.h5,L784441_SAP000_B056_S0_P000_bf.h5,L784441_SAP000_B056_S1_P000_bf.h5,L784441_SAP000_B056_S2_P000_bf.h5,L784441_SAP000_B056_S3_P000_bf.h5,L784441_SAP000_B057_S0_P000_bf.h5,L784441_SAP000_B057_S1_P000_bf.h5,L784441_SAP000_B057_S2_P000_bf.h5,L784441_SAP000_B057_S3_P000_bf.h5,L784441_SAP000_B058_S0_P000_bf.h5,L784441_SAP000_B058_S1_P000_bf.h5,L784441_SAP000_B058_S2_P000_bf.h5,L784441_SAP000_B058_S3_P000_bf.h5,L784441_SAP000_B059_S0_P000_bf.h5,L784441_SAP000_B059_S1_P000_bf.h5,L784441_SAP000_B059_S2_P000_bf.h5,L784441_SAP000_B059_S3_P000_bf.h5,L784441_SAP000_B060_S0_P000_bf.h5,L784441_SAP000_B060_S1_P000_bf.h5,L784441_SAP000_B060_S2_P000_bf.h5,L784441_SAP000_B060_S3_P000_bf.h5,L784441_SAP000_B061_S0_P000_bf.h5,L784441_SAP000_B061_S1_P000_bf.h5,L784441_SAP000_B061_S2_P000_bf.h5,L784441_SAP000_B061_S3_P000_bf.h5,L784441_SAP000_B062_S0_P000_bf.h5,L784441_SAP000_B062_S1_P000_bf.h5,L784441_SAP000_B062_S2_P000_bf.h5,L784441_SAP000_B062_S3_P000_bf.h5,L784441_SAP000_B063_S0_P000_bf.h5,L784441_SAP000_B063_S1_P000_bf.h5,L784441_SAP000_B063_S2_P000_bf.h5,L784441_SAP000_B063_S3_P000_bf.h5,L784441_SAP000_B064_S0_P000_bf.h5,L784441_SAP000_B064_S1_P000_bf.h5,L784441_SAP000_B064_S2_P000_bf.h5,L784441_SAP000_B064_S3_P000_bf.h5,L784441_SAP000_B065_S0_P000_bf.h5,L784441_SAP000_B065_S1_P000_bf.h5,L784441_SAP000_B065_S2_P000_bf.h5,L784441_SAP000_B065_S3_P000_bf.h5,L784441_SAP000_B066_S0_P000_bf.h5,L784441_SAP000_B066_S1_P000_bf.h5,L784441_SAP000_B066_S2_P000_bf.h5,L784441_SAP000_B066_S3_P000_bf.h5,L784441_SAP000_B067_S0_P000_bf.h5,L784441_SAP000_B067_S1_P000_bf.h5,L784441_SAP000_B067_S2_P000_bf.h5,L784441_SAP000_B067_S3_P000_bf.h5,L784441_SAP000_B068_S0_P000_bf.h5,L784441_SAP000_B068_S1_P000_bf.h5,L784441_SAP000_B068_S2_P000_bf.h5,L784441_SAP000_B068_S3_P000_bf.h5,L784441_SAP000_B069_S0_P000_bf.h5,L784441_SAP000_B069_S1_P000_bf.h5,L784441_SAP000_B069_S2_P000_bf.h5,L784441_SAP000_B069_S3_P000_bf.h5,L784441_SAP000_B070_S0_P000_bf.h5,L784441_SAP000_B070_S1_P000_bf.h5,L784441_SAP000_B070_S2_P000_bf.h5,L784441_SAP000_B070_S3_P000_bf.h5,L784441_SAP000_B071_S0_P000_bf.h5,L784441_SAP000_B071_S1_P000_bf.h5,L784441_SAP000_B071_S2_P000_bf.h5,L784441_SAP000_B071_S3_P000_bf.h5,L784441_SAP000_B072_S0_P000_bf.h5,L784441_SAP000_B072_S1_P000_bf.h5,L784441_SAP000_B072_S2_P000_bf.h5,L784441_SAP000_B072_S3_P000_bf.h5,L784441_SAP000_B073_S0_P000_bf.h5,L784441_SAP000_B073_S1_P000_bf.h5,L784441_SAP000_B073_S2_P000_bf.h5,L784441_SAP000_B073_S3_P000_bf.h5,L784441_SAP000_B074_S0_P000_bf.h5,L784441_SAP000_B074_S1_P000_bf.h5,L784441_SAP000_B074_S2_P000_bf.h5,L784441_SAP000_B074_S3_P000_bf.h5,L784441_SAP000_B075_S0_P000_bf.h5,L784441_SAP000_B075_S1_P000_bf.h5,L784441_SAP000_B075_S2_P000_bf.h5,L784441_SAP000_B075_S3_P000_bf.h5,L784441_SAP000_B076_S0_P000_bf.h5,L784441_SAP000_B076_S1_P000_bf.h5,L784441_SAP000_B076_S2_P000_bf.h5,L784441_SAP000_B076_S3_P000_bf.h5,L784441_SAP000_B077_S0_P000_bf.h5,L784441_SAP000_B077_S1_P000_bf.h5,L784441_SAP000_B077_S2_P000_bf.h5,L784441_SAP000_B077_S3_P000_bf.h5,L784441_SAP000_B078_S0_P000_bf.h5,L784441_SAP000_B078_S1_P000_bf.h5,L784441_SAP000_B078_S2_P000_bf.h5,L784441_SAP000_B078_S3_P000_bf.h5,L784441_SAP000_B079_S0_P000_bf.h5,L784441_SAP000_B079_S1_P000_bf.h5,L784441_SAP000_B079_S2_P000_bf.h5,L784441_SAP000_B079_S3_P000_bf.h5,L784441_SAP000_B080_S0_P000_bf.h5,L784441_SAP000_B080_S1_P000_bf.h5,L784441_SAP000_B080_S2_P000_bf.h5,L784441_SAP000_B080_S3_P000_bf.h5,L784441_SAP000_B081_S0_P000_bf.h5,L784441_SAP000_B081_S1_P000_bf.h5,L784441_SAP000_B081_S2_P000_bf.h5,L784441_SAP000_B081_S3_P000_bf.h5,L784441_SAP000_B082_S0_P000_bf.h5,L784441_SAP000_B082_S1_P000_bf.h5,L784441_SAP000_B082_S2_P000_bf.h5,L784441_SAP000_B082_S3_P000_bf.h5,L784441_SAP000_B083_S0_P000_bf.h5,L784441_SAP000_B083_S1_P000_bf.h5,L784441_SAP000_B083_S2_P000_bf.h5,L784441_SAP000_B083_S3_P000_bf.h5,L784441_SAP000_B084_S0_P000_bf.h5,L784441_SAP000_B084_S1_P000_bf.h5,L784441_SAP000_B084_S2_P000_bf.h5,L784441_SAP000_B084_S3_P000_bf.h5,L784441_SAP000_B085_S0_P000_bf.h5,L784441_SAP000_B085_S1_P000_bf.h5,L784441_SAP000_B085_S2_P000_bf.h5,L784441_SAP000_B085_S3_P000_bf.h5,L784441_SAP000_B086_S0_P000_bf.h5,L784441_SAP000_B086_S1_P000_bf.h5,L784441_SAP000_B086_S2_P000_bf.h5,L784441_SAP000_B086_S3_P000_bf.h5,L784441_SAP000_B087_S0_P000_bf.h5,L784441_SAP000_B087_S1_P000_bf.h5,L784441_SAP000_B087_S2_P000_bf.h5,L784441_SAP000_B087_S3_P000_bf.h5,L784441_SAP000_B088_S0_P000_bf.h5,L784441_SAP000_B088_S1_P000_bf.h5,L784441_SAP000_B088_S2_P000_bf.h5,L784441_SAP000_B088_S3_P000_bf.h5,L784441_SAP000_B089_S0_P000_bf.h5,L784441_SAP000_B089_S1_P000_bf.h5,L784441_SAP000_B089_S2_P000_bf.h5,L784441_SAP000_B089_S3_P000_bf.h5,L784441_SAP000_B090_S0_P000_bf.h5,L784441_SAP000_B090_S1_P000_bf.h5,L784441_SAP000_B090_S2_P000_bf.h5,L784441_SAP000_B090_S3_P000_bf.h5,L784441_SAP000_B091_S0_P000_bf.h5,L784441_SAP000_B091_S1_P000_bf.h5,L784441_SAP000_B091_S2_P000_bf.h5,L784441_SAP000_B091_S3_P000_bf.h5,L784441_SAP000_B092_S0_P000_bf.h5,L784441_SAP000_B092_S1_P000_bf.h5,L784441_SAP000_B092_S2_P000_bf.h5,L784441_SAP000_B092_S3_P000_bf.h5,L784441_SAP000_B093_S0_P000_bf.h5,L784441_SAP000_B093_S1_P000_bf.h5,L784441_SAP000_B093_S2_P000_bf.h5,L784441_SAP000_B093_S3_P000_bf.h5,L784441_SAP000_B094_S0_P000_bf.h5,L784441_SAP000_B094_S1_P000_bf.h5,L784441_SAP000_B094_S2_P000_bf.h5,L784441_SAP000_B094_S3_P000_bf.h5,L784441_SAP000_B095_S0_P000_bf.h5,L784441_SAP000_B095_S1_P000_bf.h5,L784441_SAP000_B095_S2_P000_bf.h5,L784441_SAP000_B095_S3_P000_bf.h5,L784441_SAP000_B096_S0_P000_bf.h5,L784441_SAP000_B096_S1_P000_bf.h5,L784441_SAP000_B096_S2_P000_bf.h5,L784441_SAP000_B096_S3_P000_bf.h5,L784441_SAP000_B097_S0_P000_bf.h5,L784441_SAP000_B097_S1_P000_bf.h5,L784441_SAP000_B097_S2_P000_bf.h5,L784441_SAP000_B097_S3_P000_bf.h5,L784441_SAP000_B098_S0_P000_bf.h5,L784441_SAP000_B098_S1_P000_bf.h5,L784441_SAP000_B098_S2_P000_bf.h5,L784441_SAP000_B098_S3_P000_bf.h5,L784441_SAP000_B099_S0_P000_bf.h5,L784441_SAP000_B099_S1_P000_bf.h5,L784441_SAP000_B099_S2_P000_bf.h5,L784441_SAP000_B099_S3_P000_bf.h5,L784441_SAP000_B100_S0_P000_bf.h5,L784441_SAP000_B100_S1_P000_bf.h5,L784441_SAP000_B100_S2_P000_bf.h5,L784441_SAP000_B100_S3_P000_bf.h5,L784441_SAP000_B101_S0_P000_bf.h5,L784441_SAP000_B101_S1_P000_bf.h5,L784441_SAP000_B101_S2_P000_bf.h5,L784441_SAP000_B101_S3_P000_bf.h5,L784441_SAP000_B102_S0_P000_bf.h5,L784441_SAP000_B102_S1_P000_bf.h5,L784441_SAP000_B102_S2_P000_bf.h5,L784441_SAP000_B102_S3_P000_bf.h5,L784441_SAP000_B103_S0_P000_bf.h5,L784441_SAP000_B103_S1_P000_bf.h5,L784441_SAP000_B103_S2_P000_bf.h5,L784441_SAP000_B103_S3_P000_bf.h5,L784441_SAP000_B104_S0_P000_bf.h5,L784441_SAP000_B104_S1_P000_bf.h5,L784441_SAP000_B104_S2_P000_bf.h5,L784441_SAP000_B104_S3_P000_bf.h5,L784441_SAP000_B105_S0_P000_bf.h5,L784441_SAP000_B105_S1_P000_bf.h5,L784441_SAP000_B105_S2_P000_bf.h5,L784441_SAP000_B105_S3_P000_bf.h5,L784441_SAP000_B106_S0_P000_bf.h5,L784441_SAP000_B106_S1_P000_bf.h5,L784441_SAP000_B106_S2_P000_bf.h5,L784441_SAP000_B106_S3_P000_bf.h5,L784441_SAP000_B107_S0_P000_bf.h5,L784441_SAP000_B107_S1_P000_bf.h5,L784441_SAP000_B107_S2_P000_bf.h5,L784441_SAP000_B107_S3_P000_bf.h5,L784441_SAP000_B108_S0_P000_bf.h5,L784441_SAP000_B108_S1_P000_bf.h5,L784441_SAP000_B108_S2_P000_bf.h5,L784441_SAP000_B108_S3_P000_bf.h5,L784441_SAP000_B109_S0_P000_bf.h5,L784441_SAP000_B109_S1_P000_bf.h5,L784441_SAP000_B109_S2_P000_bf.h5,L784441_SAP000_B109_S3_P000_bf.h5,L784441_SAP000_B110_S0_P000_bf.h5,L784441_SAP000_B110_S1_P000_bf.h5,L784441_SAP000_B110_S2_P000_bf.h5,L784441_SAP000_B110_S3_P000_bf.h5,L784441_SAP000_B111_S0_P000_bf.h5,L784441_SAP000_B111_S1_P000_bf.h5,L784441_SAP000_B111_S2_P000_bf.h5,L784441_SAP000_B111_S3_P000_bf.h5,L784441_SAP000_B112_S0_P000_bf.h5,L784441_SAP000_B112_S1_P000_bf.h5,L784441_SAP000_B112_S2_P000_bf.h5,L784441_SAP000_B112_S3_P000_bf.h5,L784441_SAP000_B113_S0_P000_bf.h5,L784441_SAP000_B113_S1_P000_bf.h5,L784441_SAP000_B113_S2_P000_bf.h5,L784441_SAP000_B113_S3_P000_bf.h5,L784441_SAP000_B114_S0_P000_bf.h5,L784441_SAP000_B114_S1_P000_bf.h5,L784441_SAP000_B114_S2_P000_bf.h5,L784441_SAP000_B114_S3_P000_bf.h5,L784441_SAP000_B115_S0_P000_bf.h5,L784441_SAP000_B115_S1_P000_bf.h5,L784441_SAP000_B115_S2_P000_bf.h5,L784441_SAP000_B115_S3_P000_bf.h5,L784441_SAP000_B116_S0_P000_bf.h5,L784441_SAP000_B116_S1_P000_bf.h5,L784441_SAP000_B116_S2_P000_bf.h5,L784441_SAP000_B116_S3_P000_bf.h5,L784441_SAP000_B117_S0_P000_bf.h5,L784441_SAP000_B117_S1_P000_bf.h5,L784441_SAP000_B117_S2_P000_bf.h5,L784441_SAP000_B117_S3_P000_bf.h5,L784441_SAP000_B118_S0_P000_bf.h5,L784441_SAP000_B118_S1_P000_bf.h5,L784441_SAP000_B118_S2_P000_bf.h5,L784441_SAP000_B118_S3_P000_bf.h5,L784441_SAP000_B119_S0_P000_bf.h5,L784441_SAP000_B119_S1_P000_bf.h5,L784441_SAP000_B119_S2_P000_bf.h5,L784441_SAP000_B119_S3_P000_bf.h5,L784441_SAP000_B120_S0_P000_bf.h5,L784441_SAP000_B120_S1_P000_bf.h5,L784441_SAP000_B120_S2_P000_bf.h5,L784441_SAP000_B120_S3_P000_bf.h5,L784441_SAP000_B121_S0_P000_bf.h5,L784441_SAP000_B121_S1_P000_bf.h5,L784441_SAP000_B121_S2_P000_bf.h5,L784441_SAP000_B121_S3_P000_bf.h5,L784441_SAP000_B122_S0_P000_bf.h5,L784441_SAP000_B122_S1_P000_bf.h5,L784441_SAP000_B122_S2_P000_bf.h5,L784441_SAP000_B122_S3_P000_bf.h5,L784441_SAP000_B123_S0_P000_bf.h5,L784441_SAP000_B123_S1_P000_bf.h5,L784441_SAP000_B123_S2_P000_bf.h5,L784441_SAP000_B123_S3_P000_bf.h5,L784441_SAP000_B124_S0_P000_bf.h5,L784441_SAP000_B124_S1_P000_bf.h5,L784441_SAP000_B124_S2_P000_bf.h5,L784441_SAP000_B124_S3_P000_bf.h5,L784441_SAP000_B125_S0_P000_bf.h5,L784441_SAP000_B125_S1_P000_bf.h5,L784441_SAP000_B125_S2_P000_bf.h5,L784441_SAP000_B125_S3_P000_bf.h5,L784441_SAP000_B126_S0_P000_bf.h5,L784441_SAP000_B126_S1_P000_bf.h5,L784441_SAP000_B126_S2_P000_bf.h5,L784441_SAP000_B126_S3_P000_bf.h5,L784441_SAP001_B000_S0_P000_bf.h5,L784441_SAP001_B000_S1_P000_bf.h5,L784441_SAP001_B000_S2_P000_bf.h5,L784441_SAP001_B000_S3_P000_bf.h5]
+Observation.DataProducts.Output_CoherentStokes.locations=[512*localhost:output/784441]
+Observation.DataProducts.Output_CoherentStokes.storageClusterName="localhost"
+
+Observation.DataProducts.Output_Correlated.enabled=true
+Observation.DataProducts.Output_Correlated.filenames=[L784441_SAP000_SB000_uv.MS,L784441_SAP000_SB001_uv.MS,L784441_SAP000_SB002_uv.MS,L784441_SAP000_SB003_uv.MS,L784441_SAP000_SB004_uv.MS,L784441_SAP000_SB005_uv.MS,L784441_SAP000_SB006_uv.MS,L784441_SAP000_SB007_uv.MS,L784441_SAP000_SB008_uv.MS,L784441_SAP000_SB009_uv.MS,L784441_SAP000_SB010_uv.MS,L784441_SAP000_SB011_uv.MS,L784441_SAP000_SB012_uv.MS,L784441_SAP000_SB013_uv.MS,L784441_SAP000_SB014_uv.MS,L784441_SAP000_SB015_uv.MS,L784441_SAP000_SB016_uv.MS,L784441_SAP000_SB017_uv.MS,L784441_SAP000_SB018_uv.MS,L784441_SAP000_SB019_uv.MS,L784441_SAP000_SB020_uv.MS,L784441_SAP000_SB021_uv.MS,L784441_SAP000_SB022_uv.MS,L784441_SAP000_SB023_uv.MS,L784441_SAP000_SB024_uv.MS,L784441_SAP000_SB025_uv.MS,L784441_SAP000_SB026_uv.MS,L784441_SAP000_SB027_uv.MS,L784441_SAP000_SB028_uv.MS,L784441_SAP000_SB029_uv.MS,L784441_SAP000_SB030_uv.MS,L784441_SAP000_SB031_uv.MS,L784441_SAP000_SB032_uv.MS,L784441_SAP000_SB033_uv.MS,L784441_SAP000_SB034_uv.MS,L784441_SAP000_SB035_uv.MS,L784441_SAP000_SB036_uv.MS,L784441_SAP000_SB037_uv.MS,L784441_SAP000_SB038_uv.MS,L784441_SAP000_SB039_uv.MS,L784441_SAP000_SB040_uv.MS,L784441_SAP000_SB041_uv.MS,L784441_SAP000_SB042_uv.MS,L784441_SAP000_SB043_uv.MS,L784441_SAP000_SB044_uv.MS,L784441_SAP000_SB045_uv.MS,L784441_SAP000_SB046_uv.MS,L784441_SAP000_SB047_uv.MS,L784441_SAP000_SB048_uv.MS,L784441_SAP000_SB049_uv.MS,L784441_SAP000_SB050_uv.MS,L784441_SAP000_SB051_uv.MS,L784441_SAP000_SB052_uv.MS,L784441_SAP000_SB053_uv.MS,L784441_SAP000_SB054_uv.MS,L784441_SAP000_SB055_uv.MS,L784441_SAP000_SB056_uv.MS,L784441_SAP000_SB057_uv.MS,L784441_SAP000_SB058_uv.MS,L784441_SAP000_SB059_uv.MS,L784441_SAP001_SB060_uv.MS,L784441_SAP001_SB061_uv.MS,L784441_SAP001_SB062_uv.MS,L784441_SAP001_SB063_uv.MS,L784441_SAP001_SB064_uv.MS,L784441_SAP001_SB065_uv.MS,L784441_SAP001_SB066_uv.MS,L784441_SAP001_SB067_uv.MS,L784441_SAP001_SB068_uv.MS,L784441_SAP001_SB069_uv.MS,L784441_SAP001_SB070_uv.MS,L784441_SAP001_SB071_uv.MS,L784441_SAP001_SB072_uv.MS,L784441_SAP001_SB073_uv.MS,L784441_SAP001_SB074_uv.MS,L784441_SAP001_SB075_uv.MS,L784441_SAP001_SB076_uv.MS,L784441_SAP001_SB077_uv.MS,L784441_SAP001_SB078_uv.MS,L784441_SAP001_SB079_uv.MS,L784441_SAP001_SB080_uv.MS,L784441_SAP001_SB081_uv.MS,L784441_SAP001_SB082_uv.MS,L784441_SAP001_SB083_uv.MS,L784441_SAP001_SB084_uv.MS,L784441_SAP001_SB085_uv.MS,L784441_SAP001_SB086_uv.MS,L784441_SAP001_SB087_uv.MS,L784441_SAP001_SB088_uv.MS,L784441_SAP001_SB089_uv.MS,L784441_SAP001_SB090_uv.MS,L784441_SAP001_SB091_uv.MS,L784441_SAP001_SB092_uv.MS,L784441_SAP001_SB093_uv.MS,L784441_SAP001_SB094_uv.MS,L784441_SAP001_SB095_uv.MS,L784441_SAP001_SB096_uv.MS,L784441_SAP001_SB097_uv.MS,L784441_SAP001_SB098_uv.MS,L784441_SAP001_SB099_uv.MS,L784441_SAP001_SB100_uv.MS,L784441_SAP001_SB101_uv.MS,L784441_SAP001_SB102_uv.MS,L784441_SAP001_SB103_uv.MS,L784441_SAP001_SB104_uv.MS,L784441_SAP001_SB105_uv.MS,L784441_SAP001_SB106_uv.MS,L784441_SAP001_SB107_uv.MS,L784441_SAP001_SB108_uv.MS,L784441_SAP001_SB109_uv.MS,L784441_SAP001_SB110_uv.MS,L784441_SAP001_SB111_uv.MS,L784441_SAP001_SB112_uv.MS,L784441_SAP001_SB113_uv.MS,L784441_SAP001_SB114_uv.MS,L784441_SAP001_SB115_uv.MS,L784441_SAP001_SB116_uv.MS,L784441_SAP001_SB117_uv.MS,L784441_SAP001_SB118_uv.MS,L784441_SAP001_SB119_uv.MS]
+Observation.DataProducts.Output_Correlated.locations=[120*localhost:output/784441]
+Observation.DataProducts.Output_Correlated.storageClusterName="localhost"
+
+Observation.DataProducts.Output_IncoherentStokes.enabled=false
+Observation.DataProducts.Output_InstrumentModel.enabled=false
+Observation.DataProducts.Output_Pulsar.enabled=false
+Observation.DataProducts.Output_SkyImage.enabled=false
+
+#
+# ----- Dataslot settings, remove? -----
+#
+Observation.Dataslots.CS001LBA.DataslotList=[0..119]
+Observation.Dataslots.CS001LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS002LBA.DataslotList=[0..119]
+Observation.Dataslots.CS002LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS003LBA.DataslotList=[0..119]
+Observation.Dataslots.CS003LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS004LBA.DataslotList=[0..119]
+Observation.Dataslots.CS004LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS005LBA.DataslotList=[0..119]
+Observation.Dataslots.CS005LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS006LBA.DataslotList=[0..119]
+Observation.Dataslots.CS006LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS007LBA.DataslotList=[0..119]
+Observation.Dataslots.CS007LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS011LBA.DataslotList=[0..119]
+Observation.Dataslots.CS011LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS013LBA.DataslotList=[0..119]
+Observation.Dataslots.CS013LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS017LBA.DataslotList=[0..119]
+Observation.Dataslots.CS017LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS021LBA.DataslotList=[0..119]
+Observation.Dataslots.CS021LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS024LBA.DataslotList=[0..119]
+Observation.Dataslots.CS024LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS026LBA.DataslotList=[0..119]
+Observation.Dataslots.CS026LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS028LBA.DataslotList=[0..119]
+Observation.Dataslots.CS028LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS030LBA.DataslotList=[0..119]
+Observation.Dataslots.CS030LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS031LBA.DataslotList=[0..119]
+Observation.Dataslots.CS031LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS032LBA.DataslotList=[0..119]
+Observation.Dataslots.CS032LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS101LBA.DataslotList=[0..119]
+Observation.Dataslots.CS101LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS103LBA.DataslotList=[0..119]
+Observation.Dataslots.CS103LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS201LBA.DataslotList=[0..119]
+Observation.Dataslots.CS201LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS301LBA.DataslotList=[0..119]
+Observation.Dataslots.CS301LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS302LBA.DataslotList=[0..119]
+Observation.Dataslots.CS302LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS401LBA.DataslotList=[0..119]
+Observation.Dataslots.CS401LBA.RSPBoardList=[120*0]
+Observation.Dataslots.CS501LBA.DataslotList=[0..119]
+Observation.Dataslots.CS501LBA.RSPBoardList=[120*0]
+Observation.Dataslots.DataslotInfo.DataslotList=[]
+Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+Observation.Dataslots.RS106LBA.DataslotList=[0..119]
+Observation.Dataslots.RS106LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS205LBA.DataslotList=[0..119]
+Observation.Dataslots.RS205LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS208LBA.DataslotList=[0..119]
+Observation.Dataslots.RS208LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS305LBA.DataslotList=[0..119]
+Observation.Dataslots.RS305LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS306LBA.DataslotList=[0..119]
+Observation.Dataslots.RS306LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS307LBA.DataslotList=[0..119]
+Observation.Dataslots.RS307LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS406LBA.DataslotList=[0..119]
+Observation.Dataslots.RS406LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS407LBA.DataslotList=[0..119]
+Observation.Dataslots.RS407LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS409LBA.DataslotList=[0..119]
+Observation.Dataslots.RS409LBA.RSPBoardList=[120*0]
+Observation.Dataslots.RS503LBA.DataslotList=[0..119]
+Observation.Dataslots.RS503LBA.RSPBoardList=[120*0]
+
+#
+# ----- Observation settings -----
+#
+Observation.ObsID=784441
+Observation.VirtualInstrument.minimalNrStations=1
+Observation.VirtualInstrument.stationList=[CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS101,CS103,CS201,CS301,CS501,RS106,RS205,RS208,RS305,RS306,RS406,RS407,RS409,RS503]
+Observation.VirtualInstrument.stationSet=Custom
+Observation.antennaArray=LBA
+Observation.antennaSet=LBA_OUTER
+Observation.bandFilter=LBA_10_90
+Observation.claimPeriod=35
+Observation.clockMode=<<Clock200
+Observation.momID=1021615
+Observation.nrAnaBeams=0
+Observation.nrBeams=2
+Observation.nrBitsPerSample=8
+Observation.nrTBBSettings=0
+Observation.originID=650576
+Observation.otdbID=784441
+Observation.preparePeriod=20
+Observation.processSubtype=Beam Observation
+Observation.processType=Observation
+Observation.sampleClock=200
+Observation.startTime=2020-06-10 08:30:00
+Observation.stopTime=2020-06-10 10:08:00
+Observation.strategy=default
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-796954.parset b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-796954.parset
new file mode 100644
index 0000000000000000000000000000000000000000..edf3093cd7a234600fc21d2262bc529c521dcca8
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-796954.parset
@@ -0,0 +1,280 @@
+#
+# ----- Info -----
+#
+# Purpose: LOFAR reference test
+# LOFAR Two-metre Sky Survey, HBA sky survey
+# Intended to be used with gpu_load test for GPU kernel performance benchmarking
+
+# Enable performance benchmarking
+Cobalt.Benchmark.enabled=true
+Cobalt.Benchmark.file=Benchmarks/796954.csv
+
+# These figures are used by gpu_load to predict the load across the whole cluster.
+# Set available processing resources for the full COBALT 2.0 system (2 GPUs/sockets x 11 nodes)
+Cobalt.Nodes = [ 22*localhost ]
+
+#
+# ----- Beamformer settings -----
+#
+Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=1
+Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=1
+Cobalt.BeamFormer.CoherentStokes.which=I
+Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=1
+Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=1
+Cobalt.BeamFormer.IncoherentStokes.which=I
+Cobalt.BeamFormer.coherentDedisperseChannels=false
+Cobalt.BeamFormer.flysEye=false
+Cobalt.BeamFormer.nrDelayCompensationChannels=256
+Cobalt.BeamFormer.nrHighResolutionChannels=256
+Cobalt.BeamFormer.stationList=[]
+
+#
+# ----- Correlator settings -----
+#
+Cobalt.Correlator.integrationTime=1.00139
+Cobalt.Correlator.nrBlocksPerIntegration=1
+Cobalt.Correlator.nrChannelsPerSubband=64
+Cobalt.Correlator.nrIntegrationsPerBlock=1
+
+#
+# ----- Misc settings -----
+#
+Cobalt.blockSize=195584
+Cobalt.correctBandPass=true
+Cobalt.correctClocks=true
+Cobalt.delayCompensation=true
+Cobalt.realTime=true
+
+#
+# ----- Pointings -----
+#
+Observation.Beam[0].angle1=6.200755188298173
+Observation.Beam[0].angle2=0.10093142101546985
+Observation.Beam[0].directionType=J2000
+Observation.Beam[0].duration=0
+Observation.Beam[0].momID=1036146
+Observation.Beam[0].nrTabRings=0
+Observation.Beam[0].nrTiedArrayBeams=0
+Observation.Beam[0].startTime=
+Observation.Beam[0].subbandList=[256]
+Observation.Beam[0].tabRingSize=0
+Observation.Beam[0].target=P354P355REF
+Observation.Beam[1].angle1=6.1928137583964675
+Observation.Beam[1].angle2=0.057049092446257534
+Observation.Beam[1].directionType=J2000
+Observation.Beam[1].duration=0
+Observation.Beam[1].momID=1036147
+Observation.Beam[1].nrTabRings=0
+Observation.Beam[1].nrTiedArrayBeams=0
+Observation.Beam[1].startTime=
+Observation.Beam[1].subbandList=[104..136,138..163,165..180,182..184,187..209,212..213,215..240,242..255,257..273,275..300,302..328,330..347,349,364,372,380,388,396,404,413,421,430,438,447]
+Observation.Beam[1].tabRingSize=0
+Observation.Beam[1].target=P354+03
+Observation.Beam[2].angle1=6.208696618199876
+Observation.Beam[2].angle2=0.14481374958468218
+Observation.Beam[2].directionType=J2000
+Observation.Beam[2].duration=0
+Observation.Beam[2].momID=1036148
+Observation.Beam[2].nrTabRings=0
+Observation.Beam[2].nrTiedArrayBeams=0
+Observation.Beam[2].startTime=
+Observation.Beam[2].subbandList=[104..136,138..163,165..180,182..184,187..209,212..213,215..240,242..255,257..273,275..300,302..328,330..347,349,364,372,380,388,396,404,413,421,430,438,447]
+Observation.Beam[2].tabRingSize=0
+Observation.Beam[2].target=P355+08
+
+#
+# ----- Output settings -----
+#
+Observation.DataProducts.Output_CoherentStokes.enabled=false
+
+Observation.DataProducts.Output_Correlated.enabled=true
+Observation.DataProducts.Output_Correlated.filenames=[L796954_SAP000_SB000_uv.MS,L796954_SAP001_SB001_uv.MS,L796954_SAP001_SB002_uv.MS,L796954_SAP001_SB003_uv.MS,L796954_SAP001_SB004_uv.MS,L796954_SAP001_SB005_uv.MS,L796954_SAP001_SB006_uv.MS,L796954_SAP001_SB007_uv.MS,L796954_SAP001_SB008_uv.MS,L796954_SAP001_SB009_uv.MS,L796954_SAP001_SB010_uv.MS,L796954_SAP001_SB011_uv.MS,L796954_SAP001_SB012_uv.MS,L796954_SAP001_SB013_uv.MS,L796954_SAP001_SB014_uv.MS,L796954_SAP001_SB015_uv.MS,L796954_SAP001_SB016_uv.MS,L796954_SAP001_SB017_uv.MS,L796954_SAP001_SB018_uv.MS,L796954_SAP001_SB019_uv.MS,L796954_SAP001_SB020_uv.MS,L796954_SAP001_SB021_uv.MS,L796954_SAP001_SB022_uv.MS,L796954_SAP001_SB023_uv.MS,L796954_SAP001_SB024_uv.MS,L796954_SAP001_SB025_uv.MS,L796954_SAP001_SB026_uv.MS,L796954_SAP001_SB027_uv.MS,L796954_SAP001_SB028_uv.MS,L796954_SAP001_SB029_uv.MS,L796954_SAP001_SB030_uv.MS,L796954_SAP001_SB031_uv.MS,L796954_SAP001_SB032_uv.MS,L796954_SAP001_SB033_uv.MS,L796954_SAP001_SB034_uv.MS,L796954_SAP001_SB035_uv.MS,L796954_SAP001_SB036_uv.MS,L796954_SAP001_SB037_uv.MS,L796954_SAP001_SB038_uv.MS,L796954_SAP001_SB039_uv.MS,L796954_SAP001_SB040_uv.MS,L796954_SAP001_SB041_uv.MS,L796954_SAP001_SB042_uv.MS,L796954_SAP001_SB043_uv.MS,L796954_SAP001_SB044_uv.MS,L796954_SAP001_SB045_uv.MS,L796954_SAP001_SB046_uv.MS,L796954_SAP001_SB047_uv.MS,L796954_SAP001_SB048_uv.MS,L796954_SAP001_SB049_uv.MS,L796954_SAP001_SB050_uv.MS,L796954_SAP001_SB051_uv.MS,L796954_SAP001_SB052_uv.MS,L796954_SAP001_SB053_uv.MS,L796954_SAP001_SB054_uv.MS,L796954_SAP001_SB055_uv.MS,L796954_SAP001_SB056_uv.MS,L796954_SAP001_SB057_uv.MS,L796954_SAP001_SB058_uv.MS,L796954_SAP001_SB059_uv.MS,L796954_SAP001_SB060_uv.MS,L796954_SAP001_SB061_uv.MS,L796954_SAP001_SB062_uv.MS,L796954_SAP001_SB063_uv.MS,L796954_SAP001_SB064_uv.MS,L796954_SAP001_SB065_uv.MS,L796954_SAP001_SB066_uv.MS,L796954_SAP001_SB067_uv.MS,L796954_SAP001_SB068_uv.MS,L796954_SAP001_SB069_uv.MS,L796954_SAP001_SB070_uv.MS,L796954_SAP001_SB071_uv.MS,L796954_SAP001_SB072_uv.MS,L796954_SAP001_SB073_uv.MS,L796954_SAP001_SB074_uv.MS,L796954_SAP001_SB075_uv.MS,L796954_SAP001_SB076_uv.MS,L796954_SAP001_SB077_uv.MS,L796954_SAP001_SB078_uv.MS,L796954_SAP001_SB079_uv.MS,L796954_SAP001_SB080_uv.MS,L796954_SAP001_SB081_uv.MS,L796954_SAP001_SB082_uv.MS,L796954_SAP001_SB083_uv.MS,L796954_SAP001_SB084_uv.MS,L796954_SAP001_SB085_uv.MS,L796954_SAP001_SB086_uv.MS,L796954_SAP001_SB087_uv.MS,L796954_SAP001_SB088_uv.MS,L796954_SAP001_SB089_uv.MS,L796954_SAP001_SB090_uv.MS,L796954_SAP001_SB091_uv.MS,L796954_SAP001_SB092_uv.MS,L796954_SAP001_SB093_uv.MS,L796954_SAP001_SB094_uv.MS,L796954_SAP001_SB095_uv.MS,L796954_SAP001_SB096_uv.MS,L796954_SAP001_SB097_uv.MS,L796954_SAP001_SB098_uv.MS,L796954_SAP001_SB099_uv.MS,L796954_SAP001_SB100_uv.MS,L796954_SAP001_SB101_uv.MS,L796954_SAP001_SB102_uv.MS,L796954_SAP001_SB103_uv.MS,L796954_SAP001_SB104_uv.MS,L796954_SAP001_SB105_uv.MS,L796954_SAP001_SB106_uv.MS,L796954_SAP001_SB107_uv.MS,L796954_SAP001_SB108_uv.MS,L796954_SAP001_SB109_uv.MS,L796954_SAP001_SB110_uv.MS,L796954_SAP001_SB111_uv.MS,L796954_SAP001_SB112_uv.MS,L796954_SAP001_SB113_uv.MS,L796954_SAP001_SB114_uv.MS,L796954_SAP001_SB115_uv.MS,L796954_SAP001_SB116_uv.MS,L796954_SAP001_SB117_uv.MS,L796954_SAP001_SB118_uv.MS,L796954_SAP001_SB119_uv.MS,L796954_SAP001_SB120_uv.MS,L796954_SAP001_SB121_uv.MS,L796954_SAP001_SB122_uv.MS,L796954_SAP001_SB123_uv.MS,L796954_SAP001_SB124_uv.MS,L796954_SAP001_SB125_uv.MS,L796954_SAP001_SB126_uv.MS,L796954_SAP001_SB127_uv.MS,L796954_SAP001_SB128_uv.MS,L796954_SAP001_SB129_uv.MS,L796954_SAP001_SB130_uv.MS,L796954_SAP001_SB131_uv.MS,L796954_SAP001_SB132_uv.MS,L796954_SAP001_SB133_uv.MS,L796954_SAP001_SB134_uv.MS,L796954_SAP001_SB135_uv.MS,L796954_SAP001_SB136_uv.MS,L796954_SAP001_SB137_uv.MS,L796954_SAP001_SB138_uv.MS,L796954_SAP001_SB139_uv.MS,L796954_SAP001_SB140_uv.MS,L796954_SAP001_SB141_uv.MS,L796954_SAP001_SB142_uv.MS,L796954_SAP001_SB143_uv.MS,L796954_SAP001_SB144_uv.MS,L796954_SAP001_SB145_uv.MS,L796954_SAP001_SB146_uv.MS,L796954_SAP001_SB147_uv.MS,L796954_SAP001_SB148_uv.MS,L796954_SAP001_SB149_uv.MS,L796954_SAP001_SB150_uv.MS,L796954_SAP001_SB151_uv.MS,L796954_SAP001_SB152_uv.MS,L796954_SAP001_SB153_uv.MS,L796954_SAP001_SB154_uv.MS,L796954_SAP001_SB155_uv.MS,L796954_SAP001_SB156_uv.MS,L796954_SAP001_SB157_uv.MS,L796954_SAP001_SB158_uv.MS,L796954_SAP001_SB159_uv.MS,L796954_SAP001_SB160_uv.MS,L796954_SAP001_SB161_uv.MS,L796954_SAP001_SB162_uv.MS,L796954_SAP001_SB163_uv.MS,L796954_SAP001_SB164_uv.MS,L796954_SAP001_SB165_uv.MS,L796954_SAP001_SB166_uv.MS,L796954_SAP001_SB167_uv.MS,L796954_SAP001_SB168_uv.MS,L796954_SAP001_SB169_uv.MS,L796954_SAP001_SB170_uv.MS,L796954_SAP001_SB171_uv.MS,L796954_SAP001_SB172_uv.MS,L796954_SAP001_SB173_uv.MS,L796954_SAP001_SB174_uv.MS,L796954_SAP001_SB175_uv.MS,L796954_SAP001_SB176_uv.MS,L796954_SAP001_SB177_uv.MS,L796954_SAP001_SB178_uv.MS,L796954_SAP001_SB179_uv.MS,L796954_SAP001_SB180_uv.MS,L796954_SAP001_SB181_uv.MS,L796954_SAP001_SB182_uv.MS,L796954_SAP001_SB183_uv.MS,L796954_SAP001_SB184_uv.MS,L796954_SAP001_SB185_uv.MS,L796954_SAP001_SB186_uv.MS,L796954_SAP001_SB187_uv.MS,L796954_SAP001_SB188_uv.MS,L796954_SAP001_SB189_uv.MS,L796954_SAP001_SB190_uv.MS,L796954_SAP001_SB191_uv.MS,L796954_SAP001_SB192_uv.MS,L796954_SAP001_SB193_uv.MS,L796954_SAP001_SB194_uv.MS,L796954_SAP001_SB195_uv.MS,L796954_SAP001_SB196_uv.MS,L796954_SAP001_SB197_uv.MS,L796954_SAP001_SB198_uv.MS,L796954_SAP001_SB199_uv.MS,L796954_SAP001_SB200_uv.MS,L796954_SAP001_SB201_uv.MS,L796954_SAP001_SB202_uv.MS,L796954_SAP001_SB203_uv.MS,L796954_SAP001_SB204_uv.MS,L796954_SAP001_SB205_uv.MS,L796954_SAP001_SB206_uv.MS,L796954_SAP001_SB207_uv.MS,L796954_SAP001_SB208_uv.MS,L796954_SAP001_SB209_uv.MS,L796954_SAP001_SB210_uv.MS,L796954_SAP001_SB211_uv.MS,L796954_SAP001_SB212_uv.MS,L796954_SAP001_SB213_uv.MS,L796954_SAP001_SB214_uv.MS,L796954_SAP001_SB215_uv.MS,L796954_SAP001_SB216_uv.MS,L796954_SAP001_SB217_uv.MS,L796954_SAP001_SB218_uv.MS,L796954_SAP001_SB219_uv.MS,L796954_SAP001_SB220_uv.MS,L796954_SAP001_SB221_uv.MS,L796954_SAP001_SB222_uv.MS,L796954_SAP001_SB223_uv.MS,L796954_SAP001_SB224_uv.MS,L796954_SAP001_SB225_uv.MS,L796954_SAP001_SB226_uv.MS,L796954_SAP001_SB227_uv.MS,L796954_SAP001_SB228_uv.MS,L796954_SAP001_SB229_uv.MS,L796954_SAP001_SB230_uv.MS,L796954_SAP001_SB231_uv.MS,L796954_SAP001_SB232_uv.MS,L796954_SAP001_SB233_uv.MS,L796954_SAP001_SB234_uv.MS,L796954_SAP001_SB235_uv.MS,L796954_SAP001_SB236_uv.MS,L796954_SAP001_SB237_uv.MS,L796954_SAP001_SB238_uv.MS,L796954_SAP001_SB239_uv.MS,L796954_SAP001_SB240_uv.MS,L796954_SAP001_SB241_uv.MS,L796954_SAP001_SB242_uv.MS,L796954_SAP001_SB243_uv.MS,L796954_SAP002_SB244_uv.MS,L796954_SAP002_SB245_uv.MS,L796954_SAP002_SB246_uv.MS,L796954_SAP002_SB247_uv.MS,L796954_SAP002_SB248_uv.MS,L796954_SAP002_SB249_uv.MS,L796954_SAP002_SB250_uv.MS,L796954_SAP002_SB251_uv.MS,L796954_SAP002_SB252_uv.MS,L796954_SAP002_SB253_uv.MS,L796954_SAP002_SB254_uv.MS,L796954_SAP002_SB255_uv.MS,L796954_SAP002_SB256_uv.MS,L796954_SAP002_SB257_uv.MS,L796954_SAP002_SB258_uv.MS,L796954_SAP002_SB259_uv.MS,L796954_SAP002_SB260_uv.MS,L796954_SAP002_SB261_uv.MS,L796954_SAP002_SB262_uv.MS,L796954_SAP002_SB263_uv.MS,L796954_SAP002_SB264_uv.MS,L796954_SAP002_SB265_uv.MS,L796954_SAP002_SB266_uv.MS,L796954_SAP002_SB267_uv.MS,L796954_SAP002_SB268_uv.MS,L796954_SAP002_SB269_uv.MS,L796954_SAP002_SB270_uv.MS,L796954_SAP002_SB271_uv.MS,L796954_SAP002_SB272_uv.MS,L796954_SAP002_SB273_uv.MS,L796954_SAP002_SB274_uv.MS,L796954_SAP002_SB275_uv.MS,L796954_SAP002_SB276_uv.MS,L796954_SAP002_SB277_uv.MS,L796954_SAP002_SB278_uv.MS,L796954_SAP002_SB279_uv.MS,L796954_SAP002_SB280_uv.MS,L796954_SAP002_SB281_uv.MS,L796954_SAP002_SB282_uv.MS,L796954_SAP002_SB283_uv.MS,L796954_SAP002_SB284_uv.MS,L796954_SAP002_SB285_uv.MS,L796954_SAP002_SB286_uv.MS,L796954_SAP002_SB287_uv.MS,L796954_SAP002_SB288_uv.MS,L796954_SAP002_SB289_uv.MS,L796954_SAP002_SB290_uv.MS,L796954_SAP002_SB291_uv.MS,L796954_SAP002_SB292_uv.MS,L796954_SAP002_SB293_uv.MS,L796954_SAP002_SB294_uv.MS,L796954_SAP002_SB295_uv.MS,L796954_SAP002_SB296_uv.MS,L796954_SAP002_SB297_uv.MS,L796954_SAP002_SB298_uv.MS,L796954_SAP002_SB299_uv.MS,L796954_SAP002_SB300_uv.MS,L796954_SAP002_SB301_uv.MS,L796954_SAP002_SB302_uv.MS,L796954_SAP002_SB303_uv.MS,L796954_SAP002_SB304_uv.MS,L796954_SAP002_SB305_uv.MS,L796954_SAP002_SB306_uv.MS,L796954_SAP002_SB307_uv.MS,L796954_SAP002_SB308_uv.MS,L796954_SAP002_SB309_uv.MS,L796954_SAP002_SB310_uv.MS,L796954_SAP002_SB311_uv.MS,L796954_SAP002_SB312_uv.MS,L796954_SAP002_SB313_uv.MS,L796954_SAP002_SB314_uv.MS,L796954_SAP002_SB315_uv.MS,L796954_SAP002_SB316_uv.MS,L796954_SAP002_SB317_uv.MS,L796954_SAP002_SB318_uv.MS,L796954_SAP002_SB319_uv.MS,L796954_SAP002_SB320_uv.MS,L796954_SAP002_SB321_uv.MS,L796954_SAP002_SB322_uv.MS,L796954_SAP002_SB323_uv.MS,L796954_SAP002_SB324_uv.MS,L796954_SAP002_SB325_uv.MS,L796954_SAP002_SB326_uv.MS,L796954_SAP002_SB327_uv.MS,L796954_SAP002_SB328_uv.MS,L796954_SAP002_SB329_uv.MS,L796954_SAP002_SB330_uv.MS,L796954_SAP002_SB331_uv.MS,L796954_SAP002_SB332_uv.MS,L796954_SAP002_SB333_uv.MS,L796954_SAP002_SB334_uv.MS,L796954_SAP002_SB335_uv.MS,L796954_SAP002_SB336_uv.MS,L796954_SAP002_SB337_uv.MS,L796954_SAP002_SB338_uv.MS,L796954_SAP002_SB339_uv.MS,L796954_SAP002_SB340_uv.MS,L796954_SAP002_SB341_uv.MS,L796954_SAP002_SB342_uv.MS,L796954_SAP002_SB343_uv.MS,L796954_SAP002_SB344_uv.MS,L796954_SAP002_SB345_uv.MS,L796954_SAP002_SB346_uv.MS,L796954_SAP002_SB347_uv.MS,L796954_SAP002_SB348_uv.MS,L796954_SAP002_SB349_uv.MS,L796954_SAP002_SB350_uv.MS,L796954_SAP002_SB351_uv.MS,L796954_SAP002_SB352_uv.MS,L796954_SAP002_SB353_uv.MS,L796954_SAP002_SB354_uv.MS,L796954_SAP002_SB355_uv.MS,L796954_SAP002_SB356_uv.MS,L796954_SAP002_SB357_uv.MS,L796954_SAP002_SB358_uv.MS,L796954_SAP002_SB359_uv.MS,L796954_SAP002_SB360_uv.MS,L796954_SAP002_SB361_uv.MS,L796954_SAP002_SB362_uv.MS,L796954_SAP002_SB363_uv.MS,L796954_SAP002_SB364_uv.MS,L796954_SAP002_SB365_uv.MS,L796954_SAP002_SB366_uv.MS,L796954_SAP002_SB367_uv.MS,L796954_SAP002_SB368_uv.MS,L796954_SAP002_SB369_uv.MS,L796954_SAP002_SB370_uv.MS,L796954_SAP002_SB371_uv.MS,L796954_SAP002_SB372_uv.MS,L796954_SAP002_SB373_uv.MS,L796954_SAP002_SB374_uv.MS,L796954_SAP002_SB375_uv.MS,L796954_SAP002_SB376_uv.MS,L796954_SAP002_SB377_uv.MS,L796954_SAP002_SB378_uv.MS,L796954_SAP002_SB379_uv.MS,L796954_SAP002_SB380_uv.MS,L796954_SAP002_SB381_uv.MS,L796954_SAP002_SB382_uv.MS,L796954_SAP002_SB383_uv.MS,L796954_SAP002_SB384_uv.MS,L796954_SAP002_SB385_uv.MS,L796954_SAP002_SB386_uv.MS,L796954_SAP002_SB387_uv.MS,L796954_SAP002_SB388_uv.MS,L796954_SAP002_SB389_uv.MS,L796954_SAP002_SB390_uv.MS,L796954_SAP002_SB391_uv.MS,L796954_SAP002_SB392_uv.MS,L796954_SAP002_SB393_uv.MS,L796954_SAP002_SB394_uv.MS,L796954_SAP002_SB395_uv.MS,L796954_SAP002_SB396_uv.MS,L796954_SAP002_SB397_uv.MS,L796954_SAP002_SB398_uv.MS,L796954_SAP002_SB399_uv.MS,L796954_SAP002_SB400_uv.MS,L796954_SAP002_SB401_uv.MS,L796954_SAP002_SB402_uv.MS,L796954_SAP002_SB403_uv.MS,L796954_SAP002_SB404_uv.MS,L796954_SAP002_SB405_uv.MS,L796954_SAP002_SB406_uv.MS,L796954_SAP002_SB407_uv.MS,L796954_SAP002_SB408_uv.MS,L796954_SAP002_SB409_uv.MS,L796954_SAP002_SB410_uv.MS,L796954_SAP002_SB411_uv.MS,L796954_SAP002_SB412_uv.MS,L796954_SAP002_SB413_uv.MS,L796954_SAP002_SB414_uv.MS,L796954_SAP002_SB415_uv.MS,L796954_SAP002_SB416_uv.MS,L796954_SAP002_SB417_uv.MS,L796954_SAP002_SB418_uv.MS,L796954_SAP002_SB419_uv.MS,L796954_SAP002_SB420_uv.MS,L796954_SAP002_SB421_uv.MS,L796954_SAP002_SB422_uv.MS,L796954_SAP002_SB423_uv.MS,L796954_SAP002_SB424_uv.MS,L796954_SAP002_SB425_uv.MS,L796954_SAP002_SB426_uv.MS,L796954_SAP002_SB427_uv.MS,L796954_SAP002_SB428_uv.MS,L796954_SAP002_SB429_uv.MS,L796954_SAP002_SB430_uv.MS,L796954_SAP002_SB431_uv.MS,L796954_SAP002_SB432_uv.MS,L796954_SAP002_SB433_uv.MS,L796954_SAP002_SB434_uv.MS,L796954_SAP002_SB435_uv.MS,L796954_SAP002_SB436_uv.MS,L796954_SAP002_SB437_uv.MS,L796954_SAP002_SB438_uv.MS,L796954_SAP002_SB439_uv.MS,L796954_SAP002_SB440_uv.MS,L796954_SAP002_SB441_uv.MS,L796954_SAP002_SB442_uv.MS,L796954_SAP002_SB443_uv.MS,L796954_SAP002_SB444_uv.MS,L796954_SAP002_SB445_uv.MS,L796954_SAP002_SB446_uv.MS,L796954_SAP002_SB447_uv.MS,L796954_SAP002_SB448_uv.MS,L796954_SAP002_SB449_uv.MS,L796954_SAP002_SB450_uv.MS,L796954_SAP002_SB451_uv.MS,L796954_SAP002_SB452_uv.MS,L796954_SAP002_SB453_uv.MS,L796954_SAP002_SB454_uv.MS,L796954_SAP002_SB455_uv.MS,L796954_SAP002_SB456_uv.MS,L796954_SAP002_SB457_uv.MS,L796954_SAP002_SB458_uv.MS,L796954_SAP002_SB459_uv.MS,L796954_SAP002_SB460_uv.MS,L796954_SAP002_SB461_uv.MS,L796954_SAP002_SB462_uv.MS,L796954_SAP002_SB463_uv.MS,L796954_SAP002_SB464_uv.MS,L796954_SAP002_SB465_uv.MS,L796954_SAP002_SB466_uv.MS,L796954_SAP002_SB467_uv.MS,L796954_SAP002_SB468_uv.MS,L796954_SAP002_SB469_uv.MS,L796954_SAP002_SB470_uv.MS,L796954_SAP002_SB471_uv.MS,L796954_SAP002_SB472_uv.MS,L796954_SAP002_SB473_uv.MS,L796954_SAP002_SB474_uv.MS,L796954_SAP002_SB475_uv.MS,L796954_SAP002_SB476_uv.MS,L796954_SAP002_SB477_uv.MS,L796954_SAP002_SB478_uv.MS,L796954_SAP002_SB479_uv.MS,L796954_SAP002_SB480_uv.MS,L796954_SAP002_SB481_uv.MS,L796954_SAP002_SB482_uv.MS,L796954_SAP002_SB483_uv.MS,L796954_SAP002_SB484_uv.MS,L796954_SAP002_SB485_uv.MS,L796954_SAP002_SB486_uv.MS]
+Observation.DataProducts.Output_Correlated.locations=[487*localhost:output/796954]
+Observation.DataProducts.Output_Correlated.storageClusterName="localhost"
+
+Observation.DataProducts.Output_IncoherentStokes.enabled=false
+Observation.DataProducts.Output_InstrumentModel.enabled=false
+Observation.DataProducts.Output_Pulsar.enabled=false
+Observation.DataProducts.Output_SkyImage.enabled=false
+
+#
+# ----- Dataslot settings, remove? -----
+#
+Observation.Dataslots.CS001HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS001HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS001HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS001HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS002HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS002HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS002HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS002HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS003HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS003HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS003HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS003HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS004HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS004HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS004HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS004HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS005HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS005HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS005HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS005HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS006HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS006HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS006HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS006HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS007HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS007HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS007HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS007HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS011HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS011HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS011HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS011HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS013HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS013HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS013HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS013HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS017HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS017HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS017HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS017HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS021HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS021HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS021HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS021HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS024HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS024HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS024HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS024HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS026HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS026HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS026HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS026HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS028HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS028HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS028HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS028HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS030HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS030HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS030HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS030HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS031HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS031HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS031HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS031HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS101HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS101HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS101HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS101HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS103HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS103HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS103HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS103HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS201HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS201HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS201HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS201HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS301HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS301HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS301HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS301HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS302HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS302HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS302HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS302HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS401HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS401HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS401HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS401HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS501HBA0.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS501HBA0.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.CS501HBA1.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.CS501HBA1.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.DE601HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.DE601HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.DE602HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.DE602HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.DE603HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.DE603HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.DE604HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.DE604HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.DE605HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.DE605HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.DE609HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.DE609HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.DataslotInfo.DataslotList=[]
+Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+Observation.Dataslots.FR606HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.FR606HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.IE613HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.IE613HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.LV614HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.LV614HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.PL610HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.PL610HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.PL612HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.PL612HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS106HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS106HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS205HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS205HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS208HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS208HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS210HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS210HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS305HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS305HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS306HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS306HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS307HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS307HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS310HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS310HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS406HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS406HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS407HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS407HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS409HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS409HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS503HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS503HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS508HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS508HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.RS509HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.RS509HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.SE607HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.SE607HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+Observation.Dataslots.UK608HBA.DataslotList=[0..121,0..121,0..121,0..120]
+Observation.Dataslots.UK608HBA.RSPBoardList=[122*0,122*1,122*2,121*3]
+
+#
+# ----- Observation settings -----
+#
+Observation.ObsID=796954
+Observation.VirtualInstrument.minimalNrStations=1
+Observation.VirtualInstrument.stationList=[CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS101,CS103,CS201,CS301,CS302,CS401,CS501,DE601,DE602,DE603,DE604,DE605,DE609,FR606,IE613,LV614,PL610,PL612,RS106,RS205,RS208,RS210,RS305,RS306,RS307,RS310,RS406,RS407,RS409,RS503,RS508,RS509,SE607,UK608]
+Observation.VirtualInstrument.stationSet=Custom
+Observation.antennaArray=HBA
+Observation.antennaSet=HBA_DUAL_INNER
+Observation.bandFilter=HBA_110_190
+Observation.claimPeriod=35
+Observation.clockMode=<<Clock200
+Observation.momID=1036145
+Observation.nrAnaBeams=1
+Observation.nrBeams=3
+Observation.nrBitsPerSample=8
+Observation.nrTBBSettings=0
+Observation.originID=650576
+Observation.otdbID=796954
+Observation.preparePeriod=20
+Observation.processSubtype=Beam Observation
+Observation.processType=Observation
+Observation.referencePhaseCenter=[3826577.066, 461022.948, 5064892.786]
+Observation.sampleClock=200
+Observation.startTime=2020-10-24 19:58:44
+Observation.stopTime=2020-10-24 21:58:44
+Observation.strategy=default
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-797086.parset b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-797086.parset
new file mode 100644
index 0000000000000000000000000000000000000000..35884cd6cfb7d71f3491f7e5864e1c07a50a9855
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-797086.parset
@@ -0,0 +1,251 @@
+#
+# ----- Info -----
+#
+# Purpose: LOFAR reference test
+# Simultaneous imaging and beamforming
+# Intended to be used with gpu_load test for GPU kernel performance benchmarking
+
+# Enable performance benchmarking
+Cobalt.Benchmark.enabled=true
+Cobalt.Benchmark.file=Benchmarks/797086.csv
+
+# These figures are used by gpu_load to predict the load across the whole cluster.
+# Set available processing resources for the full COBALT 2.0 system (2 GPUs/sockets x 11 nodes)
+Cobalt.Nodes = [ 22*localhost ]
+
+#
+# ----- Beamformer settings -----
+#
+Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=1
+Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=20
+Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=1
+Cobalt.BeamFormer.CoherentStokes.which=XXYY
+Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=1
+Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=1
+Cobalt.BeamFormer.IncoherentStokes.which=I
+Cobalt.BeamFormer.coherentDedisperseChannels=false
+Cobalt.BeamFormer.flysEye=false
+Cobalt.BeamFormer.nrDelayCompensationChannels=256
+Cobalt.BeamFormer.nrHighResolutionChannels=256
+Cobalt.BeamFormer.stationList=[CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS031,CS101,CS103,CS201,CS301,CS302,CS401,CS501]
+
+#
+# ----- Correlator settings -----
+#
+Cobalt.Correlator.integrationTime=1.00663
+Cobalt.Correlator.nrBlocksPerIntegration=1
+Cobalt.Correlator.nrChannelsPerSubband=64
+Cobalt.Correlator.nrIntegrationsPerBlock=1
+Cobalt.FinalMetaDataGatherer.enabled=true
+
+#
+# ----- Misc settings -----
+#
+Cobalt.blockSize=196608
+Cobalt.correctBandPass=true
+Cobalt.correctClocks=true
+Cobalt.delayCompensation=true
+Cobalt.realTime=true
+
+#
+# ----- Pointings -----
+#
+Observation.Beam[0].TiedArrayBeam[0].angle1=5.126590518467983
+Observation.Beam[0].TiedArrayBeam[0].angle2=0.3821698981723873
+Observation.Beam[0].TiedArrayBeam[0].coherent=true
+Observation.Beam[0].TiedArrayBeam[0].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[0].dispersionMeasure=0
+Observation.Beam[0].TiedArrayBeam[1].angle1=5.147234854636991
+Observation.Beam[0].TiedArrayBeam[1].angle2=0.377388665649285
+Observation.Beam[0].TiedArrayBeam[1].coherent=true
+Observation.Beam[0].TiedArrayBeam[1].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[1].dispersionMeasure=0
+Observation.Beam[0].angle1=5.126590518467983
+Observation.Beam[0].angle2=0.3821698981723873
+Observation.Beam[0].directionType=J2000
+Observation.Beam[0].duration=0
+Observation.Beam[0].momID=1036291
+Observation.Beam[0].nrTabRings=0
+Observation.Beam[0].nrTiedArrayBeams=2
+Observation.Beam[0].startTime=
+Observation.Beam[0].subbandList=[104..347]
+Observation.Beam[0].tabRingSize=0
+Observation.Beam[0].target=SGR 1935+2154
+
+#
+# ----- Output settings -----
+#
+Observation.DataProducts.Output_CoherentStokes.enabled=true
+Observation.DataProducts.Output_CoherentStokes.filenames=[L797086_SAP000_B000_S0_P000_bf.h5,L797086_SAP000_B000_S0_P001_bf.h5,L797086_SAP000_B000_S0_P002_bf.h5,L797086_SAP000_B000_S0_P003_bf.h5,L797086_SAP000_B000_S0_P004_bf.h5,L797086_SAP000_B000_S0_P005_bf.h5,L797086_SAP000_B000_S0_P006_bf.h5,L797086_SAP000_B000_S0_P007_bf.h5,L797086_SAP000_B000_S0_P008_bf.h5,L797086_SAP000_B000_S0_P009_bf.h5,L797086_SAP000_B000_S0_P010_bf.h5,L797086_SAP000_B000_S0_P011_bf.h5,L797086_SAP000_B000_S0_P012_bf.h5,L797086_SAP000_B000_S1_P000_bf.h5,L797086_SAP000_B000_S1_P001_bf.h5,L797086_SAP000_B000_S1_P002_bf.h5,L797086_SAP000_B000_S1_P003_bf.h5,L797086_SAP000_B000_S1_P004_bf.h5,L797086_SAP000_B000_S1_P005_bf.h5,L797086_SAP000_B000_S1_P006_bf.h5,L797086_SAP000_B000_S1_P007_bf.h5,L797086_SAP000_B000_S1_P008_bf.h5,L797086_SAP000_B000_S1_P009_bf.h5,L797086_SAP000_B000_S1_P010_bf.h5,L797086_SAP000_B000_S1_P011_bf.h5,L797086_SAP000_B000_S1_P012_bf.h5,L797086_SAP000_B000_S2_P000_bf.h5,L797086_SAP000_B000_S2_P001_bf.h5,L797086_SAP000_B000_S2_P002_bf.h5,L797086_SAP000_B000_S2_P003_bf.h5,L797086_SAP000_B000_S2_P004_bf.h5,L797086_SAP000_B000_S2_P005_bf.h5,L797086_SAP000_B000_S2_P006_bf.h5,L797086_SAP000_B000_S2_P007_bf.h5,L797086_SAP000_B000_S2_P008_bf.h5,L797086_SAP000_B000_S2_P009_bf.h5,L797086_SAP000_B000_S2_P010_bf.h5,L797086_SAP000_B000_S2_P011_bf.h5,L797086_SAP000_B000_S2_P012_bf.h5,L797086_SAP000_B000_S3_P000_bf.h5,L797086_SAP000_B000_S3_P001_bf.h5,L797086_SAP000_B000_S3_P002_bf.h5,L797086_SAP000_B000_S3_P003_bf.h5,L797086_SAP000_B000_S3_P004_bf.h5,L797086_SAP000_B000_S3_P005_bf.h5,L797086_SAP000_B000_S3_P006_bf.h5,L797086_SAP000_B000_S3_P007_bf.h5,L797086_SAP000_B000_S3_P008_bf.h5,L797086_SAP000_B000_S3_P009_bf.h5,L797086_SAP000_B000_S3_P010_bf.h5,L797086_SAP000_B000_S3_P011_bf.h5,L797086_SAP000_B000_S3_P012_bf.h5,L797086_SAP000_B001_S0_P000_bf.h5,L797086_SAP000_B001_S0_P001_bf.h5,L797086_SAP000_B001_S0_P002_bf.h5,L797086_SAP000_B001_S0_P003_bf.h5,L797086_SAP000_B001_S0_P004_bf.h5,L797086_SAP000_B001_S0_P005_bf.h5,L797086_SAP000_B001_S0_P006_bf.h5,L797086_SAP000_B001_S0_P007_bf.h5,L797086_SAP000_B001_S0_P008_bf.h5,L797086_SAP000_B001_S0_P009_bf.h5,L797086_SAP000_B001_S0_P010_bf.h5,L797086_SAP000_B001_S0_P011_bf.h5,L797086_SAP000_B001_S0_P012_bf.h5,L797086_SAP000_B001_S1_P000_bf.h5,L797086_SAP000_B001_S1_P001_bf.h5,L797086_SAP000_B001_S1_P002_bf.h5,L797086_SAP000_B001_S1_P003_bf.h5,L797086_SAP000_B001_S1_P004_bf.h5,L797086_SAP000_B001_S1_P005_bf.h5,L797086_SAP000_B001_S1_P006_bf.h5,L797086_SAP000_B001_S1_P007_bf.h5,L797086_SAP000_B001_S1_P008_bf.h5,L797086_SAP000_B001_S1_P009_bf.h5,L797086_SAP000_B001_S1_P010_bf.h5,L797086_SAP000_B001_S1_P011_bf.h5,L797086_SAP000_B001_S1_P012_bf.h5,L797086_SAP000_B001_S2_P000_bf.h5,L797086_SAP000_B001_S2_P001_bf.h5,L797086_SAP000_B001_S2_P002_bf.h5,L797086_SAP000_B001_S2_P003_bf.h5,L797086_SAP000_B001_S2_P004_bf.h5,L797086_SAP000_B001_S2_P005_bf.h5,L797086_SAP000_B001_S2_P006_bf.h5,L797086_SAP000_B001_S2_P007_bf.h5,L797086_SAP000_B001_S2_P008_bf.h5,L797086_SAP000_B001_S2_P009_bf.h5,L797086_SAP000_B001_S2_P010_bf.h5,L797086_SAP000_B001_S2_P011_bf.h5,L797086_SAP000_B001_S2_P012_bf.h5,L797086_SAP000_B001_S3_P000_bf.h5,L797086_SAP000_B001_S3_P001_bf.h5,L797086_SAP000_B001_S3_P002_bf.h5,L797086_SAP000_B001_S3_P003_bf.h5,L797086_SAP000_B001_S3_P004_bf.h5,L797086_SAP000_B001_S3_P005_bf.h5,L797086_SAP000_B001_S3_P006_bf.h5,L797086_SAP000_B001_S3_P007_bf.h5,L797086_SAP000_B001_S3_P008_bf.h5,L797086_SAP000_B001_S3_P009_bf.h5,L797086_SAP000_B001_S3_P010_bf.h5,L797086_SAP000_B001_S3_P011_bf.h5,L797086_SAP000_B001_S3_P012_bf.h5]
+Observation.DataProducts.Output_CoherentStokes.locations=[104*localhost:output/797086]
+Observation.DataProducts.Output_CoherentStokes.storageClusterName="localhost"
+Observation.DataProducts.Output_CoherentStokes.storageClusterPartition=/data/projects/
+
+Observation.DataProducts.Output_Correlated.enabled=true
+Observation.DataProducts.Output_Correlated.filenames=[L797086_SAP000_SB000_uv.MS,L797086_SAP000_SB001_uv.MS,L797086_SAP000_SB002_uv.MS,L797086_SAP000_SB003_uv.MS,L797086_SAP000_SB004_uv.MS,L797086_SAP000_SB005_uv.MS,L797086_SAP000_SB006_uv.MS,L797086_SAP000_SB007_uv.MS,L797086_SAP000_SB008_uv.MS,L797086_SAP000_SB009_uv.MS,L797086_SAP000_SB010_uv.MS,L797086_SAP000_SB011_uv.MS,L797086_SAP000_SB012_uv.MS,L797086_SAP000_SB013_uv.MS,L797086_SAP000_SB014_uv.MS,L797086_SAP000_SB015_uv.MS,L797086_SAP000_SB016_uv.MS,L797086_SAP000_SB017_uv.MS,L797086_SAP000_SB018_uv.MS,L797086_SAP000_SB019_uv.MS,L797086_SAP000_SB020_uv.MS,L797086_SAP000_SB021_uv.MS,L797086_SAP000_SB022_uv.MS,L797086_SAP000_SB023_uv.MS,L797086_SAP000_SB024_uv.MS,L797086_SAP000_SB025_uv.MS,L797086_SAP000_SB026_uv.MS,L797086_SAP000_SB027_uv.MS,L797086_SAP000_SB028_uv.MS,L797086_SAP000_SB029_uv.MS,L797086_SAP000_SB030_uv.MS,L797086_SAP000_SB031_uv.MS,L797086_SAP000_SB032_uv.MS,L797086_SAP000_SB033_uv.MS,L797086_SAP000_SB034_uv.MS,L797086_SAP000_SB035_uv.MS,L797086_SAP000_SB036_uv.MS,L797086_SAP000_SB037_uv.MS,L797086_SAP000_SB038_uv.MS,L797086_SAP000_SB039_uv.MS,L797086_SAP000_SB040_uv.MS,L797086_SAP000_SB041_uv.MS,L797086_SAP000_SB042_uv.MS,L797086_SAP000_SB043_uv.MS,L797086_SAP000_SB044_uv.MS,L797086_SAP000_SB045_uv.MS,L797086_SAP000_SB046_uv.MS,L797086_SAP000_SB047_uv.MS,L797086_SAP000_SB048_uv.MS,L797086_SAP000_SB049_uv.MS,L797086_SAP000_SB050_uv.MS,L797086_SAP000_SB051_uv.MS,L797086_SAP000_SB052_uv.MS,L797086_SAP000_SB053_uv.MS,L797086_SAP000_SB054_uv.MS,L797086_SAP000_SB055_uv.MS,L797086_SAP000_SB056_uv.MS,L797086_SAP000_SB057_uv.MS,L797086_SAP000_SB058_uv.MS,L797086_SAP000_SB059_uv.MS,L797086_SAP000_SB060_uv.MS,L797086_SAP000_SB061_uv.MS,L797086_SAP000_SB062_uv.MS,L797086_SAP000_SB063_uv.MS,L797086_SAP000_SB064_uv.MS,L797086_SAP000_SB065_uv.MS,L797086_SAP000_SB066_uv.MS,L797086_SAP000_SB067_uv.MS,L797086_SAP000_SB068_uv.MS,L797086_SAP000_SB069_uv.MS,L797086_SAP000_SB070_uv.MS,L797086_SAP000_SB071_uv.MS,L797086_SAP000_SB072_uv.MS,L797086_SAP000_SB073_uv.MS,L797086_SAP000_SB074_uv.MS,L797086_SAP000_SB075_uv.MS,L797086_SAP000_SB076_uv.MS,L797086_SAP000_SB077_uv.MS,L797086_SAP000_SB078_uv.MS,L797086_SAP000_SB079_uv.MS,L797086_SAP000_SB080_uv.MS,L797086_SAP000_SB081_uv.MS,L797086_SAP000_SB082_uv.MS,L797086_SAP000_SB083_uv.MS,L797086_SAP000_SB084_uv.MS,L797086_SAP000_SB085_uv.MS,L797086_SAP000_SB086_uv.MS,L797086_SAP000_SB087_uv.MS,L797086_SAP000_SB088_uv.MS,L797086_SAP000_SB089_uv.MS,L797086_SAP000_SB090_uv.MS,L797086_SAP000_SB091_uv.MS,L797086_SAP000_SB092_uv.MS,L797086_SAP000_SB093_uv.MS,L797086_SAP000_SB094_uv.MS,L797086_SAP000_SB095_uv.MS,L797086_SAP000_SB096_uv.MS,L797086_SAP000_SB097_uv.MS,L797086_SAP000_SB098_uv.MS,L797086_SAP000_SB099_uv.MS,L797086_SAP000_SB100_uv.MS,L797086_SAP000_SB101_uv.MS,L797086_SAP000_SB102_uv.MS,L797086_SAP000_SB103_uv.MS,L797086_SAP000_SB104_uv.MS,L797086_SAP000_SB105_uv.MS,L797086_SAP000_SB106_uv.MS,L797086_SAP000_SB107_uv.MS,L797086_SAP000_SB108_uv.MS,L797086_SAP000_SB109_uv.MS,L797086_SAP000_SB110_uv.MS,L797086_SAP000_SB111_uv.MS,L797086_SAP000_SB112_uv.MS,L797086_SAP000_SB113_uv.MS,L797086_SAP000_SB114_uv.MS,L797086_SAP000_SB115_uv.MS,L797086_SAP000_SB116_uv.MS,L797086_SAP000_SB117_uv.MS,L797086_SAP000_SB118_uv.MS,L797086_SAP000_SB119_uv.MS,L797086_SAP000_SB120_uv.MS,L797086_SAP000_SB121_uv.MS,L797086_SAP000_SB122_uv.MS,L797086_SAP000_SB123_uv.MS,L797086_SAP000_SB124_uv.MS,L797086_SAP000_SB125_uv.MS,L797086_SAP000_SB126_uv.MS,L797086_SAP000_SB127_uv.MS,L797086_SAP000_SB128_uv.MS,L797086_SAP000_SB129_uv.MS,L797086_SAP000_SB130_uv.MS,L797086_SAP000_SB131_uv.MS,L797086_SAP000_SB132_uv.MS,L797086_SAP000_SB133_uv.MS,L797086_SAP000_SB134_uv.MS,L797086_SAP000_SB135_uv.MS,L797086_SAP000_SB136_uv.MS,L797086_SAP000_SB137_uv.MS,L797086_SAP000_SB138_uv.MS,L797086_SAP000_SB139_uv.MS,L797086_SAP000_SB140_uv.MS,L797086_SAP000_SB141_uv.MS,L797086_SAP000_SB142_uv.MS,L797086_SAP000_SB143_uv.MS,L797086_SAP000_SB144_uv.MS,L797086_SAP000_SB145_uv.MS,L797086_SAP000_SB146_uv.MS,L797086_SAP000_SB147_uv.MS,L797086_SAP000_SB148_uv.MS,L797086_SAP000_SB149_uv.MS,L797086_SAP000_SB150_uv.MS,L797086_SAP000_SB151_uv.MS,L797086_SAP000_SB152_uv.MS,L797086_SAP000_SB153_uv.MS,L797086_SAP000_SB154_uv.MS,L797086_SAP000_SB155_uv.MS,L797086_SAP000_SB156_uv.MS,L797086_SAP000_SB157_uv.MS,L797086_SAP000_SB158_uv.MS,L797086_SAP000_SB159_uv.MS,L797086_SAP000_SB160_uv.MS,L797086_SAP000_SB161_uv.MS,L797086_SAP000_SB162_uv.MS,L797086_SAP000_SB163_uv.MS,L797086_SAP000_SB164_uv.MS,L797086_SAP000_SB165_uv.MS,L797086_SAP000_SB166_uv.MS,L797086_SAP000_SB167_uv.MS,L797086_SAP000_SB168_uv.MS,L797086_SAP000_SB169_uv.MS,L797086_SAP000_SB170_uv.MS,L797086_SAP000_SB171_uv.MS,L797086_SAP000_SB172_uv.MS,L797086_SAP000_SB173_uv.MS,L797086_SAP000_SB174_uv.MS,L797086_SAP000_SB175_uv.MS,L797086_SAP000_SB176_uv.MS,L797086_SAP000_SB177_uv.MS,L797086_SAP000_SB178_uv.MS,L797086_SAP000_SB179_uv.MS,L797086_SAP000_SB180_uv.MS,L797086_SAP000_SB181_uv.MS,L797086_SAP000_SB182_uv.MS,L797086_SAP000_SB183_uv.MS,L797086_SAP000_SB184_uv.MS,L797086_SAP000_SB185_uv.MS,L797086_SAP000_SB186_uv.MS,L797086_SAP000_SB187_uv.MS,L797086_SAP000_SB188_uv.MS,L797086_SAP000_SB189_uv.MS,L797086_SAP000_SB190_uv.MS,L797086_SAP000_SB191_uv.MS,L797086_SAP000_SB192_uv.MS,L797086_SAP000_SB193_uv.MS,L797086_SAP000_SB194_uv.MS,L797086_SAP000_SB195_uv.MS,L797086_SAP000_SB196_uv.MS,L797086_SAP000_SB197_uv.MS,L797086_SAP000_SB198_uv.MS,L797086_SAP000_SB199_uv.MS,L797086_SAP000_SB200_uv.MS,L797086_SAP000_SB201_uv.MS,L797086_SAP000_SB202_uv.MS,L797086_SAP000_SB203_uv.MS,L797086_SAP000_SB204_uv.MS,L797086_SAP000_SB205_uv.MS,L797086_SAP000_SB206_uv.MS,L797086_SAP000_SB207_uv.MS,L797086_SAP000_SB208_uv.MS,L797086_SAP000_SB209_uv.MS,L797086_SAP000_SB210_uv.MS,L797086_SAP000_SB211_uv.MS,L797086_SAP000_SB212_uv.MS,L797086_SAP000_SB213_uv.MS,L797086_SAP000_SB214_uv.MS,L797086_SAP000_SB215_uv.MS,L797086_SAP000_SB216_uv.MS,L797086_SAP000_SB217_uv.MS,L797086_SAP000_SB218_uv.MS,L797086_SAP000_SB219_uv.MS,L797086_SAP000_SB220_uv.MS,L797086_SAP000_SB221_uv.MS,L797086_SAP000_SB222_uv.MS,L797086_SAP000_SB223_uv.MS,L797086_SAP000_SB224_uv.MS,L797086_SAP000_SB225_uv.MS,L797086_SAP000_SB226_uv.MS,L797086_SAP000_SB227_uv.MS,L797086_SAP000_SB228_uv.MS,L797086_SAP000_SB229_uv.MS,L797086_SAP000_SB230_uv.MS,L797086_SAP000_SB231_uv.MS,L797086_SAP000_SB232_uv.MS,L797086_SAP000_SB233_uv.MS,L797086_SAP000_SB234_uv.MS,L797086_SAP000_SB235_uv.MS,L797086_SAP000_SB236_uv.MS,L797086_SAP000_SB237_uv.MS,L797086_SAP000_SB238_uv.MS,L797086_SAP000_SB239_uv.MS,L797086_SAP000_SB240_uv.MS,L797086_SAP000_SB241_uv.MS,L797086_SAP000_SB242_uv.MS,L797086_SAP000_SB243_uv.MS]
+Observation.DataProducts.Output_Correlated.locations=[244*localhost:/output/797086]
+Observation.DataProducts.Output_Correlated.storageClusterName="localhost"
+
+Observation.DataProducts.Output_IncoherentStokes.enabled=false
+Observation.DataProducts.Output_InstrumentModel.enabled=false
+Observation.DataProducts.Output_Pulsar.enabled=false
+Observation.DataProducts.Output_SkyImage.enabled=false
+
+#
+# ----- Dataslot settings, remove? -----
+#
+Observation.Dataslots.CS001HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS001HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS001HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS001HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS002HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS002HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS002HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS002HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS003HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS003HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS003HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS003HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS004HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS004HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS004HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS004HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS005HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS005HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS005HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS005HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS006HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS006HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS006HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS006HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS007HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS007HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS007HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS007HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS011HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS011HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS011HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS011HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS013HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS013HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS013HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS013HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS017HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS017HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS017HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS017HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS021HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS021HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS021HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS021HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS024HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS024HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS024HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS024HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS026HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS026HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS026HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS026HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS028HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS028HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS028HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS028HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS030HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS030HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS030HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS030HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS031HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS031HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS031HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS031HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS032HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS032HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS032HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS032HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS101HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS101HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS101HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS101HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS103HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS103HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS103HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS103HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS201HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS201HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS201HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS201HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS301HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS301HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS301HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS301HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS302HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS302HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS302HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS302HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS401HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS401HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS401HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS401HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS501HBA0.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS501HBA0.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.CS501HBA1.DataslotList=[0..121,0..121]
+Observation.Dataslots.CS501HBA1.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.DataslotInfo.DataslotList=[]
+Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+Observation.Dataslots.RS106HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS106HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS205HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS205HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS208HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS208HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS210HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS210HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS305HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS305HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS306HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS306HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS307HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS307HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS310HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS310HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS406HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS406HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS407HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS407HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS409HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS409HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS503HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS503HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS508HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS508HBA.RSPBoardList=[122*0,122*1]
+Observation.Dataslots.RS509HBA.DataslotList=[0..121,0..121]
+Observation.Dataslots.RS509HBA.RSPBoardList=[122*0,122*1]
+
+#
+# ----- Observation settings -----
+#
+Observation.ObsID=797086
+Observation.VirtualInstrument.minimalNrStations=1
+Observation.VirtualInstrument.stationList=[CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS101,CS103,CS201,CS301,CS302,CS401,CS501,RS106,RS205,RS208,RS210,RS305,RS306,RS307,RS310,RS406,RS407,RS409,RS503,RS508,RS509]
+Observation.VirtualInstrument.stationSet=Custom
+Observation.antennaArray=HBA
+Observation.antennaSet=HBA_DUAL_INNER
+Observation.bandFilter=HBA_110_190
+Observation.claimPeriod=35
+Observation.clockMode=<<Clock200
+Observation.momID=1036290
+Observation.nrAnaBeams=1
+Observation.nrBeams=1
+Observation.nrBitsPerSample=8
+Observation.nrTBBSettings=0
+Observation.originID=650576
+Observation.otdbID=797086
+Observation.preparePeriod=20
+Observation.processSubtype=Beam Observation
+Observation.processType=Observation
+Observation.referencePhaseCenter=[3826577.066, 461022.948, 5064892.786]
+Observation.sampleClock=200
+Observation.startTime=2020-10-21 16:42:00
+Observation.stopTime=2020-10-21 18:42:00
+Observation.strategy=default
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-797130.parset b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-797130.parset
new file mode 100644
index 0000000000000000000000000000000000000000..735a6d50766a4df73b1fdbbdee03f250ca5611aa
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/parsets/gpuload-797130.parset
@@ -0,0 +1,208 @@
+#
+# ----- Info -----
+#
+# Purpose: LOFAR reference test
+# Pulsar timing observation
+# Intended to be used with gpu_load test for GPU kernel performance benchmarking
+
+# Enable performance benchmarking
+Cobalt.Benchmark.enabled=true
+Cobalt.Benchmark.file=Benchmarks/797130.csv
+
+# These figures are used by gpu_load to predict the load across the whole cluster.
+# Set available processing resources for the full COBALT 2.0 system (2 GPUs/sockets x 11 nodes)
+Cobalt.Nodes = [ 22*localhost ]
+
+#
+# ----- Beamformer settings -----
+#
+Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband=1
+Cobalt.BeamFormer.CoherentStokes.subbandsPerFile=20
+Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor=1
+Cobalt.BeamFormer.CoherentStokes.which=XXYY
+Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband=1
+Cobalt.BeamFormer.IncoherentStokes.subbandsPerFile=512
+Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor=1
+Cobalt.BeamFormer.IncoherentStokes.which=I
+Cobalt.BeamFormer.coherentDedisperseChannels=false
+Cobalt.BeamFormer.flysEye=false
+Cobalt.BeamFormer.nrDelayCompensationChannels=256
+Cobalt.BeamFormer.nrHighResolutionChannels=256
+Cobalt.BeamFormer.stationList=[]
+
+#
+# ----- Correlator settings -----
+#
+Cobalt.Correlator.integrationTime=1.00663
+Cobalt.Correlator.nrBlocksPerIntegration=1
+Cobalt.Correlator.nrChannelsPerSubband=1
+Cobalt.Correlator.nrIntegrationsPerBlock=1
+
+#
+# ----- Misc settings -----
+#
+Cobalt.blockSize=196608
+Cobalt.correctBandPass=true
+Cobalt.correctClocks=true
+Cobalt.delayCompensation=true
+Cobalt.realTime=true
+
+#
+# ----- Pointings -----
+#
+Observation.Beam[0].TiedArrayBeam[0].angle1=3.06917167072
+Observation.Beam[0].TiedArrayBeam[0].angle2=0.525315016029
+Observation.Beam[0].TiedArrayBeam[0].coherent=true
+Observation.Beam[0].TiedArrayBeam[0].directionType=J2000
+Observation.Beam[0].TiedArrayBeam[0].dispersionMeasure=0
+Observation.Beam[0].angle1=3.069171670713514
+Observation.Beam[0].angle2=0.5253150160288448
+Observation.Beam[0].directionType=J2000
+Observation.Beam[0].duration=1200
+Observation.Beam[0].momID=1036317
+Observation.Beam[0].nrTabRings=0
+Observation.Beam[0].nrTiedArrayBeams=1
+Observation.Beam[0].startTime=
+Observation.Beam[0].subbandList=[51..450]
+Observation.Beam[0].tabRingSize=0.0
+Observation.Beam[0].target=ILT_J114324.1+300554
+
+#
+# ----- Output settings -----
+#
+Observation.DataProducts.Output_CoherentStokes.enabled=true
+Observation.DataProducts.Output_CoherentStokes.filenames=[L797130_SAP000_B000_S0_P000_bf.h5,L797130_SAP000_B000_S0_P001_bf.h5,L797130_SAP000_B000_S0_P002_bf.h5,L797130_SAP000_B000_S0_P003_bf.h5,L797130_SAP000_B000_S0_P004_bf.h5,L797130_SAP000_B000_S0_P005_bf.h5,L797130_SAP000_B000_S0_P006_bf.h5,L797130_SAP000_B000_S0_P007_bf.h5,L797130_SAP000_B000_S0_P008_bf.h5,L797130_SAP000_B000_S0_P009_bf.h5,L797130_SAP000_B000_S0_P010_bf.h5,L797130_SAP000_B000_S0_P011_bf.h5,L797130_SAP000_B000_S0_P012_bf.h5,L797130_SAP000_B000_S0_P013_bf.h5,L797130_SAP000_B000_S0_P014_bf.h5,L797130_SAP000_B000_S0_P015_bf.h5,L797130_SAP000_B000_S0_P016_bf.h5,L797130_SAP000_B000_S0_P017_bf.h5,L797130_SAP000_B000_S0_P018_bf.h5,L797130_SAP000_B000_S0_P019_bf.h5,L797130_SAP000_B000_S1_P000_bf.h5,L797130_SAP000_B000_S1_P001_bf.h5,L797130_SAP000_B000_S1_P002_bf.h5,L797130_SAP000_B000_S1_P003_bf.h5,L797130_SAP000_B000_S1_P004_bf.h5,L797130_SAP000_B000_S1_P005_bf.h5,L797130_SAP000_B000_S1_P006_bf.h5,L797130_SAP000_B000_S1_P007_bf.h5,L797130_SAP000_B000_S1_P008_bf.h5,L797130_SAP000_B000_S1_P009_bf.h5,L797130_SAP000_B000_S1_P010_bf.h5,L797130_SAP000_B000_S1_P011_bf.h5,L797130_SAP000_B000_S1_P012_bf.h5,L797130_SAP000_B000_S1_P013_bf.h5,L797130_SAP000_B000_S1_P014_bf.h5,L797130_SAP000_B000_S1_P015_bf.h5,L797130_SAP000_B000_S1_P016_bf.h5,L797130_SAP000_B000_S1_P017_bf.h5,L797130_SAP000_B000_S1_P018_bf.h5,L797130_SAP000_B000_S1_P019_bf.h5,L797130_SAP000_B000_S2_P000_bf.h5,L797130_SAP000_B000_S2_P001_bf.h5,L797130_SAP000_B000_S2_P002_bf.h5,L797130_SAP000_B000_S2_P003_bf.h5,L797130_SAP000_B000_S2_P004_bf.h5,L797130_SAP000_B000_S2_P005_bf.h5,L797130_SAP000_B000_S2_P006_bf.h5,L797130_SAP000_B000_S2_P007_bf.h5,L797130_SAP000_B000_S2_P008_bf.h5,L797130_SAP000_B000_S2_P009_bf.h5,L797130_SAP000_B000_S2_P010_bf.h5,L797130_SAP000_B000_S2_P011_bf.h5,L797130_SAP000_B000_S2_P012_bf.h5,L797130_SAP000_B000_S2_P013_bf.h5,L797130_SAP000_B000_S2_P014_bf.h5,L797130_SAP000_B000_S2_P015_bf.h5,L797130_SAP000_B000_S2_P016_bf.h5,L797130_SAP000_B000_S2_P017_bf.h5,L797130_SAP000_B000_S2_P018_bf.h5,L797130_SAP000_B000_S2_P019_bf.h5,L797130_SAP000_B000_S3_P000_bf.h5,L797130_SAP000_B000_S3_P001_bf.h5,L797130_SAP000_B000_S3_P002_bf.h5,L797130_SAP000_B000_S3_P003_bf.h5,L797130_SAP000_B000_S3_P004_bf.h5,L797130_SAP000_B000_S3_P005_bf.h5,L797130_SAP000_B000_S3_P006_bf.h5,L797130_SAP000_B000_S3_P007_bf.h5,L797130_SAP000_B000_S3_P008_bf.h5,L797130_SAP000_B000_S3_P009_bf.h5,L797130_SAP000_B000_S3_P010_bf.h5,L797130_SAP000_B000_S3_P011_bf.h5,L797130_SAP000_B000_S3_P012_bf.h5,L797130_SAP000_B000_S3_P013_bf.h5,L797130_SAP000_B000_S3_P014_bf.h5,L797130_SAP000_B000_S3_P015_bf.h5,L797130_SAP000_B000_S3_P016_bf.h5,L797130_SAP000_B000_S3_P017_bf.h5,L797130_SAP000_B000_S3_P018_bf.h5,L797130_SAP000_B000_S3_P019_bf.h5]
+Observation.DataProducts.Output_CoherentStokes.locations=[80*localhost:output/797130]
+Observation.DataProducts.Output_CoherentStokes.storageClusterName="localhost"
+
+Observation.DataProducts.Output_Correlated.enabled=false
+Observation.DataProducts.Output_IncoherentStokes.enabled=false
+Observation.DataProducts.Output_InstrumentModel.enabled=false
+Observation.DataProducts.Output_Pulsar.enabled=false
+Observation.DataProducts.Output_SkyImage.enabled=false
+
+#
+# ----- Dataslot settings, remove? -----
+#
+Observation.Dataslots.CS001HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS001HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS001HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS001HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS002HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS002HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS002HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS002HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS003HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS003HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS003HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS003HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS004HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS004HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS004HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS004HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS005HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS005HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS005HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS005HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS006HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS006HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS006HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS006HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS007HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS007HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS007HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS007HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS011HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS011HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS011HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS011HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS013HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS013HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS013HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS013HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS017HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS017HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS017HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS017HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS021HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS021HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS021HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS021HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS024HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS024HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS024HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS024HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS026HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS026HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS026HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS026HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS028HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS028HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS028HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS028HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS030HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS030HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS030HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS030HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS031HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS031HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS031HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS031HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS101HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS101HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS101HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS101HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS103HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS103HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS103HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS103HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS201HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS201HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS201HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS201HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS301HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS301HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS301HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS301HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS302HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS302HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS302HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS302HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS401HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS401HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS401HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS401HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS501HBA0.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS501HBA0.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.CS501HBA1.DataslotList=[0..121,0..121,0..121,0..33]
+Observation.Dataslots.CS501HBA1.RSPBoardList=[122*0,122*1,122*2,34*3]
+Observation.Dataslots.DataslotInfo.DataslotList=[]
+Observation.Dataslots.DataslotInfo.RSPBoardList=[]
+
+#
+# ----- Observation settings -----
+#
+Observation.ObsID=797130
+Observation.VirtualInstrument.minimalNrStations=1
+Observation.VirtualInstrument.stationList=[CS001,CS002,CS003,CS004,CS005,CS006,CS007,CS011,CS013,CS017,CS021,CS024,CS026,CS028,CS030,CS031,CS101,CS103,CS201,CS301,CS302,CS401,CS501]
+Observation.VirtualInstrument.stationSet=Custom
+Observation.antennaArray=HBA
+Observation.antennaSet=HBA_DUAL
+Observation.bandFilter=HBA_110_190
+Observation.claimPeriod=35
+Observation.clockMode=<<Clock200
+Observation.momID=1036316
+Observation.nrAnaBeams=1
+Observation.nrBeams=1
+Observation.nrBitsPerSample=8
+Observation.nrTBBSettings=0
+Observation.originID=650576
+Observation.otdbID=797130
+Observation.preparePeriod=20
+Observation.processSubtype=Beam Observation
+Observation.processType=Observation
+Observation.referencePhaseCenter=[3826577.066, 461022.948, 5064892.786]
+Observation.sampleClock=200
+Observation.startTime=2020-10-25 08:50:00
+Observation.stopTime=2020-10-25 09:10:00
+Observation.strategy=default
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks.py b/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks.py
new file mode 100755
index 0000000000000000000000000000000000000000..f3ea906b3cbb2fea0158d3da3f047134e8ec2d39
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+#Script to run muliple consecutive benchmarks and store output
+#Runs a set of representative benchmarks to test performance regression of the GPU kernels for the different pipelines.
+#Tests are specified through a set of LOFAR parsets and are passed to the gpu_load test that sets up a relative simple pipeline with performance benchmarking enabled.
+
+import subprocess
+import time
+import sys
+import os
+import shutil
+
+import argparse
+
+class Error(Exception):
+ #Base class for custom exceptions
+ pass
+
+
+class ParsetError(Error):
+ #Error in input parset
+ pass
+
+def create_arg_parser():
+ # Creates and returns the ArgumentParser object
+ parser = argparse.ArgumentParser(description='Batch process gpu_load performance benchmarks and specify output location')
+ parser.add_argument('executableDirectory', nargs='?', default="",
+ help='The directory with the binary for the gpu_load test e.g. .../opt/lofar/bin')
+ parser.add_argument('parsetDirectory', nargs='?', default="",
+ help='The directory with the parsets for the benchmarks')
+ parser.add_argument('resultsDirectory', nargs='?', default="",
+ help='The directory to store the test results')
+ parser.add_argument('--nIterations', type=int,
+ help='Specify a number of iterations for the gpu_load test, default is 1000')
+ parser.add_argument('--storeParsets', action='store_true',
+ help='Enable storing of the parsets with the test results')
+ parser.add_argument('--selectParsets', nargs='*', default="",
+ help='Only run these parsets from the directory with parsets, specify one or more files')
+ # parser.add_argument('--GPU',
+ # help='Which GPU to use') # set environment variable
+ parser.add_argument('--dryRun', action='store_true',
+ help='Dry Run')
+ parser.add_argument('--verbose', action='store_true',
+ help='Enable verbose printing')
+ return parser
+
+def enableBenchmarking(parsetFullPath, benchmarkResultsFullPath, verbosePrint):
+ #Looking for:
+ # # Enable performance benchmarking
+ # Cobalt.Benchmark.enabled=true
+ # Cobalt.Benchmark.file=Benchmarks/763847.csv
+ #Check if these parset keys are already there,
+ # return list of tuples withmatched string, line numbers and lines where string is found
+ keysBenchmark = ['Cobalt.Benchmark.enabled', 'Cobalt.Benchmark.file']
+ strKeyEnable = keysBenchmark[0]+"=true\n"
+ strKeyFile = keysBenchmark[1] + "=" + benchmarkResultsFullPath + "\n"
+
+ #Check if enable key is there
+ searchResultsEnabled = findString(parsetFullPath, keysBenchmark[0])
+ # if verbosePrint: print(searchResultsEnabled)
+ if len(searchResultsEnabled)>1:
+ print("Found too many occurrences of Parset Benchmark key")
+ raise(ParsetError)
+ elif len(searchResultsEnabled)==1:
+ #assuming that either both keys are there or both are not there, overwrite both
+ #find location of Benchmark key as well
+ searchResultsFile = findString(parsetFullPath, keysBenchmark[1])
+ # if verbosePrint: print(searchResultsFile)
+ if len(searchResultsFile)>1:
+ print("Found too many occurrences of Parset Benchmark key")
+ raise(ParsetError)
+ if len(searchResultsFile)<1:
+ print("Did not find Parset Benchmark key")
+ raise(ParsetError)
+ #overwrite
+ if verbosePrint: print("Overwiting parset keys on lines {} and {}".format(searchResultsEnabled[0][0],searchResultsFile[0][0]))
+ fh = open(parsetFullPath, "r")
+ listOfLines = fh.readlines()
+ fh.close()
+ listOfLines[searchResultsEnabled[0][0]-1] = strKeyEnable
+ listOfLines[searchResultsFile[0][0]-1] = strKeyFile
+ fh = open(parsetFullPath, "w")
+ fh.writelines(listOfLines)
+ fh.close()
+ else:
+ #keys are not there, append them to file
+ print("Appending parset keys")
+ stringToWrite = "\n\n# Enable performance benchmarking\n" + strKeyEnable + strKeyFile
+ fh = open(parsetFullPath, "a")
+ fh.write(stringToWrite)
+ fh.close()
+
+ return
+
+def findString(fullPath, searchString):
+ #Search for a given string in the file and return all line numbers and lines containing that string
+ lineNumber = 0
+ searchResults = []
+ # Open the file
+ with open(fullPath, 'r') as read_obj:
+ # Read all lines in the file one by one
+ for line in read_obj:
+ # For each line, check if line contains the string
+ lineNumber += 1
+ if searchString in line:
+ # If yes, then add the line number & line as a tuple in the list
+ searchResults.append((lineNumber, line.rstrip()))
+ # Return list of tuples containing line numbers and lines where string is found
+ return searchResults
+
+if __name__ == "__main__":
+ #Run application and capture output
+
+ argParser = create_arg_parser()
+ parsedArgs = argParser.parse_args(sys.argv[1:])
+ verbosePrint = parsedArgs.verbose
+ nIterationsGPU_LOAD = 1000
+ if parsedArgs.nIterations: nIterationsGPU_LOAD = parsedArgs.nIterations
+
+ #Checking
+ executableDirectory = parsedArgs.executableDirectory
+ if not os.path.exists(executableDirectory):
+ print("Directory " + executableDirectory + " does not exist")
+ raise(FileNotFoundError)
+ executableFullPath = os.path.join(executableDirectory, "gpu_load")
+
+ parsetDirectory = parsedArgs.parsetDirectory
+ if not os.path.exists(parsetDirectory):
+ print("Directory " + parsetDirectory + " does not exist")
+ raise(FileNotFoundError)
+
+ resultsDirectory = parsedArgs.resultsDirectory
+ if not os.path.exists(resultsDirectory):
+ print("Directory " + resultsDirectory + " does not exist")
+ raise(FileNotFoundError)
+ # Or create directory?
+
+ if parsedArgs.selectParsets:
+ #Take these as file list
+ parsetList = parsedArgs.selectParsets
+ else:
+ #Get list of files from the parset directory
+ parsetList = os.listdir(parsetDirectory)
+ nparsets = len(parsetList)
+ if verbosePrint:
+ print('Using {} parsets from directory {} :'.format(nparsets, parsetDirectory))
+ print(parsetList)
+
+ # #Optionally set which GPU to use
+ # FIXME: gpu_load test still selects gpu '0' even if set to '1' here.
+ # Instead propagate this variable to gpu_load and set device there, if we wish to use this in the future
+ # if (parsedArgs.GPU):
+ # print('Set CUDA_VISIBILE_DEVICES to {}'.format(str(parsedArgs.GPU)))
+ # os.environ['CUDA_VISIBILE_DEVICES'] = str(parsedArgs.GPU)
+ # if verbosePrint: print('Read back environment variable CUDA_VISIBILE_DEVICES = {}'.format(os.environ.get('CUDA_VISIBILE_DEVICES')))
+
+ #Get start times and format them
+ starttime=time.localtime()
+ applicationStartTime = time.time()
+ string_datetime = time.strftime("%Y_%m_%d-%H_%M_%S", starttime)
+
+ #Create directory with time and data to store results
+ # Check paths, create directory, set path variables
+ resultsFullPath = os.path.join(resultsDirectory, 'PerformanceTestResults_'+string_datetime)
+ try:
+ os.mkdir(resultsFullPath)
+ except OSError as error:
+ print(error)
+ if verbosePrint: print('Created directory {} for bechmark run results'.format(resultsFullPath))
+
+ #Create file to store output
+ logFullPath = os.path.join(resultsFullPath,string_datetime+".log")
+ try:
+ f = open(logFullPath, "w")
+ except OSError:
+ print('Could not open file: {}'.format(logFullPath))
+
+ tmpString='Starting application at {}'.format(time.strftime("%a %b %d %H:%M:%S %Y",starttime))
+ if verbosePrint: print(tmpString)
+ f.write(tmpString+'\n')
+
+ f.write('### Benchmark started with:\n### Arguments: {}\n### Parsets: {}\n\n'.format(parsedArgs, parsetList))
+
+ #Loop over all tests
+ for i, P in enumerate(parsetList):
+ #Log
+ tmpString='### {} Running test: {} out of {} : {}'.format(time.strftime("%Y_%m_%d-%H_%M_%S", time.localtime()),i+1,nparsets,P)
+ if verbosePrint: print(tmpString)
+ f.write(tmpString+'\n\n')
+
+ #Copy parset to results directory
+ baseParsetFullPath = os.path.join(parsetDirectory, P)
+ testcaseParsetFullPath = os.path.join(resultsFullPath, P)
+ benchmarkResultsFullPath = os.path.join(resultsFullPath, P.split('.')[0] + ".csv")
+ try:
+ shutil.copy(baseParsetFullPath, testcaseParsetFullPath)
+ except OSError:
+ print('Could not copy from {} to {}'.format(baseParsetFullPath, testcaseParsetFullPath))
+
+ #Adapt settings in parset
+ enableBenchmarking(testcaseParsetFullPath, benchmarkResultsFullPath, verbosePrint)
+
+ #Run the application and capture output
+ if(not parsedArgs.dryRun):
+ executeCommand = executableFullPath + ' ' + testcaseParsetFullPath + ' {}'.format(nIterationsGPU_LOAD)
+ output = subprocess.getoutput(executeCommand)
+ #Save to file
+ f.write(output+'\n')
+
+ #If not specified to store parsets, delete the adapted parset
+ if(not parsedArgs.storeParsets):
+ os.remove(testcaseParsetFullPath)
+ #End loop over all tests
+
+ #Wrap up
+ applicationTime = time.time()-applicationStartTime
+ tmpString='### Ending application at {} processing took {:.2f} seconds'.format(time.strftime("%a %b %d %H:%M:%S %Y",time.localtime()),applicationTime)
+ if verbosePrint:print(tmpString)
+ f.write(tmpString)
+ f.close()
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks_analysis.py b/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks_analysis.py
new file mode 100755
index 0000000000000000000000000000000000000000..9844de60541252daecc5ba803130da6b4579d3e1
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks_analysis.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+#Script analyze results from gpu_load tests
+
+import time
+import sys
+import os
+
+import argparse
+
+
+def create_arg_parser():
+ # Creates and returns the ArgumentParser object
+ parser = argparse.ArgumentParser(description='Analyze results from gpu_load performance benchmarks')
+ parser.add_argument('benchmarkDirectory', nargs='?', default="",
+ help='The directory with the parsets for the benchmarks')
+ parser.add_argument('--selectFiles', nargs='*', default="",
+ help='Only use these files for analysis instead of all files in the direcotry, specify one or more files')
+ parser.add_argument('--verbose', action='store_true',
+ help='Enable verbose printing')
+ return parser
+
+def findString(fullPath, searchString):
+ #Search for a given string in the file and return a list of lines containing that string
+ searchResults = []
+ # Open the file
+ with open(fullPath, 'r') as read_obj:
+ # Read all lines in the file one by one
+ for line in read_obj:
+ # For each line, check if line contains the string
+ if searchString in line:
+ # If yes, then add the line in the list
+ searchResults.append(line.rstrip())
+ return searchResults
+
+if __name__ == "__main__":
+ #Run application and capture output
+
+ argParser = create_arg_parser()
+ parsedArgs = argParser.parse_args(sys.argv[1:])
+ verbosePrint = parsedArgs.verbose
+
+ #Checking
+ benchmarkDirectory = parsedArgs.benchmarkDirectory
+ if not os.path.exists(benchmarkDirectory):
+ print("Directory " + benchmarkDirectory + " does not exist")
+ raise(FileNotFoundError)
+
+ #If not there yet create sub-directory for analysis results
+ analysisDirectory = os.path.join(benchmarkDirectory, 'analysis')
+ if not os.path.exists(analysisDirectory):os.mkdir(analysisDirectory)
+
+ summaryFullPath = os.path.join(analysisDirectory,"summary.csv")
+
+ if parsedArgs.selectFiles:
+ #Take these as file list
+ benchmarkList = parsedArgs.selectFiles
+ else:
+ #Get list of all .csv files from the benchmark directory
+ benchmarkList = [f for f in os.listdir(benchmarkDirectory) if f.endswith('.csv')]
+ nbench = len(benchmarkList)
+ if verbosePrint:
+ print('Using {} benchmark results from directory {} :'.format(nbench, benchmarkDirectory))
+ print(benchmarkList)
+
+ #Get start times and format them
+ starttime=time.localtime()
+ applicationStartTime = time.time()
+
+ #Create file to store output
+ #tmp = benchmarkDirectory.split('/') #get name of directory
+ #logFullPath = os.path.join(benchmarkDirectory,tmp[len(tmp)-1]+"_analysis.log")
+ logFullPath = os.path.join(analysisDirectory,"analysis.log")
+ try:
+ logFile = open(logFullPath, "w")
+ except OSError:
+ print('Could not open file: {}'.format(logFullPath))
+
+ try:
+ summaryFile = open(summaryFullPath, "w")
+ except OSError:
+ print('Could not open file: {}'.format(summaryFullPath))
+ summaryFile.write('{:26}{:32}{:10}{:10}{:10}{:10}{:10}{:10}\n'.format("benchmarkID;", "kernelName;", "count;", "mean;", "stDev;", "min;", "max;", "unit;"))
+
+ meansFullPath = os.path.join(analysisDirectory,"means.csv")
+ try:
+ meansFile = open(meansFullPath, "w")
+ except OSError:
+ print('Could not open file: {}'.format(meansFullPath))
+
+ tmpString='Starting application at {}'.format(time.strftime("%a %b %d %H:%M:%S %Y",starttime))
+ if verbosePrint: print(tmpString)
+ logFile.write(tmpString+'\n')
+
+ logFile.write('### Analysis started with:\n### Arguments: {}\n### Benchmarks: {}\n\n'.format(parsedArgs, benchmarkList))
+
+ #means = {} #dict to store kernelIndex and mean values
+
+ #Loop over all benchmarks
+ for i, B in enumerate(benchmarkList):
+ #Log
+ tmpString='### {} Running test: {} out of {} : {}'.format(time.strftime("%Y_%m_%d-%H_%M_%S", time.localtime()),i+1,nbench,B)
+ if verbosePrint: print(tmpString)
+ logFile.write(tmpString+'\n\n')
+
+ benchmarkFullPath = os.path.join(benchmarkDirectory, B)
+
+ #Read "PerformanceCounter" results from benchmark
+ perfCounters = findString(benchmarkFullPath,"PerformanceCounter")
+ #print(perfCounters)
+
+ #Re-format results for further processing
+ for line in perfCounters:
+ lineSplit = line.split(';')
+ benchName = B.split('.')[0] #strip name from <name>.csv
+ kernelNameFormatted = lineSplit[1].replace(" ","") #remove spaces to be able to use it as index
+ kernelIndex = benchName + '_' + kernelNameFormatted
+ #means[kernelIndex] = lineSplit[3] #dict with mean values
+ #Write results to means and summary file with benchmark prefix
+ meansFile.write('{:45}{:10}\n'.format(kernelIndex + '; ', lineSplit[3]))
+ tmpString = '{:25}'.format(benchName + ';')
+ tmpString = tmpString + '{:32}'.format(lineSplit[1] + ';')
+ for element in lineSplit[2:]:
+ tmpString = tmpString + '{:10}'.format(element + ';')
+ tmpString = tmpString + '\n'
+ summaryFile.write(tmpString)
+ #print(means)
+ #End loop over all tests
+
+ #Wrap up
+ applicationTime = time.time()-applicationStartTime
+ tmpString='### Ending application at {} processing took {:.2f} seconds'.format(time.strftime("%a %b %d %H:%M:%S %Y",time.localtime()),applicationTime)
+ if verbosePrint:print(tmpString)
+ logFile.write(tmpString)
+ meansFile.close()
+ summaryFile.close()
+ logFile.close()
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks_analysis_compare.py b/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks_analysis_compare.py
new file mode 100755
index 0000000000000000000000000000000000000000..ca18cbb862469e2c06ef2d4ab9d5f1605a9b977e
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/PerformanceTest/run_benchmarks_analysis_compare.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+#Script to compare analysis mean results from different runs, e.g. performance regression
+
+import subprocess
+import numpy as np
+import time
+import sys
+import os
+
+import argparse
+
+
+def create_arg_parser():
+ # Creates and returns the ArgumentParser object
+ parser = argparse.ArgumentParser(description='Compare analysis mean results from different runs from gpu_load performance benchmarks')
+ parser.add_argument('fileReference', nargs='?', default="",
+ help='The full path to the reference file')
+ parser.add_argument('fileCurrent', nargs='?', default="",
+ help='The full path to the current file, the comparison results are written here as well')
+ parser.add_argument('--verbose', action='store_true',
+ help='Enable verbose printing')
+ return parser
+
+def readMeans(fullPath):
+ #Search for all strings containing ';' separated values and return those strings as dict with kernelIndex and mean value
+ searchResults = {}
+ searchString = ';'
+ # Open the file
+ with open(fullPath, 'r') as read_obj:
+ # Read all lines in the file one by one
+ for line in read_obj:
+ # For each line, check if line contains ';' #filtering out unintended input like empyt lines etc
+ if searchString in line:
+ # If yes, then split in key and value and add to the dict
+ splitLine = line.split(';') #assuming splitLine now contains: ['kernelIndex','xx.yyyyyy']
+ searchResults[splitLine[0].rstrip()] = float(splitLine[1])
+ return searchResults
+
+if __name__ == "__main__":
+ #Run application and capture output
+
+ argParser = create_arg_parser()
+ parsedArgs = argParser.parse_args(sys.argv[1:])
+ verbosePrint = parsedArgs.verbose
+
+ #Checking
+ fileReferenceFullPath = parsedArgs.fileReference
+ if not os.path.exists(fileReferenceFullPath):
+ print("File " + fileReferenceFullPath + " does not exist")
+ raise(FileNotFoundError)
+
+ fileCurrentFullPath = parsedArgs.fileCurrent
+ if not os.path.exists(fileCurrentFullPath):
+ print("File " + fileCurrentFullPath + " does not exist")
+ raise(FileNotFoundError)
+
+ #Get start times and format them
+ starttime=time.localtime()
+ applicationStartTime = time.time()
+
+ #Create file to store output
+ fileCurrentBasePath = os.path.split(fileCurrentFullPath)[0] #get head of path
+ logFullPath = os.path.join(fileCurrentBasePath,"compare.log")
+ try:
+ logFile = open(logFullPath, "w")
+ except OSError:
+ print('Could not open file: {}'.format(logFullPath))
+
+ compareSummaryFullPath = os.path.join(fileCurrentBasePath,"compareMeans.csv")
+ try:
+ logFile = open(logFullPath, "w")
+ except OSError:
+ print('Could not open file: {}'.format(logFullPath))
+
+ logFile.write('### Analysis started with:\n### Arguments: {}\n\n'.format(parsedArgs))
+
+ tmpString='### Starting application at {}'.format(time.strftime("%a %b %d %H:%M:%S %Y",starttime))
+ if verbosePrint: print(tmpString)
+ logFile.write(tmpString+'\n')
+
+ try:
+ compareSummaryFile = open(compareSummaryFullPath, "w")
+ except OSError:
+ print('Could not open file: {}'.format(compareSummaryFile))
+ headerString = '{:45}; {:>15}; {:>15}; {:>15};'.format("kernelIndex", "Reference mean", "Current mean", "Difference mean")
+ compareSummaryFile.write(headerString + '\n')
+
+ if verbosePrint: print('Writing comparision results to: {}'.format(compareSummaryFullPath))
+
+
+ #dicts with kernelIndex and mean values
+ meansReference = readMeans(fileReferenceFullPath)
+ meansCurrent = readMeans(fileCurrentFullPath)
+
+ #lists to keep track of keys not found and deviating
+ keysNotFound = []
+ keysDeviating = []
+
+ #percentual bound allowed for deviation
+ bound = 5 #%
+ boundFloat = (100-bound)/100
+
+ if verbosePrint: print('### Comparing referece and current, difference = current - reference')
+ #Check size of dicts
+ meansReferenceLen = len(meansReference)
+ meansCurrentLen = len(meansCurrent)
+ if meansCurrentLen is not meansReferenceLen:
+ tmpString = '### Note amount of values in Reference and Current is not equal, found {} values in reference and {} values in current'.format(meansReferenceLen, meansCurrentLen)
+ if verbosePrint: print(tmpString)
+ logFile.write(tmpString)
+ #Compare means
+ for key in sorted(meansReference):
+ #Check if the key also exists in our Current file
+ if key in meansCurrent.keys():
+ diff = meansCurrent[key] - meansReference[key]
+ tmpString = '{:45}; {:>15}; {:>15}; {:>15};'.format(key, '{:.5f}'.format(meansReference[key]), '{:.5f}'.format(meansCurrent[key]), '{:.5f}'.format(diff))
+ #keep list of keys that deviate from reference by more than defined bound
+ absdiff = np.abs(diff)
+ if (meansReference[key] - absdiff) / meansReference[key] < boundFloat:
+ keysDeviating.append(key)
+ tmpString = tmpString + ' <--'
+ else:
+ #keep list of keys that were not found
+ keysNotFound.append(key)
+ tmpString = '{:45}; {:>15}; {:>15}; {:>15};'.format(key, '{:.5f}'.format(meansReference[key]), np.nan, np.nan)
+ if verbosePrint: print(tmpString)
+ compareSummaryFile.write(tmpString+'\n')
+
+ #Checking
+ tmpString = '\n### Summary:'
+ if verbosePrint: print(tmpString)
+ logFile.write(tmpString+'\n')
+
+ keysNotFoundLen = len(keysNotFound)
+ if keysNotFoundLen > 0:
+ tmpString = '### Did not find these {} keys from the reference file in the current file: '.format(keysNotFoundLen)
+ for key in sorted(keysNotFound):
+ tmpString = tmpString + key + ', '
+ if verbosePrint: print(tmpString)
+ logFile.write(tmpString+'\n')
+
+ keysDeviatingLen = len(keysDeviating)
+ if keysDeviatingLen > 0:
+ tmpString = '### Found {} keys that deviate by more than {}%: '.format(keysDeviatingLen, bound)
+ headerString = headerString + '{:>15};'.format('Perc. diff')
+ if verbosePrint:
+ print(tmpString)
+ print(headerString)
+ logFile.write(tmpString + '\n' + headerString + '\n')
+ for key in sorted(keysDeviating):
+ diff = meansCurrent[key] - meansReference[key]
+ percDiff = (1 - ((meansReference[key] - np.abs(diff)) / meansReference[key]))*100
+ tmpString = '{:45}; {:>15}; {:>15}; {:>15}; {:>15};'.format(key, '{:.5f}'.format(meansReference[key]), '{:.5f}'.format(meansCurrent[key]), '{:.5f}'.format(diff), '{:.2f}'.format(percDiff))
+ if verbosePrint:
+ print(tmpString)
+ logFile.write(tmpString + '\n')
+
+ #Wrap up
+ applicationTime = time.time()-applicationStartTime
+ tmpString='### Ending application at {} processing took {:.2f} seconds'.format(time.strftime("%a %b %d %H:%M:%S %Y",time.localtime()),applicationTime)
+ if verbosePrint:print('\n'+tmpString)
+ logFile.write(tmpString)
+ compareSummaryFile.close()
+ logFile.close()
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.cc b/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.cc
index 3fcafeaf8d0cab3c2fc0fca15402072e2a794e71..b113cc0e7c69a982dadfe55a00b6749fee9ecbc4 100644
--- a/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.cc
+++ b/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.cc
@@ -141,7 +141,6 @@ namespace LOFAR
outputThreads("Pipeline::outputThreads"),
mpiPool(pool),
writePool(subbandIndices.size()),
- factories(ps, nrSubbandsPerSubbandProc),
// Each work queue needs an output element for each subband it processes, because the GPU output can
// be in bulk: if processing is cheap, all subbands will be output right after they have been received.
@@ -193,7 +192,8 @@ namespace LOFAR
void Pipeline::allocateResources()
{
for (size_t i = 0; i < writePool.size(); i++) {
- writePool[i].queue = new Queue< SmartPtr<SubbandProcOutputData> >(str(boost::format("Pipeline::writePool [local subband %u]") % i));
+ writePool[i].queue.reset(
+ new Queue<std::shared_ptr<SubbandProcOutputData>>(str(boost::format("Pipeline::writePool [local subband %u]") % i)));
}
// Create the SubbandProcs, which in turn allocate the GPU buffers and
@@ -201,7 +201,7 @@ namespace LOFAR
for (size_t i = 0; i < subbandProcs.size(); ++i) {
gpu::Context context(devices[i % devices.size()]);
- subbandProcs[i] = new SubbandProc(ps, context, factories, nrSubbandsPerSubbandProc);
+ subbandProcs[i].reset(new SubbandProc(ps, context, nrSubbandsPerSubbandProc));
}
}
@@ -352,7 +352,7 @@ namespace LOFAR
template<typename SampleT>
void Pipeline::transposeInput()
{
- SmartPtr<struct MPIRecvData> input;
+ std::shared_ptr<struct MPIRecvData> input;
BudgetTimer copyTimer(
"transposeInput",
@@ -375,14 +375,14 @@ namespace LOFAR
(struct MPIProtocol::MetaData*)input->metaData.get(), false);
// The set of InputData objects we're using for this block.
- vector< SmartPtr<SubbandProcInputData> > inputDatas(subbandIndices.size());
+ vector<std::shared_ptr<SubbandProcInputData>> inputDatas(subbandIndices.size());
for (size_t subbandIdx = 0; subbandIdx < subbandIndices.size(); ++subbandIdx) {
// Fetch an input object to store this subband.
SubbandProc &queue = *subbandProcs[subbandIdx % subbandProcs.size()];
// Fetch an input object to fill from the selected queue.
- SmartPtr<SubbandProcInputData> subbandData = queue.inputPool.free.remove();
+ std::shared_ptr<SubbandProcInputData> subbandData = queue.inputPool.free.remove();
// Annotate the block
struct BlockID id;
@@ -429,6 +429,7 @@ namespace LOFAR
#endif
mpiPool.free.append(input);
+ input.reset();
ASSERT(!input);
// Report flags per antenna field
@@ -478,7 +479,7 @@ namespace LOFAR
void Pipeline::preprocessSubbands(SubbandProc &subbandProc)
{
- SmartPtr<SubbandProcInputData> input;
+ std::shared_ptr<SubbandProcInputData> input;
BudgetTimer preprocessTimer(
"preprocess",
@@ -506,6 +507,7 @@ namespace LOFAR
// Hand off output to processing
subbandProc.processPool.filled.append(input);
+ input.reset();
ASSERT(!input);
LOG_DEBUG_STR("[" << id << "] Forwarded input to processing");
@@ -515,7 +517,7 @@ namespace LOFAR
void Pipeline::processSubbands(SubbandProc &subbandProc)
{
- SmartPtr<SubbandProcInputData> input;
+ std::shared_ptr<SubbandProcInputData> input;
BudgetTimer processTimer(
"process",
@@ -529,7 +531,7 @@ namespace LOFAR
LOG_DEBUG_STR("[" << id << "] Processing start");
// Also fetch an output object to store results
- SmartPtr<SubbandProcOutputData> output = subbandProc.outputPool.free.remove();
+ std::shared_ptr<SubbandProcOutputData> output = subbandProc.outputPool.free.remove();
// Only _we_ signal end-of-data, so we should _never_ receive it
ASSERT(output != NULL);
@@ -539,6 +541,7 @@ namespace LOFAR
// Perform calculations
processTimer.start();
subbandProc.processSubband(*input, *output);
+ subbandProc.synchronize();
processTimer.stop();
if (id.block < 0) {
@@ -548,10 +551,12 @@ namespace LOFAR
// Hand off output to post processing
subbandProc.outputPool.filled.append(output);
}
+ output.reset();
ASSERT(!output);
// Give back input data for a refill
subbandProc.inputPool.free.append(input);
+ input.reset();
ASSERT(!input);
LOG_DEBUG_STR("[" << id << "] Forwarded output to post processing");
@@ -561,7 +566,7 @@ namespace LOFAR
void Pipeline::postprocessSubbands(SubbandProc &subbandProc)
{
- SmartPtr<SubbandProcOutputData> output;
+ std::shared_ptr<SubbandProcOutputData> output;
BudgetTimer postprocessTimer(
"postprocess",
@@ -581,6 +586,7 @@ namespace LOFAR
struct Output &pool = writePool[id.localSubbandIdx];
pool.queue->append(output);
+ output.reset();
ASSERT(!output);
LOG_DEBUG_STR("[" << id << "] Forwarded output to writer");
@@ -589,10 +595,10 @@ namespace LOFAR
void Pipeline::writeOutput(
unsigned globalSubbandIdx,
- Queue< SmartPtr<SubbandProcOutputData> > &inputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &outputQueue )
+ Queue<std::shared_ptr<SubbandProcOutputData>> &inputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &outputQueue )
{
- Queue< SmartPtr<SubbandProcOutputData> > queue(str(boost::format("Pipeline::writeOutput [subband %u]") % globalSubbandIdx));
+ Queue<std::shared_ptr<SubbandProcOutputData>> queue(str(boost::format("Pipeline::writeOutput [subband %u]") % globalSubbandIdx));
# pragma omp parallel sections num_threads(2)
{
@@ -620,9 +626,9 @@ namespace LOFAR
// All output corresponds to the subband indexed by globalSubbandIdx in the list of sb.
void Pipeline::writeBeamformedOutput(
unsigned globalSubbandIdx,
- Queue< SmartPtr<SubbandProcOutputData> > &inputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &outputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &spillQueue )
+ Queue<std::shared_ptr<SubbandProcOutputData>> &inputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &outputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &spillQueue )
{
NSTimer transposeTimer(str(format("Pipeline::writeBeamFormedOutput(subband %u) transpose/file") % globalSubbandIdx), true, true);
NSTimer forwardTimer(str(format("Pipeline::writeBeamFormedOutput(subband %u) forward/file") % globalSubbandIdx), true, true);
@@ -639,7 +645,7 @@ namespace LOFAR
struct LossStatistics correlatorLoss = {false, 0, 0};
struct LossStatistics beamFormerLoss = {false, 0, 0};
- SmartPtr<SubbandProcOutputData> data;
+ std::shared_ptr<SubbandProcOutputData> data;
// Process pool elements until end-of-output
while ((data = inputQueue.remove()) != NULL)
@@ -676,11 +682,17 @@ namespace LOFAR
// Note that the 'file' encodes 1 Stokes of 1 TAB, so each TAB we've
// produced can be visited 1 or 4 times.
+ // Find the current TAB and beamFormer pipeline for this 'file'
+ auto& tab = ps.settings.beamFormer.SAPs[file.sapNr].TABs[file.tabNr];
+
// Compute shape of block
const ObservationSettings::BeamFormer::StokesSettings &stokes =
file.coherent
- ? ps.settings.beamFormer.coherentSettings
- : ps.settings.beamFormer.incoherentSettings;
+ ? ps.settings.beamFormer.pipelines[tab.pipelineNr].coherentSettings
+ : ps.settings.beamFormer.pipelines[tab.pipelineNr].incoherentSettings;
+
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings &quantizer =
+ stokes.quantizerSettings;
const size_t nrChannels = stokes.nrChannels;
const size_t nrSamples = stokes.nrSamples;
@@ -694,8 +706,8 @@ namespace LOFAR
//
// We create a copy to be able to release outputData, since our
// slices can be blocked by writes to any number of outputProcs.
- SmartPtr<struct TABTranspose::Subband> subband =
- new TABTranspose::Subband(nrSamples, nrChannels);
+ std::shared_ptr<struct TABTranspose::Subband> subband(
+ new TABTranspose::Subband(nrSamples, nrChannels, quantizer.enabled));
// These 3 values are guarded with ASSERTSTR() on the other side at
// outputProc (Block::addSubband()).
@@ -707,23 +719,63 @@ namespace LOFAR
subband->id.block = id.block;
// Create view of subarray
- MultiDimArray<float, 2> srcData(
+ size_t coherentTAB = file.coherentIdxInSAP;
+ size_t incoherentTAB = file.incoherentIdxInSAP;
+ unsigned int stokesNr = file.stokesNr;
+
+ auto& coherentData = data->coherentData(tab.pipelineNr);
+ auto& incoherentData = data->incoherentData(tab.pipelineNr);
+
+ if (!quantizer.enabled) { // not quantized
+ MultiDimArray<float, 2> srcData(
+ boost::extents[nrSamples][nrChannels],
+ file.coherent
+ ? coherentData.get_data_origin(coherentTAB, stokesNr)
+ : incoherentData.get_data_origin(incoherentTAB, stokesNr),
+ false);
+
+ // Copy data to block
+ transposeTimer.start();
+ subband->data.assign(srcData.origin(), srcData.origin() + srcData.num_elements());
+ transposeTimer.stop();
+ } else { // data is quantized
+ MultiDimArray<int8_t, 2> srcQData(
boost::extents[nrSamples][nrChannels],
file.coherent
- ? data->coherentData[file.coherentIdxInSAP][file.stokesNr].origin()
- : data->incoherentData[file.incoherentIdxInSAP][file.stokesNr].origin(),
+ ? coherentData.get_qdata_origin(coherentTAB, stokesNr)
+ : incoherentData.get_qdata_origin(incoherentTAB, stokesNr),
false);
- // Copy data to block
- transposeTimer.start();
- subband->data.assign(srcData.origin(), srcData.origin() + srcData.num_elements());
- transposeTimer.stop();
+ MultiDimArray<float, 1> srcQOffsets(
+ boost::extents[nrChannels],
+ file.coherent
+ ? coherentData.get_qoffsets_origin(coherentTAB, stokesNr)
+ : incoherentData.get_qoffsets_origin(incoherentTAB, stokesNr),
+ false);
+
+ MultiDimArray<float, 1> srcQScales(
+ boost::extents[nrChannels],
+ file.coherent
+ ? coherentData.get_qscales_origin(coherentTAB, stokesNr)
+ : incoherentData.get_qscales_origin(incoherentTAB, stokesNr),
+ false);
+
+ // Copy data to block
+ transposeTimer.start();
+ subband->qdata.assign(srcQData.origin(), srcQData.origin() + srcQData.num_elements());
+ subband->qoffsets.assign(srcQOffsets.origin(), srcQOffsets.origin() + srcQOffsets.num_elements());
+ subband->qscales.assign(srcQScales.origin(), srcQScales.origin() + srcQScales.num_elements());
+ transposeTimer.stop();
+ }
// Forward block to MultiSender, who takes ownership.
forwardTimer.start();
if (multiSender.append(subband)) {
// Added a block
beamFormerLoss.blocksWritten++;
+
+ // We have passed the subband to the multiSender, explicitly release the shared_ptr here
+ subband.reset();
} else {
// Dropped a block
beamFormerLoss.dropping = true;
@@ -762,6 +814,7 @@ namespace LOFAR
spillQueue.append(data);
}
+ data.reset();
ASSERT(!data);
/*
@@ -797,8 +850,8 @@ namespace LOFAR
void Pipeline::writeCorrelatedOutput(
unsigned globalSubbandIdx,
- Queue< SmartPtr<SubbandProcOutputData> > &inputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &outputQueue )
+ Queue<std::shared_ptr<SubbandProcOutputData>> &inputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &outputQueue )
{
BudgetTimer writeTimer(
str(format("[subband %u] write correlated output") % globalSubbandIdx),
@@ -810,7 +863,7 @@ namespace LOFAR
// Register our thread to be killable at exit
OMPThreadSet::ScopedRun sr(outputThreads);
- SmartPtr<Stream> outputStream;
+ std::shared_ptr<Stream> outputStream;
bool outputEnabled = true;
if (ps.settings.correlator.enabled) {
@@ -818,14 +871,14 @@ namespace LOFAR
hostID < ps.settings.nodes.size() ? ps.settings.nodes[hostID].out_nic : "");
try {
- outputStream = createStream(desc, false, 0);
+ outputStream.reset(createStream(desc, false, 0));
} catch (Exception &ex) {
LOG_ERROR_STR("Error writing subband " << globalSubbandIdx << ", dropping all subsequent blocks: " << ex.what());
outputEnabled = false;
}
}
- SmartPtr<SubbandProcOutputData> data;
+ std::shared_ptr<SubbandProcOutputData> data;
// Process pool elements until end-of-output
while ((data = inputQueue.remove()) != NULL) {
@@ -859,6 +912,7 @@ namespace LOFAR
}
outputQueue.append(data);
+ data.reset();
ASSERT(!data);
}
diff --git a/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.h b/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.h
index b09516353d02eed582d95e36bb683953b9fe1ba1..4dfb3d1bddd667a72203bca2b16ffd9957237fd8 100644
--- a/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.h
+++ b/RTCP/Cobalt/GPUProc/src/Pipelines/Pipeline.h
@@ -27,15 +27,14 @@
#include <Common/LofarTypes.h>
#include <MACIO/RTmetadata.h>
#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/Pool.h>
#include <CoInterface/OMPThread.h>
#include <CoInterface/TABTranspose.h>
+#include <CoInterface/Queue.h>
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/PerformanceCounter.h>
#include <GPUProc/SubbandProcs/SubbandProc.h>
-#include <GPUProc/SubbandProcs/KernelFactories.h>
#include <GPUProc/MPIReceiver.h>
@@ -72,10 +71,10 @@ namespace LOFAR
struct Output
{
// output data queue
- SmartPtr< Queue< SmartPtr<SubbandProcOutputData> > > queue;
+ std::unique_ptr<Queue<std::shared_ptr<SubbandProcOutputData>>> queue;
};
- std::vector< SmartPtr<SubbandProc> > subbandProcs;
+ std::vector<std::unique_ptr<SubbandProc>> subbandProcs;
protected:
const Parset &ps;
@@ -100,8 +99,6 @@ namespace LOFAR
std::vector<struct Output> writePool; // [localSubbandIdx]
- KernelFactories factories;
-
// For each block, transpose all subbands from all stations, and divide the
// work over the subbandProcs
void transposeInput();
@@ -118,21 +115,21 @@ namespace LOFAR
void writeBeamformedOutput(
unsigned globalSubbandIdx,
- Queue< SmartPtr<SubbandProcOutputData> > &inputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &outputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &spillQueue );
+ Queue<std::shared_ptr<SubbandProcOutputData>> &inputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &outputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &spillQueue );
void writeCorrelatedOutput(
unsigned globalSubbandIdx,
- Queue< SmartPtr<SubbandProcOutputData> > &inputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &outputQueue );
+ Queue<std::shared_ptr<SubbandProcOutputData>> &inputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &outputQueue );
public:
// Send subbands to Storage
void writeOutput(
unsigned globalSubbandIdx,
- Queue< SmartPtr<SubbandProcOutputData> > &inputQueue,
- Queue< SmartPtr<SubbandProcOutputData> > &outputQueue );
+ Queue<std::shared_ptr<SubbandProcOutputData>> &inputQueue,
+ Queue<std::shared_ptr<SubbandProcOutputData>> &outputQueue );
// Output send engine, takes care of the host connections and the multiplexing.
TABTranspose::MultiSender multiSender;
diff --git a/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.cc b/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.cc
index 8caeddd04b95050310b510fc18f0a744b32267b7..3e0739980cd0b993b8559b161540294fd9f0abcf 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.cc
+++ b/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.cc
@@ -36,7 +36,7 @@ namespace LOFAR {
namespace Cobalt {
RSPRawSender::RSPRawSender() :
- itsStream(NULL),
+ itsStream(nullptr),
itsSentMsgSizes(0),
itsMaxNrBeamletsToSend(0),
itsNrDroppedPackets(0)
diff --git a/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.h b/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.h
index 040e3ee1fcf38f8f11cbe21779d9cdae66f8f769..ff3e108c4b17f0fd442905026b61eb42838b8dc4 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.h
+++ b/RTCP/Cobalt/GPUProc/src/Station/RSPRawSender.h
@@ -29,7 +29,6 @@
#include <string>
#include <Stream/SocketStream.h>
#include <CoInterface/RSP.h>
-#include <CoInterface/SmartPtr.h>
namespace LOFAR
{
@@ -62,7 +61,7 @@ private:
void trySendByteStream(struct RSP *packets, unsigned packetSize, unsigned nrPackets);
void trySendPending();
- SmartPtr<Stream> itsStream;
+ std::shared_ptr<Stream> itsStream;
std::vector<unsigned> itsSentMsgSizes;
unsigned itsMaxNrBeamletsToSend;
unsigned itsNrDroppedPackets;
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
index e2c0fa35b764715f142dcf3541544f0ad8672892..7a96bc727e347f7a8517865a90acd95780fd4df5 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationInput.cc
@@ -102,7 +102,10 @@ namespace LOFAR {
// Each element represents 1 block of buffer.
for (size_t i = 0; i < 10; ++i)
- metaDataPool.free.append(new MPIData<SampleT>(startTime, ps.settings.subbands.size(), nrSamples), false);
+ {
+ std::shared_ptr<MPIData<SampleT>> data(new MPIData<SampleT>(startTime, ps.settings.subbands.size(), nrSamples));
+ metaDataPool.free.append(data, false);
+ }
/*
* Set up delay compensation.
@@ -137,7 +140,7 @@ namespace LOFAR {
delays.getNextDelays(*delaysAfterEnd);
// INPUT
- SmartPtr< MPIData<SampleT> > mpiData = metaDataPool.free.remove();
+ std::shared_ptr<MPIData<SampleT>> mpiData = metaDataPool.free.remove();
// Annotate
mpiData->reset(block);
@@ -150,6 +153,7 @@ namespace LOFAR {
// OUTPUT
metaDataPool.filled.append(mpiData);
+ mpiData.reset();
ASSERT(!mpiData);
// Swap delay sets to accomplish delaysAtBegin = delaysAfterEnd
@@ -186,8 +190,10 @@ namespace LOFAR {
LOG_INFO_STR(logPrefix << "Input streams: " << ps.settings.antennaFields.at(stationIdx).inputStreams);
for (unsigned i = 0; i < nrBoards; ++i) {
- rspDataPool.push_back(new Pool<RSPData>(str(format("StationInput::rspDataPool[%u] [station %s]") % i % stationID.name()),
- ps.settings.realTime));
+ std::unique_ptr<Pool<RSPData>> pool(
+ new Pool<RSPData>(str(format("StationInput::rspDataPool[%u] [station %s]") % i % stationID.name()),
+ ps.settings.realTime));
+ rspDataPool.push_back(std::move(pool));
}
if (ps.settings.rspRaw.enabled &&
@@ -238,9 +244,9 @@ namespace LOFAR {
}
- SmartPtr<Stream> StationInput::inputStream(unsigned board) const
+ std::unique_ptr<Stream> StationInput::inputStream(unsigned board) const
{
- SmartPtr<Stream> stream;
+ std::unique_ptr<Stream> stream;
// Connect to specified input stream
const string &desc = ps.settings.antennaFields.at(stationIdx).inputStreams.at(board);
@@ -261,17 +267,17 @@ namespace LOFAR {
const struct BoardMode mode(ps.settings.nrBitsPerSample, ps.settings.clockMHz);
PacketFactory factory(mode);
- stream = new PacketStream(factory, from, to, board);
+ stream.reset(new PacketStream(factory, from, to, board));
} else {
if (ps.settings.realTime) {
try {
- stream = createStream(desc, true, 0, str(format("from %s-RSP%u") % stationID.name() % board));
+ stream.reset(createStream(desc, true, 0, str(format("from %s-RSP%u") % stationID.name() % board)));
} catch (Exception &ex) {
LOG_ERROR_STR(logPrefix << "Caught exception creating stream (continuing on /dev/null): " << ex.what());
- stream = new FileStream("/dev/null"); /* will read end-of-stream: avoid spamming illegal packets */
+ stream.reset(new FileStream("/dev/null")); /* will read end-of-stream: avoid spamming illegal packets */
}
} else { // non real time: un-tried call, so no rethrow (changes exc backtrace)
- stream = createStream(desc, true);
+ stream.reset(createStream(desc, true));
}
}
@@ -322,17 +328,17 @@ namespace LOFAR {
*/
try {
- SmartPtr<Stream> stream = inputStream(board);
+ std::unique_ptr<Stream> stream(inputStream(board));
PacketReader reader(str(format("%s RSP%s ") % logPrefix % board),
*stream, mode);
- Queue< SmartPtr<RSPData> > &inputQueue = rspDataPool[board]->free;
- Queue< SmartPtr<RSPData> > &outputQueue = rspDataPool[board]->filled;
+ auto &inputQueue = rspDataPool[board]->free;
+ auto &outputQueue = rspDataPool[board]->filled;
for(size_t i = 1 /* avoid printing statistics immediately */; true; i++) {
// Fill rspDataPool elements with RSP packets
- SmartPtr<RSPData> rspData = inputQueue.remove();
-
+ std::shared_ptr<RSPData> rspData = inputQueue.remove();
+
// Abort condition needed to avoid getting stuck in free.remove()
if (!rspData)
break;
@@ -395,13 +401,13 @@ namespace LOFAR {
//NSTimer copyRSPTimer(str(format("%s [board %i] copy RSP -> block") % logPrefix % board), true, true);
OMPThread::ScopedName sn(str(format("%s wr %u") % ps.settings.antennaFields.at(stationIdx).name % board));
- Queue< SmartPtr<RSPData> > &inputQueue = rspDataPool[board]->filled;
- Queue< SmartPtr<RSPData> > &outputQueue = rspDataPool[board]->free;
+ Queue<std::shared_ptr<RSPData>> &inputQueue = rspDataPool[board]->filled;
+ Queue<std::shared_ptr<RSPData>> &outputQueue = rspDataPool[board]->free;
const ssize_t *beamletIndices = &this->beamletIndices[board][0];
const size_t nrBeamletIndices = mode.nrBeamletsPerBoard();
- SmartPtr<RSPData> rspData;
+ std::shared_ptr<RSPData> rspData;
while ((rspData = inputQueue.remove(deadline, NULL)) != NULL) {
// Write valid packets to the current and/or next packet
@@ -458,12 +464,12 @@ namespace LOFAR {
void StationInput::readRSPNonRealTime( MACIO::RTmetadata &mdLogger,
const string &mdKeyPrefix )
{
- vector< SmartPtr<Stream> > streams(nrBoards);
- vector< SmartPtr<PacketReader> > readers(nrBoards);
+ vector<std::unique_ptr<Stream>> streams(nrBoards);
+ vector<std::unique_ptr<PacketReader>> readers(nrBoards);
for (size_t i = 0; i < nrBoards; ++i) {
streams[i] = inputStream(i);
- readers[i] = new PacketReader(logPrefix, *streams[i], mode);
+ readers[i].reset(new PacketReader(logPrefix, *streams[i], mode));
}
/* Since the boards will be read at different speeds, we need to
@@ -513,7 +519,7 @@ namespace LOFAR {
break;
// Emit youngest packet
- SmartPtr<RSPData> data = rspDataPool[0]->free.remove();
+ std::shared_ptr<RSPData> data = rspDataPool[0]->free.remove();
// Abort of writer does not desire any more data
if (!data) {
@@ -533,7 +539,7 @@ namespace LOFAR {
}
// Signal EOD by inserting a packet beyond obs end
- SmartPtr<RSPData> data = rspDataPool[0]->free.remove();
+ std::shared_ptr<RSPData> data = rspDataPool[0]->free.remove();
// Abort if writer does not desire any more data
if (!data) {
@@ -560,7 +566,7 @@ namespace LOFAR {
const size_t nrBeamletIndices = mode.nrBeamletsPerBoard();
for(;;) {
- SmartPtr<RSPData> data = rspDataPool[0]->filled.remove();
+ std::shared_ptr<RSPData> data = rspDataPool[0]->filled.remove();
if (!data) {
LOG_DEBUG_STR(logPrefix << "writeRSPNonRealTime: received EOS");
@@ -580,6 +586,7 @@ namespace LOFAR {
if (!next || next->write(data->packets[0], beamletIndices, nrBeamletIndices)) {
// Data is even later than next? Put this data back for a future block.
rspDataPool[0]->filled.prepend(data);
+ data.reset();
ASSERT(!data);
return;
}
@@ -594,14 +601,15 @@ namespace LOFAR {
}
rspDataPool[0]->free.append(data);
+ data.reset();
ASSERT(!data);
}
}
template <typename SampleT>
- void StationInput::processInput( Queue< SmartPtr< MPIData<SampleT> > > &inputQueue,
- Queue< SmartPtr< MPIData<SampleT> > > &outputQueue,
+ void StationInput::processInput( Queue<std::shared_ptr<MPIData<SampleT>>> &inputQueue,
+ Queue<std::shared_ptr<MPIData<SampleT>>> &outputQueue,
MACIO::RTmetadata &mdLogger, const string &mdKeyPrefix )
{
OMPThreadSet packetReaderThreads("packetReaders");
@@ -610,11 +618,17 @@ namespace LOFAR {
// Each board has its own pool to reduce lock contention
for (size_t board = 0; board < nrBoards; ++board)
for (size_t i = 0; i < 48; ++i)
- rspDataPool[board]->free.append(new RSPData(RT_PACKET_BATCH_SIZE), false);
+ {
+ std::shared_ptr<RSPData> data(new RSPData(RT_PACKET_BATCH_SIZE));
+ rspDataPool[board]->free.append(data, false);
+ }
} else {
// We just process one packet at a time, merging all the streams into rspDataPool[0].
for (size_t i = 0; i < 16; ++i)
- rspDataPool[0]->free.append(new RSPData(NONRT_PACKET_BATCH_SIZE), false);
+ {
+ std::shared_ptr<RSPData> data(new RSPData(NONRT_PACKET_BATCH_SIZE));
+ rspDataPool[0]->free.append(data, false);
+ }
}
// Make sure we only read RSP packets when we're ready to actually process them. Otherwise,
@@ -680,7 +694,7 @@ namespace LOFAR {
* Also, data can arrive slightly out-of-order.
*/
- SmartPtr< MPIData<SampleT> > current, next;
+ std::shared_ptr<MPIData<SampleT>> current, next;
while((current = next ? next : inputQueue.remove()) != NULL) {
next = inputQueue.remove();
@@ -689,12 +703,13 @@ namespace LOFAR {
startSwitch.trigger();
if (ps.settings.realTime) {
- writeRSPRealTime<SampleT>(*current, next);
+ writeRSPRealTime<SampleT>(*current.get(), next.get());
} else {
- writeRSPNonRealTime<SampleT>(*current, next);
+ writeRSPNonRealTime<SampleT>(*current.get(), next.get());
}
outputQueue.append(current);
+ current.reset();
ASSERT(!current);
if (!next) {
@@ -740,7 +755,7 @@ namespace LOFAR {
LOG_INFO_STR(logPrefix << "Processing station data");
- Queue< SmartPtr< MPIData<SampleT> > > mpiQueue(str(format(
+ Queue<std::shared_ptr<MPIData<SampleT>>> mpiQueue(str(format(
"sendInputToPipeline::mpiQueue [station %s]") % stationID.name()));
MPISender sender(logPrefix, stationIdx, subbandDistribution, ps.settings.blockDuration());
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationInput.h b/RTCP/Cobalt/GPUProc/src/Station/StationInput.h
index 6b21f4d737ee1845a4a5db53e7de312885a12845..785d9a467808ce5fc168e4a795c40d334cfe4013 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationInput.h
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationInput.h
@@ -81,8 +81,8 @@ namespace LOFAR {
unsigned hostID = 0 );
template <typename SampleT>
- void processInput( Queue< SmartPtr< MPIData<SampleT> > > &inputQueue,
- Queue< SmartPtr< MPIData<SampleT> > > &outputQueue,
+ void processInput( Queue<std::shared_ptr<MPIData<SampleT>>> &inputQueue,
+ Queue<std::shared_ptr<MPIData<SampleT>>> &outputQueue,
MACIO::RTmetadata &mdLogger, const string &mdKeyPrefix );
private:
@@ -112,7 +112,7 @@ namespace LOFAR {
const BoardMode mode;
const unsigned nrBoards;
- std::vector< SmartPtr< Pool< RSPData > > > rspDataPool; // [nrBoards]
+ std::vector<std::unique_ptr<Pool<RSPData>>>rspDataPool; // [nrBoards]
std::vector<RSPRawSender> rspRawSenders; // [nrBoards] if RSP raw enabled and the antenna field is selected, else []
// Whether we emitted certain errors (to prevent log spam)
@@ -128,7 +128,7 @@ namespace LOFAR {
MultiDimArray<ssize_t, 2> generateBeamletIndices();
- SmartPtr<Stream> inputStream(unsigned board) const;
+ std::unique_ptr<Stream> inputStream(unsigned board) const;
void initRspRaw(unsigned hostID);
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationNodeAllocation.h b/RTCP/Cobalt/GPUProc/src/Station/StationNodeAllocation.h
index 00265f0cf9faa47f329bf573196240d55807edfe..9860fca5aff7aed8a9a0d2931775eb5b29aebe6f 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationNodeAllocation.h
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationNodeAllocation.h
@@ -26,7 +26,6 @@
#include <vector>
#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
#include <Stream/Stream.h>
#include <InputProc/Buffer/StationID.h>
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc
index e055c88443dab1ab2c14a4c7e807730904e211af..027b2bbfeec18d3dd21881394c301273a9a55e4b 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.cc
@@ -246,11 +246,11 @@ namespace LOFAR {
template <typename SampleT>
- void MPISender::sendBlocks( Queue< SmartPtr< MPIData<SampleT> > > &inputQueue,
- Queue< SmartPtr< MPIData<SampleT> > > &outputQueue,
+ void MPISender::sendBlocks( Queue<std::shared_ptr<MPIData<SampleT>>> &inputQueue,
+ Queue<std::shared_ptr<MPIData<SampleT>>> &outputQueue,
bool skipDataTransfers )
{
- SmartPtr< MPIData<SampleT> > mpiData;
+ std::shared_ptr<MPIData<SampleT>> mpiData;
NSTimer mpiSendTimer(str(format("%s MPI send data") % logPrefix), true, true);
BudgetTimer copyTimer(
@@ -289,6 +289,7 @@ namespace LOFAR {
}
outputQueue.append(mpiData);
+ mpiData.reset();
ASSERT(!mpiData);
}
@@ -309,14 +310,14 @@ namespace LOFAR {
LOG_INFO_STR(logPrefix << str(format("Average data loss/flagged: %.4f%%") % avgloss));
}
- template void MPISender::sendBlocks( Queue< SmartPtr< MPIData< SampleType<i16complex> > > > &inputQueue,
- Queue< SmartPtr< MPIData< SampleType<i16complex> > > > &outputQueue,
+ template void MPISender::sendBlocks( Queue<std::shared_ptr<MPIData< SampleType<i16complex> > > > &inputQueue,
+ Queue<std::shared_ptr<MPIData< SampleType<i16complex> > > > &outputQueue,
bool skipDataTransfers );
- template void MPISender::sendBlocks( Queue< SmartPtr< MPIData< SampleType<i8complex> > > > &inputQueue,
- Queue< SmartPtr< MPIData< SampleType<i8complex> > > > &outputQueue,
+ template void MPISender::sendBlocks( Queue<std::shared_ptr<MPIData< SampleType<i8complex> > > > &inputQueue,
+ Queue<std::shared_ptr<MPIData< SampleType<i8complex> > > > &outputQueue,
bool skipDataTransfers );
- template void MPISender::sendBlocks( Queue< SmartPtr< MPIData< SampleType<i4complex> > > > &inputQueue,
- Queue< SmartPtr< MPIData< SampleType<i4complex> > > > &outputQueue,
+ template void MPISender::sendBlocks( Queue<std::shared_ptr<MPIData< SampleType<i4complex> > > > &inputQueue,
+ Queue<std::shared_ptr<MPIData< SampleType<i4complex> > > > &outputQueue,
bool skipDataTransfers );
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h
index 413f91ad6d8e114ab4a0cbfa6883428e8cddb67c..e6d809e5a1f0a0b6a403518a40970dfe2ea000f7 100644
--- a/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h
+++ b/RTCP/Cobalt/GPUProc/src/Station/StationTranspose.h
@@ -143,8 +143,8 @@ namespace LOFAR {
* But always transfer the EOS block. Always do the queue operations.
*/
template <typename SampleT>
- void sendBlocks( Queue< SmartPtr< MPIData<SampleT> > > &inputQueue,
- Queue< SmartPtr< MPIData<SampleT> > > &outputQueue,
+ void sendBlocks( Queue<std::shared_ptr<MPIData<SampleT>>> &inputQueue,
+ Queue<std::shared_ptr<MPIData<SampleT>>> &outputQueue,
bool skipDataTransfers );
private:
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc
index 7e9d4f7455157d2ea0ac59ac758342d326fefc24..4244fde6ed0e2632ab17e2675fa0e683f6ad8e4d 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.cc
@@ -110,7 +110,7 @@ namespace LOFAR
{
ASSERTSTR(!itsThread, "StorageProcess has already been started");
- itsThread = new Thread(this, &StorageProcess::controlThread, str(boost::format("%s ctrl") % itsHostname), itsLogPrefix + "[ControlThread] ");
+ itsThread.reset(new Thread(this, &StorageProcess::controlThread, str(boost::format("%s ctrl") % itsHostname), itsLogPrefix + "[ControlThread] "));
}
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h
index a6e49c8429a9e9d48739861f26eab0b678568c0f..4b41153b9e6f1ed3d456b630f3122212aae2ab10 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcess.h
@@ -27,7 +27,6 @@
#include <Common/Thread/Thread.h>
#include <Common/Thread/Semaphore.h>
#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/FinalMetaData.h>
namespace LOFAR
@@ -88,7 +87,7 @@ namespace LOFAR
FinalMetaData itsFinalMetaData;
Semaphore itsFinalMetaDataAvailable;
- SmartPtr<Thread> itsThread;
+ std::unique_ptr<Thread> itsThread;
};
}
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc
index 148c18dc8cf6bac77f56f03d69811122da6fed1b..ab65389e25a1c22391df591ab3bb5d61bf803c51 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.cc
@@ -68,7 +68,7 @@ namespace LOFAR
// Start all processes
for (unsigned rank = 0; rank < itsStorageProcesses.size(); rank++) {
- itsStorageProcesses[rank] = new StorageProcess(itsParset, itsLogPrefix, rank, hostnames[rank]);
+ itsStorageProcesses[rank].reset(new StorageProcess(itsParset, itsLogPrefix, rank, hostnames[rank]));
itsStorageProcesses[rank]->start();
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h
index 2e59471ed5cbec36b60917a4f8ef265dc33a7fde..3f81f45edded0dbc514a0c419118163eae6d2102 100644
--- a/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h
+++ b/RTCP/Cobalt/GPUProc/src/Storage/StorageProcesses.h
@@ -27,7 +27,6 @@
#include <Common/Thread/Thread.h>
#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
#include "StorageProcess.h"
@@ -84,7 +83,7 @@ namespace LOFAR
const Parset &itsParset;
const std::string itsLogPrefix;
- std::vector<SmartPtr<StorageProcess> > itsStorageProcesses;
+ std::vector<std::unique_ptr<StorageProcess> > itsStorageProcesses;
// start the processes and control threads
void start();
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.cc
index 3e850b5fd73afa24858398a4cb8b0ad58981c6a9..47923639a7fd3d7bd355b6f79d4e2b1a6b205d21 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.cc
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.cc
@@ -24,12 +24,12 @@
#include <Common/LofarLogger.h>
#include <boost/shared_ptr.hpp>
+#include <boost/format.hpp>
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/global_defines.h>
#include <GPUProc/MultiDimArrayHostBuffer.h>
#include <CoInterface/BlockID.h>
-#include <CoInterface/Parset.h>
#include "SubbandProc.h"
#include "BeamFormerCoherentStep.h"
@@ -41,151 +41,259 @@ namespace LOFAR
namespace Cobalt
{
- BeamFormerCoherentStep::Factories::Factories(const Parset &ps, size_t nrSubbandsPerSubbandProc) :
- beamFormer(BeamFormerKernel::Parameters(ps)),
- coherentTranspose(CoherentStokesTransposeKernel::Parameters(ps)),
+ BeamFormerCoherentStep::Factories::Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::BeamFormer& bfParameters,
+ const KernelParameters::Cobalt& cobParameters,
+ size_t nrSubbandsPerSubbandProc) :
+ beamFormer(BeamFormerKernel::Parameters(
+ bfParameters.preStationIndices, // stationIndices
+ bfParameters.obsStationIndices, // delayIndices
+ obsParameters.nrStations, // nrDelays
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ bfParameters.nrSAPs,
+ bfParameters.maxNrCoherentTABsPerSAP,
+ obsParameters.subbandWidth,
+ bfParameters.doFlysEye,
+ cobParameters.kernel.dumpBeamFormerKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFCoh_BeamFormerKernel.dat") % obsParameters.observationID)
+ )),
+ coherentTranspose(CoherentStokesTransposeKernel::Parameters(
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ bfParameters.maxNrCoherentTABsPerSAP,
+ cobParameters.kernel.dumpCoherentStokesTransposeKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFCoh_CoherentStokesTransposeKernel.dat") % obsParameters.observationID)
+ )),
coherentInverseFFT(FFT_Kernel::Parameters(
- ps.settings.beamFormer.nrDelayCompensationChannels,
- ps.settings.beamFormer.maxNrCoherentTABsPerSAP() * NR_POLARIZATIONS * ps.settings.blockSize,
+ preParameters.nrDelayCompensationChannels,
+ bfParameters.maxNrCoherentTABsPerSAP * NR_POLARIZATIONS * obsParameters.blockSize,
false,
"FFT (coherent, inverse)")),
- coherentInverseFFTShift(FFTShiftKernel::Parameters(ps,
- ps.settings.beamFormer.maxNrCoherentTABsPerSAP(),
- ps.settings.beamFormer.nrDelayCompensationChannels,
- "FFT-shift (coherent, inverse)")),
+ coherentInverseFFTShift(FFTShiftKernel::Parameters(
+ bfParameters.maxNrCoherentTABsPerSAP,
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ "FFT-shift (coherent, inverse)",
+ cobParameters.kernel.dumpFFTShiftKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFCoh_FFTShiftKernel.dat") % obsParameters.observationID)
+ )),
coherentFirFilter(
- ps.settings.beamFormer.coherentSettings.nrChannels > 1
- ? new KernelFactory<FIR_FilterKernel>(FIR_FilterKernel::Parameters(ps,
- ps.settings.beamFormer.maxNrCoherentTABsPerSAP(),
- false,
+ bfParameters.coherentSettings.nrChannels > 1
+ ? new KernelFactory<FIR_FilterKernel>(FIR_FilterKernel::Parameters(
+ bfParameters.maxNrCoherentTABsPerSAP,
+ obsParameters.nrBitsPerSample,
+ false, // inputIsStationData
nrSubbandsPerSubbandProc,
- ps.settings.beamFormer.coherentSettings.nrChannels,
- static_cast<float>(ps.settings.beamFormer.coherentSettings.nrChannels),
- "FIR (coherent, final)"))
+ bfParameters.coherentSettings.nrChannels,
+ obsParameters.blockSize / bfParameters.coherentSettings.nrChannels,
+ static_cast<float>(bfParameters.coherentSettings.nrChannels),
+ "FIR (coherent, final)",
+ cobParameters.kernel.dumpFIR_FilterKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFCoh_FIR_FilterKernel.dat") % obsParameters.observationID)
+ ))
: NULL),
coherentFinalFFT(
- ps.settings.beamFormer.coherentSettings.nrChannels > 1
+ bfParameters.coherentSettings.nrChannels > 1
? new KernelFactory<FFT_Kernel>(FFT_Kernel::Parameters(
- ps.settings.beamFormer.coherentSettings.nrChannels,
- ps.settings.beamFormer.maxNrCoherentTABsPerSAP() * NR_POLARIZATIONS * ps.settings.blockSize,
- true,
+ bfParameters.coherentSettings.nrChannels,
+ bfParameters.maxNrCoherentTABsPerSAP * NR_POLARIZATIONS * obsParameters.blockSize,
+ true, // forward
"FFT (coherent, final)"))
: NULL),
- coherentStokes(CoherentStokesKernel::Parameters(ps))
+ coherentStokes(CoherentStokesKernel::Parameters(
+ bfParameters.coherentSettings.nrChannels,
+ obsParameters.blockSize / bfParameters.coherentSettings.nrChannels,
+ bfParameters.maxNrCoherentTABsPerSAP,
+ bfParameters.coherentSettings.nrStokes,
+ bfParameters.coherentSettings.type == STOKES_XXYY,
+ bfParameters.coherentSettings.timeIntegrationFactor,
+ bfParameters.coherentSettings.quantizerSettings.enabled,
+ cobParameters.kernel.dumpCoherentStokesKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFCoh_CoherentStokesKernel.dat") % obsParameters.observationID)
+ )),
+
+ quantizeOutput(bfParameters.coherentSettings.quantizerSettings.enabled
+ ? new KernelFactory<QuantizeOutputKernel>(QuantizeOutputKernel::Parameters(
+ bfParameters.coherentSettings.nrChannels,
+ obsParameters.blockSize /
+ (bfParameters.coherentSettings.nrChannels
+ * bfParameters.coherentSettings.timeIntegrationFactor),
+ bfParameters.maxNrCoherentTABsPerSAP,
+ bfParameters.coherentSettings.nrStokes,
+ bfParameters.coherentSettings.type == STOKES_XXYY,
+ bfParameters.coherentSettings.quantizerSettings.nrBits,
+ bfParameters.coherentSettings.quantizerSettings.scaleMax,
+ bfParameters.coherentSettings.quantizerSettings.scaleMin,
+ bfParameters.coherentSettings.quantizerSettings.sIpositive,
+ cobParameters.kernel.dumpQuantizeOutputKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFCoh_QuantizeOutputKernel.dat") % obsParameters.observationID)
+ ))
+ : NULL)
{
}
-
+
BeamFormerCoherentStep::BeamFormerCoherentStep(
- const Parset &parset,
- gpu::Stream &i_queue,
+ const KernelParameters::Observation& obsParameters,
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devB)
+ std::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devD,
+ std::shared_ptr<gpu::DeviceMemory> i_devE)
:
- ProcessStep(parset, i_queue),
+ ProcessStep(i_htod_stream, i_dtoh_stream, i_execute_stream, context),
+ obsParameters(obsParameters),
coherentStokesPPF(factories.coherentFirFilter != NULL),
- devC(context, factories.beamFormer.bufferSize(BeamFormerKernel::OUTPUT_DATA)),
- devD(context, factories.beamFormer.bufferSize(BeamFormerKernel::OUTPUT_DATA)),
+ quantizeOutput(factories.quantizeOutput != NULL),
+ devO(context, factories.coherentTranspose.bufferSize(CoherentStokesTransposeKernel::OUTPUT_DATA)),
outputCounter(context, "output (coherent)")
{
devB = i_devB;
-
- beamFormerKernel = std::unique_ptr<BeamFormerKernel>(
- factories.beamFormer.create(queue, *devB, devD));
-
- // transpose after beamforming: A -> C
- coherentTransposeKernel = std::unique_ptr<CoherentStokesTransposeKernel>(
- factories.coherentTranspose.create(
- queue, devD, devC));
-
- // inverse FFT: C -> C (in-place)
- inverseFFT = std::unique_ptr<FFT_Kernel>(
- factories.coherentInverseFFT.create(
- queue, devC, devC));
-
- // fftshift: C -> C (in-place)
- inverseFFTShiftKernel = std::unique_ptr<FFTShiftKernel>(
- factories.coherentInverseFFTShift.create(
- queue, devC, devC));
-
- if (coherentStokesPPF) {
- // FIR filter: C -> D
- firFilterKernel = std::unique_ptr<FIR_FilterKernel>(
- factories.coherentFirFilter->create(queue, devC, devD));
-
- // final FFT: D -> C
- coherentFinalFFT = std::unique_ptr<FFT_Kernel>(
- factories.coherentFinalFFT->create(queue, devD, devC));
+ devD = i_devD;
+ devE = i_devE;
+
+ marker_gpu.reset(new gpu::Marker("coherentStep", gpu::Marker::green));
+
+ beamFormerKernel = std::unique_ptr<BeamFormerKernel>(
+ factories.beamFormer.create(*executeStream, *devB, *devD));
+
+ // transpose after beamforming: D -> O
+ coherentTransposeKernel = std::unique_ptr<CoherentStokesTransposeKernel>(
+ factories.coherentTranspose.create(
+ *executeStream, *devD, devO));
+
+ // inverse FFT: O -> D
+ inverseFFT = std::unique_ptr<FFT_Kernel>(
+ factories.coherentInverseFFT.create(
+ *executeStream, devO, *devD));
+
+ // fftshift: D -> D (in place)
+ inverseFFTShiftKernel = std::unique_ptr<FFTShiftKernel>(
+ factories.coherentInverseFFTShift.create(
+ *executeStream, *devD, *devD));
+
+ if (coherentStokesPPF) {
+ // FIR filter: D -> O
+ firFilterKernel = std::unique_ptr<FIR_FilterKernel>(
+ factories.coherentFirFilter->create(*executeStream, *devD, devO));
+
+ // final FFT: O -> D
+ coherentFinalFFT = std::unique_ptr<FFT_Kernel>(
+ factories.coherentFinalFFT->create(*executeStream, devO, *devD));
+ }
+
+ sizeof_data = factories.coherentStokes.bufferSize(CoherentStokesKernel::OUTPUT_DATA);
+
+ if (quantizeOutput) {
+ // coherentStokes: D -> E
+ coherentStokesKernel = std::unique_ptr<CoherentStokesKernel>(
+ factories.coherentStokes.create(*executeStream,
+ *devD, *devE));
+
+ // quantizeOutput: E -> O
+ quantizeOutputKernel = std::unique_ptr<QuantizeOutputKernel>(
+ factories.quantizeOutput->create(*executeStream,
+ *devE, devO));
+
+ sizeof_qdata = factories.quantizeOutput->bufferSize(QuantizeOutputKernel::OUTPUT_VALUES);
+ sizeof_qmeta = factories.quantizeOutput->bufferSize(QuantizeOutputKernel::OUTPUT_METADATA);
+ } else {
+ // coherentStokes: D -> O
+ coherentStokesKernel = std::unique_ptr<CoherentStokesKernel>(
+ factories.coherentStokes.create(*executeStream,
+ *devD, devO));
+ }
}
- // coherentStokes: C -> D
- coherentStokesKernel = std::unique_ptr<CoherentStokesKernel>(
- factories.coherentStokes.create(queue,
- devC, devD));
- }
-
- gpu::DeviceMemory BeamFormerCoherentStep::outputBuffer() {
- return devD;
- }
-
-
-size_t BeamFormerCoherentStep::nrCoherent(const BlockID &blockID) const
-{
- unsigned SAP = ps.settings.subbands[blockID.globalSubbandIdx].SAP;
-
- return ps.settings.beamFormer.SAPs[SAP].nrCoherent;
-}
-
+ gpu::DeviceMemory BeamFormerCoherentStep::outputBuffer() {
+ return devO;
+ }
-void BeamFormerCoherentStep::writeInput(const SubbandProcInputData &input)
-{
- if (nrCoherent(input.blockID) == 0)
- return;
- // Upload the new beamformerDelays (pointings) to the GPU
- queue.writeBuffer(beamFormerKernel->beamFormerDelays, input.tabDelays, false);
-}
+ void BeamFormerCoherentStep::writeInput(const MultiDimArrayHostBuffer<double, 3>& tabDelays)
+ {
+ // Upload the new beamformerDelays (pointings) to the GPU
+ htodStream->waitEvent(executeFinished);
+ htodStream->writeBuffer(beamFormerKernel->beamFormerDelays, tabDelays, false);
+ htodStream->recordEvent(inputFinished);
+ }
-void BeamFormerCoherentStep::process(const SubbandProcInputData &input)
-{
- if (nrCoherent(input.blockID) == 0)
- return;
+ void BeamFormerCoherentStep::process(const SubbandProcInputData &input)
+ {
+ executeStream->waitEvent(inputFinished);
+ executeStream->waitEvent(outputFinished);
- // The centralFrequency and SAP immediate kernel args must outlive kernel runs.
- beamFormerKernel->enqueue(input.blockID,
- ps.settings.subbands[input.blockID.globalSubbandIdx].centralFrequency,
- ps.settings.subbands[input.blockID.globalSubbandIdx].SAP);
+ // The centralFrequency and SAP immediate kernel args must outlive kernel runs.
+ beamFormerKernel->enqueue(input.blockID,
+ obsParameters.subbands[input.blockID.globalSubbandIdx].centralFrequency,
+ obsParameters.subbands[input.blockID.globalSubbandIdx].SAP);
- coherentTransposeKernel->enqueue(input.blockID);
+ coherentTransposeKernel->enqueue(input.blockID);
- inverseFFT->enqueue(input.blockID);
+ inverseFFT->enqueue(input.blockID);
- inverseFFTShiftKernel->enqueue(input.blockID);
+ inverseFFTShiftKernel->enqueue(input.blockID);
- if (coherentStokesPPF) {
- // The subbandIdx immediate kernel arg must outlive kernel runs.
- firFilterKernel->enqueue(input.blockID,
- input.blockID.subbandProcSubbandIdx);
- coherentFinalFFT->enqueue(input.blockID);
- }
+ if (coherentStokesPPF) {
+ // The subbandIdx immediate kernel arg must outlive kernel runs.
+ firFilterKernel->enqueue(input.blockID,
+ input.blockID.subbandProcSubbandIdx);
+ coherentFinalFFT->enqueue(input.blockID);
+ }
- coherentStokesKernel->enqueue(input.blockID);
-}
+ coherentStokesKernel->enqueue(input.blockID);
+ if (quantizeOutput) {
+ quantizeOutputKernel->enqueue(input.blockID);
+ }
-void BeamFormerCoherentStep::readOutput(SubbandProcOutputData &output)
-{
- if (nrCoherent(output.blockID) == 0)
- return;
+ executeStream->recordEvent(executeFinished);
- output.coherentData.resizeOneDimensionInplace(0, nrCoherent(output.blockID));
- queue.readBuffer(output.coherentData, outputBuffer(), outputCounter, false);
-}
+ marker_mutex.unlock();
+ }
+ void BeamFormerCoherentStep::readOutput(
+ SubbandProcOutputData::TABOutputData &output)
+ {
+ dtohStream->waitEvent(executeFinished);
+
+ if (!quantizeOutput) {
+ dtohStream->readBuffer(
+ output.get_data(), outputBuffer(),
+ 0, 0,
+ sizeof_data,
+ outputCounter, false);
+ } else {
+ struct QuantizeOutputKernel::GPUBufferOffsets v = quantizeOutputKernel->bufferOffsets();
+ dtohStream->readBuffer(
+ output.get_qdata(), outputBuffer(),
+ 0, v.data,
+ sizeof_qdata,
+ outputCounter, false);
+ dtohStream->readBuffer(
+ output.get_qoffsets(), outputBuffer(),
+ 0, v.offset,
+ sizeof_qmeta,
+ outputCounter, false);
+ dtohStream->readBuffer(
+ output.get_qscales(), outputBuffer(),
+ 0, v.scale,
+ sizeof_qmeta,
+ outputCounter, false);
+ }
+
+ dtohStream->synchronize();
+ dtohStream->recordEvent(outputFinished);
+ }
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.h
index 5c022679896ebd30799980c6e39448683375b008..c871db064dfaeaa455da6e9bf60aa5e9e06f0f65 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerCoherentStep.h
@@ -27,7 +27,6 @@
#include <Common/LofarLogger.h>
#include <CoInterface/Parset.h>
-#include <boost/shared_ptr.hpp>
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/MultiDimArrayHostBuffer.h>
@@ -44,52 +43,70 @@
#include <GPUProc/Kernels/FFT_Kernel.h>
#include <GPUProc/Kernels/FIR_FilterKernel.h>
#include <GPUProc/Kernels/CoherentStokesKernel.h>
+#include <GPUProc/Kernels/QuantizeOutputKernel.h>
namespace LOFAR
{
namespace Cobalt
{
- //# Forward declarations
- struct KernelFactories;
-
class BeamFormerCoherentStep: public ProcessStep
{
+ friend class SubbandProc;
+
public:
struct Factories
{
- Factories(const Parset &ps, size_t nrSubbandsPerSubbandProc = 1);
+ Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::BeamFormer& bfParameters,
+ const KernelParameters::Cobalt& cobParameters,
+ size_t nrSubbandsPerSubbandProc);
KernelFactory<BeamFormerKernel> beamFormer;
KernelFactory<CoherentStokesTransposeKernel> coherentTranspose;
KernelFactory<FFT_Kernel> coherentInverseFFT;
KernelFactory<FFTShiftKernel> coherentInverseFFTShift;
- SmartPtr< KernelFactory<FIR_FilterKernel> > coherentFirFilter;
- SmartPtr< KernelFactory<FFT_Kernel> > coherentFinalFFT;
+ std::unique_ptr<KernelFactory<FIR_FilterKernel>> coherentFirFilter;
+ std::unique_ptr<KernelFactory<FFT_Kernel>> coherentFinalFFT;
KernelFactory<CoherentStokesKernel> coherentStokes;
+ std::unique_ptr<KernelFactory<QuantizeOutputKernel>> quantizeOutput;
};
- BeamFormerCoherentStep(const Parset &parset,
- gpu::Stream &i_queue,
+ BeamFormerCoherentStep(
+ const KernelParameters::Observation& obsParameters,
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devB);
+ std::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devD,
+ std::shared_ptr<gpu::DeviceMemory> i_devE);
gpu::DeviceMemory outputBuffer();
- void writeInput(const SubbandProcInputData &input);
+ void writeInput(const MultiDimArrayHostBuffer<double, 3>& tabDelays);
void process(const SubbandProcInputData &input);
- void readOutput(SubbandProcOutputData &output);
+ void readOutput(
+ SubbandProcOutputData::TABOutputData &output);
private:
+ // Parameters
+ KernelParameters::Observation obsParameters;
const bool coherentStokesPPF;
+ // flag to remember if quantization is enabled or not
+ const bool quantizeOutput;
+
// Data members
- boost::shared_ptr<gpu::DeviceMemory> devB;
- gpu::DeviceMemory devC;
- gpu::DeviceMemory devD;
+ std::shared_ptr<gpu::DeviceMemory> devB;
+ std::shared_ptr<gpu::DeviceMemory> devD;
+ std::shared_ptr<gpu::DeviceMemory> devE;
+ gpu::DeviceMemory devO;
// Kernel members
std::unique_ptr<BeamFormerKernel> beamFormerKernel;
@@ -112,7 +129,12 @@ namespace LOFAR
PerformanceCounter outputCounter;
- size_t nrCoherent(const BlockID &blockID) const;
+ size_t sizeof_data;
+
+ // Quantizer
+ std::unique_ptr<QuantizeOutputKernel> quantizeOutputKernel;
+ size_t sizeof_qdata;
+ size_t sizeof_qmeta;
};
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.cc
index 27cc4f80f938ca50c1950431e495614d499bf20a..bbc2a73fd3b29436f0055feb2af3a6de5c49c3f5 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.cc
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.cc
@@ -29,7 +29,6 @@
#include <GPUProc/global_defines.h>
#include <GPUProc/MultiDimArrayHostBuffer.h>
#include <CoInterface/BlockID.h>
-#include <CoInterface/Parset.h>
#include "SubbandProc.h"
@@ -41,105 +40,160 @@ namespace LOFAR
{
namespace Cobalt
{
- BeamFormerIncoherentStep::Factories::Factories(const Parset &ps, size_t nrSubbandsPerSubbandProc) :
- incoherentStokesTranspose(IncoherentStokesTransposeKernel::Parameters(ps)),
+ BeamFormerIncoherentStep::Factories::Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::BeamFormer& bfParameters,
+ const KernelParameters::Cobalt& cobParameters,
+ size_t nrSubbandsPerSubbandProc) :
+ incoherentStokesTranspose(IncoherentStokesTransposeKernel::Parameters(
+ preParameters.obsStationIndices.size(), // nrInputStations
+ bfParameters.preStationIndices, // stationIndices
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ true, // beamFormerStationSubset
+ cobParameters.kernel.dumpIncoherentStokesTransposeKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFIcoh_IncoherentStokesTransposeKernel.dat") % obsParameters.observationID)
+ )),
incoherentInverseFFT(FFT_Kernel::Parameters(
- ps.settings.beamFormer.nrDelayCompensationChannels,
- ps.settings.beamFormer.antennaFieldNames.size() * NR_POLARIZATIONS * ps.settings.blockSize, false,
+ preParameters.nrDelayCompensationChannels,
+ bfParameters.preStationIndices.size() * NR_POLARIZATIONS * obsParameters.blockSize, false,
"FFT (incoherent, inverse)")),
- incoherentInverseFFTShift(FFTShiftKernel::Parameters(ps,
- ps.settings.beamFormer.antennaFieldNames.size(),
- ps.settings.beamFormer.nrDelayCompensationChannels,
- "FFT-shift (incoherent, inverse)")),
+ incoherentInverseFFTShift(FFTShiftKernel::Parameters(
+ bfParameters.preStationIndices.size(),
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ "FFT-shift (incoherent, inverse)",
+ cobParameters.kernel.dumpFFTShiftKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFIcoh_FFTShiftKernel.dat") % obsParameters.observationID)
+ )),
incoherentFirFilter(
- ps.settings.beamFormer.incoherentSettings.nrChannels > 1
- ? new KernelFactory<FIR_FilterKernel>(FIR_FilterKernel::Parameters(ps,
- ps.settings.beamFormer.antennaFieldNames.size(),
+ bfParameters.incoherentSettings.nrChannels > 1
+ ? new KernelFactory<FIR_FilterKernel>(FIR_FilterKernel::Parameters(
+ bfParameters.preStationIndices.size(),
+ obsParameters.nrBitsPerSample,
false,
nrSubbandsPerSubbandProc,
- ps.settings.beamFormer.incoherentSettings.nrChannels,
- static_cast<float>(ps.settings.beamFormer.incoherentSettings.nrChannels),
+ bfParameters.incoherentSettings.nrChannels,
+ obsParameters.blockSize / bfParameters.incoherentSettings.nrChannels,
+ static_cast<float>(bfParameters.incoherentSettings.nrChannels),
"FIR (incoherent, final)"))
- : NULL ),
+ : NULL),
incoherentFinalFFT(
- ps.settings.beamFormer.incoherentSettings.nrChannels > 1
+ bfParameters.incoherentSettings.nrChannels > 1
? new KernelFactory<FFT_Kernel>(FFT_Kernel::Parameters(
- ps.settings.beamFormer.incoherentSettings.nrChannels,
- ps.settings.beamFormer.antennaFieldNames.size() * NR_POLARIZATIONS * ps.settings.blockSize, true,
+ bfParameters.incoherentSettings.nrChannels,
+ bfParameters.preStationIndices.size() * NR_POLARIZATIONS * obsParameters.blockSize, true,
"FFT (incoherent, final)"))
: NULL),
- incoherentStokes(IncoherentStokesKernel::Parameters(ps))
+ incoherentStokes(IncoherentStokesKernel::Parameters(
+ bfParameters.preStationIndices.size(),
+ bfParameters.incoherentSettings.nrChannels,
+ obsParameters.blockSize / bfParameters.incoherentSettings.nrChannels,
+ bfParameters.incoherentSettings.nrStokes,
+ bfParameters.incoherentSettings.timeIntegrationFactor,
+ bfParameters.incoherentSettings.quantizerSettings.enabled,
+ cobParameters.kernel.dumpIncoherentStokesKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFIcoh_IncoherentStokesKernel.dat") % obsParameters.observationID)
+ )),
+
+ quantizeOutput(bfParameters.incoherentSettings.quantizerSettings.enabled
+ ? new KernelFactory<QuantizeOutputKernel>(QuantizeOutputKernel::Parameters(
+ bfParameters.incoherentSettings.nrChannels,
+ obsParameters.blockSize / (bfParameters.incoherentSettings.nrChannels
+ * bfParameters.incoherentSettings.timeIntegrationFactor),
+ 1, // nrTABs
+ bfParameters.incoherentSettings.nrStokes,
+ 1, // nrStokes
+ bfParameters.incoherentSettings.quantizerSettings.nrBits,
+ bfParameters.incoherentSettings.quantizerSettings.scaleMax,
+ bfParameters.incoherentSettings.quantizerSettings.scaleMin,
+ bfParameters.coherentSettings.quantizerSettings.sIpositive,
+ cobParameters.kernel.dumpQuantizeOutputKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFIcoh_QuantizeOutputKernel.dat") % obsParameters.observationID)
+ ))
+ : NULL)
{
}
BeamFormerIncoherentStep::BeamFormerIncoherentStep(
- const Parset &parset,
- gpu::Stream &i_queue,
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devA,
- boost::shared_ptr<gpu::DeviceMemory> i_devB )
+ std::shared_ptr<gpu::DeviceMemory> i_devA,
+ std::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devE)
:
- ProcessStep(parset, i_queue),
+ ProcessStep(i_htod_stream, i_dtoh_stream, i_execute_stream, context),
incoherentStokesPPF(factories.incoherentFirFilter != NULL),
+ quantizeOutput(factories.quantizeOutput != NULL),
+ devO(context, incoherentStokesPPF
+ ? factories.incoherentFirFilter->bufferSize(FIR_FilterKernel::OUTPUT_DATA)
+ : factories.incoherentStokes.bufferSize(IncoherentStokesKernel::OUTPUT_DATA)),
outputCounter(context, "output (incoherent)")
{
devA = i_devA;
devB = i_devB;
+ devE = i_devE;
+
+ marker_gpu.reset(new gpu::Marker("incoherentStep", gpu::Marker::blue));
// Transpose: B -> A
incoherentTranspose = std::unique_ptr<IncoherentStokesTransposeKernel>(
- factories.incoherentStokesTranspose.create(queue,
- *devB, *devA));
+ factories.incoherentStokesTranspose.create(*executeStream, *devB, *devA));
- // inverse FFT: A -> A
+ // inverse FFT: A -> A (in place)
incoherentInverseFFT = std::unique_ptr<FFT_Kernel>(
- factories.incoherentInverseFFT.create(queue, *devA, *devA));
+ factories.incoherentInverseFFT.create(*executeStream, *devA, *devA));
- // inverse FFTShift: A -> A
+ // inverse FFTShift: A -> A (in place)
incoherentInverseFFTShiftKernel = std::unique_ptr<FFTShiftKernel>(
- factories.incoherentInverseFFTShift.create(queue, *devA, *devA));
+ factories.incoherentInverseFFTShift.create(*executeStream, *devA, *devA));
if (incoherentStokesPPF) {
- // final FIR: A -> B
+ // final FIR: A -> O
incoherentFirFilterKernel = std::unique_ptr<FIR_FilterKernel>(
- factories.incoherentFirFilter->create(
- queue, *devA, *devB));
+ factories.incoherentFirFilter->create(*executeStream, *devA, devO));
- // final FFT: B -> B
+ // final FFT: O -> A
incoherentFinalFFT = std::unique_ptr<FFT_Kernel>(
- factories.incoherentFinalFFT->create(queue, *devB, *devA));
+ factories.incoherentFinalFFT->create(*executeStream, devO, *devA));
}
- // Incoherent Stokes kernel: A-> B
- //
- // 1ch: input comes from incoherentInverseFFT in A, output in B
- // Nch: input comes from incoherentFinalFFT in B, output in A
- incoherentStokesKernel = std::unique_ptr<IncoherentStokesKernel>(
- factories.incoherentStokes.create(
- queue, *devA, *devB));
- }
+ sizeof_data = factories.incoherentStokes.bufferSize(IncoherentStokesKernel::OUTPUT_DATA);
- gpu::DeviceMemory BeamFormerIncoherentStep::outputBuffer() {
- return *devB;
- }
+ if (quantizeOutput) {
+ // incoherent Stokes: A -> E
+ incoherentStokesKernel = std::unique_ptr<IncoherentStokesKernel>(
+ factories.incoherentStokes.create(*executeStream, *devA, *devE));
+ // quantizeOutput: A -> O
+ quantizeOutputKernel = std::unique_ptr<QuantizeOutputKernel>(
+ factories.quantizeOutput->create(*executeStream, *devE, devO));
- size_t BeamFormerIncoherentStep::nrIncoherent(const BlockID &blockID) const
- {
- unsigned SAP = ps.settings.subbands[blockID.globalSubbandIdx].SAP;
+ sizeof_qdata = factories.quantizeOutput->bufferSize(QuantizeOutputKernel::OUTPUT_VALUES);
+ sizeof_qmeta = factories.quantizeOutput->bufferSize(QuantizeOutputKernel::OUTPUT_METADATA);
+ } else {
+ // incoherent Stokes: A -> O
+ incoherentStokesKernel = std::unique_ptr<IncoherentStokesKernel>(
+ factories.incoherentStokes.create(*executeStream, *devA, devO));
+ }
+ }
- return ps.settings.beamFormer.SAPs[SAP].nrIncoherent;
+ gpu::DeviceMemory BeamFormerIncoherentStep::outputBuffer() {
+ return devO;
}
void BeamFormerIncoherentStep::process(const SubbandProcInputData &input)
{
- if (nrIncoherent(input.blockID) == 0)
- return;
+ executeStream->waitEvent(inputFinished);
+ executeStream->waitEvent(outputFinished);
// ********************************************************************
// incoherent stokes kernels
@@ -159,16 +213,49 @@ namespace LOFAR
}
incoherentStokesKernel->enqueue(input.blockID);
+
+ if (quantizeOutput) {
+ quantizeOutputKernel->enqueue(input.blockID);
+ }
+
+ executeStream->recordEvent(executeFinished);
+
+ marker_mutex.unlock();
}
- void BeamFormerIncoherentStep::readOutput(SubbandProcOutputData &output)
+ void BeamFormerIncoherentStep::readOutput(
+ SubbandProcOutputData::TABOutputData &output)
{
- if (nrIncoherent(output.blockID) == 0)
- return;
+ dtohStream->waitEvent(executeFinished);
+
+ if (!quantizeOutput) {
+ dtohStream->readBuffer(
+ output.get_data(), outputBuffer(),
+ 0, 0,
+ sizeof_data,
+ outputCounter, false);
+ } else {
+ struct QuantizeOutputKernel::GPUBufferOffsets v = quantizeOutputKernel->bufferOffsets();
+ dtohStream->readBuffer(
+ output.get_qdata(), outputBuffer(),
+ 0, v.data,
+ sizeof_qdata,
+ outputCounter, false);
+ dtohStream->readBuffer(
+ output.get_qoffsets(), outputBuffer(),
+ 0, v.offset,
+ sizeof_qmeta,
+ outputCounter, false);
+ dtohStream->readBuffer(
+ output.get_qscales(), outputBuffer(),
+ 0, v.scale,
+ sizeof_qmeta,
+ outputCounter, false);
+ }
- output.incoherentData.resizeOneDimensionInplace(0, nrIncoherent(output.blockID));
- queue.readBuffer(output.incoherentData, outputBuffer(), outputCounter, false);
+ dtohStream->synchronize();
+ dtohStream->recordEvent(outputFinished);
}
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.h
index 04cfcf5bf6f86ac9e2fd790ed58ff04fc3c94e1f..5130ea9f21c28c64f22bdca74f91c75115726b31 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerIncoherentStep.h
@@ -25,9 +25,7 @@
#include <complex>
#include <Common/LofarLogger.h>
-#include <CoInterface/Parset.h>
-#include <boost/shared_ptr.hpp>
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/MultiDimArrayHostBuffer.h>
@@ -42,6 +40,7 @@
#include <GPUProc/Kernels/FIR_FilterKernel.h>
#include <GPUProc/Kernels/IncoherentStokesKernel.h>
#include <GPUProc/Kernels/IncoherentStokesTransposeKernel.h>
+#include <GPUProc/Kernels/QuantizeOutputKernel.h>
namespace LOFAR
@@ -50,43 +49,57 @@ namespace LOFAR
{
class BeamFormerIncoherentStep : public ProcessStep
{
+ friend class SubbandProc;
+
public:
struct Factories
{
- Factories(const Parset &ps, size_t nrSubbandsPerSubbandProc = 1);
+ Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::BeamFormer& bfParameters,
+ const KernelParameters::Cobalt& cobParameters,
+ size_t nrSubbandsPerSubbandProc = 1);
KernelFactory<IncoherentStokesTransposeKernel> incoherentStokesTranspose;
KernelFactory<FFT_Kernel> incoherentInverseFFT;
KernelFactory<FFTShiftKernel> incoherentInverseFFTShift;
- SmartPtr< KernelFactory<FIR_FilterKernel> > incoherentFirFilter;
- SmartPtr< KernelFactory<FFT_Kernel> > incoherentFinalFFT;
+ std::unique_ptr<KernelFactory<FIR_FilterKernel>> incoherentFirFilter;
+ std::unique_ptr<KernelFactory<FFT_Kernel>> incoherentFinalFFT;
KernelFactory<IncoherentStokesKernel> incoherentStokes;
+ std::unique_ptr<KernelFactory<QuantizeOutputKernel>> quantizeOutput;
};
- BeamFormerIncoherentStep(const Parset &parset,
- gpu::Stream &i_queue,
+ BeamFormerIncoherentStep(
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devA,
- boost::shared_ptr<gpu::DeviceMemory> i_devB
- );
+ std::shared_ptr<gpu::DeviceMemory> i_devA,
+ std::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devE);
gpu::DeviceMemory outputBuffer();
void process(const SubbandProcInputData &input);
- void readOutput(SubbandProcOutputData &output);
+ void readOutput(
+ SubbandProcOutputData::TABOutputData &output);
private:
- // Data members
- boost::shared_ptr<gpu::DeviceMemory> devA;
- boost::shared_ptr<gpu::DeviceMemory> devB;
-
- // *****************************************************************
- // Objects needed to produce incoherent stokes output
const bool incoherentStokesPPF;
+ // flag to remember if quantization is enabled or not
+ const bool quantizeOutput;
+
+ // Data members
+ std::shared_ptr<gpu::DeviceMemory> devA;
+ std::shared_ptr<gpu::DeviceMemory> devB;
+ std::shared_ptr<gpu::DeviceMemory> devE;
+ gpu::DeviceMemory devO;
+
// Transpose
std::unique_ptr<IncoherentStokesTransposeKernel> incoherentTranspose;
@@ -105,7 +118,12 @@ namespace LOFAR
PerformanceCounter outputCounter;
- size_t nrIncoherent(const BlockID &blockID) const;
+ size_t sizeof_data;
+
+ // Quantizer
+ std::unique_ptr<QuantizeOutputKernel> quantizeOutputKernel;
+ size_t sizeof_qdata;
+ size_t sizeof_qmeta;
};
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.cc
index 16e0bbae341d619a2f7b757885cc77e4c89ad574..51d7922d12c18ffaf550aad8dd8afe31694bf8f8 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.cc
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.cc
@@ -26,7 +26,6 @@
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/Flagger.h>
-#include <CoInterface/Parset.h>
#include <ApplCommon/PosixTime.h>
#include <Common/LofarLogger.h>
@@ -36,84 +35,142 @@ namespace LOFAR
{
namespace Cobalt
{
- BeamFormerPreprocessingStep::Factories::Factories(const Parset &ps) :
+ BeamFormerPreprocessingStep::Factories::Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::Cobalt& cobParameters) :
+
+ nrSTABs(preParameters.obsStationIndices.size()),
+
intToFloat(IntToFloatKernel::Parameters(
- ps,
- ps.settings.beamFormer.nrDelayCompensationChannels > 1,
- true)),
+ obsParameters.nrStations,
+ preParameters.obsStationIndices,
+ obsParameters.nrBitsPerSample,
+ obsParameters.blockSize,
+ preParameters.nrDelayCompensationChannels > 1,
+ true, // beamFormerStationSubset
+ cobParameters.kernel.dumpIntToFloatKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFPre_IntToFloatKernel.dat") % obsParameters.observationID)
+ )),
firstFFT(FFT_Kernel::Parameters(
- ps.settings.beamFormer.nrDelayCompensationChannels,
- ps.settings.beamFormer.antennaFieldNames.size() * NR_POLARIZATIONS * ps.settings.blockSize,
+ preParameters.nrDelayCompensationChannels,
+ nrSTABs * NR_POLARIZATIONS * obsParameters.blockSize,
true,
"FFT (beamformer, 1st)")),
- fftShift(FFTShiftKernel::Parameters(ps,
- ps.settings.beamFormer.antennaFieldNames.size(),
- ps.settings.beamFormer.nrDelayCompensationChannels,
- "FFT-shift (beamformer)")),
-
- zeroing(ZeroingKernel::Parameters(ps,
- ps.settings.beamFormer.antennaFieldNames.size(),
- ps.settings.beamFormer.nrDelayCompensationChannels,
- "Zeroing (beamformer)")),
- delayCompensation(DelayAndBandPassKernel::Parameters(ps, false)),
-
- bandPassCorrection(BandPassCorrectionKernel::Parameters(ps))
+ fftShift(FFTShiftKernel::Parameters(
+ nrSTABs,
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ "FFT-shift (beamformer)",
+ cobParameters.kernel.dumpFFTShiftKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFPre_FFTShiftKernel_pre.dat") % obsParameters.observationID)
+ )),
+
+ zeroing(ZeroingKernel::Parameters(
+ nrSTABs,
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ "Zeroing (beamformer)",
+ cobParameters.kernel.dumpZeroingKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFPre_ZeroingKernel.dat") % obsParameters.observationID)
+ )),
+
+ delayCompensation(DelayAndBandPassKernel::Parameters(
+ nrSTABs, //nrStations
+ preParameters.obsStationIndices,
+ obsParameters.nrStations, //nrDelays
+ obsParameters.nrBitsPerSample,
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ obsParameters.subbandWidth,
+ obsParameters.nrSAPs,
+ false, // correlator
+ preParameters.delayCompensationEnabled,
+ false , // correctBandPass
+ false, // transpose
+ cobParameters.kernel.dumpDelayAndBandPassKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFPre_DelayAndBandPassKernel_%c%c%c.dat") %
+ obsParameters.observationID %
+ (false ? "B" : "b") % //correctBandpass
+ (preParameters.delayCompensationEnabled ? "D" : "d") %
+ (false ? "T" : "t")) //transpose
+ )),
+
+ bandPassCorrection(BandPassCorrectionKernel::Parameters(
+ nrSTABs,
+ preParameters.nrDelayCompensationChannels,
+ obsParameters.blockSize / preParameters.nrDelayCompensationChannels,
+ preParameters.bandPassCorrectionEnabled,
+ cobParameters.kernel.dumpBandPassCorrectionKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_BFPre_BandPassCorrectionKernel.dat") % obsParameters.observationID)))
{
}
BeamFormerPreprocessingStep::BeamFormerPreprocessingStep(
- const Parset &parset,
- gpu::Stream &i_queue,
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devA,
- boost::shared_ptr<gpu::DeviceMemory> i_devB)
+ std::shared_ptr<gpu::DeviceMemory> i_devA,
+ std::shared_ptr<gpu::DeviceMemory> i_devB)
:
- ProcessStep(parset, i_queue),
- flagsPerChannel(boost::extents[parset.settings.antennaFields.size()])
+ ProcessStep(i_htod_stream, i_dtoh_stream, i_execute_stream, context),
+ obsParameters(obsParameters),
+ preParameters(preParameters),
+ flagsPerChannel(boost::extents[preParameters.obsStationIndices.size()])
{
devA=i_devA;
devB=i_devB;
(void)context;
+ marker_gpu.reset(new gpu::Marker("preprocessingStep", gpu::Marker::yellow));
+
// intToFloat + FFTShift: A -> B
intToFloatKernel = std::unique_ptr<IntToFloatKernel>(
- factories.intToFloat.create(queue, *devA, *devB));
+ factories.intToFloat.create(*executeStream, *devA, *devB));
// FFT: B -> B
firstFFT = std::unique_ptr<FFT_Kernel>(
- factories.firstFFT.create(queue, *devB, *devB));
+ factories.firstFFT.create(*executeStream, *devB, *devB));
// zeroing: B -> B
zeroingKernel = std::unique_ptr<ZeroingKernel>(
- factories.zeroing.create(queue, *devB, *devB));
+ factories.zeroing.create(*executeStream, *devB, *devB));
// delayComp: B -> A
delayCompensationKernel = std::unique_ptr<DelayAndBandPassKernel>(
- factories.delayCompensation.create(queue, *devB, *devA));
+ factories.delayCompensation.create(*executeStream, *devB, *devA));
// bandPass: A -> B
bandPassCorrectionKernel = std::unique_ptr<BandPassCorrectionKernel>(
- factories.bandPassCorrection.create(queue, *devA, *devB));
+ factories.bandPassCorrection.create(*executeStream, *devA, *devB));
}
void BeamFormerPreprocessingStep::writeInput(const SubbandProcInputData &input)
{
- if (ps.settings.delayCompensation.enabled)
+ htodStream->waitEvent(executeFinished);
+
+ if (preParameters.delayCompensationEnabled)
{
- queue.writeBuffer(delayCompensationKernel->delaysAtBegin,
+ htodStream->writeBuffer(delayCompensationKernel->delaysAtBegin,
input.delaysAtBegin, false);
- queue.writeBuffer(delayCompensationKernel->delaysAfterEnd,
+ htodStream->writeBuffer(delayCompensationKernel->delaysAfterEnd,
input.delaysAfterEnd, false);
- queue.writeBuffer(delayCompensationKernel->phase0s,
+ htodStream->writeBuffer(delayCompensationKernel->phase0s,
input.phase0s, false);
}
+
+ htodStream->recordEvent(inputFinished);
}
void BeamFormerPreprocessingStep::process(const SubbandProcInputData &input)
{
+ executeStream->waitEvent(inputFinished);
//****************************************
// Enqueue the kernels
@@ -125,10 +182,11 @@ namespace LOFAR
// Convert input flags to channel flags
Flagger::convertFlagsToChannelFlags(
+ preParameters.obsStationIndices,
input.inputFlags,
flagsPerChannel,
- ps.settings.blockSize,
- ps.settings.beamFormer.nrDelayCompensationChannels,
+ obsParameters.blockSize,
+ preParameters.nrDelayCompensationChannels,
0);
zeroingKernel->enqueue(
@@ -138,11 +196,15 @@ namespace LOFAR
// The centralFrequency and SAP immediate kernel args must outlive kernel runs.
delayCompensationKernel->enqueue(
input.blockID,
- ps.settings.subbands[input.blockID.globalSubbandIdx].centralFrequency,
- ps.settings.subbands[input.blockID.globalSubbandIdx].SAP);
+ obsParameters.subbands[input.blockID.globalSubbandIdx].centralFrequency,
+ obsParameters.subbands[input.blockID.globalSubbandIdx].SAP);
bandPassCorrectionKernel->enqueue(
input.blockID);
+
+ executeStream->recordEvent(executeFinished);
+
+ marker_mutex.unlock();
}
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.h
index 5c7e6f39cd52c2b9840bbc9aae3b9e3f00322492..1e0c46b381989ee938e2a876ef52994f3a36271a 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/BeamFormerPreprocessingStep.h
@@ -24,9 +24,8 @@
#include <complex>
#include <Common/LofarLogger.h>
-#include <CoInterface/Parset.h>
-#include <boost/shared_ptr.hpp>
+#include <boost/format.hpp>
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/MultiDimArrayHostBuffer.h>
@@ -39,6 +38,7 @@
#include "ProcessStep.h"
#include <GPUProc/KernelFactory.h>
+#include <GPUProc/KernelParameters.h>
#include <GPUProc/Kernels/BandPassCorrectionKernel.h>
#include <GPUProc/Kernels/DelayAndBandPassKernel.h>
#include <GPUProc/Kernels/FFTShiftKernel.h>
@@ -53,8 +53,15 @@ namespace LOFAR
class BeamFormerPreprocessingStep: public ProcessStep
{
public:
+ friend class SubbandProc;
+
struct Factories {
- Factories(const Parset &ps);
+ Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::Cobalt& cobParameters);
+
+ unsigned nrSTABs;
KernelFactory<IntToFloatKernel> intToFloat;
@@ -68,24 +75,32 @@ namespace LOFAR
KernelFactory<BandPassCorrectionKernel> bandPassCorrection;
};
- BeamFormerPreprocessingStep(const Parset &parset,
- gpu::Stream &i_queue,
+ BeamFormerPreprocessingStep(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devA,
- boost::shared_ptr<gpu::DeviceMemory> i_devB);
+ std::shared_ptr<gpu::DeviceMemory> i_devA,
+ std::shared_ptr<gpu::DeviceMemory> i_devB);
void writeInput(const SubbandProcInputData &input);
void process(const SubbandProcInputData &input);
private:
+ // Parameters
+ KernelParameters::Observation obsParameters;
+ KernelParameters::Preprocessor preParameters;
+
// Preallocated flags for FFT-ed data -- used locally
MultiDimArray<SparseSet<unsigned>, 1> flagsPerChannel;
- //Data members
- boost::shared_ptr<gpu::DeviceMemory> devA;
- boost::shared_ptr<gpu::DeviceMemory> devB;
+ // Data members
+ std::shared_ptr<gpu::DeviceMemory> devA;
+ std::shared_ptr<gpu::DeviceMemory> devB;
// Int -> Float conversion
std::unique_ptr<IntToFloatKernel> intToFloatKernel;
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.cc
index 05c98e8b306fc8a65704e9b59f89d6342143921a..038deb9dc8b2841050d5a7552f10a4423ce45f91 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.cc
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.cc
@@ -37,20 +37,27 @@ namespace LOFAR
namespace Cobalt
{
- CorrelatorStep::Factories::Factories(const Parset &ps, size_t nrSubbandsPerSubbandProc) :
- fft(ps.settings.correlator.nrChannels > 1
+ CorrelatorStep::Factories::Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Correlator& corParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::Cobalt& cobParameters,
+ size_t nrSubbandsPerSubbandProc) :
+ fft(corParameters.nrChannels > 1
? new KernelFactory<FFT_Kernel>(FFT_Kernel::Parameters(
- ps.settings.correlator.nrChannels,
- ps.settings.antennaFields.size() * NR_POLARIZATIONS * ps.settings.blockSize,
+ corParameters.nrChannels,
+ obsParameters.nrStations * NR_POLARIZATIONS * obsParameters.blockSize,
true,
"FFT (correlator)"))
: NULL),
- firFilter(ps.settings.correlator.nrChannels > 1
- ? new KernelFactory<FIR_FilterKernel>(FIR_FilterKernel::Parameters(ps,
- ps.settings.antennaFields.size(),
- true,
+ firFilter(corParameters.nrChannels > 1
+ ? new KernelFactory<FIR_FilterKernel>(FIR_FilterKernel::Parameters(
+ obsParameters.nrStations,
+ obsParameters.nrBitsPerSample,
+ true, // inputIsStationData
nrSubbandsPerSubbandProc,
- ps.settings.correlator.nrChannels,
+ corParameters.nrChannels,
+ obsParameters.blockSize / corParameters.nrChannels,
// Scale to always output visibilities or stokes with the same flux scale.
// With the same bandwidth, twice the (narrower) channels _average_ (not
@@ -58,22 +65,60 @@ namespace LOFAR
// total bandwidth) _average_ to the _same_ flux, but noise * 1/sqrt(2).
// Note: FFTW/CUFFT do not normalize, correlation or stokes calculation
// effectively squares, integr on fewer channels averages over more values.
- std::sqrt((double)ps.settings.correlator.nrChannels),
+ std::sqrt((double)corParameters.nrChannels),
"FIR (correlator)"))
: NULL),
- intToFloat(ps.settings.correlator.nrChannels == 1
- ? new KernelFactory<IntToFloatKernel>(IntToFloatKernel::Parameters(ps, false, false))
+ intToFloat(corParameters.nrChannels == 1
+ ? new KernelFactory<IntToFloatKernel>(IntToFloatKernel::Parameters(
+ obsParameters.nrStations,
+ {}, // stationIndices
+ obsParameters.nrBitsPerSample,
+ obsParameters.blockSize,
+ false, // fftShift
+ false, // beamFormerStationSubset
+ cobParameters.kernel.dumpIntToFloatKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_Cor_IntToFloatKernel.dat") % obsParameters.observationID)))
: NULL),
- zeroing(ZeroingKernel::Parameters(ps,
- ps.settings.antennaFields.size(),
- ps.settings.correlator.nrChannels,
- "Zeroing (correlator)")),
-
- delayAndBandPass(DelayAndBandPassKernel::Parameters(ps, true)),
-
- correlator(ps)
+ zeroing(ZeroingKernel::Parameters(
+ obsParameters.nrStations,
+ corParameters.nrChannels,
+ obsParameters.blockSize / corParameters.nrChannels,
+ "Zeroing (correlator)",
+ cobParameters.kernel.dumpZeroingKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_Cor_ZeroingKernel.dat") % obsParameters.observationID)
+ )),
+
+ delayAndBandPass(DelayAndBandPassKernel::Parameters(
+ obsParameters.nrStations, // nrStations
+ {}, // delayIndices are set in the constructor
+ obsParameters.nrStations, // nrDelays
+ obsParameters.nrBitsPerSample,
+ corParameters.nrChannels,
+ obsParameters.blockSize / corParameters.nrChannels,
+ obsParameters.subbandWidth,
+ obsParameters.nrSAPs,
+ preParameters.delayCompensationEnabled,
+ true, // correlator
+ preParameters.bandPassCorrectionEnabled,
+ true, // transpose
+ cobParameters.kernel.dumpDelayAndBandPassKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_Cor_DelayAndBandPassKernel_%c%c%c.dat") %
+ obsParameters.observationID %
+ (preParameters.bandPassCorrectionEnabled ? "B" : "b") %
+ (preParameters.delayCompensationEnabled ? "D" : "d") %
+ (true ? "T" : "t"))
+ )),
+
+ correlator(CorrelatorKernel::Parameters(
+ obsParameters.nrStations,
+ corParameters.nrChannels,
+ corParameters.nrSamplesPerBlock / corParameters.nrIntegrationsPerBlock,
+ corParameters.nrIntegrationsPerBlock,
+ cobParameters.kernel.dumpCorrelatorKernel,
+ str(boost::format("L%d_SB%%03d_BL%%03d_Cor_CorrelatorKernel.dat") % obsParameters.observationID)
+ ))
{
}
@@ -88,7 +133,10 @@ namespace LOFAR
}
template<typename T> void CorrelatorStep::Flagger::calcNrValidSamples(
- Parset const &parset,
+ unsigned nrStations,
+ unsigned nrSamplesPerBlock,
+ unsigned nrIntegrationsPerBlock,
+ unsigned nrChannels,
MultiDimArray<SparseSet<unsigned>, 1>const & flagsPerChannel,
SubbandProcOutputData::CorrelatedData &output)
{
@@ -98,12 +146,10 @@ namespace LOFAR
*/
// The number of samples per integration within this block.
- const unsigned nrSamples =
- parset.settings.correlator.nrSamplesPerBlock /
- parset.settings.correlator.nrIntegrationsPerBlock;
+ const unsigned nrSamples = nrSamplesPerBlock / nrIntegrationsPerBlock;
// loop the stations
- for (unsigned stat1 = 0; stat1 < parset.settings.correlator.stations.size(); stat1 ++) {
+ for (unsigned stat1 = 0; stat1 < nrStations; stat1 ++) {
for (unsigned stat2 = 0; stat2 <= stat1; stat2 ++) {
const unsigned bl = baseline(stat1, stat2);
@@ -112,7 +158,7 @@ namespace LOFAR
const SparseSet<unsigned> flags =
flagsPerChannel[stat1] | flagsPerChannel[stat2];
- for (size_t i = 0; i < parset.settings.correlator.nrIntegrationsPerBlock; ++i) {
+ for (size_t i = 0; i < nrIntegrationsPerBlock; ++i) {
LOFAR::Cobalt::CorrelatedData &correlatedData = *output.subblocks[i];
// Count the flags for this subblock
@@ -120,14 +166,14 @@ namespace LOFAR
nrSamples - flags.count(i * nrSamples, (i+1) * nrSamples);
// Channel zero is invalid, unless we have only one channel
- if (parset.settings.correlator.nrChannels > 1) {
+ if (nrChannels > 1) {
correlatedData.nrValidSamples<T>(bl, 0) = 0;
} else {
correlatedData.nrValidSamples<T>(bl, 0) = nrValidSamples;
}
// Set the channels from 1 onward
- for(unsigned ch = 1; ch < parset.settings.correlator.nrChannels; ch ++) {
+ for(unsigned ch = 1; ch < nrChannels; ch ++) {
correlatedData.nrValidSamples<T>(bl, ch) = nrValidSamples;
}
}
@@ -137,21 +183,30 @@ namespace LOFAR
void CorrelatorStep::Flagger::calcNrValidSamples(
- Parset const &parset,
+ unsigned nrStations,
+ unsigned nrSamplesPerBlock,
+ unsigned nrIntegrationsPerBlock,
+ unsigned nrChannels,
MultiDimArray<SparseSet<unsigned>, 1>const & flagsPerChannel,
SubbandProcOutputData::CorrelatedData &output)
{
switch (output.subblocks[0]->itsNrBytesPerNrValidSamples) {
case 4:
- calcNrValidSamples<uint32_t>(parset, flagsPerChannel, output);
+ calcNrValidSamples<uint32_t>(
+ nrStations, nrSamplesPerBlock, nrIntegrationsPerBlock,
+ nrChannels, flagsPerChannel, output);
break;
case 2:
- calcNrValidSamples<uint16_t>(parset, flagsPerChannel, output);
+ calcNrValidSamples<uint16_t>(
+ nrStations, nrSamplesPerBlock, nrIntegrationsPerBlock,
+ nrChannels, flagsPerChannel, output);
break;
case 1:
- calcNrValidSamples<uint8_t>(parset, flagsPerChannel, output);
+ calcNrValidSamples<uint8_t>(
+ nrStations, nrSamplesPerBlock, nrIntegrationsPerBlock,
+ nrChannels, flagsPerChannel, output);
break;
}
}
@@ -181,11 +236,12 @@ namespace LOFAR
}
- template<typename T> void
- CorrelatorStep::Flagger::applyNrValidSamples(Parset const &parset,
- LOFAR::Cobalt::CorrelatedData &output)
+ template<typename T> void
+ CorrelatorStep::Flagger::applyNrValidSamples(
+ unsigned nrChannels,
+ LOFAR::Cobalt::CorrelatedData &output)
{
- const bool singleChannel = parset.settings.correlator.nrChannels == 1;
+ const bool singleChannel = nrChannels == 1;
for (unsigned bl = 0; bl < output.itsNrBaselines; ++bl)
{
@@ -195,101 +251,120 @@ namespace LOFAR
const T nrValidSamples = output.nrValidSamples<T>(bl, singleChannel ? 0 : 1);
// If all samples flagged, weights is zero.
- const float weight = nrValidSamples ? 1.0f / nrValidSamples : 0;
+ const float weight = nrValidSamples ? 1.0f / nrValidSamples : 0;
// Apply the weight to this sample, turning the visibilities into the
// average visibility over the non-flagged samples.
//
// This step thus normalises the visibilities for any integration time.
- applyWeight(bl, parset.settings.correlator.nrChannels, weight, output);
+ applyWeight(bl, nrChannels, weight, output);
}
}
- void CorrelatorStep::Flagger::applyNrValidSamples(Parset const &parset,
- LOFAR::Cobalt::CorrelatedData &output)
+ void CorrelatorStep::Flagger::applyNrValidSamples(
+ unsigned nrChannels,
+ LOFAR::Cobalt::CorrelatedData &output)
{
switch (output.itsNrBytesPerNrValidSamples) {
case 4:
- applyNrValidSamples<uint32_t>(parset, output);
+ applyNrValidSamples<uint32_t>(nrChannels, output);
break;
case 2:
- applyNrValidSamples<uint16_t>(parset, output);
+ applyNrValidSamples<uint16_t>(nrChannels, output);
break;
case 1:
- applyNrValidSamples<uint8_t>(parset, output);
+ applyNrValidSamples<uint8_t>(nrChannels, output);
break;
}
}
CorrelatorStep::CorrelatorStep(
- const Parset &parset,
- gpu::Stream &i_queue,
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::Correlator& corParameters,
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devA,
- boost::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devA,
+ std::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devE,
size_t nrSubbandsPerSubbandProc)
:
- ProcessStep(parset, i_queue),
- correlatorPPF(ps.settings.correlator.nrChannels > 1),
- flagsWithHistorySamples(boost::extents[parset.settings.antennaFields.size()]),
- flagsPerChannel(boost::extents[parset.settings.antennaFields.size()]),
- devE(context, std::max(factories.correlator.bufferSize(CorrelatorKernel::INPUT_DATA),
- factories.correlator.bufferSize(CorrelatorKernel::OUTPUT_DATA))),
+ ProcessStep(i_htod_stream, i_dtoh_stream, i_execute_stream, context),
+ obsParameters(obsParameters),
+ preParameters(preParameters),
+ corParameters(corParameters),
+ correlatorPPF(corParameters.nrChannels > 1),
+ flagsWithHistorySamples(boost::extents[obsParameters.nrStations]),
+ flagsPerChannel(boost::extents[obsParameters.nrStations]),
outputCounter(context, "output (correlator)"),
integratedData(nrSubbandsPerSubbandProc)
{
devA=i_devA;
devB=i_devB;
+ devE=i_devE;
+
+ marker_gpu.reset(new gpu::Marker("correlatorStep (gpu)", gpu::Marker::red));
+ marker_cpu.reset(new gpu::Marker("correlatorStep (cpu)", gpu::Marker::black));
if (correlatorPPF) {
// FIR filter: A -> B
- firFilterKernel = factories.firFilter->create(queue, *devA, *devB);
+ firFilterKernel.reset(factories.firFilter->create(*executeStream, *devA, *devB));
// FFT: B -> E
- fftKernel = factories.fft->create(queue, *devB, devE);
+ fftKernel.reset(factories.fft->create(*executeStream, *devB, *devE));
} else {
// intToFloat: A -> E
- intToFloatKernel = factories.intToFloat->create(queue, *devA, devE);
+ intToFloatKernel.reset(factories.intToFloat->create(*executeStream, *devA, *devE));
}
// Zeroing: E -> E
- zeroingKernel = factories.zeroing.create(queue, devE, devE);
+ zeroingKernel.reset(factories.zeroing.create(*executeStream, *devE, *devE));
// Delay and Bandpass: E -> B
- delayAndBandPassKernel = std::unique_ptr<DelayAndBandPassKernel>(factories.delayAndBandPass.create(queue, devE, *devB));
+ delayAndBandPassKernel.reset(factories.delayAndBandPass.create(*executeStream, *devE, *devB));
// Correlator: B -> E
- correlatorKernel = std::unique_ptr<CorrelatorKernel>(factories.correlator.create(queue,
- *devB, devE));
+ correlatorKernel.reset(factories.correlator.create(*executeStream, *devB, *devE));
// Initialize the output buffers for the long-time integration
for (size_t i = 0; i < integratedData.size(); i++) {
integratedData[i] =
// Note that we always integrate complete blocks
- make_pair(0, new LOFAR::Cobalt::CorrelatedData(ps.settings.antennaFields.size(),
- ps.settings.correlator.nrChannels,
- ps.settings.correlator.nrSamplesPerBlock));
+ make_pair(0, std::unique_ptr<LOFAR::Cobalt::CorrelatedData>(
+ new LOFAR::Cobalt::CorrelatedData(
+ obsParameters.nrStations,
+ corParameters.nrChannels,
+ corParameters.nrSamplesPerBlock)));
}
}
void CorrelatorStep::writeInput(const SubbandProcInputData &input)
{
- if (ps.settings.delayCompensation.enabled) {
- queue.writeBuffer(delayAndBandPassKernel->delaysAtBegin,
+ htodStream->waitEvent(executeFinished);
+
+ if (preParameters.delayCompensationEnabled) {
+ htodStream->writeBuffer(delayAndBandPassKernel->delaysAtBegin,
input.delaysAtBegin, false);
- queue.writeBuffer(delayAndBandPassKernel->delaysAfterEnd,
+ htodStream->writeBuffer(delayAndBandPassKernel->delaysAfterEnd,
input.delaysAfterEnd, false);
- queue.writeBuffer(delayAndBandPassKernel->phase0s,
+ htodStream->writeBuffer(delayAndBandPassKernel->phase0s,
input.phase0s, false);
}
+
+ htodStream->recordEvent(inputFinished);
}
void CorrelatorStep::process(const SubbandProcInputData &input)
{
+ executeStream->waitEvent(inputFinished);
+ executeStream->waitEvent(outputFinished);
+
if (correlatorPPF) {
// The subbandIdx immediate kernel arg must outlive kernel runs.
firFilterKernel->enqueue(input.blockID,
@@ -305,8 +380,8 @@ namespace LOFAR
Cobalt::Flagger::convertFlagsToChannelFlags(
flagsWithHistorySamples,
flagsPerChannel,
- ps.settings.blockSize,
- ps.settings.correlator.nrChannels,
+ obsParameters.blockSize,
+ corParameters.nrChannels,
NR_TAPS - 1);
// Zero the output of each FFT that had flagged input samples
@@ -323,25 +398,34 @@ namespace LOFAR
//
// The centralFrequency and SAP immediate kernel args must outlive kernel runs.
delayAndBandPassKernel->enqueue(
- input.blockID,
- ps.settings.subbands[input.blockID.globalSubbandIdx].centralFrequency,
- ps.settings.subbands[input.blockID.globalSubbandIdx].SAP);
+ input.blockID,
+ obsParameters.subbands[input.blockID.globalSubbandIdx].centralFrequency,
+ obsParameters.subbands[input.blockID.globalSubbandIdx].SAP);
correlatorKernel->enqueue(input.blockID);
+
+ executeStream->recordEvent(executeFinished);
+
+ marker_mutex.unlock();
}
void CorrelatorStep::readOutput(SubbandProcOutputData &output)
{
// Read data back from the kernel
- queue.readBuffer(output.correlatedData.data, devE, outputCounter, false);
+ dtohStream->waitEvent(executeFinished);
+ dtohStream->readBuffer(output.correlatedData.data, *devE, outputCounter, false);
+ dtohStream->synchronize();
+ dtohStream->recordEvent(outputFinished);
}
void CorrelatorStep::processCPU(const SubbandProcInputData &input, SubbandProcOutputData &output)
{
- // Propagate the flags.
+ outputFinished.wait();
+ marker_cpu->start(outputFinished);
+ // Propagate the flags.
if (correlatorPPF) {
flagsWithHistorySamples = input.inputFlags;
@@ -349,32 +433,36 @@ namespace LOFAR
firFilterKernel->prefixHistoryFlags(
flagsWithHistorySamples, input.blockID.subbandProcSubbandIdx);
- // Transform the flags to channel flags: taking in account
+ // Transform the flags to channel flags: taking in account
// reduced resolution in time and the size of the filter
Cobalt::Flagger::convertFlagsToChannelFlags(
flagsWithHistorySamples,
flagsPerChannel,
- ps.settings.blockSize,
- ps.settings.correlator.nrChannels,
+ obsParameters.blockSize,
+ corParameters.nrChannels,
NR_TAPS - 1);
-
- // Calculate the number of flags per baseline and assign to
- // output object.
- Flagger::calcNrValidSamples(ps, flagsPerChannel, output.correlatedData);
- } else {
- // Calculate the number of flags per baseline and assign to
- // output object.
- Flagger::calcNrValidSamples(ps, input.inputFlags, output.correlatedData);
}
+
+ // Calculate the number of flags per baseline and assign to
+ // output object.
+ Flagger::calcNrValidSamples(
+ corParameters.stations.size(),
+ corParameters.nrSamplesPerBlock,
+ corParameters.nrIntegrationsPerBlock,
+ corParameters.nrChannels,
+ correlatorPPF ? flagsPerChannel : input.inputFlags,
+ output.correlatedData);
+
+ marker_cpu->end();
}
bool CorrelatorStep::integrate(SubbandProcOutputData &output)
{
const size_t idx = output.blockID.subbandProcSubbandIdx;
- const size_t nblock = ps.settings.correlator.nrBlocksPerIntegration;
- const size_t nsubblock = ps.settings.correlator.nrIntegrationsPerBlock;
-
+ const size_t nblock = corParameters.nrBlocksPerIntegration;
+ const size_t nsubblock = corParameters.nrIntegrationsPerBlock;
+
// We don't want to copy the data if we don't need to integrate.
if (nblock == 1) {
for (size_t i = 0; i < nsubblock; ++i) {
@@ -417,8 +505,8 @@ namespace LOFAR
// The flags are already copied to the correct location
// now the flagged amount should be applied to the visibilities
- for (size_t i = 0; i < ps.settings.correlator.nrIntegrationsPerBlock; ++i) {
- Flagger::applyNrValidSamples(ps, *output.correlatedData.subblocks[i]);
+ for (size_t i = 0; i < corParameters.nrIntegrationsPerBlock; ++i) {
+ Flagger::applyNrValidSamples(corParameters.nrChannels, *output.correlatedData.subblocks[i]);
}
return true;
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.h
index 152e6c3af1a0541a1dd96e93dfed0aeb450d1311..eef6c3f210f6927cfdc0b0f7161f13dc8e97176e 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/CorrelatorStep.h
@@ -25,10 +25,6 @@
#include <vector>
#include <utility> // for std::pair
-#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
-
-#include <boost/shared_ptr.hpp>
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/MultiDimArrayHostBuffer.h>
@@ -56,13 +52,20 @@ namespace LOFAR
{
class CorrelatorStep: public ProcessStep
{
+ friend class SubbandProc;
+
public:
struct Factories {
- Factories(const Parset &ps, size_t nrSubbandsPerSubbandProc);
+ Factories(
+ const KernelParameters::Observation& obsParameters,
+ const KernelParameters::Correlator& corParameters,
+ const KernelParameters::Preprocessor& preParameters,
+ const KernelParameters::Cobalt& cobParameters,
+ size_t nrSubbandsPerSubbandProc);
- SmartPtr< KernelFactory<FFT_Kernel> > fft;
- SmartPtr< KernelFactory<FIR_FilterKernel> > firFilter;
- SmartPtr< KernelFactory<IntToFloatKernel> > intToFloat;
+ std::unique_ptr<KernelFactory<FFT_Kernel>> fft;
+ std::unique_ptr<KernelFactory<FIR_FilterKernel>> firFilter;
+ std::unique_ptr<KernelFactory<IntToFloatKernel>> intToFloat;
KernelFactory<ZeroingKernel> zeroing;
@@ -71,12 +74,18 @@ namespace LOFAR
KernelFactory<CorrelatorKernel> correlator;
};
- CorrelatorStep(const Parset &parset,
- gpu::Stream &i_queue,
+ CorrelatorStep(
+ const KernelParameters::Observation &obsParameters,
+ const KernelParameters::Preprocessor &preParameters,
+ const KernelParameters::Correlator &corParameters,
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
gpu::Context &context,
Factories &factories,
- boost::shared_ptr<gpu::DeviceMemory> i_devA,
- boost::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devA,
+ std::shared_ptr<gpu::DeviceMemory> i_devB,
+ std::shared_ptr<gpu::DeviceMemory> i_devE,
size_t nrSubbandsPerSubbandProc);
void writeInput(const SubbandProcInputData &input);
@@ -101,15 +110,21 @@ namespace LOFAR
// 2. Calculate the weight based on the number of flags and apply this
// weighting to all output values
- static void applyNrValidSamples(Parset const &parset, LOFAR::Cobalt::CorrelatedData &output);
+ static void applyNrValidSamples(
+ unsigned nrChannels,
+ LOFAR::Cobalt::CorrelatedData &output);
// 1.2 Calculate the number of flagged samples and set this on the
// output dataproduct This function is aware of the used filter width a
// corrects for this.
static void
- calcNrValidSamples(Parset const &parset,
- MultiDimArray<SparseSet<unsigned>, 1> const &flagsPerChannel,
- SubbandProcOutputData::CorrelatedData &output);
+ calcNrValidSamples(
+ unsigned nrStations,
+ unsigned nrSamplesPerBlock,
+ unsigned nrIntegrationsPerBlock,
+ unsigned nrChannels,
+ MultiDimArray<SparseSet<unsigned>, 1> const &flagsPerChannel,
+ SubbandProcOutputData::CorrelatedData &output);
// 2.1 Apply the supplied weight to the complex values in the channels
// for the given baseline. If nrChannels > 1, visibilities for channel 0 will be set to 0.0.
@@ -117,17 +132,28 @@ namespace LOFAR
float weight, LOFAR::Cobalt::CorrelatedData &output);
private:
template<typename T>
- static void applyNrValidSamples(Parset const &parset, LOFAR::Cobalt::CorrelatedData &output);
+ static void applyNrValidSamples(
+ unsigned nrChannels,
+ LOFAR::Cobalt::CorrelatedData &output);
template<typename T>
static void
- calcNrValidSamples(Parset const &parset,
- MultiDimArray<SparseSet<unsigned>, 1> const &flagsPerChannel,
- SubbandProcOutputData::CorrelatedData &output);
+ calcNrValidSamples(
+ unsigned nrStations,
+ unsigned nrSamplesPerBlock,
+ unsigned nrIntegrationsPerBlock,
+ unsigned nrChannels,
+ MultiDimArray<SparseSet<unsigned>, 1> const &flagsPerChannel,
+ SubbandProcOutputData::CorrelatedData &output);
};
private:
+ // Parameters
+ KernelParameters::Observation obsParameters;
+ KernelParameters::Preprocessor preParameters;
+ KernelParameters::Correlator corParameters;
+
const bool correlatorPPF;
// Preallocated flags for input data, to prefix with FIR history -- used locally
@@ -137,25 +163,25 @@ namespace LOFAR
MultiDimArray<SparseSet<unsigned>, 1> flagsPerChannel;
//Data members
- boost::shared_ptr<gpu::DeviceMemory> devA;
- boost::shared_ptr<gpu::DeviceMemory> devB;
- gpu::DeviceMemory devE;
+ std::shared_ptr<gpu::DeviceMemory> devA;
+ std::shared_ptr<gpu::DeviceMemory> devB;
+ std::shared_ptr<gpu::DeviceMemory> devE;
/*
* Kernels
*/
// FIR filter
- SmartPtr<FIR_FilterKernel> firFilterKernel;
+ std::unique_ptr<FIR_FilterKernel> firFilterKernel;
// FFT
- SmartPtr<FFT_Kernel> fftKernel;
+ std::unique_ptr<FFT_Kernel> fftKernel;
// IntToFloat (in case of no FFT)
- SmartPtr<IntToFloatKernel> intToFloatKernel;
+ std::unique_ptr<IntToFloatKernel> intToFloatKernel;
// Zeroing
- SmartPtr<ZeroingKernel> zeroingKernel;
+ std::unique_ptr<ZeroingKernel> zeroingKernel;
// Delay and Bandpass
std::unique_ptr<DelayAndBandPassKernel> delayAndBandPassKernel;
@@ -169,7 +195,7 @@ namespace LOFAR
// will be processed by this class instance. Each element of the vector
// contains a counter that tracks the number of additions made to the data
// buffer and the data buffer itself.
- std::vector< std::pair< size_t, SmartPtr<LOFAR::Cobalt::CorrelatedData> > >
+ std::vector<std::pair<size_t, std::unique_ptr<LOFAR::Cobalt::CorrelatedData>>>
integratedData;
bool integrate(SubbandProcOutputData &output);
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/KernelFactories.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/KernelFactories.cc
deleted file mode 100644
index 1f4791af45ed0697ce99dd804330ea95dfd8098f..0000000000000000000000000000000000000000
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/KernelFactories.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-//# KernelFactories.cc
-//#
-//# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#include <lofar_config.h>
-
-#include "KernelFactories.h"
-
-namespace LOFAR
-{
- namespace Cobalt
- {
- KernelFactories::KernelFactories(const Parset &ps,
- size_t nrSubbandsPerSubbandProc) :
- correlator(ps.settings.correlator.enabled
- ? new CorrelatorStep::Factories(ps, nrSubbandsPerSubbandProc)
- : NULL),
-
- preprocessing(ps.settings.beamFormer.enabled
- ? new BeamFormerPreprocessingStep::Factories(ps)
- : NULL),
-
- coherentStokes(ps.settings.beamFormer.anyCoherentTABs()
- ? new BeamFormerCoherentStep::Factories(ps, nrSubbandsPerSubbandProc)
- : NULL),
-
- incoherentStokes(ps.settings.beamFormer.anyIncoherentTABs()
- ? new BeamFormerIncoherentStep::Factories(ps, nrSubbandsPerSubbandProc)
- : NULL)
- {
- }
- }
-}
-
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/KernelFactories.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/KernelFactories.h
deleted file mode 100644
index c58eaaf914f60d33d3d8b718175b779d5581cc70..0000000000000000000000000000000000000000
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/KernelFactories.h
+++ /dev/null
@@ -1,53 +0,0 @@
-//# KernelFactories.h
-//#
-//# Copyright (C) 2012-2013 ASTRON (Netherlands Institute for Radio Astronomy)
-//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
-//#
-//# This file is part of the LOFAR software suite.
-//# The LOFAR software suite is free software: you can redistribute it and/or
-//# modify it under the terms of the GNU General Public License as published
-//# by the Free Software Foundation, either version 3 of the License, or
-//# (at your option) any later version.
-//#
-//# The LOFAR software suite is distributed in the hope that it will be useful,
-//# but WITHOUT ANY WARRANTY; without even the implied warranty of
-//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//# GNU General Public License for more details.
-//#
-//# You should have received a copy of the GNU General Public License along
-//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
-//#
-//# $Id$
-
-#ifndef LOFAR_GPUPROC_CUDA_BEAM_FORMER_FACTORIES_H
-#define LOFAR_GPUPROC_CUDA_BEAM_FORMER_FACTORIES_H
-
-#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
-
-#include "CorrelatorStep.h"
-#include "BeamFormerPreprocessingStep.h"
-#include "BeamFormerCoherentStep.h"
-#include "BeamFormerIncoherentStep.h"
-
-namespace LOFAR
-{
- namespace Cobalt
- {
- struct KernelFactories
- {
- KernelFactories(const Parset &ps,
- size_t nrSubbandsPerSubbandProc = 1);
-
- SmartPtr<CorrelatorStep::Factories> correlator;
-
- SmartPtr<BeamFormerPreprocessingStep::Factories> preprocessing;
-
- SmartPtr<BeamFormerCoherentStep::Factories> coherentStokes;
- SmartPtr<BeamFormerIncoherentStep::Factories> incoherentStokes;
- };
-
- }
-}
-
-#endif
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/ProcessStep.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/ProcessStep.h
index 195fd34108ac320e1d2a49f0b019693570a7d1b3..2a616c81cf5de7c56f42fd05bdbc317bbc3ce15b 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/ProcessStep.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/ProcessStep.h
@@ -21,9 +21,11 @@
#ifndef LOFAR_GPUPROC_CUDA_PROCESS_STEP_H
#define LOFAR_GPUPROC_CUDA_PROCESS_STEP_H
-#include <CoInterface/Parset.h>
+#include <thread>
+#include <memory>
#include <GPUProc/gpu_wrapper.h>
+#include <GPUProc/KernelParameters.h>
#include "SubbandProcInputData.h"
@@ -33,22 +35,63 @@ namespace LOFAR
{
class ProcessStep
{
+ friend class SubbandProc;
+
public:
virtual void process(const SubbandProcInputData &input) = 0;
virtual ~ProcessStep()
- {}
+ {
+ marker_thread_enabled = false;
+ marker_mutex.unlock();
+ if (marker_thread.joinable())
+ {
+ marker_thread.join();
+ }
+ }
protected:
- ProcessStep(const Parset &parset,
- gpu::Stream &queue)
+ ProcessStep(
+ std::shared_ptr<gpu::Stream> i_htod_stream,
+ std::shared_ptr<gpu::Stream> i_dtoh_stream,
+ std::shared_ptr<gpu::Stream> i_execute_stream,
+ gpu::Context &context)
:
- ps(parset),
- queue(queue)
- {};
+ htodStream(i_htod_stream),
+ dtohStream(i_dtoh_stream),
+ executeStream(i_execute_stream),
+ inputFinished(context),
+ executeFinished(context),
+ outputFinished(context)
+ {
+ marker_mutex.lock();
+ marker_thread = std::thread([&]{
+ while (true)
+ {
+ marker_mutex.lock();
+ if (!marker_thread_enabled)
+ {
+ break;
+ }
+ marker_gpu->start(inputFinished);
+ marker_gpu->end(executeFinished);
+ }
+ });
+ };
+
+ std::shared_ptr<gpu::Stream> htodStream;
+ std::shared_ptr<gpu::Stream> dtohStream;
+ std::shared_ptr<gpu::Stream> executeStream;
+
+ gpu::Event inputFinished;
+ gpu::Event executeFinished;
+ gpu::Event outputFinished;
- const Parset ps;
- gpu::Stream queue;
+ std::unique_ptr<gpu::Marker> marker_cpu;
+ std::unique_ptr<gpu::Marker> marker_gpu;
+ std::thread marker_thread;
+ std::mutex marker_mutex;
+ bool marker_thread_enabled = true;
};
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.cc
index 4a35c04a6db39bd0b6c48c897810acd1bf7dabbe..ff8084153860611b0abdc2e4543cf9fec09cdd42 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.cc
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.cc
@@ -21,7 +21,6 @@
#include <lofar_config.h>
#include "SubbandProc.h"
-#include "KernelFactories.h"
#include <GPUProc/global_defines.h>
#include <GPUProc/gpu_wrapper.h>
@@ -33,6 +32,7 @@
#include <Common/LofarLogger.h>
#include <iomanip>
+#include <thread>
namespace LOFAR
{
@@ -41,7 +41,6 @@ namespace LOFAR
SubbandProc::SubbandProc(
const Parset &ps,
gpu::Context &context,
- KernelFactories &factories,
size_t nrSubbandsPerSubbandProc)
:
inputPool("SubbandProc::inputPool", ps.settings.realTime),
@@ -49,10 +48,12 @@ namespace LOFAR
outputPool("SubbandProc::outputPool", ps.settings.realTime),
ps(ps),
+ context(context),
nrSubbandsPerSubbandProc(nrSubbandsPerSubbandProc),
- queue(gpu::Stream(context)),
+ htodStream(new gpu::Stream(context)),
+ dtohStream(new gpu::Stream(context)),
+ executeStream(new gpu::Stream(context)),
prevBlock(-1),
- prevSAP(-1),
totalCounter(context, "total"),
inputCounter(context, "input"),
processCPUTimer("processCPU", ps.settings.blockDuration() / nrSubbandsPerSubbandProc, true, true)
@@ -60,83 +61,235 @@ namespace LOFAR
// See doc/bf-pipeline.txt
size_t devA_size = 0;
size_t devB_size = 0;
+ size_t devD_size = 0;
+ size_t devE_size = 0;
+
+ bool correlatorEnabled = ps.settings.correlator.enabled;
+ bool preprocessingEnabled = ps.settings.beamFormer.enabled;
+ KernelParameters::Observation obsParameters(ps.settings);
+ KernelParameters::Correlator corParameters = ps.settings.correlator;
+ KernelParameters::Preprocessor preParameters(
+ ps.settings.antennaFieldNames,
+ ps.settings.beamFormer.antennaFieldNames,
+ ps.settings.delayCompensation.enabled,
+ ps.settings.corrections.bandPass,
+ ps.settings.beamFormer.nrDelayCompensationChannels);
+ KernelParameters::Cobalt cobParameters(ps.settings.cobalt);
- if (factories.correlator) {
- CorrelatorStep::Factories &cf = *factories.correlator;
+ //################################################
+ // Create objects containing the kernel and device buffers
+ // for the correlator pipeline (if any)
+
+ if (correlatorEnabled)
+ {
+ correlatorFactory.reset(
+ new CorrelatorStep::Factories(
+ obsParameters,
+ corParameters,
+ preParameters,
+ cobParameters,
+ nrSubbandsPerSubbandProc));
devA_size = std::max(devA_size,
- cf.firFilter ? cf.firFilter->bufferSize(FIR_FilterKernel::INPUT_DATA)
- : cf.delayAndBandPass.bufferSize(DelayAndBandPassKernel::INPUT_DATA));
+ correlatorFactory->firFilter ? correlatorFactory->firFilter->bufferSize(FIR_FilterKernel::INPUT_DATA)
+ : correlatorFactory->delayAndBandPass.bufferSize(DelayAndBandPassKernel::INPUT_DATA));
devB_size = std::max(devB_size,
- cf.correlator.bufferSize(CorrelatorKernel::INPUT_DATA));
+ correlatorFactory->correlator.bufferSize(CorrelatorKernel::INPUT_DATA));
+ devE_size = std::max(correlatorFactory->correlator.bufferSize(CorrelatorKernel::INPUT_DATA),
+ correlatorFactory->correlator.bufferSize(CorrelatorKernel::OUTPUT_DATA));
}
- if (factories.preprocessing) {
+ //################################################
+ // Create objects containing the kernel and device buffers
+ // for the beamformer preprocessing pipeline (if any)
+
+ if (preprocessingEnabled)
+ {
+ preprocessingFactory.reset(
+ new BeamFormerPreprocessingStep::Factories(
+ obsParameters,
+ preParameters,
+ cobParameters));
+
+ devA_size = std::max(devA_size,
+ preprocessingFactory->intToFloat.bufferSize(IntToFloatKernel::INPUT_DATA));
devA_size = std::max(devA_size,
- factories.preprocessing->intToFloat.bufferSize(IntToFloatKernel::OUTPUT_DATA));
+ preprocessingFactory->intToFloat.bufferSize(IntToFloatKernel::OUTPUT_DATA));
devB_size = std::max(devB_size,
- factories.preprocessing->intToFloat.bufferSize(IntToFloatKernel::OUTPUT_DATA));
+ preprocessingFactory->intToFloat.bufferSize(IntToFloatKernel::OUTPUT_DATA));
}
- if (factories.incoherentStokes) {
- ASSERT(factories.preprocessing);
- /* incoherentStokes uses devA and devB, but the sizes provided b the preprocessing
- pipeline are already sufficient. */
+ //################################################
+ // Create objects containing the kernel and device buffers
+ // for the beamformer coherent stokes and incoherent stokes pipelines
+
+ auto& bfPipelines = ps.settings.beamFormer.pipelines;
+
+ coherentFactories.resize(bfPipelines.size());
+ incoherentFactories.resize(bfPipelines.size());
+ coherentSteps.resize(bfPipelines.size());
+ incoherentSteps.resize(bfPipelines.size());
+
+ std::vector<std::unique_ptr<KernelParameters::BeamFormer>> bfParameters(bfPipelines.size());
+
+ for (unsigned i = 0; i < bfPipelines.size(); i++)
+ {
+ auto& bfPipeline = bfPipelines[i];
+
+ bfParameters[i].reset(new KernelParameters::BeamFormer(
+ ps.settings.antennaFieldNames,
+ ps.settings.beamFormer.antennaFieldNames,
+ bfPipeline));
+
+ if (bfPipeline.maxNrCoherentTABsPerSAP() > 0) {
+ coherentFactories[i].reset(
+ new BeamFormerCoherentStep::Factories(
+ obsParameters,
+ preParameters,
+ *bfParameters[i],
+ cobParameters,
+ nrSubbandsPerSubbandProc));
+
+ devB_size = std::max(devB_size,
+ coherentFactories[i]->beamFormer.bufferSize(BeamFormerKernel::INPUT_DATA));
+
+ devD_size = std::max(devD_size,
+ coherentFactories[i]->beamFormer.bufferSize(BeamFormerKernel::OUTPUT_DATA));
+
+ devE_size = std::max(devE_size,
+ coherentFactories[i]->coherentStokes.bufferSize(CoherentStokesKernel::OUTPUT_DATA));
+ }
+
+ if (bfPipeline.maxNrIncoherentTABsPerSAP() > 0) {
+ incoherentFactories[i].reset(
+ new BeamFormerIncoherentStep::Factories(
+ obsParameters,
+ preParameters,
+ *bfParameters[i],
+ cobParameters,
+ nrSubbandsPerSubbandProc));
+
+ /* incoherentStokes uses devA and devB, but the sizes provided by the preprocessing
+ pipeline are already sufficient. */
+
+ devE_size = std::max(devE_size,
+ incoherentFactories[i]->incoherentStokes.bufferSize(IncoherentStokesKernel::OUTPUT_DATA));
+ }
}
// NOTE: For an explanation of the different buffers being used, please refer
// to the document pipeline-buffers.txt in the GPUProc/doc directory.
devA.reset(new gpu::DeviceMemory(context, devA_size));
devB.reset(new gpu::DeviceMemory(context, devB_size));
+ devD.reset(new gpu::DeviceMemory(context, devD_size));
+ devE.reset(new gpu::DeviceMemory(context, devE_size));
//################################################
- // Create objects containing the kernel and device buffers
+ // Now that the maximum sizes of the device buffers are known,
+ // initialize the process steps.
- if (factories.correlator) {
- correlatorStep = std::unique_ptr<CorrelatorStep>(
- new CorrelatorStep(ps, queue, context, *factories.correlator,
- devA, devB, nrSubbandsPerSubbandProc));
+ if (correlatorFactory.get())
+ {
+ correlatorStep.reset(
+ new CorrelatorStep(
+ obsParameters,
+ preParameters,
+ corParameters,
+ htodStream, dtohStream, executeStream, context, *correlatorFactory,
+ devA, devB, devE, nrSubbandsPerSubbandProc));
}
- if (factories.preprocessing) {
- preprocessingStep = std::unique_ptr<BeamFormerPreprocessingStep>(
- new BeamFormerPreprocessingStep(ps, queue, context, *factories.preprocessing,
+ if (preprocessingFactory.get())
+ {
+ preprocessingStep.reset(
+ new BeamFormerPreprocessingStep(
+ obsParameters,
+ preParameters,
+ htodStream, dtohStream, executeStream, context, *preprocessingFactory,
devA, devB));
}
- if (factories.coherentStokes) {
- coherentStep = std::unique_ptr<BeamFormerCoherentStep>(
- new BeamFormerCoherentStep(ps, queue, context, *factories.coherentStokes,
- devB));
- }
+ for (unsigned i = 0; i < bfPipelines.size(); i++)
+ {
+ if (coherentFactories[i]) {
+ coherentSteps[i].reset(
+ new BeamFormerCoherentStep(
+ obsParameters,
+ htodStream, dtohStream, executeStream, context, *coherentFactories[i],
+ devB, devD, devE));
+ }
- if (factories.incoherentStokes) {
- incoherentStep = std::unique_ptr<BeamFormerIncoherentStep>(
- new BeamFormerIncoherentStep(ps, queue, context, *factories.incoherentStokes,
- devA, devB));
+ if (incoherentFactories[i]) {
+ incoherentSteps[i].reset(
+ new BeamFormerIncoherentStep(
+ htodStream, dtohStream, executeStream, context, *incoherentFactories[i],
+ devA, devB, devE));
+ }
}
-
- LOG_INFO_STR("Pipeline configuration: "
- << (correlatorStep.get() ? "[correlator] " : "")
- << (preprocessingStep.get() ? "[bf preproc] " : "")
- << (coherentStep.get() ? "[coh stokes] " : "")
- << (incoherentStep.get() ? "[incoh stokes] " : "")
- );
+ LOG_INFO_STR("Pipeline configuration: ");
+ if (correlatorStep.get())
+ {
+ LOG_INFO_STR("[correlator]");
+ }
+ if (preprocessingStep.get())
+ {
+ std::stringstream info;
+ info << "[bf preproc]";
+ info << " " << ps.settings.beamFormer.antennaFieldNames.size() << " stations: [ ";
+ for (unsigned stationIdx : preParameters.obsStationIndices)
+ {
+ info << stationIdx << " ";
+ }
+ info << "]";
+ LOG_INFO_STR(info.str());
+ }
+ for (unsigned i = 0; i < bfPipelines.size(); i++)
+ {
+ std::stringstream info;
+ info << "[bf]";
+ auto& bfp = bfPipelines[i];
+ if (bfPipelines.size() > 1)
+ {
+ info << " " << i << ":";
+ }
+ if (coherentSteps[i].get())
+ {
+ info << " [coh stokes] ";
+ info << bfp.maxNrCoherentTABsPerSAP() << " TABs";
+ }
+ if (incoherentSteps[i].get())
+ {
+ info << " [incoh stokes] ";
+ info << bfp.maxNrIncoherentTABsPerSAP() << " TABs";
+ }
+ info << ", " << bfp.antennaFieldNames.size() << " stations: [ ";
+ ASSERTSTR(bfp.antennaFieldNames.size() == bfParameters[i]->preStationIndices.size(), "Mismatch in station selection");
+ for (unsigned stationIdx : bfParameters[i]->preStationIndices)
+ {
+ info << stationIdx << " ";
+ }
+ info << "]";
+ LOG_INFO_STR(info.str());
+ }
// put enough objects in the inputPool to operate
//
// At least 3 items are needed for a smooth Pool operation.
+ LOG_INFO_STR("Initializing input pool");
size_t nrInputDatas = std::max(3UL, 2 * nrSubbandsPerSubbandProc);
for (size_t i = 0; i < nrInputDatas; ++i) {
- inputPool.free.append(new SubbandProcInputData(ps, context), false);
+ std::shared_ptr<SubbandProcInputData> data(new SubbandProcInputData(ps, context));
+ inputPool.free.append(data, false);
}
-
+
// put enough objects in the outputPool to operate
+ LOG_INFO_STR("Initializing output pool");
for (size_t i = 0; i < nrOutputElements(); ++i)
{
- outputPool.free.append(new SubbandProcOutputData(ps, context), false);
+ std::shared_ptr<SubbandProcOutputData> data(new SubbandProcOutputData(ps, context));
+ outputPool.free.append(data, false);
}
}
@@ -150,6 +303,28 @@ namespace LOFAR
// Report how many subbands would yield up to 99% load
LOG_INFO_STR("[GPU] I can process at most " << static_cast<int>(floor(0.99 * blockDuration / averageGPURunTime)) << " subbands per GPU at real time.");
+
+ // Report kernel statistics for correlator step
+ LOG_INFO_STR("[GPU] Correlator step");
+ correlatorStep.reset();
+
+ // Report kernel statistics for preprocessing step
+ LOG_INFO_STR("[GPU] Preprocessing step");
+ preprocessingStep.reset();
+
+ // Report kernel statistics for coherent steps
+ for (unsigned int i = 0; i < coherentSteps.size(); i++)
+ {
+ LOG_INFO_STR("[GPU] Coherent step " << i);
+ coherentSteps[i].reset();
+ }
+
+ // Report kernel statistics for incoherent steps
+ for (unsigned int i = 0; i < incoherentSteps.size(); i++)
+ {
+ LOG_INFO_STR("[GPU] Incoherent step " << i);
+ incoherentSteps[i].reset();
+ }
}
@@ -175,18 +350,32 @@ namespace LOFAR
//*******************************************************************
// calculate some variables depending on the input subband
size_t block = input.blockID.block;
- unsigned SAP = ps.settings.subbands[input.blockID.globalSubbandIdx].SAP;
+ unsigned subbandNr = input.blockID.globalSubbandIdx;
+ auto &subband = ps.settings.subbands[subbandNr];
+ auto &obsSap = ps.settings.SAPs[subband.SAP];
- totalCounter.recordStart(queue);
+ totalCounter.recordStart(*executeStream);
+
+ //****************************************
+ // Wait for input buffer to be free
+ if (correlatorStep.get())
+ {
+ htodStream->waitEvent(correlatorStep->executeFinished);
+ }
+ if (preprocessingStep.get())
+ {
+ htodStream->waitEvent(preprocessingStep->executeFinished);
+ }
//****************************************
// Send inputs to GPU
- queue.writeBuffer(*devA, input.inputSamples, inputCounter, true);
+ htodStream->writeBuffer(*devA, input.inputSamples, inputCounter, true);
// Some additional buffers
// Only upload delays if they changed w.r.t. the previous subband.
- if ((int)SAP != prevSAP || (ssize_t)block != prevBlock) {
- if (correlatorStep.get()) {
+ bool uploadDelays = (ssize_t) block != prevBlock;
+ if (uploadDelays) {
+ if (obsSap.correlatorEnabled) {
correlatorStep->writeInput(input);
}
@@ -194,53 +383,73 @@ namespace LOFAR
preprocessingStep->writeInput(input);
}
- if (coherentStep.get()) {
- coherentStep->writeInput(input);
- }
-
- prevSAP = SAP;
prevBlock = block;
}
// ************************************************
- // Start the GPU processing
+ // Start the correlator step
- if (correlatorStep.get()) {
+ if (obsSap.correlatorEnabled) {
correlatorStep->process(input);
correlatorStep->readOutput(output);
}
+ // ************************************************
+ // Start the beamFormer step
+
if (preprocessingStep.get()) {
preprocessingStep->process(input);
}
- if (coherentStep.get())
- {
- coherentStep->process(input);
- coherentStep->readOutput(output);
- }
- if (incoherentStep.get())
+ for (unsigned pipelineNr = 0; pipelineNr < ps.settings.beamFormer.pipelines.size(); pipelineNr++)
{
- incoherentStep->process(input);
- incoherentStep->readOutput(output);
+ // Skip this beamFormer pipeline when the current subband is not specified
+ auto& bfSap = ps.settings.beamFormer.pipelines[pipelineNr].SAPs[subband.SAP];
+ if (std::find(bfSap.subbandIndices.begin(), bfSap.subbandIndices.end(), subbandNr) == bfSap.subbandIndices.end())
+ {
+ continue;
+ }
+
+ // Start coherent step
+ auto& csStep = coherentSteps[pipelineNr];
+ if (csStep.get()) {
+ if (uploadDelays) {
+ csStep->writeInput(*input.tabDelays[pipelineNr]);
+ }
+
+ if (csStep->quantizeOutput && correlatorStep.get()) {
+ executeStream->waitEvent(correlatorStep->outputFinished);
+ }
+
+ csStep->process(input);
+ csStep->readOutput(output.coherentData(pipelineNr));
+ }
+
+ // Start incoherent step
+ auto& isStep = incoherentSteps[pipelineNr];
+ if (isStep.get()) {
+ if (isStep->quantizeOutput && correlatorStep.get()) {
+ executeStream->waitEvent(correlatorStep->outputFinished);
+ }
+
+ isStep->process(input);
+ isStep->readOutput(output.incoherentData(pipelineNr));
+ }
}
- totalCounter.recordStop(queue);
+ totalCounter.recordStop(*executeStream);
// ************************************************
// Do CPU computations while the GPU is working
processCPUTimer.start();
- if (correlatorStep.get()) {
+ if (obsSap.correlatorEnabled) {
correlatorStep->processCPU(input, output);
}
processCPUTimer.stop();
-
- // Synchronise to assure that all the work in the data is done
- queue.synchronize();
}
@@ -252,6 +461,13 @@ namespace LOFAR
output.emit_correlatedData = false;
}
}
+
+ void SubbandProc::synchronize()
+ {
+ htodStream->synchronize();
+ executeStream->synchronize();
+ dtohStream->synchronize();
+ }
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.h
index f1b70855fc72abadc3cc927506437838ff63e7f0..21e7bd07b85684b35d859b19b75c8ea0f140e8b6 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProc.h
@@ -26,13 +26,11 @@
#include <complex>
#include <memory>
-#include <boost/shared_ptr.hpp>
#include <Common/LofarLogger.h>
#include <CoInterface/BudgetTimer.h>
#include <CoInterface/CorrelatedData.h>
#include <CoInterface/Parset.h>
#include <CoInterface/Pool.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/BlockID.h>
#include <CoInterface/Config.h>
#include <CoInterface/SubbandMetaData.h>
@@ -54,9 +52,6 @@ namespace LOFAR
{
namespace Cobalt
{
- //# Forward declarations
- struct KernelFactories;
-
/*
* The SubbandProc does the following transformation:
* SubbandProcInputData -> SubbandProcOutputData
@@ -72,7 +67,7 @@ namespace LOFAR
* The data flows as follows:
*
* // Fetch the next input object to fill
- * SmartPtr<SubbandProcInputData> input = queue.inputPool.free.remove();
+ * unique_ptr<SubbandProcInputData> input = queue.inputPool.free.remove();
*
* // Provide input
* receiveInput(input);
@@ -83,7 +78,7 @@ namespace LOFAR
* input->blockID.localSubbandIdx = subbandIdx;
*
* // Fetch the next output object to fill
- * SmartPtr<SubbandProcOutputData> output = queue.outputPool.free.remove();
+ * unique_ptr<SubbandProcOutputData> output = queue.outputPool.free.remove();
*
* // Process block
* queue.processSubband(input, output);
@@ -98,9 +93,10 @@ namespace LOFAR
*/
class SubbandProc {
public:
- SubbandProc(const Parset &ps, gpu::Context &context,
- KernelFactories &factories,
- size_t nrSubbandsPerSubbandProc = 1);
+ SubbandProc(
+ const Parset &ps,
+ gpu::Context &context,
+ size_t nrSubbandsPerSubbandProc = 1);
~SubbandProc();
@@ -121,34 +117,48 @@ namespace LOFAR
// Do post processing on the CPU.
void postprocessSubband(SubbandProcOutputData &output);
+ // Wait for any asynchronous operations to complete
+ void synchronize();
+
protected:
const Parset &ps;
+ gpu::Context &context;
const size_t nrSubbandsPerSubbandProc;
- gpu::Stream queue;
+ std::shared_ptr<gpu::Stream> htodStream;
+ std::shared_ptr<gpu::Stream> dtohStream;
+ std::shared_ptr<gpu::Stream> executeStream;
// The previously processed SAP/block, or -1 if nothing has been
// processed yet. Used in order to determine if new delays have
// to be uploaded.
ssize_t prevBlock;
- signed int prevSAP;
// @{
// Device memory buffers. These buffers are used interleaved. For details,
// please refer to the document bf-pipeline.txt in the directory
// GPUProc/doc.
- boost::shared_ptr<gpu::DeviceMemory> devA;
- boost::shared_ptr<gpu::DeviceMemory> devB;
+ std::shared_ptr<gpu::DeviceMemory> devA;
+ std::shared_ptr<gpu::DeviceMemory> devB;
+ std::shared_ptr<gpu::DeviceMemory> devD;
+ std::shared_ptr<gpu::DeviceMemory> devE;
// @}
PerformanceCounter totalCounter;
PerformanceCounter inputCounter;
BudgetTimer processCPUTimer;
+ // Kernel factories
+ std::unique_ptr<CorrelatorStep::Factories> correlatorFactory;
+ std::unique_ptr<BeamFormerPreprocessingStep::Factories> preprocessingFactory;
+ std::vector<std::unique_ptr<BeamFormerCoherentStep::Factories>> coherentFactories;
+ std::vector<std::unique_ptr<BeamFormerIncoherentStep::Factories>> incoherentFactories;
+
+ // Process steps
std::unique_ptr<CorrelatorStep> correlatorStep;
std::unique_ptr<BeamFormerPreprocessingStep> preprocessingStep;
- std::unique_ptr<BeamFormerCoherentStep> coherentStep;
- std::unique_ptr<BeamFormerIncoherentStep> incoherentStep;
+ std::vector<std::unique_ptr<BeamFormerCoherentStep>> coherentSteps;
+ std::vector<std::unique_ptr<BeamFormerIncoherentStep>> incoherentSteps;
// Returns the number of output elements to create to get a smooth
// running pipeline.
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.cc
index 4a01bdca66162ed159787dfa18903a2d01bb8bb4..5440d53d5c52b84255c577a1b6445290beaf57fc 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.cc
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.cc
@@ -28,28 +28,7 @@ namespace LOFAR
{
namespace Cobalt
{
- SubbandProcInputData::SubbandProcInputData(size_t n_beams, size_t n_stations,
- size_t n_polarizations, size_t n_coherent_tabs,
- size_t n_samples, size_t bytes_per_complex_sample,
- gpu::Context &context,
- unsigned int hostBufferFlags)
- :
- delaysAtBegin(boost::extents[n_beams][n_stations][n_polarizations],
- context, hostBufferFlags),
- delaysAfterEnd(boost::extents[n_beams][n_stations][n_polarizations],
- context, hostBufferFlags),
- phase0s(boost::extents[n_stations][n_polarizations],
- context, hostBufferFlags),
- tabDelays(boost::extents[n_beams][n_stations][n_coherent_tabs],
- context, hostBufferFlags),
- inputSamples(boost::extents[n_stations][n_samples][n_polarizations][bytes_per_complex_sample],
- context, hostBufferFlags), // TODO: The size of the buffer is NOT validated
- inputFlags(boost::extents[n_stations]),
- metaData(n_stations)
- {
- }
-
- // Short-hand constructor pulling all relevant values from a Parset
+ // Constructor pulling all relevant values from a Parset
SubbandProcInputData::SubbandProcInputData(const Parset &ps,
gpu::Context &context,
unsigned int hostBufferFlags)
@@ -60,15 +39,27 @@ namespace LOFAR
context, hostBufferFlags),
phase0s(boost::extents[ps.settings.antennaFields.size()][NR_POLARIZATIONS],
context, hostBufferFlags),
- tabDelays(boost::extents[ps.settings.SAPs.size()][ps.settings.antennaFields.size()][ps.settings.beamFormer.maxNrCoherentTABsPerSAP()],
- context, hostBufferFlags),
inputSamples(boost::extents[ps.settings.antennaFields.size()][ps.settings.blockSize][NR_POLARIZATIONS][ps.nrBytesPerComplexSample()],
context, hostBufferFlags), // TODO: The size of the buffer is NOT validated
inputFlags(boost::extents[ps.settings.antennaFields.size()]),
metaData(ps.settings.antennaFields.size())
{
- }
+ auto& bfPipelines = ps.settings.beamFormer.pipelines;
+ tabDelays.resize(bfPipelines.size());
+
+ for (unsigned i = 0; i < bfPipelines.size(); i++)
+ {
+ auto& bfPipeline = bfPipelines[i];
+
+ tabDelays[i].reset(
+ new MultiDimArrayHostBuffer<double, 3>(
+ boost::extents[bfPipeline.SAPs.size()][ps.settings.antennaFields.size()][bfPipeline.maxNrCoherentTABsPerSAP()],
+ context,
+ hostBufferFlags)
+ );
+ }
+ }
void SubbandProcInputData::applyMetaData(const Parset &ps,
unsigned station, unsigned SAP,
@@ -98,22 +89,42 @@ namespace LOFAR
double compensatedDelay = (metaData.stationBeam.delayAfterEnd +
metaData.stationBeam.delayAtBegin) * 0.5;
- size_t nrTABs = ps.settings.beamFormer.SAPs[SAP].nrCoherent;
-
- ASSERTSTR(metaData.TABs.size() == nrTABs, "Need delays for " << nrTABs << " coherent TABs, but got delays for " << metaData.TABs.size() << " TABs");
+ // Count the number of coherent TABs as a sanity check
+ size_t nrTABs = 0;
- // Note: We only get delays for the coherent TABs
- for (unsigned tab = 0; tab < nrTABs; tab++)
+ for (unsigned pipelineNr = 0; pipelineNr < ps.settings.beamFormer.pipelines.size(); pipelineNr++)
{
- // subtract the delay that was already compensated for
- tabDelays[SAP][station][tab] = (metaData.TABs[tab].delayAtBegin +
- metaData.TABs[tab].delayAfterEnd) * 0.5 -
- compensatedDelay;
+ auto& pipeline = ps.settings.beamFormer.pipelines[pipelineNr];
+
+ // The coherentIdx for this pipeline
+ unsigned coherentIdxInSAP = 0;
+
+ // Note: We only get delays for the coherent TABs
+ for (unsigned tabNr = 0; tabNr < pipeline.SAPs[SAP].TABs.size(); tabNr++)
+ {
+ auto& tab = pipeline.SAPs[SAP].TABs[tabNr];
+
+ if (tab.coherent)
+ {
+ // subtract the delay that was already compensated for
+ (*tabDelays[pipelineNr])[SAP][station][coherentIdxInSAP] = (metaData.TABs[tab.coherentIdxInSAP].delayAtBegin +
+ metaData.TABs[tab.coherentIdxInSAP].delayAfterEnd) * 0.5 -
+ compensatedDelay;
+ coherentIdxInSAP++;
+ nrTABs++;
+ }
+
+ ASSERTSTR(pipelineNr == tab.pipelineNr,
+ "Mismatch in pipeline index: " << pipelineNr << " != " << tab.pipelineNr);
+ }
+
+ // Zero padding entries that exist because we always produce maxNrCoherentTABsPerSAP for any subband
+ for (unsigned tab = pipeline.SAPs[SAP].TABs.size(); tab < pipeline.maxNrCoherentTABsPerSAP(); tab++)
+ (*tabDelays[pipelineNr])[SAP][station][tab] = 0.0;
}
- // Zero padding entries that exist because we always produce maxNrCoherentTABsPerSAP for any subband
- for (unsigned tab = nrTABs; tab < ps.settings.beamFormer.maxNrCoherentTABsPerSAP(); tab++)
- tabDelays[SAP][station][tab] = 0.0;
+ ASSERTSTR(nrTABs == ps.settings.beamFormer.SAPs[SAP].nrCoherent,
+ "Mismatch in coherent TAB count: " << nrTABs << " != " << ps.settings.beamFormer.SAPs[SAP].nrCoherent);
}
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.h
index d36c27877393bf6644f4f1bdbab12baa81147f66..6250168144ecbd893585d3605dad11839dfc0041 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcInputData.h
@@ -61,7 +61,7 @@ namespace LOFAR
MultiDimArrayHostBuffer<double, 2> phase0s;
//!< Delays for TABs (aka pencil beams) after station beam correction
- MultiDimArrayHostBuffer<double, 3> tabDelays;
+ std::vector<std::shared_ptr<MultiDimArrayHostBuffer<double, 3>>> tabDelays;
// inputdata with flagged data set to zero
MultiDimArrayHostBuffer<char, 4> inputSamples;
@@ -74,13 +74,6 @@ namespace LOFAR
// Create the inputData object we need shared host/device memory on the
// supplied devicequeue
- SubbandProcInputData(size_t n_beams, size_t n_stations,
- size_t n_polarizations, size_t n_coherent_tabs,
- size_t n_samples, size_t bytes_per_complex_sample,
- gpu::Context &context,
- unsigned int hostBufferFlags = 0);
-
- // Short-hand constructor pulling all relevant values from a Parset
SubbandProcInputData(const Parset &ps,
gpu::Context &context,
unsigned int hostBufferFlags = 0);
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.cc b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.cc
index 35e798dd22522ec31b8dccce1d9978d1851baead..534de151fe95b67fcd04cd9f4979fae7ee15af3f 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.cc
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.cc
@@ -29,21 +29,6 @@ namespace LOFAR
SubbandProcOutputData::SubbandProcOutputData(
const Parset &ps,
gpu::Context &context) :
- coherentData(ps.settings.beamFormer.anyCoherentTABs()
- ? boost::extents[ps.settings.beamFormer.maxNrCoherentTABsPerSAP()]
- [ps.settings.beamFormer.coherentSettings.nrStokes]
- [ps.settings.beamFormer.coherentSettings.nrSamples]
- [ps.settings.beamFormer.coherentSettings.nrChannels]
- : boost::extents[0][0][0][0],
- context, 0),
-
- incoherentData(ps.settings.beamFormer.anyIncoherentTABs()
- ? boost::extents[ps.settings.beamFormer.maxNrIncoherentTABsPerSAP()]
- [ps.settings.beamFormer.incoherentSettings.nrStokes]
- [ps.settings.beamFormer.incoherentSettings.nrSamples]
- [ps.settings.beamFormer.incoherentSettings.nrChannels]
- : boost::extents[0][0][0][0],
- context, 0),
correlatedData(ps.settings.correlator.enabled ? ps.settings.correlator.nrIntegrationsPerBlock : 0,
ps.settings.correlator.enabled ? ps.settings.antennaFields.size() : 0,
@@ -52,6 +37,34 @@ namespace LOFAR
context),
emit_correlatedData(false)
{
+ auto& bfPipelines = ps.settings.beamFormer.pipelines;
+
+ coherentDatas.resize(bfPipelines.size());
+ incoherentDatas.resize(bfPipelines.size());
+
+ for (unsigned i = 0; i < bfPipelines.size(); i++)
+ {
+ auto& bfPipeline = bfPipelines[i];
+ coherentDatas[i].reset(
+ new TABOutputData(
+ bfPipeline.coherentSettings.quantizerSettings.enabled,
+ bfPipeline.maxNrCoherentTABsPerSAP(),
+ bfPipeline.coherentSettings.nrStokes,
+ bfPipeline.coherentSettings.nrSamples,
+ bfPipeline.coherentSettings.nrChannels,
+ context
+ ));
+
+ incoherentDatas[i].reset(
+ new TABOutputData(
+ bfPipeline.incoherentSettings.quantizerSettings.enabled,
+ bfPipeline.maxNrIncoherentTABsPerSAP(),
+ bfPipeline.incoherentSettings.nrStokes,
+ bfPipeline.incoherentSettings.nrSamples,
+ bfPipeline.incoherentSettings.nrChannels,
+ context
+ ));
+ }
}
@@ -72,12 +85,120 @@ namespace LOFAR
for (size_t i = 0; i < nrIntegrations; ++i) {
const size_t num_elements = data.strides()[0];
- subblocks[i] = new LOFAR::Cobalt::CorrelatedData(
+ subblocks[i].reset(new LOFAR::Cobalt::CorrelatedData(
nrStations, nrChannels, maxNrValidSamples,
&data[i][0][0][0][0], num_elements,
- heapAllocator, 1);
+ heapAllocator, 1));
}
}
+
+
+
+ SubbandProcOutputData::TABOutputData::TABOutputData(
+ bool quantized,
+ long unsigned nrTABs,
+ unsigned nrStokes,
+ long unsigned nrSamples,
+ unsigned nrChannels,
+ gpu::Context &context)
+ :
+ quantized(quantized),
+ nrStokes(nrStokes),
+ data((!quantized ? boost::extents[nrTABs*nrStokes][nrSamples][nrChannels]
+ : boost::extents[0][0][0]),
+ context, 0),
+ qdata((quantized ? boost::extents[nrTABs*nrStokes][nrSamples][nrChannels]
+ : boost::extents[0][0][0]),
+ context, 0),
+ qoffsets((quantized ? boost::extents[nrTABs*nrStokes][nrChannels]
+ : boost::extents[0][0]),
+ context, 0),
+ qscales((quantized ? boost::extents[nrTABs*nrStokes][nrChannels]
+ : boost::extents[0][0]),
+ context, 0)
+ {
+ }
+
+ float SubbandProcOutputData::TABOutputData::get_data(
+ unsigned tab,
+ unsigned stokes,
+ unsigned sample,
+ unsigned channel) const
+ {
+ return data[tab * nrStokes + stokes][sample][channel];
+ }
+
+ float* SubbandProcOutputData::TABOutputData::get_data_origin(
+ unsigned tab,
+ unsigned stokes)
+ {
+ return data[tab * nrStokes + stokes].origin();
+ }
+
+ int8_t* SubbandProcOutputData::TABOutputData::get_qdata_origin(
+ unsigned tab,
+ unsigned stokes)
+ {
+ return qdata[tab * nrStokes + stokes].origin();
+ }
+
+ float* SubbandProcOutputData::TABOutputData::get_qoffsets_origin(
+ unsigned tab,
+ unsigned stokes)
+ {
+ return qoffsets[tab * nrStokes + stokes].origin();
+ }
+
+ float* SubbandProcOutputData::TABOutputData::get_qscales_origin(
+ unsigned tab,
+ unsigned stokes)
+ {
+ return qscales[tab * nrStokes + stokes].origin();
+ }
+
+ void SubbandProcOutputData::TABOutputData::resize(
+ long unsigned nrTABs,
+ long unsigned nrStokes)
+ {
+ if (!quantized)
+ {
+ data.resizeOneDimensionInplace(0, nrTABs * nrStokes);
+ } else {
+ qdata.resizeOneDimensionInplace(0, nrTABs * nrStokes);
+ qoffsets.resizeOneDimensionInplace(0, nrTABs * nrStokes);
+ qscales.resizeOneDimensionInplace(0, nrTABs * nrStokes);
+ }
+ }
+
+ size_t SubbandProcOutputData::TABOutputData::data_offset(
+ unsigned long tabStokes)
+ {
+ unsigned nrSamples = data.shape()[1];
+ unsigned nrChannels = data.shape()[2];
+ return tabStokes * nrSamples * nrChannels * sizeof(float);
+ }
+
+ size_t SubbandProcOutputData::TABOutputData::qdata_offset(
+ unsigned long tabStokes)
+ {
+ unsigned nrSamples = qdata.shape()[1];
+ unsigned nrChannels = qdata.shape()[2];
+ return tabStokes * nrSamples * nrChannels * sizeof(int8_t);
+ }
+
+ size_t SubbandProcOutputData::TABOutputData::qoffsets_offset(
+ unsigned long tabStokes)
+ {
+ unsigned nrChannels = qoffsets.shape()[1];
+ return tabStokes * nrChannels * sizeof(float);
+ }
+
+ size_t SubbandProcOutputData::TABOutputData::qscales_offset(
+ unsigned long tabStokes)
+ {
+ unsigned nrChannels = qscales.shape()[1];
+ return tabStokes * nrChannels * sizeof(float);
+ }
}
}
diff --git a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.h b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.h
index 11f5225727ac3d5622307fc15d54e65e2d151f58..ded976150936a183254b9dc1d895c65ff4dea70d 100644
--- a/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.h
+++ b/RTCP/Cobalt/GPUProc/src/SubbandProcs/SubbandProcOutputData.h
@@ -24,7 +24,6 @@
#include <CoInterface/BlockID.h>
#include <CoInterface/Parset.h>
#include <CoInterface/CorrelatedData.h>
-#include <CoInterface/SmartPtr.h>
#include <GPUProc/gpu_wrapper.h>
#include <GPUProc/MultiDimArrayHostBuffer.h>
@@ -41,8 +40,73 @@ namespace LOFAR
public:
struct BlockID blockID;
- MultiDimArrayHostBuffer<float, 4> coherentData;
- MultiDimArrayHostBuffer<float, 4> incoherentData;
+ struct TABOutputData
+ {
+ TABOutputData(
+ bool quantized,
+ long unsigned nrTABs,
+ unsigned nrStokes,
+ long unsigned nrSamples,
+ unsigned nrChannels,
+ gpu::Context &context);
+
+ float get_data(
+ unsigned tab,
+ unsigned stokes,
+ unsigned smaple,
+ unsigned channel) const;
+
+ bool is_quantized() const { return quantized; };
+
+ float* get_data_origin(
+ unsigned tab = 0,
+ unsigned stokes = 0);
+
+ int8_t* get_qdata_origin(
+ unsigned tab = 0,
+ unsigned stokes = 0);
+
+ float* get_qoffsets_origin(
+ unsigned tab = 0,
+ unsigned stokes = 0);
+
+ float* get_qscales_origin(
+ unsigned tab = 0,
+ unsigned stokes = 0);
+
+ size_t data_num_elements() const { return data.num_elements(); };
+ size_t qdata_num_elements() const { return qdata.num_elements(); };
+ size_t qoffsets_num_elements() const { return qoffsets.num_elements(); };
+ size_t qscales_num_elements() const { return qscales.num_elements(); };
+
+ void resize(
+ long unsigned nrTABs,
+ long unsigned nrStokes);
+
+ gpu::HostMemory& get_data() { return data; }
+ gpu::HostMemory& get_qdata() { return qdata; }
+ gpu::HostMemory& get_qoffsets() { return qoffsets; }
+ gpu::HostMemory& get_qscales() { return qscales; }
+
+ size_t data_offset(unsigned long tabStokes);
+ size_t qdata_offset(unsigned long tabStokes);
+ size_t qoffsets_offset(unsigned long tabStokes);
+ size_t qscales_offset(unsigned long tabStokes);
+
+ private:
+ bool quantized;
+ unsigned nrStokes;
+ MultiDimArrayHostBuffer<float, 3> data;
+ MultiDimArrayHostBuffer<int8_t, 3> qdata;
+ MultiDimArrayHostBuffer<float, 2> qoffsets;
+ MultiDimArrayHostBuffer<float, 2> qscales;
+ };
+
+ TABOutputData& coherentData(unsigned idx = 0) { return *coherentDatas[idx]; };
+ TABOutputData& incoherentData(unsigned idx = 0) { return *incoherentDatas[idx]; };
+
+ std::vector<std::shared_ptr<TABOutputData>> coherentDatas;
+ std::vector<std::shared_ptr<TABOutputData>> incoherentDatas;
struct CorrelatedData
{
@@ -53,7 +117,7 @@ namespace LOFAR
gpu::Context &context);
MultiDimArrayHostBuffer<fcomplex, 5> data;
- std::vector< SmartPtr<LOFAR::Cobalt::CorrelatedData> > subblocks;
+ std::vector<std::unique_ptr<LOFAR::Cobalt::CorrelatedData>> subblocks;
};
CorrelatedData correlatedData;
diff --git a/RTCP/Cobalt/GPUProc/src/gpu_load.cc b/RTCP/Cobalt/GPUProc/src/gpu_load.cc
index f3df161f40c566d5412a59b33e2be1d85fde78a2..52d818a71f808ed36d45b2ead615d5a5baa98275 100644
--- a/RTCP/Cobalt/GPUProc/src/gpu_load.cc
+++ b/RTCP/Cobalt/GPUProc/src/gpu_load.cc
@@ -26,19 +26,27 @@
#include <CoInterface/fpequals.h>
#include <CoInterface/Parset.h>
#include <GPUProc/gpu_utils.h>
-#include <GPUProc/SubbandProcs/KernelFactories.h>
#include <GPUProc/SubbandProcs/SubbandProc.h>
+#include <GPUProc/PerformanceCounter.h>
+
+#include <fstream>
+#include <ctime>
using namespace std;
using namespace LOFAR;
using namespace LOFAR::Cobalt;
int main(int argc, char **argv) {
- if (argc != 2) {
- cout << "Usage: " << argv[0] << " parset" << endl;
+ if ((argc < 2) || (argc > 3)) {
+ cout << "Usage: " << argv[0] << " parset" << " and (optional) [int] nr of iterations" << endl;
return 1;
}
+ uint nIterations = 10; // Default value
+ if (argc == 3){ // Overwrite default
+ nIterations = strToUint(argv[2]);
+ }
+
try {
gpu::Platform pf;
cout << "Detected " << pf.size() << " CUDA devices" << endl;
@@ -62,7 +70,6 @@ int main(int argc, char **argv) {
const size_t maxNrTABsPerSAP = ps.settings.beamFormer.maxNrCoherentTABsPerSAP();
const size_t nrSamplesPerSubband = ps.settings.blockSize;
const size_t nrBitsPerSample = ps.settings.nrBitsPerSample;
- const size_t nrBytesPerComplexSample = ps.nrBytesPerComplexSample();
const fcomplex inputValue(1,1);
// Output info
@@ -70,17 +77,15 @@ int main(int argc, char **argv) {
const size_t nrBlocksPerIntegration =
ps.settings.correlator.enabled ? ps.settings.correlator.nrBlocksPerIntegration : 1;
const size_t nrChannelsPerSubband = ps.settings.correlator.nrChannels;
- const size_t integrationSteps = ps.settings.correlator.nrSamplesPerIntegration();
+ const size_t integrationSteps =
+ ps.settings.correlator.enabled ? ps.settings.correlator.nrSamplesPerIntegration() : 1;
// Assume each node has as many GPUs as us.
const size_t nrSubbandsPerSubbandProc = ceilDiv(ceilDiv(ps.settings.subbands.size(), ps.settings.nodes.size()), devices.size());
- KernelFactories factories(ps, nrSubbandsPerSubbandProc);
- SubbandProc cwq(ps, ctx, factories, nrSubbandsPerSubbandProc);
+ SubbandProc cwq(ps, ctx, nrSubbandsPerSubbandProc);
- SubbandProcInputData in(
- nrBeams, nrStations, nrPolarisations, maxNrTABsPerSAP,
- nrSamplesPerSubband, nrBytesPerComplexSample, ctx);
+ SubbandProcInputData in(ps, ctx);
SubbandProcOutputData out(ps, ctx);
@@ -106,10 +111,34 @@ int main(int argc, char **argv) {
"\n Total bytes = " << out.correlatedData.subblocks[0]->visibilities.size() * sizeof(fcomplex)); // TODO: wrong!
}
+ // If enabled, we log GPU kernel performance counters to a benchmark file in csv format.
+ // Here we create the file (or overwrite old) for logging of benchmark data and we print metadata.
+ // The user should make sure that the path to the file exists, otherwise we log a warning
+ // and don't enable csv logging of performance counters.
+ if(ps.settings.cobalt.benchmark.enabled)
+ {
+ string benchmarkFilePath = ps.settings.cobalt.benchmark.file;
+ ofstream ofs (benchmarkFilePath, ofstream::trunc);
+ if(ofs.good())
+ {
+ time_t now = time(nullptr);
+ ofs << asctime(localtime(&now));
+ ofs << "info; Test gpu_load: " << argv[0] << " with parset: " << argv[1] << " with nIterations: " << nIterations << endl;
+ ofs << "format; kernelName; count; mean; stDev; min; max; unit" << endl;
+ ofs.close();
+ // enable and set file path for logging of benchmark data from FerformanceCounters
+ PerformanceCounter::enableCSVLog(benchmarkFilePath);
+ }
+ else
+ {
+ LOG_WARN_STR("gpu_load: Logging of benchmark data was enabled, but failed to open file: " << benchmarkFilePath);
+ }
+ }
+
// Initialize subbands partitioning administration (struct BlockID). We only
// do the 1st block of whatever.
LOG_INFO("Processing ...");
- for(size_t iteration = 0; iteration < 10; iteration++) {
+ for(uint iteration = 0; iteration < nIterations; iteration++) {
in.blockID.block = 0;
in.blockID.globalSubbandIdx = 0;
in.blockID.localSubbandIdx = 0;
diff --git a/RTCP/Cobalt/GPUProc/src/gpu_load_lmm.cc b/RTCP/Cobalt/GPUProc/src/gpu_load_lmm.cc
new file mode 100644
index 0000000000000000000000000000000000000000..7add525771d9dcde26faa3ba73dafbf64f936609
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/src/gpu_load_lmm.cc
@@ -0,0 +1,142 @@
+//# gpu_load.cc
+//#
+//# Copyright (C) 2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include <Common/LofarLogger.h>
+#include <Common/LofarTypes.h>
+#include <CoInterface/fpequals.h>
+#include <CoInterface/Parset.h>
+#include <GPUProc/gpu_utils.h>
+#include <GPUProc/SubbandProcs/SubbandProc.h>
+#include <GPUProc/PerformanceCounter.h>
+
+#include <fstream>
+#include <ctime>
+
+using namespace std;
+using namespace LOFAR;
+using namespace LOFAR::Cobalt;
+
+int main(int argc, char **argv) {
+ if (argc != 2) {
+ cout << "Usage: " << argv[0] << " parset" << endl;
+ return 1;
+ }
+
+ try {
+ gpu::Platform pf;
+ cout << "Detected " << pf.size() << " CUDA devices" << endl;
+ } catch (gpu::CUDAException& e) {
+ cerr << e.what() << endl;
+ return 1;
+ }
+
+ INIT_LOGGER("gpu_load_lmm");
+
+ Parset ps(argv[1]);
+
+ gpu::Device device(0);
+ vector<gpu::Device> devices(1, device);
+ gpu::Context ctx(device);
+
+ // Input info
+ const size_t nrBeams = ps.settings.SAPs.size();
+ const size_t nrStations = ps.settings.antennaFields.size();
+ const size_t nrPolarisations = ps.settings.nrPolarisations;
+ const size_t maxNrTABsPerSAP = ps.settings.beamFormer.maxNrCoherentTABsPerSAP();
+ const size_t nrSamplesPerSubband = ps.settings.blockSize;
+ const size_t nrBitsPerSample = ps.settings.nrBitsPerSample;
+ const fcomplex inputValue(1,1);
+
+ // Output info
+ const size_t nrBaselines = nrStations * (nrStations + 1) / 2;
+ const size_t nrBlocksPerIntegration =
+ ps.settings.correlator.enabled ? ps.settings.correlator.nrBlocksPerIntegration : 1;
+ const size_t nrChannelsPerSubband = ps.settings.correlator.nrChannels;
+ const size_t integrationSteps =
+ ps.settings.correlator.enabled ? ps.settings.correlator.nrSamplesPerIntegration() : 1;
+
+ // Assume each node has as many GPUs as us.
+ const size_t nrSubbandsPerSubbandProc = ceilDiv(ceilDiv(ps.settings.subbands.size(), ps.settings.nodes.size()), devices.size());
+
+ SubbandProc cwq(ps, ctx, nrSubbandsPerSubbandProc);
+
+ SubbandProcInputData in(ps, ctx);
+
+ SubbandProcOutputData out(ps, ctx);
+
+ LOG_INFO_STR(
+ "\nInput info:" <<
+ "\n nrBeams = " << nrBeams <<
+ "\n nrStations = " << nrStations <<
+ "\n nrPolarisations = " << nrPolarisations <<
+ "\n maxNrTABsPerSAP = " << maxNrTABsPerSAP <<
+ "\n nrSamplesPerSubband = " << nrSamplesPerSubband <<
+ "\n nrBitsPerSample = " << nrBitsPerSample <<
+ "\n ----------------------------" <<
+ "\n Total bytes = " << in.inputSamples.size());
+
+ if (ps.settings.correlator.enabled) {
+ LOG_INFO_STR(
+ "\nOutput info:" <<
+ "\n nrBaselines = " << nrBaselines <<
+ "\n nrBlockPerIntegration = " << nrBlocksPerIntegration <<
+ "\n nrChannelsPerSubband = " << nrChannelsPerSubband <<
+ "\n integrationSteps = " << integrationSteps <<
+ "\n ----------------------------" <<
+ "\n Total bytes = " << out.correlatedData.subblocks[0]->visibilities.size() * sizeof(fcomplex)); // TODO: wrong!
+ }
+
+ // Initialize subbands partitioning administration (struct BlockID). We only
+ // do the 1st block for all SAPs (beams) and subbands. This way the load is representative
+ // for the load in an LMM observation where the global list of subbands (i.e. the list of subbands
+ // for all SAPs) is distributed in round-robin fashion over the GPUs.
+ in.blockID.block = 0;
+ in.blockID.globalSubbandIdx = 0;
+ in.blockID.localSubbandIdx = 0;
+ in.blockID.subbandProcSubbandIdx = 0;
+ out.emit_correlatedData = false;
+
+ LOG_INFO("Processing ...");
+ for (size_t block = 0; block < nrBlocksPerIntegration && !out.emit_correlatedData; block++) {
+ LOG_INFO_STR("Processing block #" << block);
+
+ for (size_t sapIdx = 0; sapIdx < ps.settings.SAPs.size(); sapIdx++)
+ {
+ LOG_INFO_STR("Processing SAP #" << sapIdx);
+ auto& obsSAP = ps.settings.SAPs[sapIdx];
+
+ // Select the first subband of the current SAP
+ for (auto subbandIdx : obsSAP.subbandIndices())
+ {
+ in.blockID.globalSubbandIdx = subbandIdx;
+
+ // Start processing
+ cwq.processSubband(in, out);
+ cwq.postprocessSubband(out);
+ }
+ }
+ }
+
+ return 0;
+}
+
diff --git a/RTCP/Cobalt/GPUProc/src/gpu_utils.cc b/RTCP/Cobalt/GPUProc/src/gpu_utils.cc
index 92a37f20027d6daf17682d7ffb6237fe99bcd6fe..4ac8fa37fd44e9893c36192f1eb1c8e1338290b8 100644
--- a/RTCP/Cobalt/GPUProc/src/gpu_utils.cc
+++ b/RTCP/Cobalt/GPUProc/src/gpu_utils.cc
@@ -38,6 +38,8 @@
#include "cuda_config.h"
+#include <iterator>
+#include <nvrtc.h>
namespace LOFAR
{
namespace Cobalt
@@ -239,30 +241,135 @@ namespace LOFAR
return os;
}
- string doCreatePTX(const string& source,
- const CompileFlags& flags,
+
+#define checkNVRTCCall(func) \
+ do { \
+ nvrtcResult result = func; \
+ if (result != NVRTC_SUCCESS) { \
+ THROW (GPUProcException, \
+ # func << ": " << nvrtcGetErrorString(result)); \
+ } \
+ } while(0)
+
+ string doCreatePTXusingNVRTC(const string& source,
+ const CompileFlags& flags,
+ const CompileDefinitions& defs)
+ {
+ LOG_INFO_STR("Starting runtime compilation:\n\t"
+ << source << flags << defs);
+
+ // Append compile flags and compile definitions to get compile options
+ std::stringstream options_stream;
+ options_stream << flags << defs;
+ istringstream iss(options_stream.str());
+
+ // Create vector of compile options with strings
+ vector<string> options_vector{istream_iterator<string>{iss}, istream_iterator<string>{}};
+
+ // Add the kernel directory as include directory
+ std::string include_flags("--include-path=" + prefixPath());
+ options_vector.push_back(include_flags);
+ options_vector.push_back("--include-path=/usr/include");
+ options_vector.push_back("--include-path=/usr/local/cuda/include");
+ options_vector.push_back("--device-c");
+ options_vector.push_back("-rdc=true");
+ options_vector.push_back("-D__x86_64__");
+ options_vector.push_back("-D__CUDACC__");
+ options_vector.push_back("-default-device");
+
+ // Create vector of compile options with C-style strings
+ vector<const char*> options_c_vector;
+ options_c_vector.reserve(options_vector.size());
+ for (auto &option : options_vector) {
+ options_c_vector.push_back(option.c_str());
+ }
+
+ // Open source file
+ std::ifstream input_stream;
+ input_stream.open(source, std::ifstream::in);
+ if (input_stream.fail()) {
+ THROW(GPUProcException, "Failed to open file:\n\t" << source);
+ }
+ std::ostringstream file_contents;
+ file_contents << input_stream.rdbuf();
+ input_stream.close();
+
+ // Try to compile source into NVRTC program
+ nvrtcProgram prog;
+ checkNVRTCCall(nvrtcCreateProgram(&prog, file_contents.str().c_str(), source.c_str(), 0, NULL, NULL));
+ nvrtcResult compilerResult = nvrtcCompileProgram(prog, options_c_vector.size(), &options_c_vector[0]);
+
+ // Print compilation log (if any)
+ size_t logSize;
+ checkNVRTCCall(nvrtcGetProgramLogSize(prog, &logSize));
+ if (logSize > 1) // ignore logs with only trailing NULL
+ {
+ char log[logSize];
+ checkNVRTCCall(nvrtcGetProgramLog(prog, &log[0]));
+ LOG_INFO_STR(log);
+ }
+
+ // Make sure that compilation succeeded
+ checkNVRTCCall(compilerResult);
+
+ size_t ptxSize;
+ checkNVRTCCall(nvrtcGetPTXSize(prog, &ptxSize));
+ if (!ptxSize) {
+ THROW(GPUProcException, "PTX empty!\n\t" << source);
+ }
+
+ // Get PTX
+ char ptx_c[ptxSize];
+ checkNVRTCCall(nvrtcGetPTX(prog, &ptx_c[0]));
+ string ptx_string(ptx_c);
+
+ // Destroy the program
+ checkNVRTCCall(nvrtcDestroyProgram(&prog));
+
+ return ptx_string;
+ }
+
+ string doCreatePTXusingNVCC(const string& source,
+ const CompileFlags& flags_,
const CompileDefinitions& defs)
{
+ // Add NVCC specific compiler flags
+ CompileFlags flags = flags_;
+ flags.insert("-ptx");
+ flags.insert("-O3");
+ flags.insert(str(format("-I%s") % includePath()));
+
+ // Try to compile source to PTX and store the result in a temporary file
+ char tmpfile_cname[] = "/tmp/cobalt_nvcc_ptx.XXXXXX";
+ int tmpfd=mkstemp(tmpfile_cname);
+ std::string tmpfile_name(tmpfile_cname);
+ if (tmpfd==-1) {
+ THROW(GPUProcException, "Temporary file "<<tmpfile_name<<" creation failed!");
+ }
+ close(tmpfd);
// TODO: first try 'nvcc', then this path.
- ostringstream oss;
- oss << CUDA_TOOLKIT_ROOT_DIR << "/bin/nvcc " << source << flags << defs;
- string cmd(oss.str());
+ stringstream cmd_stream;
+ cmd_stream << CUDA_TOOLKIT_ROOT_DIR << "/bin/nvcc " << source << flags << defs << " -o " << tmpfile_name;
+ string cmd(cmd_stream.str());
LOG_INFO_STR("Starting runtime compilation:\n\t" << cmd);
-
- string ptx;
- char buffer [1024];
- FILE * stream = popen(cmd.c_str(), "r");
- if (!stream) {
- THROW_SYSCALL("popen");
- }
- while (!feof(stream)) { // NOTE: We do not get stderr (TODO)
- if (fgets(buffer, sizeof buffer, stream) != NULL) {
- ptx += buffer;
- }
+ if (system(cmd.c_str()) != 0) {
+ THROW(GPUProcException, "Runtime compilation failed!\n\t" << cmd);
}
- if (pclose(stream) || ptx.empty()) {
+
+ // Try to open the result
+ std::ifstream input_stream;
+ input_stream.open(tmpfile_name, std::ifstream::in);
+ if (input_stream.fail()) {
THROW(GPUProcException, "Runtime compilation failed!\n\t" << cmd);
}
+
+ // Read and return contents
+ std::ostringstream file_contents;
+ file_contents << input_stream.rdbuf();
+ input_stream.close();
+ std::string ptx = file_contents.str();
+ // Remove tmpfile
+ unlink(tmpfile_cname);
return ptx;
}
@@ -278,26 +385,28 @@ namespace LOFAR
CompileFlags defaultCompileFlags()
{
CompileFlags flags;
- flags.insert("-o /dev/stdout");
- flags.insert("-ptx");
// For now, keep optimisations the same to detect changes in
// output with reference.
flags.insert("--restrict");
- flags.insert("-O3");
- flags.insert(str(format("-I%s") % includePath()));
+ // Enable source-file mappings to aid profiling
+ // This flag slightly increases compilation times and size
+ // of the generated PTX, but does not affect kernel performance.
+ flags.insert("-lineinfo");
+
return flags;
}
string createPTX(string srcFilename,
CompileDefinitions definitions,
CompileFlags flags,
- const vector<gpu::Device> &devices)
+ const vector<gpu::Device> &devices,
+ bool useNVRTC)
{
// The CUDA code is assumed to be written for the architecture of the
// oldest device.
- flags.insert(str(format("--gpu-architecture %s") %
+ flags.insert(str(format("--gpu-architecture=%s") %
get_virtarch(computeTarget(devices))));
// Add default definitions and flags
@@ -329,7 +438,17 @@ namespace LOFAR
srcFilename = prefixPath() + "/" + srcFilename;
}
- return doCreatePTX(srcFilename, flags, definitions);
+ // If useNVRTC=false, check env variable to see
+ // if it is still enabled in runtime
+ if (!useNVRTC) {
+ useNVRTC=static_cast<bool>(getenv("COBALT_USE_NVRTC"));
+ }
+ if (useNVRTC)
+ {
+ return doCreatePTXusingNVRTC(srcFilename, flags, definitions);
+ } else {
+ return doCreatePTXusingNVCC(srcFilename, flags, definitions);
+ }
}
diff --git a/RTCP/Cobalt/GPUProc/src/gpu_utils.h b/RTCP/Cobalt/GPUProc/src/gpu_utils.h
index f378eadda89a224f57bf74db8df4fff17ce47d32..2f94850a78e62a60e49677669f05de22644ab07c 100644
--- a/RTCP/Cobalt/GPUProc/src/gpu_utils.h
+++ b/RTCP/Cobalt/GPUProc/src/gpu_utils.h
@@ -71,6 +71,7 @@ namespace LOFAR
// will be selected as target for the PTX code. If no devices are
// given, then the program is compiled for the least capable device
// that is available on the current platform.
+ // \par useNVRCT : if true, use NVRTC for PTX generation instead of NVCC
// \note The arguments \a srcFilename, \a flags and \a definitions are
// passed by value intentionally, because they will be modified by
// this method.
@@ -78,7 +79,8 @@ namespace LOFAR
createPTX(std::string srcFilename,
CompileDefinitions definitions = CompileDefinitions(),
CompileFlags flags = CompileFlags(),
- const GPUDevices &devices = gpu::Platform().devices());
+ const GPUDevices &devices = gpu::Platform().devices(),
+ bool useNVRTC = false);
// Create a Module from a PTX (string).
// \par context The context that the Module should be associated with.
diff --git a/RTCP/Cobalt/GPUProc/src/gpu_wrapper.cc b/RTCP/Cobalt/GPUProc/src/gpu_wrapper.cc
index ac82561b7bf561a398615ce6e787258dff44c13a..923c9a4dbf55e9dfbe68c9b5a73d87a259207514 100644
--- a/RTCP/Cobalt/GPUProc/src/gpu_wrapper.cc
+++ b/RTCP/Cobalt/GPUProc/src/gpu_wrapper.cc
@@ -1091,6 +1091,38 @@ namespace LOFAR
counter.recordStop(*this);
}
+ void Stream::readBuffer(const HostMemory &hostMem, const DeviceMemory &devMem,
+ size_t offset,
+ PerformanceCounter &counter, bool synchronous) const
+ {
+ size_t hostOffset = 0;
+ size_t devOffset = offset;
+ size_t size = std::min(hostMem.size()-hostOffset, devMem.size()-devOffset);
+ readBuffer(hostMem, devMem, hostOffset, devOffset, size, counter, synchronous);
+ }
+
+ void Stream::readBuffer(const HostMemory &hostMem, const DeviceMemory &devMem,
+ size_t hostOffset, size_t devOffset, size_t size,
+ PerformanceCounter &counter, bool synchronous) const
+ {
+ counter.recordStart(*this);
+
+ ASSERT(devMem.size()>devOffset);
+ ASSERT(hostMem.size()>hostOffset);
+ ASSERT(devMem.size()-devOffset >= size);
+ ASSERT(hostMem.size()-hostOffset >= size);
+
+ _impl->memcpyDtoHAsync(
+ static_cast<void *>(static_cast<unsigned char*>(hostMem.get<void>())+hostOffset),
+ (CUdeviceptr)(static_cast<unsigned char*>(devMem.get())+devOffset),
+ size);
+
+ if (synchronous || force_synchronous)
+ {
+ synchronize();
+ }
+ counter.recordStop(*this);
+ }
void Stream::launchKernel(const Function &function,
const Grid &grid, const Block &block) const
@@ -1142,6 +1174,53 @@ namespace LOFAR
return force_synchronous;
}
+ Marker::Marker(
+ const char *message,
+ Color color)
+ {
+ _attributes.version = NVTX_VERSION;
+ _attributes.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
+ _attributes.colorType = NVTX_COLOR_ARGB;
+ _attributes.color = convert(color);
+ _attributes.messageType = NVTX_MESSAGE_TYPE_ASCII;
+ _attributes.message.ascii = message;
+ }
+
+ void Marker::start()
+ {
+ _id = nvtxRangeStartEx(&_attributes);
+ }
+
+ void Marker::end()
+ {
+ nvtxRangeEnd(_id);
+ }
+
+ void Marker::start(
+ gpu::Event& event)
+ {
+ event.wait();
+ start();
+ }
+
+ void Marker::end(
+ gpu::Event& event)
+ {
+ event.wait();
+ end();
+ }
+
+ unsigned int Marker::convert(Color color)
+ {
+ switch (color) {
+ case red : return 0xffff0000;
+ case green : return 0xff00ff00;
+ case blue : return 0xff0000ff;
+ case yellow : return 0xffffff00;
+ case black : return 0xff000000;
+ default: return 0xff00ff00;
+ }
+ }
} // namespace gpu
} // namespace Cobalt
diff --git a/RTCP/Cobalt/GPUProc/src/gpu_wrapper.h b/RTCP/Cobalt/GPUProc/src/gpu_wrapper.h
index 6f19a4f10ee0ce013a963d634ad30f6320953b38..3eaf6da831abcda71b4aedeb70e8f6777e1a9652 100644
--- a/RTCP/Cobalt/GPUProc/src/gpu_wrapper.h
+++ b/RTCP/Cobalt/GPUProc/src/gpu_wrapper.h
@@ -36,9 +36,9 @@
#include <map>
#include <iosfwd>
-#include <boost/shared_ptr.hpp>
#include "gpu_incl.h" // ideally, this goes into the .cc, but too much leakage
#include <cufft.h>
+#include <nvToolsExt.h>
#include <Common/Exception.h>
#include <GPUProc/gpu_wrapper.h> // GPUException
@@ -254,7 +254,7 @@ namespace LOFAR
class Impl;
// Reference counted pointer to the implementation class.
- boost::shared_ptr<Impl> _impl;
+ std::shared_ptr<Impl> _impl;
friend class ScopedCurrentContext;
};
@@ -311,7 +311,7 @@ namespace LOFAR
class Impl;
// Reference counted pointer to the implementation class.
- boost::shared_ptr<Impl> _impl;
+ std::shared_ptr<Impl> _impl;
};
@@ -348,7 +348,7 @@ namespace LOFAR
class Impl;
// Reference counted pointer to the implementation class.
- boost::shared_ptr<Impl> _impl;
+ std::shared_ptr<Impl> _impl;
};
@@ -401,7 +401,7 @@ namespace LOFAR
class Impl;
// Reference counted pointer to the implementation class.
- boost::shared_ptr<Impl> _impl;
+ std::shared_ptr<Impl> _impl;
};
// Wrap a CUDA Device Function. This is the equivalent of an OpenCL
@@ -504,7 +504,7 @@ namespace LOFAR
class Impl;
// Reference counted pointer to the implementation class.
- boost::shared_ptr<Impl> _impl;
+ std::shared_ptr<Impl> _impl;
};
@@ -560,6 +560,18 @@ namespace LOFAR
void readBuffer(const HostMemory &hostMem, const DeviceMemory &devMem,
PerformanceCounter &counter, bool synchronous = false) const;
+ // \param offset: read from devMem+offset
+ void readBuffer(const HostMemory &hostMem, const DeviceMemory &devMem,
+ size_t offset,
+ PerformanceCounter &counter, bool synchronous = false) const;
+
+ // \param devOffset: read from devMem+devOffset
+ // \param hostOffset: write to hostMem+hostOffset
+ void readBuffer(const HostMemory &hostMem, const DeviceMemory &devMem,
+ size_t hostOffset, size_t devOffset, size_t size,
+ PerformanceCounter &counter, bool synchronous = false) const;
+
+
// Transfer data from device memory \a devSource to device memory \a devTarget.
// \param devTarget Device memory that will be copied to.
// \param devSource Device memory that will be copied from.
@@ -614,12 +626,52 @@ namespace LOFAR
class Impl;
// Reference counted pointer to the implementation class.
- boost::shared_ptr<Impl> _impl;
+ std::shared_ptr<Impl> _impl;
// Force synchronous transfers and kernel launches
bool force_synchronous;
};
+
+ // Wrap a NVTX range. This class helps to annotate the timeline
+ // in the NVIDIA Visual Profiler.
+ class Marker {
+ public:
+
+ // A number of predefined colors. These values
+ // can be converted to the format used in NVTX
+ // by using the convert() function.
+ enum Color {
+ red , green, blue, yellow, black
+ };
+
+ // Create a marker
+ Marker(
+ const char *message,
+ Marker::Color color = Color::red);
+
+ // Mark the begininng of the range
+ void start();
+
+ // Mark the end of the range
+ void end();
+
+ // Mark the begininng of the range,
+ // after waiting for the event
+ void start(
+ gpu::Event& event);
+
+ // Mark the end of the range,
+ // after waiting for the event
+ void end(
+ gpu::Event& event);
+
+ private:
+ unsigned int convert(Color color);
+ nvtxEventAttributes_t _attributes;
+ nvtxRangeId_t _id;
+ };
+
} // namespace gpu
} // namespace Cobalt
} // namespace LOFAR
diff --git a/RTCP/Cobalt/GPUProc/src/rtcp.cc b/RTCP/Cobalt/GPUProc/src/rtcp.cc
index 507f134ad52302c42bb2abf48e33df9bf6cfff0c..2295118bf2b8a54bd36210c156c2be59e8c64742 100644
--- a/RTCP/Cobalt/GPUProc/src/rtcp.cc
+++ b/RTCP/Cobalt/GPUProc/src/rtcp.cc
@@ -353,32 +353,32 @@ int main(int argc, char **argv)
ps.settings.blockSize,
ps.settings.antennaFields.size(),
ps.nrBitsPerSample());
-
- SmartPtr<Pipeline> pipeline;
+
+ std::unique_ptr<Pipeline> pipeline;
// Creation of pipelines cause fork/exec, which we need to
// do before we start doing anything fancy with libraries and threads.
if ((ps.settings.correlator.enabled || ps.settings.beamFormer.enabled) &&
!subbandIndices.empty()) {
- pipeline = new Pipeline(ps, subbandIndices, devices,
- MPI_receive_pool, mdLogger, mdKeyPrefix, mpi.rank());
+ pipeline.reset(new Pipeline(ps, subbandIndices, devices,
+ MPI_receive_pool, mdLogger, mdKeyPrefix, mpi.rank()));
} else {
// RSP raw data output or piggy-backing only, or software test without pipeline
- pipeline = NULL;
- }
+ pipeline.reset(nullptr);
+ }
// After pipeline creation (post-fork()), allow creation of a thread to send
// data points for monitoring (PVSS).
//mdLogger.start();
// Only ONE host should start the Storage processes
- SmartPtr<StorageProcesses> storageProcesses;
+ std::unique_ptr<StorageProcesses> storageProcesses;
if (mpi.rank() == 0) {
LOG_INFO("----- Starting OutputProc");
// NOTE: This fork()s.
- storageProcesses = new StorageProcesses(ps, "");
+ storageProcesses.reset(new StorageProcesses(ps, ""));
}
/*
@@ -438,7 +438,7 @@ int main(int argc, char **argv)
LOG_INFO_STR("Processing subbands " << subbandDistribution[mpi.rank()]);
Trigger stopSwitch;
- SmartPtr<CommandThread> commandThread;
+ std::unique_ptr<CommandThread> commandThread;
#pragma omp parallel sections num_threads(2)
{
@@ -466,7 +466,7 @@ int main(int argc, char **argv)
OMPThread::ScopedName sn("CommandThr bcast");
if (mpi.rank() == 0) {
- commandThread = new CommandThread(ps.settings.commandStream);
+ commandThread.reset(new CommandThread(ps.settings.commandStream));
}
string command;
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/CMakeLists.txt b/RTCP/Cobalt/GPUProc/test/Kernels/CMakeLists.txt
index b57f2cf655f81ab8357305828dc74b88a7b0b568..241faed571f0aa34d005f7b7d596146fa9e28fef 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/CMakeLists.txt
@@ -36,6 +36,8 @@ if(UNITTEST++_FOUND AND BUILD_TESTING)
lofar_add_test(tKernelFunctions tKernelFunctions.cc)
lofar_add_test(tCoherentStokesKernel
tCoherentStokesKernel.cc KernelTestHelpers.cc)
+ lofar_add_test(tQuantizeOutputKernel
+ tQuantizeOutputKernel.cc KernelTestHelpers.cc)
set_tests_properties(
tCorrelatorKernel
@@ -45,6 +47,7 @@ if(UNITTEST++_FOUND AND BUILD_TESTING)
tFFTShiftKernel
tKernelFunctions
tCoherentStokesKernel
+ tQuantizeOutputKernel
tZeroingKernel
PROPERTIES ENVIRONMENT "LOFARROOT=${PACKAGE_SOURCE_DIR}"
)
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.cc b/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.cc
index 1d23506ce405a056a0641df6a687cfc10b97afc9..6ac155d516ffe9525d3a41d2c2e2032c40a03c30 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.cc
@@ -51,7 +51,7 @@ void usage(char const *testName)
}
-KernelParameters::KernelParameters()
+KernelTestParameters::KernelTestParameters()
{
nrTabs = 127;
nrChannels = 64;
@@ -65,7 +65,7 @@ KernelParameters::KernelParameters()
timeIntegrationFactor = 1;
}
-void KernelParameters::print()
+void KernelTestParameters::print()
{
cout << "Used Kernel parameters:" << endl
<< "nrTabs : " << nrTabs << endl
@@ -77,7 +77,7 @@ void KernelParameters::print()
}
-void parseCommandlineParameters(int argc, char *argv[], Parset &ps, KernelParameters ¶ms, const char *testName)
+void parseCommandlineParameters(int argc, char *argv[], Parset &ps, KernelTestParameters ¶ms, const char *testName)
{
int opt;
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.h b/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.h
index 557dfa072c9fa104af83bc4fd3576ba242a841ef..3e64b7267d1fd946248b3af35d5a7f6e1bf1d574 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.h
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/KernelTestHelpers.h
@@ -26,7 +26,7 @@ void usage(char const *testName);
// Helper struct to collect some kernel/pipeline parameters collected from the
// command line
-struct KernelParameters
+struct KernelTestParameters
{
unsigned nrTabs;
unsigned nrChannels;
@@ -39,7 +39,7 @@ struct KernelParameters
bool parameterParsed;
std::string stokesType; // I, XXYY, IQUV
- KernelParameters();
+ KernelTestParameters();
void print();
};
@@ -48,4 +48,4 @@ struct KernelParameters
// The parameters struct is used for values not parsed from the command line
void parseCommandlineParameters(
int argc, char *argv[],
- LOFAR::Cobalt::Parset &ps, KernelParameters ¶meters, const char *testName);
+ LOFAR::Cobalt::Parset &ps, KernelTestParameters ¶meters, const char *testName);
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel.cc
index 2f3726fd083a44adacf37503c7691ff3b401a640..0ddae56fb229b954a4e60c8f54d751cdbe6ef175 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel.cc
@@ -49,11 +49,17 @@ int main() {
gpu::Stream stream(ctx);
Parset ps("tBandPassCorrectionKernel.in_parset");
- BandPassCorrectionKernel::Parameters params(ps);
- params.nrDelayCompensationChannels = 64; // unused
- params.nrHighResolutionChannels = 4096;
- params.nrSamplesPerChannel =
- ps.settings.blockSize / params.nrHighResolutionChannels;
+ unsigned nrStations = ps.settings.beamFormer.antennaFieldNames.size();
+ unsigned nrHighResolutionChannels = 4096;
+ unsigned nrSamplesPerChannel =
+ ps.settings.blockSize / nrHighResolutionChannels;
+ bool correctBandPass = ps.settings.corrections.bandPass;
+
+ BandPassCorrectionKernel::Parameters params(
+ nrStations,
+ nrHighResolutionChannels,
+ nrSamplesPerChannel,
+ correctBandPass);
KernelFactory<BandPassCorrectionKernel> factory(params);
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel2.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel2.cc
index 1ee32b5035204f8dc49486e3e23e73035f678997..b642d18fcfa24191450a4e3ba6af67585b9771ef 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel2.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tBandPassCorrectionKernel2.cc
@@ -31,16 +31,23 @@ using namespace LOFAR::Cobalt;
struct TestFixture
{
- TestFixture() :
- ps("tBandPassCorrectionKernel2.in_parset"),
- params(ps),
+ TestFixture() :
+ ps("tBandPassCorrectionKernel2.in_parset"),
factory()
{
+ unsigned nrStations = ps.settings.beamFormer.antennaFieldNames.size();
+ bool correctBandPass = ps.settings.corrections.bandPass;
+
// Default parameters parsed from parset are not correct
- params.nrDelayCompensationChannels = 64; // unused
- params.nrHighResolutionChannels = 4096;
- params.nrSamplesPerChannel =
- ps.settings.blockSize / params.nrHighResolutionChannels;
+ unsigned nrHighResolutionChannels = 4096;
+ unsigned nrSamplesPerChannel =
+ ps.settings.blockSize / nrHighResolutionChannels;
+
+ params = BandPassCorrectionKernel::Parameters(
+ nrStations,
+ nrHighResolutionChannels,
+ nrSamplesPerChannel,
+ correctBandPass);
factory = new KernelFactory<BandPassCorrectionKernel>(params);
}
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tBeamFormerKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tBeamFormerKernel.cc
index 62f8c5a563ad3cd48cef7f8c37df01ba555688b8..5e77681fa35ed4da65ffb62690db98f0b9953cac 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tBeamFormerKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tBeamFormerKernel.cc
@@ -33,6 +33,8 @@
#include <stdlib.h>
#include <unistd.h>
+#include <boost/format.hpp>
+
#include "KernelTestHelpers.h"
using namespace std;
@@ -46,7 +48,7 @@ int main(int argc, char *argv[])
INIT_LOGGER(testName);
// parse command line arguments to parset
Parset ps;
- KernelParameters params;
+ KernelTestParameters params;
parseCommandlineParameters(argc, argv, ps, params, testName);
// Set up gpu environment
@@ -64,7 +66,18 @@ int main(int argc, char *argv[])
gpu::Stream stream(ctx);
// Create the factory
- BeamFormerKernel::Parameters bfparams(ps);
+ BeamFormerKernel::Parameters bfparams(
+ ObservationSettings::AntennaFieldName::indices(ps.settings.beamFormer.antennaFieldNames, ps.settings.antennaFieldNames),
+ ObservationSettings::AntennaFieldName::indices(ps.settings.beamFormer.antennaFieldNames, ps.settings.antennaFieldNames),
+ ps.settings.antennaFieldNames.size(),
+ ps.settings.beamFormer.nrDelayCompensationChannels,
+ ps.settings.blockSize / ps.settings.beamFormer.nrDelayCompensationChannels,
+ ps.settings.SAPs.size(),
+ ps.settings.beamFormer.maxNrCoherentTABsPerSAP(),
+ ps.settings.subbandWidth(),
+ ps.settings.beamFormer.doFlysEye
+ );
+
KernelFactory<BeamFormerKernel> factory(bfparams);
DeviceMemory
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tCoherentStokesKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tCoherentStokesKernel.cc
index d759792a0051d637454771c5d0323850414a39aa..303912d791060d9dcb01bf1d383736d1535d1a22 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tCoherentStokesKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tCoherentStokesKernel.cc
@@ -57,7 +57,9 @@ struct ParsetSUT
nrStations,
nrInputSamples,
blockSize,
- nrDelayCompensationChannels;
+ nrDelayCompensationChannels,
+ nrTabs,
+ nrStokes;
Parset parset;
@@ -66,6 +68,7 @@ struct ParsetSUT
size_t inrStations = 43,
size_t inrTabs = 21,
size_t itimeIntegrationFactor = 1,
+ size_t inrStokes = 4,
string stokes = "IQUV")
:
timeIntegrationFactor(itimeIntegrationFactor),
@@ -74,7 +77,9 @@ struct ParsetSUT
nrStations(inrStations),
nrInputSamples(nrOutputSamples * timeIntegrationFactor),
blockSize(nrChannels * nrInputSamples),
- nrDelayCompensationChannels(64)
+ nrDelayCompensationChannels(64),
+ nrTabs(inrTabs),
+ nrStokes(inrStokes)
{
size_t nr_files = inrTabs * 4; // 4 for number of stokes
parset.add("Observation.DataProducts.Output_CoherentStokes.enabled", "true");
@@ -107,7 +112,7 @@ struct ParsetSUT
TEST(BufferSizes)
{
LOG_INFO("Test BufferSizes");
- ParsetSUT sut;
+ ParsetSUT sut;
const ObservationSettings::BeamFormer::StokesSettings &settings =
sut.parset.settings.beamFormer.coherentSettings;
CHECK_EQUAL(sut.timeIntegrationFactor, settings.timeIntegrationFactor);
@@ -122,7 +127,15 @@ TEST(KernelFactory)
{
LOG_INFO("Test KernelFactory");
ParsetSUT sut;
- KernelFactory<CoherentStokesKernel> kf(CoherentStokesKernel::Parameters(sut.parset));
+ KernelFactory<CoherentStokesKernel> kf(CoherentStokesKernel::Parameters(
+ sut.nrChannels,
+ sut.nrInputSamples,
+ sut.nrTabs,
+ sut.nrStokes,
+ false, // outputComplexVoltages
+ sut.timeIntegrationFactor,
+ false // quantizeOutput
+ ));
}
struct SUTWrapper: ParsetSUT
@@ -132,6 +145,7 @@ struct SUTWrapper: ParsetSUT
gpu::Stream stream;
size_t nrStokes;
size_t nrTabs;
+ CoherentStokesKernel::Parameters parameters;
KernelFactory<CoherentStokesKernel> factory;
MultiDimArrayHostBuffer<fcomplex, 4> hInput;
MultiDimArrayHostBuffer<float, 4> hOutput;
@@ -152,7 +166,15 @@ struct SUTWrapper: ParsetSUT
stream(context),
nrStokes(parset.settings.beamFormer.coherentSettings.nrStokes),
nrTabs(parset.settings.beamFormer.maxNrTABsPerSAP()),
- factory(parset),
+ factory(CoherentStokesKernel::Parameters(
+ nrChannels,
+ nrInputSamples,
+ nrTabs,
+ nrStokes,
+ false, // outputComplexVoltages
+ timeIntegrationFactor,
+ false // quantizeOutput
+ )),
hInput(
boost::extents[nrTabs][NR_POLARIZATIONS][nrInputSamples][nrChannels],
context),
@@ -498,7 +520,7 @@ int main(int argc, char *argv[])
INIT_LOGGER(testName);
Parset ps;
- KernelParameters params;
+ KernelTestParameters params;
parseCommandlineParameters(argc, argv, ps, params, testName);
// If no arguments were parsed
try
@@ -523,7 +545,16 @@ int main(int argc, char *argv[])
gpu::Stream stream(ctx);
// Create the factory
- CoherentStokesKernel::Parameters csparams(ps);
+ ParsetSUT sut;
+ CoherentStokesKernel::Parameters csparams(
+ sut.nrChannels,
+ sut.nrInputSamples,
+ sut.nrTabs,
+ sut.nrStokes,
+ false, // outputComplexVoltages
+ sut.timeIntegrationFactor,
+ false // quantizeOutput
+ );
KernelFactory<CoherentStokesKernel> factory(csparams);
DeviceMemory coherentStokesInputMem(ctx,
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tCorrelatorKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tCorrelatorKernel.cc
index 3ff57bfdb128614bbfa4698de0fc728e21e05f9e..0b048f6d2f419681ba86f09acb7adc03cb7d7c75 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tCorrelatorKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tCorrelatorKernel.cc
@@ -36,7 +36,14 @@ using boost::format;
struct TestFixture
{
- TestFixture() : ps("tCorrelatorKernel.in_parset"), factory(ps) {}
+ TestFixture() :
+ ps("tCorrelatorKernel.in_parset"),
+ factory(CorrelatorKernel::Parameters(
+ ps.settings.antennaFields.size(),
+ ps.settings.correlator.nrChannels,
+ ps.settings.correlator.nrSamplesPerBlock / ps.settings.correlator.nrIntegrationsPerBlock,
+ ps.settings.correlator.nrIntegrationsPerBlock
+ )) {}
~TestFixture() {}
Parset ps;
KernelFactory<CorrelatorKernel> factory;
@@ -95,7 +102,12 @@ TEST(DataValidity)
ps.updateSettings();
// Configuration
- KernelFactory<CorrelatorKernel> factory(ps);
+ KernelFactory<CorrelatorKernel> factory(
+ CorrelatorKernel::Parameters(
+ ps.settings.antennaFields.size(),
+ ps.settings.correlator.nrChannels,
+ ps.settings.correlator.nrSamplesPerBlock / ps.settings.correlator.nrIntegrationsPerBlock,
+ ps.settings.correlator.nrIntegrationsPerBlock));
// Stream for all I/O
gpu::Platform platform;
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel.cc
index 743e9b4a346ff1783a7251cb06ad9b3c8217925a..a66c5d83185810fe627d0cca697fe1d74331e94f 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel.cc
@@ -38,7 +38,7 @@ int main(int argc, char *argv[])
const char * testName = "tDelayAndBandPassKernel";
INIT_LOGGER(testName);
Parset ps;
- KernelParameters params;
+ KernelTestParameters params;
parseCommandlineParameters(argc, argv, ps, params, testName);
// Set up gpu environment
@@ -55,8 +55,25 @@ int main(int argc, char *argv[])
gpu::Context ctx(device);
gpu::Stream stream(ctx);
- DelayAndBandPassKernel::Parameters dbpparams(ps, false);
- KernelFactory<DelayAndBandPassKernel> factory(dbpparams);
+ bool correlator = false;
+ DelayAndBandPassKernel::Parameters dbpparams(
+
+ );
+ KernelFactory<DelayAndBandPassKernel> factory(
+ DelayAndBandPassKernel::Parameters(
+ ps.settings.beamFormer.antennaFieldNames.size(), // nrStations
+ ObservationSettings::AntennaFieldName::indices(ps.settings.beamFormer.antennaFieldNames, ps.settings.antennaFieldNames),
+ ps.settings.antennaFieldNames.size(), // nrDelays
+ ps.settings.nrBitsPerSample,
+ ps.settings.beamFormer.nrDelayCompensationChannels,
+ ps.settings.blockSize / ps.settings.beamFormer.nrDelayCompensationChannels,
+ ps.settings.subbandWidth(),
+ ps.settings.SAPs.size(),
+ ps.settings.delayCompensation.enabled,
+ correlator,
+ false, // correctBandPass
+ false // transpose
+ ));
gpu::DeviceMemory
inputData(ctx, factory.bufferSize(DelayAndBandPassKernel::INPUT_DATA)),
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel2.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel2.cc
index 62b08af52efd252e92f90b0de451c46bb6f721eb..6f5445d23f808feb3a85117dd2db12e528d457a5 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel2.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tDelayAndBandPassKernel2.cc
@@ -30,7 +30,20 @@ using namespace LOFAR::Cobalt;
struct TestFixture
{
- TestFixture() : ps("tDelayAndBandPassKernel2.in_parset"), factory(DelayAndBandPassKernel::Parameters(ps, true)) {}
+ TestFixture() : ps("tDelayAndBandPassKernel2.in_parset"), factory(DelayAndBandPassKernel::Parameters(
+ ps.settings.antennaFieldNames.size(), // nrStations
+ {}, // delayIndices are set in the constructor
+ ps.settings.antennaFieldNames.size(), // nrDelays
+ ps.settings.nrBitsPerSample,
+ ps.settings.correlator.nrChannels,
+ ps.settings.blockSize / ps.settings.correlator.nrChannels,
+ ps.settings.subbandWidth(),
+ ps.settings.SAPs.size(),
+ ps.settings.delayCompensation.enabled,
+ true, // correlator
+ ps.settings.corrections.bandPass, // correctBandPass
+ true // transpose
+ )) {}
~TestFixture() {}
Parset ps;
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tFFTShiftKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tFFTShiftKernel.cc
index d23611bf8c37329a18a919b6b5a8210645b924eb..f069023da916268c4da4c475e612fdb22892216e 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tFFTShiftKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tFFTShiftKernel.cc
@@ -125,7 +125,7 @@ struct SUTWrapper : ParsetSUT
stream(context),
nrStokes(parset.settings.beamFormer.coherentSettings.nrStokes),
nrTabs(parset.settings.beamFormer.maxNrTABsPerSAP()),
- factory(FFTShiftParams(parset)),
+ factory(FFTShiftKernel::Parameters(inrStations, nrChannels, inrOutputinSamplesPerSuband)),
hInput(
boost::extents[inrStations][NR_POLARIZATIONS][nrChannels][inrOutputinSamplesPerSuband],
context),
@@ -140,16 +140,6 @@ struct SUTWrapper : ParsetSUT
initializeHostBuffers();
}
- // Needed to adapt the number of channels in the parameterset
- // These values are normaly added in the SubbandProc (compile time default).
- FFTShiftKernel::Parameters FFTShiftParams(Parset parset)
- {
- FFTShiftKernel::Parameters params(parset, nrStations, nrChannels);
-
- return params;
- }
-
-
// Initialize all the elements of the input host buffer to zero, and all
// elements of the output host buffer to NaN.
void initializeHostBuffers()
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tFIR_FilterKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tFIR_FilterKernel.cc
index 837bd8585a1bfac740e248328505d927ef5b9265..2ce8f9ffc4c9ac071a43e26b1a57a506dc7a8936 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tFIR_FilterKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tFIR_FilterKernel.cc
@@ -51,11 +51,13 @@ TEST(FIR_FilterKernel)
ps.add("Observation.DataProducts.Output_Correlated.locations", "[localhost:.]");
ps.updateSettings();
- FIR_FilterKernel::Parameters params(ps,
+ FIR_FilterKernel::Parameters params(
ps.settings.antennaFields.size(),
+ ps.settings.nrBitsPerSample,
true,
1,
ps.settings.correlator.nrChannels,
+ ps.settings.blockSize / ps.settings.correlator.nrChannels,
1.0f
);
@@ -130,11 +132,13 @@ TEST(HistoryFlags)
ps.add("Observation.DataProducts.Output_Correlated.locations", "[localhost:.]");
ps.updateSettings();
- FIR_FilterKernel::Parameters params(ps,
+ FIR_FilterKernel::Parameters params(
ps.settings.antennaFields.size(),
+ ps.settings.nrBitsPerSample,
true,
1,
ps.settings.correlator.nrChannels,
+ ps.settings.blockSize / ps.settings.correlator.nrChannels,
1.0f
);
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tIntToFloatKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tIntToFloatKernel.cc
index a95ab183ca17fa2b3ac3bbb19a4e667f102bd000..cc1cabf0330649b2b331372ca66c42829b1f3832 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tIntToFloatKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tIntToFloatKernel.cc
@@ -49,7 +49,13 @@ int main() {
gpu::Stream stream(ctx);
Parset ps("tIntToFloatKernel.in_parset");
- KernelFactory<IntToFloatKernel> factory(IntToFloatKernel::Parameters(ps, true, true));
+ KernelFactory<IntToFloatKernel> factory(IntToFloatKernel::Parameters(
+ ps.settings.antennaFields.size(),
+ ObservationSettings::AntennaFieldName::indices(ps.settings.beamFormer.antennaFieldNames, ps.settings.antennaFieldNames),
+ ps.settings.nrBitsPerSample,
+ ps.settings.blockSize,
+ true,
+ true));
size_t nSampledData = factory.bufferSize(IntToFloatKernel::INPUT_DATA) / sizeof(char);
size_t sizeSampledData = nSampledData * sizeof(char);
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tKernelFunctions.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tKernelFunctions.cc
index 0251498765830d2671ab8c5f50ffd3c89665044e..37c1a8f62c283ef5f7b3ac8cda59eee14c3b579e 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tKernelFunctions.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tKernelFunctions.cc
@@ -53,11 +53,13 @@ TEST(tKernelFunctions)
ps.add("Observation.DataProducts.Output_Correlated.locations", "[localhost:.]");
ps.updateSettings();
- FIR_FilterKernel::Parameters params(ps,
+ FIR_FilterKernel::Parameters params(
ps.settings.antennaFields.size(),
+ ps.settings.nrBitsPerSample,
true,
1,
ps.settings.correlator.nrChannels,
+ ps.settings.blockSize / ps.settings.correlator.nrChannels,
1.0f);
KernelFactory<FIR_FilterKernel> factory(params);
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tQuantizeOutputKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tQuantizeOutputKernel.cc
new file mode 100644
index 0000000000000000000000000000000000000000..cd777e19bc828af5da8ac12deb8b7837a607ca87
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tQuantizeOutputKernel.cc
@@ -0,0 +1,587 @@
+//# tQuantizeOutputKernel.cc: test QuantizeOutputKernel class
+//#
+//# Copyright (C) 2013 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include <GPUProc/Kernels/CoherentStokesKernel.h>
+#include <GPUProc/Kernels/QuantizeOutputKernel.h>
+#include <GPUProc/MultiDimArrayHostBuffer.h>
+#include <GPUProc/gpu_wrapper.h>
+#include <CoInterface/BlockID.h>
+#include <CoInterface/Config.h>
+#include <CoInterface/Parset.h>
+#include <Common/LofarLogger.h>
+
+#include <UnitTest++.h>
+#include <boost/format.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/scoped_ptr.hpp>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <random>
+
+
+#include "KernelTestHelpers.h"
+
+using namespace std;
+using namespace boost;
+using namespace LOFAR::Cobalt;
+using namespace LOFAR::Cobalt::gpu;
+
+typedef complex<float> fcomplex;
+
+// Fixture for testing correct translation of parset values
+struct ParsetSUT
+{
+ size_t
+ timeIntegrationFactor,
+ nrChannels,
+ nrOutputSamples,
+ nrStations,
+ nrInputSamples,
+ blockSize,
+ nrDelayCompensationChannels,
+ nrTabs,
+ nrStokes,
+ nrQuantizeBits;
+
+ float
+ quantizeScaleMax,
+ quantizeScaleMin;
+
+ Parset parset;
+
+ ParsetSUT(
+ size_t inrChannels = 16,
+ size_t inrOutputSamples = 1024,
+ size_t inrStations = 43,
+ size_t inrTabs = 21,
+ size_t itimeIntegrationFactor = 1,
+ size_t inrStokes = 4,
+ string stokes = "IQUV",
+ size_t nrQuantizeBits = 8,
+ float quantizeScaleMax = 5,
+ float quantizeScaleMin = -5)
+ :
+ timeIntegrationFactor(itimeIntegrationFactor),
+ nrChannels(inrChannels),
+ nrOutputSamples(inrOutputSamples),
+ nrStations(inrStations),
+ nrInputSamples(nrOutputSamples * timeIntegrationFactor),
+ blockSize(nrChannels * nrInputSamples),
+ nrDelayCompensationChannels(64),
+ nrTabs(inrTabs),
+ nrStokes(inrStokes),
+ nrQuantizeBits(nrQuantizeBits),
+ quantizeScaleMax(quantizeScaleMax),
+ quantizeScaleMin(quantizeScaleMin)
+ {
+ size_t nr_files = inrTabs * 4; // 4 for number of stokes
+ parset.add("Observation.DataProducts.Output_CoherentStokes.enabled", "true");
+ parset.add("Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor",
+ lexical_cast<string>(timeIntegrationFactor));
+ parset.add("Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband",
+ lexical_cast<string>(nrChannels));
+ parset.add("Cobalt.BeamFormer.CoherentStokes.which", stokes);
+ parset.add("Observation.VirtualInstrument.stationList",
+ str(format("[%d*RS000]") % nrStations));
+ parset.add("Cobalt.blockSize",
+ lexical_cast<string>(blockSize));
+ parset.add("Observation.Beam[0].subbandList", "[0]");
+ parset.add("Observation.Beam[0].nrTiedArrayBeams",lexical_cast<string>(inrTabs));
+ parset.add("Observation.DataProducts.Output_CoherentStokes.filenames",
+ str(format("[%d*dummy.raw]") % nr_files));
+ parset.add("Observation.DataProducts.Output_CoherentStokes.locations", str(format("[%d*:.]") % nr_files));
+ parset.add("Cobalt.BeamFormer.nrDelayCompensationChannels",
+ lexical_cast<string>(nrDelayCompensationChannels));
+ parset.add("Cobalt.BeamFormer.CoherentStokes.quantize", lexical_cast<string>(true));
+ parset.add("Cobalt.BeamFormer.CoherentStokes.quantizeBits", lexical_cast<string>(nrQuantizeBits));
+ parset.add("Cobalt.BeamFormer.CoherentStokes.quantizeScaleMax", lexical_cast<string>(quantizeScaleMax));
+ parset.add("Cobalt.BeamFormer.CoherentStokes.quantizeScaleMin", lexical_cast<string>(quantizeScaleMin));
+ parset.add("Cobalt.BeamFormer.CoherentStokes.quantizeIpositive", lexical_cast<string>(false));
+ parset.updateSettings();
+ }
+};
+
+
+// Test if we can succesfully create a KernelFactory
+TEST(KernelFactory)
+{
+ LOG_INFO("Test KernelFactory");
+ ParsetSUT sut;
+
+ CoherentStokesKernel::Parameters csParameters(
+ sut.nrChannels,
+ sut.nrInputSamples,
+ sut.nrTabs,
+ sut.nrStokes,
+ false, // outputComplexVoltages
+ sut.timeIntegrationFactor,
+ false // quantizeOutput
+ );
+
+ QuantizeOutputKernel::Parameters qoParameters(
+ sut.nrChannels,
+ sut.nrInputSamples,
+ sut.nrTabs,
+ sut.nrStokes,
+ false, // outputComplexVoltages
+ sut.nrQuantizeBits,
+ sut.quantizeScaleMax,
+ sut.quantizeScaleMin,
+ false // quantizeIpositive
+ );
+
+ KernelFactory<CoherentStokesKernel> kfcoherentstokes(csParameters);
+ KernelFactory<QuantizeOutputKernel> kfquantization(qoParameters);
+ }
+
+struct SUTWrapper: ParsetSUT
+{
+ gpu::Device device;
+ gpu::Context context;
+ gpu::Stream stream;
+ size_t nrStokes;
+ size_t nrTabs;
+ CoherentStokesKernel::Parameters csparameters;
+ QuantizeOutputKernel::Parameters qoparameters;
+ KernelFactory<CoherentStokesKernel> csfactory;
+ KernelFactory<QuantizeOutputKernel> qofactory;
+ MultiDimArrayHostBuffer<fcomplex, 4> hCoherentStokesInput;
+ MultiDimArrayHostBuffer<float, 4> hCoherentStokesOutput;
+ MultiDimArrayHostBuffer<float, 4> hCoherentStokesOutputTransposed;
+ MultiDimArrayHostBuffer<float, 4> hQuantizedOutput;
+ gpu::DeviceMemory inputBuffer;
+ gpu::DeviceMemory outputBuffer;
+ scoped_ptr<CoherentStokesKernel> cskernel;
+ scoped_ptr<QuantizeOutputKernel> qokernel;
+
+ SUTWrapper(size_t inrChannels = 16,
+ size_t inrOutputSamples = 1024,
+ size_t inrStations = 43,
+ size_t inrTabs = 21,
+ size_t itimeIntegrationFactor = 1,
+ size_t quantizeBits = 8,
+ float quantizeScaleMax = 5,
+ float quantizeScaleMin = -5) :
+ ParsetSUT(inrChannels, inrOutputSamples, inrStations ,
+ inrTabs,itimeIntegrationFactor, 4, "IQUV",
+ quantizeBits, quantizeScaleMax, quantizeScaleMin),
+ device(gpu::Platform().devices()[0]),
+ context(device),
+ stream(context),
+ nrStokes(parset.settings.beamFormer.coherentSettings.nrStokes),
+ nrTabs(parset.settings.beamFormer.maxNrTABsPerSAP()),
+ csfactory(CoherentStokesKernel::Parameters(
+ nrChannels,
+ nrInputSamples,
+ nrTabs,
+ nrStokes,
+ false, // outputComplexVoltages
+ timeIntegrationFactor,
+ false // quantizeOutput
+ )),
+ qofactory(QuantizeOutputKernel::Parameters(
+ nrChannels,
+ nrInputSamples,
+ nrTabs,
+ nrStokes,
+ false, // outputComplexVoltages
+ nrQuantizeBits,
+ quantizeScaleMax,
+ quantizeScaleMin,
+ false // quantizeIpositive
+ )),
+ hCoherentStokesInput(
+ boost::extents[nrTabs][NR_POLARIZATIONS][nrInputSamples][nrChannels],
+ context),
+ hCoherentStokesOutput(
+ boost::extents[nrTabs][nrStokes][nrOutputSamples][nrChannels],
+ context),
+ hCoherentStokesOutputTransposed(
+ boost::extents[nrTabs][nrStokes][nrChannels][nrOutputSamples],
+ context),
+ hQuantizedOutput(
+ boost::extents[nrTabs][nrStokes][nrOutputSamples][nrChannels],
+ context),
+ inputBuffer(context, csfactory.bufferSize(CoherentStokesKernel::INPUT_DATA)),
+ outputBuffer(context, csfactory.bufferSize(CoherentStokesKernel::OUTPUT_DATA)),
+ qokernel(qofactory.create(stream, outputBuffer, inputBuffer))
+ {
+ initializeHostBuffers();
+ }
+
+ // Initialize all the elements of the input host buffer to zero, and all
+ // elements of the output host buffer to NaN.
+ void initializeHostBuffers()
+ {
+ cout << "\nInitializing host buffers..."
+ << "\n buffers.input.size() = " << setw(7) << inputBuffer.size()
+ << "\n buffers.output.size() = " << setw(7) << outputBuffer.size()
+ << endl;
+ CHECK_EQUAL(inputBuffer.size(), hCoherentStokesInput.size());
+ CHECK_EQUAL(outputBuffer.size(), hQuantizedOutput.size());
+ fill(hCoherentStokesInput.data(), hCoherentStokesInput.data() + hCoherentStokesInput.num_elements(), 0.0f);
+ fill(hCoherentStokesOutput.data(), hCoherentStokesOutput.data() + hCoherentStokesOutput.num_elements(), 0.0f);
+ fill(hCoherentStokesOutputTransposed.data(), hCoherentStokesOutputTransposed.data() + hCoherentStokesOutputTransposed.num_elements(), 0.0f);
+ fill(hQuantizedOutput.data(), hQuantizedOutput.data() + hQuantizedOutput.num_elements(), 0.0f);
+ }
+
+ void runKernel()
+ {
+ // Dummy BlockID
+ BlockID blockId;
+ // Copy reference output data from host- to device buffer synchronously
+ stream.writeBuffer(outputBuffer, hCoherentStokesOutputTransposed, true);
+ // Launch the quantize output kernel
+ std::cout << "launch quantize output kernel" << std::endl;
+ qokernel->enqueue(blockId);
+ // Copy output data from device- to host buffer synchronously
+ stream.readBuffer(hCoherentStokesInput, inputBuffer, true);
+ }
+
+};
+
+void initialize(
+ MultiDimArray<float, 4>& ref_float,
+ bool outputComplexVoltages = false)
+{
+ // Get parameters
+ unsigned nrTABs = ref_float.shape()[0];
+ unsigned nrStokes = ref_float.shape()[1];
+ unsigned nrSamples = ref_float.shape()[2];
+ unsigned nrChannels = ref_float.shape()[3];
+
+ // Initialize random number generators
+ std::mt19937 mt(0);
+ std::normal_distribution<double> dist_signed(-1, 1);
+ std::normal_distribution<double> dist_unsigned(0, 1);
+
+ for (size_t tab = 0; tab < nrTABs; tab++)
+ {
+ for (size_t stokes = 0; stokes < nrStokes; stokes++)
+ {
+ // Determine the type of quantization to apply
+ bool stokes_quv = (stokes > 0);
+ bool zero_mean = (outputComplexVoltages || stokes_quv);
+
+ for (size_t sample = 0; sample < nrSamples; sample++)
+ {
+ for (size_t channel = 0; channel < nrChannels; channel++)
+ {
+ float ref = zero_mean ? dist_signed(mt) : dist_unsigned(mt);
+ ref_float[tab][stokes][sample][channel] = ref;
+ }
+ }
+ }
+ }
+}
+
+void transpose4D(
+ MultiDimArray<float, 4>& dst,
+ MultiDimArray<float, 4>& src)
+{
+ // Sanity check
+ assert(dst.num_elements() == src.num_elements());
+
+ // Get parameters
+ unsigned x_dim = src.shape()[0];
+ unsigned y_dim = src.shape()[1];
+ unsigned z_dim = src.shape()[2];
+ unsigned w_dim = src.shape()[3];
+
+ for (size_t x = 0; x < x_dim; x++)
+ {
+ for (size_t y = 0; y < y_dim; y++)
+ {
+ for (size_t z = 0; z < z_dim; z++)
+ {
+ for (size_t w = 0; w < w_dim; w++)
+ {
+ dst[x][y][w][z] = src[x][y][z][w];
+ }
+ }
+ }
+ }
+}
+
+template<typename INT, typename UINT>
+void zeroClipped(
+ MultiDimArray<float, 4>& ref_float,
+ MultiDimArray<float, 4>& x_float,
+ MultiDimArray<INT, 4>& x_int,
+ bool outputComplexVoltages = false)
+{
+ // Sanity check
+ assert(ref_float.num_elements() == x_float.num_elements());
+ assert(ref_float.num_elements() == x_int.num_elements());
+
+ // Get parameters
+ unsigned nrTABs = ref_float.shape()[0];
+ unsigned nrStokes = ref_float.shape()[1];
+ unsigned nrSamples = ref_float.shape()[2];
+ unsigned nrChannels = ref_float.shape()[3];
+ unsigned nrQuantizeBits = sizeof(INT) * 8;
+
+ unsigned int nrClipped = 0;
+
+ for (size_t tab = 0; tab < nrTABs; tab++)
+ {
+ for (size_t stokes = 0; stokes < nrStokes; stokes++)
+ {
+ // Determine the type of quantization to apply
+ bool stokes_quv = (stokes > 0);
+ bool zero_mean = (outputComplexVoltages || stokes_quv);
+
+ // Determine min and max values
+ int min_unsigned = 0;
+ int max_unsigned = (1 << nrQuantizeBits) - 1;
+ int min_signed = -1 * (1 << (nrQuantizeBits - 1));
+ int max_signed = (1 << (nrQuantizeBits - 1)) - 1;
+ int max_value = zero_mean ? max_signed : max_unsigned;
+ int min_value = zero_mean ? min_signed : min_unsigned;
+
+ for (size_t sample = 0; sample < nrSamples; sample++)
+ {
+ for (size_t channel = 0; channel < nrChannels; channel++)
+ {
+ auto x_ptr = &(x_int[tab][stokes][sample][channel]);
+ auto x = zero_mean ? *((INT *) x_ptr) : *((UINT *) x_ptr);
+
+ if (x == min_value || x == max_value)
+ {
+ ref_float[tab][stokes][sample][channel] = 0.0f;
+ x_float[tab][stokes][sample][channel] = 0.0f;
+ nrClipped++;
+ }
+ }
+ }
+ }
+ }
+
+ // Report
+ float fractionClipped = ((float) nrClipped) / ref_float.num_elements();
+ float percentageClipped = ((int) (fractionClipped * 1000)) / 10.;
+ std::cout << "Number of clipped values set to zero: " << nrClipped;
+ std::cout << " (" << percentageClipped << "%)" << std::endl;
+}
+
+// CHECK_ARRAY_CLOSE makes it very hard to inspect differences
+// between the expected and actual output. This method
+// prints only the incorrect values.
+bool checkArrayClose(
+ const float* ref_float,
+ const float* x_float,
+ size_t n,
+ float tolerance)
+{
+ size_t nnz = 0;
+ double error = 0;
+
+ for (size_t idx = 0; idx < n; ++idx)
+ {
+ float ref = ref_float[idx];
+ float x = x_float[idx];
+ float diff = ref - x;
+ if (abs(diff) > tolerance)
+ {
+ std::cerr << "[" << idx << "] ";
+ std::cerr << "ref: " << ref << ", ";
+ std::cerr << "x: " << x << ", ";
+ std::cerr << "diff: " << diff << std::endl;
+ }
+ nnz += ref != 0;
+ error += diff * diff;
+ }
+
+ nnz = nnz == 0 ? 1 : nnz;
+ error = sqrt(error) / nnz;
+ std::clog << "Error: " << error << std::endl;
+
+ return error < tolerance;
+}
+
+TEST(QuantizeOutputRunTest)
+{
+ SUTWrapper sut(8, // channels
+ 32, // inrOutputSamples
+ 1, // inrStations
+ 1, // inrTabs
+ 1); // itimeIntegrationFactor
+
+ // Run the kernels
+ sut.runKernel();
+}
+
+template<typename INT, typename UINT>
+void testQuantizeOutput(
+ unsigned int nrChannels,
+ unsigned int nrSamples,
+ float tolerance)
+{
+ size_t NR_TABS = 64;
+ size_t INTEGRATION_SIZE = 1;
+ size_t QUANTIZE_BITS = sizeof(INT) * 8;
+ float QUANTIZE_SCALE = 2;
+
+ SUTWrapper sut(nrChannels,
+ nrSamples,
+ 1,
+ NR_TABS,
+ INTEGRATION_SIZE,
+ QUANTIZE_BITS,
+ QUANTIZE_SCALE,
+ -QUANTIZE_SCALE);
+
+ // Set the reference output
+ initialize(sut.hCoherentStokesOutput);
+ transpose4D(sut.hCoherentStokesOutputTransposed, sut.hCoherentStokesOutput);
+
+ // Run the kernels
+ sut.runKernel();
+
+ // Reconstruct hOutput from hInput
+ sut.qokernel->convertI2F(sut.hQuantizedOutput, sut.hCoherentStokesInput);
+
+ // Get reference to quantized data
+ auto qdata = sut.qokernel->getQuantizedData<INT>(sut.hCoherentStokesInput);
+
+ // Set clipped values to zero
+ zeroClipped<INT, UINT>(
+ sut.hCoherentStokesOutput,
+ sut.hQuantizedOutput,
+ qdata);
+
+ checkArrayClose(
+ sut.hCoherentStokesOutput.data(),
+ sut.hQuantizedOutput.data(),
+ sut.hQuantizedOutput.num_elements(),
+ tolerance);
+
+ CHECK_ARRAY_CLOSE(sut.hCoherentStokesOutput.data(),
+ sut.hQuantizedOutput.data(),
+ sut.hQuantizedOutput.num_elements(),
+ tolerance);
+}
+
+TEST(QuantizeOutput8Bit12288Samples)
+{
+ LOG_INFO("Test QuantizeOutput 8-bit 12288 samples");
+ testQuantizeOutput<int8_t, uint8_t>(16, 12288, 1e-1);
+}
+
+TEST(QuantizeOutput8Bit196608Samples)
+{
+ LOG_INFO("Test QuantizeOutput 8-bit 196608 samples");
+ testQuantizeOutput<int8_t, uint8_t>(1, 196608, 1e-1);
+}
+
+TEST(QuantizeOutput8Bit49152Samples)
+{
+ LOG_INFO("Test QuantizeOutput 8-bit 49152 samples");
+ testQuantizeOutput<int8_t, uint8_t>(4, 49152, 1e-1);
+}
+
+TEST(QuantizeOutput8Bit384Samples)
+{
+ LOG_INFO("Test QuantizeOutput 8-bit 384 samples");
+ testQuantizeOutput<int8_t, uint8_t>(512, 384, 1e-1);
+}
+
+TEST(QuantizeOutput16Bit12288Samples)
+{
+ LOG_INFO("Test QuantizeOutput 16-bit 12288 samples");
+ testQuantizeOutput<int16_t, uint16_t>(16, 12288, 1e-3);
+}
+
+TEST(QuantizeOutput16Bit49152Samples)
+{
+ LOG_INFO("Test QuantizeOutput 16-bit 49152 samples");
+ testQuantizeOutput<int16_t, uint16_t>(4, 49152, 1e-1);
+}
+
+TEST(QuantizeOutput16Bit196608Samples)
+{
+ LOG_INFO("Test QuantizeOutput 16-bit 196608 samples");
+ testQuantizeOutput<int16_t, uint16_t>(1, 196608, 1e-1);
+}
+
+TEST(QuantizeOutput16Bit384Samples)
+{
+ LOG_INFO("Test QuantizeOutput 16-bit 384 samples");
+ testQuantizeOutput<int16_t, uint16_t>(512, 384, 1e-1);
+}
+
+int main(int argc, char *argv[])
+{
+ const char * testName = "tCoherentStokesKernel";
+ INIT_LOGGER(testName);
+
+ Parset ps;
+ KernelTestParameters params;
+ parseCommandlineParameters(argc, argv, ps, params, testName);
+ // If no arguments were parsed
+ try
+ {
+ gpu::Platform pf;
+ }
+ catch (gpu::GPUException&)
+ {
+ cerr << "No GPU device(s) found. Skipping tests." << endl;
+ return 3;
+ }
+
+ if (!params.parameterParsed)
+ {
+ cout << "Running unittests" << endl;
+ return UnitTest::RunAllTests() == 0 ? 0 : 1;
+ }
+
+ gpu::Device device(0);
+ vector<gpu::Device> devices(1, device);
+ gpu::Context ctx(device);
+ gpu::Stream stream(ctx);
+
+ SUTWrapper sut;
+
+ // Setup CoherentStokesKernel
+ CoherentStokesKernel::Parameters& csparams = sut.csparameters;
+ KernelFactory<CoherentStokesKernel> csfactory(csparams);
+
+ DeviceMemory coherentStokesInputMem(ctx, csfactory.bufferSize(CoherentStokesKernel::INPUT_DATA));
+ DeviceMemory coherentStokesOutputMem(ctx, csfactory.bufferSize(CoherentStokesKernel::OUTPUT_DATA));
+
+ unique_ptr<CoherentStokesKernel> coherentStokesKernel(csfactory.create(stream, coherentStokesInputMem, coherentStokesOutputMem));
+
+ // Setup QuantizeOutputKernel
+ QuantizeOutputKernel::Parameters& qoparams = sut.qoparameters;
+ KernelFactory<QuantizeOutputKernel> qofactory(qoparams);
+
+ unique_ptr<QuantizeOutputKernel> quantizeOutputKernel(qofactory.create(stream, coherentStokesOutputMem, coherentStokesInputMem));
+
+ BlockID blockId;
+
+ // run
+ coherentStokesKernel->enqueue(blockId);
+ quantizeOutputKernel->enqueue(blockId);
+ stream.synchronize();
+}
diff --git a/RTCP/Cobalt/GPUProc/test/Kernels/tZeroingKernel.cc b/RTCP/Cobalt/GPUProc/test/Kernels/tZeroingKernel.cc
index 7179ddf2da920930d81881462e8689f549ab69fb..40ae7553e56fa9bb24f31fd50986e125e8aa4277 100644
--- a/RTCP/Cobalt/GPUProc/test/Kernels/tZeroingKernel.cc
+++ b/RTCP/Cobalt/GPUProc/test/Kernels/tZeroingKernel.cc
@@ -100,7 +100,7 @@ struct SUTWrapper : ParsetSUT
context(device),
stream(context),
nrSTABs(parset.settings.antennaFields.size()),
- factory(ZeroingKernel::Parameters(parset, nrStations, nrChannels)),
+ factory(ZeroingKernel::Parameters(nrStations, nrChannels, nrSamples / nrChannels)),
hData(
boost::extents[nrStations][NR_POLARIZATIONS][nrSamples / nrChannels][nrChannels],
context),
diff --git a/RTCP/Cobalt/GPUProc/test/Pipelines/tCorrelatorPipelineProcessObs.cc b/RTCP/Cobalt/GPUProc/test/Pipelines/tCorrelatorPipelineProcessObs.cc
index 478fa951ff3d5a4462d3857dd7b2d9f6c8bd9a65..ea1fef046c21e0ac4d9e6e29f70aff169e22ed02 100644
--- a/RTCP/Cobalt/GPUProc/test/Pipelines/tCorrelatorPipelineProcessObs.cc
+++ b/RTCP/Cobalt/GPUProc/test/Pipelines/tCorrelatorPipelineProcessObs.cc
@@ -79,8 +79,8 @@ int main(int argc, char *argv[]) {
// So do kernel compilation (reqs fork()) first.
// Don't bother passing a hostname to (or start()ing) the mdLogger.
MACIO::RTmetadata rtmd(ps.settings.observationID, "", "");
- SmartPtr<Pipeline> pipeline = new Pipeline(ps, subbands, devices,
- MPI_receive_pool, rtmd, "rtmd key prefix");
+ std::unique_ptr<Pipeline> pipeline(new Pipeline(ps, subbands, devices,
+ MPI_receive_pool, rtmd, "rtmd key prefix"));
mpi.init(argc, argv);
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/CMakeLists.txt b/RTCP/Cobalt/GPUProc/test/SubbandProcs/CMakeLists.txt
index 4a693506fbf7af9f111e17e629077a78c26d2de3..6210350972baeb682c7e77ec3a7f635dd68bcc20 100644
--- a/RTCP/Cobalt/GPUProc/test/SubbandProcs/CMakeLists.txt
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/CMakeLists.txt
@@ -12,6 +12,9 @@ endif()
# Understand the proble
lofar_add_test(tBeamFormerSubbandProcProcessSb tBeamFormerSubbandProcProcessSb.cc)
+lofar_add_test(tBeamFormerStationSubsetSubbandProcProcessSb tBeamFormerStationSubsetSubbandProcProcessSb.cc)
+lofar_add_test(tBeamFormerSubbandSubsetSubbandProcProcessSb tBeamFormerSubbandSubsetSubbandProcProcessSb.cc)
+
lofar_add_test(tCoherentStokesBeamFormerSubbandProcProcessSb
tCoherentStokesBeamFormerSubbandProcProcessSb.cc ../Kernels/KernelTestHelpers.cc)
lofar_add_test(tFlysEyeBeamFormerSubbandProcProcessSb
@@ -22,6 +25,8 @@ lofar_add_test(tFlysEyeBeamFormerSubbandProcProcessSb
if(BUILD_TESTING)
set_tests_properties(
tBeamFormerSubbandProcProcessSb
+ tBeamFormerStationSubsetSubbandProcProcessSb
+ tBeamFormerSubbandSubsetSubbandProcProcessSb
tCorrelatorSubbandProcProcessSb
tCoherentStokesBeamFormerSubbandProcProcessSb
tFlysEyeBeamFormerSubbandProcProcessSb
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.cc
new file mode 100644
index 0000000000000000000000000000000000000000..3690f1ca235706820287c3c83ad9a8e82ea9ec42
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.cc
@@ -0,0 +1,235 @@
+//# tBeamFormerStationSubsetSubbandProcProcessSb: test of Beamformer subband processor.
+//#
+//# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include <complex>
+#include <cmath>
+#include <iomanip>
+
+#include <Common/LofarLogger.h>
+#include <CoInterface/Parset.h>
+#include <CoInterface/fpequals.h>
+#include <GPUProc/gpu_utils.h>
+#include <GPUProc/SubbandProcs/SubbandProc.h>
+
+using namespace std;
+using namespace LOFAR::Cobalt;
+using namespace LOFAR::TYPES;
+
+void initialize_input(
+ Parset& ps,
+ SubbandProcInputData& in)
+{
+ const size_t nrStations = ps.settings.antennaFieldNames.size();
+ const size_t nrSamplesPerSubband = ps.settings.blockSize;
+ const size_t nrPolarisations = ps.settings.nrPolarisations;
+ const size_t nrBitsPerSample = ps.settings.nrBitsPerSample;
+
+ // Initialize synthetic input to input signal
+ for (size_t st = 0; st < nrStations; st++) {
+ for (size_t i = 0; i < nrSamplesPerSubband; i++) {
+ size_t pol = i % nrPolarisations;
+ if (st == 0)
+ {
+ switch(nrBitsPerSample) {
+ case 8:
+ reinterpret_cast<i8complex&>(in.inputSamples[st][i][pol][0]) = (st + 1);
+ break;
+ case 16:
+ reinterpret_cast<i16complex&>(in.inputSamples[st][i][pol][0]) = (st + 1);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ }
+
+ // Initialize subbands partitioning administration (struct BlockID). We only
+ // do the 1st block of whatever.
+
+ // Block number: 0 .. inf
+ in.blockID.block = 0;
+
+ // Subband index in the observation: [0, ps.nrSubbands())
+ in.blockID.globalSubbandIdx = 0;
+
+ // Subband index for this pipeline/workqueue: [0, subbandIndices.size())
+ in.blockID.localSubbandIdx = 0;
+
+ // Subband index for this SubbandProc
+ in.blockID.subbandProcSubbandIdx = 0;
+
+ // Initialize delays. We skip delay compensation, but init anyway,
+ // so we won't copy uninitialized data to the device.
+ for (size_t i = 0; i < in.delaysAtBegin.num_elements(); i++)
+ in.delaysAtBegin.get<float>()[i] = 0.0f;
+ for (size_t i = 0; i < in.delaysAfterEnd.num_elements(); i++)
+ in.delaysAfterEnd.get<float>()[i] = 0.0f;
+ for (size_t i = 0; i < in.phase0s.num_elements(); i++)
+ in.phase0s.get<float>()[i] = 0.0f;
+ for (auto tabDelays : in.tabDelays) {
+ for (size_t i = 0; i < tabDelays->num_elements(); i++)
+ tabDelays->get<float>()[i] = 0.0f;
+ }
+}
+
+void initialize_output(
+ SubbandProcOutputData& out)
+{
+ for (size_t i = 0; i < out.coherentData().data_num_elements(); i++)
+ out.coherentData().get_data_origin()[i] = 42.0f;
+ for (size_t i = 0; i < out.incoherentData().data_num_elements(); i++)
+ out.incoherentData().get_data_origin()[i] = 42.0f;
+}
+
+
+int main() {
+ INIT_LOGGER("tBeamFormerStationSubsetSubbandProcProcessSb");
+
+ try {
+ gpu::Platform pf;
+ cout << "Detected " << pf.size() << " CUDA devices" << endl;
+ } catch (gpu::CUDAException& e) {
+ cerr << e.what() << endl;
+ return 3;
+ }
+
+ gpu::Device device(0);
+ vector<gpu::Device> devices(1, device);
+ gpu::Context ctx(device);
+
+ Parset ps("tBeamFormerStationSubsetSubbandProcProcessSb.parset");
+
+ // Set the stations to process in the preprocessor
+ ps.replace("Cobalt.BeamFormer.stationList", "[CS002,CS003,CS004,CS005,CS006,CS007]");
+ ps.replace("Cobalt.BeamFormer.Pipeline[0].stationList", "[CS002,CS003,CS004,CS005,CS006,CS007]");
+ ps.updateSettings();
+
+ // Input array sizes
+ const size_t nrBeams = ps.settings.SAPs.size();
+ const size_t nrStations = ps.settings.antennaFieldNames.size();
+ const size_t nrBFStations = ps.settings.beamFormer.pipelines[0].antennaFieldNames.size();
+ const size_t nrPolarisations = ps.settings.nrPolarisations;
+ const size_t nrSamplesPerSubband = ps.settings.blockSize;
+ const size_t nrBitsPerSample = ps.settings.nrBitsPerSample;
+ const size_t nrBytesPerComplexSample = ps.nrBytesPerComplexSample();
+
+ const unsigned fftSize = ps.settings.beamFormer.nrDelayCompensationChannels;
+
+ // We only support 8-bit or 16-bit input samples
+ ASSERT(nrBitsPerSample == 8 || nrBitsPerSample == 16);
+
+ LOG_INFO_STR(
+ "Input info:" <<
+ "\n nrBeams = " << nrBeams <<
+ "\n nrStations = " << nrStations <<
+ "\n nrBFStations = " << nrBFStations <<
+ "\n nrPolarisations = " << nrPolarisations <<
+ "\n nrSamplesPerSubband = " << nrSamplesPerSubband <<
+ "\n nrBitsPerSample = " << nrBitsPerSample <<
+ "\n nrBytesPerComplexSample = " << nrBytesPerComplexSample <<
+ "\n fftSize = " << fftSize);
+
+ // Create very simple kernel programs, with predictable output. Skip as much
+ // as possible. Nr of channels/sb from the parset is 1, so the PPF will not
+ // even run. Parset also has turned of delay compensation and bandpass
+ // correction (but that kernel will run to convert int to float and to
+ // transform the data order).
+
+ // Initialize reference parset (copy of the base parset)
+ auto ps_ref = ps;
+
+ // Initialize parset with station subset, the preprocessor step processes
+ // one station more than the coherent and incoherent steps
+ auto ps_sub = ps;
+ ps_sub.replace("Cobalt.BeamFormer.stationList", "[CS001,CS002,CS003,CS004,CS005,CS006,CS007]");
+ ps_sub.updateSettings();
+
+ // Initialize input data
+ SubbandProcInputData in_ref(ps_ref, ctx);
+ initialize_input(ps_ref, in_ref);
+ SubbandProcInputData in_sub(ps_sub, ctx);
+ initialize_input(ps_sub, in_sub);
+
+ // Sanity check: the input data objects should be the same size
+ ASSERTSTR(in_ref.inputSamples.num_elements() == in_sub.inputSamples.num_elements(), "length(in_ref) != length(in_sub)");
+
+ // Initialize output data
+ SubbandProcOutputData out_ref(ps_ref, ctx);
+ initialize_output(out_ref);
+ SubbandProcOutputData out_sub(ps_sub, ctx);
+ initialize_output(out_sub);
+
+ // Sanity check: the output data objects should be the same size
+ ASSERTSTR(out_ref.coherentData().data_num_elements() == out_sub.coherentData().data_num_elements(), "length(out_ref) != length(out_sub)");
+
+ // Initialize SubbandProcs
+ SubbandProc bwq_ref(ps_ref, ctx);
+ SubbandProc bwq_sub(ps_sub, ctx);
+
+ // Don't bother initializing out.blockID; processSubband() doesn't need it.
+
+ cout << "processSubband()" << endl;
+ bwq_ref.processSubband(in_ref, out_ref);
+ bwq_ref.synchronize();
+ bwq_sub.processSubband(in_sub, out_sub);
+ bwq_sub.synchronize();
+ cout << "processSubband() done" << endl;
+
+ cout << "Output: " << endl;
+
+ // Output verification
+
+ // *** COHERENT STOKES ***
+
+ for (size_t t = 0; t < ps.settings.beamFormer.coherentSettings.nrSamples; t++)
+ {
+ for (size_t c = 0; c < ps.settings.beamFormer.coherentSettings.nrChannels; c++)
+ {
+ auto result_ref = out_ref.coherentData().get_data(0,0,t,c);
+ auto result_sub = out_sub.coherentData().get_data(0,0,t,c);
+ ASSERTSTR(fpEquals(result_ref, result_sub, 1e-4f),
+ "out.coherentData.data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result_sub <<
+ "; reference = " << result_ref);
+ }
+ }
+
+ // *** INCOHERENT STOKES ***
+
+ for (size_t t = 0; t < ps.settings.beamFormer.incoherentSettings.nrSamples; t++)
+ {
+ for (size_t c = 0; c < ps.settings.beamFormer.incoherentSettings.nrChannels; c++)
+ {
+ auto result_ref = out_ref.incoherentData().get_data(0,0,t,c);
+ auto result_sub = out_sub.incoherentData().get_data(0,0,t,c);
+ ASSERTSTR(fpEquals(result_ref, result_sub, 1e-4f),
+ "out.incoherentData.data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result_sub <<
+ "; reference = " << result_ref);
+ }
+ }
+
+ return 0;
+}
+
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.parset b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.parset
new file mode 100644
index 0000000000000000000000000000000000000000..cc40336318ac9461fb5f8b30b268029054e3529b
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.parset
@@ -0,0 +1,36 @@
+Cobalt.blockSize = 196608
+Cobalt.correctBandPass = T
+Cobalt.delayCompensation = T
+Observation.sampleClock = 200
+Observation.nrBitsPerSample = 8
+
+Observation.VirtualInstrument.stationList = [CS101,CS030,CS301,CS103,CS302,CS017,CS005,CS026,CS501,CS401,CS028,CS003,CS021,CS031,CS201,CS006,CS001,CS011,CS024,CS002,CS004,CS007,CS032]
+Observation.antennaSet = HBA_DUAL
+Observation.bandFilter = HBA_110_190
+
+Observation.nrBeams = 1
+Observation.Beam[0].subbandList = [101..110]
+Observation.Beam[0].nrTiedArrayBeams = 2
+Observation.Beam[0].TiedArrayBeam[0].angle1 = 0
+Observation.Beam[0].TiedArrayBeam[0].angle2 = 0
+Observation.Beam[0].TiedArrayBeam[0].coherent = F
+Observation.Beam[0].TiedArrayBeam[1].angle1 = 0
+Observation.Beam[0].TiedArrayBeam[1].angle2 = 0
+Observation.Beam[0].TiedArrayBeam[1].coherent = T
+
+Cobalt.BeamFormer.nrDelayCompensationChannels = 256
+Cobalt.BeamFormer.nrHighResolutionChannels = 256
+Cobalt.BeamFormer.CoherentStokes.which = I
+Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor = 1
+Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband = 1
+Cobalt.BeamFormer.IncoherentStokes.which = I
+Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor = 1
+Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband = 1
+
+Observation.DataProducts.Output_CoherentStokes.enabled=true
+Observation.DataProducts.Output_CoherentStokes.filenames=[tab1.raw]
+Observation.DataProducts.Output_CoherentStokes.locations=[1*:.]
+
+Observation.DataProducts.Output_IncoherentStokes.enabled=true
+Observation.DataProducts.Output_IncoherentStokes.filenames=[tab0.raw]
+Observation.DataProducts.Output_IncoherentStokes.locations=[1*:.]
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.sh b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.sh
new file mode 100755
index 0000000000000000000000000000000000000000..3fc9b4c37522c783aed012914a85ef3b9f058b2e
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerStationSubsetSubbandProcProcessSb.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+./runctest.sh tBeamFormerStationSubsetSubbandProcProcessSb
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandProcProcessSb.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandProcProcessSb.cc
index b3df8475e33705d1a39f405574af16ae8f47897c..a96042277fe8759908bd43b50e7fa1bcf6dccb61 100644
--- a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandProcProcessSb.cc
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandProcProcessSb.cc
@@ -30,7 +30,6 @@
#include <CoInterface/fpequals.h>
#include <GPUProc/gpu_utils.h>
#include <GPUProc/SubbandProcs/SubbandProc.h>
-#include <GPUProc/SubbandProcs/KernelFactories.h>
using namespace std;
using namespace LOFAR::Cobalt;
@@ -112,8 +111,7 @@ int main() {
// correction (but that kernel will run to convert int to float and to
// transform the data order).
- KernelFactories factories(ps);
- SubbandProc bwq(ps, ctx, factories);
+ SubbandProc bwq(ps, ctx);
SubbandProcInputData in(ps, ctx);
@@ -159,20 +157,23 @@ int main() {
in.delaysAfterEnd.get<float>()[i] = 0.0f;
for (size_t i = 0; i < in.phase0s.num_elements(); i++)
in.phase0s.get<float>()[i] = 0.0f;
- for (size_t i = 0; i < in.tabDelays.num_elements(); i++)
- in.tabDelays.get<float>()[i] = 0.0f;
+ for (auto tabDelays : in.tabDelays) {
+ for (size_t i = 0; i < tabDelays->num_elements(); i++)
+ tabDelays->get<float>()[i] = 0.0f;
+ }
SubbandProcOutputData out(ps, ctx);
- for (size_t i = 0; i < out.coherentData.num_elements(); i++)
- out.coherentData.get<float>()[i] = 42.0f;
- for (size_t i = 0; i < out.incoherentData.num_elements(); i++)
- out.incoherentData.get<float>()[i] = 42.0f;
+ for (size_t i = 0; i < out.coherentData().data_num_elements(); i++)
+ out.coherentData().get_data_origin()[i] = 42.0f;
+ for (size_t i = 0; i < out.incoherentData().data_num_elements(); i++)
+ out.incoherentData().get_data_origin()[i] = 42.0f;
// Don't bother initializing out.blockID; processSubband() doesn't need it.
cout << "processSubband()" << endl;
bwq.processSubband(in, out);
+ bwq.synchronize();
cout << "processSubband() done" << endl;
cout << "Output: " << endl;
@@ -197,11 +198,16 @@ int main() {
cout << "coherent outVal = " << coh_outVal << endl;
for (size_t t = 0; t < ps.settings.beamFormer.coherentSettings.nrSamples; t++)
+ {
for (size_t c = 0; c < ps.settings.beamFormer.coherentSettings.nrChannels; c++)
- ASSERTSTR(fpEquals(out.coherentData[0][0][t][c], coh_outVal, 1e-4f),
- "out.coherentData[0][0][" << t << "][" << c << "] = " <<
- setprecision(12) << out.coherentData[0][0][t][c] <<
+ {
+ auto result = out.coherentData().get_data(0,0,t,c);
+ ASSERTSTR(fpEquals(result, coh_outVal, 1e-4f),
+ "out.coherentData.data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result <<
"; outVal = " << coh_outVal);
+ }
+ }
// *** INCOHERENT STOKES ***
@@ -221,12 +227,17 @@ int main() {
cout << "incoherent outVal = " << incoh_outVal << endl;
for (size_t t = 0; t < ps.settings.beamFormer.incoherentSettings.nrSamples; t++)
+ {
for (size_t c = 0; c < ps.settings.beamFormer.incoherentSettings.nrChannels; c++)
- ASSERTSTR(fpEquals(out.incoherentData[0][0][t][c], incoh_outVal, 1e-4f),
- "out.incoherentData[0][0][" << t << "][" << c << "] = " <<
- setprecision(12) << out.incoherentData[0][0][t][c] <<
+ {
+ auto result = out.incoherentData().get_data(0,0,t,c);
+ ASSERTSTR(fpEquals(result, incoh_outVal, 1e-4f),
+ "out.incoherentData.data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result <<
"; outVal = " << incoh_outVal);
-
+ }
+ }
+
return 0;
}
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.cc
new file mode 100644
index 0000000000000000000000000000000000000000..6c418181843602f796562abc683d883cbc0b580e
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.cc
@@ -0,0 +1,280 @@
+//# tBeamFormerSubbandSubsetSubbandProcProcessSb: test of Beamformer subband processor.
+//#
+//# Copyright (C) 2020 ASTRON (Netherlands Institute for Radio Astronomy)
+//# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+//#
+//# This file is part of the LOFAR software suite.
+//# The LOFAR software suite is free software: you can redistribute it and/or
+//# modify it under the terms of the GNU General Public License as published
+//# by the Free Software Foundation, either version 3 of the License, or
+//# (at your option) any later version.
+//#
+//# The LOFAR software suite is distributed in the hope that it will be useful,
+//# but WITHOUT ANY WARRANTY; without even the implied warranty of
+//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//# GNU General Public License for more details.
+//#
+//# You should have received a copy of the GNU General Public License along
+//# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+//#
+//# $Id$
+
+#include <lofar_config.h>
+
+#include <complex>
+#include <cmath>
+#include <iomanip>
+
+#include <Common/LofarLogger.h>
+#include <CoInterface/Parset.h>
+#include <CoInterface/fpequals.h>
+#include <GPUProc/gpu_utils.h>
+#include <GPUProc/SubbandProcs/SubbandProc.h>
+
+using namespace std;
+using namespace LOFAR::Cobalt;
+using namespace LOFAR::TYPES;
+
+#define DUMMY_VALUE 42.0f
+
+template<typename T> T inputSignal(size_t t)
+{
+ size_t nrBits = sizeof(T) / 2 * 8;
+ double amp = (1 << (nrBits - 1)) - 1;
+ // Sine wave
+ double freq = (2 * 64.0 + 17.0) / 4096.0; // in samples
+ double angle = (double)t * 2.0 * M_PI * freq;
+ double s = ::sin(angle);
+ double c = ::cos(angle);
+ return T(::round(amp * c), ::round(amp * s));
+}
+
+void initialize_input(
+ Parset& ps,
+ SubbandProcInputData& in,
+ size_t subband)
+{
+ const size_t nrStations = ps.settings.antennaFieldNames.size();
+ const size_t nrSamplesPerSubband = ps.settings.blockSize;
+ const size_t nrPolarisations = ps.settings.nrPolarisations;
+ const size_t nrBitsPerSample = ps.settings.nrBitsPerSample;
+
+ // Initialize synthetic input to input signal
+ for (size_t st = 0; st < nrStations; st++) {
+ for (size_t i = 0; i < nrSamplesPerSubband; i++) {
+ size_t pol = i % nrPolarisations;
+ if (st == 0)
+ {
+ switch(nrBitsPerSample) {
+ case 8:
+ reinterpret_cast<i8complex&>(in.inputSamples[st][i][pol][0]) = inputSignal<i8complex>(i);
+ break;
+ case 16:
+ reinterpret_cast<i16complex&>(in.inputSamples[st][i][pol][0]) = inputSignal<i16complex>(i);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ }
+
+ // Initialize subbands partitioning administration (struct BlockID).
+
+ // Block number: 0 .. inf
+ in.blockID.block = 0;
+
+ // Subband index in the observation: [0, ps.nrSubbands())
+ in.blockID.globalSubbandIdx = subband;
+
+ // Subband index for this pipeline/workqueue: [0, subbandIndices.size())
+ in.blockID.localSubbandIdx = subband;
+
+ // Subband index for this SubbandProc
+ in.blockID.subbandProcSubbandIdx = subband;
+
+ // Initialize delays. We skip delay compensation, but init anyway,
+ // so we won't copy uninitialized data to the device.
+ for (size_t i = 0; i < in.delaysAtBegin.num_elements(); i++)
+ in.delaysAtBegin.get<float>()[i] = 0.0f;
+ for (size_t i = 0; i < in.delaysAfterEnd.num_elements(); i++)
+ in.delaysAfterEnd.get<float>()[i] = 0.0f;
+ for (size_t i = 0; i < in.phase0s.num_elements(); i++)
+ in.phase0s.get<float>()[i] = 0.0f;
+ for (auto tabDelays : in.tabDelays) {
+ for (size_t i = 0; i < tabDelays->num_elements(); i++)
+ tabDelays->get<float>()[i] = 0.0f;
+ }
+}
+
+void initialize_output(
+ SubbandProcOutputData& out)
+{
+ for (size_t p = 0; p < out.coherentDatas.size(); p++)
+ for (size_t i = 0; i < out.coherentData().data_num_elements(); i++)
+ out.coherentData(p).get_data_origin()[i] = DUMMY_VALUE;
+ for (size_t p = 0; p < out.incoherentDatas.size(); p++)
+ for (size_t i = 0; i < out.incoherentData().data_num_elements(); i++)
+ out.incoherentData(p).get_data_origin()[i] = DUMMY_VALUE;
+}
+
+
+int main() {
+ INIT_LOGGER("tBeamFormerStationSubsetSubbandProcProcessSb");
+
+ try {
+ gpu::Platform pf;
+ cout << "Detected " << pf.size() << " CUDA devices" << endl;
+ } catch (gpu::CUDAException& e) {
+ cerr << e.what() << endl;
+ return 3;
+ }
+
+ gpu::Device device(0);
+ vector<gpu::Device> devices(1, device);
+ gpu::Context ctx(device);
+
+ Parset ps("tBeamFormerSubbandSubsetSubbandProcProcessSb.parset");
+
+ // Input array sizes
+ const size_t nrBeams = ps.settings.SAPs.size();
+ const size_t nrStations = ps.settings.antennaFieldNames.size();
+ const size_t nrBFStations = ps.settings.beamFormer.pipelines[0].antennaFieldNames.size();
+ const size_t nrPolarisations = ps.settings.nrPolarisations;
+ const size_t nrSamplesPerSubband = ps.settings.blockSize;
+ const size_t nrBitsPerSample = ps.settings.nrBitsPerSample;
+ const size_t nrBytesPerComplexSample = ps.nrBytesPerComplexSample();
+
+ const unsigned fftSize = ps.settings.beamFormer.nrDelayCompensationChannels;
+
+ // We only support 8-bit or 16-bit input samples
+ ASSERT(nrBitsPerSample == 8 || nrBitsPerSample == 16);
+
+ LOG_INFO_STR(
+ "Input info:" <<
+ "\n nrBeams = " << nrBeams <<
+ "\n nrStations = " << nrStations <<
+ "\n nrBFStations = " << nrBFStations <<
+ "\n nrPolarisations = " << nrPolarisations <<
+ "\n nrSamplesPerSubband = " << nrSamplesPerSubband <<
+ "\n nrBitsPerSample = " << nrBitsPerSample <<
+ "\n nrBytesPerComplexSample = " << nrBytesPerComplexSample <<
+ "\n fftSize = " << fftSize);
+
+ // Create very simple kernel programs, with predictable output. Skip as much
+ // as possible. Nr of channels/sb from the parset is 1, so the PPF will not
+ // even run. Parset also has turned of delay compensation and bandpass
+ // correction (but that kernel will run to convert int to float and to
+ // transform the data order).
+
+ // Initialize reference parset (copy of the base parset)
+ auto ps_ref = ps;
+
+ // Initialize parset with two beamFormer pipelines, each with a different subband selection
+ auto ps_sub = ps;
+ ps_sub.replace("Cobalt.BeamFormer.nrPipelines", "2");
+ ps_sub.replace("Cobalt.BeamFormer.Pipeline[0].Beam[0].subbandList", "[101..110]");
+ ps_sub.replace("Cobalt.BeamFormer.Pipeline[1].Beam[0].subbandList", "[105]");
+ ps_sub.updateSettings();
+
+ // Initialize SubbandProcs
+ SubbandProc bwq_ref(ps_ref, ctx);
+ SubbandProc bwq_sub(ps_sub, ctx);
+
+ // Iterate all subbands
+ for (auto subband : ps.settings.subbandIndices())
+ {
+ // Initialize input data
+ SubbandProcInputData in_ref(ps_ref, ctx);
+ initialize_input(ps_ref, in_ref, subband);
+ SubbandProcInputData in_sub(ps_sub, ctx);
+ initialize_input(ps_sub, in_sub, subband);
+
+ // Sanity check: the input data objects should be the same size
+ ASSERTSTR(in_ref.inputSamples.num_elements() == in_sub.inputSamples.num_elements(), "length(in_ref) != length(in_sub)");
+
+ // Initialize output data
+ SubbandProcOutputData out_ref(ps_ref, ctx);
+ initialize_output(out_ref);
+ SubbandProcOutputData out_sub(ps_sub, ctx);
+ initialize_output(out_sub);
+
+ // Process subbands
+ cout << "processSubband(" << subband << ")" << endl;
+ bwq_ref.processSubband(in_ref, out_ref);
+ bwq_ref.synchronize();
+ bwq_sub.processSubband(in_sub, out_sub);
+ bwq_sub.synchronize();
+ cout << "processSubband(" << subband << ") done" << endl;
+
+ // Output verification
+
+ // *** COHERENT STOKES ***
+
+ for (size_t t = 0; t < ps.settings.beamFormer.coherentSettings.nrSamples; t++)
+ {
+ for (size_t c = 0; c < ps.settings.beamFormer.coherentSettings.nrChannels; c++)
+ {
+ auto result_ref = out_ref.coherentData().get_data(0,0,t,c);
+
+ // The first pipeline should produce the same output as the reference
+ auto result_sub = out_sub.coherentData(0).get_data(0,0,t,c);
+ ASSERTSTR(fpEquals(result_ref, result_sub, 1e-4f),
+ "out.coherentData(0).data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result_sub <<
+ "; reference = " << result_ref);
+ ASSERTSTR(result_ref != 0, "Oei: " << result_ref);
+
+ // The second pipeline should only produce output for one subband,
+ // for other subbands it should not touch the output.
+ result_sub = out_sub.coherentData(1).get_data(0,0,t,c);
+ if (subband == 4)
+ {
+ ASSERTSTR(fpEquals(result_ref, result_sub, 1e-4f),
+ "out.coherentData(1).data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result_sub <<
+ "; reference = " << result_ref);
+ } else {
+ ASSERTSTR(DUMMY_VALUE == result_sub,
+ "out.coherentData(1).data[0][0][" << t << "][" << c << "] = " <<
+ result_sub << "; reference = " << DUMMY_VALUE);
+ }
+ }
+ }
+
+ // *** INCOHERENT STOKES ***
+
+ for (size_t t = 0; t < ps.settings.beamFormer.incoherentSettings.nrSamples; t++)
+ {
+ for (size_t c = 0; c < ps.settings.beamFormer.incoherentSettings.nrChannels; c++)
+ {
+ auto result_ref = out_ref.incoherentData().get_data(0,0,t,c);
+
+ // The first pipeline should produce the same output as the reference
+ auto result_sub = out_sub.incoherentData(0).get_data(0,0,t,c);
+ ASSERTSTR(fpEquals(result_ref, result_sub, 1e-4f),
+ "out.incoherentData(0).data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result_sub <<
+ "; reference = " << result_ref);
+ ASSERTSTR(result_ref != 0, "Oei: " << result_ref);
+
+ // The second pipeline should only produce output for one subband,
+ // for other subbands it should not touch the output.
+ result_sub = out_sub.incoherentData(1).get_data(0,0,t,c);
+ if (subband == 4)
+ {
+ ASSERTSTR(fpEquals(result_ref, result_sub, 1e-4f),
+ "out.incoherentData(1).data[0][0][" << t << "][" << c << "] = " <<
+ setprecision(12) << result_sub <<
+ "; reference = " << result_ref);
+ } else {
+ ASSERTSTR(DUMMY_VALUE == result_sub,
+ "out.incoherentData(1).data[0][0][" << t << "][" << c << "] = " <<
+ result_sub << "; reference = " << DUMMY_VALUE);
+ }}
+ }
+ }
+
+ return 0;
+}
+
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.parset b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.parset
new file mode 100644
index 0000000000000000000000000000000000000000..69e0f044be98fa11b856368120f8d05c0a64dfb7
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.parset
@@ -0,0 +1,36 @@
+Cobalt.correctBandPass = F
+Cobalt.delayCompensation = F
+Observation.sampleClock = 200
+Observation.nrBitsPerSample = 16
+Cobalt.blockSize = 65536
+Cobalt.BeamFormer.nrDelayCompensationChannels = 64
+
+Observation.VirtualInstrument.stationList = [CS001,CS002,CS003]
+Observation.antennaSet = HBA_ZERO
+Observation.nrBeams = 1
+Observation.Beam[0].subbandList = [101..110]
+Observation.Beam[0].nrTiedArrayBeams = 2
+Observation.Beam[0].TiedArrayBeam[0].angle1 = 0
+Observation.Beam[0].TiedArrayBeam[0].angle2 = 0
+Observation.Beam[0].TiedArrayBeam[0].coherent = F
+Observation.Beam[0].TiedArrayBeam[1].angle1 = 0
+Observation.Beam[0].TiedArrayBeam[1].angle2 = 0
+Observation.Beam[0].TiedArrayBeam[1].coherent = T
+
+Cobalt.BeamFormer.stationList = [CS001,CS003]
+Cobalt.BeamFormer.CoherentStokes.which = I # IQUV
+Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor = 1
+Cobalt.BeamFormer.CoherentStokes.nrChannelsPerSubband = 1
+Cobalt.BeamFormer.IncoherentStokes.which = I # IQUV
+Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor = 1
+Cobalt.BeamFormer.IncoherentStokes.nrChannelsPerSubband = 1
+Observation.rspBoardList = [10*0]
+Observation.rspSlotList = [10*0]
+
+Observation.DataProducts.Output_CoherentStokes.enabled=true
+Observation.DataProducts.Output_CoherentStokes.filenames=[ctab0.raw,ctab1.raw]
+Observation.DataProducts.Output_CoherentStokes.locations=[2*:.]
+
+Observation.DataProducts.Output_IncoherentStokes.enabled=true
+Observation.DataProducts.Output_IncoherentStokes.filenames=[itab0.raw,itab1.raw]
+Observation.DataProducts.Output_IncoherentStokes.locations=[2*:.]
\ No newline at end of file
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.sh b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.sh
new file mode 100755
index 0000000000000000000000000000000000000000..9dabd1b45638f22f2b3d92f75628c95e3f33758d
--- /dev/null
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tBeamFormerSubbandSubsetSubbandProcProcessSb.sh
@@ -0,0 +1,2 @@
+#!/bin/sh
+./runctest.sh tBeamFormerSubbandSubsetSubbandProcProcessSb
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCoherentStokesBeamFormerSubbandProcProcessSb.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCoherentStokesBeamFormerSubbandProcProcessSb.cc
index 3220febe8e078307ae10cb9d5038da035d42ad8f..a941e90bce02fcecc955353ec2485cbd9a307a1c 100644
--- a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCoherentStokesBeamFormerSubbandProcProcessSb.cc
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCoherentStokesBeamFormerSubbandProcProcessSb.cc
@@ -30,7 +30,6 @@
#include <CoInterface/fpequals.h>
#include <GPUProc/gpu_utils.h>
#include <GPUProc/SubbandProcs/SubbandProc.h>
-#include <GPUProc/SubbandProcs/KernelFactories.h>
#include "../Kernels/KernelTestHelpers.h"
@@ -76,7 +75,7 @@ int main(/*int argc, char *argv[]*/) {
Parset ps("tCoherentStokesBeamFormerSubbandProcProcessSb.parset");
// Parset ps;
- KernelParameters params;
+ KernelTestParameters params;
// override the faults
/*
params.nStation = 5;
@@ -139,12 +138,9 @@ int main(/*int argc, char *argv[]*/) {
// correction (but that kernel will run to convert int to float and to
// transform the data order).
- KernelFactories factories(ps);
- SubbandProc bwq(ps, ctx, factories);
+ SubbandProc bwq(ps, ctx);
- SubbandProcInputData in(
- nrBeams, nrStations, nrPolarisations, maxNrTABsPerSAP,
- nrSamplesPerSubband, nrBytesPerComplexSample, ctx);
+ SubbandProcInputData in(ps, ctx);
// Initialize synthetic input to input signal
for (size_t st = 0; st < nrStations; st++) {
@@ -188,18 +184,21 @@ int main(/*int argc, char *argv[]*/) {
in.delaysAfterEnd.get<float>()[i] = 0.0f;
for (size_t i = 0; i < in.phase0s.num_elements(); i++)
in.phase0s.get<float>()[i] = 0.0f;
- for (size_t i = 0; i < in.tabDelays.num_elements(); i++)
- in.tabDelays.get<float>()[i] = 0.0f;
+ for (auto tabDelays : in.tabDelays) {
+ for (size_t i = 0; i < tabDelays->num_elements(); i++)
+ tabDelays->get<float>()[i] = 0.0f;
+ }
SubbandProcOutputData out(ps, ctx);
- for (size_t i = 0; i < out.coherentData.num_elements(); i++)
- out.coherentData.get<float>()[i] = 42.0f;
+ for (size_t i = 0; i < out.coherentData().data_num_elements(); i++)
+ out.coherentData().get_data_origin()[i] = 42.0f;
// Don't bother initializing out.blockID; processSubband() doesn't need it.
cout << "processSubband()" << endl;
bwq.processSubband(in, out);
+ bwq.synchronize();
cout << "processSubband() done" << endl;
cout << "Output: " << endl;
@@ -231,9 +230,10 @@ int main(/*int argc, char *argv[]*/) {
for (size_t t = 0; t < nrSamples; t++)
for (size_t c = 0; c < nrChannels; c++)
{
- ASSERTSTR(fpEquals(out.coherentData[tab][s][t][c], outVal, 1e-4f),
- "out.coherentData[" << tab << "][" << s << "][" << t << "][" << c << "] = " << setprecision(12) <<
- out.coherentData[tab][s][t][c] << "; outVal = " << outVal);
+ float result = out.coherentData().get_data(tab, s, t, c);
+ ASSERTSTR(fpEquals(result, outVal, 1e-4f),
+ "out.coherentData.data[" << tab << "][" << s << "][" << t << "][" << c << "] = " << setprecision(12) <<
+ result << "; outVal = " << outVal);
}
}
return 0;
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorStep.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorStep.cc
index eb1850b2e75ef7bfbd6a5b1abbc909af05c4d574..2e18b21e8fa4f256c2af197bef645ad1d54a0692 100644
--- a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorStep.cc
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorStep.cc
@@ -158,7 +158,12 @@ TEST(propagateFlags)
0);
// calculate nr of valid samples per baseline
- CorrelatorStep::Flagger::calcNrValidSamples(parset, flagsPerChannel, correlatedData);
+ CorrelatorStep::Flagger::calcNrValidSamples(
+ parset.settings.correlator.stations.size(),
+ parset.settings.correlator.nrSamplesPerIntegration(),
+ parset.settings.correlator.nrIntegrationsPerBlock,
+ parset.settings.correlator.nrChannels,
+ flagsPerChannel, correlatedData);
processCPUTimer.stop();
}
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProc.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProc.cc
index faba46123b8a11e881710bac73a9c332a244471b..759058706146b548718f5e4ca89caee4522c3e83 100644
--- a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProc.cc
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProc.cc
@@ -100,10 +100,17 @@ TEST(propagateFlags)
NR_TAPS - 1);
// calculate nr of valid samples per baseline
- CorrelatorStep::Flagger::calcNrValidSamples(parset, flagsPerChannel, output);
+ CorrelatorStep::Flagger::calcNrValidSamples(
+ parset.settings.correlator.stations.size(),
+ parset.settings.correlator.nrSamplesPerBlock,
+ parset.settings.correlator.nrIntegrationsPerBlock,
+ parset.settings.correlator.nrChannels,
+ flagsPerChannel, output);
// now perform weighting of the data based on the number of valid samples
- CorrelatorStep::Flagger::applyNrValidSamples(parset, *output.subblocks[0]);
+ CorrelatorStep::Flagger::applyNrValidSamples(
+ parset.settings.correlator.nrChannels,
+ *output.subblocks[0]);
// *********************************************************************************************
// Now validate the functionality:
@@ -202,10 +209,15 @@ TEST(calcNrValidSamples4Channels)
//insert same cases
flagsPerChannel[0].include(100,111);//A.
flagsPerChannel[1].include(111,120);//E. second station flags
-
+
//propageFlags
- CorrelatorStep::Flagger::calcNrValidSamples(parset, flagsPerChannel, output);
-
+ CorrelatorStep::Flagger::calcNrValidSamples(
+ parset.settings.correlator.stations.size(),
+ parset.settings.correlator.nrSamplesPerBlock,
+ parset.settings.correlator.nrIntegrationsPerBlock,
+ parset.settings.correlator.nrChannels,
+ flagsPerChannel, output);
+
// Now check that the flags are correctly set in the ouput object
CHECK_EQUAL(256u - 11u, output.subblocks[0]->getNrValidSamples(0,1)); // 11 flagged in station 1
@@ -254,10 +266,15 @@ TEST(calcNrValidSamples1Channels)
//insert same cases
flagsPerChannel[0].include(100,111);//A.
flagsPerChannel[1].include(111,120);//E. second station flags
-
+
//propageFlags
- CorrelatorStep::Flagger::calcNrValidSamples(parset, flagsPerChannel, output);
-
+ CorrelatorStep::Flagger::calcNrValidSamples(
+ parset.settings.correlator.stations.size(),
+ parset.settings.correlator.nrSamplesPerBlock,
+ parset.settings.correlator.nrIntegrationsPerBlock,
+ parset.settings.correlator.nrChannels,
+ flagsPerChannel, output);
+
// Now check that the flags are correctly set in the ouput object
// channel is 1 so no time resolution loss!!
CHECK_EQUAL(245u, output.subblocks[0]->getNrValidSamples(0,0)); // 11 flagged in station 1
@@ -305,7 +322,7 @@ TEST(applyNrValidSamples)
output.subblocks[0]->setNrValidSamples(0,1,n_valid_samples); //baseline 0, channel 1
output.subblocks[0]->setNrValidSamples(1,1,256); //baseline 1, channel 1
output.subblocks[0]->setNrValidSamples(2,1,0); //baseline 0, channel 1
- CorrelatorStep::Flagger::applyNrValidSamples(parset, *output.subblocks[0]);
+ CorrelatorStep::Flagger::applyNrValidSamples(parset.settings.correlator.nrChannels, *output.subblocks[0]);
// 4 channels: therefore the chanel zero should be zero
CHECK_EQUAL(std::complex<float>(0,0), visibilities[0][0][0][0]);
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProcProcessSb.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProcProcessSb.cc
index 6244d056365b1a0e5e47135dc73973ac2e16ff4b..fdaf16820fbeb729974c1e91dae43a4088ccca3f 100644
--- a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProcProcessSb.cc
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tCorrelatorSubbandProcProcessSb.cc
@@ -26,7 +26,6 @@
#include <CoInterface/fpequals.h>
#include <CoInterface/Parset.h>
#include <GPUProc/gpu_utils.h>
-#include <GPUProc/SubbandProcs/KernelFactories.h>
#include <GPUProc/SubbandProcs/SubbandProc.h>
#include <UnitTest++.h>
@@ -46,7 +45,6 @@ struct SubbandProcWrapper {
gpu::Context ctx;
Parset ps;
- KernelFactories factories;
SubbandProc cwq;
SubbandProcInputData in;
SubbandProcOutputData out;
@@ -57,15 +55,8 @@ struct SubbandProcWrapper {
device(0),
ctx(device),
ps(ps),
- factories(ps, 1),
- cwq(ps, ctx, factories),
- in(ps.settings.SAPs.size(),
- ps.settings.antennaFields.size(),
- ps.settings.nrPolarisations,
- ps.settings.beamFormer.maxNrTABsPerSAP(),
- ps.settings.blockSize,
- ps.nrBytesPerComplexSample(),
- ctx),
+ cwq(ps, ctx),
+ in(ps, ctx),
out(ps, ctx),
inputValue(1,1)
@@ -162,6 +153,7 @@ struct SubbandProcWrapper {
in.blockID.block = block;
cwq.processSubband(in, out);
+ cwq.synchronize();
if (block >= 0)
cwq.postprocessSubband(out);
}
diff --git a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tFlysEyeBeamFormerSubbandProcProcessSb.cc b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tFlysEyeBeamFormerSubbandProcProcessSb.cc
index 642398919d064b0be7225d5f291f161044ff12c8..462cb16aa15276b9624347818758c159f202cafb 100644
--- a/RTCP/Cobalt/GPUProc/test/SubbandProcs/tFlysEyeBeamFormerSubbandProcProcessSb.cc
+++ b/RTCP/Cobalt/GPUProc/test/SubbandProcs/tFlysEyeBeamFormerSubbandProcProcessSb.cc
@@ -31,7 +31,6 @@
#include <CoInterface/fpequals.h>
#include <GPUProc/gpu_utils.h>
#include <GPUProc/SubbandProcs/SubbandProc.h>
-#include <GPUProc/SubbandProcs/KernelFactories.h>
using namespace std;
using namespace LOFAR::Cobalt;
@@ -127,12 +126,10 @@ int main() {
// correction (but that kernel will run to convert int to float and to
// transform the data order).
- KernelFactories factories(ps);
- SubbandProc bwq(ps, ctx, factories);
- SubbandProcInputData in(
- nrBeams, nrStations, nrPolarisations, nrTABs,
- nrSamplesPerSubband, nrBytesPerComplexSample, ctx);
+ SubbandProc bwq(ps, ctx);
+
+ SubbandProcInputData in(ps, ctx);
// Initialize synthetic input to input signal
for (size_t st = 0; st < nrStations; st++) {
@@ -176,19 +173,23 @@ int main() {
in.delaysAfterEnd.get<float>()[i] = 0.0f;
for (size_t i = 0; i < in.phase0s.num_elements(); i++)
in.phase0s.get<float>()[i] = 0.0f;
- for (size_t i = 0; i < in.tabDelays.num_elements(); i++)
- in.tabDelays.get<float>()[i] = 0.0f;
+ for (auto& tabDelays : in.tabDelays) {
+ for (size_t i = 0; i < tabDelays->num_elements(); i++)
+ tabDelays->get<float>()[i] = 0.0f;
+ }
+
// Allocate buffer for output signal
SubbandProcOutputData out(ps, ctx);
- for (size_t i = 0; i < out.coherentData.num_elements(); i++)
- out.coherentData.get<float>()[i] = 42.0f;
+ for (size_t i = 0; i < out.coherentData().data_num_elements(); i++)
+ out.coherentData().get_data_origin()[i] = 42.0f;
// Don't bother initializing out.blockID; processSubband() doesn't need it.
cout << "processSubband()" << endl;
bwq.processSubband(in, out);
+ bwq.synchronize();
cout << "processSubband() done" << endl;
cout << "Output: " << endl;
@@ -215,10 +216,14 @@ int main() {
for (size_t s = 0; s < nrStokes; s++)
for (size_t t = 0; t < nrSamples; t++)
for (size_t c = 0; c < nrChannels; c++)
- ASSERTSTR(fpEquals(out.coherentData[tab][s][t][c], outVal, 1e-4f),
- "out.coherentData[" << tab << "][" << s << "][" << t << "][" << c <<
- "] = " << setprecision(12) << out.coherentData[tab][s][t][c] <<
+ {
+ float result = out.coherentData().get_data(tab,s,t,c);
+ ASSERTSTR(fpEquals(result, outVal, 1e-4f),
+ "out.coherentData.data[" << tab << "][" << s << "][" << t << "][" << c <<
+ "] = " << setprecision(12) << result <<
"; outVal = " << outVal);
+ }
+
return 0;
}
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tBandPassCorrection.cc b/RTCP/Cobalt/GPUProc/test/cuda/tBandPassCorrection.cc
index 866a4e9cd1655d97b1701694645cb4577670a267..0cc21a49b8874fc2bf3033c3ea9ec5b1f1f5fbce 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tBandPassCorrection.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tBandPassCorrection.cc
@@ -50,13 +50,12 @@ gpu::Stream *stream;
// default compile definitions
const unsigned NR_STATIONS = 48;
-const unsigned NR_CHANNELS_1 = 64;
-const unsigned NR_CHANNELS_2 = 32;
+const unsigned NR_CHANNELS = 64;
const unsigned NR_SAMPLES_PER_CHANNEL = 32;
const unsigned NR_BITS_PER_SAMPLE = 8;
const unsigned NR_POLARIZATIONS = 2;
-const unsigned nrResultVals = NR_STATIONS * NR_CHANNELS_1 * NR_CHANNELS_2*
+const unsigned nrResultVals = NR_STATIONS * NR_CHANNELS *
NR_SAMPLES_PER_CHANNEL * NR_POLARIZATIONS;
// Initialize input AND output before calling runKernel().
@@ -65,7 +64,7 @@ const unsigned nrResultVals = NR_STATIONS * NR_CHANNELS_1 * NR_CHANNELS_2*
template <typename T>
void runKernel(gpu::Function kfunc,
MultiDimArrayHostBuffer<fcomplex, 4> &outputData,
- MultiDimArrayHostBuffer<T, 5> &inputData,
+ MultiDimArrayHostBuffer<T, 4> &inputData,
MultiDimArrayHostBuffer<float, 1> &bandPassFactors)
{
gpu::Context ctx(stream->getContext());
@@ -80,9 +79,7 @@ void runKernel(gpu::Function kfunc,
// This test is fragile for it no uses the Kernel class interface
gpu::Grid globalWorkSize(NR_SAMPLES_PER_CHANNEL / 16,
- NR_CHANNELS_1 / 16,
- NR_CHANNELS_2
- );
+ NR_CHANNELS / 16);
gpu::Block localWorkSize(16, 16, 1);
@@ -118,10 +115,8 @@ CompileDefinitions getDefaultCompileDefinitions()
defs["NR_STATIONS"] =
boost::lexical_cast<string>(NR_STATIONS);
- defs["NR_CHANNELS_1"] =
- boost::lexical_cast<string>(NR_CHANNELS_1);
- defs["NR_CHANNELS_2"] =
- boost::lexical_cast<string>(NR_CHANNELS_2);
+ defs["NR_CHANNELS"] =
+ boost::lexical_cast<string>(NR_CHANNELS);
defs["NR_SAMPLES_PER_CHANNEL"] =
boost::lexical_cast<string>(NR_SAMPLES_PER_CHANNEL);
defs["NR_POLARIZATIONS"] =
@@ -142,27 +137,26 @@ vector<fcomplex> runTest(const CompileDefinitions& compileDefs)
gpu::Context ctx(stream->getContext());
boost::scoped_ptr<MultiDimArrayHostBuffer<fcomplex, 4> > outputData;
- boost::scoped_ptr<MultiDimArrayHostBuffer<T, 5> > inputData;
+ boost::scoped_ptr<MultiDimArrayHostBuffer<T, 4> > inputData;
outputData.reset(
new MultiDimArrayHostBuffer<fcomplex, 4>(boost::extents
[NR_STATIONS]
- [NR_CHANNELS_1 * NR_CHANNELS_2]
+ [NR_CHANNELS]
[NR_SAMPLES_PER_CHANNEL]
[NR_POLARIZATIONS],
ctx));
inputData.reset(
- new MultiDimArrayHostBuffer<fcomplex, 5>(boost::extents
+ new MultiDimArrayHostBuffer<fcomplex, 4>(boost::extents
[NR_STATIONS]
[NR_POLARIZATIONS]
- [NR_CHANNELS_1]
- [NR_SAMPLES_PER_CHANNEL]
- [NR_CHANNELS_2],
+ [NR_CHANNELS]
+ [NR_SAMPLES_PER_CHANNEL],
ctx));
MultiDimArrayHostBuffer<float, 1> bandPassFactors(
- boost::extents [NR_CHANNELS_1 * NR_CHANNELS_2], ctx);
+ boost::extents [NR_CHANNELS], ctx);
// set inputs
for (size_t i = 0; i < inputData->num_elements(); i++) {
@@ -192,7 +186,7 @@ vector<fcomplex> runTest(const CompileDefinitions& compileDefs)
if(false)
{
for(size_t idx1 = 0; idx1 < NR_STATIONS; ++ idx1)
- for(size_t idx2 = 0; idx2 < NR_CHANNELS_1 * NR_CHANNELS_2; ++ idx2)
+ for(size_t idx2 = 0; idx2 < NR_CHANNELS; ++ idx2)
{
for(size_t idx3 = 0; idx3 < NR_SAMPLES_PER_CHANNEL; ++ idx3)
{
@@ -223,7 +217,7 @@ TEST(BandPass)
CHECK_CLOSE(0.0, results[0].real(), 0.000001);
// Last element is (array_elements - 1) / array_elements index start at 0 until max-1
- CHECK_CLOSE((NR_CHANNELS_1 * NR_CHANNELS_2 * 1.0 - 1.0) / (NR_CHANNELS_1 * NR_CHANNELS_2),
+ CHECK_CLOSE((NR_CHANNELS * 1.0 - 1.0) / (NR_CHANNELS),
results[1].real(), 0.000001);
}
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tBeamFormer.cc b/RTCP/Cobalt/GPUProc/test/cuda/tBeamFormer.cc
index df9e5e773e8f1ad054402d900a7942b79d9d7a50..184581384a17209595da7cf8c2fd29c955886c3d 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tBeamFormer.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tBeamFormer.cc
@@ -111,7 +111,8 @@ HostMemory runTest(Context ctx,
// Compile to ptx
// Set op string string pairs to be provided to the compiler as defines
definitions["NVIDIA_CUDA"] = "";
- definitions["NR_STATIONS"] = lexical_cast<string>(NR_STATIONS);
+ definitions["NR_INPUT_STATIONS"] = lexical_cast<string>(NR_STATIONS);
+ definitions["NR_OUTPUT_STATIONS"] = lexical_cast<string>(NR_STATIONS);
definitions["NR_CHANNELS"] = lexical_cast<string>(NR_CHANNELS);
definitions["NR_SAMPLES_PER_CHANNEL"] = lexical_cast<string>(NR_SAMPLES_PER_CHANNEL);
definitions["NR_SAPS"] = lexical_cast<string>(NR_SAPS);
@@ -164,10 +165,11 @@ HostMemory runTest(Context ctx,
// Run the kernel on the created data
hKernel.setArg(0, devComplexVoltagesMemory);
hKernel.setArg(1, devBandPassCorrectedMemory);
- hKernel.setArg(2, devDelaysMemory);
- hKernel.setArg(3, devStationIndices);
- hKernel.setArg(4, subbandFrequency);
- hKernel.setArg(5, sap);
+ hKernel.setArg(2, devStationIndices);
+ hKernel.setArg(3, devDelaysMemory);
+ hKernel.setArg(4, devStationIndices);
+ hKernel.setArg(5, subbandFrequency);
+ hKernel.setArg(6, sap);
// Calculate the number of threads in total and per block
Grid globalWorkSize(1, 1, 1);
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tCoherentStokesTranspose.cc b/RTCP/Cobalt/GPUProc/test/cuda/tCoherentStokesTranspose.cc
index 3ed91b4de84dbacebdb62ebed6c3edff1156e904..7f4a17ab44b9a721ddbe2e8ed97e6c760b21a6c7 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tCoherentStokesTranspose.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tCoherentStokesTranspose.cc
@@ -62,7 +62,10 @@ void runTest( Context &ctx, Stream &stream )
ps.add("Observation.DataProducts.Output_CoherentStokes.locations", "[localhost:.]");
ps.updateSettings();
- CoherentStokesTransposeKernel::Parameters params(ps);
+ CoherentStokesTransposeKernel::Parameters params(
+ NR_CHANNELS,
+ NR_SAMPLES_PER_CHANNEL,
+ NR_TABS);
params.nrChannels = NR_CHANNELS;
params.nrSamplesPerChannel = NR_SAMPLES_PER_CHANNEL;
params.nrTABs = NR_TABS;
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tCorrelator2.cc b/RTCP/Cobalt/GPUProc/test/cuda/tCorrelator2.cc
index 793402232852a1894ae71597811c921b64a63b4e..1537b330d427f43ef1cdf28ef89955ce7b53d72f 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tCorrelator2.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tCorrelator2.cc
@@ -78,7 +78,11 @@ void runTest(
ps.add("Cobalt.Correlator.nrIntegrationsPerBlock", str(format("%u") % NR_INTEGRATIONS));
ps.updateSettings();
- CorrelatorKernel::Parameters params(ps);
+ CorrelatorKernel::Parameters params(
+ ps.settings.antennaFields.size(),
+ ps.settings.correlator.nrChannels,
+ ps.settings.correlator.nrSamplesPerBlock / ps.settings.correlator.nrIntegrationsPerBlock,
+ ps.settings.correlator.nrIntegrationsPerBlock);
KernelFactory<CorrelatorKernel> factory(params);
// Define dummy Block-ID
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tCorrelatorPerformance.cc b/RTCP/Cobalt/GPUProc/test/cuda/tCorrelatorPerformance.cc
index fd5477af2e5dbd4c2b5961714f12ef2a58a30d96..71627f10964e7e36f9eb118256ea14d8f64a6758 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tCorrelatorPerformance.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tCorrelatorPerformance.cc
@@ -72,7 +72,11 @@ void runTest(
ps.add("Cobalt.Correlator.nrIntegrationsPerBlock", str(format("%u") % NR_INTEGRATIONS));
ps.updateSettings();
- CorrelatorKernel::Parameters params(ps);
+ CorrelatorKernel::Parameters params(
+ ps.settings.antennaFields.size(),
+ ps.settings.correlator.nrChannels,
+ ps.settings.correlator.nrSamplesPerBlock / ps.settings.correlator.nrIntegrationsPerBlock,
+ ps.settings.correlator.nrIntegrationsPerBlock);
KernelFactory<CorrelatorKernel> factory(params);
// Define dummy Block-ID
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tFFT.cc b/RTCP/Cobalt/GPUProc/test/cuda/tFFT.cc
index e5e73b360fb2453215cbe49e8b037a75ab9f843d..881cff517f3f78a6fda7d8871aa402954c951735 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tFFT.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tFFT.cc
@@ -30,7 +30,6 @@
#include <Common/LofarLogger.h>
#include <Common/LofarTypes.h>
#include <CoInterface/BlockID.h>
-#include <CoInterface/SmartPtr.h>
#include <GPUProc/Kernels/FFT_Kernel.h>
#include <GPUProc/KernelFactory.h>
#include <GPUProc/PerformanceCounter.h>
@@ -95,8 +94,8 @@ int main() {
KernelFactory<FFT_Kernel> factoryFwd(FFT_Kernel::Parameters(fftSize, size, true));
KernelFactory<FFT_Kernel> factoryBwd(FFT_Kernel::Parameters(fftSize, size, false));
- SmartPtr<FFT_Kernel> fftFwdKernel(factoryFwd.create(stream, d_inout, d_inout));
- SmartPtr<FFT_Kernel> fftBwdKernel(factoryBwd.create(stream, d_inout, d_inout));
+ std::unique_ptr<FFT_Kernel> fftFwdKernel(factoryFwd.create(stream, d_inout, d_inout));
+ std::unique_ptr<FFT_Kernel> fftBwdKernel(factoryBwd.create(stream, d_inout, d_inout));
// FFTW buffers and plans
ASSERT(fftw_init_threads() != 0);
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tGPUWrapper.cc b/RTCP/Cobalt/GPUProc/test/cuda/tGPUWrapper.cc
index 1e1bb0f2ca3fa373403c1d4ab48d847224f58554..c055627494ab45d3e2da9a8574a7812a656df146 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tGPUWrapper.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tGPUWrapper.cc
@@ -231,6 +231,46 @@ SUITE(Memory) {
}
}
}
+
+ TEST(TransferWithOffset) {
+ Platform pf;
+ vector<Device> devices(pf.devices());
+
+ for (vector<Device>::const_iterator i = devices.begin(); i != devices.end(); ++i) {
+ Context ctx(*i);
+ Stream s(ctx);
+
+ PerformanceCounter counter(ctx, "readBuffer");
+ // Allocate memory on host and device
+ const size_t size = 1024;
+ HostMemory hm(ctx, size);
+ DeviceMemory dm(ctx, size);
+
+ // Fill the host memory
+ memset(hm.get<void>(), 42, size);
+
+ // Transfer to device
+ s.writeBuffer(dm, hm, counter, true);
+
+ // Clear host memory
+ memset(hm.get<void>(), 0, size);
+
+ // Transfer back in different offsets and sizes
+ vector<size_t> offsets = {0, 7, 100, size-10};
+ size_t idx=0;
+ while (idx<offsets.size()-1) {
+ s.readBuffer(hm, dm, offsets[idx], offsets[idx], offsets[idx+1]-offsets[idx], counter, false);
+ idx++;
+ }
+ // last copy needs syn flag true
+ s.readBuffer(hm, dm, offsets[idx], offsets[idx], size-offsets[idx], counter, true);
+
+ // Check results
+ for (size_t i = 0; i < size; ++i) {
+ CHECK_EQUAL(42, hm.get<char>()[i]);
+ }
+ }
+ }
}
int main(int, char **) {
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokes.cc b/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokes.cc
index f0dbcb45820c6b6cdd18679e50ffe815897c7ed0..7dc9120fd9bc8101541997df08858b6008d81061 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokes.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokes.cc
@@ -58,7 +58,8 @@ struct ParsetFixture
nrInputSamples = nrOutputSamples * timeIntegrationFactor,
blockSize = nrChannels * nrInputSamples,
nrStations = 43,
- nrDelayCompensationChannels = 64;
+ nrDelayCompensationChannels = 64,
+ nrStokes = 4;
Parset parset;
@@ -112,7 +113,15 @@ TEST_FIXTURE(ParsetFixture, BufferSizes)
// Test if we can succesfully create a KernelFactory
TEST_FIXTURE(ParsetFixture, KernelFactory)
{
- KernelFactory<IncoherentStokesKernel> kf(parset);
+ IncoherentStokesKernel::Parameters parameters(
+ nrStations,
+ nrChannels,
+ nrInputSamples,
+ nrStokes,
+ timeIntegrationFactor,
+ false // quantizeOutput
+ );
+ KernelFactory<IncoherentStokesKernel> kf(parameters);
}
@@ -136,7 +145,15 @@ struct KernelFixture : ParsetFixture
context(device),
stream(context),
nrStokes(parset.settings.beamFormer.incoherentSettings.nrStokes),
- factory(parset),
+ factory(
+ IncoherentStokesKernel::Parameters(
+ nrStations,
+ nrChannels,
+ nrInputSamples,
+ nrStokes,
+ timeIntegrationFactor,
+ false // quantizeOutput
+ )),
hInput(
boost::extents[nrStations][NR_POLARIZATIONS][nrInputSamples][nrChannels],
context),
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokesTranspose.cc b/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokesTranspose.cc
index 465d7b9ad26e3e15f2ecec7778bc629049c30bdc..f4f49f5837d2e57d931a4b2984bf8e2b314c904d 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokesTranspose.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tIncoherentStokesTranspose.cc
@@ -59,12 +59,24 @@ void runTest( gpu::Context &ctx, gpu::Stream &stream )
ps.add("Observation.DataProducts.Output_IncoherentStokes.locations", "[localhost:.]");
ps.updateSettings();
- IncoherentStokesTransposeKernel::Parameters params(ps);
+ std::vector<unsigned> stationIndices(NR_STATIONS);
+ for (unsigned stationIdx = 0; stationIdx < NR_STATIONS; stationIdx++)
+ {
+ stationIndices[stationIdx] = stationIdx;
+ }
+
+ IncoherentStokesTransposeKernel::Parameters params(
+ NR_STATIONS,
+ stationIndices,
+ NR_CHANNELS,
+ NR_SAMPLES_PER_CHANNEL,
+ true
+ );
// Update the parameters ourselves.
params.nrChannels = NR_CHANNELS;
params.nrSamplesPerChannel = NR_SAMPLES_PER_CHANNEL;
- params.nrStations = NR_STATIONS;
+ params.nrInputStations = NR_STATIONS;
KernelFactory<IncoherentStokesTransposeKernel> factory(params);
diff --git a/RTCP/Cobalt/GPUProc/test/cuda/tTranspose2.cc b/RTCP/Cobalt/GPUProc/test/cuda/tTranspose2.cc
index 8f15b3437805b6eacd43452eed9757974e65e30f..bacc617a606083fb5a53bc11706f5c98bad8b934 100644
--- a/RTCP/Cobalt/GPUProc/test/cuda/tTranspose2.cc
+++ b/RTCP/Cobalt/GPUProc/test/cuda/tTranspose2.cc
@@ -62,10 +62,10 @@ void runTest( Context &ctx, Stream &stream )
ps.add("Observation.DataProducts.Output_CoherentStokes.locations", "[localhost:.]");
ps.updateSettings();
- BeamFormerTransposeKernel::Parameters params(ps);
- params.nrChannels = NR_CHANNELS;
- params.nrSamplesPerChannel = NR_SAMPLES_PER_CHANNEL;
- params.nrTABs = NR_TABS;
+ BeamFormerTransposeKernel::Parameters params(
+ (unsigned) NR_CHANNELS,
+ (unsigned) NR_SAMPLES_PER_CHANNEL,
+ (unsigned) NR_TABS);
KernelFactory<BeamFormerTransposeKernel> factory(params);
diff --git a/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc b/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc
index 88673b74b50e0b997b09f61d6fdfec05a6ed31bf..a81481a89086b648fe5641ea9c2095658ca8cc29 100644
--- a/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc
+++ b/RTCP/Cobalt/GPUProc/test/tMPIReceive.cc
@@ -121,7 +121,7 @@ int main(int argc, char **argv)
// empty the pool
#pragma omp section
{
- SmartPtr<struct MPIRecvData> input;
+ std::shared_ptr<struct MPIRecvData> input;
while ((input = MPI_receive_pool.filled.remove()) != NULL)
{
cout << "Block freed" << endl;
diff --git a/RTCP/Cobalt/GPUProc/test/tStationInput.cc b/RTCP/Cobalt/GPUProc/test/tStationInput.cc
index 197ca6af63fba390554be4e1c968dfdd537c48bf..514236b59ff32eaafa2e1ace419ecfa0d393b712 100644
--- a/RTCP/Cobalt/GPUProc/test/tStationInput.cc
+++ b/RTCP/Cobalt/GPUProc/test/tStationInput.cc
@@ -270,7 +270,7 @@ SUITE(StationMetaData) {
// Validate the blocks
for (ssize_t block = -1; block < 3; ++block) {
ASSERT(!sm.metaDataPool.filled.empty());
- SmartPtr< MPIData<SampleT> > data = sm.metaDataPool.filled.remove();
+ std::shared_ptr<MPIData<SampleT>> data = sm.metaDataPool.filled.remove();
ASSERT(data != NULL);
CHECK_EQUAL(block, data->block);
@@ -278,7 +278,7 @@ SUITE(StationMetaData) {
// computeMetaData closes off with a NULL
ASSERT(!sm.metaDataPool.filled.empty());
- SmartPtr< MPIData<SampleT> > data = sm.metaDataPool.filled.remove();
+ std::shared_ptr<MPIData<SampleT>> data = sm.metaDataPool.filled.remove();
CHECK(data == NULL);
CHECK(sm.metaDataPool.filled.empty());
diff --git a/RTCP/Cobalt/GPUProc/test/t_gpu_utils.cc b/RTCP/Cobalt/GPUProc/test/t_gpu_utils.cc
index 3939faedbb66a276d21b3f492a50c67f8c088743..fa020f4c2f90ce75d1cecfff38f32da51a153be0 100644
--- a/RTCP/Cobalt/GPUProc/test/t_gpu_utils.cc
+++ b/RTCP/Cobalt/GPUProc/test/t_gpu_utils.cc
@@ -27,6 +27,7 @@
#include <vector>
#include <fstream>
#include <stdexcept>
+#include <string>
#include <UnitTest++.h>
#include <Common/LofarLogger.h>
#include <CoInterface/Exceptions.h>
@@ -79,7 +80,7 @@ TEST_FIXTURE(CreateFixture, CreatePtxWrongExtraDef)
TEST_FIXTURE(CreateFixture, CreatePtxExtraFlag)
{
CompileFlags flags;
- flags.insert("--source-in-ptx");
+ flags.insert("--use_fast_math");
createPTX(srcFile, defaultCompileDefinitions(), flags);
}
@@ -116,6 +117,67 @@ TEST_FIXTURE(CreateFixture, CreateModuleHighestArch)
gpu::GPUException);
}
+std::string getPathToSrcFile(std::string srcFilename)
+{
+ const char* env = getenv("LOFARROOT");
+ return ((env ? string(env) : string())
+ + "/share/gpu/kernels/" + srcFilename).c_str();
+}
+
+unsigned int countSubstring(const std::string& str, const std::string& sub)
+{
+ if (sub.length() == 0)
+ {
+ return 0;
+ }
+ int count = 0;
+ for (size_t offset = str.find(sub); offset != std::string::npos;
+ offset = str.find(sub, offset + sub.length()))
+ {
+ count++;
+ }
+ return count;
+}
+
+#if 0
+// The NVRT PTX creation produces different output compared to the existing
+// compilation method using NVCC. Compilation using NVRTC is experimental and
+// not used by default. Therefore, this test, is disabled for now.
+TEST_FIXTURE(CreateFixture, CreatePtxUsingNVRTC)
+{
+
+ CompileDefinitions definitions;
+ definitions["NR_STATIONS"] = "2";
+ definitions["NR_CHANNELS"] = "16";
+ definitions["NR_INTEGRATIONS"] = "1";
+ definitions["NR_SAMPLES_PER_INTEGRATION"] = "128";
+ definitions["NR_POLARIZATIONS"] = "2";
+ definitions["NR_STATIONS_PER_THREAD"] = "1";
+
+
+ auto ptxRef = createPTX(getPathToSrcFile("Correlator.cu"),
+ definitions,
+ defaultCompileFlags(),
+ vector<gpu::Device>());
+
+ auto ptxNVRTC = createPTX(getPathToSrcFile("Correlator.cu"),
+ definitions,
+ defaultCompileFlags(),
+ vector<gpu::Device>(),
+ true); // useNVRTC
+
+ // Count the number of add instructions
+ unsigned nrAddRef = countSubstring(ptxRef, "add");
+ unsigned nrAddNVRTC = countSubstring(ptxNVRTC, "add");
+ CHECK_EQUAL(nrAddRef, nrAddNVRTC);
+
+ // Count the number of mul instructions
+ unsigned nrMulRef = countSubstring(ptxRef, "mul");
+ unsigned nrMulNVRTC = countSubstring(ptxNVRTC, "mul");
+ CHECK_EQUAL(nrMulRef, nrMulNVRTC);
+}
+#endif
+
TEST(DumpBuffer)
{
typedef unsigned element_type;
diff --git a/RTCP/Cobalt/InputProc/src/Delays/Delays.cc b/RTCP/Cobalt/InputProc/src/Delays/Delays.cc
index 41bac56e8c06e6b57c869a30f4baebef169cf8bb..5a2e0d52f3efa5ba4df89e294cbe4c285a952bb0 100644
--- a/RTCP/Cobalt/InputProc/src/Delays/Delays.cc
+++ b/RTCP/Cobalt/InputProc/src/Delays/Delays.cc
@@ -77,12 +77,12 @@ namespace LOFAR
SAPs.resize(parset.settings.SAPs.size());
for (size_t sap = 0; sap < parset.settings.SAPs.size(); ++sap) {
- if (parset.settings.beamFormer.enabled) {
- const struct ObservationSettings::BeamFormer::SAP &bfSap = parset.settings.beamFormer.SAPs[sap];
+ if (parset.settings.beamFormer.enabled) {
+ const struct ObservationSettings::BeamFormer::SAP &bfSap = parset.settings.beamFormer.SAPs[sap];
- // Reserve room for coherent TABs only
- SAPs[sap].TABs.resize(bfSap.nrCoherent);
- }
+ // Reserve room for coherent TABs only
+ SAPs[sap].TABs.resize(bfSap.nrCoherent);
+ }
}
}
diff --git a/RTCP/Cobalt/InputProc/src/Delays/Delays.h b/RTCP/Cobalt/InputProc/src/Delays/Delays.h
index cd85ef8b4e962b5e580b7f20b36796ac887f6a4e..71912b5b63797fa6381f0941304b5a6ca6560fff 100644
--- a/RTCP/Cobalt/InputProc/src/Delays/Delays.h
+++ b/RTCP/Cobalt/InputProc/src/Delays/Delays.h
@@ -35,7 +35,6 @@
#include <CoInterface/MultiDimArray.h>
#include <CoInterface/Parset.h>
#include <CoInterface/SubbandMetaData.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/RSPTimeStamp.h>
#ifdef HAVE_CASACORE
diff --git a/RTCP/Cobalt/InputProc/src/Station/Generator.cc b/RTCP/Cobalt/InputProc/src/Station/Generator.cc
index a1afd6551384cc21718b792ec128f3e3b08e08ec..f587ba8dddb06a66a184356ca3d885c08df9bb51 100644
--- a/RTCP/Cobalt/InputProc/src/Station/Generator.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/Generator.cc
@@ -26,7 +26,6 @@
#include <Common/LofarLogger.h>
#include <Stream/Stream.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/Stream.h>
@@ -35,7 +34,7 @@ namespace LOFAR
namespace Cobalt
{
- Generator::Generator( const StationID &stationID, const std::vector< SmartPtr<Stream> > &outputStreams_, PacketFactory &packetFactory, const TimeStamp &from, const TimeStamp &to )
+ Generator::Generator( const StationID &stationID, const std::vector<std::shared_ptr<Stream>> &outputStreams_, PacketFactory &packetFactory, const TimeStamp &from, const TimeStamp &to )
:
RSPBoards(str(boost::format("[station %s] [Generator] ") % stationID.name()), outputStreams_.size()),
stationID(stationID),
diff --git a/RTCP/Cobalt/InputProc/src/Station/Generator.h b/RTCP/Cobalt/InputProc/src/Station/Generator.h
index a503382f8ea00f29a41573dca3e9ac8836052819..55ac0c58afac04809399379354a4fe590434ace0 100644
--- a/RTCP/Cobalt/InputProc/src/Station/Generator.h
+++ b/RTCP/Cobalt/InputProc/src/Station/Generator.h
@@ -25,7 +25,6 @@
#include <vector>
#include <Stream/Stream.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/RSP.h>
#include <CoInterface/RSPTimeStamp.h>
@@ -44,11 +43,11 @@ namespace LOFAR
class Generator : public RSPBoards
{
public:
- Generator( const StationID &stationID, const std::vector< SmartPtr<Stream> > &outputStreams, PacketFactory &packetFactory, const TimeStamp &from, const TimeStamp &to );
+ Generator( const StationID &stationID, const std::vector<std::shared_ptr<Stream>> &outputStreams, PacketFactory &packetFactory, const TimeStamp &from, const TimeStamp &to );
protected:
const StationID stationID;
- std::vector<Stream *> outputStreams;
+ std::vector<std::shared_ptr<Stream>> outputStreams;
PacketFactory &packetFactory;
std::vector<size_t> nrSent;
diff --git a/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc b/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc
index 86809656a229a8594b5ce46ba8d4f2e3eda531b7..ec450f6ba7076953a2d8e5bd3266f9b9467ab877 100644
--- a/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/filterRSP.cc
@@ -32,7 +32,6 @@
#include <Common/LofarLogger.h>
#include <Stream/StreamFactory.h>
#include <ApplCommon/PosixTime.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/RSP.h>
#include "PacketReader.h"
@@ -131,19 +130,18 @@ int main(int argc, char **argv)
}
// create in- and output streams
- SmartPtr<Stream> inputStream = createStream(inputStreamDesc, true);
- SmartPtr<Stream> outputStream = createStream(outputStreamDesc, false);
+ std::unique_ptr<Stream> inputStream(createStream(inputStreamDesc, true));
+ std::unique_ptr<Stream> outputStream(createStream(outputStreamDesc, false));
PacketReader reader("", *inputStream);
// create packet queues between reader and writer
- Queue< SmartPtr<packetSet> > readQueue;
- Queue< SmartPtr<packetSet> > writeQueue;
+ Queue<std::shared_ptr<packetSet>> readQueue;
+ Queue<std::shared_ptr<packetSet>> writeQueue;
for (size_t i = 0; i < 256; ++i) {
- SmartPtr<packetSet> p = new packetSet;
+ std::unique_ptr<packetSet> p(new packetSet);
p->packets.resize(256);
-
- readQueue.append(p);
+ readQueue.append(std::move(p));
}
/*
@@ -164,7 +162,7 @@ int main(int argc, char **argv)
try {
Thread::ScopedPriority sp(SCHED_FIFO, 10);
- SmartPtr<packetSet> p;
+ std::shared_ptr<packetSet> p;
while (!writerDone && (p = readQueue.remove()) != NULL) {
// Read packets and queue them
@@ -179,7 +177,7 @@ int main(int argc, char **argv)
# pragma omp section
{
- SmartPtr<packetSet> p;
+ std::shared_ptr<packetSet> p;
// Keep reading until NULL
while ((p = writeQueue.remove()) != NULL) {
diff --git a/RTCP/Cobalt/InputProc/src/Station/generate.cc b/RTCP/Cobalt/InputProc/src/Station/generate.cc
index c19c524fad42f9c24b14eae5b66aad4c91bc458a..0f7a4c1269f262423bf807ac4c08b845067589d2 100644
--- a/RTCP/Cobalt/InputProc/src/Station/generate.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/generate.cc
@@ -149,7 +149,7 @@ int main( int argc, char **argv )
continue;
}
- vector< SmartPtr<Stream> > outputStreams;
+ vector<std::shared_ptr<Stream>> outputStreams;
for (size_t s = 0; s < field.inputStreams.size(); ++s) {
string desc = field.inputStreams[s];
@@ -160,7 +160,8 @@ int main( int argc, char **argv )
}
LOG_INFO_STR("[" << field.name << "] Creating stream to " << desc);
- outputStreams.push_back(createStream(desc, false));
+ std::shared_ptr<Stream> stream(createStream(desc, false));
+ outputStreams.push_back(stream);
LOG_INFO_STR("[" << field.name << "] Created stream to " << desc);
}
diff --git a/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc b/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc
index 315bf18f8fb7571bf5f0d8e7cc2603911419b24a..8e6f052f99bb25828c23d4bb7ab730d4f0f7251f 100644
--- a/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/generateRSP.cc
@@ -32,7 +32,6 @@
#include <Common/LofarLogger.h>
#include <ApplCommon/PosixTime.h>
#include <Stream/StreamFactory.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/RSP.h>
#include <CoInterface/RSPTimeStamp.h>
#include <InputProc/Buffer/BoardMode.h>
@@ -144,7 +143,7 @@ int main(int argc, char **argv)
}
ifstream inStream("/dev/stdin");
- SmartPtr<Stream> outStream = createStream(streamdesc, false);
+ std::unique_ptr<Stream> outStream(createStream(streamdesc, false));
BoardMode boardMode(bitmode, clockmode);
unsigned boardNr(0);
diff --git a/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc b/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc
index ca400d6f8caad09cafb4c71174f3bdfe297193b7..a8b09ebc232579928cbff7e6a35c909eecce5736 100644
--- a/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc
+++ b/RTCP/Cobalt/InputProc/src/Station/repairRSP.cc
@@ -32,7 +32,6 @@
#include <Common/LofarLogger.h>
#include <ApplCommon/PosixTime.h>
#include <Stream/StreamFactory.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/RSP.h>
#include "PacketReader.h"
@@ -105,8 +104,8 @@ int main(int argc, char **argv)
}
// create in- and output streams
- SmartPtr<Stream> inputStream = createStream("file:/dev/stdin", true);
- SmartPtr<Stream> outputStream = createStream("file:/dev/stdout", false);
+ std::unique_ptr<Stream> inputStream(createStream("file:/dev/stdin", true));
+ std::unique_ptr<Stream> outputStream(createStream("file:/dev/stdout", false));
try {
for(;;) {
diff --git a/RTCP/Cobalt/InputProc/src/Transpose/MPIReceiveStations.h b/RTCP/Cobalt/InputProc/src/Transpose/MPIReceiveStations.h
index d4e71f65cc3a44a2852ed557da09d83671268037..cbbb3611818110c28d81c9d37bd9e8afe51be570 100644
--- a/RTCP/Cobalt/InputProc/src/Transpose/MPIReceiveStations.h
+++ b/RTCP/Cobalt/InputProc/src/Transpose/MPIReceiveStations.h
@@ -26,7 +26,6 @@
#include <Common/LofarTypes.h>
#include <CoInterface/MultiDimArray.h>
-#include <CoInterface/SmartPtr.h>
#include "MPIProtocol.h"
#include "MPIUtil.h"
diff --git a/RTCP/Cobalt/InputProc/src/Transpose/MPISendStation.h b/RTCP/Cobalt/InputProc/src/Transpose/MPISendStation.h
index aacf5e61d6a3652a801e94509a70c095508340ee..5af47f600e3ca451a56427a0c674baec3f5a5c6a 100644
--- a/RTCP/Cobalt/InputProc/src/Transpose/MPISendStation.h
+++ b/RTCP/Cobalt/InputProc/src/Transpose/MPISendStation.h
@@ -27,7 +27,6 @@
#include <Common/LofarTypes.h>
#include <CoInterface/MultiDimArray.h>
#include <CoInterface/SubbandMetaData.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/RSPTimeStamp.h>
#include "MPIProtocol.h"
diff --git a/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc b/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc
index 6e08585ce1f9cc6741fe873e3d66bf3327da9b1c..91aebac729d3f514431f871873a8b91aa1e61080 100644
--- a/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc
+++ b/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.cc
@@ -19,7 +19,6 @@
#include <boost/format.hpp>
#include <Common/Thread/Mutex.h>
#include <Common/Timer.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/TimeFuncs.h>
#include <Common/SystemCallException.h>
@@ -314,7 +313,7 @@ namespace LOFAR {
started = true;
- thread = new Thread(this, &MPIPoll::pollThread, "MPIPoll", "MPIPoll::pollThread");
+ thread.reset(new Thread(this, &MPIPoll::pollThread, "MPIPoll", "MPIPoll::pollThread"));
}
diff --git a/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.h b/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.h
index 2695190d81c0218a76f48b065fd6444aeac28c6d..b5a22bbe6c8dbfb4d39f589e3c42e84ee6e2c9ac 100644
--- a/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.h
+++ b/RTCP/Cobalt/InputProc/src/Transpose/MPIUtil.h
@@ -8,7 +8,6 @@
#include <Common/Thread/Thread.h>
#include <Common/Thread/Condition.h>
#include <CoInterface/Allocator.h>
-#include <CoInterface/SmartPtr.h>
namespace LOFAR {
@@ -184,7 +183,7 @@ namespace LOFAR {
Condition newRequest;
// The thread that periodically polls MPI
- SmartPtr<Thread> thread;
+ std::unique_ptr<Thread> thread;
// Unregister a set of MPI requests (doesn't grab mutex)
void _remove( RequestSet *set );
diff --git a/RTCP/Cobalt/InputProc/test/tGenerator.cc b/RTCP/Cobalt/InputProc/test/tGenerator.cc
index 9df011fab948d79ca62e9221b0d6c1ac2779296c..d2a7be1f5cada729bebce74e99ea910f8e3f50ce 100644
--- a/RTCP/Cobalt/InputProc/test/tGenerator.cc
+++ b/RTCP/Cobalt/InputProc/test/tGenerator.cc
@@ -54,16 +54,16 @@ int main( int, char **argv )
const string desc = "tcp:localhost:54321";
- vector< SmartPtr<Stream> > inputStreams(1);
- vector< SmartPtr<Stream> > outputStreams(1);
+ vector<std::shared_ptr<Stream> > inputStreams(1);
+ vector<std::shared_ptr<Stream> > outputStreams(1);
#pragma omp parallel sections num_threads(2)
{
#pragma omp section
- inputStreams[0] = createStream(desc, true);
+ inputStreams[0].reset(createStream(desc, true));
#pragma omp section
- outputStreams[0] = createStream(desc, false);
+ outputStreams[0].reset(createStream(desc, false));
}
struct StationID stationID("RS106", "LBA");
diff --git a/RTCP/Cobalt/OutputProc/src/FastFileStream.cc b/RTCP/Cobalt/OutputProc/src/FastFileStream.cc
index 8db289c0d2cb700b0bc0cd1d86c50e847508ab2b..7491efa0f442e72171bcaa01f7bf0445495cb83a 100644
--- a/RTCP/Cobalt/OutputProc/src/FastFileStream.cc
+++ b/RTCP/Cobalt/OutputProc/src/FastFileStream.cc
@@ -43,7 +43,7 @@ namespace LOFAR
:
FileStream(name.c_str(), flags | O_DIRECT | O_SYNC, mode),
bufsize(0),
- buffer(0),
+ buffer(nullptr),
remainder(0)
{
// alignment must be a power of two for easy calculations
@@ -83,7 +83,7 @@ namespace LOFAR
size_t nrpadbytes = alignment - remainder;
ensureBuffer(alignment);
memset(buffer.get() + remainder, 0, nrpadbytes);
- forceWrite(buffer, alignment);
+ forceWrite(buffer.get(), alignment);
remainder = 0;
@@ -110,7 +110,7 @@ namespace LOFAR
memcpy(buf, buffer.get(), remainder);
}
- buffer = static_cast<char*>(buf); // SmartPtr will take care of deleting the old buffer
+ buffer.reset(static_cast<char*>(buf));
bufsize = newsize;
}
diff --git a/RTCP/Cobalt/OutputProc/src/FastFileStream.h b/RTCP/Cobalt/OutputProc/src/FastFileStream.h
index 07aa449251ac53805a4dc4585c14c456cd85afc4..8b063d9332a6cabaa5f2ccbf2c1389fc429323a9 100644
--- a/RTCP/Cobalt/OutputProc/src/FastFileStream.h
+++ b/RTCP/Cobalt/OutputProc/src/FastFileStream.h
@@ -23,7 +23,6 @@
#include <string>
#include <Stream/FileStream.h>
-#include <CoInterface/SmartPtr.h>
namespace LOFAR
{
@@ -65,7 +64,7 @@ namespace LOFAR
void forceWrite(const void *ptr, size_t size);
size_t bufsize;
- SmartPtr<char, SmartPtrFree<char> > buffer;
+ std::unique_ptr<char> buffer;
size_t remainder;
};
diff --git a/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc b/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
index c1275ece96a0f10f30e65765860090764511160a..23dbdbdeee8deb0a932bbd021e1a52b39898cda5 100644
--- a/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
+++ b/RTCP/Cobalt/OutputProc/src/GPUProcIO.cc
@@ -50,7 +50,6 @@
#include <CoInterface/Parset.h>
#include <CoInterface/FinalMetaData.h>
#include <CoInterface/Stream.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/SelfDestructTimer.h>
#include "SubbandWriter.h"
#include "RSPRawWriter.h"
@@ -155,9 +154,9 @@ bool process(Stream &controlStream)
{
// make sure "parset" and "mdLogger" stay in scope for the lifetime of the SubbandWriters
- vector<SmartPtr<SubbandWriter> > subbandWriters;
- vector<SmartPtr<TABOutputThread> > tabWriters;
- vector<SmartPtr<RSPRawWriter> > rspRawWriters;
+ vector<std::unique_ptr<SubbandWriter> > subbandWriters;
+ vector<std::unique_ptr<TABOutputThread> > tabWriters;
+ vector<std::unique_ptr<RSPRawWriter> > rspRawWriters;
/*
* Construct writers
@@ -180,16 +179,16 @@ bool process(Stream &controlStream)
string logPrefix = str(format("[stream %3u file %s] ")
% fileIdx % file.location.filename);
- SubbandWriter *writer = new SubbandWriter(parset, fileIdx, mdLogger, mdKeyPrefix, logPrefix);
- subbandWriters.push_back(writer);
+ std::unique_ptr<SubbandWriter> writer(new SubbandWriter(parset, fileIdx, mdLogger, mdKeyPrefix, logPrefix));
+ subbandWriters.push_back(std::move(writer));
LOG_DEBUG_STR("done with fileIdx " << fileIdx);
}
}
- map<size_t, SmartPtr<Arena> > arenas;
- map<size_t, SmartPtr<Allocator> > allocators;
- map<size_t, SmartPtr<Pool<TABTranspose::BeamformedData> > > outputPools;
+ map<size_t, std::unique_ptr<Arena> > arenas;
+ map<size_t, std::unique_ptr<Allocator> > allocators;
+ map<size_t, std::unique_ptr<Pool<TABTranspose::BeamformedData> > > outputPools;
TABTranspose::Receiver::CollectorMap collectors;
// Process beam-formed data
@@ -207,9 +206,15 @@ bool process(Stream &controlStream)
LOG_DEBUG_STR("Allocating transpose buffers for " << file.location.filename);
- struct ObservationSettings::BeamFormer::StokesSettings &stokes =
- file.coherent ? parset.settings.beamFormer.coherentSettings
- : parset.settings.beamFormer.incoherentSettings;
+ auto& tab = parset.settings.beamFormer.SAPs[file.sapNr].TABs[file.tabNr];
+ auto& pipeline = parset.settings.beamFormer.pipelines[tab.pipelineNr];
+
+ const struct ObservationSettings::BeamFormer::StokesSettings &stokes =
+ file.coherent ? pipeline.coherentSettings
+ : pipeline.incoherentSettings;
+
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings &quantizer =
+ stokes.quantizerSettings;
const size_t nrSubbands = file.lastSubbandIdx - file.firstSubbandIdx;
const size_t nrChannels = stokes.nrChannels;
@@ -218,14 +223,21 @@ bool process(Stream &controlStream)
// poolSize is the buffer size (in #blocks) for the queue to disk
const size_t poolSize = 60;
-
- arenas[fileIdx] = new MallocedArena(poolSize * (MultiDimArray<float,3>::nrElements(boost::extents[nrSamples][nrSubbands][nrChannels]) * sizeof(float) + alignment), alignment);
- allocators[fileIdx] = new SparseSetAllocator(*arenas[fileIdx]);
- outputPools[fileIdx] = new Pool<TABTranspose::BeamformedData>(str(format("process::outputPool [stream %u]") % fileIdx), parset.settings.realTime);
+ // arena size can be made smaller if quantized data is processed
+ if (!quantizer.enabled) {
+ arenas[fileIdx].reset(new MallocedArena(poolSize * (MultiDimArray<float,3>::nrElements(boost::extents[nrSamples][nrSubbands][nrChannels]) * sizeof(float) + alignment), alignment));
+ } else {
+ arenas[fileIdx].reset(new MallocedArena(poolSize * (MultiDimArray<int8_t,3>::nrElements(boost::extents[nrSamples][nrSubbands][nrChannels]) * sizeof(int8_t) + 2*MultiDimArray<float,2>::nrElements(boost::extents[nrSubbands][nrChannels]) * sizeof(float) + 3*alignment), alignment));
+ }
+ allocators[fileIdx].reset(new SparseSetAllocator(*arenas[fileIdx]));
+ outputPools[fileIdx].reset(new Pool<TABTranspose::BeamformedData>(str(format("process::outputPool [stream %u]") % fileIdx), parset.settings.realTime));
// Create and fill an outputPool for this fileIdx
for (size_t i = 0; i < poolSize; ++i)
- outputPools[fileIdx]->free.append(new TABTranspose::BeamformedData(nrSubbands, nrChannels, nrSamples, *allocators[fileIdx]), false);
+ {
+ std::shared_ptr<TABTranspose::BeamformedData> data(new TABTranspose::BeamformedData(nrSubbands, nrChannels, nrSamples, quantizer.enabled, *allocators[fileIdx]));
+ outputPools[fileIdx]->free.append(data, false);
+ }
string logPrefix = str(format("[stream %3u file %s] ")
% fileIdx % file.location.filename);
@@ -233,13 +245,13 @@ bool process(Stream &controlStream)
// Create a collector for this fileIdx
// #blocks here is the number of blocks that can be collected in parallel on
// the input side (what the correlator sends)
- collectors[fileIdx] = new TABTranspose::BlockCollector(
- *outputPools[fileIdx], fileIdx, nrSubbands, nrChannels, nrSamples, parset.settings.nrBlocks(), parset.settings.realTime ? 5 : 0, logPrefix, parset.settings.blockDuration());
+ collectors[fileIdx].reset(new TABTranspose::BlockCollector(
+ *outputPools[fileIdx], fileIdx, nrSubbands, nrChannels, nrSamples, parset.settings.nrBlocks(), parset.settings.realTime ? 5 : 0, logPrefix, parset.settings.blockDuration()));
LOG_DEBUG_STR("Setting up writer for " << file.location.filename);
- TABOutputThread *writer = new TABOutputThread(parset, fileIdx, *outputPools[fileIdx], mdLogger, mdKeyPrefix, logPrefix);
- tabWriters.push_back(writer);
+ std::unique_ptr<TABOutputThread> writer(new TABOutputThread(parset, fileIdx, *outputPools[fileIdx], mdLogger, mdKeyPrefix, logPrefix));
+ tabWriters.push_back(std::move(writer));
}
}
@@ -264,8 +276,8 @@ bool process(Stream &controlStream)
string logPrefix = str(format("[obs %u RSP raw stream %3u] ")
% parset.settings.observationID % fileIdx);
- RSPRawWriter *writer = new RSPRawWriter(parset, fileIdx, mdLogger, mdKeyPrefix, logPrefix);
- rspRawWriters.push_back(writer);
+ std::unique_ptr<RSPRawWriter> writer(new RSPRawWriter(parset, fileIdx, mdLogger, mdKeyPrefix, logPrefix));
+ rspRawWriters.push_back(std::move(writer));
LOG_INFO_STR("done with RSP raw fileIdx " << fileIdx);
}
diff --git a/RTCP/Cobalt/OutputProc/src/InputThread.cc b/RTCP/Cobalt/OutputProc/src/InputThread.cc
index cbc606c6e8d07db97aadbd608aeabdf78d16fce8..99a0feb2c2fc6c8c259e46e23cac5b07dcc1239f 100644
--- a/RTCP/Cobalt/OutputProc/src/InputThread.cc
+++ b/RTCP/Cobalt/OutputProc/src/InputThread.cc
@@ -56,11 +56,11 @@ namespace LOFAR
{
try {
LOG_INFO_STR(itsLogPrefix << "Creating connection from " << itsInputDescriptor << "..." );
- SmartPtr<Stream> streamFromION(createStream(itsInputDescriptor, true, itsDeadline));
+ std::unique_ptr<Stream> streamFromION(createStream(itsInputDescriptor, true, itsDeadline));
LOG_INFO_STR(itsLogPrefix << "Creating connection from " << itsInputDescriptor << ": done" );
- for(SmartPtr<StreamableData> data; (data = itsOutputPool.free.remove()) != NULL; itsOutputPool.filled.append(data)) {
- data->read(streamFromION, 1); // Cobalt writes with an alignment of 1
+ for (std::shared_ptr<StreamableData> data; (data = itsOutputPool.free.remove()) != NULL; itsOutputPool.filled.append(data)) {
+ data->read(streamFromION.get(), 1); // Cobalt writes with an alignment of 1
++itsNrIntegrationsReceived; // for RSP_RAW_DATA we need to count bytes, but N/A here
LOG_DEBUG_STR(itsLogPrefix << "Received data block with seq nr " << data->sequenceNumber());
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc b/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc
index c919506d7741aa8cce9f437a376e3eda296cef25..1239fe5eb3992575540f29203f17d0021299188e 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.cc
@@ -83,7 +83,7 @@ namespace LOFAR
string seqfilename = stagePath(str(format("%s-table.f0seqnr") % msName), parset.settings.correlator.files[subbandIndex].dataStagingDirTargetFS);
try {
- itsSequenceNumbersFile = new FileStream(seqfilename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+ itsSequenceNumbersFile.reset(new FileStream(seqfilename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH));
} catch (...) {
LOG_WARN_STR(itsLogPrefix << "Could not open sequence numbers file " << seqfilename);
}
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.h b/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.h
index 99f2556bd93e7ac3ec298fc43e433c4ab989a387..821e85e644f23b3bbda311df259ac25b8ea8cf0f 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.h
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterCorrelated.h
@@ -26,7 +26,6 @@
#include <Stream/FileStream.h>
#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/StreamableData.h>
#include "MSWriterFile.h"
@@ -55,7 +54,7 @@ namespace LOFAR
const Parset &itsParset;
const unsigned itsSubbandIndex;
- SmartPtr<FileStream> itsSequenceNumbersFile;
+ std::unique_ptr<FileStream> itsSequenceNumbersFile;
};
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc
index 169ad137ec51ffa0aa5456b08e6cb7986b928e0c..f8b86e8991e5bb1f6592a4f2da31becdd347d6ed 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.cc
@@ -39,7 +39,6 @@
#include <Common/Thread/Mutex.h>
#include <CoInterface/StreamableData.h>
#include <CoInterface/LTAFeedback.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/CommonLofarAttributes.h>
#include <OutputProc/Package__Version.h>
@@ -93,9 +92,18 @@ namespace LOFAR
const struct ObservationSettings::BeamFormer::File &f = itsParset.settings.beamFormer.files[itsFileNr];
+
+ auto& tab = itsParset.settings.beamFormer.SAPs[f.sapNr].TABs[f.tabNr];
+ auto& pipeline = itsParset.settings.beamFormer.pipelines[tab.pipelineNr];
+
const struct ObservationSettings::BeamFormer::StokesSettings &stokesSet =
- f.coherent ? itsParset.settings.beamFormer.coherentSettings
- : itsParset.settings.beamFormer.incoherentSettings;
+ f.coherent ? pipeline.coherentSettings
+ : pipeline.incoherentSettings;
+
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings &quantizer =
+ stokesSet.quantizerSettings;
+
+ itsQuantizerEnabled=quantizer.enabled;
// All subbands in the SAP that we store in this file.
// We could have multiple SAPs and/or have split up the subbands over multiple files (parts).
@@ -111,6 +119,19 @@ namespace LOFAR
LTAFeedback fb(itsParset.settings);
itsConfiguration.adoptCollection(fb.beamFormedFeedback(itsFileNr));
itsConfigurationPrefix = fb.beamFormedPrefix(itsFileNr);
+ if (itsQuantizerEnabled) {
+ string rawSfilename = forceextension(itsFilename, ".scale");
+ string rawOfilename = forceextension(itsFilename, ".offset");
+ int file_flags=O_RDWR | O_CREAT | O_TRUNC;
+ int file_mode=S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
+#ifdef USE_O_DIRECT
+ itsSFile=std::unique_ptr<FastFileStream>(new FastFileStream(rawSfilename, file_flags, file_mode));
+ itsOFile=std::unique_ptr<FastFileStream>(new FastFileStream(rawOfilename, file_flags, file_mode));
+#else
+ itsSFile=std::unique_ptr<FileStream>(new FileStream(rawSfilename, file_flags, file_mode));
+ itsOFile=std::unique_ptr<FileStream>(new FileStream(rawOfilename, file_flags, file_mode));
+#endif
+ }
}
template <typename T,unsigned DIM, unsigned FLAGS_DIM>
@@ -131,10 +152,11 @@ namespace LOFAR
const unsigned sapNr = f.sapNr;
const unsigned beamNr = f.tabNr;
const unsigned stokesNr = f.stokesNr;
+ auto& tab = itsParset.settings.beamFormer.SAPs.at(f.sapNr).TABs.at(f.tabNr);
const struct ObservationSettings::BeamFormer::StokesSettings &stokesSet =
- f.coherent ? itsParset.settings.beamFormer.coherentSettings
- : itsParset.settings.beamFormer.incoherentSettings;
+ f.coherent ? itsParset.settings.beamFormer.pipelines[tab.pipelineNr].coherentSettings
+ : itsParset.settings.beamFormer.pipelines[tab.pipelineNr].incoherentSettings;
//*******************************
@@ -236,7 +258,15 @@ namespace LOFAR
sap.pointDEC().value = beamDir.angle2 * 180.0 / M_PI;
sap.pointDECUnit().value = "deg";
- sap.observationNofBeams().value = itsParset.settings.beamFormer.SAPs[sapNr].TABs.size();
+ unsigned nofBeams = 0;
+ for (auto& pipeline : itsParset.settings.beamFormer.pipelines)
+ {
+ for (auto& sap : pipeline.SAPs)
+ {
+ nofBeams += sap.TABs.size();
+ }
+ }
+ sap.observationNofBeams().value = nofBeams;
sap.nofBeams().value = 1;
BF_ProcessingHistory sapHistory = sap.processHistory();
@@ -255,12 +285,16 @@ namespace LOFAR
beam.create();
beam.groupType().value = "Beam";
- if (itsParset.settings.beamFormer.doFlysEye) {
+ auto& pipeline = itsParset.settings.beamFormer.pipelines[tab.pipelineNr];
+ auto& antennaFieldNames = pipeline.antennaFieldNames;
+
+ if (pipeline.doFlysEye) {
beam.nofStations().value = 1;
- beam.stationsList().value = vector<string>(1, itsParset.settings.beamFormer.antennaFieldNames[beamNr].fullName());
+ auto& antennaFieldName = itsParset.settings.beamFormer.pipelines.at(tab.pipelineNr).antennaFieldNames.at(tab.localTabNr);
+ beam.stationsList().value = vector<string>(1, antennaFieldName.fullName());
} else {
- beam.nofStations().value = itsParset.settings.beamFormer.antennaFieldNames.size();
- beam.stationsList().value = ObservationSettings::AntennaFieldName::names(itsParset.settings.beamFormer.antennaFieldNames);
+ beam.nofStations().value = antennaFieldNames.size();
+ beam.stationsList().value = ObservationSettings::AntennaFieldName::names(antennaFieldNames);
}
const vector<string> beamtargets(1, itsParset.settings.SAPs[sapNr].target);
@@ -268,7 +302,7 @@ namespace LOFAR
beam.targets().value = beamtargets;
beam.tracking().value = itsParset.settings.SAPs[sapNr].direction.type;
- const struct ObservationSettings::Direction &tabDir = itsParset.settings.beamFormer.SAPs[sapNr].TABs[beamNr].direction;
+ const struct ObservationSettings::Direction &tabDir = tab.direction;
beam.pointRA().value = tabDir.angle1 * 180.0 / M_PI;
beam.pointRAUnit().value = "deg";
beam.pointDEC().value = tabDir.angle2 * 180.0 / M_PI;
@@ -316,7 +350,7 @@ namespace LOFAR
beam.beamFrequencyCenter().value = (beamCenterFrequencySum / nrSubbands - frequencyOffsetPPF) / 1e6;
beam.beamFrequencyCenterUnit().value = "MHz";
- const double DM = itsParset.settings.corrections.dedisperse ? itsParset.settings.beamFormer.SAPs[sapNr].TABs[beamNr].dispersionMeasure : 0.0;
+ const double DM = itsParset.settings.corrections.dedisperse ? tab.dispersionMeasure : 0.0;
beam.foldedData().value = false;
beam.foldPeriod().value = 0.0;
@@ -337,10 +371,6 @@ namespace LOFAR
beam.complexVoltage().value = stokesSet.type == STOKES_XXYY;
beam.signalSum().value = stokesSet.coherent ? "COHERENT" : "INCOHERENT";
- beam.stokesComponents().value = stokesComponents;
- beam.complexVoltage().value = stokesSet.type == STOKES_XXYY;
- beam.signalSum().value = stokesSet.coherent ? "COHERENT" : "INCOHERENT";
-
BF_ProcessingHistory beamHistory = beam.processHistory();
beamHistory.create();
@@ -370,7 +400,7 @@ namespace LOFAR
const vector<double> unitvector(1, 1);
- SmartPtr<TimeCoordinate> timeCoordinate = dynamic_cast<TimeCoordinate*>(coordinates.coordinate(0));
+ std::unique_ptr<TimeCoordinate> timeCoordinate(dynamic_cast<TimeCoordinate*>(coordinates.coordinate(0)));
timeCoordinate.get()->create();
timeCoordinate.get()->groupType().value = "TimeCoord";
@@ -394,7 +424,7 @@ namespace LOFAR
timeCoordinate.get()->axisValuesPixel().value = vector<unsigned>(1, 0); // not used
timeCoordinate.get()->axisValuesWorld().value = vector<double>(1, 0.0); // not used
- SmartPtr<SpectralCoordinate> spectralCoordinate = dynamic_cast<SpectralCoordinate*>(coordinates.coordinate(1));
+ std::unique_ptr<SpectralCoordinate> spectralCoordinate(dynamic_cast<SpectralCoordinate*>(coordinates.coordinate(1)));
spectralCoordinate.get()->create();
spectralCoordinate.get()->groupType().value = "SpectralCoord";
@@ -421,7 +451,6 @@ namespace LOFAR
// NOTE: channel 0 will be wrongly annotated if nrChannels > 1, because it is a combination of the
// highest and the lowest frequencies (half a channel each).
-
for (unsigned ch = 0; ch < stokesSet.nrChannels; ch++) {
spectralPixels.push_back(spectralPixels.size());
spectralWorld.push_back(subbandBeginFreq + ch * channelBandwidth);
@@ -431,7 +460,6 @@ namespace LOFAR
spectralCoordinate.get()->axisValuesPixel().value = spectralPixels;
spectralCoordinate.get()->axisValuesWorld().value = spectralWorld;
- BF_StokesDataset stokesDS = beam.stokes(stokesNr);
vector<ssize_t> dims(2), maxdims(2);
@@ -441,14 +469,48 @@ namespace LOFAR
maxdims[0] = -1;
maxdims[1] = itsNrChannels;
- stokesDS.create(dims, maxdims, LOFAR::basename(rawfilename), BF_StokesDataset::LITTLE);
- stokesDS.groupType().value = "bfData";
- stokesDS.dataType().value = "float";
+ const vector<int> stokesQuantized(1, itsQuantizerEnabled);
+ beam.quantized().value=stokesQuantized;
+ if (!itsQuantizerEnabled) {
+ BF_StokesDataset stokesDS = beam.stokes(stokesNr);
+ stokesDS.create(dims, maxdims, LOFAR::basename(rawfilename), BF_StokesDataset::LITTLE);
+ stokesDS.groupType().value = "bfData";
+ stokesDS.dataType().value = "float";
+ stokesDS.stokesComponent().value = stokesVars[stokesNr];
+ stokesDS.nofChannels().value = vector<unsigned>(nrSubbands, stokesSet.nrChannels);
+ stokesDS.nofSubbands().value = nrSubbands;
+ stokesDS.nofSamples().value = dims[0];
+ } else {
+ string rawSfilename = forceextension(itsFilename, ".scale");
+ string rawOfilename = forceextension(itsFilename, ".offset");
+ vector<ssize_t> sodims(2);
+ sodims[0]=itsNrExpectedBlocks; // one value for each NrSamples
+ sodims[1]=itsNrChannels;
+
+ bool unsignedData= stokesVars[stokesNr]=="I";
+ BF_QStokesDataset qstokes(beam.qstokes(stokesNr));
+ qstokes.create();
+ qstokes.groupType().value = "bfData";
+ qstokes.dataType().value = unsignedData?"unsigned char":"signed char"; //from dal/hdf5/types/h5typemap.h
+ qstokes.scaleOffsetType().value = "float";
+ qstokes.stokesComponent().value = stokesVars[stokesNr];
+ qstokes.nofChannels().value = vector<unsigned>(nrSubbands, stokesSet.nrChannels);
+ qstokes.nofSubbands().value = nrSubbands;
+ qstokes.nofSamples().value = dims[0];
+ qstokes.nofSamplesPerBlock().value=itsNrSamples;
+ if (unsignedData) {
+ dal::Dataset<uint8_t> qdata=qstokes.data<uint8_t>();
+ qdata.create(dims, maxdims, LOFAR::basename(rawfilename), Dataset<uint8_t>::LITTLE);
+ } else {
+ dal::Dataset<int8_t> qdata=qstokes.data<int8_t>();
+ qdata.create(dims, maxdims, LOFAR::basename(rawfilename), Dataset<int8_t>::LITTLE);
+ }
+ dal::Dataset<float> qscale=qstokes.scale();
+ qscale.create(sodims, maxdims, LOFAR::basename(rawSfilename), Dataset<float>::LITTLE);
+ dal::Dataset<float> qoffset=qstokes.offset();
+ qoffset.create(sodims, maxdims, LOFAR::basename(rawOfilename), Dataset<float>::LITTLE);
+ }
- stokesDS.stokesComponent().value = stokesVars[stokesNr];
- stokesDS.nofChannels().value = vector<unsigned>(nrSubbands, stokesSet.nrChannels);
- stokesDS.nofSubbands().value = nrSubbands;
- stokesDS.nofSamples().value = dims[0];
}
template <typename T,unsigned DIM, unsigned FLAGS_DIM>
@@ -464,17 +526,37 @@ namespace LOFAR
ASSERT( data );
ASSERT( sdata );
- ASSERTSTR( sdata->samples.num_elements() >= itsBlockSize,
+ if (!itsQuantizerEnabled) {
+ ASSERTSTR( sdata->samples.num_elements() >= itsBlockSize,
"A block is at least " << itsBlockSize <<
" elements, but provided sdata only has " <<
sdata->samples.num_elements() << " elements" );
+ } else {
+ ASSERTSTR( sdata->qsamples.num_elements() >= itsBlockSize,
+ "A block is at least " << itsBlockSize <<
+ " elements, but provided sdata only has " <<
+ sdata->qsamples.num_elements() << " elements" );
+ ASSERTSTR( sdata->qscales.num_elements() == sdata->qoffsets.num_elements() && sdata->qoffsets.num_elements() >= 1 && sdata->qoffsets.num_elements() < itsBlockSize,
+ "Offset/scale should be valid for " << itsBlockSize <<
+ " elements, but provided offset/scale has " <<
+ sdata->qoffsets.num_elements() <<","<< sdata->qscales.num_elements() << " elements" );
+ }
unsigned seqNr = data->sequenceNumber();
- unsigned bytesPerBlock = itsBlockSize * sizeof(T);
+ unsigned bytesPerBlock = itsBlockSize * (!itsQuantizerEnabled?sizeof(T):sizeof(int8_t));
+ // per each block, there is only one scale/offset
+ unsigned bytesPerBlockSO = sizeof(float);
+ if (itsQuantizerEnabled) {
+ bytesPerBlockSO *=sdata->qoffsets.num_elements();
+ }
// fill in zeroes for lost blocks
if (itsNextSeqNr < seqNr) {
itsFile.skip((seqNr - itsNextSeqNr) * bytesPerBlock);
+ if (itsQuantizerEnabled) {
+ itsSFile->skip((seqNr - itsNextSeqNr) * bytesPerBlockSO);
+ itsOFile->skip((seqNr - itsNextSeqNr) * bytesPerBlockSO);
+ }
// the write below can throw, so record that we've
// skipped this far to prevent skipping twice.
@@ -482,7 +564,13 @@ namespace LOFAR
}
// make sure we skip |2 in the highest dimension
- itsFile.write(sdata->samples.origin(), bytesPerBlock);
+ if (!itsQuantizerEnabled) {
+ itsFile.write(sdata->samples.origin(), bytesPerBlock);
+ } else {
+ itsSFile->write(sdata->qscales.origin(), bytesPerBlockSO);
+ itsOFile->write(sdata->qoffsets.origin(), bytesPerBlockSO);
+ itsFile.write(sdata->qsamples.origin(), bytesPerBlock);
+ }
itsNextSeqNr = seqNr + 1;
itsNrBlocksWritten++;
@@ -491,9 +579,8 @@ namespace LOFAR
itsConfiguration.replace(itsConfigurationPrefix + "percentageWritten", str(format("%u") % percentageWritten()));
}
- // specialisation for FinalBeamFormedData
+ // specialisations for FinalBeamFormedData
template class MSWriterDAL<float,3>;
-
} // namespace Cobalt
} // namespace LOFAR
diff --git a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.h b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.h
index ca25c939e1f5bda3f25865a58bbe533d31ac5265..b8b9012d340cf1125eccab914c9619ea3de70297 100644
--- a/RTCP/Cobalt/OutputProc/src/MSWriterDAL.h
+++ b/RTCP/Cobalt/OutputProc/src/MSWriterDAL.h
@@ -57,6 +57,16 @@ namespace LOFAR
unsigned itsNextSeqNr;
const unsigned itsFileNr;
unsigned itsBlockSize; // the size of StreamableData::samples, in T
+ bool itsQuantizerEnabled;
+ //create additional output streams if needed
+#ifdef USE_O_DIRECT
+ std::unique_ptr<FastFileStream> itsSFile;
+ std::unique_ptr<FastFileStream> itsOFile;
+#else
+ std::unique_ptr<FileStream> itsSFile;
+ std::unique_ptr<FileStream> itsOFile;
+#endif
+
};
}
}
diff --git a/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc b/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc
index 8b226b2e9c348ef3b61d9db157751fd49fbb81f6..d00ae1631be49b7e5ce44b890e4fe6d8bb783685 100644
--- a/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc
+++ b/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.cc
@@ -100,7 +100,7 @@ namespace LOFAR
stationNames(itsPS.mergedStationNames()),
antPos(itsPS.positions()),
itsNrAnt(stationNames.size()),
- itsMS(0),
+ itsMS(nullptr),
itsAlignment(alignment)
{
if (itsPS.nrTabStations() > 0) {
@@ -122,7 +122,7 @@ namespace LOFAR
{
ScopedLock scopedLock(sharedMutex);
- itsMS = 0;
+ itsMS.reset();
}
@@ -139,7 +139,7 @@ namespace LOFAR
createMSMetaFile(MSname, subband);
// Release itsMS, we don't need it anymore
- itsMS = 0;
+ itsMS.reset();
}
@@ -170,7 +170,7 @@ namespace LOFAR
// MSLofar() constructor needs a NEW table, to avoid checks
// for subtables that aren't yet there.
- itsMS = new MSLofar(newtab);
+ itsMS.reset(new MSLofar(newtab));
itsMS->createDefaultSubtables(Table::New);
Block<MPosition> antMPos(itsNrAnt);
diff --git a/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.h b/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.h
index bbc186ec67e239ce41849d8b80ceecb30baaa2b3..0e6240e830841b8f8dcbf8ed7c17a88d94874a00 100644
--- a/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.h
+++ b/RTCP/Cobalt/OutputProc/src/MeasurementSetFormat.h
@@ -28,7 +28,6 @@
#include <Common/Thread/Mutex.h>
#include <MSLofar/MSLofar.h>
#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/OutputTypes.h> //for LofarStManVersion
#include <casacore/casa/aips.h>
@@ -89,7 +88,7 @@ namespace LOFAR
double itsTimeStep;
- SmartPtr<MSLofar> itsMS;
+ std::shared_ptr<MSLofar> itsMS;
const uint32 itsAlignment;
diff --git a/RTCP/Cobalt/OutputProc/src/OutputThread.cc b/RTCP/Cobalt/OutputProc/src/OutputThread.cc
index 6f9438c3763dda8e5857a75473cee871d4d301f2..c237ca629bfdfe849184b2426aa49189831b865d 100644
--- a/RTCP/Cobalt/OutputProc/src/OutputThread.cc
+++ b/RTCP/Cobalt/OutputProc/src/OutputThread.cc
@@ -128,23 +128,23 @@ namespace LOFAR
itsBlockDuration,
true, true);
- for (SmartPtr<T> data; (data = itsOutputPool.filled.remove()) != 0; itsOutputPool.free.append(data)) {
+ for (std::shared_ptr<T> data; (data = itsOutputPool.filled.remove()) != 0; itsOutputPool.free.append(data)) {
if (itsParset.settings.realTime) {
try {
BudgetTimer::StartStop ss(writeTimer);
if (itsParset.settings.writeToDisk)
- itsWriter->write(data);
+ itsWriter->write(data.get());
} catch (SystemCallException &ex) {
LOG_WARN_STR(itsLogPrefix << "OutputThread caught non-fatal exception: " << ex.what());
itsBlocksDroppedByMe++;
continue;
}
} else { // no try/catch: any loss (e.g. disk full) is fatal in non-real-time mode
- itsWriter->write(data);
+ itsWriter->write(data.get());
}
- checkForDroppedData(data);
+ checkForDroppedData(data.get());
// print debug info for the other blocks
LOG_DEBUG_STR(itsLogPrefix << "Written block with seqno = " << data->sequenceNumber() << "(which was " << (100.0 - 100.0 * data->outputLossFraction()) << "% complete), " << itsBlocksWritten << " blocks written, " << itsBlocksDropped << " blocks not received");
@@ -321,21 +321,21 @@ namespace LOFAR
if (itsParset.settings.realTime) {
try {
- itsWriter = new MSWriterCorrelated(itsLogPrefix, path, itsParset, itsStreamNr);
+ itsWriter.reset(new MSWriterCorrelated(itsLogPrefix, path, itsParset, itsStreamNr));
logInitialStreamMetadataEvents("Correlated", fileName, directoryName);
} catch (Exception &ex) {
LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex.what());
- itsWriter = new MSWriterNull(itsParset);
+ itsWriter.reset(new MSWriterNull(itsParset));
#if defined HAVE_AIPSPP
} catch (casacore::AipsError &ex) {
LOG_ERROR_STR(itsLogPrefix << "Caught AipsError: " << ex.what());
- itsWriter = new MSWriterNull(itsParset);
+ itsWriter.reset(new MSWriterNull(itsParset));
#endif
}
} else { // don't handle exception in non-RT: it is fatal: avoid rethrow for a clean stracktrace
- itsWriter = new MSWriterCorrelated(itsLogPrefix, path, itsParset, itsStreamNr);
+ itsWriter.reset(new MSWriterCorrelated(itsLogPrefix, path, itsParset, itsStreamNr));
logInitialStreamMetadataEvents("Correlated", fileName, directoryName);
}
@@ -380,27 +380,27 @@ namespace LOFAR
if (itsParset.settings.realTime) {
try {
#ifdef HAVE_DAL
- itsWriter = new MSWriterDAL<float,3>(path, itsParset, itsStreamNr);
+ itsWriter.reset(new MSWriterDAL<float,3>(path, itsParset, itsStreamNr));
#else
- itsWriter = new MSWriterFile(path);
+ itsWriter.reset(new MSWriterFile(path));
#endif
logInitialStreamMetadataEvents("Beamformed", fileName, directoryName);
} catch (Exception &ex) {
LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex.what());
- itsWriter = new MSWriterNull(itsParset);
+ itsWriter.reset(new MSWriterNull(itsParset));
#if defined HAVE_AIPSPP
} catch (casacore::AipsError &ex) {
LOG_ERROR_STR(itsLogPrefix << "Caught AipsError: " << ex.what());
- itsWriter = new MSWriterNull(itsParset);
+ itsWriter.reset(new MSWriterNull(itsParset));
#endif
}
} else { // don't handle exception in non-RT: it is fatal: avoid rethrow for a clean stracktrace
#ifdef HAVE_DAL
- itsWriter = new MSWriterDAL<float,3,1>(path, itsParset, itsStreamNr);
+ itsWriter.reset(new MSWriterDAL<float,3,1>(path, itsParset, itsStreamNr));
#else
- itsWriter = new MSWriterFile(path);
+ itsWriter.reset(new MSWriterFile(path));
#endif
logInitialStreamMetadataEvents("Beamformed", fileName, directoryName);
}
@@ -445,17 +445,17 @@ namespace LOFAR
if (itsParset.settings.realTime) {
try {
- itsWriter = new MSWriterFile(path);
+ itsWriter.reset(new MSWriterFile(path));
rspRawParset.writeFile(path + ".parset"); // relies on (recursive) mkdir by MSWriterFile()
// The rest of the system doesn't know about RSP raw data output, but if monitoring did, enable this:
//logInitialStreamMetadataEvents("RSPRaw", fileName, directoryName);
} catch (Exception& ex) {
LOG_ERROR_STR(itsLogPrefix << "Cannot open " << path << ": " << ex.what());
- itsWriter = new MSWriterNull(itsParset);
+ itsWriter.reset(new MSWriterNull(itsParset));
}
} else { // don't handle exception in non-RT: it is fatal: avoid rethrow for a clean stracktrace
- itsWriter = new MSWriterFile(path);
+ itsWriter.reset(new MSWriterFile(path));
rspRawParset.writeFile(path + ".parset"); // relies on (recursive) mkdir by MSWriterFile()
// The rest of the system doesn't know about RSP raw data output, but if monitoring did, enable this:
diff --git a/RTCP/Cobalt/OutputProc/src/OutputThread.h b/RTCP/Cobalt/OutputProc/src/OutputThread.h
index 1d9d5694d665fe9d62b3c6172b5f94fa82bda680..50af3b08ba3ce1caaf99e90afc83090adaa0534e 100644
--- a/RTCP/Cobalt/OutputProc/src/OutputThread.h
+++ b/RTCP/Cobalt/OutputProc/src/OutputThread.h
@@ -29,7 +29,6 @@
#include <Stream/FileStream.h>
#include <MACIO/RTmetadata.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/StreamableData.h>
#include <CoInterface/TABTranspose.h>
#include <CoInterface/FinalMetaData.h>
@@ -104,7 +103,7 @@ namespace LOFAR
Pool<T> &itsOutputPool;
- SmartPtr<MSWriter> itsWriter;
+ std::unique_ptr<MSWriter> itsWriter;
};
diff --git a/RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc b/RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc
index 09b8246ea7bc90814065ab9e9c2573dd92a37d45..31b3bb2a235efbe9617c0c7afb28c3a1d7a1d27c 100644
--- a/RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc
+++ b/RTCP/Cobalt/OutputProc/src/RSPRawWriter.cc
@@ -42,7 +42,7 @@ namespace LOFAR
itsOutputThread(parset, streamNr, itsOutputPool, mdLogger, mdKeyPrefix, logPrefix)
{
for (unsigned i = 0; i < preAllocateReceiveQueue; i++) {
- RSPRawData *data = new RSPRawData();
+ std::shared_ptr<RSPRawData> data(new RSPRawData());
itsOutputPool.free.append(data);
}
}
diff --git a/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc b/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc
index 7f679c482a9ee43d73584b87b8e0735b546c8409..92ed67e21e3984cc8d6e7db8a78ae8938afe59eb 100644
--- a/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc
+++ b/RTCP/Cobalt/OutputProc/src/SubbandWriter.cc
@@ -40,8 +40,8 @@ namespace LOFAR
const std::string &logPrefix)
:
itsStreamNr(streamNr),
- itsArena(0),
- itsAllocator(0),
+ itsArena(nullptr),
+ itsAllocator(nullptr),
itsOutputPool(str(format("SubbandWriter::itsOutputPool [stream %u]") % streamNr), parset.settings.realTime),
itsInputThread(parset, CORRELATED_DATA, streamNr, itsOutputPool, logPrefix),
itsOutputThread(parset, streamNr, itsOutputPool, mdLogger, mdKeyPrefix, logPrefix),
@@ -53,12 +53,12 @@ namespace LOFAR
ASSERT(preAllocateReceiveQueue <= maxReceiveQueueSize);
// We alloc all memory at once to avoid maxReceiveQueueSize malloc() calls, which occasionally stall on CEP4
- itsArena = new MallocedArena(maxReceiveQueueSize * CorrelatedData::size(itsNrStations, itsNrChannels, itsNrSamples, itsAlignment), itsAlignment);
+ itsArena.reset(new MallocedArena(maxReceiveQueueSize * CorrelatedData::size(itsNrStations, itsNrChannels, itsNrSamples, itsAlignment), itsAlignment));
- itsAllocator = new SparseSetAllocator(*itsArena);
+ itsAllocator.reset(new SparseSetAllocator(*itsArena));
for (unsigned i = 0; i < preAllocateReceiveQueue; i++) {
- CorrelatedData *data = new CorrelatedData(itsNrStations, itsNrChannels, itsNrSamples, *itsAllocator, itsAlignment);
+ std::shared_ptr<CorrelatedData> data(new CorrelatedData(itsNrStations, itsNrChannels, itsNrSamples, *itsAllocator, itsAlignment));
itsOutputPool.free.append(data);
}
}
@@ -73,7 +73,7 @@ namespace LOFAR
OMPThread::ScopedName sn(str(format("allocator %u") % itsStreamNr));
for (unsigned i = preAllocateReceiveQueue; i < maxReceiveQueueSize; i++) {
- CorrelatedData *data = new CorrelatedData(itsNrStations, itsNrChannels, itsNrSamples, *itsAllocator, itsAlignment);
+ std::shared_ptr<CorrelatedData> data(new CorrelatedData(itsNrStations, itsNrChannels, itsNrSamples, *itsAllocator, itsAlignment));
itsOutputPool.free.append(data);
}
}
diff --git a/RTCP/Cobalt/OutputProc/src/SubbandWriter.h b/RTCP/Cobalt/OutputProc/src/SubbandWriter.h
index 9127148ec04bcc24376f66a2263b7f706ed3ef5e..cbc9c0e96c2b7be67d29f45b19120aab9a94e8f6 100644
--- a/RTCP/Cobalt/OutputProc/src/SubbandWriter.h
+++ b/RTCP/Cobalt/OutputProc/src/SubbandWriter.h
@@ -27,7 +27,6 @@
#include <CoInterface/Parset.h>
#include <CoInterface/Pool.h>
#include <CoInterface/Allocator.h>
-#include <CoInterface/SmartPtr.h>
#include <CoInterface/StreamableData.h>
#include <CoInterface/FinalMetaData.h>
#include "InputThread.h"
@@ -67,8 +66,8 @@ namespace LOFAR
const unsigned itsStreamNr;
- SmartPtr<Arena> itsArena;
- SmartPtr<Allocator> itsAllocator;
+ std::unique_ptr<Arena> itsArena;
+ std::unique_ptr<Allocator> itsAllocator;
Pool<StreamableData> itsOutputPool;
InputThread itsInputThread;
diff --git a/RTCP/Cobalt/OutputProc/test/tMSWriterDAL.cc b/RTCP/Cobalt/OutputProc/test/tMSWriterDAL.cc
index 3dfc126944eafcea2a95e7000ade4a0f1ec9c231..15197f24516b6cfc3000371f197e7532c71fcbe6 100644
--- a/RTCP/Cobalt/OutputProc/test/tMSWriterDAL.cc
+++ b/RTCP/Cobalt/OutputProc/test/tMSWriterDAL.cc
@@ -40,7 +40,6 @@ int main()
{
const size_t fileNo = 0;
- MSWriterDAL<float,3> writer("tMSWriterDAL_tmp.h5", parset, fileNo);
const ObservationSettings::BeamFormer::File &file =
parset.settings.beamFormer.files[fileNo];
@@ -52,16 +51,34 @@ int main()
const size_t nrSubbands = parset.settings.SAPs[file.sapNr].subbands.size();
- SampleData<float,3,1> data(
- boost::extents[sset.nrSamples][nrSubbands][sset.nrChannels],
- boost::extents[nrSubbands]);
+ const ObservationSettings::BeamFormer::StokesSettings::QuantizerSettings
+ &qset = sset.quantizerSettings;
+
+ if (!qset.enabled) {
+ MSWriterDAL<float,3> writer("tMSWriterDAL_tmp.h5", parset, fileNo);
+ SampleData<float,3,1> data(
+ boost::extents[sset.nrSamples][nrSubbands][sset.nrChannels],
+ boost::extents[nrSubbands]);
+ memset(data.samples.origin(), 0, data.samples.num_elements() * sizeof *data.samples.origin());
+ writer.init();
+ writer.write(&data);
+ // Dump feedback data to stdout
+ cout << writer.configuration() << endl;
+ } else {
+ MSWriterDAL<float,3> writer("tMSWriterDAL_tmp.h5", parset, fileNo);
+ SampleData<float,3,1> data(
+ nrSubbands,sset.nrChannels,sset.nrSamples,
+ true);
+ memset(data.qsamples.origin(), 0, data.qsamples.num_elements() * sizeof *data.qsamples.origin());
+ memset(data.qscales.origin(), 1, data.qscales.num_elements() * sizeof *data.qscales.origin());
+ memset(data.qoffsets.origin(), 2, data.qoffsets.num_elements() * sizeof *data.qoffsets.origin());
+ writer.init();
+ writer.write(&data);
+ // Dump feedback data to stdout
+ cout << writer.configuration() << endl;
+ }
- memset(data.samples.origin(), 0, data.samples.num_elements() * sizeof *data.samples.origin());
- writer.write(&data);
-
- // Dump feedback data to stdout
- cout << writer.configuration() << endl;
}
#else
cout << "Built without DAL, skipped actual test code." << endl;
diff --git a/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc b/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc
index 6d1ee9c0fd60a154340b465974d58f0598b827ee..501310866450d8c7873d3cdfaffbbd4e961b0a3b 100644
--- a/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc
+++ b/RTCP/Cobalt/OutputProc/test/tSubbandWriter.cc
@@ -110,7 +110,7 @@ SUITE(SubbandWriter)
{
string sendDesc = getStreamDescriptorBetweenIONandStorage(ps, CORRELATED_DATA, 0);
- SmartPtr<Stream> inputStream = createStream(sendDesc, false, 0);
+ std::unique_ptr<Stream> inputStream(createStream(sendDesc, false, 0));
CorrelatedData data(ps.nrMergedStations(), ps.settings.correlator.nrChannels, ps.settings.correlator.nrSamplesPerIntegration(), heapAllocator, 512);
@@ -118,7 +118,7 @@ SUITE(SubbandWriter)
*(data.visibilities.origin() + i) = complex<float>(i, 2*i);
}
- data.write(inputStream, 1);
+ data.write(inputStream.get(), 1);
}
}
@@ -152,7 +152,7 @@ SUITE(SubbandWriter)
{
/* connect & disconnect */
string sendDesc = getStreamDescriptorBetweenIONandStorage(ps, CORRELATED_DATA, 0);
- SmartPtr<Stream> inputStream = createStream(sendDesc, false, 0);
+ std::shared_ptr<Stream> inputStream(createStream(sendDesc, false, 0));
}
}
diff --git a/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.cc b/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.cc
index 7ad78e692b6e446ad9ebe2971987b8ef6c3ad5c1..a75184883ef40df43c37229678e6a78151e1c58a 100644
--- a/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.cc
+++ b/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.cc
@@ -892,25 +892,29 @@ namespace LOFAR
else
{
dpMeas = -1.0;
- for(unsigned sap = 0;
- sap < parset.settings.beamFormer.SAPs.size(); sap++)
+ for (auto& pipeline : parset.settings.beamFormer.pipelines)
{
- for(unsigned tab = 0;
- tab < parset.settings.beamFormer.SAPs[sap].TABs.size();
- tab++)
+ auto& SAPs = pipeline.SAPs;
+
+ for(unsigned sap = 0;
+ sap < SAPs.size(); sap++)
{
- const ObservationSettings::BeamFormer::TAB &t = parset
- .settings.beamFormer.SAPs[sap].TABs[tab];
- if(t.coherent)
+ for(unsigned tab = 0;
+ tab < SAPs[sap].TABs.size();
+ tab++)
{
- dpMeas = t.dispersionMeasure;
- break;
+ const ObservationSettings::BeamFormer::TAB &t = SAPs[sap].TABs[tab];
+ if(t.coherent)
+ {
+ dpMeas = t.dispersionMeasure;
+ break;
+ }
}
- }
- if(dpMeas != -1.0)
- {
- break;
+ if(dpMeas != -1.0)
+ {
+ break;
+ }
}
}
}
diff --git a/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.h b/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.h
index e68ce7a3b3b053a384740f329c3f2cfa6c1812fe..b250326d7387d00e8b642550670336e306071fa4 100644
--- a/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.h
+++ b/RTCP/Cobalt/TBBWriter/src/TBB_Dipole.h
@@ -31,7 +31,6 @@
#include <Common/Thread/Mutex.h>
#include <Stream/FileStream.h>
#include <CoInterface/Parset.h>
-#include <CoInterface/SmartPtr.h> // when switching to C++11, replace SmartPtr with unique_ptr
#include <dal/lofar/TBB_File.h> // https://github.com/nextgen-astrodata/DAL
@@ -56,7 +55,7 @@ namespace LOFAR
std::string itsH5Filename;
struct DumpInfo {
- SmartPtr<LOFAR::FileStream> itsRawFile;
+ std::unique_ptr<LOFAR::FileStream> itsRawFile;
std::vector<dal::Range> itsFlagOffsets;
uint64_t itsDatasetLen; // in values, including holes from missing data
diff --git a/RTCP/Cobalt/TBBWriter/src/TBB_Writer.cc b/RTCP/Cobalt/TBBWriter/src/TBB_Writer.cc
index 8b80c112fc8d329b9e6d18a55c3640e04dc39171..69fff8b83de4b6ddb0fce26487b554cef01a895b 100644
--- a/RTCP/Cobalt/TBBWriter/src/TBB_Writer.cc
+++ b/RTCP/Cobalt/TBBWriter/src/TBB_Writer.cc
@@ -61,7 +61,7 @@ namespace LOFAR
itsStreamWriters.reserve(inputStreamNames.size());
for (unsigned i = 0; i < inputStreamNames.size(); i++) {
- itsStreamWriters.push_back(SmartPtr<TBB_StreamWriter>(new TBB_StreamWriter(*this,
+ itsStreamWriters.push_back(std::unique_ptr<TBB_StreamWriter>(new TBB_StreamWriter(*this,
inputStreamNames[i], logPrefix, thrExitStatus.at(2 * i), thrExitStatus.at(2 * i + 1))));
}
}
@@ -108,7 +108,7 @@ namespace LOFAR
TBB_Station *TBB_Writer::getStation(const TBB_Header& header)
{
ScopedLock sl(itsStationsMutex); // protect against insert below
- map<unsigned, SmartPtr<TBB_Station> >::iterator stIt(itsStations.find(header.stationID));
+ map<unsigned, std::unique_ptr<TBB_Station>>::iterator stIt(itsStations.find(header.stationID));
if (stIt != itsStations.end()) {
LOG_DEBUG_STR("TBB_Writer::getStation: found known station for id "
<< static_cast< uint32_t >(header.stationID));
@@ -123,16 +123,16 @@ namespace LOFAR
// If not found, station is not participating in the observation. Should not happen, but don't panic.
const StationMetaData& stMetaData = stMdIt == itsStationMetaDataMap.end() ? itsUnknownStationMetaData : stMdIt->second;
- SmartPtr<TBB_Station> station;
+ std::unique_ptr<TBB_Station> station;
{
ScopedLock slH5(itsH5Mutex);
- station = new TBB_Station(stationName, itsH5Mutex, itsParset,
- itsAllSubbandCentralFreqs, stMetaData, h5Filename, subbandSize);
+ station.reset(new TBB_Station(stationName, itsH5Mutex, itsParset,
+ itsAllSubbandCentralFreqs, stMetaData, h5Filename, subbandSize));
}
LOG_DEBUG_STR("TBB_Writer::getStation: returning new station for id "
<< static_cast< uint32_t >(header.stationID));
- return itsStations.insert(make_pair(header.stationID, station)).first->second.get();
+ return itsStations.insert(make_pair(header.stationID, std::move(station))).first->second.get();
}
string TBB_Writer::createNewTBB_H5Filename(const TBB_Header& header, const string& stationName)
diff --git a/RTCP/Cobalt/TBBWriter/src/TBB_Writer.h b/RTCP/Cobalt/TBBWriter/src/TBB_Writer.h
index 34adf441ad7330e2d12c9dd4ae1a53c893cef3da..f9df93bffb6e11a083bfaa1f41d16f847d7ef4ff 100644
--- a/RTCP/Cobalt/TBBWriter/src/TBB_Writer.h
+++ b/RTCP/Cobalt/TBBWriter/src/TBB_Writer.h
@@ -38,7 +38,7 @@ namespace LOFAR
Mutex itsH5Mutex; // used in ~Station(), so declared before itsStations below for proper destruction order
// Usually, we handle only 1 station per writer, but users have requested to support multiple concurrently (still needed?).
- std::map<unsigned, SmartPtr<TBB_Station> > itsStations; // stationID -> SmartPtr<TBB_Station>
+ std::map<unsigned, std::unique_ptr<TBB_Station> > itsStations; // stationID -> std::unique_ptr<TBB_Station>
Mutex itsStationsMutex;
const Parset& itsParset;
@@ -51,7 +51,7 @@ namespace LOFAR
unsigned itsRunNr;
- std::vector<SmartPtr<TBB_StreamWriter> > itsStreamWriters;
+ std::vector<std::unique_ptr<TBB_StreamWriter> > itsStreamWriters;
// Note: do not add vars here; leave itsStreamWriters last for safe thread destruction!
// do not use
diff --git a/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py b/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
index 3c536222dd32d5b566e16a3028d6935139fb01c1..ae626422a05db7bd5344961749d60c4f3e8d72aa 100755
--- a/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
+++ b/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
@@ -503,7 +503,7 @@ class RAtoOTDBTranslator():
## control software, giving everyone a puzzled expression on their face and a big headache when figuring
## out why the system was sometimes behaving so funny...
# FIXME: please find a better way to do this or remove this hack when not necessary any more!
- if project_name in ['IPS_Commissioning', 'LC6_001', 'LC7_001', 'LC8_001', 'LC9_001', 'LT10_001', 'LT10_002', 'LT10_006', 'LC14_001']:
+ if project_name in ['IPS_Commissioning', 'LC6_001', 'LC7_001', 'LC8_001', 'LC9_001', 'LT10_001', 'LT10_002', 'LT10_006', 'LC14_001', 'LC15_001']:
logging.info("CreateParset: Overwriting inspectionProgram parset key for dynspec")
parset[PREFIX+'ObservationControl.OnlineControl.inspectionProgram'] = '/data/home/lofarsys/dynspec/scripts/inspection-dynspec-observation.sh'
diff --git a/SAS/TMSS/docker-compose-scu199.yml b/SAS/TMSS/docker-compose-scu199.yml
index 85cfd2d27d6fd292129294551405937b511a07bf..227ca832e0a291a189d1f60c098439ead86ec1a8 100644
--- a/SAS/TMSS/docker-compose-scu199.yml
+++ b/SAS/TMSS/docker-compose-scu199.yml
@@ -7,7 +7,7 @@ services:
env_file:
- ./.env
network_mode: "host"
- command: bash -c 'source /opt/lofar/lofarinit.sh && ALLOWED_HOSTS=* tmss_test_environment -H 0.0.0.0 -P `hostname -f` -p 8008 -sSd'
+ command: bash -c 'source /opt/lofar/lofarinit.sh && ALLOWED_HOSTS=* tmss_test_environment -H 0.0.0.0 -P `hostname -f` -p 8008 -sSdv'
ports:
- "8008:8008"
testprovider:
diff --git a/SAS/TMSS/frontend/tmss_webapp/.vscode/launch.json b/SAS/TMSS/frontend/tmss_webapp/.vscode/launch.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6b35a0b639d037c20e30a924f2df13e783620c5
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/.vscode/launch.json
@@ -0,0 +1,15 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "type": "chrome",
+ "request": "launch",
+ "name": "Launch Chrome against localhost",
+ "url": "http://localhost:3000",
+ "webRoot": "${workspaceFolder}"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/package.json b/SAS/TMSS/frontend/tmss_webapp/package.json
index 705a5183e8801882a780f93350b30333e5bd3582..c32b29542e63179bd7481faa6b90dfa4b122b27c 100644
--- a/SAS/TMSS/frontend/tmss_webapp/package.json
+++ b/SAS/TMSS/frontend/tmss_webapp/package.json
@@ -22,6 +22,8 @@
"font-awesome": "^4.7.0",
"history": "^5.0.0",
"interactjs": "^1.9.22",
+ "js-cookie": "^2.2.1",
+ "katex": "^0.12.0",
"lodash": "^4.17.19",
"match-sorter": "^4.1.0",
"moment": "^2.27.0",
@@ -57,7 +59,7 @@
"test": "react-scripts test",
"eject": "react-scripts eject"
},
- "proxy": "http://127.0.0.1:8008/",
+ "proxy": "http://localhost:8008/",
"eslintConfig": {
"extends": "react-app"
},
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/App.js b/SAS/TMSS/frontend/tmss_webapp/src/App.js
index 28dc4939d93ef1dcc6a112bb1da40a4a8fa79067..3abca18eb4682b9a8debe753161544e2b4d0c0ea 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/App.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/App.js
@@ -1,4 +1,5 @@
import React, {Component} from 'react';
+import { Redirect} from 'react-router-dom';
import { BrowserRouter as Router } from 'react-router-dom';
import classNames from 'classnames';
import {AppTopbar} from './layout/components/AppTopbar';
@@ -15,40 +16,41 @@ import './layout/layout.scss';
import 'primeflex/primeflex.css';
import './App.scss';
import './App.css';
+import Auth from'./authenticate/auth';
+import { Login } from './authenticate/login';
class App extends Component {
constructor() {
- super();
- this.state = {
- layoutMode: 'static',
- currentMenu: '',
- currentPath: '/',
- PageTitle:'',
- staticMenuInactive: localStorage.getItem('staticMenuInactive') === 'true' ? true : false,
- overlayMenuActive: localStorage.getItem('overlayMenuActive') === 'true' ? true : false,
- mobileMenuActive: localStorage.getItem('mobileMenuActive') === 'true' ? true : false,
- };
- this.onWrapperClick = this.onWrapperClick.bind(this);
+ super();
+ this.state = {
+ layoutMode: 'static',
+ currentMenu: '',
+ currentPath: '/',
+ PageTitle:'',
+ staticMenuInactive: localStorage.getItem('staticMenuInactive') === 'true' ? true : false,
+ overlayMenuActive: localStorage.getItem('overlayMenuActive') === 'true' ? true : false,
+ mobileMenuActive: localStorage.getItem('mobileMenuActive') === 'true' ? true : false,
+ authenticated: Auth.isAuthenticated(),
+ redirect: (Auth.isAuthenticated() && window.location.pathname === "/login")?"/":window.location.pathname
+ };
+ this.onWrapperClick = this.onWrapperClick.bind(this);
this.onToggleMenu = this.onToggleMenu.bind(this);
this.onSidebarClick = this.onSidebarClick.bind(this);
this.onMenuItemClick = this.onMenuItemClick.bind(this);
this.setPageTitle = this.setPageTitle.bind(this);
-
- this.menu = [
- {label: 'Dashboard', icon: 'pi pi-fw pi-home', to:'/dashboard',section: 'dashboard'},
- {label: 'Cycle', icon:'pi pi-fw pi-spinner', to:'/cycle',section: 'cycle'},
- {label: 'Project', icon: 'fab fa-fw fa-wpexplorer', to:'/project',section: 'project'},
- {label: 'Scheduling Units', icon: 'pi pi-fw pi-calendar', to:'/schedulingunit',section: 'schedulingunit'},
- {label: 'Timeline', icon: 'pi pi-fw pi-clock', to:'/su/timelineview',section: 'su/timelineview'},
- // {label: 'Tasks', icon: 'pi pi-fw pi-check-square', to:'/task'},
-
-
- ];
+ this.loggedIn = this.loggedIn.bind(this);
+ this.logout = this.logout.bind(this);
- // this.menuComponent = {'Dashboard': Dashboard}
- }
-
- onWrapperClick(event) {
+ this.menu = [ {label: 'Dashboard', icon: 'pi pi-fw pi-home', to:'/dashboard',section: 'dashboard'},
+ {label: 'Cycle', icon:'pi pi-fw pi-spinner', to:'/cycle',section: 'cycle'},
+ {label: 'Project', icon: 'fab fa-fw fa-wpexplorer', to:'/project',section: 'project'},
+ {label: 'Scheduling Units', icon: 'pi pi-fw pi-calendar', to:'/schedulingunit',section: 'schedulingunit'},
+ {label: 'Timeline', icon: 'pi pi-fw pi-clock', to:'/su/timelineview',section: 'su/timelineview'},
+ // {label: 'Tasks', icon: 'pi pi-fw pi-check-square', to:'/task'},
+ ];
+ }
+
+ onWrapperClick(event) {
if (!this.menuClick) {
this.setState({
overlayMenuActive: false,
@@ -94,9 +96,9 @@ class App extends Component {
this.menuClick = true;
}
- onMenuItemClick(event) {
- this.setState({currentMenu:event.item.label, currentPath: event.item.path});
- }
+ onMenuItemClick(event) {
+ this.setState({currentMenu:event.item.label, currentPath: event.item.path});
+ }
isDesktop() {
return window.innerWidth > 1024;
@@ -107,36 +109,72 @@ class App extends Component {
this.setState({ PageTitle })
}
}
-
- render() {
- const wrapperClass = classNames('layout-wrapper', {
- 'layout-overlay': this.state.layoutMode === 'overlay',
- 'layout-static': this.state.layoutMode === 'static',
- 'layout-static-sidebar-inactive': this.state.staticMenuInactive && this.state.layoutMode === 'static',
- 'layout-overlay-sidebar-active': this.state.overlayMenuActive && this.state.layoutMode === 'overlay',
- 'layout-mobile-sidebar-active': this.state.mobileMenuActive
- });
- const AppBreadCrumbWithRouter = withRouter(AppBreadcrumb);
-
- return (
- <React.Fragment>
- <div className="App">
- {/* <div className={wrapperClass} onClick={this.onWrapperClick}> */}
- <div className={wrapperClass}>
- <AppTopbar onToggleMenu={this.onToggleMenu}></AppTopbar>
- <Router basename={ this.state.currentPath }>
- <AppMenu model={this.menu} onMenuItemClick={this.onMenuItemClick} layoutMode={this.state.la} active={this.state.menuActive}/>
- <div className="layout-main">
- <AppBreadCrumbWithRouter setPageTitle={this.setPageTitle} />
- <RoutedContent />
- </div>
- </Router>
- <AppFooter></AppFooter>
- </div>
+
+ /**
+ * Callback function from login page to set the authentication state to true amd redirect to the
+ * original requested URL.
+ */
+ loggedIn() {
+ const redirect = this.state.redirect;
+ this.setState({authenticated: true, redirect: redirect==="/login"?"/":redirect});
+ }
+
+ /**
+ * Logout and redirect to login page.
+ */
+ logout() {
+ Auth.logout();
+ this.setState({authenticated: false, redirect:"/"});
+ }
+
+ render() {
+ const wrapperClass = classNames('layout-wrapper', {
+ 'layout-overlay': this.state.layoutMode === 'overlay',
+ 'layout-static': this.state.layoutMode === 'static',
+ 'layout-static-sidebar-inactive': this.state.staticMenuInactive && this.state.layoutMode === 'static',
+ 'layout-overlay-sidebar-active': this.state.overlayMenuActive && this.state.layoutMode === 'overlay',
+ 'layout-mobile-sidebar-active': this.state.mobileMenuActive
+ });
+ const AppBreadCrumbWithRouter = withRouter(AppBreadcrumb);
+ console.log(this.props);
+ return (
+ <React.Fragment>
+ <div className="App">
+ {/* <div className={wrapperClass} onClick={this.onWrapperClick}> */}
+ <div className={wrapperClass}>
+
+ {/* Load main routes and application only if the application is authenticated */}
+ {this.state.authenticated &&
+ <>
+ <AppTopbar onToggleMenu={this.onToggleMenu} isLoggedIn={this.state.authenticated} onLogout={this.logout}></AppTopbar>
+ <Router basename={ this.state.currentPath }>
+ <AppMenu model={this.menu} onMenuItemClick={this.onMenuItemClick} layoutMode={this.state.la} active={this.state.menuActive}/>
+ <div className="layout-main">
+ {this.state.redirect &&
+ <Redirect to={{pathname: this.state.redirect}} />}
+ <AppBreadCrumbWithRouter setPageTitle={this.setPageTitle} />
+ <RoutedContent />
+ </div>
+ </Router>
+ <AppFooter></AppFooter>
+ </>
+ }
+
+ {/* If not authenticated, show only login page */}
+ {!this.state.authenticated &&
+ <>
+ <Router basename={ this.state.currentPath }>
+ <Redirect to={{pathname: "/login"}} />
+ <Login onLogin={this.loggedIn} />
+ </Router>
+ </>
+ }
+
+ </div>
</div>
- </React.Fragment>
- );
- }
+ </React.Fragment>
+ );
+ }
}
export default App;
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/authenticate/auth.js b/SAS/TMSS/frontend/tmss_webapp/src/authenticate/auth.js
new file mode 100644
index 0000000000000000000000000000000000000000..c5e845a6f3d5b5433750ad193d8dc2ce628ef466
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/authenticate/auth.js
@@ -0,0 +1,37 @@
+// import AuthService from "../services/auth.service";
+
+/**
+ * Global functions to authenticate user and get user details from browser local storage.
+ */
+const Auth = {
+ /** To check if user already logged in and the token is available in the browser local storage */
+ isAuthenticated: () => {
+ let user = localStorage.getItem("user");
+ if (user) {
+ user = JSON.parse(user);
+ if (user.token) {
+ return true;
+ }
+ }
+ return false;
+ },
+ /** Gets user details from browser local storage */
+ getUser: () => {
+ return JSON.parse(localStorage.getItem("user"));
+ },
+ /** Authenticate user from the backend and store user details in local storage */
+ login: async(user, pass) => {
+ // const user = await AuthService.authenticate();
+ if (user) {
+ //TODO set token and username
+ }
+ localStorage.setItem("user", JSON.stringify({name:user, token: "ABCDEFGHIJ"}));
+ return true;
+ },
+ /** Remove user details from localstorage on logout */
+ logout: () => {
+ localStorage.removeItem("user");
+ }
+}
+
+export default Auth;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/authenticate/login.js b/SAS/TMSS/frontend/tmss_webapp/src/authenticate/login.js
new file mode 100644
index 0000000000000000000000000000000000000000..4512989fd19744d8476290430ad3a77819e656c5
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/authenticate/login.js
@@ -0,0 +1,133 @@
+import { InputText } from 'primereact/inputtext';
+import React, {Component} from 'react';
+import { Redirect } from 'react-router-dom';
+import Auth from '../authenticate/auth';
+
+/**
+ * Component to authenticate users in the application.
+ */
+export class Login extends Component {
+ constructor(props){
+ super(props);
+ this.state = {
+ username: null,
+ password: null,
+ redirect: Auth.isAuthenticated()?"/":null, //If already logged in, redirect to home page
+ error: null,
+ errors: {},
+ validFields: {}
+ };
+ this.login = this.login.bind(this);
+ this.setCredentials = this.setCredentials.bind(this);
+ }
+
+ /**
+ * To set form field values.
+ * @param {String} field
+ * @param {String} value
+ */
+ setCredentials(field, value) {
+ let state = this.state;
+ let errors = state.errors;
+ let validFields = state.validFields;
+ // If value is null or empty set error field and remove from valid field and vice versa
+ if (!value) {
+ errors[field] = `${field} required`;
+ delete validFields[field];
+ } else {
+ delete errors[field];
+ validFields[field] = field;
+ }
+ state[field] = value;
+ state.errors = errors;
+ state.validFields = validFields;
+ this.setState(state);
+ }
+
+ /**
+ * Login function called on click of 'Login' button.
+ * If authenticated, callback parent component function.
+ */
+ async login() {
+ const loggedIn = await Auth.login(this.state.username, this.state.password);
+ if (loggedIn) {
+ this.setState({error: false});
+ this.props.onLogin();
+ } else {
+ this.setState({error: true});
+ }
+ }
+
+ render() {
+ if (this.state.redirect) {
+ return (<Redirect to={{pathname: this.state.redirect}} />);
+ }
+ return (
+ <>
+ <div className="container-fluid bg-login">
+ <div className="container">
+ <div className="row">
+ <div className="col-lg-9 col-md-12 login-card">
+ <div className="row">
+ {/* Left panel with image and TMSS title */}
+ <div className="col-md-5 detail-part">
+ <h3>Telescope Manager Specification System</h3>
+ <p>By ASTRON</p>
+ </div>
+ {/* Right panel with login form */}
+ <div className="col-md-7 logn-part">
+ <div className="row">
+ <div className="col-lg-10 col-md-12 mx-auto">
+ <div className="logo-cover">
+ {/* <img src="./logo.png" alt="" /> */}
+ </div>
+ <div className="login-form">
+ <h4>Login</h4>
+ <div className="form-field">
+ <span className="p-float-label">
+ <InputText id="inputtext" className={`${this.state.errors.username?"input-error ":""} form-control`}
+ value={this.state.username} onChange={(e) => this.setCredentials('username', e.target.value)} />
+ <label htmlFor="inputtext"><i className="fa fa-user"></i>Enter Username</label>
+ </span>
+ <label className={this.state.errors.username?"error":""}>
+ {this.state.errors.username?this.state.errors.username : ""}
+ </label>
+ </div>
+ <div className="form-field">
+ <span className="p-float-label">
+ <InputText id="inputtext" className={`${this.state.errors.password?"input-error ":""} form-control`}
+ type="password" value={this.state.password} onChange={(e) => this.setCredentials('password', e.target.value )} />
+ <label htmlFor="inputtext"><i className="fa fa-key"></i>Enter Password</label>
+ </span>
+ <label className={this.state.errors.password?"error":""}>
+ {this.state.errors.password?this.state.errors.password : ""}
+ </label>
+ </div>
+ <div className="row form-footer">
+ <div className="col-md-6 forget-paswd">
+
+ </div>
+ <div className="col-md-6 button-div">
+ <button className="btn btn-primary"
+ disabled={Object.keys(this.state.validFields).length<2}
+ onClick={this.login}>Login</button>
+ </div>
+ </div>
+ {this.state.error &&
+ <div className="row error">
+ Unable to login, please try with different Username and/or Password.
+ </div>
+ }
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </>
+ );
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/CustomPageSpinner.js b/SAS/TMSS/frontend/tmss_webapp/src/components/CustomPageSpinner.js
new file mode 100644
index 0000000000000000000000000000000000000000..cc794fe8ee4f3178c5d61a281cb6357abdde97ca
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/CustomPageSpinner.js
@@ -0,0 +1,20 @@
+import React, {Component} from 'react';
+import { ProgressSpinner } from 'primereact/progressspinner';
+
+/**
+ * Custom spinner component for the whole page.
+ */
+export class CustomPageSpinner extends Component {
+ render() {
+ return (
+ <>
+ {this.props.visible &&
+ <div style={{position: 'fixed', left:0, top: 0, width:'100%', height:'100%', backgroundColor: 'grey', zIndex: 9999, opacity: '0.5'}}>
+ <span style={{position: 'absolute', top: '50%', left:'50%', '-ms-transform': 'translateY(-50%)', transform: 'translateY(-50%)', backgroundColor:'white' }}>
+ <ProgressSpinner /></span>
+ </div>
+ }
+ </>
+ );
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js b/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
index 1a4f0d4290cde1e5be6ce84333dad9622678693a..7cc46ca9851a7529b85dd822b454b4164aa53c20 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
@@ -40,17 +40,25 @@ function Jeditor(props) {
if (property["$ref"] && !property["$ref"].startsWith("#")) { // 1st level reference of the object
const refUrl = property["$ref"];
let newRef = refUrl.substring(refUrl.indexOf("#"));
- if (refUrl.endsWith("/pointing")) { // For type pointing
- schema.definitions["pointing"] = (await $RefParser.resolve(refUrl)).get(newRef);
- property["$ref"] = newRef;
- } else { // General object to resolve if any reference in child level
- property = await resolveSchema((await $RefParser.resolve(refUrl)).get(newRef));
- }
+ //>>>>>> TODO if pointin works fine, remove these commented lines
+ // if (refUrl.endsWith("/pointing")) { // For type pointing
+ // schema.definitions["pointing"] = (await $RefParser.resolve(refUrl)).get(newRef);
+ // property["$ref"] = newRef;
+ // } else { // General object to resolve if any reference in child level
+ // property = await resolveSchema((await $RefParser.resolve(refUrl)).get(newRef));
+ // }
+ let defKey = refUrl.substring(refUrl.lastIndexOf("/")+1);
+ schema.definitions[defKey] = (await $RefParser.resolve(refUrl)).get(newRef);
+ property["$ref"] = newRef;
} else if(property["type"] === "array") { // reference in array items definition
let resolvedItems = await resolveSchema(property["items"]);
schema.definitions = {...schema.definitions, ...resolvedItems.definitions};
delete resolvedItems['definitions'];
property["items"] = resolvedItems;
+ } else if(property["type"] === "object" && property.properties) {
+ property = await resolveSchema(property);
+ schema.definitions = {...schema.definitions, ...property.definitions};
+ delete property['definitions'];
}
properties[propertyKey] = property;
}
@@ -64,19 +72,34 @@ function Jeditor(props) {
} else if (schema["$ref"] && !schema["$ref"].startsWith("#")) { //reference in oneOf list item
const refUrl = schema["$ref"];
let newRef = refUrl.substring(refUrl.indexOf("#"));
- if (refUrl.endsWith("/pointing")) {
- schema.definitions["pointing"] = (await $RefParser.resolve(refUrl)).get(newRef);
- schema["$ref"] = newRef;
- } else {
- schema = await resolveSchema((await $RefParser.resolve(refUrl)).get(newRef));
+ //>>>>>> TODO: If pointing works fine, remove these commented lines
+ // if (refUrl.endsWith("/pointing")) {
+ // schema.definitions["pointing"] = (await $RefParser.resolve(refUrl)).get(newRef);
+ // schema["$ref"] = newRef;
+ // } else {
+ // schema = await resolveSchema((await $RefParser.resolve(refUrl)).get(newRef));
+ // }
+ let defKey = refUrl.substring(refUrl.lastIndexOf("/")+1);
+ schema.definitions[defKey] = (await $RefParser.resolve(refUrl)).get(newRef);
+ if (schema.definitions[defKey].properties) {
+ let property = await resolveSchema(schema.definitions[defKey]);
+ schema.definitions = {...schema.definitions, ...property.definitions};
+ delete property['definitions'];
+ schema.definitions[defKey] = property;
}
+ schema["$ref"] = newRef;
}
return schema;
}
const init = async () => {
- const element = document.getElementById('editor_holder');
+ const element = document.getElementById(props.id?props.id:'editor_holder');
let schema = await resolveExternalRef();
+ /** If any formatting is done at the parent/implementation component pass the resolved schema
+ and get the formatted schema like adding validation type, field ordering, etc.,*/
+ if (props.defintionFormatter) {
+ props.defintionFormatter(schema);
+ }
pointingProps = [];
// Customize the pointing property to capture angle1 and angle2 to specified format
for (const definitionKey in schema.definitions) {
@@ -541,7 +564,7 @@ function Jeditor(props) {
return (
<React.Fragment>
- <div id='editor_holder'></div>
+ <div id={props.id?props.id:'editor_holder'}></div>
</React.Fragment>
);
};
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/BetweenEditor.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/BetweenEditor.js
new file mode 100644
index 0000000000000000000000000000000000000000..644620bc7655a8ab4a00b9e3e2ee1aff7d5e44bf
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/BetweenEditor.js
@@ -0,0 +1,172 @@
+import React, { Component } from 'react';
+
+import {Calendar} from 'primereact/calendar';
+import { Dialog } from 'primereact/dialog';
+import { Button } from 'primereact/button';
+
+import moment from 'moment';
+import _ from 'lodash';
+
+const DATE_TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+
+export default class BetweenEditor extends Component {
+ constructor(props) {
+ super(props);
+ this.tmpRowData = [];
+
+ this.state = {
+ showDialog: false,
+ dialogTitle: '',
+ };
+
+ this.copyDateValue = this.copyDateValue.bind(this);
+ }
+
+ isPopup() {
+ return true;
+ }
+
+ /**
+ * Init the date value if exists
+ */
+ async componentDidMount(){
+ let parentRows = this.props.agGridReact.props.rowData[this.props.node.rowIndex];
+ let parentCellData = parentRows[this.props.colDef.field];
+ this.tmpRowData = [];
+ if(parentCellData){
+ let cellDataArray = _.split(parentCellData, '|');
+ await cellDataArray.forEach(dataVal =>{
+ let line = {};
+ let dataValue = _.split(dataVal, ',');
+ line['from'] = (dataValue[0])? moment(dataValue[0]).toDate():'';
+ line['until'] = ( dataValue[1])? moment(dataValue[1]).toDate():'';
+ this.tmpRowData.push(line);
+ });
+ }
+ if(this.tmpRowData.length>0){
+ let row = this.tmpRowData[this.tmpRowData.length-1];
+ if((row['from'] !== '' && row['from'] !== 'undefined') && (row['until'] !== '' && row['until'] !== 'undefined')){
+ let line = {'from': '', 'until': ''};
+ this.tmpRowData.push(line);
+ }
+ }else{
+ let line = {'from': '', 'until': ''};
+ this.tmpRowData.push(line);
+ }
+
+ await this.setState({
+ rowData: this.tmpRowData,
+ dialogTitle: (this.props.colDef.field === 'between') ? this.props.colDef.field : 'Not-Between',
+ showDialog: true,
+ });
+
+ }
+
+ /*isCancelAfterEnd(){console.log('after')
+ console.log('called')
+ this.copyDateValue();
+ }*/
+
+ /**
+ * Call the function on click Esc or Close the dialog
+ */
+async copyDateValue(){
+ let consolidateDates = '';
+ this.state.rowData.map(row =>{
+ if((row['from'] !== '' && row['from'] !== 'undefined') && (row['until'] !== '' && row['until'] !== 'undefined')){
+ consolidateDates += ((row['from'] !== '')?moment(row['from']).format(DATE_TIME_FORMAT):'' )+","+((row['until'] !== '')?moment(row['until']).format(DATE_TIME_FORMAT):'')+"|";
+ }
+ });
+ await this.props.context.componentParent.updateTime(
+ this.props.node.rowIndex,this.props.colDef.field, consolidateDates
+ );
+ this.setState({ showDialog: false});
+
+}
+
+/*
+ Set value in relevant field
+ */
+updateDateChanges(rowIndex, field, e){
+ let tmpRows = this.state.rowData;
+ let row = tmpRows[rowIndex];
+ row[field] = e.value;
+ tmpRows[rowIndex] = row;
+ if(this.state.rowData.length === rowIndex+1){
+ let line = {'from': '', 'until': ''};
+ tmpRows.push(line);
+ }
+ this.setState({
+ rowData: tmpRows
+ })
+}
+
+/*
+ Remove the the row from dialog
+*/
+removeInput(rowIndex){
+ let tmpRows = this.state.rowData;
+ delete tmpRows[rowIndex];
+ this.setState({
+ rowData: tmpRows
+ })
+}
+
+render() {
+ return (
+ <>
+ {this.state.rowData && this.state.rowData.length > 0 &&
+ <Dialog header={_.startCase(this.state.dialogTitle)} visible={this.state.showDialog} maximized={false}
+ onHide={() => {this.copyDateValue()}} inputId="confirm_dialog"
+ footer={<div>
+ <Button key="back" label="Close" onClick={() => {this.copyDateValue()}} />
+ </div>
+ } >
+ <div className="ag-theme-balham" style={{ height: '500px', width: '600px', paddingLeft: '20px' }}>
+ <div className="p-field p-grid" >
+ <React.Fragment>
+ <label key={'labelFrom'} className="col-lg-6 col-md-6 col-sm-12">From</label>
+ <label key={'labelUntil'} className="col-lg-4 col-md-5 col-sm-12">Until</label>
+ <label key={'labelRemove'} className="col-lg-2 col-md-2 col-sm-12">Remove</label>
+ </React.Fragment>
+ </div>
+ {this.state.rowData.map((bdate, index) => (
+ <React.Fragment key={index}>
+ <div className="p-field p-grid" >
+ <Calendar
+ d dateFormat="dd-M-yy"
+ value= {this.state.rowData[index].from}
+ onChange= {e => {this.updateDateChanges(index, 'from', e)}}
+ // onBlur= {e => {this.updateDateChanges(index, 'from', e)}}
+ showTime={true}
+ showSeconds={true}
+ hourFormat="24"
+ showIcon={true}
+ />
+ <Calendar
+ d dateFormat="dd-M-yy"
+ value= {this.state.rowData[index].until}
+ onChange= {e => {this.updateDateChanges(index, 'until', e)}}
+ // onBlur= {e => {this.updateDateChanges(index, 'until', e)}}
+ showTime={true}
+ showSeconds={true}
+ hourFormat="24"
+ showIcon={true}
+ style={{marginLeft:'60px'}}
+ />
+ {this.state.rowData.length !== (index+1) &&
+ <button className="p-link" style={{marginLeft: '6vw'}} onClick={(e) => this.removeInput(index)} >
+ <i className="fa fa-trash pi-error"></i></button>
+ }
+ </div>
+
+ </React.Fragment>
+ ))}
+ </div>
+ </Dialog>
+ }
+ </>
+ );
+}
+
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/BetweenRenderer.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/BetweenRenderer.js
new file mode 100644
index 0000000000000000000000000000000000000000..90a8ca3d7fc4ca9f22084fd5c9e6db063e80366d
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/BetweenRenderer.js
@@ -0,0 +1,18 @@
+import React, { Component } from 'react';
+
+export default class BetweenRenderer extends Component {
+ constructor(props) {
+ super(props);
+ }
+
+ /**
+ Show cell value in grid
+ */
+ render() {
+ let row = this.props.agGridReact.props.rowData[this.props.node.rowIndex];
+ let value = row[this.props.colDef.field];
+ return <> {value &&
+ value
+ }</>;
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComp.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComp.js
new file mode 100644
index 0000000000000000000000000000000000000000..e92fe5f4719adac8aebd00fcc41aa4b5cfa5e444
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComp.js
@@ -0,0 +1,85 @@
+import React, { Component } from 'react';
+import {Calendar} from 'primereact/calendar';
+import moment from 'moment';
+
+const DATE_TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+
+export default class CustomDateComp extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ date: '',
+ };
+ }
+
+ componentDidMount(){
+ let parentRows = this.props.agGridReact.props.rowData[this.props.node.rowIndex];
+ let parentCellData = parentRows[this.props.colDef.field];
+ this.setState({
+ date:parentCellData
+ })
+ }
+
+ isPopup() {
+ return true;
+ }
+
+ isCancelAfterEnd(){
+ let date = (this.state.date !== '' && this.state.date !== 'undefined')? moment(this.state.date).format(DATE_TIME_FORMAT) :'';
+ this.props.context.componentParent.updateTime(
+ this.props.node.rowIndex,this.props.colDef.field, date
+ );
+ }
+
+ render() {
+ return (
+ <Calendar
+ d dateFormat="dd-M-yy"
+ value= {this.state.date}
+ onChange= {e => {this.updateDateChanges(e)}}
+ onBlur= {e => {this.updateDateChanges(e)}}
+ //data-testid="start"
+ showTime= {true}
+ showSeconds= {true}
+ hourFormat= "24"
+ showIcon= {true}
+ />
+ );
+ }
+
+
+ updateDateChanges(e){
+ if(e.value){
+ this.setState({date : e.value});
+ }
+ }
+
+ ondatechange(e){
+ this.setState({date : e.value});
+ }
+
+ getDate() {
+ return this.state.date;
+ }
+
+ setDate(date) {
+ this.setState({ date });
+ this.picker.setDate(date);
+ }
+
+ updateAndNotifyAgGrid(date) {
+ this.setState(
+ {
+ date,
+ },
+ this.props.onDateChanged
+ );
+ }
+
+
+ onDateChanged = (selectedDates) => {
+ this.props.context.componentParent.updateTime(
+ this.props.node.rowIndex,this.props.colDef.field,selectedDates[0]
+ );
+ };
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComponent.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComponent.js
new file mode 100644
index 0000000000000000000000000000000000000000..7e0c18e9b6926bb138c3c6b7667d67f7fa76d930
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/CustomDateComponent.js
@@ -0,0 +1,114 @@
+import React, { Component } from 'react';
+import flatpickr from 'flatpickr';
+import "flatpickr/dist/flatpickr.css";
+
+export default class CustomDateComponent extends Component {
+ constructor(props) {
+ super(props);
+
+ this.state = {
+ date: null,
+ };
+ }
+
+ isPopup() {
+ return true;
+ }
+
+ render() {
+ //Inlining styles to make simpler the component
+ return (
+ <div
+ className="ag-input-wrapper custom-date-filter"
+ role="presentation"
+ ref="flatpickr"
+ style={{ height: '50px', width: '90%' }}>
+
+ <input type="text" ref="eInput" data-input style={{ width: '100%',}} />
+ <a className="input-button" title="clear" data-clear>
+ <i className="fa fa-times"></i>
+ </a>
+ </div>
+ );
+ }
+
+ componentDidMount() {
+ this.picker = flatpickr(this.refs.flatpickr, {
+ onChange: this.onDateChanged.bind(this),
+ dateFormat: 'd-M-Y',
+ timeFormat: "h:m:d",
+ wrap: true,
+ showClearButton: false,
+ inlineHideInput: true,
+ defaultHour: 0,
+ defaultMinute: 1,
+ enableSeconds: true,
+ defaultSecond: 0,
+ hourIncrement: 1,
+ minuteIncrement: 1,
+ secondIncrement: 5,
+ time_24hr: true,
+ allowInput: true
+ });
+
+ this.eInput = this.refs.eInput;
+
+ this.picker.calendarContainer.classList.add('ag-custom-component-popup');
+ }
+
+ //*********************************************************************************
+ // METHODS REQUIRED BY AG-GRID
+ //*********************************************************************************
+
+ getDate() {
+ //ag-grid will call us here when in need to check what the current date value is hold by this
+ //component.
+ return this.state.date;
+ }
+
+ setDate(date) {
+ //ag-grid will call us here when it needs this component to update the date that it holds.
+ this.setState({ date });
+ this.picker.setDate(date);
+ }
+
+ //*********************************************************************************
+ // AG-GRID OPTIONAL METHODS
+ //*********************************************************************************
+
+ setInputPlaceholder(placeholder) {
+ this.eInput.setAttribute('placeholder', placeholder);
+ }
+
+ setInputAriaLabel(label) {
+ this.eInput.setAttribute('aria-label', label);
+ }
+
+ //*********************************************************************************
+ // LINKS THE INTERNAL STATE AND AG-GRID
+ //*********************************************************************************
+
+ updateAndNotifyAgGrid(date) {
+ this.setState(
+ {
+ date,
+ },
+ //Callback after the state is set. This is where we tell ag-grid that the date has changed so
+ //it will proceed with the filtering and we can then expect ag-Grid to call us back to getDate
+ this.props.onDateChanged
+ );
+ }
+
+ //*********************************************************************************
+ // LINKING THE UI, THE STATE AND AG-GRID
+ //*********************************************************************************
+ onDateChanged = (selectedDates) => {
+ //console.log('>>', selectedDates[0])
+ this.props.context.componentParent.updateTime(
+ this.props.node.rowIndex,this.props.colDef.field,selectedDates[0]
+ );
+
+
+ // this.updateAndNotifyAgGrid(selectedDates[0]);
+ };
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/DegreeInputmask.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/DegreeInputmask.js
index 151b9edb3ccc07af675b1ab0bdae39d96bc3d38e..320f815503edfbd9d8f203daaa97f21c84ab9af0 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/DegreeInputmask.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/DegreeInputmask.js
@@ -2,19 +2,25 @@ import React, { Component } from 'react';
import { InputMask } from 'primereact/inputmask';
import Validator from '../../utils/validator';
+const BG_COLOR= '#f878788f';
+
export default class DegreeInputMask extends Component {
constructor(props) {
super(props);
this.callbackUpdateAngle = this.callbackUpdateAngle.bind(this);
}
+ /**
+ * Update Angle value
+ * @param {*} e
+ */
callbackUpdateAngle(e) {
let isValid = false;
if(Validator.validateAngle(e.value)){
e.originalEvent.target.style.backgroundColor = '';
isValid = true;
}else{
- e.originalEvent.target.style.backgroundColor = 'orange';
+ e.originalEvent.target.style.backgroundColor = BG_COLOR;
}
this.props.context.componentParent.updateAngle(
this.props.node.rowIndex,this.props.colDef.field,e.value,false,isValid
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/MultiSelector.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/MultiSelector.js
new file mode 100644
index 0000000000000000000000000000000000000000..c6df1658da13b941ace35509c758b001731f6f2e
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/MultiSelector.js
@@ -0,0 +1,74 @@
+import React, { Component } from 'react';
+import {MultiSelect} from 'primereact/multiselect';
+import _ from 'lodash';
+
+export default class SkySllector extends Component {
+ constructor(props) {
+ super(props);
+
+ this.dailyOptions= [
+ {name: 'require_day', value: 'require_day'},
+ {name: 'require_night', value: 'require_night'},
+ {name: 'avoid_twilight', value: 'avoid_twilight'},
+ ];
+ this.state= {
+ daily: [],
+
+ }
+
+ this.callbackUpdateDailyCell = this.callbackUpdateDailyCell.bind(this);
+ }
+
+ async componentDidMount(){
+ let selectedValues = this.props.data['daily'];
+ if(selectedValues && selectedValues.length>0){
+ let tmpDailyValue = _.split(selectedValues, ",");
+ await this.setState({
+ daily: tmpDailyValue,
+ });
+ }
+
+ console.log('this.props.props',this.props.data['daily'], this.state.daily)
+
+ // this.props.props.
+ /* console.log('---',this.props.data['daily'])
+ await this.setState({
+ daily: this.props.data['daily']
+ })*/
+ }
+
+ async callbackUpdateDailyCell(e) {
+ let isValid = false;
+ this.setState({
+ daily: e.value
+ })
+ let dailyValue = '';
+ let selectedValues = e.value;
+ await selectedValues.forEach( key =>{
+ dailyValue += key+",";
+ })
+ dailyValue = _.trim(dailyValue)
+ dailyValue = dailyValue.replace(/,([^,]*)$/, '' + '$1')
+
+ this.props.context.componentParent.updateDailyCell(
+ this.props.node.rowIndex,this.props.colDef.field,dailyValue
+ );
+
+ }
+
+ afterGuiAttached(){
+ // this.input.input.focus();
+ }
+ isPopup() {
+ return true;
+ }
+ render() {
+ return (
+ <div className="col-sm-6">
+ <MultiSelect optionLabel="name" value={this.state.daily} options={this.dailyOptions}
+ optionLabel="value" optionValue="value" filter={true}
+ onChange={this.callbackUpdateDailyCell} />
+ </div>
+ );
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/RenderTimeInputmask.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/RenderTimeInputmask.js
deleted file mode 100644
index c11bfe84597cf6d471a1cc8653be5556f5eccf64..0000000000000000000000000000000000000000
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/RenderTimeInputmask.js
+++ /dev/null
@@ -1,60 +0,0 @@
-import React, { Component } from 'react';
-import { InputMask } from 'primereact/inputmask';
-import Validator from '../../utils/validator';
-
-export default class RenderTimeInputmask extends Component{
- constructor(props) {
- super(props);
- this.callbackUpdateAngle = this.callbackUpdateAngle.bind(this);
- }
-
- callbackUpdateAngle(e) {
- let isValid = false;
- if(Validator.validateTime(e.value)){
- e.originalEvent.target.style.backgroundColor = '';
- isValid = true;
- }else{
- e.originalEvent.target.style.backgroundColor = 'orange';
- }
-
- this.props.context.componentParent.updateAngle(
- this.props.node.rowIndex,this.props.colDef.field,e.value,false,isValid
- );
-
- }
-
- /*
- isPopup(){}
- isCancelBeforeStart(){}
-
- focusIn(){}
- focusOut(){}
- destroy(){}
- */
-
- isCancelAfterEnd(){
- // console.log('params', this.props);
-
- // return false;
- }
- afterGuiAttached(){
- //this.input.input.focus();
- }
-
-
- getValue(){
- // console.log(this.input.value)
- }
- render() {
- return (
- <InputMask
- value={this.props.value}
- mask="99:99:99"
- placeholder="HH:mm:ss"
- className="inputmask"
- onComplete={this.callbackUpdateAngle}
- ref={input =>{this.input = input}}
- />
- );
- }
-}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/StationEditor.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/StationEditor.js
new file mode 100644
index 0000000000000000000000000000000000000000..66aa263f2c1e43af512d069dbfa176df8d31fa3b
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/StationEditor.js
@@ -0,0 +1,181 @@
+import React, { Component } from 'react';
+
+import { Dialog } from 'primereact/dialog';
+import { Button } from 'primereact/button';
+import Stations from '../../routes/Scheduling/Stations';
+
+import moment from 'moment';
+import _ from 'lodash';
+
+const DATE_TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+
+export default class StationEditor extends Component {
+ constructor(props) {
+ super(props);
+ this.tmpRowData = [];
+
+ this.state = {
+ schedulingUnit: {},
+ showDialog: false,
+ dialogTitle: 'Station Group',
+ missingStationFieldsErrors: [],
+ stationGroup: [],
+ customSelectedStations: []
+ };
+ this.formRules = {
+ name: {required: true, message: "Name can not be empty"},
+ description: {required: true, message: "Description can not be empty"},
+ };
+ }
+
+ isPopup() {
+ return true;
+ }
+
+ /**
+ * Init the date value if exists
+ */
+ async componentDidMount(){
+ let tmpStationGroups = [];
+ let tmpStationGroup = {};
+
+ let rowSU = this.props.agGridReact.props.rowData[this.props.node.rowIndex];
+ let sgCellValue = rowSU[this.props.colDef.field];
+
+ if(sgCellValue && sgCellValue.length >0){
+ let stationGroups = _.split(sgCellValue, "|");
+ stationGroups.map(stationGroup =>{
+ tmpStationGroup = {};
+ let sgValue = _.split(stationGroup, ":");
+ if(sgValue && sgValue[0].length>0){
+ let stationArray = _.split(sgValue[0], ",");
+
+ tmpStationGroup['stations'] = stationArray;
+ tmpStationGroup['max_nr_missing'] = sgValue[1];
+ tmpStationGroups.push(tmpStationGroup);
+ }
+
+ })
+ this.setState({
+ stationGroup: tmpStationGroups,
+ showDialog: true
+ });
+ }else{
+ let defaultSGs = this.props.context.componentParent.state.defaultStationGroups;
+ if(defaultSGs){
+ this.setState({
+ stationGroup: defaultSGs,
+ selectedStations: defaultSGs,
+ showDialog: true
+ });
+ }
+ }
+ }
+
+validateForm(fieldName) {
+ let validForm = false;
+ let errors = this.state.errors;
+ let validFields = this.state.validFields;
+ if (fieldName) {
+ delete errors[fieldName];
+ delete validFields[fieldName];
+ if (this.formRules[fieldName]) {
+ const rule = this.formRules[fieldName];
+ const fieldValue = this.state.schedulingUnit[fieldName];
+ if (rule.required) {
+ if (!fieldValue) {
+ errors[fieldName] = rule.message?rule.message:`${fieldName} is required`;
+ } else {
+ validFields[fieldName] = true;
+ }
+ }
+ }
+ } else {
+ errors = {};
+ validFields = {};
+ for (const fieldName in this.formRules) {
+ const rule = this.formRules[fieldName];
+ const fieldValue = this.state.schedulingUnit[fieldName];
+ if (rule.required) {
+ if (!fieldValue) {
+ errors[fieldName] = rule.message?rule.message:`${fieldName} is required`;
+ } else {
+ validFields[fieldName] = true;
+ }
+ }
+ }
+ }
+ this.setState({errors: errors, validFields: validFields});
+ if (Object.keys(validFields).length === Object.keys(this.formRules).length) {
+ validForm = true;
+ }
+ return validForm && !this.state.missingStationFieldsErrors;
+}
+
+async updateStationGroup(){
+ let stationValue = '';
+ const station_groups = [];
+ (this.state.selectedStations || []).forEach(key => {
+ let station_group = {};
+ const stations = this.state[key] ? this.state[key].stations : [];
+ const max_nr_missing = parseInt(this.state[key] ? this.state[key].missing_StationFields : 0);
+ station_group = {
+ stations,
+ max_nr_missing
+ };
+ station_groups.push(station_group);
+ });
+ this.state.customSelectedStations.forEach(station => {
+ station_groups.push({
+ stations: station.stations,
+ max_nr_missing: parseInt(station.max_nr_missing)
+ });
+ });
+ if(station_groups){
+ station_groups.map(stationGroup =>{
+ stationValue += stationGroup.stations+':'+stationGroup.max_nr_missing+"|";
+ })
+ }
+ await this.props.context.componentParent.updateDailyCell(
+ this.props.node.rowIndex,this.props.colDef.field, stationValue
+ );
+ this.setState({ showDialog: false});
+
+}
+
+onUpdateStations = (state, selectedStations, missingStationFieldsErrors, customSelectedStations) => {
+ this.setState({
+ ...state,
+ selectedStations,
+ missingStationFieldsErrors,
+ customSelectedStations
+ }, () => {
+ this.setState({
+ validForm: this.validateForm()
+ });
+ });
+};
+
+render() {
+ return (
+ <>
+
+ <Dialog header={_.startCase(this.state.dialogTitle)} visible={this.state.showDialog} maximized={false}
+ onHide={() => {this.updateStationGroup()}} inputId="confirm_dialog"
+ footer={<div>
+ <Button key="back" label="Close" onClick={() => {this.updateStationGroup()}} />
+ </div>
+ } >
+ <div className="ag-theme-balham" style={{ height: '90%', width: '1000px', paddingLeft: '20px' }}>
+ <Stations
+ stationGroup={this.state.stationGroup}
+ onUpdateStations={this.onUpdateStations.bind(this)}
+ />
+ </div>
+ </Dialog>
+
+ </>
+ );
+}
+
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/TimeInputmask.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/TimeInputmask.js
index 685d11cc442e4f9bf1082706aad01a2fdc1c9186..ef773a00181db906bc02b311308c08e8cab813d0 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/TimeInputmask.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/TimeInputmask.js
@@ -2,6 +2,8 @@ import React, { Component } from 'react';
import { InputMask } from 'primereact/inputmask';
import Validator from '../../utils/validator';
+const BG_COLOR= '#f878788f';
+
export default class TimeInputMask extends Component {
constructor(props) {
super(props);
@@ -14,7 +16,7 @@ export default class TimeInputMask extends Component {
e.originalEvent.target.style.backgroundColor = '';
isValid = true;
}else{
- e.originalEvent.target.style.backgroundColor = 'orange';
+ e.originalEvent.target.style.backgroundColor = BG_COLOR;
}
this.props.context.componentParent.updateAngle(
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/numericEditor.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/numericEditor.js
index e759fd958dd2540ec4c2276793ac10443b586500..1662daa58c07829ad70ed7cd7c56416cbfb12124 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/numericEditor.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Spreadsheet/numericEditor.js
@@ -12,9 +12,7 @@ export default class NumericEditor extends Component {
this.cancelBeforeStart =
this.props.charPress && '1234567890'.indexOf(this.props.charPress) < 0;
-
this.state = this.createInitialState(props);
-
this.onKeyDown = this.onKeyDown.bind(this);
this.handleChange = this.handleChange.bind(this);
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
index 180744f4df30abb3edb16851ca9db17313ef356d..8eadc296893366b76c4255e561f964433a734020 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
@@ -802,6 +802,9 @@ export class CalendarTimeline extends Component {
const result = await this.props.dateRangeCallback(startTime, endTime, refreshData);
if (!this.props.showSunTimings && this.state.viewType === UIConstants.timeline.types.NORMAL) {
result.items = await this.addStationSunTimes(startTime, endTime, result.group, result.items);
+ } else if (this.state.viewType === UIConstants.timeline.types.WEEKVIEW) {
+ let group = DEFAULT_GROUP.concat(result.group);
+ result.items = await this.addWeekSunTimes(startTime, endTime, group, result.items);
}
return result;
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js b/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js
index 00f02fc49b31ac94ad2bae9182a4cc2a5291df50..3428a67afc894027d9fb81d49d12ededd102db17 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/ViewTable.js
@@ -1,5 +1,5 @@
import React, {useRef, useState } from "react";
-import { useSortBy, useTable, useFilters, useGlobalFilter, useAsyncDebounce, usePagination } from 'react-table'
+import { useSortBy, useTable, useFilters, useGlobalFilter, useAsyncDebounce, usePagination, useRowSelect } from 'react-table'
import matchSorter from 'match-sorter'
import _ from 'lodash';
import moment from 'moment';
@@ -15,13 +15,15 @@ import { Button } from "react-bootstrap";
import { InputNumber } from "primereact/inputnumber";
let tbldata =[], filteredData = [] ;
+let selectedRows = [];
let isunittest = false;
let showTopTotal = true;
let showGlobalFilter = true;
let showColumnFilter = true;
let allowColumnSelection = true;
+let allowRowSelection = false;
let columnclassname =[];
-let parentCallbackFunction;
+let parentCallbackFunction, parentCBonSelection;
// Define a default UI for filtering
function GlobalFilter({
@@ -406,6 +408,7 @@ const defaultColumn = React.useMemo(
setHiddenColumns,
gotoPage,
setPageSize,
+ selectedFlatRows,
} = useTable(
{
columns,
@@ -419,7 +422,8 @@ const defaultColumn = React.useMemo(
useFilters,
useGlobalFilter,
useSortBy,
- usePagination
+ usePagination,
+ useRowSelect
);
React.useEffect(() => {
setHiddenColumns(
@@ -478,6 +482,15 @@ const defaultColumn = React.useMemo(
if (parentCallbackFunction) {
parentCallbackFunction(filteredData);
}
+
+ /* Select only rows than can be selected. This is required when ALL is selected */
+ selectedRows = _.filter(selectedFlatRows, selectedRow => { return (selectedRow.original.canSelect===undefined || selectedRow.original.canSelect)});
+ /* Take only the original values passed to the component */
+ selectedRows = _.map(selectedRows, 'original');
+ /* Callback the parent function if available to pass the selected records on selection */
+ if (parentCBonSelection) {
+ parentCBonSelection(selectedRows)
+ }
return (
<>
@@ -524,10 +537,12 @@ const defaultColumn = React.useMemo(
setGlobalFilter={setGlobalFilter}
/>
}
- </div>
- { showTopTotal &&
+ </div>
+ { showTopTotal && filteredData.length === data.length &&
<div className="total_records_top_label"> <label >Total records ({data.length})</label></div>
}
+ { showTopTotal && filteredData.length < data.length &&
+ <div className="total_records_top_label" ><label >Filtered {filteredData.length} from {data.length}</label></div>}
</div>
<div className="tmss-table table_container">
@@ -575,7 +590,10 @@ const defaultColumn = React.useMemo(
</table>
</div>
<div className="pagination p-grid" >
- <div className="total_records_bottom_label" ><label >Total records ({data.length})</label></div>
+ {filteredData.length === data.length &&
+ <div className="total_records_bottom_label" ><label >Total records ({data.length})</label></div>}
+ {filteredData.length < data.length &&
+ <div className="total_records_bottom_label" ><label >Filtered {filteredData.length} from {data.length}</label></div>}
<div>
<Paginator rowsPerPageOptions={[10,25,50,100]} first={currentpage} rows={currentrows} totalRecords={rows.length} onPageChange={onPagination}></Paginator>
</div>
@@ -612,12 +630,14 @@ function ViewTable(props) {
// Data to show in table
tbldata = props.data;
parentCallbackFunction = props.filterCallback;
+ parentCBonSelection = props.onRowSelection;
isunittest = props.unittest;
columnclassname = props.columnclassname;
showTopTotal = props.showTopTotal===undefined?true:props.showTopTotal;
showGlobalFilter = props.showGlobalFilter===undefined?true:props.showGlobalFilter;
showColumnFilter = props.showColumnFilter===undefined?true:props.showColumnFilter;
allowColumnSelection = props.allowColumnSelection===undefined?true:props.allowColumnSelection;
+ allowRowSelection = props.allowRowSelection===undefined?false:props.allowRowSelection;
// Default Header to show in table and other columns header will not show until user action on UI
let defaultheader = props.defaultcolumns;
let optionalheader = props.optionalcolumns;
@@ -631,6 +651,33 @@ function ViewTable(props) {
let columns = [];
let defaultdataheader = Object.keys(defaultheader[0]);
let optionaldataheader = Object.keys(optionalheader[0]);
+
+ /* If allowRowSelection property is true for the component, add checkbox column as 1st column.
+ If the record has property to select, enable the checkbox */
+ if (allowRowSelection) {
+ columns.push({
+ Header: ({ getToggleAllRowsSelectedProps }) => { return (
+ <div>
+ <IndeterminateCheckbox {...getToggleAllRowsSelectedProps()} style={{width:'15px', height:'15px'}}/>
+ </div>
+ )},
+ id:'Select',
+ accessor: props.keyaccessor,
+ Cell: ({ row }) => { return (
+ <div>
+ {(row.original.canSelect===undefined || row.original.canSelect) &&
+ <IndeterminateCheckbox {...row.getToggleRowSelectedProps()} style={{width:'15px', height:'15px'}}/>
+ }
+ {row.original.canSelect===false &&
+ <input type="checkbox" checked={false} disabled style={{width:'15px', height:'15px'}}></input>
+ }
+ </div>
+ )},
+ disableFilters: true,
+ disableSortBy: true,
+ isVisible: defaultdataheader.includes(props.keyaccessor),
+ });
+ }
if(props.showaction === 'true') {
columns.push({
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/images/login-bg-1.jpg b/SAS/TMSS/frontend/tmss_webapp/src/images/login-bg-1.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..2707a9905eb1b32168203380be4687f0fa764d8a
Binary files /dev/null and b/SAS/TMSS/frontend/tmss_webapp/src/images/login-bg-1.jpg differ
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/images/login-bg-2.jpg b/SAS/TMSS/frontend/tmss_webapp/src/images/login-bg-2.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..405247dc8b9ec3a4be9f9e19250598e9efd929a3
Binary files /dev/null and b/SAS/TMSS/frontend/tmss_webapp/src/images/login-bg-2.jpg differ
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss
index 4ec1204d72a8ead8c5565e6457231059a9e82108..2a374f424a3c437691b9f1f98916b3fcdd7665c2 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/_overrides.scss
@@ -22,6 +22,12 @@
border-top: none;
}
}
+.ag-root-wrapper{
+ /*
+ calendar is overlaped by AG-grid table, so table props update to show calendar
+ */
+ overflow: inherit;
+}
.tmss-table {
overflow:auto;
// since calendar getting inserted to table, because of above overflow, not getting visible
@@ -137,10 +143,46 @@
.plots{
padding-left: 2px;
}
+.list-stations-summary {
+ max-height: 200px;
+ overflow: auto;
+}
+.block-list {
+ > a {
+ display: block;
+ }
+}
+.se-resizing-bar .se-navigation.sun-editor-common {
+ display: none;
+}
+.se-component.se-image-container {
+ img {
+ width: 100%;
+ }
+}
+.p-button.tooltip-wrapper {
+ margin: 0;
+ padding: 0;
+ border: none;
+ background: none;
+ background-color: transparent;
+ box-shadow: none;
+ font-size: 1rem;
+ text-align: left;
+ color: #000;
+ .p-button-text.p-c{
+ display: none;
+ }
+}
+.p-button.tooltip-wrapper:enabled:hover {
+ background-color: transparent;
+ color: #000;
+ border: none;
+}
-
-
-
-
-
-
+.overlay-panel-header {
+ font-size: 14px;
+ font-weight: 600;
+ color: #004B93;
+ text-align: center;
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppTopbar.js b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppTopbar.js
index e45be034ce26e4db68ac6446a00f3517bc5f9184..f112943d779cdedc9448a0f7ff2f42ce10fab3c2 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppTopbar.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/AppTopbar.js
@@ -4,24 +4,27 @@ import 'primeicons/primeicons.css';
import 'primereact/resources/themes/nova-light/theme.css';
import 'primereact/resources/primereact.css';
import 'primeflex/primeflex.css';
- import { PropTypes } from 'prop-types';
+import { PropTypes } from 'prop-types';
+import Auth from '../../authenticate/auth';
- export class AppTopbar extends Component {
+export class AppTopbar extends Component {
- // constructor(props) {
- // super(props);
- // }
+ constructor(props) {
+ super(props);
+ this.state = {
+ username: Auth.getUser().name
+ };
+ }
static defaultProps = {
onToggleMenu: null
- }
-
-
- static propTypes = {
- onToggleMenu: PropTypes.func.isRequired
- }
+ }
+
+ static propTypes = {
+ onToggleMenu: PropTypes.func.isRequired
+ }
render() {
return (
@@ -31,7 +34,15 @@ import 'primeflex/primeflex.css';
<button className="p-link layout-menu-button" onClick={this.props.onToggleMenu}>
<i className="pi pi-bars"></i></button>
<span className="header-title">TMSS</span>
+ {this.props.isLoggedIn &&
+ <div className="top-right-bar">
+ <span><i className="fa fa-user"></i>{this.state.username}</span>
+ <button className="p-link layout-menu-button" onClick={this.props.onLogout}>
+ <i className="pi pi-power-off"></i></button>
+ </div>
+ }
</div>
+
</div>
</React.Fragment>
)
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/CustomDialog.js b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/CustomDialog.js
new file mode 100644
index 0000000000000000000000000000000000000000..ea013dca232d1dd5a0cc4e1dcda11542f79af1ce
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/CustomDialog.js
@@ -0,0 +1,63 @@
+import React, {Component} from 'react';
+import { Button } from 'primereact/button';
+import { Dialog } from 'primereact/dialog';
+
+/**
+ * Custom Dialog component to get user input before doing something. It can be warning information or confirmation or error message and based on the
+ * user's input will proceed to next step.
+ */
+export class CustomDialog extends Component {
+
+ constructor(props) {
+ super(props);
+ this.state = {
+ visible: props.visible===undefined?true:props.visible
+ }
+ }
+
+ render() {
+ const isConfirm = this.props.type.toLowerCase()==='confirmation';
+ const isWarning = this.props.type.toLowerCase()==='warning';
+ const isSuccess = this.props.type.toLowerCase()==='success';
+ // const isError = this.props.type.toLowerCase()==='error';
+ let iconClass = isConfirm?"pi-question-circle pi-warning":(isWarning?"pi-info-circle pi-warning": (isSuccess?"pi-check-circle pi-success":"pi-times-circle pi-danger"));
+ return (
+ <div className="p-grid" data-testid="confirm_dialog">
+ <Dialog header={this.props.header} visible={this.props.visible} style={{width: this.props.width?this.props.width:'25vw'}}
+ inputId="confirm_dialog"
+ modal={true} onHide={this.props.onClose}
+ footer={<div>
+ {/* Action buttons based on 'type' props. If 'actions' passed as props, then type is ignored */}
+ {!this.props.actions &&
+ <>
+ {isConfirm &&
+ <Button key="back" onClick={this.props.onCancel} label="No" />
+ }
+ <Button key="submit" type="primary" onClick={this.props.onSubmit?this.props.onSubmit:this.props.onClose} label={isConfirm?'Yes':'Ok'} />
+ </>
+ }
+ {/* Action button based on the 'actions' props */}
+ {this.props.actions && this.props.actions.map((action, index) => {
+ return (
+ <Button key={action.id} label={action.title} onClick={action.callback} />);
+ })}
+ </div>
+ } >
+ <div className="p-grid">
+ <div className="col-lg-2 col-md-2 col-sm-2">
+ <span style={{position: 'absolute', top: '50%', '-ms-transform': 'translateY(-50%)', transform: 'translateY(-50%)'}}>
+ <i className={`pi pi-large ${iconClass}`}></i>
+ </span>
+ </div>
+ <div className="col-lg-10 col-md-10 col-sm-10">
+ {/* Display message passed */}
+ {this.props.message?this.props.message:""}
+ {/* Render subcomponent passed as function */}
+ {this.props.content?this.props.content():""}
+ </div>
+ </div>
+ </Dialog>
+ </div>
+ );
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/PageHeader.js b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/PageHeader.js
index fb95ec75a094fc8a2d86bdf74ba78fab8c885a39..02de326d2c7ab3829d12003304e28da3f77fa090 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/components/PageHeader.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/components/PageHeader.js
@@ -42,7 +42,7 @@ export default ({ title, subTitle, actions, ...props}) => {
{(actions || []).map((action, index) =>{
if (action.type === 'button') {
return (
- <button className="p-link" key={index}>
+ <button className="p-link" key={index} title={action.title || ''}>
<i className={`fa ${action.icon}`}
onMouseOver={(e) => onButtonMouseOver(e, action)}
onClick={(e) => onButtonClick(e, action)} />
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss
index 302208b3dad39e6c4ebf092e614e502fbd131e37..1f4526c7df85ed761cf12441b914192264ac4fa3 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_layout.scss
@@ -16,4 +16,5 @@
@import "./timeline";
@import "./_aggrid";
@import "./suSummary";
+@import "./login";
// @import "./splitpane";
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_login.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_login.scss
new file mode 100644
index 0000000000000000000000000000000000000000..d1df063e62e02a4846902f89d81c5a9faf1de83b
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_login.scss
@@ -0,0 +1,90 @@
+.login-page {
+ background-image: url(../../images/login-bg-1.jpg);
+ height: 100%;
+}
+
+.bg-login{
+ background-image: url(../../images/login-bg-1.jpg);
+ min-height: 100vh;
+ background-size: 100%;
+ padding: 20px;
+}
+.login-card{
+ background-color: #fff;
+ float: none;
+ margin: auto;
+ box-shadow: 0 1px 15px rgba(0,0,0,.04), 0 1px 6px rgba(0,0,0,.04);
+ margin-top: 8%;
+ margin-bottom: 5%;
+}
+.detail-part{
+ background-image: url(../../images/login-bg-2.jpg);
+ padding: 30px;
+ background-size: cover;
+ background-repeat: no-repeat;
+}
+.detail-part h3{
+ color: #fff;
+ padding-top: 10%;
+ font-size: 2.5rem;
+}
+.detail-part p{
+ color: #fff;
+ margin-top: 30px;
+ padding-left: 70%;
+}
+.logn-part{
+ padding: 5%;
+}
+.logo-cover img{
+ margin-bottom: 30px;
+}
+.form-cover h6{
+ margin-bottom: 30px;
+}
+.form-cover input{
+ margin-bottom: 30px;
+ border-radius: 0px;
+ background-color: #cccccc38;
+}
+.form-footer .forget-paswd{
+ text-align: left;
+}
+.button-div{
+ text-align: right;
+}
+.form-footer{
+ margin-bottom: 50px;
+}
+.button-div .btn{
+ background-color: #922c88 !important;
+ border-color: #922c88 !important;
+}
+.button-div .btn:hover{
+ background-color: #922c88 !important;
+ border-color: #922c88 !important;
+}
+.button-div .btn:active{
+ background-color: #922c88 !important;
+ border-color: #922c88 !important;
+}
+.button-div .btn:focus{
+ background-color: #922c88 !important;
+ border-color: #922c88 !important;
+}
+.login-form .error {
+ text-transform: capitalize;
+}
+.login-form .form-field {
+ padding-top: 20px;
+ padding-bottom: 0px;
+}
+.login-form .form-field i {
+ padding-right: 5px;
+}
+
+@media screen and (max-width: 1100px){
+ .bg-login{
+ background-image: url(../../images/login-bg-1.jpg);
+ }
+}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_stations.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_stations.scss
index e0c2e01575b7d261bb353d311a22730592c8af06..18a08aae503368a4e75fdeea786a99eccaa0d8eb 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_stations.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_stations.scss
@@ -23,8 +23,13 @@
top: 2px;
span {
font-size: 14px !important;
- padding: 0 !important;
+ padding-right: 20px !important;
+ top: 10px;
}
+ padding-left: 5px;
+ padding-top: 3px;
+ cursor: pointer;
+ font-size: 14px;
}
.text-caps {
text-transform: capitalize;
@@ -33,9 +38,10 @@
padding: 10px;
max-height: 200px;
overflow-y: auto;
- label {
+ span {
display: block;
}
+ background-color: #d1cdd936;
}
.custom-label {
padding-left: 8px !important;
@@ -71,7 +77,7 @@
}
.custom-remove {
position: absolute;
- left: -12px;
+ left: 5px;
background-color: transparent !important;
border: none !important;
padding: 0;
@@ -82,3 +88,9 @@
padding: 0 !important;
}
}
+/**
+* Class to set margin-left for (i) and remove button in station.js
+*/
+// .icon-left{
+// margin-left: 20%
+// }
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss
index 76b02736fb1959096ef3067f75b67f2ad1a72336..ab73c0560d155915f052c300f41c36280658e4ac 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss
@@ -45,4 +45,11 @@
.json-to-table::-webkit-scrollbar-thumb
{
background-color: #0000007c;
+ }
+
+ .station-list {
+ max-height: 150px;
+ overflow-y: scroll;
+ border: 1px solid lightgrey;
+ padding: 0px 10px 10px 10px;
}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_topbar.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_topbar.scss
index e687ef7dae3dec800531b48174fbc34b7790b0dc..678f58f4053d806555173ff111674d31bd7cf47e 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_topbar.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_topbar.scss
@@ -130,4 +130,17 @@
button {
cursor: pointer;
}
+}
+
+.top-right-bar {
+ float: right;
+ padding-top: 5px
+}
+
+.top-right-bar span>i {
+ padding-right: 5px;
+}
+
+.top-right-bar button {
+ padding-left: 5px;
}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
index 1bcbcefcbd4a7cd6a5bbb2c2cf96eb7c4b7c2800..fb4232f12ba8e52cbbb9851ef37f428012dba601 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
@@ -117,20 +117,31 @@ export default (props) => {
list.push('disable-field');
}
ref.editors['root.time.at'].container.className = list.join(' ');
- Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('input')).forEach(input => input.disabled = true);
- Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('button')).forEach(button => button.disabled = true);
+ if (ref.editors['root.time.at'].control) {
+ Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('input')).forEach(input => input.disabled = true);
+ Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('button')).forEach(button => button.disabled = true);
+ }
} else {
ref.editors['root.time.at'].container.className = ref.editors['root.time.at'].container.className.replace('disable-field', '');
- Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('input')).forEach(input => input.disabled = false);
- Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('button')).forEach(button => button.disabled = false);
+ if (ref.editors['root.time.at'].control) {
+ Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('input')).forEach(input => input.disabled = false);
+ Array.prototype.slice.call(ref.editors['root.time.at'].control.getElementsByTagName('button')).forEach(button => button.disabled = false);
+ }
}
if (props.callback) {
+ // Remove 'time' fields if it is empty
+ for (const key of _.keys(jsonOutput.time)) {
+ if (!jsonOutput.time[key]) {
+ delete jsonOutput.time[key];
+ }
+ }
props.callback(jsonOutput, errors);
}
}
const constraintStrategy = () => {
- const constraintTemplate = { ...props.constraintTemplate }
+ // const constraintTemplate = { ...props.constraintTemplate }
+ const constraintTemplate = _.cloneDeep(props.constraintTemplate);
if (constraintTemplate.schema) {
configureProperties(constraintTemplate.schema.properties);
configureDefinitions(constraintTemplate.schema);
@@ -183,13 +194,15 @@ export default (props) => {
return (
<>
{constraintSchema && React.createElement(Jeditor, {
+ id: "constraint_editor",
title: "Scheduling Constraints specification",
schema: constraintSchema.schema,
callback: onEditForm,
initValue: initialValue,
disabled: props.disable,
formatOutput: props.formatOutput,
- parentFunction: parentFunction
+ parentFunction: parentFunction,
+ defintionFormatter: configureDefinitions
})}
</>
);
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js
index 570ca6388bd7f10954ccb6fe2731e0b424f92435..f64b1133eb4b63b6787e5913e8e4a2f75f48452f 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/SchedulingUnitList.js
@@ -56,6 +56,8 @@ class SchedulingUnitList extends Component{
}],
defaultSortColumn: [{id: "Name", desc: false}],
}
+ this.onRowSelection = this.onRowSelection.bind(this);
+ this.reloadData = this.reloadData.bind(this);
}
async getSchedulingUnitList () {
@@ -86,6 +88,7 @@ class SchedulingUnitList extends Component{
blueP['created_at'] = moment(blueP['created_at'], moment.ISO_8601).format("YYYY-MMM-DD HH:mm:ss");
blueP['updated_at'] = moment(blueP['updated_at'], moment.ISO_8601).format("YYYY-MMM-DD HH:mm:ss");
blueP.project = project.name;
+ blueP.canSelect = false;
return blueP;
});
output.push(...blueprintdata);
@@ -95,11 +98,13 @@ class SchedulingUnitList extends Component{
scheduleunit['created_at'] = moment(scheduleunit['created_at'], moment.ISO_8601).format("YYYY-MMM-DD HH:mm:ss");
scheduleunit['updated_at'] = moment(scheduleunit['updated_at'], moment.ISO_8601).format("YYYY-MMM-DD HH:mm:ss");
scheduleunit.project = project.name;
+ scheduleunit.canSelect = true;
output.push(scheduleunit);
}
this.setState({
scheduleunit: output, isLoading: false
});
+ this.selectedRows = [];
})
}
}
@@ -109,6 +114,22 @@ class SchedulingUnitList extends Component{
}
+ /**
+ * Callback function passed to ViewTable component to pass back the selected rows.
+ * @param {Array} selectedRows - Subset of data passed to the ViewTable component based on selection.
+ */
+ onRowSelection(selectedRows) {
+ this.selectedRows = selectedRows;
+ }
+
+ /**
+ * Funtion to reload data. This function can be called from the implementing component.
+ */
+ reloadData() {
+ this.setState({isLoading: true});
+ this.getSchedulingUnitList();
+ }
+
render(){
if (this.state.isLoading) {
return <AppLoader/>
@@ -139,6 +160,8 @@ class SchedulingUnitList extends Component{
paths={this.state.paths}
unittest={this.state.unittest}
tablename="scheduleunit_list"
+ allowRowSelection={this.props.allowRowSelection}
+ onRowSelection = {this.onRowSelection}
/>
:<div>No scheduling unit found </div>
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Stations.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Stations.js
index 7a7a354a7fd5872a6ee6256abba1b92749a22538..2cc9aab744b87021b6d2d5ffda604c1859ce1fc3 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Stations.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Stations.js
@@ -1,5 +1,3 @@
-
-
import React, { useState, useEffect } from 'react';
import _ from 'lodash';
import {MultiSelect} from 'primereact/multiselect';
@@ -225,9 +223,10 @@ export default (props) => {
const updateSchedulingComp = (param_State, param_SelectedStations, param_missing_StationFieldsErrors, param_Custom_selected_options) => {
const isError = param_missing_StationFieldsErrors.length || param_Custom_selected_options.filter(i => i.error).length;
- debugger
+ // debugger
props.onUpdateStations(param_State, param_SelectedStations, isError, param_Custom_selected_options);
};
+
/**
* Method to remove the custom stations
* @param {*} index number
@@ -239,98 +238,123 @@ export default (props) => {
updateSchedulingComp(state, selectedStations, missing_StationFieldsErrors, custom_selected_options);
};
+ const isPopup =() =>{
+ return true;
+ }
return (
- <div className="p-field p-grid grouping p-fluid">
+ <div className={`p-field p-grid grouping p-fluid ${props.isSummary && 'p-col-12'}`}>
<fieldset>
<legend>
<label>Stations<span style={{color:'red'}}>*</span></label>
</legend>
- {!props.view && <div className="col-sm-12 p-field p-grid" data-testid="stations">
- <div className="col-md-6 d-flex">
- <label htmlFor="stationgroup" className="col-sm-6 station_header">Station Groups</label>
- <div className="col-sm-6">
- <MultiSelect data-testid="stations" id="stations" optionLabel="value" optionValue="value" filter={true}
- tooltip="Select Stations" tooltipOptions={tooltipOptions}
- value={selectedStations}
- options={stationOptions}
- placeholder="Select Stations"
- onChange={(e) => setSelectedStationGroup(e.value)}
- />
+ {!props.isSummary && <>
+ {!props.view && <div className="col-sm-12 p-field p-grid" data-testid="stations">
+ <div className="col-md-6 d-flex">
+ <label htmlFor="stationgroup" className="col-sm-6 station_header">Station Groups</label>
+ <div className="col-sm-6">
+ <MultiSelect data-testid="stations" id="stations" optionLabel="value" optionValue="value" filter={true}
+ tooltip="Select Stations" tooltipOptions={tooltipOptions}
+ value={selectedStations}
+ options={stationOptions}
+ placeholder="Select Stations"
+ onChange={(e) => setSelectedStationGroup(e.value)}
+ />
+ </div>
</div>
- </div>
- <div className="add-custom">
- <Button onClick={addCustom} label="Add Custom" icon="pi pi-plus" disabled={!stationOptions.length}/>
- </div>
- </div>}
- {selectedStations.length ? <div className="col-sm-12 selected_stations" data-testid="selected_stations">
- {<div className="col-sm-12"><label style={{paddingLeft: '8px'}}>Maximum number of stations that can be missed in the selected groups</label></div>}
- <div className="col-sm-12 p-0 d-flex flex-wrap">
- {selectedStations.map(i => (
- <div className="p-field p-grid col-md-6" key={i}>
- <label className="col-sm-6 text-caps">
- {i}
- <Button icon="pi pi-info-circle" className="p-button-rounded p-button-secondary p-button-text info" onClick={(e) => showStations(e, i)} />
- </label>
- <div className="col-sm-6">
- <InputText id="missingstation" data-testid="name"
- className={(state[i] && state[i].error) ?'input-error':''}
- tooltip="Max No. of Missing Stations" tooltipOptions={tooltipOptions} maxLength="128"
- placeholder="Max No. of Missing Stations"
- value={state[i] ? state[i].missing_StationFields : ''}
- disabled={props.view}
- onChange={(e) => setNoOfmissing_StationFields(i, e.target.value)}/>
- {(state[i] && state[i].error) && <span className="error-message">{state[i].missing_StationFields ? `Max. no of missing stations is ${state[i] ? state[i].stations.length : 0}` : 'Max. no of missing stations required'}</span>}
- </div>
- </div>
- ))}
- {customStations.map((stat, index) => (
- <div className="p-field p-grid col-md-12 custom-station-wrapper" key={index}>
- {!props.view && <Button icon="pi pi-trash" className="p-button-secondary p-button-text custom-remove" onClick={() => removeCustomStations(index)} />}
-
- <div className="col-md-6 p-field p-grid">
- <label className="col-sm-6 text-caps custom-label">
- Custom {index + 1}
+ <div className="add-custom">
+ <Button onClick={addCustom} label="Add Custom" icon="pi pi-plus" disabled={!stationOptions.length}/>
+ </div>
+ </div>}
+ {selectedStations.length ? <div className="col-sm-12 selected_stations" data-testid="selected_stations">
+ {<div className="col-sm-12"><label style={{paddingLeft: '8px'}}>Maximum number of stations that can be missed in the selected groups</label></div>}
+ <div className="col-sm-12 p-0 d-flex flex-wrap">
+ {selectedStations.map(i => (
+ <div className="p-field p-grid col-md-6" key={i}>
+ <label className="col-sm-6 text-caps">
+ {i}
+ <i className="pi pi-info-circle info label-icon" onClick={(e) => showStations(e, i)} />
</label>
- <div className="col-sm-6 pr-8 custom-value">
- <MultiSelect data-testid="custom_stations" id="custom_stations" filter
- tooltip="Select Stations" tooltipOptions={tooltipOptions}
- value={stat.stations}
- options={customStationsOptions}
- placeholder="Select Stations"
+ <div className="col-sm-6">
+ <InputText id="missingstation" data-testid="name"
+ className={(state[i] && state[i].error) ?'input-error':''}
+ tooltip="Max No. of Missing Stations" tooltipOptions={tooltipOptions} maxLength="128"
+ placeholder="Max No. of Missing Stations"
+ value={state[i] ? state[i].missing_StationFields : ''}
disabled={props.view}
- optionLabel="value"
- optionValue="value"
- onChange={(e) => onChangeCustomSelectedStations(e.value, index)}
- />
+ onChange={(e) => setNoOfmissing_StationFields(i, e.target.value)}/>
+ {(state[i] && state[i].error) && <span className="error-message">{state[i].missing_StationFields ? `Max. no of missing stations is ${state[i] ? state[i].stations.length : 0}` : 'Max. no of missing stations required'}</span>}
</div>
</div>
- <div className="col-md-6 p-field p-grid">
- <label className="col-sm-6 customMissingStationLabel">
- Maximum No. Of Missing Stations
- </label>
- <div className="col-sm-6 pr-8 custom-field">
- <InputText id="missingStation" data-testid="name"
- className={(stat.error && stat.touched) ?'input-error':''}
- tooltip="Max Number of Missing Stations" tooltipOptions={tooltipOptions}
- placeholder="Max Number of Missing Stations"
- value={stat.max_nr_missing}
- disabled={props.view}
- onChange={(e) => setMissingFieldsForCustom(e.target.value, index)}/>
- {(stat.error && stat.touched) && <span className="error-message">{stat.max_nr_missing ? `Max. no of missing stations is ${stat.stations.length}` : 'Max. no of missing stations required'}</span>}
- {/* {props.view &&
- <span className="info">Max No. of Missing Stations</span>} */}
-
- </div>
+ ))}
+ {customStations.map((stat, index) => (
+ <div className="p-field p-grid col-md-12 custom-station-wrapper" key={index}>
+ {!props.view && <Button icon="pi pi-trash" className="p-button-secondary p-button-text custom-remove" onClick={() => removeCustomStations(index)} />}
+
+ <div className="col-md-6 p-field p-grid">
+ <label className="col-sm-6 text-caps custom-label">
+ Custom {index + 1}
+ </label>
+ <div className="col-sm-6 pr-8 custom-value">
+ <MultiSelect data-testid="custom_stations" id="custom_stations" filter
+ tooltip="Select Stations" tooltipOptions={tooltipOptions}
+ value={stat.stations}
+ options={customStationsOptions}
+ placeholder="Select Stations"
+ disabled={props.view}
+ optionLabel="value"
+ optionValue="value"
+ onChange={(e) => onChangeCustomSelectedStations(e.value, index)}
+ />
+ </div>
+ </div>
+ <div className="col-md-6 p-field p-grid">
+ <label className="col-sm-6 customMissingStationLabel">
+ Maximum No. Of Missing Stations
+ </label>
+ <div className="col-sm-6 pr-8 custom-field">
+ <InputText id="missingStation" data-testid="name"
+ className={(stat.error && stat.touched) ?'input-error':''}
+ tooltip="Max Number of Missing Stations" tooltipOptions={tooltipOptions}
+ placeholder="Max Number of Missing Stations"
+ value={stat.max_nr_missing}
+ disabled={props.view}
+ onChange={(e) => setMissingFieldsForCustom(e.target.value, index)}/>
+ {(stat.error && stat.touched) && <span className="error-message">{stat.max_nr_missing ? `Max. no of missing stations is ${stat.stations.length}` : 'Max. no of missing stations required'}</span>}
+ {/* {props.view &&
+ <span className="info">Max No. of Missing Stations</span>} */}
+
+ </div>
+ </div>
</div>
- </div>
- ))}
+ ))}
+ </div>
+
+ </div> : null}
+ </>}
+ {/* For timeline view, displaying all stations in list */}
+ {props.isSummary && (
+ <div className="list-stations-summary">
+ {state && Object.keys(state).map(key => {
+ if (key !== 'Custom') {
+ return (
+ <>
+ {state[key].stations.map((station, index) => <div key={index}>{station}</div>)}
+ </>
+ )
+ }
+ })}
+ {customStations.map((stat, index) => (
+ <>
+ {stat.stations.map(station => <div key={index}>{station}</div>)}
+ </>
+ ))}
</div>
-
- </div> : null}
- <OverlayPanel ref={(el) => op = el} dismissable style={{width: '450px'}}>
+ )}
+ <OverlayPanel ref={(el) => op = el} dismissable style={{width: '200px'}}>
+ <h6 className="overlay-panel-header">Stations in group</h6>
<div className="station-container">
{(stations || []).map(i => (
- <label>{i}</label>
+ <span>{i}</span>
))}
</div>
</OverlayPanel>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js
index a76501c5c154cadef3782037eb877ff3db06c865..5a25a62050f2652843210ebe61992ffb126ac661 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/ViewSchedulingUnit.js
@@ -14,6 +14,10 @@ import SchedulingConstraint from './Scheduling.Constraints';
import { Dialog } from 'primereact/dialog';
import TaskStatusLogs from '../Task/state_logs';
import Stations from './Stations';
+import { Redirect } from 'react-router-dom';
+import { CustomDialog } from '../../layout/components/CustomDialog';
+import { CustomPageSpinner } from '../../components/CustomPageSpinner';
+import { Growl } from 'primereact/components/growl/Growl';
class ViewSchedulingUnit extends Component{
constructor(props){
@@ -34,6 +38,7 @@ class ViewSchedulingUnit extends Component{
filter:"select"
},
id: "ID",
+ subTaskID: 'Control ID',
name:"Name",
description:"Description",
created_at:{
@@ -65,6 +70,7 @@ class ViewSchedulingUnit extends Component{
"Status Logs": "filter-input-0",
"Type":"filter-input-75",
"ID":"filter-input-50",
+ "Control ID":"filter-input-75",
"Cancelled":"filter-input-50",
"Duration (HH:mm:ss)":"filter-input-75",
"Template ID":"filter-input-50",
@@ -73,42 +79,47 @@ class ViewSchedulingUnit extends Component{
"Relative End Time (HH:mm:ss)": "filter-input-75",
"Status":"filter-input-100"
}],
- stationGroup: []
+ stationGroup: [],
+ dialog: {header: 'Confirm', detail: 'Do you want to create a Scheduling Unit Blueprint?'},
+ dialogVisible: false
}
- this.actions = [
- {icon: 'fa-window-close',title:'Click to Close Scheduling Unit View', link: this.props.history.goBack}
- ];
+ this.actions = [];
this.stations = [];
this.constraintTemplates = [];
- if (this.props.match.params.type === 'draft') {
- this.actions.unshift({icon: 'fa-edit', title: 'Click to edit', props : { pathname:`/schedulingunit/edit/${ this.props.match.params.id}`}
- });
- } else {
- this.actions.unshift({icon: 'fa-sitemap',title :'View Workflow',props :{pathname:`/schedulingunit/${this.props.match.params.id}/workflow`}});
- this.actions.unshift({icon: 'fa-lock', title: 'Cannot edit blueprint'});
- }
- if (this.props.match.params.id) {
- this.state.scheduleunitId = this.props.match.params.id;
- }
- if (this.props.match.params.type) {
- this.state.scheduleunitType = this.props.match.params.type;
- }
+ this.checkAndCreateBlueprint = this.checkAndCreateBlueprint.bind(this);
+ this.createBlueprintTree = this.createBlueprintTree.bind(this);
+ this.closeDialog = this.closeDialog.bind(this);
+
+ }
+
+ componentDidUpdate(prevProps, prevState) {
+ if (this.state.scheduleunit && this.props.match.params &&
+ (this.state.scheduleunitId !== this.props.match.params.id ||
+ this.state.scheduleunitType !== this.props.match.params.type)) {
+ this.getSchedulingUnitDetails(this.props.match.params.type, this.props.match.params.id);
}
+ }
async componentDidMount(){
- let schedule_id = this.state.scheduleunitId;
- let schedule_type = this.state.scheduleunitType;
+ let schedule_id = this.props.match.params.id;
+ let schedule_type = this.props.match.params.type;
if (schedule_type && schedule_id) {
- const subtaskComponent = (task)=> {
- return (
- <button className="p-link" onClick={(e) => {this.setState({showStatusLogs: true, task: task})}}>
- <i className="fa fa-history"></i>
- </button>
- );
- };
this.stations = await ScheduleService.getStationGroup();
this.setState({stationOptions: this.stations});
- this.getScheduleUnit(schedule_type, schedule_id)
+ this.getSchedulingUnitDetails(schedule_type, schedule_id);
+ }
+ }
+
+ subtaskComponent = (task)=> {
+ return (
+ <button className="p-link" onClick={(e) => {this.setState({showStatusLogs: true, task: task})}}>
+ <i className="fa fa-history"></i>
+ </button>
+ );
+ };
+
+ getSchedulingUnitDetails(schedule_type, schedule_id) {
+ this.getScheduleUnit(schedule_type, schedule_id)
.then(schedulingUnit =>{
if (schedulingUnit) {
ScheduleService.getSchedulingConstraintTemplates().then((response) => {
@@ -118,15 +129,22 @@ class ViewSchedulingUnit extends Component{
this.getScheduleUnitTasks(schedule_type, schedulingUnit)
.then(tasks =>{
tasks.map(task => {
- task.status_logs = task.tasktype === "Blueprint"?subtaskComponent(task):"";
+ task.status_logs = task.tasktype === "Blueprint"?this.subtaskComponent(task):"";
+ //Displaying SubTask ID of the 'control' Task
+ const subTaskIds = task.subTasks?task.subTasks.filter(sTask => sTask.subTaskTemplate.name.indexOf('control') > 1):[];
+ task.subTaskID = subTaskIds.length ? subTaskIds[0].id : '';
return task;
});
const targetObservation = _.find(tasks, (task)=> {return task.template.type_value==='observation' && task.tasktype.toLowerCase()===schedule_type && task.specifications_doc.station_groups});
this.setState({
+ scheduleunitId: schedule_id,
scheduleunit : schedulingUnit,
+ scheduleunitType: schedule_type,
schedule_unit_task : tasks,
isLoading: false,
- stationGroup: targetObservation?targetObservation.specifications_doc.station_groups:[]
+ stationGroup: targetObservation?targetObservation.specifications_doc.station_groups:[],
+ redirect: null,
+ dialogVisible: false
}, this.getAllStations);
});
} else {
@@ -135,41 +153,76 @@ class ViewSchedulingUnit extends Component{
});
}
});
- }
+ this.actions = [
+ {icon: 'fa-window-close',title:'Click to Close Scheduling Unit View', link: this.props.history.goBack}
+ ];
+ if (this.props.match.params.type === 'draft') {
+ this.actions.unshift({icon: 'fa-edit', title: 'Click to edit', props : { pathname:`/schedulingunit/edit/${ this.props.match.params.id}`}
+ });
+ this.actions.unshift({icon:'fa-stamp', title: 'Create Blueprint', type:'button',
+ actOn:'click', props : { callback: this.checkAndCreateBlueprint},
+ });
+ } else {
+ this.actions.unshift({icon: 'fa-sitemap',title :'View Workflow',props :{pathname:`/schedulingunit/${this.props.match.params.id}/workflow`}});
+ this.actions.unshift({icon: 'fa-lock', title: 'Cannot edit blueprint'});
+ }
}
getScheduleUnitTasks(type, scheduleunit){
if(type === 'draft')
- return ScheduleService.getTasksBySchedulingUnit(scheduleunit.id, true);
+ return ScheduleService.getTasksBySchedulingUnit(scheduleunit.id, true, true, true);
else
- return ScheduleService.getTaskBlueprintsBySchedulingUnit(scheduleunit, true);
+ return ScheduleService.getTaskBPWithSubtaskTemplateOfSU(scheduleunit);
}
+
getScheduleUnit(type, id){
if(type === 'draft')
return ScheduleService.getSchedulingUnitDraftById(id)
else
return ScheduleService.getSchedulingUnitBlueprintById(id)
}
+
+ /**
+ * Checks if the draft scheduling unit has existing blueprints and alerts. If confirms to create, creates blueprint.
+ */
+ checkAndCreateBlueprint() {
+ if (this.state.scheduleunit) {
+ let dialog = this.state.dialog;
+ if (this.state.scheduleunit.scheduling_unit_blueprints.length>0) {
+ dialog.detail = "Blueprint(s) already exist for this Scheduling Unit. Do you want to create another one?";
+ }
+ dialog.actions = [{id: 'yes', title: 'Yes', callback: this.createBlueprintTree},
+ {id: 'no', title: 'No', callback: this.closeDialog}];
+ this.setState({dialogVisible: true, dialog: dialog});
+ }
+ }
+
+ /**
+ * Funtion called to create blueprint on confirmation.
+ */
+ createBlueprintTree() {
+ this.setState({dialogVisible: false, showSpinner: true});
+ ScheduleService.createSchedulingUnitBlueprintTree(this.state.scheduleunit.id)
+ .then(blueprint => {
+ this.growl.show({severity: 'success', summary: 'Success', detail: 'Blueprint created successfully!'});
+ this.setState({showSpinner: false, redirect: `/schedulingunit/view/blueprint/${blueprint.id}`, isLoading: true});
+ });
+ }
+
+ /**
+ * Callback function to close the dialog prompted.
+ */
+ closeDialog() {
+ this.setState({dialogVisible: false});
+ }
- render(){
+ render(){
+ if (this.state.redirect) {
+ return <Redirect to={ {pathname: this.state.redirect} }></Redirect>
+ }
return(
<>
- {/*} <div className="p-grid">
- <div className="p-col-10">
- <h2>Scheduling Unit - Details </h2>
- </div>
- <div className="p-col-2">
- <Link to={{ pathname: '/schedulingunit'}} title="Close"
- style={{float:'right'}}>
- <i className="fa fa-times" style={{marginTop: "10px", marginLeft: '5px'}}></i>
- </Link>
- <Link to={{ pathname: '/schedulingunit/edit', state: {id: this.state.scheduleunit?this.state.scheduleunit.id:''}}} title="Edit"
- style={{float:'right'}}>
- <i className="fa fa-edit" style={{marginTop: "10px"}}></i>
- </Link>
- </div>
- </div> */
- /*TMSS-363 Blueprint icon changes */}
+ <Growl ref={(el) => this.growl = el} />
<PageHeader location={this.props.location} title={'Scheduling Unit - Details'}
actions={this.actions}/>
{ this.state.isLoading ? <AppLoader/> :this.state.scheduleunit &&
@@ -256,6 +309,12 @@ class ViewSchedulingUnit extends Component{
<TaskStatusLogs taskId={this.state.task.id}></TaskStatusLogs>
</Dialog>
}
+ {/* Dialog component to show messages and get confirmation */}
+ <CustomDialog type="confirmation" visible={this.state.dialogVisible}
+ header={this.state.dialog.header} message={this.state.dialog.detail} actions={this.state.dialog.actions}
+ onClose={this.closeDialog} onCancel={this.closeDialog} onSubmit={this.createBlueprintTree}></CustomDialog>
+ {/* Show spinner during backend API call */}
+ <CustomPageSpinner visible={this.state.showSpinner} />
</>
)
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
index 4dba59e7cc582d9ff47663d3130e2dd9efdea3ab..02547c1bc763eb13b960a1f5582a5b6b18b69073 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
@@ -200,7 +200,7 @@ export class SchedulingUnitCreate extends Component {
setConstraintsEditorOutput(jsonOutput, errors) {
let err = [ ...errors ];
- if (jsonOutput.scheduler === 'online') {
+ if (jsonOutput.scheduler === 'online' || jsonOutput.scheduler === 'dynamic') {
err = err.filter(e => e.path !== 'root.time.at');
}
this.constraintParamsOutput = jsonOutput;
@@ -299,7 +299,7 @@ export class SchedulingUnitCreate extends Component {
async saveSchedulingUnit() {
const constStrategy = _.cloneDeep(this.state.constraintParamsOutput);
for (let type in constStrategy.time) {
- if (constStrategy.scheduler === 'online') {
+ if (constStrategy.scheduler === 'online' || constStrategy.scheduler === 'dynamic') {
delete constStrategy.time.at;
}
if (!constStrategy.time.after) {
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.scheduleset.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.scheduleset.js
index 2e330e0fbc70c7708dd629f2a17e1229e8507b18..227e677f8a39dec732b95bd1fb45010d42b7fef2 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.scheduleset.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.scheduleset.js
@@ -1,28 +1,45 @@
import React, {Component} from 'react';
-import { Link, Redirect } from 'react-router-dom';
-import _ from 'lodash';
+import { Redirect } from 'react-router-dom';
import {Dropdown} from 'primereact/dropdown';
import { Button } from 'primereact/button';
import {Dialog} from 'primereact/components/dialog/Dialog';
import {Growl} from 'primereact/components/growl/Growl';
+import { AgGridReact } from 'ag-grid-react';
+import { AllCommunityModules } from '@ag-grid-community/all-modules';
+import $RefParser from "@apidevtools/json-schema-ref-parser";
+
+import TimeInputmask from './../../components/Spreadsheet/TimeInputmask'
+import DegreeInputmask from './../../components/Spreadsheet/DegreeInputmask'
+import NumericEditor from '../../components/Spreadsheet/numericEditor';
+import BetweenEditor from '../../components/Spreadsheet/BetweenEditor';
+import BetweenRenderer from '../../components/Spreadsheet/BetweenRenderer';
+import MultiSelector from '../../components/Spreadsheet/MultiSelector';
import AppLoader from '../../layout/components/AppLoader';
+
+import PageHeader from '../../layout/components/PageHeader';
+import { CustomDialog } from '../../layout/components/CustomDialog';
import ProjectService from '../../services/project.service';
import ScheduleService from '../../services/schedule.service';
import TaskService from '../../services/task.service';
+import CustomDateComp from '../../components/Spreadsheet/CustomDateComp';
+
+import Validator from '../../utils/validator';
+import UnitConverter from '../../utils/unit.converter'
import UIConstants from '../../utils/ui.constants';
-import $RefParser from "@apidevtools/json-schema-ref-parser";
-import TimeInputmask from './../../components/Spreadsheet/TimeInputmask'
-import DegreeInputmask from './../../components/Spreadsheet/DegreeInputmask'
-import NumericEditor from '../../components/Spreadsheet/numericEditor';
+import UnitConversion from '../../utils/unit.converter';
+import StationEditor from '../../components/Spreadsheet/StationEditor';
+
+
+import moment from 'moment';
+import _ from 'lodash';
-import { AgGridReact } from 'ag-grid-react';
-import { AllCommunityModules } from '@ag-grid-community/all-modules';
import 'ag-grid-community/dist/styles/ag-grid.css';
import 'ag-grid-community/dist/styles/ag-theme-alpine.css';
-import UnitConverter from '../../utils/unit.converter'
-import Validator from '../../utils/validator';
-import PageHeader from '../../layout/components/PageHeader';
+
+const DATE_TIME_FORMAT = 'YYYY-MM-DD HH:mm:ss';
+const BG_COLOR= '#f878788f';
+
/**
* Component to create / update Scheduling Unit Drafts using Spreadsheet
*/
@@ -33,10 +50,14 @@ export class SchedulingSetCreate extends Component {
this.gridColumnApi = ''
this.rowData = [];
this.tmpRowData = [];
-
+ this.defaultCellValues = [];
+ this.daily = [];
+
this.state = {
+ dailyOption: [],
projectDisabled: (props.match?(props.match.params.project? true:false):false),
- isLoading: true, // Flag for loading spinner
+ isLoading: true,
+ isAGLoading: false, // Flag for loading spinner
dialog: { header: '', detail: ''}, // Dialog properties
redirect: null, // URL to redirect
errors: [], // Form Validation errors
@@ -61,6 +82,11 @@ export class SchedulingSetCreate extends Component {
numericEditor: NumericEditor,
timeInputMask: TimeInputmask,
degreeInputMask: DegreeInputmask,
+ betweenRenderer: BetweenRenderer,
+ betweenEditor: BetweenEditor,
+ multiselector: MultiSelector,
+ agDateInput: CustomDateComp,
+ station: StationEditor,
},
columnTypes: {
numberValueColumn: {
@@ -79,6 +105,8 @@ export class SchedulingSetCreate extends Component {
rowIdRenderer: function (params) {
return 1 + params.rowIndex;
},
+ validCount: 0,
+ inValidCount: 0,
},
noOfSUOptions: [
{ label: '10', value: '10' },
@@ -87,6 +115,10 @@ export class SchedulingSetCreate extends Component {
{ label: '250', value: '250' },
{ label: '500', value: '500' }
],
+ customSelectedStations: [],
+ selectedStations: [],
+ defaultStationGroups: [],
+ saveDialogVisible: false,
}
this.onGridReady = this.onGridReady.bind(this);
@@ -96,7 +128,11 @@ export class SchedulingSetCreate extends Component {
this.cancelCreate = this.cancelCreate.bind(this);
this.clipboardEvent = this.clipboardEvent.bind(this);
this.reset = this.reset.bind(this);
-
+ this.close = this.close.bind(this);
+ this.saveSU = this.saveSU.bind(this);
+ this.validateGridAndSave = this.validateGridAndSave.bind(this);
+ this.showDialogContent = this.showDialogContent.bind(this);
+
this.projects = []; // All projects to load project dropdown
this.schedulingSets = []; // All scheduling sets to be filtered for project
this.observStrategies = []; // All Observing strategy templates
@@ -108,7 +144,7 @@ export class SchedulingSetCreate extends Component {
scheduling_set_id: {required: true, message: "Select the Scheduling Set"},
};
}
-
+
componentDidMount() {
const promises = [ ProjectService.getProjectList(),
ScheduleService.getSchedulingSets(),
@@ -136,7 +172,7 @@ export class SchedulingSetCreate extends Component {
const projectSchedluingSets = _.filter(this.schedulingSets, {'project_id': projectName});
let schedulingUnit = this.state.schedulingUnit;
schedulingUnit.project = projectName;
- this.setState({schedulingUnit: schedulingUnit, schedulingSets: projectSchedluingSets, validForm: this.validateForm('project')});
+ this.setState({schedulingUnit: schedulingUnit, schedulingSets: projectSchedluingSets, validForm: this.validateForm('project'), rowData: [],observStrategy: {}});
}
/**
@@ -144,57 +180,80 @@ export class SchedulingSetCreate extends Component {
* @param {string} key
* @param {object} value
*/
- async setSchedingSetParams(key, value) {
+ async setSchedulingSetParams(key, value) {
+ this.setState({isAGLoading: true});
+
let schedulingUnit = this.state.schedulingUnit;
schedulingUnit[key] = value;
let schedulingUnitList = await ScheduleService.getSchedulingBySet(value);
- if(schedulingUnitList){
+ if (schedulingUnitList) {
const schedulingSetIds = _.uniq(_.map(schedulingUnitList, 'observation_strategy_template_id'));
- if(schedulingSetIds.length === 1){
+ if (schedulingSetIds.length === 1) {
const observStrategy = _.find(this.observStrategies, {'id': schedulingUnitList[0].observation_strategy_template_id});
+ this.setDefaultStationGroup(observStrategy);
this.setState({
schedulingUnit: schedulingUnit, validForm: this.validateForm(key), validEditor: this.validateEditor(),
- schedulingUnitList: schedulingUnitList, schedulingSetId: value, selectedSchedulingSetId: value, observStrategy: observStrategy
+ schedulingUnitList: schedulingUnitList, schedulingSetId: value, selectedSchedulingSetId: value, observStrategy: observStrategy,
});
await this.prepareScheduleUnitListForGrid();
- }else{
+ } else {
/* Let user to select Observation Strategy */
this.setState({
rowData:[], schedulingUnit: schedulingUnit, validForm: this.validateForm(key), validEditor: this.validateEditor(),
schedulingUnitList:schedulingUnitList, selectedSchedulingSetId: value, observStrategy: {}
});
}
- }else{
+ } else {
this.setState({schedulingUnit: schedulingUnit, validForm: this.validateForm(key), validEditor: this.validateEditor(),
selectedSchedulingSetId: value});
}
+ this.setState({isAGLoading: false});
}
+ async setDefaultStationGroup(observStrategy) {
+ let station_group = [];
+ const tasks = observStrategy.template.tasks;
+ for (const taskName of _.keys(tasks)) {
+ const task = tasks[taskName];
+ //Resolve task from the strategy template
+ const $taskRefs = await $RefParser.resolve(task);
+ // Identify the task specification template of every task in the strategy template
+ const taskTemplate = _.find(this.taskTemplates, {'name': task['specifications_template']});
+ if (taskTemplate.type_value==='observation' && task.specifications_doc.station_groups) {
+ station_group = task.specifications_doc.station_groups;
+ }
+ }
+ await this.setState({
+ defaultStationGroups: station_group,
+ })
+ }
/**
* Function called when observation strategy template is changed.
*
* @param {number} strategyId
*/
async changeStrategy (strategyId) {
+ this.setState({isAGLoading: true});
const observStrategy = _.find(this.observStrategies, {'id': strategyId});
let schedulingUnitList= await ScheduleService.getSchedulingBySet(this.state.selectedSchedulingSetId);
schedulingUnitList = _.filter(schedulingUnitList,{'observation_strategy_template_id': strategyId}) ;
-
+ this.setDefaultStationGroup(observStrategy);
await this.setState({
schedulingUnitList: schedulingUnitList,
- observStrategy: observStrategy
+ observStrategy: observStrategy,
})
- if(schedulingUnitList && schedulingUnitList.length >0){
+ if (schedulingUnitList && schedulingUnitList.length >0){
await this.prepareScheduleUnitListForGrid();
- }else{
+ } else {
this.setState({
rowData: []
})
}
- this.state.gridApi.setRowData(this.state.rowData)
- this.state.gridApi.redrawRows();
+ // this.state.gridApi.setRowData(this.state.rowData)
+ //this.state.gridApi.redrawRows();
+ this.setState({isAGLoading: false});
}
/**
@@ -212,10 +271,10 @@ export class SchedulingSetCreate extends Component {
if (refUrl.endsWith("/pointing")) { // For type pointing
schema.definitions["pointing"] = (await $RefParser.resolve(refUrl)).get(newRef);
property["$ref"] = newRef;
- } else { // General object to resolve if any reference in child level
+ } else { // General object to resolve if any reference in child level
property = await this.resolveSchema((await $RefParser.resolve(refUrl)).get(newRef));
}
- } else if(property["type"] === "array") { // reference in array items definition
+ } else if (property["type"] === "array") { // reference in array items definition
let resolvedItems = await this.resolveSchema(property["items"]);
schema.definitions = {...schema.definitions, ...resolvedItems.definitions};
delete resolvedItems['definitions'];
@@ -243,6 +302,10 @@ export class SchedulingSetCreate extends Component {
return schema;
}
+ async getConstraintSchema(scheduleUnit){
+ let constraintSchema = await ScheduleService.getSchedulingConstraintTemplate(scheduleUnit.scheduling_constraints_template_id);
+ return constraintSchema;
+ }
/**
* Function to generate AG-Grid column definition.
@@ -251,24 +314,57 @@ export class SchedulingSetCreate extends Component {
async createGridColumns(scheduleUnit){
let schema = await this.getTaskSchema(scheduleUnit);
schema = await this.resolveSchema(schema);
+ let constraintSchema = await this.getConstraintSchema(scheduleUnit);
+ constraintSchema = await this.resolveSchema(constraintSchema);
+
// AG Grid Cell Specific Properties
- const cellProps =[];
- cellProps['angle1'] = {type:'numberValueColumn', cellRenderer: 'timeInputMask',cellEditor: 'timeInputMask', valueSetter: 'valueSetter' };
- cellProps['angle2'] = {type:'numberValueColumn', cellRenderer: 'degreeInputMask',cellEditor: 'degreeInputMask', valueSetter: 'valueSetter' };
- cellProps['angle3'] = {cellEditor: 'numericEditor',};
- cellProps['direction_type'] = {cellEditor: 'agSelectCellEditor',default: schema.definitions.pointing.properties.direction_type.default,
- cellEditorParams: {
- values: schema.definitions.pointing.properties.direction_type.enum,
- },
- };
- //Ag-grid Colums definition
+ let dailyOption= [];
+ let dailyProps = Object.keys( constraintSchema.schema.properties.daily.properties);
+ this.daily = [];
+ dailyProps.forEach(prop => {
+ dailyOption.push({'Name':prop, 'Code':prop});
+ this.daily.push(prop);
+ })
+
+ this.setState({
+ dailyOption: this.dailyOption,
+ schedulingConstraintsDoc: scheduleUnit.scheduling_constraints_doc,
+ constraintUrl: scheduleUnit.scheduling_constraints_template,
+ constraintId: scheduleUnit.scheduling_constraints_template_id,
+ daily: this.daily,
+ });
+ let cellProps =[];
+ cellProps['angle1'] = {isgroup: true, type:'numberValueColumn', cellRenderer: 'timeInputMask',cellEditor: 'timeInputMask', valueSetter: 'valueSetter', };
+ cellProps['angle2'] = {isgroup: true, type:'numberValueColumn', cellRenderer: 'degreeInputMask',cellEditor: 'degreeInputMask', valueSetter: 'valueSetter' };
+ cellProps['angle3'] = {isgroup: true, cellEditor: 'numericEditor', cellStyle: function(params) { if (params.value){
+ if (!Number(params.value)) {
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < 0|| Number(params.value) > 90) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ }}};
+ cellProps['direction_type'] = {isgroup: true, cellEditor: 'agSelectCellEditor',default: schema.definitions.pointing.properties.direction_type.default,
+ cellEditorParams: {
+ values: schema.definitions.pointing.properties.direction_type.enum,
+ },
+ };
+
+ //Ag-grid Colums definition
+ // Column order to use clipboard copy
let colKeyOrder = [];
+
+ colKeyOrder.push("suname");
+ colKeyOrder.push("sudesc");
+
let columnMap = [];
let colProperty = {};
let columnDefs = [
{ // Row Index
- headerName: '',
+ headerName: '#',
editable: false,
maxWidth: 60,
cellRenderer: 'rowIdRenderer',
@@ -280,16 +376,155 @@ export class SchedulingSetCreate extends Component {
headerName: 'Scheduling Unit',
children: [
{headerName: 'Name',field: 'suname'},
- {headerName: 'Description',field: 'sudesc'}
+ {headerName: 'Description',field: 'sudesc', cellStyle: function(params) {
+ if (params.data.suname && params.data.suname !== '' && params.value === '') {
+ return { backgroundColor: BG_COLOR};
+ } else { return { backgroundColor: ''};}
+ },
+ }
],
- }
+ },
+
+ { headerName: 'Scheduler',field: 'scheduler',cellEditor: 'agSelectCellEditor',default: constraintSchema.schema.properties.scheduler.default,
+ cellEditorParams: {
+ values: constraintSchema.schema.properties.scheduler.enum,
+ },
+ },
+ { headerName: 'Time',
+ children: [
+ { headerName: 'At', field:'timeat', editable: true, cellRenderer: 'betweenRenderer',cellEditor: 'agDateInput', valueSetter: 'newValueSetter'},
+ { headerName: 'After', field:'timeafter', editable: true, cellRenderer: 'betweenRenderer',cellEditor: 'agDateInput', valueSetter: 'newValueSetter'},
+ { headerName: 'Before', field:'timebefore', editable: true, cellRenderer: 'betweenRenderer',cellEditor: 'agDateInput', valueSetter: 'newValueSetter'},
+ ],
+ },
+
+ {headerName: 'Between',field: 'between',cellRenderer: 'betweenRenderer',cellEditor: 'betweenEditor',valueSetter: 'newValueSetter', },
+ {headerName: 'Not Between',field: 'notbetween',cellRenderer: 'betweenRenderer',cellEditor: 'betweenEditor',valueSetter: 'newValueSetter'},
+ {headerName: 'Daily',field: 'daily',cellEditor: 'multiselector', valueSetter: 'valueSetter'},
+ {
+ headerName: 'Sky',
+ children: [
+ {headerName: 'Min Target Elevation',field: 'min_target_elevation', cellStyle: function(params) {
+ if (params.value){
+ if ( !Number(params.value)){
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < 0|| Number(params.value) > 90) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ }
+ }, },
+ {headerName: 'Min Calibrator Elevation',field: 'min_calibrator_elevation', cellStyle: function(params) {
+ if (params.value){
+ if ( !Number(params.value)){
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < 0|| Number(params.value) > 90) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ }
+ }, },
+ {headerName: 'Offset Window From',field: 'offset_from', cellStyle: function(params) {
+ if (params.value){
+ if (params.value === 'undefined' || params.value === ''){
+ return { backgroundColor: ''};
+ }
+ if ( !Number(params.value)){
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < -0.20943951 || Number(params.value) > 0.20943951) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ } else {
+ return { backgroundColor: ''};
+ }
+ }, },
+ {headerName: 'Offset Window To',field: 'offset_to', cellStyle: function(params) {
+ if (params.value){
+ if (params.value === 'undefined' || params.value === ''){
+ return { backgroundColor: ''};
+ }
+ if ( !Number(params.value)){
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < -0.20943951 || Number(params.value) > 0.20943951) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ } else {
+ return { backgroundColor: ''};
+ }
+ }, },
+ ],
+ },
+ {
+ headerName: 'Min_distance',
+ children: [
+ {headerName: 'Sun',field: 'md_sun', cellStyle: function(params) {
+ if (params.value){
+ if ( !Number(params.value)){
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < 0 || Number(params.value) > 180) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ }
+
+ },},
+ {headerName: 'Moon',field: 'md_moon', cellStyle: function(params) {
+ if (params.value){
+ if ( !Number(params.value)){
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < 0 || Number(params.value) > 180) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ }
+ }, },
+ {headerName: 'Jupiter',field: 'md_jupiter', cellStyle: function(params) {
+ if (params.value){
+ if ( !Number(params.value)){
+ return { backgroundColor: BG_COLOR};
+ }
+ else if ( Number(params.value) < 0 || Number(params.value) > 180) {
+ return { backgroundColor: BG_COLOR};
+ } else{
+ return { backgroundColor: ''};
+ }
+ }
+ }, },
+ ],
+ },
];
- colKeyOrder.push("suname");
- colKeyOrder.push("sudesc");
+ colKeyOrder.push('scheduler');
+ colKeyOrder.push('timeat');
+ colKeyOrder.push('timeafter');
+ colKeyOrder.push('timebefore');
+ colKeyOrder.push('between');
+ colKeyOrder.push('notbetween');
+ colKeyOrder.push('daily');
+ colKeyOrder.push('min_target_elevation');
+ colKeyOrder.push('min_calibrator_elevation');
+ colKeyOrder.push('offset_from');
+ colKeyOrder.push('offset_to');
+ colKeyOrder.push('md_sun');
+ colKeyOrder.push('md_moon');
+ colKeyOrder.push('md_jupiter');
colProperty ={'ID':'id', 'Name':'suname', 'Description':'sudesc'};
columnMap['Scheduling Unit'] = colProperty;
-
+
let definitions = schema.definitions.pointing.properties;
let properties = schema.properties;
const propsKeys = Object.keys(properties);
@@ -319,17 +554,16 @@ export class SchedulingSetCreate extends Component {
})
columnMap[property.title] = colProperty;
}
- colProperty ={'From':'bfrom', 'Until':'buntil'};
- columnMap['Between'] = colProperty;
+ columnDefs.push({headerName: 'Stations', field: 'stations', cellRenderer: 'betweenRenderer', cellEditor: 'station', valueSetter: 'newValueSetter'});
+ colKeyOrder.push('stations');
this.setState({
columnDefs:columnDefs,
columnMap:columnMap,
colKeyOrder:colKeyOrder
})
-
}
-
- async getTaskSchema(scheduleUnit){
+
+ async getTaskSchema(scheduleUnit) {
let strategyId = scheduleUnit.observation_strategy_template_id;
let tasksToUpdate = {};
const observStrategy = _.find(this.observStrategies, {'id': strategyId});
@@ -391,18 +625,83 @@ export class SchedulingSetCreate extends Component {
}
return schema;
}
+ /**
+ * CallBack Function : update time value in master grid
+ */
+ async updateTime(rowIndex, field, value) {
+ let row = this.state.rowData[rowIndex];
+ row[field] = value;
+ let tmpRowData =this.state.rowData;
+ tmpRowData[rowIndex]= row;
+ await this.setState({
+ rowData: tmpRowData
+ });
+ this.state.gridApi.setRowData(this.state.rowData);
+ this.state.gridApi.redrawRows();
+ }
+ /**
+ * Update the Daily column value from external component
+ * @param {*} rowIndex
+ * @param {*} field
+ * @param {*} value
+ */
+ async updateDailyCell(rowIndex, field, value) {
+ let row = this.state.rowData[rowIndex];
+ row[field] = value;
+ let tmpRowData =this.state.rowData;
+ tmpRowData[rowIndex]= row;
+ await this.setState({
+ rowData: tmpRowData
+ });
+ }
+
+ async getStationGrops(schedulingUnit){
+ let stationValue = '';
+ if (schedulingUnit && schedulingUnit.id>0) {
+ const promises = await [
+ ScheduleService.getObservationStrategies(),
+ TaskService.getTaskTemplates(),
+ ScheduleService.getSchedulingUnitDraftById(schedulingUnit.id),
+ ScheduleService.getTasksDraftBySchedulingUnitId(schedulingUnit.id),
+ ScheduleService.getStationGroup()
+ ];
+ await Promise.all(promises).then(responses => {
+ this.observStrategies = responses[0];
+ this.taskTemplates = responses[1];
+ let schedulingUnit = responses[2];
+ let taskDrafts = responses[3];
+ this.stations = responses[4];
+ let stationGroups = [];
+ if (schedulingUnit && schedulingUnit.observation_strategy_template_id) {
+ let targetObservation = schedulingUnit.requirements_doc.tasks['Target Observation'];
+ if (targetObservation && targetObservation.specifications_doc.station_groups){
+ stationGroups = targetObservation?targetObservation.specifications_doc.station_groups:[];
+ } else {
+ targetObservation = taskDrafts.data.results.find(task => {return task.specifications_doc.station_groups?true:false});
+ stationGroups = targetObservation?targetObservation.specifications_doc.station_groups:[];
+ }
+ }
+
+ if (stationGroups) {
+ stationGroups.map(stationGroup =>{
+ stationValue += stationGroup.stations+':'+stationGroup.max_nr_missing+"|";
+ })
+ }
+ });
+ }
+ return stationValue;
+ }
/**
* Function to prepare ag-grid row data.
*/
async prepareScheduleUnitListForGrid(){
- if(this.state.schedulingUnitList.length===0){
+ if (this.state.schedulingUnitList.length===0) {
return;
}
this.tmpRowData = [];
let totalSU = this.state.noOfSU;
- let paramsOutput = {};
//refresh column header
await this.createGridColumns(this.state.schedulingUnitList[0]);
let observationPropsList = [];
@@ -417,21 +716,53 @@ export class SchedulingSetCreate extends Component {
let parameters = scheduleunit['requirements_doc'].parameters;
for(const parameter of parameters){
- let rurl = parameter['refs'];
- let valueItem = (await $RefParser.resolve( scheduleunit['requirements_doc'])).get(rurl[0]);
+ let refUrl = parameter['refs'];
+ let valueItem = (await $RefParser.resolve( scheduleunit['requirements_doc'])).get(refUrl[0]);
let excelColumns = this.state.columnMap[parameter.name];
let excelColumnsKeys = Object.keys(excelColumns);
for(const eColKey of excelColumnsKeys){
- if(eColKey === 'angle1'){
+ if (eColKey === 'angle1') {
observationProps[excelColumns[eColKey]] = UnitConverter.getAngleInput(valueItem[eColKey], false);
}
- else if(eColKey === 'angle2'){
+ else if (eColKey === 'angle2') {
observationProps[excelColumns[eColKey]] = UnitConverter.getAngleInput(valueItem[eColKey], true);
}
- else{
+ else {
observationProps[excelColumns[eColKey]] = valueItem[eColKey];
}
+ }
+ }
+ observationProps['stations'] = await this.getStationGrops(scheduleunit);
+ let constraint = scheduleunit.scheduling_constraints_doc;
+ if (constraint){
+ if (constraint.scheduler){
+ observationProps['scheduler'] = constraint.scheduler;
+ }
+ observationProps['timeat'] = moment.utc(constraint.time.at).format(DATE_TIME_FORMAT);
+ observationProps['timeafter'] = moment.utc(constraint.time.after).format(DATE_TIME_FORMAT);
+ observationProps['timebefore'] = moment.utc(constraint.time.before).format(DATE_TIME_FORMAT);
+ if (constraint.time.between){
+ observationProps['between'] = this.getBetweenStringValue(constraint.time.between);
+ }
+ if (constraint.time.between){
+ observationProps['notbetween'] = this.getBetweenStringValue(constraint.time.not_between);
+ }
+
+ observationProps['daily'] = this.fetchDailyFieldValue(constraint.daily);
+ UnitConversion.radiansToDegree(constraint.sky);
+ observationProps['min_target_elevation'] = constraint.sky.min_target_elevation;
+ observationProps['min_calibrator_elevation'] = constraint.sky.min_calibrator_elevation;
+ if ( constraint.sky.transit_offset ){
+ observationProps['offset_from'] = (constraint.sky.transit_offset.from)?constraint.sky.transit_offset.from:'';
+ observationProps['offset_to'] = (constraint.sky.transit_offset.to)?constraint.sky.transit_offset.to:'';
+ }
+
+ if (constraint.sky.min_distance){
+ observationProps['md_sun'] = (constraint.sky.min_distance.sun)?constraint.sky.min_distance.sun:'';
+ observationProps['md_moon'] = (constraint.sky.min_distance.moon)?constraint.sky.min_distance.moon:'';
+ observationProps['md_jupiter'] = (constraint.sky.min_distance.jupiter)?constraint.sky.min_distance.jupiter:'';
}
+
}
observationPropsList.push(observationProps);
}
@@ -440,18 +771,18 @@ export class SchedulingSetCreate extends Component {
// find No. of rows filled in array
let totalCount = this.tmpRowData.length;
// Prepare No. Of SU for rows for UI
- if(this.tmpRowData && this.tmpRowData.length>0){
+ if (this.tmpRowData && this.tmpRowData.length>0){
const paramsOutputKey = Object.keys( this.tmpRowData[0]);
const availableCount = this.tmpRowData.length;
- if(availableCount>= totalSU){
- totalSU = availableCount+10;
+ if (availableCount >= totalSU){
+ totalSU = availableCount+5;
}
for(var i = availableCount; i<totalSU; i++){
let emptyRow = {};
paramsOutputKey.forEach(key =>{
- if(key === 'id'){
+ if (key === 'id'){
emptyRow[key]= 0;
- }else{
+ } else {
emptyRow[key]= '';
}
})
@@ -466,6 +797,24 @@ export class SchedulingSetCreate extends Component {
});
}
+ /**
+ * Get Daily column value
+ * @param {*} daily
+ */
+ fetchDailyFieldValue(daily){
+ let returnValue = [];
+ if (daily.require_day === true){
+ returnValue.push('require_day');
+ }
+ if (daily.require_night === true){
+ returnValue.push('require_night');
+ }
+ if (daily.avoid_twilight === true){
+ returnValue.push('avoid_twilight');
+ }
+ return returnValue;
+ }
+
/**
* Function called back from Degree/Time Input Mask to set value in row data.
*
@@ -482,7 +831,6 @@ export class SchedulingSetCreate extends Component {
await this.setState({
rowData: tmpRowData
});
-
}
/**
@@ -491,7 +839,7 @@ export class SchedulingSetCreate extends Component {
async readClipBoard(){
try{
const queryOpts = { name: 'clipboard-read', allowWithoutGesture: true };
- const permissionStatus = await navigator.permissions.query(queryOpts);
+ await navigator.permissions.query(queryOpts);
let data = await navigator.clipboard.readText();
return data;
}catch(err){
@@ -499,28 +847,32 @@ export class SchedulingSetCreate extends Component {
}
}
- /**
- * Check the content is JSON format
- * @param {*} jsonData
- */
- async isJsonData(jsonData){
- try{
- let jsonObj = JSON.parse(jsonData);
- return true;
- }catch(err){
- console.log("error :",err)
- return false;
+ /*
+ // to resolve the invalid degree and time
+ resolveCellData(data){
+ console.log('data >',data)
+ let angleData = _.split(data, ":");
+ let returnValue ='';
+ if (angleData.length === 3){
+ returnValue = (angleData[0].length === 2)?angleData[0] :'0'+angleData[0]+":";
+ returnValue += (angleData[1].length === 2)?angleData[1] :'0'+angleData[1]+":";
+ returnValue += (angleData[2].length === 2)?angleData[2] :'0'+angleData[2];
+
}
- }
+ console.log('returnValue',returnValue)
+ return returnValue;
+ }
+ */
/**
* Copy data to/from clipboard
* @param {*} e
*/
async clipboardEvent(e){
- var key = e.which || e.keyCode; // keyCode detection
- var ctrl = e.ctrlKey ? e.ctrlKey : ((key === 17) ? true : false); // ctrl detection
- if ( key == 86 && ctrl ) {
+ //let angleCellKey = ['tp1angle1','tp1angle2','tp2angle1','tp2angle2','tpangle1','tpangle2'];
+ var key = e.which || e.keyCode;
+ var ctrl = e.ctrlKey ? e.ctrlKey : ((key === 17) ? true : false);
+ if ( key === 86 && ctrl ) {
// Ctrl+V
this.tmpRowData = this.state.rowData;
let dataRowCount = this.state.totalCount;
@@ -532,7 +884,8 @@ export class SchedulingSetCreate extends Component {
}catch(err){
console.log("error :",err);
}
- if(clipboardData){
+ if (clipboardData){
+ clipboardData = _.trim(clipboardData);
let suGridRowData= this.state.emptyRow;
clipboardData = _.trim(clipboardData);
let suRows = clipboardData.split("\n");
@@ -543,7 +896,11 @@ export class SchedulingSetCreate extends Component {
suGridRowData['id']= 0;
suGridRowData['isValid']= true;
for(const key of this.state.colKeyOrder){
- suGridRowData[key]= suRow[colCount];
+ /* if (_.includes(angleCellKey, key)){
+ suGridRowData[key]= this.resolveCellData(suRow[colCount]);
+ } else {*/
+ suGridRowData[key]= suRow[colCount];
+ // }
colCount++;
}
this.tmpRowData[dataRowCount]= (suGridRowData);
@@ -552,8 +909,8 @@ export class SchedulingSetCreate extends Component {
}
let emptyRow = this.state.emptyRow;
let tmpNoOfSU= this.state.noOfSU;
- if(dataRowCount >= tmpNoOfSU){
- tmpNoOfSU = dataRowCount+10;
+ if (dataRowCount >= tmpNoOfSU){
+ tmpNoOfSU = dataRowCount+5;
//Create additional empty row at the end
for(let i= this.tmpRowData.length; i<=tmpNoOfSU; i++){
this.tmpRowData.push(emptyRow);
@@ -573,7 +930,7 @@ export class SchedulingSetCreate extends Component {
console.error('Error: ', err);
}
- } else if ( key == 67 && ctrl ) {
+ } else if ( key === 67 && ctrl ) {
//Ctrl+C
var selectedRows = this.state.gridApi.getSelectedRows();
let clipboardData = '';
@@ -586,28 +943,168 @@ export class SchedulingSetCreate extends Component {
clipboardData += line + '\r\n';
}
clipboardData = _.trim(clipboardData);
+
const queryOpts = { name: 'clipboard-write', allowWithoutGesture: true };
await navigator.permissions.query(queryOpts);
await navigator.clipboard.writeText(clipboardData);
+ } else if ( key === 46){
+ // Delete selected rows
+ let tmpRowData = this.state.rowData;
+
+ var selectedRows = this.state.gridApi.getSelectedNodes();
+ if (selectedRows){
+ await selectedRows.map(delRow =>{
+ delete tmpRowData[delRow.rowIndex]
+ });
+ await this.setState({
+ rowData: tmpRowData
+ });
+ this.state.gridApi.setRowData(this.state.rowData);
+ this.state.gridApi.redrawRows();
+ }
}
}
+
+ /**
+ * Validate Grid values on click Save button from UI
+ */
+ async validateGridAndSave(){
+ let validCount = 0;
+ let inValidCount = 0;
+ let isValidRow = true;
+ let errorDisplay = [];
+ const mandatoryKeys = ['suname','sudesc','scheduler','min_target_elevation','min_calibrator_elevation','offset_from','offset_to','md_sun','md_moon','md_jupiter','tp1angle1','tp1angle2','tp1angle3','tp1direction_type','tp2angle1','tp2angle2','tp2angle3','tp2direction_type','tbangle1','tbangle2','tbangle3','tbdirection_type'];
+ let tmpMandatoryKeys = [];
+ let tmpRowData = this.state.rowData;
+ this.state.gridApi.forEachNode(function (node) {
+ isValidRow = true;
+ let errorMsg = 'Row Id ['+(Number(node.rowIndex)+1) +'] : ';
+ tmpMandatoryKeys = [];
+ const rowData = node.data;
+ let isManualScheduler = false;
+ if (rowData) {
+ for(const key of mandatoryKeys) {
+ if (rowData[key] === '') {
+ tmpMandatoryKeys.push(key);
+ } else if (key === 'scheduler' && rowData[key] === 'manual' ) {
+ isManualScheduler = true;
+ }
+ }
+ if (tmpMandatoryKeys.length !== mandatoryKeys.length) {
+ let rowNoColumn = {};
+ isValidRow = true;
+ for (var i = 0; i< node.columnController.gridColumns.length; i++) {
+ let column = node.columnController.gridColumns[i];
+ if (column.colId === '0'){
+ rowNoColumn = column;
+ } else {
+ if (_.includes(tmpMandatoryKeys, column.colId)){
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ //column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ //rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ } else {
+ if (column.colId === 'timeat' && isManualScheduler && rowData[column.colId] === ''){
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ // column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ // rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ } else if (column.colId === 'min_target_elevation' || column.colId === 'min_calibrator_elevation' || _.endsWith(column.colId, "angle3")){
+ if (Number(rowData[column.colId]) < 0 || Number(rowData[column.colId]) > 90){
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ // column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ // rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ }
+ } else if (column.colId === 'offset_from' || column.colId === 'offset_to'){
+ if ( !Number(rowData[column.colId])){
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ // column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ // rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ } else if ( Number(rowData[column.colId]) < -0.20943951 || Number(rowData[column.colId]) > 0.20943951) {
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ //column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ // rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ }
+ } else if (column.colId === 'md_sun' || column.colId === 'md_moon' || column.colId === 'md_jupiter'){
+ if (Number(rowData[column.colId]) < 0 || Number(rowData[column.colId]) > 180){
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ // column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ // rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ }
+ } else if (_.endsWith(column.colId, "angle1") && !Validator.validateTime(rowData[column.colId])){
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ //column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ // rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ } else if (_.endsWith(column.colId, "angle2") && !Validator.validateAngle(rowData[column.colId])){
+ isValidRow = false;
+ errorMsg += column.colDef.headerName+", ";
+ //column.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ //rowNoColumn.colDef.cellStyle = { backgroundColor: BG_COLOR};
+ }
+ }
+ }
+ }
+ }
+ }
+ if (isValidRow) {
+ validCount++;
+ tmpRowData[node.rowIndex]['isValid'] = true;
+ } else {
+ inValidCount++;
+ tmpRowData[node.rowIndex]['isValid'] = false;
+ errorDisplay.push(errorMsg.slice(0, -2));
+ }
+ });
+
+ if (validCount > 0 && inValidCount === 0) {
+ // save SU directly
+ this. saveSU();
+ } else if (validCount === 0 && inValidCount === 0) {
+ // leave with no change
+ } else {
+ this.setState({
+ validCount: validCount,
+ inValidCount: inValidCount,
+ tmpRowData: tmpRowData,
+ saveDialogVisible: true,
+ errorDisplay: errorDisplay,
+ });
+ this.state.gridApi.redrawRows();
+ }
+ }
/**
* Function to create Scheduling unit
*/
- async saveSchedulingUnit() {
+ async saveSchedulingUnit(){
+ this.validateGridAndSave();
+ }
+
+
+ /**
+ * Save/Update Scheduling Unit
+ */
+ async saveSU() {
let newSUCount = 0;
let existingSUCount = 0;
try{
+ this.setState({
+ saveDialogVisible: false
+ })
let observStrategy = _.cloneDeep(this.state.observStrategy);
const $refs = await $RefParser.resolve(observStrategy.template);
let newSU = this.state.schedulingUnit;
let parameters = this.state.schedulingUnitList[0]['requirements_doc'].parameters;
let columnMap = this.state.columnMap;
-
+
for(const suRow of this.state.rowData){
- if(!suRow['isValid']){
+ if (!suRow['isValid']){
continue;
}
let validRow = true;
@@ -618,59 +1115,199 @@ export class SchedulingSetCreate extends Component {
let result = columnMap[parameter.name];
let resultKeys = Object.keys(result);
resultKeys.forEach(key =>{
- if(key === 'angle1'){
- if(!Validator.validateTime(suRow[result[key]])){
+ if (key === 'angle1') {
+ if (!Validator.validateTime(suRow[result[key]])) {
validRow = false;
return;
}
paramOutput[key] = UnitConverter.getAngleOutput(suRow[result[key]],false);
- }else if(key === 'angle2'){
- if(!Validator.validateAngle(suRow[result[key]])){
+ } else if (key === 'angle2'){
+ if (!Validator.validateAngle(suRow[result[key]])){
validRow = false;
return;
}
paramOutput[key] = UnitConverter.getAngleOutput(suRow[result[key]],true);
- }else{
+ } else {
paramOutput[key] = suRow[result[key]];
}
})
paramsOutput['param_'+index] = paramOutput;
index++;
}
- if(!validRow){
+ if (!validRow){
continue;
}
observStrategy.template.parameters.forEach(async(param, index) => {
$refs.set(observStrategy.template.parameters[index]['refs'][0], paramsOutput['param_' + index]);
});
- if(suRow.id >0 && suRow.suname.length>0 && suRow.sudesc.length>0){
+
+ //Stations
+ let sgCellValue = suRow.stations;
+ let tmpStationGroups = [];
+ if (sgCellValue && sgCellValue.length >0){
+ tmpStationGroups = [];
+ let tmpStationGroup = {};
+ let stationGroups = _.split(sgCellValue, "|");
+ stationGroups.map(stationGroup =>{
+ tmpStationGroup = {};
+ let sgValue = _.split(stationGroup, ":");
+ if (sgValue && sgValue[0].length>0){
+ let stationArray = _.split(sgValue[0], ",");
+
+ tmpStationGroup['stations'] = stationArray;
+ tmpStationGroup['max_nr_missing'] = sgValue[1];
+ tmpStationGroups.push(tmpStationGroup);
+ }
+
+ })
+ for (const taskName in observStrategy.template.tasks) {
+ let task = observStrategy.template.tasks[taskName];
+ if (task.specifications_doc.station_groups) {
+ task.specifications_doc.station_groups = tmpStationGroups;
+ }
+ }
+ }
+
+ let between = this.getBetWeenDateValue(suRow.between);
+ let notbetween = this.getBetWeenDateValue(suRow.notbetween);
+
+ let isNewConstraint = false;
+ let newConstraint = {};
+ let constraint = null;
+ if (suRow.id >0){
+ newSU = _.find(this.state.schedulingUnitList, {'id': suRow.id});
+ constraint = newSU.scheduling_constraints_doc;
+ }
+
+ if ( constraint === null || constraint === 'undefined' || constraint === {}){
+ constraint = this.state.schedulingConstraintsDoc;
+ isNewConstraint = true;
+ }
+
+ //If No SU Constraint create default ( maintan default struc)
+ constraint['scheduler'] = suRow.scheduler;
+ if (suRow.scheduler === 'online'){
+ if (!constraint.time.at){
+ delete constraint.time.at;
+ }
+ if (!constraint.time.after) {
+ delete constraint.time.after;
+ }
+ if (!constraint.time.before) {
+ delete constraint.time.before;
+ }
+ } else {
+ constraint.time.at = `${moment(suRow.timeat).format("YYYY-MM-DDTHH:mm:ss.SSSSS", { trim: false })}Z`;
+ constraint.time.after = `${moment(suRow.timeafter).format("YYYY-MM-DDTHH:mm:ss.SSSSS", { trim: false })}Z`;
+ constraint.time.before = `${moment(suRow.timebefore).format("YYYY-MM-DDTHH:mm:ss.SSSSS", { trim: false })}Z`;
+ }
+ if (between && between.length>0){
+ constraint.time.between = between;
+ }
+ if (notbetween && notbetween.length>0){
+ constraint.time.not_between = notbetween;
+ }
+ let dailyValueSelected = _.split(suRow.daily, ",");
+ this.state.daily.forEach(daily =>{
+ if (_.includes(dailyValueSelected, daily)){
+ constraint.daily[daily] = true;
+ } else {
+ constraint.daily[daily] = false;
+ }
+ })
+ let min_distance_res = {};
+ min_distance_res['sun'] = suRow.md_sun;
+ min_distance_res['moon'] = suRow.md_moon;
+ min_distance_res['jupiter'] = suRow.md_jupiter;
+ constraint.sky.min_distance = min_distance_res;
+
+ let transit_offset_res = {};
+ transit_offset_res['from'] = +suRow.offset_from;
+ transit_offset_res['to'] = +suRow.offset_to;
+ if (transit_offset_res){
+ constraint.sky.transit_offset= transit_offset_res;
+ }
+
+ constraint.sky.min_target_elevation = suRow.min_target_elevation;
+ constraint.sky.min_calibrator_elevation = suRow.min_calibrator_elevation;
+
+ UnitConversion.degreeToRadians(constraint.sky);
+ if (isNewConstraint){
+ newSU.scheduling_constraints_doc = constraint;
+ }
+
+ if (suRow.id === 0){
+ newConstraint['scheduling_constraints_doc'] = constraint;
+ newConstraint['id'] = this.state.constraintId;
+ newConstraint['constraint'] = {'url':''};
+ newConstraint.constraint.url = this.state.constraintUrl;
+ }
+
+ if (suRow.id >0 && suRow.suname.length>0 && suRow.sudesc.length>0){
newSU = _.find(this.state.schedulingUnitList, {'id': suRow.id});
newSU['name'] = suRow.suname;
newSU['description'] = suRow.sudesc;
+
newSU.requirements_doc.tasks= observStrategy.template.tasks;
await ScheduleService.updateSUDraftFromObservStrategy(observStrategy, newSU, this.state.taskDrafts, this.state.tasksToUpdate);
existingSUCount++;
}
- else if(suRow.id === 0 && suRow.suname.length>0 && suRow.sudesc.length>0){
+ else if (suRow.id === 0 && suRow.suname.length>0 && suRow.sudesc.length>0){
newSU['id'] = suRow.id;
newSU['name'] = suRow.suname;
newSU['description'] = suRow.sudesc;
- await ScheduleService.saveSUDraftFromObservStrategy(observStrategy, newSU);
+ await ScheduleService.saveSUDraftFromObservStrategy(observStrategy, newSU, newConstraint);
newSUCount++;
}
}
- if((newSUCount+existingSUCount)>0){
+ if ((newSUCount+existingSUCount)>0){
const dialog = {header: 'Success', detail: '['+newSUCount+'] Scheduling Units are created & ['+existingSUCount+'] Scheduling Units are updated successfully.'};
this.setState({ dialogVisible: true, dialog: dialog});
- }else{
+ } else {
this.growl.show({severity: 'error', summary: 'Warning', detail: 'No Scheduling Units create/update '});
}
}catch(err){
this.growl.show({severity: 'error', summary: 'Error Occured', detail: 'Unable to create/update Scheduling Units'});
}
}
-
+
+ /**
+ * Convert the date to string value for Between And Not-Between Columns
+ * @param {*} dates
+ */
+ getBetweenStringValue(dates){
+ let returnDate = '';
+ if (dates){
+ dates.forEach(utcDateArray =>{
+ returnDate +=moment.utc(utcDateArray.from).format(DATE_TIME_FORMAT)+",";
+ returnDate +=moment.utc(utcDateArray.to).format(DATE_TIME_FORMAT)+"|";
+ })
+ }
+ return returnDate;
+ }
+
+ /**
+ * convert String to Date value for Between And Not-Between Columns
+ */
+ getBetWeenDateValue(betweenValue){
+ let returnDate = [];
+ if (betweenValue){
+ let rowDateArray = _.split(betweenValue, "|");
+ rowDateArray.forEach(betweenDates =>{
+ let betweendate = _.split(betweenDates, ",");
+ let dateres = {};
+ if (betweendate && betweendate.length === 2){
+ dateres['from'] = `${moment(betweendate[0]).format("YYYY-MM-DDTHH:mm:SS.SSSSS", { trim: false })}Z`;
+ dateres['to'] = `${moment(betweendate[1]).format("YYYY-MM-DDTHH:mm:SS.SSSSS", { trim: false })}Z`;
+ returnDate.push(dateres);
+ }
+ })
+ }
+ return returnDate;
+ }
+
+
/**
* Refresh the grid with updated data
*/
@@ -702,17 +1339,46 @@ export class SchedulingSetCreate extends Component {
}
async setNoOfSUint(value){
- if(value >= 0 && value < 501){
+ this.setState({isAGLoading: true});
+ if (value >= 0 && value < 501){
await this.setState({
noOfSU: value
})
- }else{
+ } else {
await this.setState({
noOfSU: 500
})
}
- //refresh row data
- await this.prepareScheduleUnitListForGrid();
+
+ let noOfSU = this.state.noOfSU;
+ this.tmpRowData = [];
+ let totalCount = this.state.totalCount;
+ if (this.state.rowData && this.state.rowData.length >0 && this.state.emptyRow) {
+ if (this.state.totalCount <= noOfSU) {
+ // set API data
+ for (var i = 0; i < totalCount; i++) {
+ this.tmpRowData.push(this.state.rowData[i]);
+ }
+ // add empty row
+ for(var i = this.state.totalCount; i < noOfSU; i++) {
+ this.tmpRowData.push(this.state.emptyRow);
+ }
+ this.setState({
+ rowData: this.tmpRowData,
+ noOfSU: noOfSU,
+ isAGLoading: false
+ });
+ } else {
+ this.setState({
+ isAGLoading: false
+ })
+ }
+
+ } else {
+ this.setState({
+ isAGLoading: false
+ });
+ }
}
validateForm(fieldName) {
@@ -755,6 +1421,10 @@ export class SchedulingSetCreate extends Component {
return validForm;
}
+ close(){
+ this.setState({saveDialogVisible: false})
+ }
+
/**
* This function is mainly added for Unit Tests. If this function is removed Unit Tests will fail.
*/
@@ -762,6 +1432,17 @@ export class SchedulingSetCreate extends Component {
return this.validEditor?true:false;
}
+ /**
+ * Show the content in custom dialog
+ */
+ showDialogContent(){
+ return <> Invalid Rows:- Row # and Invalid columns, <br/>{this.state.errorDisplay && this.state.errorDisplay.length>0 &&
+ this.state.errorDisplay.map((msg, index) => (
+ <React.Fragment key={index+10} className="col-lg-9 col-md-9 col-sm-12">
+ <span key={'label1-'+ index}>{msg}</span> <br />
+ </React.Fragment>
+ ))} </>
+ }
render() {
if (this.state.redirect) {
@@ -798,7 +1479,7 @@ export class SchedulingSetCreate extends Component {
tooltip="Scheduling set of the project" tooltipOptions={this.tooltipOptions}
value={this.state.schedulingUnit.scheduling_set_id}
options={this.state.schedulingSets}
- onChange={(e) => {this.setSchedingSetParams('scheduling_set_id',e.value)}}
+ onChange={(e) => {this.setSchedulingSetParams('scheduling_set_id',e.value)}}
placeholder="Select Scheduling Set" />
<label className={this.state.errors.scheduling_set_id ?"error":"info"}>
{this.state.errors.scheduling_set_id ? this.state.errors.scheduling_set_id : "Scheduling Set of the Project"}
@@ -833,27 +1514,30 @@ export class SchedulingSetCreate extends Component {
</div>
</div>
<>
- {this.state.observStrategy.id &&
- <div className="ag-theme-alpine" style={ { height: '500px', marginBottom: '10px' } } onKeyDown={this.clipboardEvent}>
- <AgGridReact
- suppressClipboardPaste={false}
- columnDefs={this.state.columnDefs}
- columnTypes={this.state.columnTypes}
- defaultColDef={this.state.defaultColDef}
- rowSelection={this.state.rowSelection}
- onGridReady={this.onGridReady}
- rowData={this.state.rowData}
- frameworkComponents={this.state.frameworkComponents}
- context={this.state.context}
- components={this.state.components}
- modules={this.state.modules}
- enableRangeSelection={true}
- rowSelection={this.state.rowSelection}
- >
-
- </AgGridReact>
- </div>
- }
+ { this.state.isAGLoading ? <AppLoader /> :
+ <>
+ {this.state.observStrategy.id &&
+ <div className="ag-theme-alpine" style={ {overflowX: 'inherit !importent', height: '500px', marginBottom: '10px' } } onKeyDown={this.clipboardEvent}>
+ <AgGridReact
+ suppressClipboardPaste={false}
+ columnDefs={this.state.columnDefs}
+ columnTypes={this.state.columnTypes}
+ defaultColDef={this.state.defaultColDef}
+ rowSelection={this.state.rowSelection}
+ onGridReady={this.onGridReady}
+ rowData={this.state.rowData}
+ frameworkComponents={this.state.frameworkComponents}
+ context={this.state.context}
+ components={this.state.components}
+ modules={this.state.modules}
+ enableRangeSelection={true}
+ rowSelection={this.state.rowSelection}
+ >
+ </AgGridReact>
+ </div>
+ }
+ </>
+ }
</>
<div className="p-grid p-justify-start">
<div className="p-col-1">
@@ -864,6 +1548,7 @@ export class SchedulingSetCreate extends Component {
<Button label="Cancel" className="p-button-danger" icon="pi pi-times" onClick={this.cancelCreate} />
</div>
</div>
+
</div>
</>
}
@@ -886,6 +1571,12 @@ export class SchedulingSetCreate extends Component {
</div>
</Dialog>
</div>
+
+ <CustomDialog type="confirmation" visible={this.state.saveDialogVisible} width="40vw"
+ header={'Save Scheduling Unit(s)'} message={' Some of the Scheduling Unit(s) has invalid data, Do you want to ignore and save valid Scheduling Unit(s) only?'}
+ content={this.showDialogContent} onClose={this.close} onCancel={this.close} onSubmit={this.saveSU}>
+ </CustomDialog>
+
</React.Fragment>
);
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js
index 8ff0d98a82fecec96b172ba1b4d85ec20d8466fc..3c4005621301b56437fac0f8ac0389dfb232510b 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/index.js
@@ -1,7 +1,13 @@
import React, {Component} from 'react';
+import _ from 'lodash';
+
import SchedulingUnitList from './SchedulingUnitList';
import PageHeader from '../../layout/components/PageHeader';
import { TieredMenu } from 'primereact/tieredmenu';
+import { CustomDialog } from '../../layout/components/CustomDialog';
+import { CustomPageSpinner } from '../../components/CustomPageSpinner';
+import ScheduleService from '../../services/schedule.service';
+import { Growl } from 'primereact/components/growl/Growl';
export class Scheduling extends Component {
constructor(props){
@@ -10,7 +16,8 @@ export class Scheduling extends Component {
scheduleunit: [],
schedule_unit_task: [] ,
isLoading:false,
- redirect: ''
+ redirect: '',
+ dialog: {header: 'Confirm', detail: 'Do you want to create blueprints for the selected drafts?'},
};
this.optionsMenu = React.createRef();
@@ -18,6 +25,11 @@ export class Scheduling extends Component {
this.showOptionMenu = this.showOptionMenu.bind(this);
this.selectOptionMenu = this.selectOptionMenu.bind(this);
+ this.checkAndCreateBlueprint = this.checkAndCreateBlueprint.bind(this);
+ this.createBlueprintTree = this.createBlueprintTree.bind(this);
+ this.createBlueprintTreeNewOnly = this.createBlueprintTreeNewOnly.bind(this);
+ this.warningContent = this.warningContent.bind(this);
+ this.closeDialog = this.closeDialog.bind(this);
}
showOptionMenu(event) {
@@ -36,20 +48,133 @@ export class Scheduling extends Component {
}
}
+ /**
+ * Subcomponet to display in the confirmation dialog.
+ */
+ warningContent() {
+ const suListWithBlueprint = this.state.schedulingUnitsWithBlueprint;
+ const suListWithoutBlueprint = _.difference(this.suList.selectedRows, suListWithBlueprint);
+ return (
+ <>
+ {suListWithBlueprint && suListWithBlueprint.length>0 &&
+ <div>
+ <hr></hr>
+ <span>Blueprint(s) already exist for the following Scheduling Units. If you want to create a blueprint for all of them click “yesâ€. If you want to create a blue print for a subset click “no†to change your selection.</span>
+ <div className="p-grid" key={`dlg-msg-head`} style={{marginTop: '10px'}}>
+ <label className="col-lg-3">ID</label>
+ <label className="col-lg-9">Name</label>
+ </div>
+ {suListWithBlueprint.map((schedulingUnit, index) => (
+ <div className="p-grid" key={`dlg-msg-${index}`} style={{marginBottom: "5px"}}>
+ <span className="col-lg-3">{schedulingUnit.id}</span>
+ <span className="col-lg-9">{schedulingUnit.name}</span>
+ </div>
+ ))}
+ </div>
+ }
+ {suListWithoutBlueprint && suListWithoutBlueprint.length>0 &&
+ <div>
+ <hr></hr>
+ <span>Selected Scheduling Unit drafts without blueprint are listed below.</span>
+ <div className="p-grid" key={`dlg-msg-head`} style={{marginTop: '10px'}}>
+ <label className="col-lg-3">ID</label>
+ <label className="col-lg-9">Name</label>
+ </div>
+ {suListWithoutBlueprint.map((schedulingUnit, index) => (
+ <div className="p-grid" key={`dlg-msg-${index}`} style={{marginBottom: "5px"}}>
+ <span className="col-lg-3">{schedulingUnit.id}</span>
+ <span className="col-lg-9">{schedulingUnit.name}</span>
+ </div>
+ ))}
+ {suListWithBlueprint && suListWithBlueprint.length>0 &&
+ <span>If you want to create blueprints for only drafts without blueprints, click 'Create Only New'</span>
+ }
+ </div>
+ }
+
+ </>
+ );
+ }
+
+ /**
+ * Function to check if blueprint already exist for the selected Scheduling Units and propmt contfirmation dialog.
+ * When confirmed will create new blueprints for the selected Scheduling Units.
+ */
+ checkAndCreateBlueprint() {
+ if (this.suList.selectedRows && this.suList.selectedRows.length>0) {
+ let dialog = this.state.dialog;
+ dialog.content = this.warningContent;
+ const schedulingUnitsWithBlueprint = _.filter(this.suList.selectedRows, schedulingUnit=> { return schedulingUnit.scheduling_unit_blueprints.length>0});
+ dialog.actions = [ {id:"yes", title: 'Yes', callback: this.createBlueprintTree},
+ {id:"no", title: 'No', callback: this.closeDialog} ]
+ /* Add this action only when both new and old drafts are selected */
+ if (schedulingUnitsWithBlueprint.length > 0 && this.suList.selectedRows.length>schedulingUnitsWithBlueprint.length) {
+ dialog.actions.unshift({id:"newOnly", title: 'Create Only New', callback: this.createBlueprintTreeNewOnly});
+ }
+ this.setState({dialogVisible: true, dialog: dialog, schedulingUnitsWithBlueprint: _.sortBy(schedulingUnitsWithBlueprint,['id'])});
+ } else {
+ this.growl.show({severity: 'info', summary: 'Select Row', detail: 'Please select one or more Scheduling Unit Draft(s)'});
+ }
+ }
+
+ /**
+ * Callback function from dialog to create blueprints for only new drafts without blueprints.
+ * @param {Event} event
+ */
+ createBlueprintTreeNewOnly(event){
+ this.createBlueprintTree(event, true);
+ }
+
+ /**
+ * Function to create actual blueprints for the selected drafts
+ * @param {Event} event
+ * @param {Boolean} excludeOld
+ */
+ async createBlueprintTree(event, excludeOld) {
+ this.setState({dialogVisible: false, showSpinner: true});
+ let selectedRows = this.suList.selectedRows;
+ // Remove old drafts from selected rows
+ if (excludeOld) {
+ selectedRows = _.difference(selectedRows, this.state.schedulingUnitsWithBlueprint);
+ }
+ for (const schedulingUnit of selectedRows) {
+ await ScheduleService.createSchedulingUnitBlueprintTree(schedulingUnit.id);
+ }
+ this.setState({showSpinner: false, schedulingUnitsWithBlueprint:null});
+ this.growl.show({severity: 'success', summary: 'Success', detail: 'Blueprint(s) created successfully!'});
+ this.suList.reloadData();
+ }
+
+ /**
+ * Callback function to close the dialog.
+ */
+ closeDialog() {
+ this.setState({dialogVisible: false});
+ }
+
render() {
return (
<>
- <TieredMenu className="app-header-menu" model={this.menuOptions} popup ref={el => this.optionsMenu = el} />
- <PageHeader location={this.props.location} title={'Scheduling Unit - List'}
+ <Growl ref={(el) => this.growl = el} style={{paddingTop:"50px"}} />
+ <TieredMenu className="app-header-menu" model={this.menuOptions} popup ref={el => this.optionsMenu = el} />
+ <PageHeader location={this.props.location} title={'Scheduling Unit - List'}
actions={[
-
+ {icon:'fa-stamp', title: 'Create Blueprint', type:'button',
+ actOn:'click', props : { callback: this.checkAndCreateBlueprint}},
{icon: 'fa fa-plus-square', title: 'Add New Scheduling Unit',
props: {pathname: '/schedulingunit/create'}},
{icon: 'fa fa-table', title: 'Add Scheduling Set',
props: {pathname: '/schedulingset/schedulingunit/create'}}]} />
{this.state.scheduleunit &&
- <SchedulingUnitList /> }
+ <SchedulingUnitList allowRowSelection={true} ref={suList => {this.suList = suList}} /> }
+ {/* Dialog component to show messages and get confirmation */}
+ <CustomDialog type="confirmation" visible={this.state.dialogVisible} width="40vw"
+ header={this.state.dialog.header} message={this.state.dialog.detail} content={this.state.dialog.content}
+ onClose={this.closeDialog} onCancel={this.closeDialog} onSubmit={this.createBlueprintTree}
+ actions={this.state.dialog.actions}></CustomDialog>
+ {/* Show spinner during backend API call */}
+ <CustomPageSpinner visible={this.state.showSpinner} />
</>
);
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js
index 4b2ef70081a130dd75f9567e8c1ec2616b186c90..53b1672422eac8ec3525cf2742413f98582656c8 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js
@@ -5,6 +5,7 @@ import _ from 'lodash';
import ViewTable from '../../components/ViewTable';
import { JsonToTable } from "react-json-to-table";
import SchedulingConstraints from './Scheduling.Constraints';
+import Stations from './Stations';
/**
* Component to view summary of the scheduling unit with limited task details
@@ -15,7 +16,7 @@ export class SchedulingUnitSummary extends Component {
super(props);
this.state = {
schedulingUnit: props.schedulingUnit || null
- }
+ };
this.constraintsOrder = ['scheduler','time','daily','sky'];
this.closeSUDets = this.closeSUDets.bind(this);
this.setConstraintsEditorOutput = this.setConstraintsEditorOutput.bind(this);
@@ -152,14 +153,28 @@ export class SchedulingUnitSummary extends Component {
}
</>
}
+
+ {/* {<Stations
+ stationGroup={this.props.stationGroup}
+ view
+ isSummary
+ />} */}
+ <div className="col-12"><label>Stations:</label></div>
+ <div className="col-12 station-list">
+ {this.props.stationGroup && this.props.stationGroup.map((station, index) => (
+ <div key={`stn-${index}`}>{station}</div>
+ ))}
+
+ </div>
+
<div className="col-12 task-summary">
<label>Tasks:</label>
<ViewTable
data={suTaskList}
- defaultcolumns={[{id: "ID", start_time:"Start Time", stop_time:"End Time", status: "Status",
+ defaultcolumns={[{id: "ID", subTaskID: 'Control ID', start_time:"Start Time", stop_time:"End Time", status: "Status",
antenna_set: "Antenna Set", band: 'Band'}]}
optionalcolumns={[{actionpath: "actionpath"}]}
- columnclassname={[{"ID": "filter-input-50", "Start Time": "filter-input-75", "End Time": "filter-input-75",
+ columnclassname={[{"ID": "filter-input-50","Control ID":"filter-input-75", "Start Time": "filter-input-75", "End Time": "filter-input-75",
"Status": "filter-input-75", "Antenna Set": "filter-input-75", "Band": "filter-input-75"}]}
defaultSortColumn= {[{id: "ID", desc: false}]}
showaction="false"
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
index 2fd2e272fc587b7ed335de24b48ed3a6c31a4352..9a0b6cc381d3dde261eaeca7a05272e03b962bac 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
@@ -43,7 +43,8 @@ export class TimelineView extends Component {
canShrinkSUList: false,
selectedItem: null,
suTaskList:[],
- isSummaryLoading: false
+ isSummaryLoading: false,
+ stationGroup: []
}
this.STATUS_BEFORE_SCHEDULED = ['defining', 'defined', 'schedulable']; // Statuses before scheduled to get station_group
this.allStationsGroup = [];
@@ -149,15 +150,19 @@ export class TimelineView extends Component {
canExtendSUList: false, canShrinkSUList:false});
if (fetchDetails) {
const suBlueprint = _.find(this.state.suBlueprints, {id: (this.state.stationView?parseInt(item.id.split('-')[0]):item.id)});
- ScheduleService.getTaskBlueprintsBySchedulingUnit(suBlueprint, true)
+ ScheduleService.getTaskBPWithSubtaskTemplateOfSU(suBlueprint)
.then(taskList => {
for (let task of taskList) {
+ //Control Task Id
+ const subTaskIds = (task.subTasks || []).filter(sTask => sTask.subTaskTemplate.name.indexOf('control') > 1);
+ task. subTaskID = subTaskIds.length ? subTaskIds[0].id : '';
if (task.template.type_value.toLowerCase() === "observation") {
task.antenna_set = task.specifications_doc.antenna_set;
task.band = task.specifications_doc.filter;
}
}
- this.setState({suTaskList: _.sortBy(taskList, "id"), isSummaryLoading: false})
+ this.setState({suTaskList: _.sortBy(taskList, "id"), isSummaryLoading: false,
+ stationGroup: this.getSUStations(suBlueprint)});
});
// Get the scheduling constraint template of the selected SU block
ScheduleService.getSchedulingConstraintTemplate(suBlueprint.suDraft.scheduling_constraints_template_id)
@@ -219,37 +224,44 @@ export class TimelineView extends Component {
* @param {Array} items
*/
getStationItemGroups(suBlueprint, timelineItem, group, items) {
- /** Get all observation tasks */
- const observtionTasks = _.filter(suBlueprint.tasks, (task) => { return task.template.type_value.toLowerCase() === "observation"});
+ /* Get stations based on SU status */
+ let stations = this.getSUStations(suBlueprint);
+
+ /* Group the items by station */
+ for (const station of stations) {
+ let stationItem = _.cloneDeep(timelineItem);
+ stationItem.id = `${stationItem.id}-${station}`;
+ stationItem.group = station;
+ items.push(stationItem);
+ }
+ }
+
+ /**
+ * Get all stations of the SU bleprint from the observation task or subtask bases on the SU status.
+ * @param {Object} suBlueprint
+ */
+ getSUStations(suBlueprint) {
let stations = [];
- for (const observtionTask of observtionTasks) {
+ /* Get all observation tasks */
+ const observationTasks = _.filter(suBlueprint.tasks, (task) => { return task.template.type_value.toLowerCase() === "observation"});
+ for (const observationTask of observationTasks) {
/** If the status of SU is before scheduled, get all stations from the station_groups from the task specification_docs */
if (this.STATUS_BEFORE_SCHEDULED.indexOf(suBlueprint.status.toLowerCase()) >= 0
- && observtionTask.specifications_doc.station_groups) {
- for (const grpStations of _.map(observtionTask.specifications_doc.station_groups, "stations")) {
+ && observationTask.specifications_doc.station_groups) {
+ for (const grpStations of _.map(observationTask.specifications_doc.station_groups, "stations")) {
stations = _.concat(stations, grpStations);
}
} else if (this.STATUS_BEFORE_SCHEDULED.indexOf(suBlueprint.status.toLowerCase()) < 0
- && observtionTask.subTasks) {
+ && observationTask.subTasks) {
/** If the status of SU is scheduled or after get the stations from the subtask specification tasks */
- for (const subtask of observtionTask.subTasks) {
+ for (const subtask of observationTask.subTasks) {
if (subtask.specifications_doc.stations) {
stations = _.concat(stations, subtask.specifications_doc.stations.station_list);
}
}
}
}
- stations = _.uniq(stations);
- /** Group the items by station */
- for (const station of stations) {
- let stationItem = _.cloneDeep(timelineItem);
- stationItem.id = `${stationItem.id}-${station}`;
- stationItem.group = station;
- items.push(stationItem);
- // if (!_.find(group, {'id': station})) {
- // group.push({'id': station, title: station});
- // }
- }
+ return _.uniq(stations);
}
/**
@@ -369,6 +381,7 @@ export class TimelineView extends Component {
{this.state.isSummaryLoading?<AppLoader /> :
<SchedulingUnitSummary schedulingUnit={suBlueprint} suTaskList={this.state.suTaskList}
constraintsTemplate={this.state.suConstraintTemplate}
+ stationGroup={this.state.stationGroup}
closeCallback={this.closeSUDets}></SchedulingUnitSummary>
}
</div>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/week.view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/week.view.js
index b4b3ae65ec5a753e2b52bff6c251f59c3d60f71e..67e9ef5c7e7439c1fc5c4399014a7bc4ac9126f4 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/week.view.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/week.view.js
@@ -44,7 +44,8 @@ export class WeekTimelineView extends Component {
canShrinkSUList: false,
selectedItem: null,
suTaskList:[],
- isSummaryLoading: false
+ isSummaryLoading: false,
+ stationGroup: []
}
this.onItemClick = this.onItemClick.bind(this);
@@ -166,15 +167,27 @@ export class WeekTimelineView extends Component {
canExtendSUList: false, canShrinkSUList:false});
if (fetchDetails) {
const suBlueprint = _.find(this.state.suBlueprints, {id: parseInt(item.id.split('-')[0])});
- ScheduleService.getTaskBlueprintsBySchedulingUnit(suBlueprint, true)
+ ScheduleService.getTaskBPWithSubtaskTemplateOfSU(suBlueprint)
.then(taskList => {
+ const observationTask = _.find(taskList, (task)=> {return task.template.type_value==='observation' && task.specifications_doc.station_groups});
for (let task of taskList) {
+ //Control Task ID
+ const subTaskIds = (task.subTasks || []).filter(sTask => sTask.subTaskTemplate.name.indexOf('control') > 1);
+ task. subTaskID = subTaskIds.length ? subTaskIds[0].id : '';
if (task.template.type_value.toLowerCase() === "observation") {
task.antenna_set = task.specifications_doc.antenna_set;
task.band = task.specifications_doc.filter;
}
}
- this.setState({suTaskList: _.sortBy(taskList, "id"), isSummaryLoading: false})
+ let stations = [];
+ //>>>>>> TODO: Station groups from subtasks based on the status of SU
+ if (observationTask) {
+ for (const grpStations of _.map(observationTask.specifications_doc.station_groups, "stations")) {
+ stations = _.concat(stations, grpStations);
+ }
+ }
+ this.setState({suTaskList: _.sortBy(taskList, "id"), isSummaryLoading: false,
+ stationGroup: _.uniq(stations)})
});
// Get the scheduling constraint template of the selected SU block
ScheduleService.getSchedulingConstraintTemplate(suBlueprint.suDraft.scheduling_constraints_template_id)
@@ -366,6 +379,7 @@ export class WeekTimelineView extends Component {
<SchedulingUnitSummary schedulingUnit={suBlueprint} suTaskList={this.state.suTaskList}
constraintsTemplate={this.state.suConstraintTemplate}
closeCallback={this.closeSUDets}
+ stationGroup={this.state.stationGroup}
location={this.props.location}></SchedulingUnitSummary>
}
</div>
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/QAreporting.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/QAreporting.js
deleted file mode 100644
index 5ffe01ea22288a4b72aa794753358c400d8862c6..0000000000000000000000000000000000000000
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/QAreporting.js
+++ /dev/null
@@ -1,97 +0,0 @@
-import React, { Component } from 'react';
-import PageHeader from '../../layout/components/PageHeader';
-import {Growl} from 'primereact/components/growl/Growl';
-import { Button } from 'primereact/button';
-// import AppLoader from '../../layout/components/AppLoader';
-import SunEditor from 'suneditor-react';
-import 'suneditor/dist/css/suneditor.min.css'; // Import Sun Editor's CSS File
-import {Dropdown} from 'primereact/dropdown';
-// import {InputText} from 'primereact/inputtext';
-import ScheduleService from '../../services/schedule.service';
-import { Link } from 'react-router-dom';
-
-class QAreporting extends Component{
-
- constructor(props) {
- super(props);
- this.state={};
- }
-
- componentDidMount() {
- ScheduleService.getSchedulingUnitBlueprintById(this.props.match.params.id)
- .then(schedulingUnit => {
- this.setState({schedulingUnit: schedulingUnit});
- })
- }
-
- render() {
- return (
- <React.Fragment>
- <Growl ref={(el) => this.growl = el} />
- <PageHeader location={this.props.location} title={'QA Reporting (TO)'} actions={[{icon:'fa-window-close',link:this.props.history.goBack, title:'Click to Close Workflow', props:{ pathname: '/schedulingunit/view'}}]}/>
- {this.state.schedulingUnit &&
- <>
- <div>
- <div className="p-fluid">
- <div className="p-field p-grid">
- <label htmlFor="suStatus" className="col-lg-2 col-md-2 col-sm-12">Scheduling Unit</label>
- <div className="col-lg-3 col-md-3 col-sm-12">
- <Link to={ { pathname:`/schedulingunit/view/blueprint/${this.state.schedulingUnit.id}`}}>{this.state.schedulingUnit.name}</Link>
- </div>
- <div className="col-lg-1 col-md-1 col-sm-12"></div>
- <label htmlFor="suStatus" className="col-lg-2 col-md-2 col-sm-12">Scheduling Unit Status</label>
- <div className="col-lg-3 col-md-3 col-sm-12">
- {/* <InputText id="suStatus" data-testid="name" disabled
- value={this.state.schedulingUnit.status}/> */}
- <span>{this.state.schedulingUnit.status}</span>
- </div>
- </div>
- <div className="p-field p-grid">
- <label htmlFor="assignTo" className="col-lg-2 col-md-2 col-sm-12">Assign To </label>
- <div className="col-lg-3 col-md-3 col-sm-12" data-testid="assignTo" >
- <Dropdown inputId="projCat" optionLabel="value" optionValue="value"
- options={[{value: 'User 1'},{value: 'User 2'},{value: 'User 3'}]}
- placeholder="Assign To" />
- </div>
- <div className="col-lg-1 col-md-1 col-sm-12"></div>
- <label htmlFor="viewPlots" className="col-lg-2 col-md-2 col-sm-12">View Plots</label>
- <div className="col-lg-3 col-md-3 col-sm-12" style={{paddingLeft:'2px'}}>
- <label className="col-sm-10 " >
- <a href="https://proxy.lofar.eu/inspect/HTML/" target="_blank">Inspection plots</a>
- </label>
- <label className="col-sm-10 ">
- <a href="https://proxy.lofar.eu/qa" target="_blank">Adder plots</a>
- </label>
- <label className="col-sm-10 ">
- <a href=" https://proxy.lofar.eu/lofmonitor/" target="_blank">Station Monitor</a>
- </label>
- </div>
- </div>
- <div className="p-grid" style={{padding: '10px'}}>
- <label htmlFor="comments" >Comments</label>
- <div className="col-lg-12 col-md-12 col-sm-12"></div>
- <SunEditor height="250" enableToolbar={true}
- setOptions={{
- buttonList: [
- ['undo', 'redo', 'bold', 'underline', 'fontColor', 'table', 'link', 'image', 'video','italic', 'strike', 'subscript',
- 'superscript','outdent', 'indent','fullScreen', 'showBlocks', 'codeView','preview', 'print','removeFormat']
- ]
- }} />
- </div>
- </div>
- <div className="p-grid" style={{marginTop: '20px'}}>
- <div className="p-col-1">
- <Button label="Save" className="p-button-primary" icon="pi pi-check" />
- </div>
- <div className="p-col-1">
- <Button label="Cancel" className="p-button-danger" icon="pi pi-times" />
- </div>
- </div>
-
- </div>
- </>
- }
- </React.Fragment>
- )};
-}
-export default QAreporting;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/Scheduled.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/Scheduled.js
new file mode 100644
index 0000000000000000000000000000000000000000..5e1cad1b4c4994b239b627675ee6f85e5fc04891
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/Scheduled.js
@@ -0,0 +1,64 @@
+import React, { Component } from 'react';
+import { Button } from 'primereact/button';
+import { Link } from 'react-router-dom';
+import moment from 'moment';
+
+class Scheduled extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {};
+ this.Next = this.Next.bind(this);
+ }
+
+ /**
+ * Method will trigger on click save buton
+ * here onNext props coming from parent, where will handle redirection to other page
+ */
+ Next() {
+ this.props.onNext({});
+ }
+
+ render() {
+ return (
+ <>
+ <div>
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <label htmlFor="suStatus" className="col-lg-2 col-md-2 col-sm-12">Start Time</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <span>{this.props.schedulingUnit.start_time && moment(this.props.schedulingUnit.start_time).format("YYYY-MMM-DD HH:mm:SS")}</span>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="suStatus" className="col-lg-2 col-md-2 col-sm-12">End Time</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <span>{this.props.schedulingUnit.stop_time && moment(this.props.schedulingUnit.stop_time).format("YYYY-MMM-DD HH:mm:SS")}</span>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="suStatus" className="col-lg-2 col-md-2 col-sm-12">Timeline</label>
+ <div className="col-lg-3 col-md-3 col-sm-12 block-list">
+ <Link to={{ pathname: '/su/timelineview' }}>TimeLine View <span class="fas fa-clock"></span></Link>
+ <Link to={{ pathname: '/su/timelineview/week' }}>Week Overview <span class="fas fa-calendar-alt"></span></Link>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="suStatus" className="col-lg-2 col-md-2 col-sm-12">Scheduling Method</label>
+ <div className="col-lg-3 col-md-3 col-sm-12 block-list">
+ <span>{this.props.schedulingUnit.scheduling_constraints_doc.scheduler}</span>
+ </div>
+ </div>
+ </div>
+
+ <div className="p-grid p-justify-start">
+ <div className="p-col-1">
+ <Button label="Next" className="p-button-primary" icon="pi pi-check" onClick={ this.Next } />
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" style={{ width: '90px' }} />
+ </div>
+ </div>
+ </div>
+ </>
+ )
+ };
+
+}
+export default Scheduled;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/decide.acceptance.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/decide.acceptance.js
new file mode 100644
index 0000000000000000000000000000000000000000..7087513f513adf2350dc8fd911b7d9c7953da671
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/decide.acceptance.js
@@ -0,0 +1,115 @@
+import React, { Component } from 'react';
+import { Button } from 'primereact/button';
+import SunEditor from 'suneditor-react';
+import 'suneditor/dist/css/suneditor.min.css'; // Import Sun Editor's CSS File
+import { Checkbox } from 'primereact/checkbox';
+
+class DecideAcceptance extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ content: props.report,
+ picomment: props.picomment, //PI Comment Field
+ showEditor: false, //Sun Editor
+ checked: false, //Checkbox
+
+ };
+ this.Next = this.Next.bind(this);
+ this.handleChange = this.handleChange.bind(this);
+ this.onChangePIComment = this.onChangePIComment.bind(this);
+ }
+
+
+ // Method will trigger on change of operator report sun-editor
+ handleChange(e) {
+ this.setState({
+ content: e
+ });
+ localStorage.setItem('report_qa', e);
+ }
+
+ //PI Comment Editor
+ onChangePIComment(e) {
+ this.setState({
+ picomment: e.target.value
+ });
+ localStorage.setItem('pi_comment', e.target.value);
+ }
+
+ /**
+ * Method will trigger on click save buton
+ * here onNext props coming from parent, where will handle redirection to other page
+ */
+ Next() {
+ this.props.onNext({
+ report: this.state.content,
+ picomment: this.state.picomment
+
+ });
+ }
+
+ // Not using at present
+ cancelCreate() {
+ this.props.history.goBack();
+ }
+
+ render() {
+ return (
+ <>
+ <div>
+ <div className="p-fluid">
+ <div className="p-grid" style={{ padding: '15px' }}>
+ <label htmlFor="operatorReport" >Operator Report</label>
+ <div className="col-lg-12 col-md-12 col-sm-12"></div>
+ {this.state.showEditor && <SunEditor setDefaultStyle="min-height: 250px; height: auto;" enableToolbar={true}
+ onChange={this.handleChange}
+ setContents={this.state.content}
+ setOptions={{
+ buttonList: [
+ ['undo', 'redo', 'bold', 'underline', 'fontColor', 'table', 'link', 'image', 'video', 'italic', 'strike', 'subscript',
+ 'superscript', 'outdent', 'indent', 'fullScreen', 'showBlocks', 'codeView', 'preview', 'print', 'removeFormat']
+ ]
+ }}
+ />}
+ <div dangerouslySetInnerHTML={{ __html: this.state.content }}></div>
+ </div>
+ <div className="p-field p-grid">
+ <label htmlFor="piReport" className="col-lg-2 col-md-2 col-sm-12">PI Report</label>
+ <div className="col-lg-12 col-md-12 col-sm-12">
+ {this.state.showEditor && <SunEditor setDefaultStyle="min-height: 250px; height: auto;" enableToolbar={true}
+ onChange={this.onChangePIComment}
+ setContents={this.state.picomment}
+ setOptions={{
+ buttonList: [
+ ['undo', 'redo', 'bold', 'underline', 'fontColor', 'table', 'link', 'image', 'video', 'italic', 'strike', 'subscript',
+ 'superscript', 'outdent', 'indent', 'fullScreen', 'showBlocks', 'codeView', 'preview', 'print', 'removeFormat']
+ ]
+ }}
+ />}
+ <div className="operator-report" dangerouslySetInnerHTML={{ __html: this.state.picomment }}></div>
+ </div>
+ </div>
+ <div className="p-field p-grid">
+ <label htmlFor="piAccept" className="col-lg-2 col-md-2 col-sm-12">SDCO accepts after PI</label>
+ <div className="col-lg-3 col-md-3 col-sm-6">
+ <div className="p-field-checkbox">
+ <Checkbox inputId="binary" checked={this.state.checked} onChange={e => this.setState({ checked: e.checked })} />
+ </div>
+ </div>
+ </div>
+ </div>
+ <div className="p-grid" style={{ marginTop: '20px' }}>
+ <div className="p-col-1">
+ <Button label="Next" className="p-button-primary" icon="pi pi-check" onClick = { this.Next } />
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" style={{ width : '90px' }} />
+ </div>
+ </div>
+
+ </div>
+ </>
+ )
+ };
+}
+export default DecideAcceptance;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/index.js
new file mode 100644
index 0000000000000000000000000000000000000000..f477d2f5970a6e42ae33cc5cef7d92c16af638e2
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/index.js
@@ -0,0 +1,93 @@
+import React, { useEffect, useState } from 'react';
+import PageHeader from '../../layout/components/PageHeader';
+import {Growl} from 'primereact/components/growl/Growl';
+import { Link } from 'react-router-dom';
+import ScheduleService from '../../services/schedule.service';
+import Scheduled from './Scheduled';
+import ProcessingDone from './processing.done';
+import QAreporting from './qa.reporting';
+import QAsos from './qa.sos';
+import PIverification from './pi.verification';
+import DecideAcceptance from './decide.acceptance';
+import IngestDone from './ingest.done';
+
+//Workflow Page Title
+const pageTitle = ['Scheduled','Processing Done','QA Reporting (TO)', 'QA Reporting (SDCO)', 'PI Verification', 'Decide Acceptance','Ingest Done'];
+
+export default (props) => {
+ let growl;
+ const [state, setState] = useState({});
+ const [currentStep, setCurrentStep] = useState(1);
+ const [schedulingUnit, setSchedulingUnit] = useState();
+ const [ingestTask, setInjestTask] = useState({});
+ useEffect(() => {
+ // Clearing Localstorage on start of the page to load fresh
+ clearLocalStorage();
+ ScheduleService.getSchedulingUnitBlueprintById(props.match.params.id)
+ .then(schedulingUnit => {
+ setSchedulingUnit(schedulingUnit);
+ })
+ const promises = [ScheduleService.getSchedulingUnitBlueprintById(props.match.params.id), ScheduleService.getTaskType()]
+ Promise.all(promises).then(responses => {
+ setSchedulingUnit(responses[0]);
+ ScheduleService.getTaskBlueprintsBySchedulingUnit(responses[0], true, false).then(response => {
+ setInjestTask(response.find(task => task.template.type_value==='observation'));
+ });
+ });
+ }, []);
+
+ const clearLocalStorage = () => {
+ localStorage.removeItem('pi_comment');
+ localStorage.removeItem('report_qa');
+ }
+
+ //Pages changes step by step
+ const onNext = (content) => {
+ setState({...state, ...content});
+ setCurrentStep(currentStep + 1);
+ };
+
+ return (
+ <>
+ <Growl ref={(el) => growl = el} />
+ <PageHeader location={props.location} title={`${pageTitle[currentStep - 1]}`} actions={[{ icon: 'fa-window-close', link: props.history.goBack, title: 'Click to Close Workflow', props: { pathname: '/schedulingunit/1/workflow' } }]} />
+ {schedulingUnit &&
+ <>
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <label htmlFor="suName" className="col-lg-2 col-md-2 col-sm-12">Scheduling Unit</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <Link to={{ pathname: `/schedulingunit/view/blueprint/${schedulingUnit.id}` }}>{schedulingUnit.name}</Link>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="suStatus" className="col-lg-2 col-md-2 col-sm-12">Scheduling Unit Status</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <span>{schedulingUnit.status}</span>
+ </div>
+ <label htmlFor="viewPlots" className="col-lg-2 col-md-2 col-sm-12">View Plots</label>
+ <div className="col-lg-3 col-md-3 col-sm-12" style={{ paddingLeft: '2px' }}>
+ <label className="col-sm-10 " >
+ <a href="https://proxy.lofar.eu/inspect/HTML/" target="_blank">Inspection plots</a>
+ </label>
+ <label className="col-sm-10 ">
+ <a href="https://proxy.lofar.eu/qa" target="_blank">Adder plots</a>
+ </label>
+ <label className="col-sm-10 ">
+ <a href=" https://proxy.lofar.eu/lofmonitor/" target="_blank">Station Monitor</a>
+ </label>
+ </div>
+ </div>
+ {currentStep === 1 && <Scheduled onNext={onNext} {...state} schedulingUnit={schedulingUnit} />}
+ {currentStep === 2 && <ProcessingDone onNext={onNext} {...state}/>}
+ {currentStep === 3 && <QAreporting onNext={onNext}/>}
+ {currentStep === 4 && <QAsos onNext={onNext} {...state} />}
+ {currentStep === 5 && <PIverification onNext={onNext} {...state} />}
+ {currentStep === 6 && <DecideAcceptance onNext={onNext} {...state} />}
+ {currentStep === 7 && <IngestDone onNext={onNext}{...state} task={ingestTask} />}
+
+ </div>
+ </>
+ }
+ </>
+ )
+};
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/ingest.done.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/ingest.done.js
new file mode 100644
index 0000000000000000000000000000000000000000..db2887b1bff7c3d96fe9b0e3dcca28b3a88be890
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/ingest.done.js
@@ -0,0 +1,47 @@
+import React, { Component } from 'react';
+import { Link } from 'react-router-dom';
+
+class IngestDone extends Component {
+ constructor(props) {
+ super(props);
+ this.state = { };
+ this.onSave = this.onSave.bind(this);
+ }
+
+ onSave(){
+ this.props.onNext({
+ report: this.props.report,
+ picomment: this.props.picomment
+ });
+ }
+
+ render(){
+ return(
+ <>
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <label htmlFor="ingestTaskStatus" className="col-lg-2 col-md-2 col-sm-12">Ingest Task Status</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <span>{this.props.task.status}</span>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="ingestTask" className="col-lg-2 col-md-2 col-sm-12">Ingest Task</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <a href={`${window.location.origin}/task/view/blueprint/${this.props.task.id}`}>{this.props.task.name}</a>
+ </div>
+ <label htmlFor="ingestMonitoring" className="col-lg-2 col-md-2 col-sm-12">Ingest Monitoring</label>
+ <label className="col-sm-10 " >
+ <a href="http://lexar003.control.lofar:9632/" target="_blank">View Ingest Monitoring <span class="fas fa-desktop"></span></a>
+ </label>
+
+ {/* <div className="col-lg-3 col-md-3 col-sm-12">
+ <Link to={{ pathname: `http://lexar003.control.lofar:9632/` }}> View Ingest Monitoring <span class="fas fa-desktop"></span></Link>
+ </div> */}
+ </div>
+ </div>
+ </>
+ )
+ };
+
+}
+export default IngestDone;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/pi.verification.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/pi.verification.js
new file mode 100644
index 0000000000000000000000000000000000000000..f63a6fe0591e6dd6acd9fd5bc72c1988c33fc358
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/pi.verification.js
@@ -0,0 +1,115 @@
+import React, { Component } from 'react';
+import { Button } from 'primereact/button';
+import SunEditor from 'suneditor-react';
+import 'suneditor/dist/css/suneditor.min.css'; // Import Sun Editor's CSS File
+import { Checkbox } from 'primereact/checkbox';
+//import {InputTextarea} from 'primereact/inputtextarea';
+
+class PIverification extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ content: props.report,
+ showEditor: false,
+ };
+ this.Next = this.Next.bind(this);
+ this.handleChange = this.handleChange.bind(this);
+ this.onChangePIComment = this.onChangePIComment.bind(this);
+ }
+
+ /**
+ * Method wiill trigger on change of operator report sun-editor
+ */
+ handleChange(e) {
+ this.setState({
+ comment: e
+ });
+ localStorage.setItem('report_pi', e);
+ }
+
+ /**
+ * Method will trigger on click save buton
+ * here onNext props coming from parent, where will handle redirection to other page
+ */
+ Next(){
+ this.props.onNext({
+ report: this.state.content,
+ picomment: this.state.comment
+ });
+ }
+
+ /**
+ * Method wiill triigger on change of pi report sun-editor
+ */
+ onChangePIComment(a) {
+ this.setState({
+ comment: a
+ });
+ localStorage.setItem('comment_pi', a);
+ }
+
+ // Not using at present
+ cancelCreate() {
+ this.props.history.goBack();
+ }
+
+ render() {
+ return (
+ <>
+ <div>
+ <div className="p-fluid">
+ <div className="p-grid" style={{ padding: '10px' }}>
+ <label htmlFor="operatorReport" >Operator Report</label>
+ <div className="col-lg-12 col-md-12 col-sm-12"></div>
+ {this.state.showEditor && <SunEditor setDefaultStyle="min-height: 250px; height: auto;" enableToolbar={true}
+ onChange={this.handleChange}
+ setContents={this.state.content}
+ setOptions={{
+ buttonList: [
+ ['undo', 'redo', 'bold', 'underline', 'fontColor', 'table', 'link', 'image', 'video', 'italic', 'strike', 'subscript',
+ 'superscript', 'outdent', 'indent', 'fullScreen', 'showBlocks', 'codeView', 'preview', 'print', 'removeFormat']
+ ]
+ }}
+ />}
+ <div className="operator-report" dangerouslySetInnerHTML={{ __html: this.state.content }}></div>
+ </div>
+ <div className="p-grid" style={{ padding: '10px' }}>
+ <label htmlFor="piReport" >PI Report</label>
+ <div className="col-lg-12 col-md-12 col-sm-12"></div>
+ <SunEditor setDefaultStyle="min-height: 150px; height: auto;" enableToolbar={true}
+ setContents={this.state.comment}
+ onChange={this.onChangePIComment}
+ setOptions={{
+ buttonList: [
+ ['undo', 'redo', 'bold', 'underline', 'fontColor', 'table', 'link', 'image', 'video', 'italic', 'strike', 'subscript',
+ 'superscript', 'outdent', 'indent', 'fullScreen', 'showBlocks', 'codeView', 'preview', 'print', 'removeFormat']
+ ]
+ }} />
+ {/* <InputTextarea rows={3} cols={30}
+ tooltip="PIReport" tooltipOptions={this.tooltipOptions} maxLength="128"
+ data-testid="PIReport"
+ value={this.state.piComment}
+ onChange={this.onChangePIComment}
+ /> */}
+ </div>
+ <div className="p-field p-grid">
+ <label htmlFor="piAccept" className="col-lg-2 col-md-2 col-sm-12">PI Accept</label>
+ <div className="p-field-checkbox">
+ <Checkbox inputId="binary" checked={this.state.checked} onChange={e => this.setState({ checked: e.checked })} />
+ </div>
+ </div>
+ <div className="p-grid" style={{ marginTop: '20px' }}>
+ <div className="p-col-1">
+ <Button label="Next" className="p-button-primary" icon="pi pi-check" onClick={ this.Next } />
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" style={{ width : '90px' }} />
+ </div>
+ </div>
+ </div>
+ </div>
+ </>
+ )
+ };
+}
+export default PIverification;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/processing.done.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/processing.done.js
new file mode 100644
index 0000000000000000000000000000000000000000..07c7ca9cdca1ae96d2d8ce166d836303036ccbc0
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/processing.done.js
@@ -0,0 +1,37 @@
+import React, { Component } from 'react';
+import { Button } from 'primereact/button';
+
+class ProcessingDone extends Component {
+
+ constructor(props) {
+ super(props);
+ this.Next = this.Next.bind(this);
+ }
+
+ /**
+ * Method will trigger on click save buton
+ * here onNext props coming from parent, where will handle redirection to other page
+ */
+ Next(){
+ this.props.onNext({});
+ }
+
+ render(){
+ return(
+ <>
+ <div className="p-grid p-justify-start">
+ <div className="p-col-1">
+ <Button label="Next" className="p-button-primary" icon="pi pi-check" onClick={ this.Next }/>
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" style={{ width : '90px' }} />
+ </div>
+ </div>
+
+ </>
+ )
+ };
+
+
+}
+export default ProcessingDone
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/qa.reporting.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/qa.reporting.js
new file mode 100644
index 0000000000000000000000000000000000000000..1c4684e2c5b5b13e34cd1a2f87a27118c27f33f8
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/qa.reporting.js
@@ -0,0 +1,80 @@
+import React, { Component } from 'react';
+import { Button } from 'primereact/button';
+import SunEditor from 'suneditor-react';
+import 'suneditor/dist/css/suneditor.min.css'; // Import Sun Editor's CSS File
+import { Dropdown } from 'primereact/dropdown';
+//import katex from 'katex' // for mathematical operations on sun editor this component should be added
+//import 'katex/dist/katex.min.css'
+
+class QAreporting extends Component{
+
+ constructor(props) {
+ super(props);
+ this.state={
+ content: props.report
+ };
+ this.Next = this.Next.bind(this);
+ this.handleChange = this.handleChange.bind(this);
+ }
+
+ /**
+ * Method will trigger on click save buton
+ * here onNext props coming from parent, where will handle redirection to other page
+ */
+ Next() {
+ this.props.onNext({ report: this.state.content });
+ }
+
+ /**
+ * Method will trigger on change of operator report sun-editor
+ */
+ handleChange(e) {
+ localStorage.setItem('report_qa', e);
+ this.setState({ content: e });
+ }
+
+ //Not using at present
+ cancelCreate() {
+ this.props.history.goBack();
+ }
+
+ render() {
+ return (
+ <>
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <label htmlFor="assignTo" className="col-lg-2 col-md-2 col-sm-12">Assign To </label>
+ <div className="col-lg-3 col-md-3 col-sm-12" data-testid="assignTo" >
+ <Dropdown inputId="assignToValue" optionLabel="value" optionValue="value"
+ options={[{ value: 'User 1' }, { value: 'User 2' }, { value: 'User 3' }]}
+ placeholder="Assign To" />
+ </div>
+ </div>
+ <div className="p-grid" style={{ padding: '10px' }}>
+ <label htmlFor="comments" >Comments</label>
+ <div className="col-lg-12 col-md-12 col-sm-12"></div>
+ <SunEditor enableToolbar={true}
+ setDefaultStyle="min-height: 250px; height: auto;"
+ onChange={ this.handleChange }
+ setOptions={{
+ buttonList: [
+ ['undo', 'redo', 'bold', 'underline', 'fontColor', 'table', 'link', 'image', 'video', 'italic', 'strike', 'subscript',
+ 'superscript', 'outdent', 'indent', 'fullScreen', 'showBlocks', 'codeView', 'preview', 'print', 'removeFormat']
+ ]
+ }} />
+ </div>
+ </div>
+ <div className="p-grid p-justify-start">
+ <div className="p-col-1">
+ <Button label="Next" className="p-button-primary" icon="pi pi-check" onClick={ this.Next } />
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" style={{ width : '88px' }}/>
+ </div>
+ </div>
+ </>
+ )
+};
+
+}
+export default QAreporting;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/qa.sos.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/qa.sos.js
new file mode 100644
index 0000000000000000000000000000000000000000..59ef61e29ac4d7f2fe3eb7510fc290e65febff47
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Workflow/qa.sos.js
@@ -0,0 +1,97 @@
+import React, { Component } from 'react';
+import { Button } from 'primereact/button';
+import SunEditor from 'suneditor-react';
+import 'suneditor/dist/css/suneditor.min.css'; // Import Sun Editor's CSS File
+import { Checkbox } from 'primereact/checkbox';
+
+class QAreportingSDCO extends Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ content: props.report,
+ showEditor: false,
+ checked: false,
+ pichecked: false
+
+ };
+ this.Next = this.Next.bind(this);
+ this.handleChange = this.handleChange.bind(this);
+ }
+
+ /**
+ * Method will trigger on change of sun-editor
+ */
+ handleChange(e) {
+ this.setState({
+ content: e
+ });
+ localStorage.setItem('report_qa', e);
+ }
+
+ /**
+ * Method will trigger on click save buton
+ * here onNext props coming from parent, where will handle redirection to other page
+ */
+ Next() {
+ this.props.onNext({
+ report: this.state.content,
+ piChecked: this.state.pichecked
+ })
+ }
+
+ //Not using at present
+ cancelCreate() {
+ this.props.history.goBack();
+ }
+
+ render() {
+ return (
+ <>
+ <div>
+ <div className="p-fluid">
+ <div className="p-field p-grid">
+ <label htmlFor="qualityPolicy" className="col-lg-2 col-md-2 col-sm-12">Quality Policy</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <div className="p-field-checkbox">
+ <Checkbox inputId="binary" checked={this.state.checked} onChange={e => this.setState({ checked: e.checked })} />
+ </div>
+ </div>
+ <div className="col-lg-1 col-md-1 col-sm-12"></div>
+ <label htmlFor="sdcoAccept" className="col-lg-2 col-md-2 col-sm-12">SDCO Accept</label>
+ <div className="col-lg-3 col-md-3 col-sm-12">
+ <div className="p-field-checkbox">
+ <Checkbox inputId="secondary" pichecked={this.state.pichecked} onChange={e => this.setState({ pichecked: e.pichecked })} />
+ </div>
+ </div>
+ </div>
+ <div className="p-grid" style={{ padding: '10px' }}>
+ <label htmlFor="operatorReport" >Operator Report {!this.state.showEditor && <span className="con-edit">(Click content to edit)</span>}</label>
+ <div className="col-lg-12 col-md-12 col-sm-12"></div>
+ {this.state.showEditor && <SunEditor setDefaultStyle="min-height: 250px; height: auto" enableToolbar={true}
+ onChange={this.handleChange}
+ setContents={this.state.content}
+ setOptions={{
+ buttonList: [
+ ['undo', 'redo', 'bold', 'underline', 'fontColor', 'table', 'link', 'image', 'video', 'italic', 'strike', 'subscript',
+ 'superscript', 'outdent', 'indent', 'fullScreen', 'showBlocks', 'codeView', 'preview', 'print', 'removeFormat']
+ ]
+ }}
+ />}
+ {!this.state.showEditor && <div onClick={() => this.setState({ showEditor: !this.state.showEditor })} className="operator-report" dangerouslySetInnerHTML={{ __html: this.state.content }}></div>}
+ </div>
+ </div>
+ <div className="p-grid" style={{ marginTop: '20px' }}>
+ <div className="p-col-1">
+ <Button label="Next" className="p-button-primary" icon="pi pi-check" onClick={ this.Next } />
+ </div>
+ <div className="p-col-1">
+ <Button label="Cancel" className="p-button-danger" icon="pi pi-times" style={{ width : '90px' }} />
+ </div>
+ </div>
+ </div>
+ </>
+ )
+ };
+
+}
+export default QAreportingSDCO;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js
index a5c43e081e64a4c5801d01d50d1ef4ad9d4b35d7..156267acb0fad8c56c76a1c548a491683aacf184 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/index.js
@@ -2,7 +2,7 @@ import React from 'react';
import {
Route,
Switch,
- Redirect,
+ // Redirect,
} from 'react-router-dom';
import {NotFound} from '../layout/components/NotFound';
@@ -16,13 +16,14 @@ import EditSchedulingUnit from './Scheduling/edit';
import { CycleList, CycleCreate, CycleView, CycleEdit } from './Cycle';
import {TimelineView, WeekTimelineView} from './Timeline';
import SchedulingSetCreate from './Scheduling/create.scheduleset';
-import QAreporting from './Workflow/QAreporting';
+import Workflow from './Workflow';
+
+
export const routes = [
{
path: "/not-found",
component: NotFound,
-
},{
path: "/dashboard",
component: Dashboard,
@@ -154,17 +155,18 @@ export const routes = [
},
{
path: "/schedulingunit/:id/workflow",
- component: QAreporting,
- name: 'QA Reporting (TO)',
+ component: Workflow,
+ name: 'Workflow',
title: 'QA Reporting (TO)'
- }
+ },
+
];
export const RoutedContent = () => {
return (
<Switch>
- <Redirect from="/" to="/" exact />
- {routes.map(routeProps => <Route {...routeProps} exact key={routeProps.path} />)}
+ {/* <Redirect from="/" to="/" exact /> */}
+ {routes.map(routeProps => <Route {...routeProps} exact key={routeProps.path} />)}
</Switch>
);
}
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/auth.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/auth.service.js
new file mode 100644
index 0000000000000000000000000000000000000000..c6f2f964dab0fb646e110f7956e930d859d5539f
--- /dev/null
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/auth.service.js
@@ -0,0 +1,18 @@
+import Cookies from 'js-cookie';
+
+const axios = require('axios');
+delete axios.defaults.headers.common['Authorization'];
+const AuthService = {
+ authenticate: async() => {
+ try {
+ console.log(Cookies.get('csrftoken'));
+ const response = await axios.post("/accounts/login/", {csrfmiddlewaretoken: Cookies.get('csrftoken'), username: "test", password: "test"});
+ // const response = await axios.post("/accounts/login/", {username: "test", password: "test"});
+ console.log(response);
+ } catch(error) {
+ console.error(error);
+ }
+ }
+}
+
+export default AuthService;
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
index 1ace4022edceedeffc7a6916b1749edaf81cc4fa..3e7646d162cbd04337a6b55d24e4677976f1b408 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/schedule.service.js
@@ -8,24 +8,58 @@ axios.defaults.headers.common['Authorization'] = 'Basic dGVzdDp0ZXN0';
const ScheduleService = {
getSchedulingUnitDraft: async function (){
let res = [];
- await axios.get('/api/scheduling_unit_draft/?ordering=id')
- .then(response => {
- res= response;
- }).catch(function(error) {
+ try {
+ res = await axios.get('/api/scheduling_unit_draft/?limit=1');
+ if (res.data.count > res.data.results.length) {
+ res = await axios.get(`/api/scheduling_unit_draft/?ordering=id&limit=${res.data.count}&offset=0`);
+ }
+ } catch(error) {
console.error('[schedule.services.getSchedulingUnitDraft]',error);
- });
+ }
return res;
},
getSchedulingUnitBlueprint: async function (){
let res = [];
- await axios.get('/api/scheduling_unit_blueprint/?ordering=id')
- .then(response => {
- res= response;
- }).catch(function(error) {
+ try {
+ res = await axios.get('/api/scheduling_unit_blueprint/?limit=1');
+ if (res.data.count > res.data.results.length) {
+ res = await axios.get(`/api/scheduling_unit_blueprint/?ordering=id&limit=${res.data.count}&offset=0`);
+ }
+ } catch(error) {
console.error('[schedule.services.getSchedulingUnitBlueprint]',error);
- });
+ }
return res;
},
+ //>>>>>> TODO: Remove this method by using/modifying other functions with additional parameters
+ getTaskBPWithSubtaskTemplate: async function(id) {
+ let result;
+ try {
+ result = await axios.get('/api/task_blueprint/'+id);
+ if (result.data) {
+ result.data.template = await TaskService.getTaskTemplate(result.data.specifications_template_id);
+ }
+ if (result.data) {
+ let subTasks = [];
+ let subTasktemplate = {}
+ for (const subtaskId of result.data.subtasks_ids) {
+ const subTask = await TaskService.getSubtaskDetails(subtaskId);
+ //To avoid repeated api call for template if it has already loaded
+ if (subTasktemplate[subTask.specifications_template_id]) {
+ subTask.subTaskTemplate = subTasktemplate[subTask.specifications_template_id];
+ } else {
+ const subTaskTemplate = await TaskService.getSubtaskTemplate(subTask.specifications_template_id);
+ subTask.subTaskTemplate = subTaskTemplate;
+ subTasktemplate[subTask.specifications_template_id] = subTaskTemplate;
+ }
+ subTasks.push((subTask));
+ }
+ result.data.subTasks = subTasks;
+ }
+ } catch(error) {
+ console.error('[schedule.services.getTaskBlueprintById]',error);
+ }
+ return result;
+ },
getSchedulingUnitBlueprintById: async function (id){
try {
const response = await axios.get('/api/scheduling_unit_blueprint/'+id);
@@ -35,6 +69,7 @@ const ScheduleService = {
schedulingUnit.scheduling_set_id = schedulingUnitDraft.scheduling_set_id;
schedulingUnit.scheduling_set = schedulingUnitDraft.scheduling_set;
schedulingUnit.scheduling_set_object = schedulingUnitDraft.scheduling_set_object;
+ schedulingUnit.scheduling_constraints_doc = schedulingUnitDraft.scheduling_constraints_doc;
}
return schedulingUnit;
} catch(error) {
@@ -42,6 +77,15 @@ const ScheduleService = {
return null;
}
},
+ getTaskType: async function(){
+ try {
+ const response = await axios.get('/api/task_type');
+ return response.data.results;
+ } catch(error) {
+ console.error(error);
+ return null;
+ };
+ },
getSchedulingUnitDraftById: async function (id){
try {
const schedulingUnit = (await axios.get('/api/scheduling_unit_draft/'+id)).data;
@@ -91,7 +135,27 @@ const ScheduleService = {
}
return taskblueprintsList;
},
- getTasksBySchedulingUnit: async function(id, loadTemplate){
+ //>>>>>> TODO: Remove this method by using/modifying other functions with additional parameters
+ getTaskBPWithSubtaskTemplateOfSU: async function(scheduleunit){
+ // there no single api to fetch associated task_blueprint, so iterate the task_blueprint id to fetch associated task_blueprint
+ let taskblueprintsList = [];
+ if (scheduleunit.task_blueprints_ids){
+ for(const id of scheduleunit.task_blueprints_ids) {
+ await this.getTaskBPWithSubtaskTemplate(id).then(response =>{
+ let taskblueprint = response.data;
+ taskblueprint['tasktype'] = 'Blueprint';
+ taskblueprint['actionpath'] = '/task/view/blueprint/'+taskblueprint['id'];
+ taskblueprint['blueprint_draft'] = taskblueprint['draft'];
+ taskblueprint['relative_start_time'] = 0;
+ taskblueprint['relative_stop_time'] = 0;
+ taskblueprint.duration = moment.utc((taskblueprint.duration || 0)*1000).format('HH:mm:ss');
+ taskblueprintsList.push(taskblueprint);
+ })
+ }
+ }
+ return taskblueprintsList;
+ },
+ getTasksBySchedulingUnit: async function(id, loadTemplate, loadSubtasks, loadSubtaskTemplate){
let scheduletasklist=[];
// let taskblueprints = [];
// Common keys for Task and Blueprint
@@ -128,6 +192,7 @@ const ScheduleService = {
// if (o.draft_id === task['id']) return o;
// });
+ let subTasktemplate = {}
for(const blueprint of draftBlueprints){
let taskblueprint = [];
taskblueprint['tasktype'] = 'Blueprint';
@@ -146,6 +211,24 @@ const ScheduleService = {
if (loadTemplate) {
taskblueprint.template = scheduletask.template;
}
+ if (loadSubtasks) {
+ let subTasks = [];
+ for (const subtaskId of blueprint.subtasks_ids) {
+ const subTask = await TaskService.getSubtaskDetails(subtaskId);
+ if (loadSubtaskTemplate) {
+ //To avoid repeated api call for template if it has already loaded
+ if (subTasktemplate[subTask.specifications_template_id]) {
+ subTask.subTaskTemplate = subTasktemplate[subTask.specifications_template_id];
+ } else {
+ const subTaskTemplate = await TaskService.getSubtaskTemplate(subTask.specifications_template_id);
+ subTask.subTaskTemplate = subTaskTemplate;
+ subTasktemplate[subTask.specifications_template_id] = subTaskTemplate;
+ }
+ }
+ subTasks.push((subTask));
+ }
+ taskblueprint.subTasks = subTasks;
+ }
//Add Blue print details to array
scheduletasklist.push(taskblueprint);
}
@@ -287,7 +370,6 @@ const ScheduleService = {
},
updateSchedulingUnitDraft: async function(schedulingUnit) {
try {
- // console.log(schedulingUnit);
schedulingUnit.scheduling_constraints_doc = ( schedulingUnit.scheduling_constraints_doc == null)?"": schedulingUnit.scheduling_constraints_doc;
const suUpdateResponse = await axios.put(`/api/scheduling_unit_draft/${schedulingUnit.id}/`, schedulingUnit);
return suUpdateResponse.data;
@@ -350,6 +432,14 @@ const ScheduleService = {
console.error('[project.services.getSchedulingUnitBySet]',error);
}
},
+ createSchedulingUnitBlueprintTree: async function(id) {
+ try {
+ const response = await axios.get(`/api/scheduling_unit_draft/${id}/create_blueprints_and_subtasks`);
+ return response.data;
+ } catch(error) {
+ console.error(error);
+ }
+ },
getStationGroup: async function() {
try {
// const response = await axios.get('/api/station_type/');
diff --git a/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py b/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py
index 27c764a783f8fdf149ef3d961a7ef7a532673191..f9e2aab204e1b49dfddbb9c2019342a9150cbf9d 100644
--- a/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py
+++ b/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py
@@ -29,9 +29,11 @@ import logging
logger = logging.getLogger(__name__)
from datetime import datetime, timedelta
from dateutil import parser
+from astropy.coordinates import Angle
+import astropy.units
from lofar.sas.tmss.tmss.tmssapp import models
-from lofar.sas.tmss.tmss.tmssapp.conversions import create_astroplan_observer_for_station, Time, timestamps_and_stations_to_sun_rise_and_set, coordinates_and_timestamps_to_separation_from_bodies
+from lofar.sas.tmss.tmss.tmssapp.conversions import create_astroplan_observer_for_station, Time, timestamps_and_stations_to_sun_rise_and_set, coordinates_and_timestamps_to_separation_from_bodies, coordinates_timestamps_and_stations_to_target_rise_and_set
from lofar.sas.tmss.tmss.exceptions import TMSSException
from . import ScoredSchedulingUnit
@@ -109,7 +111,8 @@ def can_run_anywhere_within_timewindow_with_daily_constraints(scheduling_unit: m
if upper_bound < lower_bound:
raise ValueError("Provided upper_bound=%s is earlier than provided lower_bound=%s" % (upper_bound, lower_bound))
- stations = scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']['stations']
+ station_groups = scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']["station_groups"]
+ stations = list(set(sum([group['stations'] for group in station_groups], []))) # flatten all station_groups to single list
# check contraint and return false on first failure
for station in stations:
@@ -194,6 +197,9 @@ def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: mod
:return: True if all sky constraints are met over the entire time window, else False.
"""
constraints = scheduling_unit.draft.scheduling_constraints_doc
+ if not "sky" in constraints:
+ return True
+
for task in scheduling_unit.requirements_doc['tasks'].values():
if 'specifications_doc' in task:
if 'tile_beam' in task['specifications_doc']:
@@ -201,7 +207,8 @@ def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: mod
angle1 = beam['angle1']
angle2 = beam['angle2']
direction_type = beam['direction_type']
- if "sky" in constraints and 'min_distance' in constraints['sky']:
+
+ if 'min_distance' in constraints['sky']:
# currently we only check at bounds, we probably want to add some more samples in between later on
distances = coordinates_and_timestamps_to_separation_from_bodies(angle1=angle1, angle2=angle2, direction_type=direction_type, timestamps=(lower_bound, upper_bound), bodies=tuple(constraints['sky']['min_distance'].keys()))
for body, min_distance in constraints['sky']['min_distance'].items():
@@ -211,6 +218,28 @@ def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: mod
logger.info('Distance=%s from body=%s does not meet min_distance=%s constraint at timestamp=%s' % (angle.rad, body, min_distance, timestamp))
return False
+ if 'min_target_elevation' in constraints['sky'] and task['specifications_template'] == 'target observation' or \
+ 'min_calibrator_elevation' in constraints['sky'] and task['specifications_template'] == 'calibrator observation':
+ if task['specifications_template'] == 'calibrator observation':
+ min_elevation = Angle(constraints['sky']['min_calibrator_elevation'], unit=astropy.units.rad)
+ main_observation_task_name = get_target_observation_task_name_from_requirements_doc(scheduling_unit)
+ station_groups = scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']["station_groups"]
+ else:
+ min_elevation = Angle(constraints['sky']['min_target_elevation'], unit=astropy.units.rad)
+ station_groups = task['specifications_doc']['station_groups']
+ stations = list(set(sum([group['stations'] for group in station_groups], []))) # flatten all station_groups to single list
+ timestamps = (lower_bound, upper_bound)
+ # currently we only check at bounds, we probably want to add some more samples in between later on
+ target_rise_and_set_times = coordinates_timestamps_and_stations_to_target_rise_and_set(angle1=angle1, angle2=angle2, direction_type=direction_type, timestamps=timestamps, stations=tuple(stations), angle_to_horizon=min_elevation)
+ for station, times in target_rise_and_set_times.items():
+ for i in range(len(timestamps)):
+ if not (timestamps[i] > times[0]['rise'] and timestamps[i] < times[0]['set']):
+ if task['specifications_template'] == 'calibrator observation':
+ logger.info('min_calibrator_elevation=%s constraint is not met at timestamp=%s' % (min_elevation.rad, timestamps[i]))
+ else:
+ logger.info('min_target_elevation=%s constraint is not met at timestamp=%s' % (min_elevation.rad, timestamps[i]))
+ return False
+
return True
@@ -236,7 +265,8 @@ def get_earliest_possible_start_time(scheduling_unit: models.SchedulingUnitBluep
return parser.parse(constraints['time']['after'], ignoretz=True)
if constraints['daily']['require_day'] or constraints['daily']['require_night'] or constraints['daily']['avoid_twilight']:
- stations = scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']['stations']
+ station_groups = scheduling_unit.requirements_doc['tasks'][main_observation_task_name]['specifications_doc']["station_groups"]
+ stations = list(set(sum([group['stations'] for group in station_groups], []))) # flatten all station_groups to single list
all_sun_events = timestamps_and_stations_to_sun_rise_and_set(timestamps=(lower_bound,lower_bound+timedelta(days=1)), stations=tuple(stations))
start_time_per_station = {}
for station in stations:
diff --git a/SAS/TMSS/services/scheduling/test/t_dynamic_scheduling.py b/SAS/TMSS/services/scheduling/test/t_dynamic_scheduling.py
index 10cd3206512e03b8c52a20796ce75b46dfe10384..75d55fa8467b1cda9887ce2859bda2d617ff93f4 100755
--- a/SAS/TMSS/services/scheduling/test/t_dynamic_scheduling.py
+++ b/SAS/TMSS/services/scheduling/test/t_dynamic_scheduling.py
@@ -125,6 +125,7 @@ class TestDynamicScheduling(TestCase): # Note: we use django.test.TestCase inst
scheduling_unit_spec = add_defaults_to_json_object_for_schema(strategy_template.template,
strategy_template.scheduling_unit_template.schema)
scheduling_unit_spec['tasks']['Observation']['specifications_doc']['duration'] = obs_duration
+ scheduling_unit_spec['tasks']['Observation']['specifications_doc']['station_groups'][0]['stations'] = ['CS001']
# add the scheduling_unit_doc to a new SchedulingUnitDraft instance, and were ready to use it!
return models.SchedulingUnitDraft.objects.create(name=name,
@@ -360,7 +361,7 @@ class TestDailyConstraints(TestCase):
self.assertEqual(returned_time, self.sunrise_data['CS001']['day'][0]['start'])
def test_get_earliest_possible_start_time_with_daytime_constraint_returns_day_start_of_latest_station(self):
- self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['stations'] = ['CS001', 'DE601']
+ self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['station_groups'] = [{'stations': ['CS001', 'DE601']}]
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['require_day'] = True
self.scheduling_unit_blueprint.save()
self.sunrise_mock.return_value = self.sunrise_data_early_night
@@ -420,10 +421,13 @@ class TestDailyConstraints(TestCase):
lower_bound = datetime(2020, 1, 1, 8, 0, 0)
upper_bound = datetime(2020, 1, 1, 12, 0, 0)
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_daily_constraints(self.scheduling_unit_blueprint, lower_bound, upper_bound))
-
+
def test_can_run_within_timewindow_with_daytime_constraint_returns_correct_value(self):
# todo: for time ranges across dates, consider removing the mock for this because the moving window cannot be easily mocked
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky']['min_distance'] = {} # remove sky constraint
+ # remove other constraints:
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {}
+
+ # set constraint to test
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['require_day'] = True
self.scheduling_unit_blueprint.save()
@@ -449,7 +453,7 @@ class TestDailyConstraints(TestCase):
self.assertEqual(returned_time, self.sunrise_data['CS001']['night'][0]['start'])
def test_get_earliest_possible_start_time_with_nighttime_constraint_returns_night_start_of_latest_station(self):
- self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['stations'] = ['CS001', 'DE601']
+ self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['station_groups'] = [{'stations': ['CS001', 'DE601']}]
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['require_night'] = True
self.scheduling_unit_blueprint.save()
timestamp = datetime(2020, 1, 1, 14, 0, 0)
@@ -551,10 +555,13 @@ class TestDailyConstraints(TestCase):
lower_bound = datetime(2020, 1, 1, 3, 0, 0)
upper_bound = datetime(2020, 1, 1, 23, 0, 0)
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_daily_constraints(self.scheduling_unit_blueprint, lower_bound, upper_bound))
-
+
def test_can_run_within_timewindow_with_nighttime_constraint_returns_correct_value(self):
# todo: for time ranges across dates, consider removing the mock for this because the moving window cannot be easily mocked
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky']['min_distance'] = {} # remove sky constraint
+ # remove other constraints:
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {}
+
+ # set constraint to test
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['require_night'] = True
self.scheduling_unit_blueprint.save()
@@ -569,7 +576,7 @@ class TestDailyConstraints(TestCase):
lower_bound = datetime(2020, 1, 1, 15, 0, 0)
upper_bound = datetime(2020, 1, 1, 23, 0, 0)
self.assertTrue(can_run_within_timewindow(self.scheduling_unit_blueprint, lower_bound, upper_bound))
-
+
# avoid_twilight
@@ -583,7 +590,7 @@ class TestDailyConstraints(TestCase):
self.assertEqual(returned_time, self.sunrise_data['CS001']['day'][0]['start'])
def test_get_earliest_possible_start_time_with_twilight_constraint_returns_day_start_of_latest_station(self):
- self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['stations'] = ['CS001', 'DE601']
+ self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['station_groups'] = [{'stations': ['CS001', 'DE601']}]
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['avoid_twilight'] = True
self.scheduling_unit_blueprint.save()
@@ -602,7 +609,7 @@ class TestDailyConstraints(TestCase):
self.assertEqual(returned_time, self.sunrise_data['CS001']['night'][0]['start'])
def test_get_earliest_possible_start_time_with_twilight_constraint_returns_night_start_of_latest_station(self):
- self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['stations'] = ['CS001', 'DE601']
+ self.scheduling_unit_blueprint.requirements_doc['tasks']['Observation']['specifications_doc']['station_groups'] = [{'stations': ['CS001', 'DE601']}]
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['avoid_twilight'] = True
self.scheduling_unit_blueprint.save()
@@ -668,7 +675,6 @@ class TestDailyConstraints(TestCase):
self.assertFalse(tc1.can_run_anywhere_within_timewindow_with_daily_constraints(self.scheduling_unit_blueprint, lower_bound, upper_bound))
def test_can_run_anywhere_within_timewindow_with_daily_constraints_with_twilight_constraint_returns_false_when_partially_in_twilight(self):
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky']['min_distance'] = {} # remove sky constraint
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['avoid_twilight'] = True
self.scheduling_unit_blueprint.save()
@@ -684,7 +690,10 @@ class TestDailyConstraints(TestCase):
def test_can_run_within_timewindow_with_twilight_constraint_returns_correct_value(self):
# todo: for time ranges across dates, consider removing the mock for this because the moving window cannot be easily mocked
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky']['min_distance'] = {} # remove sky constraint
+ # remove other constraints:
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {}
+
+ # set constraint to test
self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['daily']['avoid_twilight'] = True
self.scheduling_unit_blueprint.save()
@@ -725,22 +734,45 @@ class TestSkyConstraints(unittest.TestCase):
self.distance_mock = self.distance_patcher.start()
self.distance_mock.return_value = self.distance_data
self.addCleanup(self.distance_patcher.stop)
+
+ self.target_rise_and_set_data = {"CS002": [{"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0)},
+ {"rise": datetime(2020, 1, 1, 8, 0, 0), "set": datetime(2020, 1, 1, 12, 30, 0)}]}
+ self.target_rise_and_set_patcher = mock.patch('lofar.sas.tmss.services.scheduling.constraints.template_constraints_v1.coordinates_timestamps_and_stations_to_target_rise_and_set')
+ self.target_rise_and_set_mock = self.target_rise_and_set_patcher.start()
+ self.target_rise_and_set_mock.return_value = self.target_rise_and_set_data
+ self.addCleanup(self.target_rise_and_set_patcher.stop)
# min_distance
def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_distance_constraint_returns_true_when_met(self):
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky']['min_distance'] = {'sun': 0.1, 'moon': 0.1, 'jupiter': 0.1}
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_distance': {'sun': 0.1, 'moon': 0.1, 'jupiter': 0.1}}
self.scheduling_unit_blueprint.save()
timestamp = datetime(2020, 1, 1, 10, 0, 0)
returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
self.assertTrue(returned_value)
def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_distance_constraint_returns_false_when_not_met(self):
- self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky']['min_distance'] = {'sun': 0.2, 'moon': 0.2, 'jupiter': 0.2}
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_distance': {'sun': 0.2, 'moon': 0.2, 'jupiter': 0.2}}
self.scheduling_unit_blueprint.save()
timestamp = datetime(2020, 1, 1, 10, 0, 0)
returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
self.assertFalse(returned_value)
+
+ # min_target_elevation
+
+ def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_constraint_returns_true_when_met(self):
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_target_elevation': 0.1}
+ self.scheduling_unit_blueprint.save()
+ timestamp = datetime(2020, 1, 1, 10, 0, 0)
+ returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
+ self.assertTrue(returned_value)
+
+ def test_can_run_anywhere_within_timewindow_with_sky_constraints_with_min_target_elevation_constraint_returns_false_when_not_met(self):
+ self.scheduling_unit_blueprint.draft.scheduling_constraints_doc['sky'] = {'min_target_elevation': 0.2}
+ self.scheduling_unit_blueprint.save()
+ timestamp = datetime(2020, 1, 1, 11, 0, 0)
+ returned_value = tc1.can_run_anywhere_within_timewindow_with_sky_constraints(self.scheduling_unit_blueprint, timestamp, timestamp + timedelta(seconds=self.obs_duration))
+ self.assertFalse(returned_value)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
diff --git a/SAS/TMSS/src/tmss/settings.py b/SAS/TMSS/src/tmss/settings.py
index 17ce37cd4ad04d34e3ed17887fb58f27b9319167..7c6334e185e7e757cad6d70c0d69e0ea5cedb546 100644
--- a/SAS/TMSS/src/tmss/settings.py
+++ b/SAS/TMSS/src/tmss/settings.py
@@ -92,7 +92,8 @@ INSTALLED_APPS = [
'drf_yasg',
'django_filters',
'material',
- 'material.frontend']
+ 'material.frontend'
+ ]
MIDDLEWARE = [
'django.middleware.gzip.GZipMiddleware',
diff --git a/SAS/TMSS/src/tmss/tmssapp/conversions.py b/SAS/TMSS/src/tmss/tmssapp/conversions.py
index 335b8937493b5c3e26fa9f0a80b798bee31107c0..40765b6998575b0cdccb3b1d11c113c527f64cb3 100644
--- a/SAS/TMSS/src/tmss/tmssapp/conversions.py
+++ b/SAS/TMSS/src/tmss/tmssapp/conversions.py
@@ -25,7 +25,8 @@ def create_astroplan_observer_for_station(station: str) -> Observer:
# default angle to the horizon at which the sunset/sunrise starts and ends, as per LOFAR definition.
SUN_SET_RISE_ANGLE_TO_HORIZON = Angle(10, unit=astropy.units.deg)
-SUN_SET_RISE_PRECISION = 30 # n_grid_points; higher is more precise but very costly; astropy defaults to 150, errors now can be in the minutes, increase if this is not good enough
+# default n_grid_points; higher is more precise but very costly; astropy defaults to 150, errors now can be in the minutes, increase if this is not good enough
+SUN_SET_RISE_PRECISION = 30
@lru_cache(maxsize=256, typed=False) # does not like lists, so use tuples to allow caching
def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tuple, angle_to_horizon: Angle=SUN_SET_RISE_ANGLE_TO_HORIZON) -> dict:
@@ -35,6 +36,7 @@ def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tup
The night is usually the one _starting_ on the date of the time stamp, unless the given timestamp falls before sunrise, in which case it is the night _ending_ on the timestamp date.
:param timestamps: tuple of datetimes, e.g. (datetime(2020, 1, 1), datetime(2020, 1, 2))
:param stations: tuple of station names, e.g. ("CS002",)
+ :param angle_to_horizon: the angle between horizon and given coordinates for which rise and set times are returned
:return A dict that maps station names to a nested dict that contains lists of start and end times for sunrise, sunset, etc, on each requested date.
E.g.
{"CS002":
@@ -55,7 +57,7 @@ def timestamps_and_stations_to_sun_rise_and_set(timestamps: tuple, stations: tup
# todo: this can probably be made faster by moving the following logic to an own function with single station/timestamp as input and putting the lru_cache on there.
# This also means that we have to strip the time from the datetime. Can this be safely done?
observer = create_astroplan_observer_for_station(station)
- sunrise_start = observer.sun_rise_time(time=Time(datetime.combine(timestamp.date(), dtime(12,0,0))), which='previous', n_grid_points=SUN_SET_RISE_PRECISION)
+ sunrise_start = observer.sun_rise_time(time=Time(datetime.combine(timestamp.date(), dtime(12,0,0))), horizon=-angle_to_horizon, which='previous', n_grid_points=SUN_SET_RISE_PRECISION)
sunrise_end = observer.sun_rise_time(time=Time(sunrise_start), horizon=angle_to_horizon, which='next', n_grid_points=SUN_SET_RISE_PRECISION)
sunset_start = observer.sun_set_time(time=sunrise_end, horizon=angle_to_horizon, which='next', n_grid_points=SUN_SET_RISE_PRECISION)
sunset_end = observer.sun_set_time(time=sunset_start, horizon=-angle_to_horizon, which='next', n_grid_points=SUN_SET_RISE_PRECISION)
@@ -91,7 +93,7 @@ def coordinates_and_timestamps_to_separation_from_bodies(angle1: float, angle2:
}
"""
if direction_type == "J2000":
- coord = astropy.coordinates.SkyCoord(ra=angle1, dec=angle2, unit=astropy.units.deg)
+ coord = astropy.coordinates.SkyCoord(ra=angle1, dec=angle2, unit=astropy.units.rad)
else:
raise ValueError("Do not know how to convert direction_type=%s to SkyCoord" % direction_type)
return_dict = {}
@@ -105,6 +107,48 @@ def coordinates_and_timestamps_to_separation_from_bodies(angle1: float, angle2:
return return_dict
+# default angle above horizon, above which the target it reporte as 'up'
+TARGET_SET_RISE_ANGLE_TO_HORIZON = Angle(0, unit=astropy.units.deg) # if default should be non-zero, should we include it explicitly in response?
+# default n_grid_points; higher is more precise but very costly; astropy defaults to 150, note that errors can be in the minutes with a lower values
+TARGET_SET_RISE_PRECISION = 150
+
+@lru_cache(maxsize=256, typed=False) # does not like lists, so use tuples to allow caching
+def coordinates_timestamps_and_stations_to_target_rise_and_set(angle1: float, angle2: float, direction_type: str, timestamps: tuple, stations: tuple, angle_to_horizon: Angle=TARGET_SET_RISE_ANGLE_TO_HORIZON) -> dict:
+ """
+ Compute rise and set times of the given coordinates above the provided horizon, for each given station and timestamp.
+ The set time is always the one following the provided timestamp.
+ This implies that if the target is up at a given timestamp, the surrounding rise and set times are returned.
+ Otherwise both rise and set times follow the timestamp.
+ :param angle1: first angle of celectial coordinates, e.g. RA
+ :param angle2: second angle of celectial coordinates, e.g. Dec
+ :param direction_type: direction_type of celectial coordinates, e.g. 'J2000'
+ :param timestamps: tuple of datetimes, e.g. (datetime(2020, 1, 1), datetime(2020, 1, 2))
+ :param stations: tuple of station names, e.g. ("CS002",)
+ :param angle_to_horizon: the angle between horizon and given coordinates for which rise and set times are returned
+ :return A dict that maps station names to a list of dicts with rise and set times for each requested date.
+ E.g.
+ {"CS002": [{"rise": datetime(2020, 1, 1, 4, 0, 0), "set": datetime(2020, 1, 1, 11, 0, 0)},
+ {"rise": datetime(2020, 1, 2, 4, 0, 0), "set": datetime(2020, 1, 2, 11, 0, 0)}]
+ }
+ """
+ if direction_type == "J2000":
+ coord = astropy.coordinates.SkyCoord(ra=angle1, dec=angle2, unit=astropy.units.rad)
+ else:
+ raise ValueError("Do not know how to convert direction_type=%s to SkyCoord" % direction_type)
+ return_dict = {}
+ for station in stations:
+ for timestamp in timestamps:
+ # todo: this can probably be made faster by moving the following logic to an own function with single station/timestamp as input and putting the lru_cache on there.
+ observer = create_astroplan_observer_for_station(station)
+ target_set = observer.target_set_time(target=coord, time=Time(timestamp), horizon=angle_to_horizon, which='next', n_grid_points=TARGET_SET_RISE_PRECISION)
+ target_rise = observer.target_rise_time(target=coord, time=Time(target_set), horizon=angle_to_horizon, which='previous', n_grid_points=TARGET_SET_RISE_PRECISION)
+
+ return_dict.setdefault(station, []).append({"rise": target_rise.to_datetime(), "set": target_set.to_datetime()})
+
+ return return_dict
+
+
+
def local_sidereal_time_for_utc_and_station(timestamp: datetime = None,
station: str = 'CS002',
field: str = 'LBA',
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index a657a6531ec6a608c64768ac2905a436e20bec7d..691a5c9ce300985aea3eb03552bc75c72f2c7aa4 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2020-11-30 08:59
+# Generated by Django 3.0.9 on 2020-12-03 10:10
from django.conf import settings
import django.contrib.postgres.fields
@@ -428,7 +428,7 @@ class Migration(migrations.Migration):
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
('description', models.CharField(help_text='Short description for this reservation, used in overviews', max_length=255)),
('start_time', models.DateTimeField(help_text='Start of this reservation.')),
- ('duration', models.IntegerField(help_text='Duration of this reservations.', null=True)),
+ ('duration', models.IntegerField(help_text='Duration of this reservation (in seconds). If null, then this reservation is indefinitely.', null=True)),
('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Properties of this reservation')),
],
options={
@@ -1200,7 +1200,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='reservation',
name='project',
- field=models.ForeignKey(help_text='Reservation will be accounted for this project.', on_delete=django.db.models.deletion.CASCADE, related_name='reservations', to='tmssapp.Project'),
+ field=models.ForeignKey(help_text='Reservation will be accounted for this project.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='reservations', to='tmssapp.Project'),
),
migrations.AddField(
model_name='reservation',
@@ -1473,4 +1473,4 @@ class Migration(migrations.Migration):
model_name='dataproduct',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_dat_tags_5932a3_gin'),
),
- ]
\ No newline at end of file
+ ]
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
index 1f5a6e3b5be0631f4c5b99ee9e4b6a2176556623..b3629f35cfd18d93ccf77af17911b7f9928271cd 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
@@ -385,34 +385,6 @@ class DefaultReservationTemplate(BasicCommon):
template = ForeignKey("ReservationTemplate", on_delete=PROTECT)
-#
-# DatabaseView objects
-#
-class TaskBlueprintSummary(Model):
- taskblueprint_id = IntegerField()
- subtask_id = IntegerField()
- substate = CharField(max_length=128)
- subtask_type = CharField(max_length=128)
-
- class Meta:
- managed = False
- db_table = 'tmssapp_taskblueprintsummary'
-
-
-class SchedulingUnitBlueprintSummary(Model):
- # Using in an id and ForeignKey is not common for a view BUT the id is a 'dummy' to be able to use in Django
- # https://resources.rescale.com/using-database-views-in-django-orm/
- # otherwise an exception will be thrown
- id = IntegerField(primary_key=True)
- sub_id = IntegerField()
- taskblueprint_id = IntegerField()
- task_type = CharField(max_length=128)
- derived_task_status = CharField(max_length=128)
-
- class Meta:
- managed = False
- db_table = 'tmssapp_schedulingunitblueprintsummary'
-
#
# Instance Objects
#
@@ -1124,13 +1096,20 @@ class TaskSchedulingRelationDraft(BasicCommon):
class Reservation(NamedCommon):
- project = ForeignKey('Project', related_name='reservations', on_delete=CASCADE, help_text='Reservation will be accounted for this project.')
+ project = ForeignKey('Project', null=True, related_name='reservations', on_delete=CASCADE, help_text='Reservation will be accounted for this project.')
description = CharField(max_length=255, help_text='Short description for this reservation, used in overviews')
start_time = DateTimeField(help_text='Start of this reservation.')
- duration = IntegerField(null=True, help_text='Duration of this reservations.')
+ duration = IntegerField(null=True, help_text='Duration of this reservation (in seconds). If null, then this reservation is indefinitely.')
specifications_doc = JSONField(help_text='Properties of this reservation')
specifications_template = ForeignKey('ReservationTemplate', on_delete=CASCADE, help_text='Schema used for specifications_doc.')
+ @property
+ def stop_time(self) -> datetime.datetime:
+ '''The stop_time based on start_time+duration if duration is known, else None'''
+ if self.duration:
+ return self.start_time + datetime.timedelta(seconds=self.duration)
+ return None
+
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json b/SAS/TMSS/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json
index cfa908a68e642538b03c398a65e6d752f2e81db2..3a9536713768ac5ff4ddb93874dcea024dcdf9ee 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/simple-observation-scheduling-unit-observation-strategy.json
@@ -24,7 +24,9 @@
},
"antenna_set": "HBA_DUAL_INNER",
"filter": "HBA_110_190",
- "stations": ["CS001"],
+ "station_groups": [ {
+ "stations": ["CS002", "CS003", "CS004", "CS005", "CS006", "CS007"]
+ }],
"tile_beam": {
"direction_type": "J2000",
"angle1": 0.42,
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
index 19afb2076dfec6e7b2644021680e1750d90db36e..55eab8b6118553a53fd2ec6f9e4e9b15cf405532 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
@@ -399,6 +399,9 @@ class DefaultReservationTemplateSerializer(RelationalHyperlinkedModelSerializer)
class ReservationSerializer(RelationalHyperlinkedModelSerializer):
+ specifications_doc = JSONEditorField(schema_source='specifications_template.schema')
+
class Meta:
model = models.Reservation
fields = '__all__'
+ extra_fields = ['stop_time']
diff --git a/SAS/TMSS/src/tmss/tmssapp/views.py b/SAS/TMSS/src/tmss/tmssapp/views.py
index b099d553958475e0d7410dc7bef529503b7f9d4d..8dabf0b06f1967e925ea8fac41e80afb84e31387 100644
--- a/SAS/TMSS/src/tmss/tmssapp/views.py
+++ b/SAS/TMSS/src/tmss/tmssapp/views.py
@@ -15,7 +15,9 @@ from django.apps import apps
from rest_framework.decorators import api_view
from datetime import datetime
import dateutil.parser
-from lofar.sas.tmss.tmss.tmssapp.conversions import local_sidereal_time_for_utc_and_station, local_sidereal_time_for_utc_and_longitude, timestamps_and_stations_to_sun_rise_and_set, coordinates_and_timestamps_to_separation_from_bodies
+from astropy.coordinates import Angle
+import astropy.units
+from lofar.sas.tmss.tmss.tmssapp.conversions import local_sidereal_time_for_utc_and_station, local_sidereal_time_for_utc_and_longitude, timestamps_and_stations_to_sun_rise_and_set, coordinates_and_timestamps_to_separation_from_bodies, coordinates_timestamps_and_stations_to_target_rise_and_set
# Note: Decorate with @api_view to get this picked up by Swagger
@@ -216,3 +218,54 @@ def get_angular_separation_from_bodies(request):
new_sep_dict.setdefault(body, {})[timestamp.isoformat()] = angle.rad
return JsonResponse(new_sep_dict)
+
+
+
+@permission_classes([AllowAny])
+@authentication_classes([AllowAny])
+@swagger_auto_schema(method='GET',
+ responses={200: 'A JSON object with rise and set times of the given coordinates above the provided horizon, for each given station and timestamp.'},
+ operation_description="Get rise and set times of the given coordinates above the provided horizon, for each given station and timestamp. \n\n"
+ "Example request: /api/util/target_rise_and_set?angle1=0.5&angle2=0.5×tamps=2020-01-01T15&horizon=0.3",
+ manual_parameters=[Parameter(name='angle1', required=True, type='string', in_='query',
+ description="first angle of celectial coordinates as float, e.g. RA"),
+ Parameter(name='angle2', required=True, type='string', in_='query',
+ description="second angle of celectial coordinates as float, e.g. RA"),
+ Parameter(name='direction_type', required=False, type='string', in_='query',
+ description="direction_type of celectial coordinates as string, e.g. J2000"),
+ Parameter(name='timestamps', required=False, type='string', in_='query',
+ description="comma-separated list of isoformat timestamps"),
+ Parameter(name='stations', required=False, type='string', in_='query',
+ description="comma-separated list of station names"),
+ Parameter(name='horizon', required=False, type='string', in_='query',
+ description="Elevation above horizon for which to return rise/set times as float")])
+@api_view(['GET'])
+def get_target_rise_and_set(request):
+ '''
+ returns rise and set times of the given coordinates above the provided horizon, for each given station and timestamp.
+ '''
+ timestamps = request.GET.get('timestamps', None)
+ angle1 = request.GET.get('angle1')
+ angle2 = request.GET.get('angle2')
+ direction_type = request.GET.get("direction_type", "J2000")
+ stations = tuple(request.GET.get('stations', "CS002").split(','))
+ horizon = request.GET.get('horizon', None)
+
+ if angle1 is None or angle2 is None:
+ raise ValueError("Please provide celestial coordinates via 'angle1', 'angle2' (and optionally 'direction_type') properties.")
+
+ if timestamps is None:
+ timestamps = (datetime.utcnow(),)
+ else:
+ timestamps = timestamps.split(',')
+ timestamps = tuple([dateutil.parser.parse(timestamp, ignoretz=True) for timestamp in timestamps]) # isot to datetime
+
+ if horizon is None:
+ horizon = Angle(0, unit=astropy.units.rad)
+ else:
+ horizon = Angle(horizon, unit=astropy.units.rad)
+
+ # calculate
+ rise_set_dict = coordinates_timestamps_and_stations_to_target_rise_and_set(angle1=angle1, angle2=angle2, direction_type=direction_type, angle_to_horizon=horizon, timestamps=timestamps, stations=stations)
+ return JsonResponse(rise_set_dict)
+
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index 81258f3d806a1fd7937d85240769739c532fbe9e..dda3767daec991e0fe14e9f8330270c2249b8ef3 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -71,6 +71,7 @@ urlpatterns = [
re_path('util/utc/?', views.utc, name="system-utc"),
re_path('util/lst/?', views.lst, name="conversion-lst"),
re_path('util/angular_separation_from_bodies/?', views.get_angular_separation_from_bodies, name='get_angular_separation_from_bodies'),
+ re_path('util/target_rise_and_set/?', views.get_target_rise_and_set, name='get_target_rise_and_set'),
]
if os.environ.get('SHOW_DJANGO_DEBUG_TOOLBAR', False):
@@ -237,6 +238,7 @@ if bool(os.environ.get('TMSS_ENABLE_VIEWFLOW', False)):
viewflow_router.register('scheduling_unit_flow/qa_pi_verification', workflow_viewsets.PIVerificationViewSet, basename='qa_pi_verification')
viewflow_router.register('scheduling_unit_flow/qa_decide_acceptance', workflow_viewsets.DecideAcceptanceViewSet, basename='qa_decide_acceptance')
viewflow_router.register('scheduling_unit_flow/qa_scheduling_unit_process', workflow_viewsets.SchedulingUnitProcessViewSet, basename='qa_scheduling_unit_process')
+ viewflow_router.register('scheduling_unit_flow/qa_scheduling_unit_task', workflow_viewsets.SchedulingUnitTaskViewSet, basename='qa_scheduling_unit_task')
viewflow_urlpatterns.extend(viewflow_router.urls)
diff --git a/SAS/TMSS/src/tmss/workflowapp/apps.py b/SAS/TMSS/src/tmss/workflowapp/apps.py
index 3ba8ab2cdb3c7e994ebf6bea7850ddc6128b9428..b4d7b07f0d004048aa3fac119f0354ee4bf5a38e 100644
--- a/SAS/TMSS/src/tmss/workflowapp/apps.py
+++ b/SAS/TMSS/src/tmss/workflowapp/apps.py
@@ -4,5 +4,5 @@ from django.apps import AppConfig
class WorkflowappConfig(AppConfig):
- name = 'lofar.sas.tmss.tmss.workflowapp'
+ name = 'workflowapp'
diff --git a/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py
index bcd8c2bdd182d7c82f438054dea08940bf124282..0bf572d1e31c9c7817a845a12d8d6abdbbb23f8f 100644
--- a/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py
@@ -66,6 +66,7 @@ class Condition(Signal):
activation.prepare()
if activation.flow_task.condition_check(activation, instance):
activation.done()
+
def ready(self):
"""Resolve internal `this`-references. and subscribe to the signal."""
@@ -205,7 +206,6 @@ class SchedulingUnitFlow(Flow):
logger.info("End of schedulingunit workflow: can_delete: %s, results_accepted: %s", activation.process.can_delete, activation.process.results_accepted)
return activation
-
def check_condition_scheduled(self, activation, instance):
if instance is None:
instance = activation.process.su
diff --git a/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py
index aa03e1a4d75db411590a57f4ec385e3e0d39bd10..8119f3254e3b5d89bd1593f16b715b9d6f2d0d7a 100644
--- a/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2020-11-26 09:54
+# Generated by Django 3.0.9 on 2020-12-02 20:16
from django.db import migrations, models
import django.db.models.deletion
@@ -9,6 +9,7 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
+ ('tmssapp', '0001_initial'),
('viewflow', '0008_jsonfield_and_artifact'),
]
@@ -45,19 +46,6 @@ class Migration(migrations.Migration):
('operator_accept', models.BooleanField(default=False)),
],
),
- migrations.CreateModel(
- name='HelloWorldProcess',
- fields=[
- ],
- options={
- 'verbose_name': 'World Request',
- 'verbose_name_plural': 'World Requests',
- 'proxy': True,
- 'indexes': [],
- 'constraints': [],
- },
- bases=('viewflow.process',),
- ),
migrations.CreateModel(
name='SchedulingUnitProcess',
fields=[
diff --git a/SAS/TMSS/src/tmss/workflowapp/models/__init__.py b/SAS/TMSS/src/tmss/workflowapp/models/__init__.py
index a0ae3713747c0b28c5595736d06f4bcb800da5b5..bfdfbc84e07beb363937412fd7fb6d5788c684d0 100644
--- a/SAS/TMSS/src/tmss/workflowapp/models/__init__.py
+++ b/SAS/TMSS/src/tmss/workflowapp/models/__init__.py
@@ -1,2 +1 @@
-from .helloworldflow import *
from .schedulingunitflow import *
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py
index c883b4fc9a3c857e2c2913df064c629d16239619..d33f462ed653833794709d701e3d0e0be47f05a4 100644
--- a/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py
@@ -1,7 +1,9 @@
# Create your models here.
from django.db.models import CharField, IntegerField,BooleanField, ForeignKey, CASCADE, Model,NullBooleanField
-from viewflow.models import Process
+from viewflow.models import Process, Task
+from viewflow.fields import FlowReferenceField
+from viewflow.compat import _
from lofar.sas.tmss.tmss.tmssapp.models import SchedulingUnitBlueprint
diff --git a/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py
index 884061c224168ac706def57a833c5ee9cb7952bd..694f7f7310cdf9476f1c32d5219737584e5b368f 100644
--- a/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py
@@ -4,6 +4,8 @@ from lofar.sas.tmss.tmss.workflowapp import models
from django.views import generic
from django.forms.models import modelform_factory
+from viewflow.models import Task
+
from .. import forms
@@ -31,5 +33,10 @@ class DecideAcceptanceSerializer(ModelSerializer):
class SchedulingUnitProcessSerializer(ModelSerializer):
class Meta:
- model = models.SchedulingUnitProcess
- fields = '__all__'
\ No newline at end of file
+ model = models.SchedulingUnitProcess
+ fields = '__all__'
+
+class SchedulingUnitTaskSerializer(ModelSerializer):
+ class Meta:
+ model = Task
+ fields = '__all__'
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py
index 530d4cb8f9c0207c4c12212c1a4bce7832aa8d29..acaa7459631c3341df848686df8f8ba371f2dbd4 100644
--- a/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py
@@ -11,6 +11,7 @@ from viewflow.flow.views.utils import get_next_task_url
from django.forms import CharField, CheckboxInput
from django.forms.models import modelform_factory
+from viewflow.models import Task
from .. import forms, models, serializers
@@ -36,6 +37,10 @@ class SchedulingUnitProcessViewSet(viewsets.ModelViewSet):
queryset = models.SchedulingUnitProcess.objects.all()
serializer_class = serializers.SchedulingUnitProcessSerializer
+class SchedulingUnitTaskViewSet(viewsets.ModelViewSet):
+ queryset = Task.objects.all()
+ serializer_class = serializers.SchedulingUnitTaskSerializer
+
class QAReportingTOView(FlowMixin, generic.CreateView):
template_name = 'qa_reporting.html'
model = models.QAReportingTO
diff --git a/SAS/TMSS/test/t_conversions.py b/SAS/TMSS/test/t_conversions.py
index 18865051aecdd7bd80946e68ef80487e58f8b815..d3f005999089466c6deceaf0745c1df416d77719 100755
--- a/SAS/TMSS/test/t_conversions.py
+++ b/SAS/TMSS/test/t_conversions.py
@@ -76,6 +76,8 @@ class SiderealTime(unittest.TestCase):
class UtilREST(unittest.TestCase):
+ # utc
+
def test_util_utc_returns_timestamp(self):
# assert local clock differs not too much from returned TMSS system clock
@@ -86,6 +88,8 @@ class UtilREST(unittest.TestCase):
delta = abs((returned_datetime - current_datetime).total_seconds())
self.assertTrue(delta < 60.0)
+ # lst
+
def test_util_lst_returns_longitude(self):
# assert returned value is a parseable hms value
@@ -128,6 +132,8 @@ class UtilREST(unittest.TestCase):
lon_str2 = r2.content.decode('utf8')
self.assertNotEqual(lon_str1, lon_str2)
+ # sun_rise_and_set
+
def test_util_sun_rise_and_set_returns_json_structure_with_defaults(self):
r = requests.get(BASE_URL + '/util/sun_rise_and_set', auth=AUTH)
self.assertEqual(r.status_code, 200)
@@ -196,6 +202,7 @@ class UtilREST(unittest.TestCase):
response_date = dateutil.parser.parse(r_dict['CS002']['night'][1]['start']).date()
self.assertEqual(expected_date, response_date)
+ # angular_separation_from_bodies
def test_util_angular_separation_from_bodies_yields_error_when_no_pointing_is_given(self):
r = requests.get(BASE_URL + '/util/angular_separation_from_bodies', auth=AUTH)
@@ -265,6 +272,94 @@ class UtilREST(unittest.TestCase):
self.assertNotEqual(angle, angle_last)
angle_last = angle
+ # target_rise_and_set
+
+ def test_util_target_rise_and_set_returns_json_structure_with_defaults(self):
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.5', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+
+ # defaults are CS002 and today
+ self.assertIn('CS002', r_dict.keys())
+
+ # assert day of timestamp matches day of returned rise
+ expected_date = datetime.date.today()
+ target_rise = dateutil.parser.parse(r_dict['CS002'][0]['rise'])
+ target_set = dateutil.parser.parse(r_dict['CS002'][0]['set'])
+ self.assertTrue(expected_date == target_rise.date() or expected_date == target_set.date())
+
+ def test_util_target_rise_and_set_considers_stations(self):
+ stations = ['CS005', 'RS305', 'DE609']
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.5&stations=%s' % ','.join(stations), auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+
+ # assert station is included in response and timestamps differ
+ target_rise_last = None
+ for station in stations:
+ self.assertIn(station, r_dict.keys())
+ target_rise = dateutil.parser.parse(r_dict[station][0]['rise'])
+ if target_rise_last:
+ self.assertNotEqual(target_rise, target_rise_last)
+ target_rise_last = target_rise
+
+ def test_util_target_rise_and_set_considers_timestamps(self):
+ timestamps = ['2020-01-01', '2020-02-22T16-00-00', '2020-3-11', '2020-01-01']
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.5×tamps=%s' % ','.join(timestamps), auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+
+ # assert all requested timestamps are included in response and either rise or set are in that day
+ for i in range(len(timestamps)):
+ expected_date = dateutil.parser.parse(timestamps[i]).date()
+ response_rise_date = dateutil.parser.parse(r_dict['CS002'][i]['rise']).date()
+ response_set_date = dateutil.parser.parse(r_dict['CS002'][i]['set']).date()
+ self.assertTrue(expected_date == response_rise_date or expected_date == response_set_date)
+
+ def test_util_target_rise_and_set_returns_correct_date_of_target_rise_and_set(self):
+ timestamps = ['2020-01-01T02-00-00']
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.5×tamps=%s' % ','.join(timestamps), auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+
+ # assert day of timestamp matches day of returned rise
+ expected_date = dateutil.parser.parse(timestamps[0]).date()
+ target_rise = dateutil.parser.parse(r_dict['CS002'][0]['rise'])
+ target_set = dateutil.parser.parse(r_dict['CS002'][0]['set'])
+ self.assertTrue(expected_date == target_rise.date() or expected_date == target_set.date())
+
+ # assert set time falls in the 24h after rise time
+ self.assertTrue(target_set - target_rise > datetime.timedelta(0) and target_set - target_rise < datetime.timedelta(days=1))
+
+ def test_util_target_rise_and_set_considers_coordinates(self):
+ test_coords = [(0.5, 0.5, "J2000"), (0.6, 0.5, "J2000"), (0.6, 0.6, "J2000")]
+ for coords in test_coords:
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=%s&angle2=%s&direction_type=%s' % coords, auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+
+ # assert all requested coordinates yield a response and times differ
+ rise_last = None
+ rise = r_dict['CS002'][0]['rise']
+ if rise_last:
+ self.assertNotEqual(rise, rise_last)
+ rise_last = rise
+
+ def test_util_target_rise_and_set_considers_horizon(self):
+ test_horizons = [0.1, 0.2, 0.3]
+ for horizon in test_horizons:
+ r = requests.get(BASE_URL + '/util/target_rise_and_set?angle1=0.5&angle2=0.5&horizon=%s' % horizon, auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ r_dict = json.loads(r.content.decode('utf-8'))
+
+ # assert all requested horizons yield a response and times differ
+ rise_last = None
+ rise = r_dict['CS002'][0]['rise']
+ if rise_last:
+ self.assertNotEqual(rise, rise_last)
+ rise_last = rise
+
+
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
unittest.main()
diff --git a/SAS/TriggerServices/test/t_trigger_service.in/trigger_testing_20_03_17.xml b/SAS/TriggerServices/test/t_trigger_service.in/trigger_testing_20_03_17.xml
index c6e358f95aaba7a66a17ea74442fee3b7ee18829..6abdbb35067fa91805a317956c2f3b9c3075c8d9 100644
--- a/SAS/TriggerServices/test/t_trigger_service.in/trigger_testing_20_03_17.xml
+++ b/SAS/TriggerServices/test/t_trigger_service.in/trigger_testing_20_03_17.xml
@@ -1,278 +1,278 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<p:trigger xsi:schemaLocation="http://www.astron.nl/LofarTrigger LofarTrigger.xsd"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p2="http://www.astron.nl/LofarSpecification"
- xmlns:p="http://www.astron.nl/LofarTrigger" xmlns:p1="http://www.astron.nl/LofarBase">
- <version>version</version>
- <name>name</name>
- <description>description</description>
- <projectReference>
- <ProjectCode>test-lofar</ProjectCode>
- </projectReference>
- <contactInformation>
- <name>name</name>
- <email>email</email>
- <phoneNumber>phoneNumber</phoneNumber>
- <affiliation>affiliation</affiliation>
- </contactInformation>
- <userName>userName</userName>
- <comment>comment</comment>
- <event>
- <identification>identification</identification>
- <description>description</description>
- <type>VOEvent</type>
- </event>
- <specification>
- <version>version</version>
- <projectReference>
- <ProjectCode>test-lofar</ProjectCode>
- </projectReference>
- <userName>userName</userName>
- <comment>comment</comment>
- <generatorName>generatorVersion</generatorName>
- <generatorVersion>generatorVersion</generatorVersion>
- <container>
- <temporaryIdentifier>
- <source>source</source>
- <identifier>100</identifier>
- <description>description</description>
- </temporaryIdentifier>
- <addToExistingContainer>false</addToExistingContainer>
- <folder>
- <name>100_name</name>
- <description>description</description>
- <topology>topology</topology>
- </folder>
- </container>
- <container>
- <temporaryIdentifier>
- <source>source</source>
- <identifier>101</identifier>
- <description>description</description>
- </temporaryIdentifier>
- <addToExistingContainer>false</addToExistingContainer>
- <folder>
- <name>101_name</name>
- <description>description</description>
- <topology>topology</topology>
- </folder>
- </container>
- <container>
- <temporaryIdentifier>
- <source>source</source>
- <identifier>110</identifier>
- <description>description</description>
- </temporaryIdentifier>
- <addToExistingContainer>false</addToExistingContainer>
- <folder>
- <name>110_name</name>
- <description>description</description>
- <topology>topology</topology>
- </folder>
- </container>
- <container>
- <temporaryIdentifier>
- <source>source</source>
- <identifier>111</identifier>
- <description>description</description>
- </temporaryIdentifier>
- <addToExistingContainer>false</addToExistingContainer>
- <folder>
- <name>111_name</name>
- <description>description</description>
- <topology>topology</topology>
- </folder>
- </container>
- <activity>
- <temporaryIdentifier>
- <source>source</source>
- <identifier>200</identifier>
- <description>description</description>
- </temporaryIdentifier>
- <observation>
- <name>name</name>
- <description>description</description>
- <processingCluster>
- <name>name</name>
- <partition>partition</partition>
- <numberOfTasks>0</numberOfTasks>
- <minRAMPerTask unit="byte">0</minRAMPerTask>
- <minScratchPerTask unit="byte">0</minScratchPerTask>
- <maxDurationPerTask>P1D</maxDurationPerTask>
- <numberOfCoresPerTask>0</numberOfCoresPerTask>
- <runSimultaneous>true</runSimultaneous>
- </processingCluster>
- <instrument>Beam Observation</instrument>
- <defaultTemplate>defaultTemplate</defaultTemplate>
- <tbbPiggybackAllowed>true</tbbPiggybackAllowed>
- <aartfaacPiggybackAllowed>true</aartfaacPiggybackAllowed>
- <correlatedData>true</correlatedData>
- <filteredData>true</filteredData>
- <beamformedData>true</beamformedData>
- <coherentStokesData>true</coherentStokesData>
- <incoherentStokesData>true</incoherentStokesData>
- <antenna>HBA Zero</antenna>
- <clock units="MHz">160</clock>
- <instrumentFilter>10-70 MHz</instrumentFilter>
- <integrationInterval>0.0</integrationInterval>
- <channelsPerSubband>0</channelsPerSubband>
- <coherentDedisperseChannels>true</coherentDedisperseChannels>
- <tiedArrayBeams>
- <flyseye>true</flyseye>
- <nrTabRings>0</nrTabRings>
- <tabRingSize>0.0</tabRingSize>
- <tiedArrayBeamList>
- <tiedArrayBeam index="0">
- <coherent>true</coherent>
- <angle1>0.0</angle1>
- <angle2>0.0</angle2>
- </tiedArrayBeam>
- </tiedArrayBeamList>
- </tiedArrayBeams>
- <stokes>
- <integrateChannels>true</integrateChannels>
- <subbandsPerFileCS>0</subbandsPerFileCS>
- <numberCollapsedChannelsCS>0</numberCollapsedChannelsCS>
- <stokesDownsamplingStepsCS>0</stokesDownsamplingStepsCS>
- <whichCS>I</whichCS>
- <subbandsPerFileIS>0</subbandsPerFileIS>
- <numberCollapsedChannelsIS>0</numberCollapsedChannelsIS>
- <stokesDownsamplingStepsIS>0</stokesDownsamplingStepsIS>
- <whichIS>I</whichIS>
- </stokes>
- <bypassPff>true</bypassPff>
- <enableSuperterp>true</enableSuperterp>
- <numberOfBitsPerSample>0</numberOfBitsPerSample>
- <stationSelectionSpecification>
- <stationSelection>
- <stationSet>Single</stationSet>
- </stationSelection>
- </stationSelectionSpecification>
- <timeWindowSpecification>
- <timeFrame />
- <duration>
- <duration>P6Y</duration>
- </duration>
- </timeWindowSpecification>
- </observation>
- <status>opened</status>
- <qualityOfService>THROUGHPUT</qualityOfService>
- <priority>10</priority>
- <triggerId>
- <source>source</source>
- <identifier>0</identifier>
- <description>description</description>
- </triggerId>
- </activity>
- <activity>
- <temporaryIdentifier>
- <source>source</source>
- <identifier>201</identifier>
- <description>description</description>
- </temporaryIdentifier>
- <pipeline xsi:type="p1:AveragingPipeline">
- <name>pipe</name>
- <description>description</description>
- <processingCluster>
- <name>name</name>
- <partition>partition</partition>
- <numberOfTasks>0</numberOfTasks>
- <minRAMPerTask unit="byte">0</minRAMPerTask>
- <minScratchPerTask unit="byte">0</minScratchPerTask>
- <maxDurationPerTask>P1D</maxDurationPerTask>
- <numberOfCoresPerTask>0</numberOfCoresPerTask>
- <runSimultaneous>true</runSimultaneous>
- </processingCluster>
- <defaultTemplate>default</defaultTemplate>
- </pipeline>
- <status>opened</status>
- <qualityOfService>THROUGHPUT</qualityOfService>
- <priority>10</priority>
- <triggerId>
- <source>source</source>
- <identifier>0</identifier>
- <description>description</description>
- </triggerId>
- </activity>
- <entity>
- <temporaryIdentifier>
- <source>source</source>
- <identifier>0</identifier>
- <description>description</description>
- </temporaryIdentifier>
- <dataproductType>UVDataProduct</dataproductType>
- <storageCluster>
- <name>name</name>
- <partition>partition</partition>
- </storageCluster>
- </entity>
- <relation xsi:type="p2:ChildRelation">
- <parent>
- <source>source</source>
- <identifier>100</identifier>
- <description>description</description>
- </parent>
- <child>
- <source>source</source>
- <identifier>101</identifier>
- <description>description</description>
- </child>
- <type>folder-folder</type>
- </relation>
- <relation xsi:type="p2:ChildRelation">
- <parent>
- <source>source</source>
- <identifier>101</identifier>
- <description>description</description>
- </parent>
- <child>
- <source>source</source>
- <identifier>110</identifier>
- <description>description</description>
- </child>
- <type>folder-folder</type>
- </relation>
- <relation xsi:type="p2:ChildRelation">
- <parent>
- <source>source</source>
- <identifier>111</identifier>
- <description>description</description>
- </parent>
- <child>
- <source>source</source>
- <identifier>200</identifier>
- <description>description</description>
- </child>
- <type>folder-activity</type>
- </relation>
- <relation xsi:type="p2:ChildRelation">
- <parent>
- <source>source</source>
- <identifier>111</identifier>
- <description>description</description>
- </parent>
- <child>
- <source>source</source>
- <identifier>201</identifier>
- <description>description</description>
- </child>
- <type>folder-activity</type>
- </relation>
- <relation xsi:type="p2:ChildRelation">
- <parent>
- <source>source</source>
- <identifier>110</identifier>
- <description>description</description>
- </parent>
- <child>
- <source>source</source>
- <identifier>111</identifier>
- <description>description</description>
- </child>
- <type>folder-folder</type>
- </relation>
- </specification>
- <generatorName></generatorName>
- <generatorVersion></generatorVersion>
-</p:trigger>
-
-
+<?xml version="1.0" encoding="UTF-8"?>
+<p:trigger xsi:schemaLocation="http://www.astron.nl/LofarTrigger LofarTrigger.xsd"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p2="http://www.astron.nl/LofarSpecification"
+ xmlns:p="http://www.astron.nl/LofarTrigger" xmlns:p1="http://www.astron.nl/LofarBase">
+ <version>version</version>
+ <name>name</name>
+ <description>description</description>
+ <projectReference>
+ <ProjectCode>test-lofar</ProjectCode>
+ </projectReference>
+ <contactInformation>
+ <name>name</name>
+ <email>email</email>
+ <phoneNumber>phoneNumber</phoneNumber>
+ <affiliation>affiliation</affiliation>
+ </contactInformation>
+ <userName>userName</userName>
+ <comment>comment</comment>
+ <event>
+ <identification>identification</identification>
+ <description>description</description>
+ <type>VOEvent</type>
+ </event>
+ <specification>
+ <version>version</version>
+ <projectReference>
+ <ProjectCode>test-lofar</ProjectCode>
+ </projectReference>
+ <userName>userName</userName>
+ <comment>comment</comment>
+ <generatorName>generatorVersion</generatorName>
+ <generatorVersion>generatorVersion</generatorVersion>
+ <container>
+ <temporaryIdentifier>
+ <source>source</source>
+ <identifier>100</identifier>
+ <description>description</description>
+ </temporaryIdentifier>
+ <addToExistingContainer>false</addToExistingContainer>
+ <folder>
+ <name>100_name</name>
+ <description>description</description>
+ <topology>topology</topology>
+ </folder>
+ </container>
+ <container>
+ <temporaryIdentifier>
+ <source>source</source>
+ <identifier>101</identifier>
+ <description>description</description>
+ </temporaryIdentifier>
+ <addToExistingContainer>false</addToExistingContainer>
+ <folder>
+ <name>101_name</name>
+ <description>description</description>
+ <topology>topology</topology>
+ </folder>
+ </container>
+ <container>
+ <temporaryIdentifier>
+ <source>source</source>
+ <identifier>110</identifier>
+ <description>description</description>
+ </temporaryIdentifier>
+ <addToExistingContainer>false</addToExistingContainer>
+ <folder>
+ <name>110_name</name>
+ <description>description</description>
+ <topology>topology</topology>
+ </folder>
+ </container>
+ <container>
+ <temporaryIdentifier>
+ <source>source</source>
+ <identifier>111</identifier>
+ <description>description</description>
+ </temporaryIdentifier>
+ <addToExistingContainer>false</addToExistingContainer>
+ <folder>
+ <name>111_name</name>
+ <description>description</description>
+ <topology>topology</topology>
+ </folder>
+ </container>
+ <activity>
+ <temporaryIdentifier>
+ <source>source</source>
+ <identifier>200</identifier>
+ <description>description</description>
+ </temporaryIdentifier>
+ <observation>
+ <name>name</name>
+ <description>description</description>
+ <processingCluster>
+ <name>name</name>
+ <partition>partition</partition>
+ <numberOfTasks>0</numberOfTasks>
+ <minRAMPerTask unit="byte">0</minRAMPerTask>
+ <minScratchPerTask unit="byte">0</minScratchPerTask>
+ <maxDurationPerTask>P1D</maxDurationPerTask>
+ <numberOfCoresPerTask>0</numberOfCoresPerTask>
+ <runSimultaneous>true</runSimultaneous>
+ </processingCluster>
+ <instrument>Beam Observation</instrument>
+ <defaultTemplate>defaultTemplate</defaultTemplate>
+ <tbbPiggybackAllowed>true</tbbPiggybackAllowed>
+ <aartfaacPiggybackAllowed>true</aartfaacPiggybackAllowed>
+ <correlatedData>true</correlatedData>
+ <filteredData>true</filteredData>
+ <beamformedData>true</beamformedData>
+ <coherentStokesData>true</coherentStokesData>
+ <incoherentStokesData>true</incoherentStokesData>
+ <antenna>HBA Zero</antenna>
+ <clock units="MHz">160</clock>
+ <instrumentFilter>10-70 MHz</instrumentFilter>
+ <integrationInterval>0.0</integrationInterval>
+ <channelsPerSubband>0</channelsPerSubband>
+ <coherentDedisperseChannels>true</coherentDedisperseChannels>
+ <tiedArrayBeams>
+ <flyseye>true</flyseye>
+ <nrTabRings>0</nrTabRings>
+ <tabRingSize>0.0</tabRingSize>
+ <tiedArrayBeamList>
+ <tiedArrayBeam index="0">
+ <coherent>true</coherent>
+ <angle1>0.0</angle1>
+ <angle2>0.0</angle2>
+ </tiedArrayBeam>
+ </tiedArrayBeamList>
+ </tiedArrayBeams>
+ <stokes>
+ <integrateChannels>true</integrateChannels>
+ <subbandsPerFileCS>0</subbandsPerFileCS>
+ <numberCollapsedChannelsCS>0</numberCollapsedChannelsCS>
+ <stokesDownsamplingStepsCS>0</stokesDownsamplingStepsCS>
+ <whichCS>I</whichCS>
+ <subbandsPerFileIS>0</subbandsPerFileIS>
+ <numberCollapsedChannelsIS>0</numberCollapsedChannelsIS>
+ <stokesDownsamplingStepsIS>0</stokesDownsamplingStepsIS>
+ <whichIS>I</whichIS>
+ </stokes>
+ <bypassPff>true</bypassPff>
+ <enableSuperterp>true</enableSuperterp>
+ <numberOfBitsPerSample>0</numberOfBitsPerSample>
+ <stationSelectionSpecification>
+ <stationSelection>
+ <stationSet>Single</stationSet>
+ </stationSelection>
+ </stationSelectionSpecification>
+ <timeWindowSpecification>
+ <timeFrame />
+ <duration>
+ <duration>P6Y</duration>
+ </duration>
+ </timeWindowSpecification>
+ </observation>
+ <status>opened</status>
+ <qualityOfService>THROUGHPUT</qualityOfService>
+ <priority>10</priority>
+ <triggerId>
+ <source>source</source>
+ <identifier>0</identifier>
+ <description>description</description>
+ </triggerId>
+ </activity>
+ <activity>
+ <temporaryIdentifier>
+ <source>source</source>
+ <identifier>201</identifier>
+ <description>description</description>
+ </temporaryIdentifier>
+ <pipeline xsi:type="p1:AveragingPipeline">
+ <name>pipe</name>
+ <description>description</description>
+ <processingCluster>
+ <name>name</name>
+ <partition>partition</partition>
+ <numberOfTasks>0</numberOfTasks>
+ <minRAMPerTask unit="byte">0</minRAMPerTask>
+ <minScratchPerTask unit="byte">0</minScratchPerTask>
+ <maxDurationPerTask>P1D</maxDurationPerTask>
+ <numberOfCoresPerTask>0</numberOfCoresPerTask>
+ <runSimultaneous>true</runSimultaneous>
+ </processingCluster>
+ <defaultTemplate>default</defaultTemplate>
+ </pipeline>
+ <status>opened</status>
+ <qualityOfService>THROUGHPUT</qualityOfService>
+ <priority>10</priority>
+ <triggerId>
+ <source>source</source>
+ <identifier>0</identifier>
+ <description>description</description>
+ </triggerId>
+ </activity>
+ <entity>
+ <temporaryIdentifier>
+ <source>source</source>
+ <identifier>0</identifier>
+ <description>description</description>
+ </temporaryIdentifier>
+ <dataproductType>UVDataProduct</dataproductType>
+ <storageCluster>
+ <name>name</name>
+ <partition>partition</partition>
+ </storageCluster>
+ </entity>
+ <relation xsi:type="p2:ChildRelation">
+ <parent>
+ <source>source</source>
+ <identifier>100</identifier>
+ <description>description</description>
+ </parent>
+ <child>
+ <source>source</source>
+ <identifier>101</identifier>
+ <description>description</description>
+ </child>
+ <type>folder-folder</type>
+ </relation>
+ <relation xsi:type="p2:ChildRelation">
+ <parent>
+ <source>source</source>
+ <identifier>101</identifier>
+ <description>description</description>
+ </parent>
+ <child>
+ <source>source</source>
+ <identifier>110</identifier>
+ <description>description</description>
+ </child>
+ <type>folder-folder</type>
+ </relation>
+ <relation xsi:type="p2:ChildRelation">
+ <parent>
+ <source>source</source>
+ <identifier>111</identifier>
+ <description>description</description>
+ </parent>
+ <child>
+ <source>source</source>
+ <identifier>200</identifier>
+ <description>description</description>
+ </child>
+ <type>folder-activity</type>
+ </relation>
+ <relation xsi:type="p2:ChildRelation">
+ <parent>
+ <source>source</source>
+ <identifier>111</identifier>
+ <description>description</description>
+ </parent>
+ <child>
+ <source>source</source>
+ <identifier>201</identifier>
+ <description>description</description>
+ </child>
+ <type>folder-activity</type>
+ </relation>
+ <relation xsi:type="p2:ChildRelation">
+ <parent>
+ <source>source</source>
+ <identifier>110</identifier>
+ <description>description</description>
+ </parent>
+ <child>
+ <source>source</source>
+ <identifier>111</identifier>
+ <description>description</description>
+ </child>
+ <type>folder-folder</type>
+ </relation>
+ </specification>
+ <generatorName></generatorName>
+ <generatorVersion></generatorVersion>
+</p:trigger>
+
+