diff --git a/.clang-tidy b/.clang-tidy index af9b18e704e61c996f61e622faeaf4a8d6ed44e0..8d0b6833f07ac0cea4f92f77b4c5106f3e68b21a 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: '-*,clang-diagnostic-*,clang-analyzer-*,-*,modernize*,performance*,readability*,bugprone*,clang-analyzer*,cppcoreguidelines*,misc*,-readability-braces-around-statements,-cppcoreguidelines-pro-bounds-array-to-pointer-decay' +Checks: '-*,clang-diagnostic-*,clang-analyzer-*,-*,modernize*,performance*,readability*,bugprone*,clang-analyzer*,cppcoreguidelines*,misc*,-readability-braces-around-statements,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-modernize-use-trailing-return-type' WarningsAsErrors: '' HeaderFilterRegex: '(src|test)/.*' AnalyzeTemporaryDtors: false diff --git a/CMakeLists.txt b/CMakeLists.txt index 47082ba0870e643886060e0d852180ca3cd984de..5ba59181c9a5407a9108459663216d639ff342c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,9 +38,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) # Build options -set(FETCH_LIBHDBPP_TAG "exp-refactor" CACHE STRING "Libhdbpp branch/tag to clone 'master'") -option(FETCH_LIBHDBPP "Download and build using a local copy of libhdb++" ON) -option(FETCH_LIBHDBPP_TAG "When FETCH_LIBHDBPP is enabled, this is the tag fetch ('master')") option(BUILD_UNIT_TESTS "Build unit tests" OFF) option(BUILD_BENCHMARK_TESTS "Build benchmarking tests (Forces RELEASE build)" OFF) option(ENABLE_CLANG "Enable clang code and layout analysis" OFF) diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 14e78c0fa104c7610389436bdcf08bbc0e1c5dc8..61faa9cce135e69fa8cb929a63c03f13b3c75025 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -3,29 +3,60 @@ project(benchmark-tests) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_COLOR_MAKEFILE ON) -set(BENCHMARK_SOURCES - ${CMAKE_CURRENT_SOURCE_DIR}/QueryBuilderTests.cpp) +add_executable(query-builder-tests ${CMAKE_CURRENT_SOURCE_DIR}/QueryBuilderTests.cpp) +target_compile_options(query-builder-tests PRIVATE -Wall -Wextra -g) + +target_link_libraries(query-builder-tests + PRIVATE + libhdbpp_timescale_static_library + TangoInterfaceLibrary + benchmark + benchmark_main + gtest + test-utils) + +target_include_directories(query-builder-tests + PRIVATE ${CMAKE_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}) -add_executable(benchmark-tests ${BENCHMARK_SOURCES}) -target_compile_options(benchmark-tests PRIVATE -Wall -Wextra -g) +target_compile_definitions(query-builder-tests + PRIVATE -DDEBUG_ENABLED) -target_link_libraries(benchmark-tests - PRIVATE libhdbpp_headers libhdbpp_timescale_shared_library TangoInterfaceLibrary benchmark gtest) +set_target_properties(query-builder-tests + PROPERTIES + LINK_FLAGS "-Wl,--no-undefined" + CXX_STANDARD 14) -target_include_directories(benchmark-tests +if(DO_CLANG_TIDY) + set_target_properties(query-builder-tests + PROPERTIES + CXX_CLANG_TIDY ${DO_CLANG_TIDY}) +endif(DO_CLANG_TIDY) + +add_executable(db-insert-tests ${CMAKE_CURRENT_SOURCE_DIR}/DbInsertionTests.cpp) +target_compile_options(db-insert-tests PRIVATE -Wall -Wextra -g) + +target_link_libraries(db-insert-tests + PRIVATE + libhdbpp_timescale_static_library + TangoInterfaceLibrary + benchmark + benchmark_main + gtest + test-utils) + +target_include_directories(db-insert-tests PRIVATE ${CMAKE_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}) -target_compile_definitions(benchmark-tests +target_compile_definitions(db-insert-tests PRIVATE -DDEBUG_ENABLED) -set_target_properties(benchmark-tests +set_target_properties(db-insert-tests PROPERTIES LINK_FLAGS "-Wl,--no-undefined" CXX_STANDARD 14) if(DO_CLANG_TIDY) - set_target_properties(unit-tests + set_target_properties(db-insert-tests PROPERTIES CXX_CLANG_TIDY ${DO_CLANG_TIDY}) endif(DO_CLANG_TIDY) - diff --git a/benchmark/DbInsertionTests.cpp b/benchmark/DbInsertionTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fb64d453db377943d6d6328c825cb9a7b4ffdf95 --- /dev/null +++ b/benchmark/DbInsertionTests.cpp @@ -0,0 +1,566 @@ +/* Copyright (C) : 2014-2019 + European Synchrotron Radiation Facility + BP 220, Grenoble 38043, FRANCE + + This file is part of libhdb++timescale. + + libhdb++timescale is free software: you can redistribute it and/or modify + it under the terms of the Lesser GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + libhdb++timescale is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Lesser + GNU General Public License for more details. + + You should have received a copy of the Lesser GNU General Public License + along with libhdb++timescale. If not, see <http://www.gnu.org/licenses/>. */ + +#include "DbConnection.hpp" +#include "HdbppDefines.hpp" +#include "QueryBuilder.hpp" +#include "TestHelpers.hpp" + +#include <benchmark/benchmark.h> +#include <memory> + +//============================================================================= +//============================================================================= +void clearTables() +{ + auto conn = make_unique<pqxx::connection>(hdbpp_test::psql_connection::postgres_db::HdbppConnectionString); + + auto traits_array = hdbpp_test::utils::getTraits(); + + { + auto query = string("TRUNCATE "); + + pqxx::work tx {*conn}; + + for (auto &traits : traits_array) + { + query += hdbpp_internal::pqxx_conn::QueryBuilder::tableName(traits); + query += ","; + } + + query += hdbpp_internal::pqxx_conn::schema::ErrTableName + ","; + query += hdbpp_internal::pqxx_conn::schema::ParamTableName + ","; + query += hdbpp_internal::pqxx_conn::schema::HistoryEventTableName + ","; + query += hdbpp_internal::pqxx_conn::schema::HistoryTableName + ","; + query += hdbpp_internal::pqxx_conn::schema::ConfTableName + " RESTART IDENTITY"; + + tx.exec(query); + tx.commit(); + } +} + +//============================================================================= +//============================================================================= +template<Tango::CmdArgType Type, Tango::AttrDataFormat Format, int Size = 0> +void bmAllocateData(benchmark::State &state) +{ + // TEST - Test data allocation time, so we can subtract it from the db write + // tests + hdbpp_internal::LogConfigurator::initLogging("test"); + hdbpp_internal::AttributeTraits traits {Tango::READ_WRITE, Format, Type}; + + for (auto _ : state) + { + for (int i = 0; i < 1000; i++) + { + if (Format == Tango::SPECTRUM) + { + auto value_r = hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size); + auto value_w = hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size); + } + else + { + auto value_r = hdbpp_test::data_gen::generateData<Type>(traits); + auto value_w = hdbpp_test::data_gen::generateData<Type>(traits); + } + } + } +} + +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_BOOLEAN, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_SHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_LONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_LONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_FLOAT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_DOUBLE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_UCHAR, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_USHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_ULONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_ULONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_STRING, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_STATE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); + +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_BOOLEAN, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_SHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_LONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_LONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_FLOAT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_DOUBLE, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_UCHAR, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_USHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_ULONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_ULONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_STRING, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmAllocateData, Tango::DEV_STATE, Tango::SCALAR)->Unit(benchmark::kMillisecond); + +//============================================================================= +//============================================================================= +template<Tango::CmdArgType Type, Tango::AttrDataFormat Format, int Size = 0> +void bmDbMultipleSQLInsert(benchmark::State &state) +{ + // TEST - Test the write speed when pushing a single event at a time + // to the database. This version inserts via strings + hdbpp_internal::LogConfigurator::initLogging("test"); + clearTables(); + + hdbpp_internal::AttributeTraits traits {Tango::READ, Format, Type}; + + hdbpp_internal::pqxx_conn::DbConnection conn(hdbpp_internal::pqxx_conn::DbConnection::DbStoreMethod::InsertString); + + conn.connect(hdbpp_test::psql_connection::postgres_db::HdbppConnectionString); + + conn.storeAttribute(hdbpp_test::attr_name::TestAttrFinalName, + hdbpp_test::attr_name::TestAttrCs, + hdbpp_test::attr_name::TestAttrDomain, + hdbpp_test::attr_name::TestAttrFamily, + hdbpp_test::attr_name::TestAttrMember, + hdbpp_test::attr_name::TestAttrName, + 0, + traits); + + struct timeval tv + {}; + + for (auto _ : state) + { + for (int i = 0; i < 1000; i++) + { + gettimeofday(&tv, nullptr); + double event_time = tv.tv_sec + tv.tv_usec / 1.0e6; + + if (Format == Tango::SPECTRUM) + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + traits); + } + else + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateData<Type>(traits)), + move(hdbpp_test::data_gen::generateData<Type>(traits)), + traits); + } + } + } + + conn.disconnect(); +} + +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_BOOLEAN, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_SHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_LONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_LONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_FLOAT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_DOUBLE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_UCHAR, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_USHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_ULONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_ULONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_STRING, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_STATE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); + +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_BOOLEAN, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_SHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_LONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_LONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_FLOAT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_DOUBLE, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_UCHAR, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_USHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_ULONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_ULONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_STRING, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultipleSQLInsert, Tango::DEV_STATE, Tango::SCALAR)->Unit(benchmark::kMillisecond); + +//============================================================================= +//============================================================================= +template<Tango::CmdArgType Type, Tango::AttrDataFormat Format, int Size = 0> +void bmDbMultiplePreparedStatementInsert(benchmark::State &state) +{ + // TEST - Test the write speed when pushing a single event at a time + // to the database. This version inserts via prepared statements + hdbpp_internal::LogConfigurator::initLogging("test"); + clearTables(); + + hdbpp_internal::AttributeTraits traits {Tango::READ, Format, Type}; + + hdbpp_internal::pqxx_conn::DbConnection conn( + hdbpp_internal::pqxx_conn::DbConnection::DbStoreMethod::PreparedStatement); + + conn.connect(hdbpp_test::psql_connection::postgres_db::HdbppConnectionString); + + conn.storeAttribute(hdbpp_test::attr_name::TestAttrFinalName, + hdbpp_test::attr_name::TestAttrCs, + hdbpp_test::attr_name::TestAttrDomain, + hdbpp_test::attr_name::TestAttrFamily, + hdbpp_test::attr_name::TestAttrMember, + hdbpp_test::attr_name::TestAttrName, + 0, + traits); + + struct timeval tv + {}; + + for (auto _ : state) + { + for (int i = 0; i < 1000; i++) + { + gettimeofday(&tv, nullptr); + double event_time = tv.tv_sec + tv.tv_usec / 1.0e6; + + if (Format == Tango::SPECTRUM) + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + traits); + } + else + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateData<Type>(traits)), + move(hdbpp_test::data_gen::generateData<Type>(traits)), + traits); + } + } + } + + conn.disconnect(); +} + +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_BOOLEAN, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_SHORT, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_LONG, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_LONG64, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_FLOAT, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_DOUBLE, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_UCHAR, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_USHORT, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_ULONG, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_ULONG64, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_STRING, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_STATE, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_BOOLEAN, Tango::SCALAR) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_SHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_LONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_LONG64, Tango::SCALAR) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_FLOAT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_DOUBLE, Tango::SCALAR) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_UCHAR, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_USHORT, Tango::SCALAR) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_ULONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_ULONG64, Tango::SCALAR) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_STRING, Tango::SCALAR) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbMultiplePreparedStatementInsert, Tango::DEV_STATE, Tango::SCALAR)->Unit(benchmark::kMillisecond); + +//============================================================================= +//============================================================================= +template<Tango::CmdArgType Type, Tango::AttrDataFormat Format, int Size = 0> +void bmDbBatchedInsert(benchmark::State &state) +{ + // TEST - Test the write speed when pushing a multiple events at once to + // the db + hdbpp_internal::LogConfigurator::initLogging("test"); + clearTables(); + + hdbpp_internal::AttributeTraits traits {Tango::READ, Format, Type}; + + hdbpp_internal::pqxx_conn::DbConnection conn( + hdbpp_internal::pqxx_conn::DbConnection::DbStoreMethod::PreparedStatement); + + conn.connect(hdbpp_test::psql_connection::postgres_db::HdbppConnectionString); + conn.buffer(true); + + conn.storeAttribute(hdbpp_test::attr_name::TestAttrFinalName, + hdbpp_test::attr_name::TestAttrCs, + hdbpp_test::attr_name::TestAttrDomain, + hdbpp_test::attr_name::TestAttrFamily, + hdbpp_test::attr_name::TestAttrMember, + hdbpp_test::attr_name::TestAttrName, + 0, + traits); + + struct timeval tv + {}; + + for (auto _ : state) + { + for (int i = 0; i < 1000; i++) + { + gettimeofday(&tv, nullptr); + double event_time = tv.tv_sec + tv.tv_usec / 1.0e6; + + if (Format == Tango::SPECTRUM) + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + traits); + } + else + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateData<Type>(traits)), + move(hdbpp_test::data_gen::generateData<Type>(traits)), + traits); + } + } + + conn.flush(); + } + + conn.buffer(false); + conn.disconnect(); +} + +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_BOOLEAN, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_SHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_LONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_LONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_FLOAT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_DOUBLE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_UCHAR, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_USHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_ULONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_ULONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_STRING, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_STATE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); + +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_BOOLEAN, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_SHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_LONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_LONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_FLOAT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_DOUBLE, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_UCHAR, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_USHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_ULONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_ULONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_STRING, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbBatchedInsert, Tango::DEV_STATE, Tango::SCALAR)->Unit(benchmark::kMillisecond); + +//============================================================================= +//============================================================================= +template<Tango::CmdArgType Type, Tango::AttrDataFormat Format, int Size = 0> +void bmDbSingleSQLInsert(benchmark::State &state) +{ + // TEST - Test the write speed when pushing a single event via sql + hdbpp_internal::LogConfigurator::initLogging("test"); + clearTables(); + + hdbpp_internal::AttributeTraits traits {Tango::READ, Format, Type}; + + hdbpp_internal::pqxx_conn::DbConnection conn(hdbpp_internal::pqxx_conn::DbConnection::DbStoreMethod::InsertString); + + conn.connect(hdbpp_test::psql_connection::postgres_db::HdbppConnectionString); + + conn.storeAttribute(hdbpp_test::attr_name::TestAttrFinalName, + hdbpp_test::attr_name::TestAttrCs, + hdbpp_test::attr_name::TestAttrDomain, + hdbpp_test::attr_name::TestAttrFamily, + hdbpp_test::attr_name::TestAttrMember, + hdbpp_test::attr_name::TestAttrName, + 0, + traits); + + struct timeval tv + {}; + + for (auto _ : state) + { + gettimeofday(&tv, nullptr); + double event_time = tv.tv_sec + tv.tv_usec / 1.0e6; + + if (Format == Tango::SPECTRUM) + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + traits); + } + else + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateData<Type>(traits)), + move(hdbpp_test::data_gen::generateData<Type>(traits)), + traits); + } + } + + conn.buffer(false); + conn.disconnect(); +} + +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_BOOLEAN, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_SHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_LONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_LONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_FLOAT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_DOUBLE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_UCHAR, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_USHORT, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_ULONG, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_ULONG64, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_STRING, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_STATE, Tango::SPECTRUM, 512)->Unit(benchmark::kMillisecond); + +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_BOOLEAN, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_SHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_LONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_LONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_FLOAT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_DOUBLE, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_UCHAR, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_USHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_ULONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_ULONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_STRING, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSingleSQLInsert, Tango::DEV_STATE, Tango::SCALAR)->Unit(benchmark::kMillisecond); + +//============================================================================= +//============================================================================= +template<Tango::CmdArgType Type, Tango::AttrDataFormat Format, int Size = 0> +void bmDbSinglePreparedStatementInsert(benchmark::State &state) +{ + // TEST - Test the write speed when pushing a single event via sql + hdbpp_internal::LogConfigurator::initLogging("test"); + clearTables(); + + hdbpp_internal::AttributeTraits traits {Tango::READ, Format, Type}; + + hdbpp_internal::pqxx_conn::DbConnection conn( + hdbpp_internal::pqxx_conn::DbConnection::DbStoreMethod::PreparedStatement); + + conn.connect(hdbpp_test::psql_connection::postgres_db::HdbppConnectionString); + + conn.storeAttribute(hdbpp_test::attr_name::TestAttrFinalName, + hdbpp_test::attr_name::TestAttrCs, + hdbpp_test::attr_name::TestAttrDomain, + hdbpp_test::attr_name::TestAttrFamily, + hdbpp_test::attr_name::TestAttrMember, + hdbpp_test::attr_name::TestAttrName, + 0, + traits); + + struct timeval tv + {}; + + for (auto _ : state) + { + gettimeofday(&tv, nullptr); + double event_time = tv.tv_sec + tv.tv_usec / 1.0e6; + + if (Format == Tango::SPECTRUM) + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + move(hdbpp_test::data_gen::generateSpectrumData<Type>(false, Size)), + traits); + } + else + { + conn.storeDataEvent(hdbpp_test::attr_name::TestAttrFinalName, + event_time, + 1, + move(hdbpp_test::data_gen::generateData<Type>(traits)), + move(hdbpp_test::data_gen::generateData<Type>(traits)), + traits); + } + } + + conn.buffer(false); + conn.disconnect(); +} + +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_BOOLEAN, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_SHORT, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_LONG, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_LONG64, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_FLOAT, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_DOUBLE, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_UCHAR, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_USHORT, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_ULONG, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_ULONG64, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_STRING, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_STATE, Tango::SPECTRUM, 512) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_BOOLEAN, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_SHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_LONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_LONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_FLOAT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_DOUBLE, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_UCHAR, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_USHORT, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_ULONG, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_ULONG64, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_STRING, Tango::SCALAR)->Unit(benchmark::kMillisecond); +BENCHMARK_TEMPLATE(bmDbSinglePreparedStatementInsert, Tango::DEV_STATE, Tango::SCALAR)->Unit(benchmark::kMillisecond); \ No newline at end of file diff --git a/benchmark/QueryBuilderTests.cpp b/benchmark/QueryBuilderTests.cpp index 24970008c1e8e951ea57d525574bc4a97c5535cb..0433583841f3821a3ff99d4ea5d31731cad89a71 100644 --- a/benchmark/QueryBuilderTests.cpp +++ b/benchmark/QueryBuilderTests.cpp @@ -16,17 +16,18 @@ You should have received a copy of the Lesser GNU General Public License along with libhdb++timescale. If not, see <http://www.gnu.org/licenses/>. */ - + #include "QueryBuilder.hpp" + #include <benchmark/benchmark.h> //============================================================================= //============================================================================= -void bmAllocateQueryBuilder(benchmark::State& state) +void bmAllocateQueryBuilder(benchmark::State &state) { // Test - Testing the time it takes to allocate a QueryBuilder, mainly for future test // reference - hdbpp_internal::LogConfigurator::initLogging(); + hdbpp_internal::LogConfigurator::initLogging("test"); for (auto _ : state) hdbpp_internal::pqxx_conn::QueryBuilder query_builder; @@ -36,10 +37,10 @@ BENCHMARK(bmAllocateQueryBuilder); //============================================================================= //============================================================================= -void bmTraitsComparator(benchmark::State& state) +void bmTraitsComparator(benchmark::State &state) { // TEST - Test the AttributeTraits comparator used in the cache inside QueryBuilder, - // the test is against a full map with every possible tango traits combination + // the test is against a full map with every possible tango traits combination std::map<hdbpp_internal::AttributeTraits, std::string> trait_cache; vector<Tango::CmdArgType> types {Tango::DEV_DOUBLE, @@ -67,8 +68,7 @@ void bmTraitsComparator(benchmark::State& state) for (auto &write : write_types) { // add to the cache for future hits - trait_cache.emplace( - hdbpp_internal::AttributeTraits{write, format, type}, + trait_cache.emplace(hdbpp_internal::AttributeTraits {write, format, type}, to_string(write) + to_string(format) + to_string(type)); } } @@ -84,25 +84,25 @@ BENCHMARK(bmTraitsComparator); //============================================================================= //============================================================================= -static void writeTypeArgs(benchmark::internal::Benchmark* b) +static void writeTypeArgs(benchmark::internal::Benchmark *b) { vector<Tango::AttrWriteType> write_types {Tango::READ, Tango::WRITE, Tango::READ_WRITE, Tango::READ_WITH_WRITE}; - for (auto & write_type : write_types) + for (auto &write_type : write_types) b->Args({static_cast<int>(write_type)}); } //============================================================================= //============================================================================= template<typename T> -void bmStoreDataEventQueryNoCache(benchmark::State& state) +void bmStoreDataEventQueryNoCache(benchmark::State &state) { // TEST - Testing how long it takes to build an Insert Data Event query with // an empty cache (this forces the full string to be built) - hdbpp_internal::LogConfigurator::initLogging(); + hdbpp_internal::LogConfigurator::initLogging("test"); - hdbpp_internal::AttributeTraits traits - {static_cast<Tango::AttrWriteType>(state.range(0)), Tango::SCALAR, Tango::DEV_DOUBLE}; + hdbpp_internal::AttributeTraits traits { + static_cast<Tango::AttrWriteType>(state.range(0)), Tango::SCALAR, Tango::DEV_DOUBLE}; for (auto _ : state) { @@ -115,14 +115,14 @@ void bmStoreDataEventQueryNoCache(benchmark::State& state) //============================================================================= //============================================================================= template<typename T> -void bmStoreDataEventQueryCache(benchmark::State& state) +void bmStoreDataEventQueryCache(benchmark::State &state) { // TEST - Testing the full lookup for an Insert Data QueryEvent query when the cache - // map is fully populated - hdbpp_internal::LogConfigurator::initLogging(); + // map is fully populated + hdbpp_internal::LogConfigurator::initLogging("test"); - hdbpp_internal::AttributeTraits traits - {static_cast<Tango::AttrWriteType>(state.range(0)), Tango::SCALAR, Tango::DEV_DOUBLE}; + hdbpp_internal::AttributeTraits traits { + static_cast<Tango::AttrWriteType>(state.range(0)), Tango::SCALAR, Tango::DEV_DOUBLE}; vector<Tango::CmdArgType> types {Tango::DEV_DOUBLE, Tango::DEV_FLOAT, @@ -147,7 +147,7 @@ void bmStoreDataEventQueryCache(benchmark::State& state) for (auto &type : types) for (auto &format : format_types) for (auto &write : write_types) - query_builder.storeDataEventStatement<T>(hdbpp_internal::AttributeTraits{write, format, type}); + query_builder.storeDataEventStatement<T>(hdbpp_internal::AttributeTraits {write, format, type}); for (auto _ : state) query_builder.storeDataEventStatement<T>(traits); @@ -155,5 +155,3 @@ void bmStoreDataEventQueryCache(benchmark::State& state) BENCHMARK_TEMPLATE(bmStoreDataEventQueryNoCache, bool)->Apply(writeTypeArgs); BENCHMARK_TEMPLATE(bmStoreDataEventQueryCache, bool)->Apply(writeTypeArgs); - -BENCHMARK_MAIN(); diff --git a/src/DbConnection.cpp b/src/DbConnection.cpp index a2463ae7b733fc2e4ca393b63e01cb38f2f158f8..c04a1ef262a8157b53a6f4928dd17d4d1a4f352f 100644 --- a/src/DbConnection.cpp +++ b/src/DbConnection.cpp @@ -364,7 +364,7 @@ namespace pqxx_conn checkConnection(LOCATION_INFO); checkAttributeExists(full_attr_name, LOCATION_INFO); - // first ensure the error message has an id inm the database, otherwise + // first ensure the error message has an id in the database, otherwise // we can not store data against it if (!_error_desc_id_cache->valueExists(error_msg)) storeErrorMsg(full_attr_name, error_msg); @@ -383,36 +383,52 @@ namespace pqxx_conn Tango::Except::throw_exception("Consistency Error", msg, LOCATION_INFO); } - try + if (_enable_buffering) { - // create and perform a pqxx transaction - pqxx::perform([&, this]() { - pqxx::work tx {(*_conn), StoreDataEventError}; - - if (!tx.prepared(_query_builder.storeDataEventErrorName(traits)).exists()) - { - tx.conn().prepare(_query_builder.storeDataEventErrorName(traits), - _query_builder.storeDataEventErrorStatement(traits)); - - spdlog::trace("Created prepared statement for: {}", _query_builder.storeDataEventErrorName(traits)); - } - - // no result expected - tx.exec_prepared0(_query_builder.storeDataEventErrorName(traits), - _conf_id_cache->value(full_attr_name), - event_time, - quality, - _error_desc_id_cache->value(error_msg)); - - tx.commit(); - }); + auto query = QueryBuilder::storeDataEventErrorString(pqxx::to_string(_conf_id_cache->value(full_attr_name)), + pqxx::to_string(event_time), + pqxx::to_string(quality), + pqxx::to_string(_error_desc_id_cache->value(error_msg)), + traits); + + query += ";"; + _sql_buffer.push_back(query); } - catch (const pqxx::pqxx_exception &ex) + else { - handlePqxxError("The attribute [" + full_attr_name + "] error message [" + error_msg + "] was not saved.", - ex.base().what(), - _query_builder.storeDataEventErrorName(traits), - LOCATION_INFO); + try + { + // create and perform a pqxx transaction + pqxx::perform([&, this]() { + pqxx::work tx {(*_conn), StoreDataEventError}; + + if (!tx.prepared(_query_builder.storeDataEventErrorName(traits)).exists()) + { + tx.conn().prepare(_query_builder.storeDataEventErrorName(traits), + _query_builder.storeDataEventErrorStatement(traits)); + + spdlog::trace( + "Created prepared statement for: {}", _query_builder.storeDataEventErrorName(traits)); + } + + // no result expected + tx.exec_prepared0(_query_builder.storeDataEventErrorName(traits), + _conf_id_cache->value(full_attr_name), + event_time, + quality, + _error_desc_id_cache->value(error_msg)); + + tx.commit(); + }); + } + catch (const pqxx::pqxx_exception &ex) + { + handlePqxxError( + "The attribute [" + full_attr_name + "] error message [" + error_msg + "] was not saved.", + ex.base().what(), + _query_builder.storeDataEventErrorName(traits), + LOCATION_INFO); + } } } @@ -573,6 +589,49 @@ namespace pqxx_conn return traits; } + //============================================================================= + //============================================================================= + void DbConnection::flush() + { + spdlog::debug("Flushing buffer of size: {}", _sql_buffer.size()); + + if (_sql_buffer.empty()) + { + spdlog::warn("Nothing to flush from the buffer, returning"); + return; + } + + try + { + pqxx::perform([&, this]() { + pqxx::work tx {(*_conn), StoreDataEvents}; + + string full_query; + + for (auto const &query : _sql_buffer) + full_query += query; + + tx.exec0(full_query); + + // commit the result + tx.commit(); + }); + } + catch (const pqxx::pqxx_exception &ex) + { + // we may try the events individually in future + _sql_buffer.clear(); + + string full_msg {"The multiple event transaction failed."}; + spdlog::error("Error: An unexpected error occurred when trying to run the database query"); + spdlog::error("Caught error at: {} Error: \"{}\"", LOCATION_INFO, ex.base().what()); + spdlog::error("Throwing storage error with message: \"{}\"", full_msg); + Tango::Except::throw_exception("Storage Error", full_msg, LOCATION_INFO); + } + + _sql_buffer.clear(); + } + //============================================================================= //============================================================================= void DbConnection::storeEvent(const std::string &full_attr_name, const std::string &event) diff --git a/src/DbConnection.hpp b/src/DbConnection.hpp index 2a1b0351f5964f720856d6e328773da9f24e5994..21adc2e52cb9016d12a9a7d2223f14c1d6b3e2af 100644 --- a/src/DbConnection.hpp +++ b/src/DbConnection.hpp @@ -64,6 +64,12 @@ namespace pqxx_conn bool isOpen() const noexcept override { return _connected; } bool isClosed() const noexcept override { return !isOpen(); } + // this API allows the connection to buffer the event data store + // requests, and send them all at once to the db, this will increase + // insert time. Flush will execute the sql and clear the buffer + void buffer(bool enable) { _enable_buffering = enable; } + void flush(); + // storage API // store a new attribute and its conf data into the database @@ -99,8 +105,8 @@ namespace pqxx_conn void storeDataEvent(const std::string &full_attr_name, double event_time, int quality, - std::unique_ptr<vector<T>> value_r, - std::unique_ptr<vector<T>> value_w, + std::unique_ptr<std::vector<T>> value_r, + std::unique_ptr<std::vector<T>> value_w, const AttributeTraits &traits); // store a data error event in the data tables @@ -160,6 +166,13 @@ namespace pqxx_conn // configured db access method DbStoreMethod _db_store_method; + + // it is possible to buffer store requests as sql strings, then flush + // them all to the database at once, this increases insert speed, since + // multiple statements can be sent across the wire at once. This vector + // holds the preapred sql until the connection is flushed + bool _enable_buffering = false; + std::vector<std::string> _sql_buffer; }; } // namespace pqxx_conn } // namespace hdbpp_internal diff --git a/src/DbConnection.tpp b/src/DbConnection.tpp index 587d51733df444831ac3db1a5a5dd5c25b048ef8..26e5ae3d23bb34444284f53b00e49bee19dce803 100644 --- a/src/DbConnection.tpp +++ b/src/DbConnection.tpp @@ -35,7 +35,7 @@ namespace pqxx_conn template<typename T> struct Store { - static void run(std::unique_ptr<std::vector<T>> &value, + static void run(const std::unique_ptr<std::vector<T>> &value, const AttributeTraits &traits, pqxx::prepare::invocation &inv, pqxx::work & /*unused*/) @@ -53,7 +53,7 @@ namespace pqxx_conn template<> struct Store<std::string> { - static void run(std::unique_ptr<std::vector<std::string>> &value, + static void run(const std::unique_ptr<std::vector<std::string>> &value, const AttributeTraits &traits, pqxx::prepare::invocation &inv, pqxx::work &tx) @@ -77,7 +77,7 @@ namespace pqxx_conn template<> struct Store<bool> { - static void run(std::unique_ptr<std::vector<bool>> &value, + static void run(const std::unique_ptr<std::vector<bool>> &value, const AttributeTraits &traits, pqxx::prepare::invocation &inv, pqxx::work & /*unused*/) @@ -119,85 +119,105 @@ namespace pqxx_conn checkConnection(LOCATION_INFO); checkAttributeExists(full_attr_name, LOCATION_INFO); - try + // if we are buffering the queries, then just save it until the buffer is flushed, + // otherwise execute directly + if (_enable_buffering) { - return pqxx::perform([&, this]() { - pqxx::work tx {(*_conn), StoreDataEvent}; + auto query = QueryBuilder::storeDataEventString<T>(pqxx::to_string(_conf_id_cache->value(full_attr_name)), + pqxx::to_string(event_time), + pqxx::to_string(quality), + value_r, + value_w, + traits); + + query += ";"; + _sql_buffer.push_back(query); + } + else + { + try + { + return pqxx::perform([&, this]() { + pqxx::work tx {(*_conn), StoreDataEvent}; - // there is a single special case here, arrays of strings need a different syntax to store, - // to avoid the quoting. Its likely we will need more for DevEncoded and DevEnum - if (_db_store_method == DbStoreMethod::InsertString || - (traits.isArray() && traits.type() == Tango::DEV_STRING)) - { - auto query = _query_builder.storeDataEventString<T>( - pqxx::to_string(_conf_id_cache->value(full_attr_name)), - pqxx::to_string(event_time), - pqxx::to_string(quality), - value_r, - value_w, - traits); - - tx.exec0(query); - } - else - { - // prepare as a prepared statement, we are going to use these - // queries often - if (!tx.prepared(_query_builder.storeDataEventName(traits)).exists()) + // there is a single special case here, arrays of strings need a different syntax to store, + // to avoid the quoting. Its likely we will need more for DevEncoded and DevEnum + if (_db_store_method == DbStoreMethod::InsertString || + (traits.isArray() && traits.type() == Tango::DEV_STRING) || _enable_buffering) { - tx.conn().prepare(_query_builder.storeDataEventName(traits), - _query_builder.storeDataEventStatement<T>(traits)); - } - - // get the pqxx prepared statement invocation object to allow us to - // bind each parameter in turn, this gives us the flexibility to bind - // conditional parameters (as long as the query string matches) - auto inv = tx.prepared(_query_builder.storeDataEventName(traits)); - - // this lambda stores the data value correctly into the invocation, - // we must treat scalar/spectrum in different ways, one is a single - // element and the other an array. Further, the unique_ptr may be - // empty and signify a null should be stored in the column instead - auto store_value = [&tx, &traits, &inv](auto &value) { - if (value && !value->empty()) - { - store_data_utils::Store<T>::run(value, traits, inv, tx); - } + auto query = QueryBuilder::storeDataEventString<T>( + pqxx::to_string(_conf_id_cache->value(full_attr_name)), + pqxx::to_string(event_time), + pqxx::to_string(quality), + value_r, + value_w, + traits); + + if (_enable_buffering) + _sql_buffer.push_back(query); else + tx.exec0(query); + } + else + { + // prepare as a prepared statement, we are going to use these + // queries often + if (!tx.prepared(_query_builder.storeDataEventName(traits)).exists()) { - // no value was given for this field, simply add a null - // instead, this allows invalid quality attributes to be saved - // with no data - inv(); + tx.conn().prepare(_query_builder.storeDataEventName(traits), + _query_builder.storeDataEventStatement<T>(traits)); } - }; - - // bind all the parameters - inv(_conf_id_cache->value(full_attr_name)); - inv(event_time); - - if (traits.hasReadData()) - store_value(value_r); - if (traits.hasWriteData()) - store_value(value_w); - - inv(quality); - - // execute - inv.exec(); - } + // get the pqxx prepared statement invocation object to allow us to + // bind each parameter in turn, this gives us the flexibility to bind + // conditional parameters (as long as the query string matches) + auto inv = tx.prepared(_query_builder.storeDataEventName(traits)); + + // this lambda stores the data value correctly into the invocation, + // we must treat scalar/spectrum in different ways, one is a single + // element and the other an array. Further, the unique_ptr may be + // empty and signify a null should be stored in the column instead + auto store_value = [&tx, &traits, &inv](auto &value) { + if (value && !value->empty()) + { + store_data_utils::Store<T>::run(value, traits, inv, tx); + } + else + { + // no value was given for this field, simply add a null + // instead, this allows invalid quality attributes to be saved + // with no data + inv(); + } + }; + + // bind all the parameters + inv(_conf_id_cache->value(full_attr_name)); + inv(event_time); + + if (traits.hasReadData()) + store_value(value_r); + + if (traits.hasWriteData()) + store_value(value_w); + + inv(quality); + + // execute + inv.exec(); + } - // commit the result - tx.commit(); - }); - } - catch (const pqxx::pqxx_exception &ex) - { - handlePqxxError("The attribute [" + full_attr_name + "] data event was not saved.", - ex.base().what(), - _query_builder.storeDataEventStatement<T>(traits), - LOCATION_INFO); + // commit the result + tx.commit(); + }); + } + catch (const pqxx::pqxx_exception &ex) + { + handlePqxxError("The attribute [" + full_attr_name + "] data event was not saved.", + ex.base().what(), + _query_builder.storeDataEventStatement<T>(traits), + LOCATION_INFO); + } } } } // namespace pqxx_conn diff --git a/src/HdbppTimescaleDbApi.cpp b/src/HdbppTimescaleDbApi.cpp index 5f30247e386bfafcc5779d2f9b728e0f7bcd7271..aadd0989f4cd7603eaade235eb692fa42f2fc1e3 100644 --- a/src/HdbppTimescaleDbApi.cpp +++ b/src/HdbppTimescaleDbApi.cpp @@ -166,56 +166,31 @@ void HdbppTimescaleDbApi::insert_event(Tango::EventData *event_data, const HdbEv assert(event_data->attr_value); spdlog::trace("Insert data event for attribute: {}", event_data->attr_name); - // if there is an error, we store an error, since there will be no data passed in - if (event_data->err) - { - spdlog::trace("Event type is error for attribute: {}", event_data->attr_name); - - // no time data is passed for errors, so make something up - struct timeval tv - {}; + // hand the call to the internal routine + doInsertEvent(event_data, data_type); +} - struct Tango::TimeVal tango_tv - {}; +//============================================================================= +//============================================================================= +void HdbppTimescaleDbApi::insert_events(vector<tuple<Tango::EventData *, HdbEventDataType>> events) +{ + _conn->buffer(true); - gettimeofday(&tv, nullptr); - tango_tv.tv_sec = tv.tv_sec; - tango_tv.tv_usec = tv.tv_usec; - tango_tv.tv_nsec = 0; + try + { + for (auto event : events) + doInsertEvent(get<0>(event), get<1>(event)); - _conn->createTx<HdbppTxDataEventError>() - .withName(event_data->attr_name) - .withTraits(static_cast<Tango::AttrWriteType>(data_type.write_type), - static_cast<Tango::AttrDataFormat>(data_type.data_format), - static_cast<Tango::CmdArgType>(data_type.data_type)) - .withError(string(event_data->errors[0].desc)) - .withEventTime(tango_tv) - .withQuality(event_data->attr_value->get_quality()) - .store(); + _conn->flush(); } - else + catch (Tango::DevFailed &e) { - spdlog::trace("Event type is data for attribute: {}", event_data->attr_name); - - // build a data event request, this will store 0 or more data elements, - // pending on type, format and quality - _conn->createTx<HdbppTxDataEvent>() - .withName(event_data->attr_name) - .withTraits(static_cast<Tango::AttrWriteType>(data_type.write_type), - static_cast<Tango::AttrDataFormat>(data_type.data_format), - static_cast<Tango::CmdArgType>(data_type.data_type)) - .withAttribute(event_data->attr_value) - .withEventTime(event_data->attr_value->get_date()) - .withQuality(event_data->attr_value->get_quality()) - .store(); + // ensure this is disabled on error + _conn->buffer(false); + throw; } -} -//============================================================================= -//============================================================================= -void HdbppTimescaleDbApi::insert_events(vector<tuple<Tango::EventData *, HdbEventDataType>> events) -{ - + _conn->buffer(false); } //============================================================================= @@ -286,4 +261,53 @@ bool HdbppTimescaleDbApi::supported(HdbppFeatures feature) return supported; } +//============================================================================= +//============================================================================= +void HdbppTimescaleDbApi::doInsertEvent(Tango::EventData *event_data, const HdbEventDataType &data_type) +{ + // if there is an error, we store an error, since there will be no data passed in + if (event_data->err) + { + spdlog::trace("Event type is error for attribute: {}", event_data->attr_name); + + // no time data is passed for errors, so make something up + struct timeval tv + {}; + + struct Tango::TimeVal tango_tv + {}; + + gettimeofday(&tv, nullptr); + tango_tv.tv_sec = tv.tv_sec; + tango_tv.tv_usec = tv.tv_usec; + tango_tv.tv_nsec = 0; + + _conn->createTx<HdbppTxDataEventError>() + .withName(event_data->attr_name) + .withTraits(static_cast<Tango::AttrWriteType>(data_type.write_type), + static_cast<Tango::AttrDataFormat>(data_type.data_format), + static_cast<Tango::CmdArgType>(data_type.data_type)) + .withError(string(event_data->errors[0].desc)) + .withEventTime(tango_tv) + .withQuality(event_data->attr_value->get_quality()) + .store(); + } + else + { + spdlog::trace("Event type is data for attribute: {}", event_data->attr_name); + + // build a data event request, this will store 0 or more data elements, + // pending on type, format and quality + _conn->createTx<HdbppTxDataEvent>() + .withName(event_data->attr_name) + .withTraits(static_cast<Tango::AttrWriteType>(data_type.write_type), + static_cast<Tango::AttrDataFormat>(data_type.data_format), + static_cast<Tango::CmdArgType>(data_type.data_type)) + .withAttribute(event_data->attr_value) + .withEventTime(event_data->attr_value->get_date()) + .withQuality(event_data->attr_value->get_quality()) + .store(); + } +} + } // namespace hdbpp diff --git a/src/HdbppTimescaleDbApi.hpp b/src/HdbppTimescaleDbApi.hpp index d4b9ecbececf3486a3508f5db354d0a79cd83054..7e632ce7aeb7f0cef7568de7b0f1d0610216dae2 100644 --- a/src/HdbppTimescaleDbApi.hpp +++ b/src/HdbppTimescaleDbApi.hpp @@ -72,6 +72,8 @@ public: bool supported(HdbppFeatures feature) override; private: + void doInsertEvent(Tango::EventData *event_data, const HdbEventDataType &data_type); + std::unique_ptr<hdbpp_internal::pqxx_conn::DbConnection> _conn; std::string _identity; }; diff --git a/src/HdbppTxDataEvent.hpp b/src/HdbppTxDataEvent.hpp index 47f352ea2c20fde3bbf0fa4c6e253fb6e77d11d1..be3c4165ac86add434564c364725a425cebce7c6 100644 --- a/src/HdbppTxDataEvent.hpp +++ b/src/HdbppTxDataEvent.hpp @@ -207,7 +207,7 @@ void HdbppTxDataEvent<Conn>::doStore(ReadFunctor extract_read, WriteFunctor extr } // release ownership of the unique_ptr back to the caller - return std::move(value); + return value; }; // attempt to store the error in the database, any exceptions are left to diff --git a/src/QueryBuilder.cpp b/src/QueryBuilder.cpp index 625eef4aad17f3bacc3bc165dc5c51282dd8d788..71e60fdcf5731d16f285c380cf6fa13d2e86158d 100644 --- a/src/QueryBuilder.cpp +++ b/src/QueryBuilder.cpp @@ -261,6 +261,31 @@ namespace pqxx_conn return result->second; } + //============================================================================= + //============================================================================= + string QueryBuilder::storeDataEventErrorString(const string &id, + const string &event_time, + const string &quality, + const string &err_id, + const AttributeTraits &traits) + { + // clang-format off + auto query = "INSERT INTO " + + QueryBuilder::tableName(traits) + " (" + + schema::DatColId + "," + + schema::DatColDataTime + "," + + schema::DatColQuality + "," + + schema::DatColErrorDescId + ") " + + "VALUES (" + + id + "," + + "TO_TIMESTAMP(" + event_time + ")," + + quality + "," + + err_id + ")"; + // clang-format on + + return query; + } + //============================================================================= //============================================================================= const string &QueryBuilder::storeErrorStatement() diff --git a/src/QueryBuilder.hpp b/src/QueryBuilder.hpp index 0dc52f9543a55347826a70a767fcd79068207844..92d4f0fda3cb9407b8f61c7db1023e87b369d685 100644 --- a/src/QueryBuilder.hpp +++ b/src/QueryBuilder.hpp @@ -68,7 +68,7 @@ namespace pqxx_conn template<typename T> struct DataToString { - static std::string run(std::unique_ptr<std::vector<T>> &value, const AttributeTraits &traits) + static std::string run(const std::unique_ptr<std::vector<T>> &value, const AttributeTraits &traits) { if (traits.isScalar()) return pqxx::to_string((*value)[0]); @@ -81,7 +81,7 @@ namespace pqxx_conn template<> struct DataToString<bool> { - static std::string run(std::unique_ptr<std::vector<bool>> &value, const AttributeTraits &traits) + static std::string run(const std::unique_ptr<std::vector<bool>> &value, const AttributeTraits &traits) { // a vector<bool> is not actually a vector<bool>, rather its some kind of bitfield. When // trying to return an element, we appear to get some kind of bitfield reference, @@ -103,7 +103,8 @@ namespace pqxx_conn template<> struct DataToString<std::string> { - static std::string run(std::unique_ptr<std::vector<std::string>> &value, const AttributeTraits &traits) + static std::string run( + const std::unique_ptr<std::vector<std::string>> &value, const AttributeTraits &traits) { // arrays of strings need both the ARRAY keywords and dollar escaping, this is so we // do not have to rely on the postgres escape functions that double and then store @@ -139,6 +140,7 @@ namespace pqxx_conn const string StoreHistoryEvent = "StoreHistoryEvent"; const string StoreParameterEvent = "StoreParameterEvent"; const string StoreDataEvent = "StoreDataEvent"; + const string StoreDataEvents = "StoreDataEvents"; const string StoreDataEventError = "StoreDataEventError"; const string StoreErrorString = "StoreErrorString"; const string StoreTtl = "StoreTtl"; @@ -189,16 +191,24 @@ namespace pqxx_conn // internal caching, so its less efficient, but can be chained in a pipe // to batch data to the database. template<typename T> - const std::string storeDataEventString(const std::string &full_attr_name, + static std::string storeDataEventString(const std::string &id, const std::string &event_time, const std::string &quality, - std::unique_ptr<vector<T>> &value_r, - std::unique_ptr<vector<T>> &value_w, + const std::unique_ptr<vector<T>> &value_r, + const std::unique_ptr<vector<T>> &value_w, const AttributeTraits &traits); // Builds a prepared statement for data event errors const std::string &storeDataEventErrorStatement(const AttributeTraits &traits); + // A vareint of storeDataEventErrorStatement that build and returns a string + // instead of a prepared statement + static std::string storeDataEventErrorString(const std::string &id, + const std::string &event_time, + const std::string &quality, + const std::string &err_id, + const AttributeTraits &traits); + // Utility void print(std::ostream &os) const noexcept; @@ -270,11 +280,11 @@ namespace pqxx_conn } template<typename T> - const std::string QueryBuilder::storeDataEventString(const std::string &full_attr_name, + std::string QueryBuilder::storeDataEventString(const std::string &id, const std::string &event_time, const std::string &quality, - std::unique_ptr<vector<T>> &value_r, - std::unique_ptr<vector<T>> &value_w, + const std::unique_ptr<vector<T>> &value_r, + const std::unique_ptr<vector<T>> &value_w, const AttributeTraits &traits) { auto query = "INSERT INTO " + QueryBuilder::tableName(traits) + " (" + schema::DatColId + "," + @@ -287,7 +297,7 @@ namespace pqxx_conn query = query + "," + schema::DatColValueW; // split to ensure increments are in the correct order - query = query + "," + schema::DatColQuality + ") VALUES ('" + full_attr_name + "'"; + query = query + "," + schema::DatColQuality + ") VALUES ('" + id + "'"; query = query + ",TO_TIMESTAMP(" + event_time + ")"; // add the read parameter with cast diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8d839445587955bd08c870cc81bdea70a0ebb113..59417c380409420f2f2f7cca30bf76f528b00b4d 100755 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -6,7 +6,6 @@ set(CMAKE_COLOR_MAKEFILE ON) # Make test executable set(TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/TestHelpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/AttributeNameTests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/AttributeTraitsTests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ColumnCacheTests.cpp @@ -20,11 +19,26 @@ set(TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/HdbppTxUpdateTtlTests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/QueryBuilderTests.cpp) +add_library(test-utils STATIC EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/TestHelpers.cpp) +target_compile_options(test-utils PRIVATE -Wall -Wextra -g) + +target_link_libraries(test-utils + PRIVATE + libhdbpp_timescale_static_library + TangoInterfaceLibrary) + +target_include_directories(test-utils + PUBLIC ${CMAKE_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}) + add_executable(unit-tests ${TEST_SOURCES}) target_compile_options(unit-tests PRIVATE -Wall -Wextra -g) target_link_libraries(unit-tests - PRIVATE libhdbpp_headers libhdbpp_timescale_shared_library Catch2 TangoInterfaceLibrary) + PRIVATE + libhdbpp_timescale_static_library + Catch2 + TangoInterfaceLibrary + test-utils) target_include_directories(unit-tests PRIVATE ${CMAKE_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}) @@ -38,4 +52,8 @@ if(DO_CLANG_TIDY) set_target_properties(unit-tests PROPERTIES CXX_CLANG_TIDY ${DO_CLANG_TIDY}) + + set_target_properties(test-utils + PROPERTIES + CXX_CLANG_TIDY ${DO_CLANG_TIDY}) endif(DO_CLANG_TIDY) \ No newline at end of file diff --git a/test/DbConnectionTests.cpp b/test/DbConnectionTests.cpp index 84a2e68079660314ede98fa7d8dd60aae1e5647d..ac2dd3757774b35f905437a197f055c72aea530d 100644 --- a/test/DbConnectionTests.cpp +++ b/test/DbConnectionTests.cpp @@ -144,8 +144,6 @@ protected: const string &att_name, const AttributeTraits &traits, const tuple<vector<T>, vector<T>> &data); QueryBuilder &queryBuilder() { return _query_builder; } - vector<AttributeTraits> getTraits() const; - vector<AttributeTraits> getTraitsImplemented() const; public: DbConnectionTestsFixture() = default; @@ -180,7 +178,7 @@ pqxx::connection &DbConnectionTestsFixture::verifyConn() //============================================================================= void DbConnectionTestsFixture::clearTables() { - vector<AttributeTraits> traits_array = getTraits(); + vector<AttributeTraits> traits_array = utils::getTraits(); { string query = "TRUNCATE "; @@ -237,70 +235,6 @@ string DbConnectionTestsFixture::storeAttributeByTraits(const AttributeTraits &t return name; } -//============================================================================= -//============================================================================= -vector<AttributeTraits> DbConnectionTestsFixture::getTraits() const -{ - vector<AttributeTraits> traits_array {}; - - vector<Tango::CmdArgType> types {Tango::DEV_BOOLEAN, - Tango::DEV_DOUBLE, - Tango::DEV_FLOAT, - Tango::DEV_STRING, - Tango::DEV_LONG, - Tango::DEV_ULONG, - Tango::DEV_LONG64, - Tango::DEV_ULONG64, - Tango::DEV_SHORT, - Tango::DEV_USHORT, - Tango::DEV_UCHAR, - Tango::DEV_STATE, - Tango::DEV_ENCODED, - Tango::DEV_ENUM}; - - vector<Tango::AttrWriteType> write_types {Tango::READ, Tango::WRITE, Tango::READ_WRITE, Tango::READ_WITH_WRITE}; - vector<Tango::AttrDataFormat> format_types {Tango::SCALAR, Tango::SPECTRUM}; - - // loop for every combination of type in Tango - for (auto &type : types) - for (auto &format : format_types) - for (auto &write : write_types) - traits_array.emplace_back(AttributeTraits {write, format, type}); - - return traits_array; -} - -//============================================================================= -//============================================================================= -vector<AttributeTraits> DbConnectionTestsFixture::getTraitsImplemented() const -{ - vector<AttributeTraits> traits_array {}; - - vector<Tango::CmdArgType> types {Tango::DEV_BOOLEAN, - Tango::DEV_DOUBLE, - Tango::DEV_FLOAT, - Tango::DEV_STRING, - Tango::DEV_LONG, - Tango::DEV_ULONG, - Tango::DEV_LONG64, - Tango::DEV_ULONG64, - Tango::DEV_SHORT, - Tango::DEV_USHORT, - Tango::DEV_UCHAR, - Tango::DEV_STATE}; - - vector<Tango::AttrWriteType> write_types {Tango::READ, Tango::WRITE, Tango::READ_WRITE, Tango::READ_WITH_WRITE}; - vector<Tango::AttrDataFormat> format_types {Tango::SCALAR, Tango::SPECTRUM}; - - // loop for every combination of type in Tango - for (auto &type : types) - for (auto &format : format_types) - for (auto &write : write_types) - traits_array.emplace_back(AttributeTraits {write, format, type}); - - return traits_array; -} - //============================================================================= //============================================================================= template<Tango::CmdArgType Type> @@ -816,10 +750,10 @@ TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, } TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, - "Storing event data for all Tango type combinations in the database (prepared statements)", + "Storing single event data for all Tango type combinations in the database (prepared statements)", "[db-access][hdbpp-db-access][db-connection]") { - auto traits_array = getTraitsImplemented(); + auto traits_array = utils::getTraitsImplemented(); REQUIRE_NOTHROW(clearTables()); resetDbAccess(DbConnection::DbStoreMethod::PreparedStatement); @@ -886,10 +820,10 @@ TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, } TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, - "Storing event data for all Tango type combinations in the database (insert strings)", + "Storing single event data for all Tango type combinations in the database (insert strings)", "[db-access][hdbpp-db-access][db-connection]") { - auto traits_array = getTraitsImplemented(); + auto traits_array = utils::getTraitsImplemented(); REQUIRE_NOTHROW(clearTables()); resetDbAccess(DbConnection::DbStoreMethod::InsertString); @@ -955,6 +889,130 @@ TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, SUCCEED("Passed"); } +TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, + "Storing multiple event data via a buffered sql statement", + "[db-access][hdbpp-db-access][db-connection]") +{ + auto traits_array = utils::getTraitsImplemented(); + REQUIRE_NOTHROW(clearTables()); + resetDbAccess(DbConnection::DbStoreMethod::InsertString); + + testConn().buffer(true); + + for (auto &traits : traits_array) + { + INFO("Inserting data for traits: " << traits); + auto name = storeAttributeByTraits(traits); + + switch (traits.type()) + { + case Tango::DEV_BOOLEAN: storeTestEventData<Tango::DEV_BOOLEAN>(name, traits); break; + + case Tango::DEV_SHORT: storeTestEventData<Tango::DEV_SHORT>(name, traits); break; + + case Tango::DEV_LONG: storeTestEventData<Tango::DEV_LONG>(name, traits); break; + + case Tango::DEV_LONG64: storeTestEventData<Tango::DEV_LONG64>(name, traits); break; + + case Tango::DEV_FLOAT: storeTestEventData<Tango::DEV_FLOAT>(name, traits); break; + + case Tango::DEV_DOUBLE: storeTestEventData<Tango::DEV_DOUBLE>(name, traits); break; + + case Tango::DEV_UCHAR: storeTestEventData<Tango::DEV_UCHAR>(name, traits); break; + + case Tango::DEV_USHORT: storeTestEventData<Tango::DEV_USHORT>(name, traits); break; + + case Tango::DEV_ULONG: storeTestEventData<Tango::DEV_ULONG>(name, traits); break; + + case Tango::DEV_ULONG64: storeTestEventData<Tango::DEV_ULONG64>(name, traits); break; + + case Tango::DEV_STRING: storeTestEventData<Tango::DEV_STRING>(name, traits); break; + + case Tango::DEV_STATE: storeTestEventData<Tango::DEV_STATE>(name, traits); break; + + default: throw "Should not be here!"; + } + } + + REQUIRE_NOTHROW(testConn().flush()); + + // check the rows, should be four in each table + for (auto &traits : traits_array) + { + pqxx::work tx {verifyConn()}; + + // now get the count in the table + auto result(tx.exec1("SELECT COUNT(*) FROM " + QueryBuilder::tableName(traits))); + tx.commit(); + + REQUIRE(result.size() == 1); + REQUIRE(result[0].as<int>() == 4); + } + + testConn().buffer(false); + SUCCEED("Passed"); +} + +TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, + "Storing large number of spectrum event data via a buffered sql statement", + "[db-access][hdbpp-db-access][db-connection]") +{ + REQUIRE_NOTHROW(clearTables()); + resetDbAccess(DbConnection::DbStoreMethod::InsertString); + + testConn().buffer(true); + + AttributeTraits traits {Tango::READ_WRITE, Tango::SPECTRUM, Tango::DEV_DOUBLE}; + auto name = storeAttributeByTraits(traits); + + for (int i = 1; i <= 1000; i++) + storeTestEventData<Tango::DEV_DOUBLE>(name, traits); + + REQUIRE_NOTHROW(testConn().flush()); + + pqxx::work tx {verifyConn()}; + + // now get the count in the table + auto result(tx.exec1("SELECT COUNT(*) FROM " + QueryBuilder::tableName(traits))); + tx.commit(); + + REQUIRE(result.size() == 1); + REQUIRE(result[0].as<int>() == 1000); + + testConn().buffer(false); + SUCCEED("Passed"); +} + +TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, + "Storing large number of scalar event data via a buffered sql statement", + "[db-access][hdbpp-db-access][db-connection]") +{ + REQUIRE_NOTHROW(clearTables()); + resetDbAccess(DbConnection::DbStoreMethod::InsertString); + + testConn().buffer(true); + + AttributeTraits traits {Tango::READ_WRITE, Tango::SCALAR, Tango::DEV_DOUBLE}; + auto name = storeAttributeByTraits(traits); + + for (int i = 0; i < 1000; i++) + storeTestEventData<Tango::DEV_DOUBLE>(name, traits); + + REQUIRE_NOTHROW(testConn().flush()); + + pqxx::work tx {verifyConn()}; + + // now get the count in the table + auto result(tx.exec1("SELECT COUNT(*) FROM " + QueryBuilder::tableName(traits))); + tx.commit(); + + REQUIRE(result.size() == 1); + REQUIRE(result[0].as<int>() == 1000); + + testConn().buffer(false); + SUCCEED("Passed"); +} + TEST_CASE_METHOD(pqxx_conn_test::DbConnectionTestsFixture, "Storing complex arrays of strings containing postgres escape characters", "[db-access][hdbpp-db-access][db-connection]") diff --git a/test/QueryBuilderTests.cpp b/test/QueryBuilderTests.cpp index 3daaf18169959df25440bb2fa93e64d507c985ef..6bb32e572ad1626c0b37eaf0432ce5cc3b859ab1 100644 --- a/test/QueryBuilderTests.cpp +++ b/test/QueryBuilderTests.cpp @@ -32,7 +32,6 @@ SCENARIO("storeDataEventString() returns the correct Value fields for the given { GIVEN("A query builder object with nothing cached") { - QueryBuilder query_builder; auto value_r = make_unique<vector<double>>(1.1, 2.2); auto value_r_empty = make_unique<vector<double>>(); auto value_w = make_unique<vector<double>>(3.3, 4.4); @@ -42,7 +41,7 @@ SCENARIO("storeDataEventString() returns the correct Value fields for the given { AttributeTraits traits {Tango::READ, Tango::SCALAR, Tango::DEV_DOUBLE}; - auto result = query_builder.storeDataEventString<double>( + auto result = QueryBuilder::storeDataEventString<double>( TestAttrFQDName, string("0"), string("1"), value_r, value_w_empty, traits); THEN("The result must include the schema::DatColValueR field only") @@ -56,7 +55,7 @@ SCENARIO("storeDataEventString() returns the correct Value fields for the given { AttributeTraits traits {Tango::WRITE, Tango::SCALAR, Tango::DEV_DOUBLE}; - auto result = query_builder.storeDataEventString<double>( + auto result = QueryBuilder::storeDataEventString<double>( TestAttrFQDName, string("0"), string("1"), value_r_empty, value_w, traits); THEN("The result must include the schema::DatColValueW field only") @@ -70,7 +69,7 @@ SCENARIO("storeDataEventString() returns the correct Value fields for the given { AttributeTraits traits {Tango::READ_WRITE, Tango::SCALAR, Tango::DEV_DOUBLE}; - auto result = query_builder.storeDataEventString<double>( + auto result = QueryBuilder::storeDataEventString<double>( TestAttrFQDName, string("0"), string("1"), value_r, value_w, traits); THEN("The result must include both the schema::DatColValueR and schema::DatColValueW field") @@ -85,7 +84,7 @@ SCENARIO("storeDataEventString() returns the correct Value fields for the given { AttributeTraits traits {Tango::READ_WITH_WRITE, Tango::SCALAR, Tango::DEV_DOUBLE}; - auto result = query_builder.storeDataEventString<double>( + auto result = QueryBuilder::storeDataEventString<double>( TestAttrFQDName, string("0"), string("1"), value_r, value_w, traits); THEN("The result must include both the schema::DatColValueR and schema::DatColValueW field") @@ -103,7 +102,6 @@ SCENARIO("storeDataEventString() adds a null when value is size zero", "[query-s { GIVEN("A query builder object with nothing cached") { - QueryBuilder query_builder; auto value_r = make_unique<vector<double>>(1.1, 2.2); auto value_r_empty = make_unique<vector<double>>(); auto value_w = make_unique<vector<double>>(3.3, 4.4); @@ -113,7 +111,7 @@ SCENARIO("storeDataEventString() adds a null when value is size zero", "[query-s { AttributeTraits traits {Tango::READ_WRITE, Tango::SCALAR, Tango::DEV_DOUBLE}; - auto result = query_builder.storeDataEventString<double>( + auto result = QueryBuilder::storeDataEventString<double>( TestAttrFQDName, string("0"), string("1"), value_r, value_w_empty, traits); THEN("The result must include both the schema::DatColValueR and schema::DatColValueW field") @@ -128,7 +126,7 @@ SCENARIO("storeDataEventString() adds a null when value is size zero", "[query-s { AttributeTraits traits {Tango::READ_WRITE, Tango::SCALAR, Tango::DEV_DOUBLE}; - auto result = query_builder.storeDataEventString<double>( + auto result = QueryBuilder::storeDataEventString<double>( TestAttrFQDName, string("0"), string("1"), value_r_empty, value_w, traits); THEN("The result must include both the schema::DatColValueR and schema::DatColValueW field") @@ -203,21 +201,20 @@ SCENARIO("storeDataEventStatement() returns the correct Value fields for the giv } } -SCENARIO("Creating valid insert queries with storeDataEventErrorQuery()", "[query-string]") +SCENARIO("Creating valid insert queries with storeDataEventErrorString()", "[query-string]") { - QueryBuilder query_builder; - GIVEN("An Attribute traits configured for scalar") { AttributeTraits traits {Tango::READ, Tango::SCALAR, Tango::DEV_DOUBLE}; - WHEN("Requesting a table name for the traits") + WHEN("Requesting an error query string") { - auto result = QueryBuilder::tableName(traits); + auto result = QueryBuilder::storeDataEventErrorString( + string("1"), string("0"), string("1"), string("An error message"), traits); - THEN("The result must include the schema::TypeScalar from the schema") + THEN("The result must include the schema::DatColValueR field only") { - REQUIRE_THAT(result, Contains(schema::TypeScalar)); + REQUIRE_THAT(result, Contains(string("An error message"))); } } } diff --git a/test/TestHelpers.cpp b/test/TestHelpers.cpp index 7450bf531916b8689b2ef2ab404d035f410a67e0..b432432f701d815baeff08742af864bb74c38aa8 100644 --- a/test/TestHelpers.cpp +++ b/test/TestHelpers.cpp @@ -39,7 +39,7 @@ namespace data_gen for (int i = 0; i < size; i++) value->push_back(d(gen)); - return move(value); + return value; } //============================================================================= @@ -78,7 +78,7 @@ namespace data_gen for (int i = 0; i < size; i++) value->push_back(d(gen)); - return move(value); + return value; } //============================================================================= @@ -93,7 +93,7 @@ namespace data_gen for (int i = 0; i < size; i++) value->push_back(d(gen)); - return move(value); + return value; } //============================================================================= @@ -108,7 +108,7 @@ namespace data_gen for (int i = 0; i < size; i++) value->push_back(d(gen)); - return move(value); + return value; } //============================================================================= @@ -216,7 +216,7 @@ namespace data_gen for (int i = 0; i < size; i++) value->push_back(strings[experimental::randint(0, ((int)strings.size()) - 1)]); - return move(value); + return value; } //============================================================================= @@ -232,7 +232,74 @@ namespace data_gen for (int i = 0; i < size; i++) value->push_back(d(gen) ? Tango::ON : Tango::OFF); - return move(value); + return value; } } // namespace data_gen + +namespace utils +{ + //============================================================================= + //============================================================================= + vector<AttributeTraits> getTraits() + { + vector<AttributeTraits> traits_array {}; + + vector<Tango::CmdArgType> types {Tango::DEV_BOOLEAN, + Tango::DEV_DOUBLE, + Tango::DEV_FLOAT, + Tango::DEV_STRING, + Tango::DEV_LONG, + Tango::DEV_ULONG, + Tango::DEV_LONG64, + Tango::DEV_ULONG64, + Tango::DEV_SHORT, + Tango::DEV_USHORT, + Tango::DEV_UCHAR, + Tango::DEV_STATE, + Tango::DEV_ENCODED, + Tango::DEV_ENUM}; + + vector<Tango::AttrWriteType> write_types {Tango::READ, Tango::WRITE, Tango::READ_WRITE, Tango::READ_WITH_WRITE}; + vector<Tango::AttrDataFormat> format_types {Tango::SCALAR, Tango::SPECTRUM}; + + // loop for every combination of type in Tango + for (auto &type : types) + for (auto &format : format_types) + for (auto &write : write_types) + traits_array.emplace_back(AttributeTraits {write, format, type}); + + return traits_array; + } + + //============================================================================= + //============================================================================= + vector<AttributeTraits> getTraitsImplemented() + { + vector<AttributeTraits> traits_array {}; + + vector<Tango::CmdArgType> types {Tango::DEV_BOOLEAN, + Tango::DEV_DOUBLE, + Tango::DEV_FLOAT, + Tango::DEV_STRING, + Tango::DEV_LONG, + Tango::DEV_ULONG, + Tango::DEV_LONG64, + Tango::DEV_ULONG64, + Tango::DEV_SHORT, + Tango::DEV_USHORT, + Tango::DEV_UCHAR, + Tango::DEV_STATE}; + + vector<Tango::AttrWriteType> write_types {Tango::READ, Tango::WRITE, Tango::READ_WRITE, Tango::READ_WITH_WRITE}; + vector<Tango::AttrDataFormat> format_types {Tango::SCALAR, Tango::SPECTRUM}; + + // loop for every combination of type in Tango + for (auto &type : types) + for (auto &format : format_types) + for (auto &write : write_types) + traits_array.emplace_back(AttributeTraits {write, format, type}); + + return traits_array; + } +} // namespace utils } // namespace hdbpp_test \ No newline at end of file diff --git a/test/TestHelpers.hpp b/test/TestHelpers.hpp index 3000fceb95a3d98fb6003cea9dd4027d02a19cce..089cb7c0f129d4f23bfc95b3ae77a2f970e26f1f 100644 --- a/test/TestHelpers.hpp +++ b/test/TestHelpers.hpp @@ -28,6 +28,7 @@ #include <memory> #include <random> #include <string> +#include <vector> namespace hdbpp_test { @@ -221,7 +222,7 @@ namespace data_gen for (int i = 0; i < size; i++) value->push_back(d(gen)); - return std::move(value); + return value; } template<Tango::CmdArgType Type> @@ -246,5 +247,11 @@ namespace data_gen return generateScalarData<Type>(empty_data); } } // namespace data_gen + +namespace utils +{ + std::vector<hdbpp_internal::AttributeTraits> getTraits(); + std::vector<hdbpp_internal::AttributeTraits> getTraitsImplemented(); +} // namespace utils } // namespace hdbpp_test #endif // _TEST_HELPERS_HPP diff --git a/test/main.cpp b/test/main.cpp index b1a62c68e36b1f776b4856ace0050f6f671b3283..85da4ef4c68ed3b1566504d6a93856f0ab0185bf 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -24,12 +24,12 @@ int main(int argc, char *argv[]) { - hdbpp_internal::LogConfigurator::initLogging(); - //hdbpp_internal::LogConfigurator::initConsoleLogging(); + hdbpp_internal::LogConfigurator::initLogging("tests"); + //hdbpp_internal::LogConfigurator::initConsoleLogging("tests"); hdbpp_internal::LogConfigurator::setLoggingLevel(spdlog::level::err); int result = Catch::Session().run(argc, argv); - hdbpp_internal::LogConfigurator::shutdownLogging(); + hdbpp_internal::LogConfigurator::shutdownLogging("tests"); return result; }