Switch to cmake

.gitignore

+Testing
+*.d
+*~

.gitmodules

 [submodule "external/cuda-wrappers"]
-	path = external/cuda-wrappers
+	path = external/cudawrappers
 	url = https://github.com/nlesc-recruit/CUDA-wrappers.git

CMakeLists.txt

+cmake_minimum_required(VERSION 3.17 FATAL_ERROR)
+
+project(
+  tensor_core_correlator
+  DESCRIPTION "Tensor-Core Correlator"
+  VERSION 0.5
+  HOMEPAGE_URL https://git.astron.nl/RD/tensor-core-correlator
+  LANGUAGES CXX CUDA)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_BUILD_TYPE Release)
+option(BUILD_SHARED_LIBS "Create shared libraries" True)
+option(BUILD_TESTING "Build tests" False)
+
+# Use CMAKE_INSTALL_PREFIX when searching for libraries
+include(GNUInstallDirs)
+list(APPEND CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR})
+
+# Add cudawrappers dependency
+add_subdirectory(external/cudawrappers)
+
+# Set up libtcc
+add_subdirectory(libtcc)
+
+# Set up tests
+include(CTest)
+if(BUILD_TESTING)
+  add_subdirectory(test)
+endif()
+
+# Install project cmake targets
+include(CMakePackageConfigHelpers)
+configure_file(cmake/${PROJECT_NAME}-config.cmake.in
+               ${PROJECT_NAME}-config.cmake @ONLY)
+write_basic_package_version_file(
+  ${PROJECT_NAME}-config-version.cmake
+  VERSION ${cudawrappers_VERSION}
+  COMPATIBILITY AnyNewerVersion)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake
+              ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+
+# --- auto-ignore build directory
+if(NOT EXISTS ${PROJECT_BINARY_DIR}/.gitignore)
+  file(WRITE ${PROJECT_BINARY_DIR}/.gitignore "*")
+endif()

Makefile

-VERSION=		0.5
-CUDA=			$(shell dirname `dirname \`which nvcc\``)
-#CUDA=			/usr/local/cuda
-CUDA_INCLUDE=		$(shell dirname `find $(CUDA) -name cuda.h`)
-CUDA_LIBDIR=		$(shell dirname `find $(CUDA) -name libcuda.so`|head -n1)
-NVRTC_LIBDIR=		$(shell dirname `find $(CUDA) -name libnvrtc.so`|head -n1)
-#POWER_SENSOR=		$(HOME)/projects/libpowersensor-master/build
-ARCH=			$(shell arch)
-CC=			gcc
-CXX=			g++ #-Wno-deprecated-declarations
-NVCC=			nvcc
-INCLUDES=		-I.
-INCLUDES+=		-I$(CUDA_INCLUDE)
-#INCLUDES+=		-I$(CUDA_INCLUDE) -I$(NVRTC_INCLUDE)
-#INCLUDES+=		-I$(POWER_SENSOR)/include
-CXXFLAGS+=		-std=c++11 -O3 -g -fpic -fopenmp $(INCLUDES) -DNDEBUG
-NVCCFLAGS=		$(INCLUDES)
-
-#CXXFLAGS+=		-march=core-avx2 -mcmodel=medium
-
-LIBTCC_SOURCES=		libtcc/CorrelatorKernel.cc\
-			libtcc/Correlator.cc\
-			libtcc/Kernel.cc
-
-
-CORRELATOR_TEST_SOURCES=test/CorrelatorTest/CorrelatorTest.cc\
-			test/CorrelatorTest/Options.cc\
-			test/Common/Record.cc\
-			test/Common/UnitTest.cc
-
-OPENCL_TEST_SOURCES=	test/OpenCLCorrelatorTest/OpenCLCorrelatorTest.cc
-
-SIMPLE_EXAMPLE_SOURCES=	test/SimpleExample/SimpleExample.cu
-
-
-LIBTCC_OBJECTS=		$(LIBTCC_SOURCES:%.cc=%.o) libtcc/TCCorrelator.o
-SIMPLE_EXAMPLE_OBJECTS=	$(SIMPLE_EXAMPLE_SOURCES:%.cu=%.o)
-CORRELATOR_TEST_OBJECTS=$(CORRELATOR_TEST_SOURCES:%.cc=%.o)
-OPENCL_TEST_OBJECTS=	$(OPENCL_TEST_SOURCES:%.cc=%.o)
-
-OBJECTS=		$(LIBTCC_OBJECTS)\
-			$(SIMPLE_EXAMPLE_OBJECTS)\
-			$(CORRELATOR_TEST_OBJECTS)\
-			$(OPENCL_TEST_OBJECTS)
-
-SHARED_OBJECTS=		libtcc/libtcc.so libtcc/libtcc.so.$(VERSION)
-
-DEPENDENCIES=		$(OBJECTS:%.o=%.d)
-
-EXECUTABLES=		test/SimpleExample/SimpleExample\
-			test/CorrelatorTest/CorrelatorTest\
-			test/OpenCLCorrelatorTest/OpenCLCorrelatorTest
-
-CUDA_WRAPPERS_DIR=       external/cuda-wrappers
-CUDA_WRAPPERS_LIB=        $(CUDA_WRAPPERS_DIR)/libcu.so
-CUDA_WRAPPERS_INCLUDE=    $(CUDA_WRAPPERS_DIR)/cu
-#LIBTCC_OBJECTS+=         $(CUDA_WRAPPERS_LIB)
-
-LIBRARIES=		-L$(CUDA_LIBDIR) -lcuda\
-			$(CUDA_WRAPPERS_LIB) \
-			-L$(NVRTC_LIBDIR) -lnvrtc #\
-			#-L$(POWER_SENSOR)/lib -lpowersensor -lnvidia-ml
-
-
-%.d:			%.cc
-			-$(CXX) $(CXXFLAGS) -MM -MT $@ -MT ${@:%.d=%.o} -MT ${@:%.d=%.s} $< -o $@
-
-%.d:			%.cu
-			-$(CXX) -x c++ $(CXXFLAGS) -MM -MT $@ -MT ${@:%.d=%.o} -MT ${@:%.d=%.s} $< -o $@
-
-%.o:			%.cc
-			$(CXX) $(CXXFLAGS) -o $@ -c $<
-
-%.o:			%.cu
-			$(NVCC) $(NVCCFLAGS) -o $@ -c $<
-
-%.s:			%.cc
-			$(CXX) $(CXXFLAGS) -o $@ -S $<
-
-%.so:			%.so.$(VERSION)
-			rm -f $@
-			ln -s $(@F).$(VERSION) $@
-
-all::			$(EXECUTABLES)
-
-clean::
-			$(RM) $(OBJECTS) $(SHARED_OBJECTS) $(DEPENDENCIES) $(EXECUTABLES)
-
-$(CUDA_WRAPPERS_LIB):
-			cd $(CUDA_WRAPPERS_DIR) && cmake .
-			cd $(CUDA_WRAPPERS_DIR) && CPATH=$(CPATH):$(CUDA_INCLUDE) make
-
-libtcc/TCCorrelator.o:	libtcc/TCCorrelator.cu	# CUDA code embedded in object file
-			ld -r -b binary -o $@ $<
-
-libtcc/TCCorrelator.d:
-			-
-
-libtcc/libtcc.so.$(VERSION):			$(LIBTCC_OBJECTS) $(CUDA_WRAPPERS_LIB)
-			$(CXX) -shared -o $@ $^ $(LIBRARIES)
-
-test/SimpleExample/SimpleExample:		$(SIMPLE_EXAMPLE_OBJECTS) libtcc/libtcc.so
-			$(NVCC) $(NVCCFLAGS) -o $@ $(SIMPLE_EXAMPLE_OBJECTS) -Xlinker -rpath=$(CUDA_WRAPPERS_DIR) -Llibtcc -ltcc $(LIBRARIES)
-
-test/CorrelatorTest/CorrelatorTest:		$(CORRELATOR_TEST_OBJECTS) libtcc/libtcc.so
-			$(CXX) $(CXXFLAGS) -o $@ $(CORRELATOR_TEST_OBJECTS) -Wl,-rpath=$(CUDA_WRAPPERS_DIR) -Llibtcc -ltcc $(LIBRARIES)
-
-test/OpenCLCorrelatorTest/OpenCLCorrelatorTest:	$(OPENCL_TEST_OBJECTS)
-			$(CXX) $(CXXFLAGS) -o $@ $(OPENCL_TEST_OBJECTS) -L$(CUDA)/lib64 -lOpenCL
-
-ifeq (0, $(words $(findstring $(MAKECMDGOALS), clean)))
--include $(DEPENDENCIES)
-endif

README.md

@@ -65,4 +65,32 @@ Limitations:
 - the amount of samples over which is integrated) must be a multiple of 128 / `NR_BITS`
   (i.e., 32, 16, or 8 for 4-bit, 8-bit, or 16-bit input, respectively).
 
+## Building, testing, and installation
+
+To build and install the project, run:
+```bash
+cmake -S . -B build
+make -C build
+make -C build install
+```
+
+To install in a custom location, e.g. `~/.local`, run:
+```bash
+cmake -S . -B build -DCMAKE_INSTALL_PREFIX=$HOME/.local
+make -C build
+make -C build install
+```
+
+To compile and run the tests, run:
+```bash
+cmake -S. -B build -DBUILD_TESTING=ON
+make -C build
+make -C build test
+```
+The tests require a GPU.
+On the DAS cluster you can request a GPU node and run the tests with the command:
+```bash
+srun -N 1 -C gpunode --gres=gpu:1 make -C build test
+```
+
 Contact John Romein (romein@astron.nl) to report bugs/feedback

cmake/Utils.cmake

+# Make it possible to #include cuda source code
+function(include_cuda_code target input_file)
+  # Save file containing cuda code as a C++ raw string literal
+  file(READ ${input_file} content)
+  set(delim "for_c++_include")
+  set(content "R\"${delim}(\n${content})${delim}\"")
+  set(output_file "${CMAKE_CURRENT_BINARY_DIR}/${input_file}")
+  file(WRITE ${output_file} "${content}")
+  # Add save path to the include directories
+  get_filename_component(output_subdir ${input_file} DIRECTORY)
+  target_include_directories(
+    ${target} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/${output_subdir}")
+endfunction(include_cuda_code)

cmake/tensor_core_correlator-config.cmake.in

+include(CMakeFindDependencyMacro)
+
+find_package(CUDAToolkit @CUDA_MIN_VERSION@ REQUIRED)
+
+foreach(component ${@PROJECT_NAME@_FIND_COMPONENTS})
+  include(${CMAKE_CURRENT_LIST_DIR}/${component}-config.cmake)
+endforeach()

cuda-wrappers @ 884fbc7c

-Subproject commit 884fbc7c69617f0b4fbc6696435272a488f49716

cudawrappers @ ffacb0e3

+Subproject commit ffacb0e3c649a4d8999b3c7959ce12dc03d25bf7

libtcc/CMakeLists.txt

+# Create tcc library
+add_library(tcc)
+# Add source files
+target_sources(tcc PRIVATE Correlator.cc CorrelatorKernel.cc Kernel.cc)
+# Add public header
+set_target_properties(tcc PROPERTIES PUBLIC_HEADER Correlator.h)
+# Add includes
+target_include_directories(
+  tcc PUBLIC $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>
+             $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>)
+# Add links
+target_link_libraries(tcc PUBLIC cudawrappers::cu cudawrappers::nvrtc)
+# Install libraries and headers
+install(
+  TARGETS tcc
+  EXPORT tcc-config # export tcc cmake targets
+  COMPONENT tcc
+  PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME})
+# Install tcc cmake targets
+install(
+  EXPORT tcc-config
+  NAMESPACE ${PROJECT_NAME}::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+# Embed CUDA code
+include(${CMAKE_SOURCE_DIR}/cmake/Utils.cmake)
+include_cuda_code(tcc kernel/TCCorrelator.cu)

libtcc/Correlator.cc

@@ -82,7 +82,9 @@ cu::Module Correlator::compileModule(unsigned nrBits,
   // embed the CUDA source code in libtcc.so, so that it need not be installed separately
   // for runtime compilation
   // copy into std::string for '\0' termination
-  std::string source(&_binary_libtcc_TCCorrelator_cu_start, &_binary_libtcc_TCCorrelator_cu_end);
+  const std::string source =
+#include "TCCorrelator.cu"
+  ;
   nvrtc::Program program(source, "TCCorrelator.cu");
 #endif
 

libtcc/Correlator.h

 #if !defined TCC_CORRELATOR_H
 #define TCC_CORRELATOR_H
 
-#include "libtcc/CorrelatorKernel.h"
-#include "external/cuda-wrappers/cu/cu.h"
-#include "external/cuda-wrappers/cu/nvrtc.h"
-
 #include <string>
 
+#include <cu.hpp>
+#include <nvrtc.hpp>
+
+#include "libtcc/CorrelatorKernel.h"
 
 namespace tcc {
   class Correlator {

libtcc/Kernel.h

 #if !defined TCC_KERNEL_H
 #define TCC_KERNEL_H
 
-#include "external/cuda-wrappers/cu/cu.h"
-
 #include <stdint.h>
 
+#include <cu.hpp>
 
 namespace tcc {
   class Kernel

test/CMakeLists.txt

+add_subdirectory(Common)
+add_subdirectory(CorrelatorTest)
+add_subdirectory(OpenCLCorrelatorTest)
+add_subdirectory(SimpleExample)

test/Common/CMakeLists.txt

+foreach(component Record UnitTest)
+  add_library(${component})
+  target_sources(${component} PRIVATE ${component}.cc)
+  target_include_directories(${component} PRIVATE ${CMAKE_SOURCE_DIR})
+  target_link_libraries(${component} PRIVATE cudawrappers::cu)
+endforeach()

test/Common/Record.h

 #if !defined RECORD_H
 #define RECORD_H
 
-#include "test/Common/Config.h"
+#include <cu.hpp>
 
-#include "external/cuda-wrappers/cu/cu.h"
+#include "test/Common/Config.h"
 
 #if defined MEASURE_POWER
 #include <powersensor/NVMLPowerSensor.h>

test/Common/UnitTest.h

 #if !defined UNIT_TEST_H
 #define UNIT_TEST_H
 
+#include <cu.hpp>
+
 #include "test/Common/Record.h"
-#include "external/cuda-wrappers/cu/cu.h"
 
 #if defined MEASURE_POWER
 #include <powersensor/NVMLPowerSensor.h>

test/CorrelatorTest/CMakeLists.txt

+find_package(OpenMP REQUIRED)
+
+add_executable(CorrelatorTest)
+target_sources(CorrelatorTest PRIVATE CorrelatorTest.cc Options.cc)
+target_include_directories(CorrelatorTest PRIVATE ${CMAKE_SOURCE_DIR})
+target_link_libraries(CorrelatorTest PRIVATE tcc Record UnitTest
+                                             OpenMP::OpenMP_CXX)
+
+# Define various combinations of parameters to test:
+set(ARGS0 -b  4 -c 1 -n 1 -N 32 -r 1 -R 1 -t 32 -V 1)
+set(ARGS1 -b  8 -c 1 -n 1 -N 48 -r 1 -R 1 -t 16 -V 1)
+set(ARGS2 -b 16 -c 1 -n 1 -N 64 -r 1 -R 1 -t  8 -V 1)
+set(ARGS3 -b 16 -c 2 -n 3 -N 32 -r 4 -R 5 -t 64 -V 1)
+
+foreach(idx RANGE 3)
+  add_test(NAME CorrelatorTest${idx} COMMAND CorrelatorTest ${ARGS${idx}})
+endforeach()

test/CorrelatorTest/CorrelatorTest.cc

-#include "test/Common/ComplexInt4.h"
-#include "test/Common/Record.h"
 #include "test/CorrelatorTest/CorrelatorTest.h"
-#include "util/ExceptionPropagator.h"
-#include "external/cuda-wrappers/cu/nvrtc.h"
 
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
 
+#include <nvrtc.hpp>
+
+#include "test/Common/ComplexInt4.h"
+#include "test/Common/Record.h"
+#include "util/ExceptionPropagator.h"
+
 #define GNU_SOURCE
 #include <link.h>
 #include <omp.h>
 
-
 CorrelatorTest::CorrelatorTest(const Options &options)
 :
   UnitTest(options.deviceNumber),
@@ -223,17 +224,21 @@ template<typename SampleType, typename VisibilityType> void CorrelatorTest::veri
 
 int main(int argc, char *argv[])
 {
+  int err{0};
   try {
     cu::init();
     Options options(argc, argv);
     CorrelatorTest test(options);
   } catch (cu::Error &error) {
     std::cerr << "cu::Error: " << error.what() << std::endl;
+    err = 1;
   } catch (nvrtc::Error &error) {
     std::cerr << "nvrtc::Error: " << error.what() << std::endl;
+    err = 1;
   } catch (Options::Error &error) {
     std::cerr << "Options::Error: " << error.what() << std::endl;
+    err = 1;
   }
 
-  return 0;
+  return err;
 }