GPU Proxy throws a CUFFT_INVALID_PLAN exception
Using the GPU from wsclean results in an exception being thrown
wsclean -gridder idg -idg-mode gpu ...
#0 0x00007fffefb06bce in __cxxabiv1::__cxa_throw (obj=0x36f0600, tinfo=0x7fffecf45670 <typeinfo for cufft::Error>, dest=0x7fffeccd95a6 <cufft::Error::~Error()>)
at /tmp/spack/spack-stage/spack-stage-gcc-9.4.0-ljb4ejaog46e2soro4seyp3ueya2uvyc/spack-src/libstdc++-v3/libsupc++/eh_throw.cc:78
#1 0x00007fffeccd88ae in cufft::FFT::checkCuFFTCall (this=0x2fdbd70, result=CUFFT_INVALID_SIZE) at /home/bvdtol/schaap/idg/build/_deps/cudawrappers-src/include/cudawrappers/cufft.hpp:123
#2 0x00007fffeccd7f4f in cufft::FFT2D<(cudaDataType_t)4>::FFT2D (this=0x2fdbd70, nx=1152, ny=1152, stride=1, dist=1327104, batch=0)
at /home/bvdtol/schaap/idg/build/_deps/cudawrappers-src/include/cudawrappers/cufft.hpp:190
#3 0x00007fffeccd8c39 in std::make_unique<cufft::FFT2D<(cudaDataType_t)4>, unsigned long const&, unsigned long const&, unsigned long const&, unsigned long const&, unsigned long const&> ()
at /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-9.4.0/gcc-9.4.0-ljb4ejaog46e2soro4seyp3ueya2uvyc/lib/gcc/x86_64-pc-linux-gnu/9.4.0/../../../../include/c++/9.4.0/bits/unique_ptr.h
:857
#4 0x00007fffeccd8250 in idg::kernel::cuda::KernelFFTImpl::KernelFFTImpl (this=0x2fa9940, stream=..., parameters=...)
at /home/bvdtol/schaap/idg/idg/idg-lib/src/CUDA/common/kernels/KernelFFT.cpp:45
#5 0x00007fffeccd89dd in std::make_unique<idg::kernel::cuda::KernelFFTImpl, cu::Stream&, idg::kernel::cuda::KernelFFT::Parameters const&> ()
at /var/software/spack/opt/spack/linux-rocky8-zen2/gcc-9.4.0/gcc-9.4.0-ljb4ejaog46e2soro4seyp3ueya2uvyc/lib/gcc/x86_64-pc-linux-gnu/9.4.0/../../../../include/c++/9.4.0/bits/unique_ptr.h
:857
#6 0x00007fffeccd816f in idg::kernel::cuda::KernelFFT::create (stream=..., parameters=...) at /home/bvdtol/schaap/idg/idg/idg-lib/src/CUDA/common/kernels/KernelFFT.cpp:36
#7 0x00007fffeccb0b0b in idg::kernel::cuda::InstanceCUDA::plan_batched_fft (this=0xc67770, size=1152, batch=0) at /home/bvdtol/schaap/idg/idg/idg-lib/src/CUDA/common/InstanceCUDA.cpp:353
#8 0x00007fffecccdc25 in idg::proxy::cuda::CUDA::run_wtiles_to_grid (this=0xdbc780, subgrid_size=56, image_size=0.223402143, w_step=320.586548, shift=..., wtile_flush_info=...)
at /home/bvdtol/schaap/idg/idg/idg-lib/src/CUDA/common/routines/WTiling.cpp:232
#9 0x00007fffecccfebd in idg::proxy::cuda::CUDA::flush_wtiles (this=0xdbc780) at /home/bvdtol/schaap/idg/idg/idg-lib/src/CUDA/common/routines/WTiling.cpp:732
#10 0x00007fffeccdc2e3 in idg::proxy::cuda::Generic::get_final_grid (this=0xdbc780) at /home/bvdtol/schaap/idg/idg/idg-lib/src/CUDA/Generic/Generic.cpp:98
#11 0x00007ffff23caf9e in idg::api::GridderBufferImpl::finished (this=0x1607330) at /home/bvdtol/schaap/idg/idg/idg-api/GridderBuffer.cpp:255
#12 0x00007ffff23aa73b in idg::api::BufferSetImpl::finished (this=0xd1ccc0) at /home/bvdtol/schaap/idg/idg/idg-api/BufferSet.cpp:762
#13 0x00000000006ac448 in IdgMsGridder::gridMeasurementSet(MSGridderBase::MSData const&) ()
#14 0x00000000006acabb in IdgMsGridder::Invert() ()
#15 0x00000000004e989c in GriddingTaskManager::runDirect(GriddingTask&&, MSGridderBase&) ()
#16 0x00000000004e9c49 in GriddingTaskManager::RunDirect(GriddingTask&&) ()
#17 0x00000000004e9c8c in GriddingTaskManager::Run(GriddingTask&&, std::function<void (GriddingResult&)>) ()
#18 0x00000000004796b5 in WSClean::imagePSF(ImagingTableEntry&) ()
#19 0x0000000000489de5 in WSClean::runIndependentGroup(ImagingTable&, std::unique_ptr<PrimaryBeam, std::default_delete<PrimaryBeam> >&) ()
#20 0x000000000048d99a in WSClean::RunClean() ()
#21 0x0000000000443663 in main ()
Edited by Bas van der Tol