1.9.0 rocthrust and thrust conflict issues

[cuihaoran21]

/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/uninitialized_copy.h:65:13: error: no matching 'operator new' function for non-allocating placement new expression; include
::new(static_cast<void*>(&out)) OutputType(in);
^~
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/parallel_for.h:72:17: note: in instantiation of function template specialization 'thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>::operator()' requested here
f(offset + tile_base + idx);
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/parallel_for.h:109:48: note: in instantiation of function template specialization 'thrust::hip_rocprim::__parallel_for::kernel<256U, thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>, long, 1U>' requested here
hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel<block_size, F, Size, items_per_thread>),
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/parallel_for.h:147:50: note: in instantiation of function template specialization 'thrust::hip_rocprim::__parallel_for::parallel_for<thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>, long>' requested here
hipError_t status = __parallel_for::parallel_for(count, f, stream);
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/parallel_for.h:135:12: note: in instantiation of member function 'thrust::hip_rocprim::parallel_for(execution_policythrust::hip_rocprim::execute_on_stream &, thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>, long)::workaround::par' requested here
struct workaround
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/uninitialized_copy.h:80:18: note: in instantiation of function template specialization 'thrust::hip_rocprim::parallel_for<thrust::hip_rocprim::execute_on_stream, thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>, long>' requested here
hip_rocprim::parallel_for(policy, functor_t(first, result), count);
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/internal/copy_cross_system.h:221:18: note: (skipping 9 contexts in backtrace; use -ftemplate-backtrace-limit=0 to see all)
hip_rocprim::uninitialized_copy_n(device_s, first, num_items, d_in_ptr.data());
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/reduce.h:150:24: note: in instantiation of function template specialization 'thrust::hip_rocprim::__reduce::reduce<thrust::hip_rocprim::execute_on_stream, thrust::hip_rocprim::transform_input_iterator_t<thrust::tuple<unsigned long, unsigned long>, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9)>, long, thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
return __reduce::reduce(
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/reduce.h:136:10: note: in instantiation of member function 'thrust::hip_rocprim::reduce_n(execution_policythrust::hip_rocprim::execute_on_stream &, thrust::hip_rocprim::transform_input_iterator_t<thrust::tuple<unsigned long, unsigned long>, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9)>, long, thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9))::workaround::par' requested here
struct workaround
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/transform_reduce.h:52:12: note: in instantiation of function template specialization 'thrust::hip_rocprim::reduce_n<thrust::hip_rocprim::execute_on_stream, thrust::hip_rocprim::transform_input_iterator_t<thrust::tuple<unsigned long, unsigned long>, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9)>, long, thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
return reduce_n(
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/detail/transform_reduce.inl:43:10: note: in instantiation of function template specialization 'thrust::hip_rocprim::transform_reduce<thrust::hip_rocprim::execute_on_stream, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9), thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
return transform_reduce(thrust::detail::derived_cast(thrust::detail::strip_const(exec)), first, last, unary_op, init, binary_op);
^
/work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:101:38: note: in instantiation of function template specialization 'thrust::transform_reduce<thrust::hip_rocprim::execute_on_stream, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9), thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
auto num_elements_pair = thrust::transform_reduce(
^
In file included from /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:7:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/binary_search.h:1898:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/detail/binary_search.inl:23:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/system/detail/generic/binary_search.h:172:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/system/detail/generic/binary_search.inl:26:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/for_each.h:277:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/detail/for_each.inl:24:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/system/detail/adl/for_each.h:44:
In file included from /public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/for_each.h:35:
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/parallel_for.h:161:17: error: no matching function for call to object of type 'thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>'
f(idx);
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/parallel_for.h:135:12: note: in instantiation of member function 'thrust::hip_rocprim::parallel_for(execution_policythrust::hip_rocprim::execute_on_stream &, thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>, long)::workaround::seq' requested here
struct workaround
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/uninitialized_copy.h:80:18: note: in instantiation of function template specialization 'thrust::hip_rocprim::parallel_for<thrust::hip_rocprim::execute_on_stream, thrust::hip_rocprim::__uninitialized_copy::functor<thrust::tuple<unsigned long, unsigned long> *, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>, long>' requested here
hip_rocprim::parallel_for(policy, functor_t(first, result), count);
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/internal/copy_cross_system.h:221:18: note: in instantiation of function template specialization 'thrust::hip_rocprim::uninitialized_copy_n<thrust::hip_rocprim::execute_on_stream, thrust::tuple<unsigned long, unsigned long> *, long, thrust::pointer<thrust::tuple<unsigned long, unsigned long>, thrust::hip_rocprim::execute_on_stream>>' requested here
hip_rocprim::uninitialized_copy_n(device_s, first, num_items, d_in_ptr.data());
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/internal/copy_cross_system.h:270:20: note: in instantiation of function template specialization 'thrust::hip_rocprim::__copy::cross_system_copy_n_dh_nt<thrust::hip_rocprim::execute_on_stream, thrust::system::cpp::detail::tag, thrust::tuple<unsigned long, unsigned long> *, long, thrust::tuple<unsigned long, unsigned long> *>' requested here
result = cross_system_copy_n_dh_nt(device_s, host_s, first, num_items, result);
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/internal/copy_cross_system.h:252:10: note: in instantiation of member function 'thrust::hip_rocprim::__copy::cross_system_copy_n(thrust::hip_rocprim::execution_policy<execute_on_stream> &, thrust::cpp::execution_policy &, thrust::tuple<unsigned long, unsigned long> *, long, thrust::tuple<unsigned long, unsigned long> *, thrust::detail::false_type)::workaround::par' requested here
struct workaround
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/internal/copy_cross_system.h:303:16: note: (skipping 6 contexts in backtrace; use -ftemplate-backtrace-limit=0 to see all)
return cross_system_copy_n(
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/reduce.h:150:24: note: in instantiation of function template specialization 'thrust::hip_rocprim::__reduce::reduce<thrust::hip_rocprim::execute_on_stream, thrust::hip_rocprim::transform_input_iterator_t<thrust::tuple<unsigned long, unsigned long>, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9)>, long, thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
return reduce::reduce(
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/reduce.h:136:10: note: in instantiation of member function 'thrust::hip_rocprim::reduce_n(execution_policythrust::hip_rocprim::execute_on_stream &, thrust::hip_rocprim::transform_input_iterator_t<thrust::tuple<unsigned long, unsigned long>, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9)>, long, thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9))::workaround::par' requested here
struct workaround
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/transform_reduce.h:52:12: note: in instantiation of function template specialization 'thrust::hip_rocprim::reduce_n<thrust::hip_rocprim::execute_on_stream, thrust::hip_rocprim::transform_input_iterator_t<thrust::tuple<unsigned long, unsigned long>, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9)>, long, thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
return reduce_n(
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/detail/transform_reduce.inl:43:10: note: in instantiation of function template specialization 'thrust::hip_rocprim::transform_reduce<thrust::hip_rocprim::execute_on_stream, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9), thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
return transform_reduce(thrust::detail::derived_cast(thrust::detail::strip_const(exec)), first, last, unary_op, init, binary_op);
^
/work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:101:38: note: in instantiation of function template specialization 'thrust::transform_reduce<thrust::hip_rocprim::execute_on_stream, thrust::zip_iterator<thrust::tuple<const unsigned long *, const unsigned long *>>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:103:9), thrust::tuple<unsigned long, unsigned long>, (lambda at /work/home/zzuchr/project/ginkgo-1.9.0/common/cuda_hip/distributed/matrix_kernels.cpp:116:9)>' requested here
auto num_elements_pair = thrust::transform_reduce(
^
/public/software/compiler/rocm/dtk-23.10/include/thrust/system/hip/detail/uninitialized_copy.h:60:41: note: candidate template ignored: substitution failure [with Size = long]
void THRUST_HIP_DEVICE_FUNCTION operator()(Size idx)
^
2 errors generated when compiling for gfx906.
gmake[2]: *** [hip/CMakeFiles/ginkgo_hip.dir/build.make:1588: hip/CMakeFiles/ginkgo_hip.dir//common/cuda_hip/distributed/matrix_kernels.cpp.o] Error 1
gmake[1]: *** [CMakeFiles/Makefile2:7379: hip/CMakeFiles/ginkgo_hip.dir/all] Error 2

[upsj]

Thanks for the report! Can you please add some more information about your build environment (ROCm version, rocThrust version, Ginkgo commit hash, CMake configuration and CMakeCache.txt file)? Otherwise we will not be able to reproduce this build failure.

[cuihaoran21]

HI！ cmake/3.23.3、devtoolset/7.3.1、gcc-7.3.1、anaconda3/5.2.0、dtk/23.10（ROCm version<5.7）

Summary of Configuration for Ginkgo (version 1.9.0 with tag main)

--
--    Ginkgo configuration:
--        CMAKE_BUILD_TYPE:                                Release
--        BUILD_SHARED_LIBS:                               ON
--        CMAKE_INSTALL_PREFIX:                          ~ /projects/ginkgo
--        PROJECT_SOURCE_DIR:                            ~ /project/ginkgo-1.9.0
--        PROJECT_BINARY_DIR:                             ~ /project/ginkgo-1.9.0/build
--        CMAKE_CXX_COMPILER:                           GNU 7.3.1 on platform Linux x86_64
--                                                         /opt/rh/devtoolset-7/root/usr/bin/c++
--    User configuration:
--      Enabled modules:
--        GINKGO_BUILD_OMP:                                ON
--        GINKGO_BUILD_MPI:                                ON
--        GINKGO_BUILD_REFERENCE:                          ON
--        GINKGO_BUILD_CUDA:                               OFF
--        GINKGO_BUILD_HIP:                                ON
--        GINKGO_BUILD_SYCL:                               OFF
--      Enabled features:
--        GINKGO_MIXED_PRECISION:                          OFF
--        GINKGO_HAVE_GPU_AWARE_MPI:                       OFF
--        GINKGO_ENABLE_HALF:                              ON
--      Tests, benchmarks and examples:
--        GINKGO_BUILD_TESTS:                              ON
--        GINKGO_FAST_TESTS:                               OFF
--        GINKGO_BUILD_EXAMPLES:                           ON
--        GINKGO_EXTLIB_EXAMPLE:                           
--        GINKGO_BUILD_BENCHMARKS:                         ON
--        GINKGO_BENCHMARK_ENABLE_TUNING:                  OFF
--      Documentation:
--        GINKGO_BUILD_DOC:                                OFF
--        GINKGO_VERBOSE_LEVEL:                            1
--    
---------------------------------------------------------------------------------------------------------
--
--      Developer Tools:
--        GINKGO_DEVEL_TOOLS:                              OFF
--        GINKGO_WITH_CLANG_TIDY:                          OFF
--        GINKGO_WITH_IWYU:                                OFF
--        GINKGO_CHECK_CIRCULAR_DEPS:                      OFF
--        GINKGO_WITH_CCACHE:                              ON
---------------------------------------------------------------------------------------------------------
--
--      Components:
--        GINKGO_BUILD_PAPI_SDE:                           OFF
--        GINKGO_BUILD_HWLOC:                              OFF

CMakeCache.txt