Error building with "-DCMAKE_BUILD_TYPE=Release" on Summit about omega_h

[cwsmith]

It looks like the line continuation character (\) is missing from this line of the script:

cmake $ohsrc -DCMAKE_BUILD_TYPE=Release

[zhangchonglin]

@cwsmith: Thanks for the quick reply. I must forgot that in the attempt to reproduce the error message for this issue. But with that fix, there are other errors coming out:

[  5%] Building CUDA object src/CMakeFiles/omega_h.dir/Omega_h_amr_transfer.cpp.o
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/core/agent_launcher.h: In instantiation of 'void thrust::cuda_cub::core::__wrapper__device_stub__kernel_agent(_0&, _1&) [with <template-parameter-1-1> = thrust::cuda_cub::__parallel_for::ParallelForAgent<thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >, int>; _0 = thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >; _1 = int]':
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/core/agent_launcher.h:313:82:   required from 'void thrust::cuda_cub::core::_kernel_agent(_0, _1) [with __T4 = thrust::cuda_cub::__parallel_for::ParallelForAgent<thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >, int>; _0 = thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >; _1 = int]'
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/core/agent_launcher.h:926:8:   required from 'void thrust::cuda_cub::core::AgentLauncher<Agent>::launch_impl(thrust::detail::true_type, _0, _1) const [with _0 = thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >; _1 = int; Agent = thrust::cuda_cub::__parallel_for::ParallelForAgent<thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >, int>; thrust::detail::true_type = thrust::detail::integral_constant<bool, true>]'
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/core/agent_launcher.h:1077:12:   required from 'void thrust::cuda_cub::core::AgentLauncher<Agent>::launch(_0, _1) const [with _0 = thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >; _1 = int; Agent = thrust::cuda_cub::__parallel_for::ParallelForAgent<thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >, int>]'
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/parallel_for.h:142:1:   required from 'cudaError_t thrust::cuda_cub::__parallel_for::parallel_for(Size, F, cudaStream_t) [with F = thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >; Size = int; cudaError_t = cudaError; cudaStream_t = CUstream_st*]'
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/parallel_for.h:164:50:   required from 'void thrust::cuda_cub::parallel_for(thrust::cuda_cub::execution_policy<Derived>&, F, Size) [with Derived = thrust::cuda_cub::par_t; F = thrust::cuda_cub::for_each_f<Omega_h::IntIterator, thrust::detail::wrapped_function<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >, void> >; Size = int]'
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/for_each.h:79:23:   required from 'Input thrust::cuda_cub::for_each_n(thrust::cuda_cub::execution_policy<Derived>&, Input, Size, UnaryOp) [with Derived = thrust::cuda_cub::par_t; Input = Omega_h::IntIterator; Size = int; UnaryOp = __nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >]'
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/for_each.h:103:28:   required from 'Input thrust::cuda_cub::for_each(thrust::cuda_cub::execution_policy<Derived>&, Input, Input, UnaryOp) [with Derived = thrust::cuda_cub::par_t; Input = Omega_h::IntIterator; UnaryOp = __nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >]'
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/detail/for_each.inl:44:16:   required from 'InputIterator thrust::for_each(const thrust::detail::execution_policy_base<DerivedPolicy>&, InputIterator, InputIterator, UnaryFunction) [with DerivedPolicy = thrust::cuda_cub::par_t; InputIterator = Omega_h::IntIterator; UnaryFunction = __nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >]'
/ccs/home/zhangc20/pumi-pic/omega_h/src/Omega_h_for.hpp:39:17:   required from 'void Omega_h::for_each(InputIterator, InputIterator, UnaryFunction&&) [with InputIterator = Omega_h::IntIterator; UnaryFunction = __nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >&]'
/ccs/home/zhangc20/pumi-pic/omega_h/src/Omega_h_for.hpp:57:18:   required from 'void Omega_h::parallel_for(Omega_h::LO, UnaryFunction&&) [with UnaryFunction = __nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Graph (*)(int, Omega_h::Adj, Omega_h::Adj, Omega_h::Read<signed char>), Omega_h::elements_across_sides, 1>, const int, const Omega_h::LOs, const Omega_h::Read<signed char>, const Omega_h::LOs, Omega_h::Write<int> >; Omega_h::LO = int]'
/ccs/home/zhangc20/pumi-pic/omega_h/src/Omega_h_adj.cpp:759:38:   required from here
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/core/agent_launcher.h:313:42: error: unused parameter '__T2' [-Werror=unused-parameter]
   void __global__ _kernel_agent(_0,_1) {}
                                          ^   
/sw/summit/cuda/10.1.243/bin/../targets/ppc64le-linux/include/thrust/system/cuda/detail/core/agent_launcher.h:313:52: error: unused parameter '__T3' [-Werror=unused-parameter]
   void __global__ _kernel_agent(_0,_1) {}
                                                    ^   
[  5%] Building CUDA object src/CMakeFiles/omega_h.dir/Omega_h_any.cpp.o
[  6%] Building CUDA object src/CMakeFiles/omega_h.dir/Omega_h_approach.cpp.o
cc1plus: all warnings being treated as errors
make[2]: *** [src/CMakeFiles/omega_h.dir/Omega_h_adj.cpp.o] Error 1
make[2]: *** Waiting for unfinished jobs....
make[1]: *** [src/CMakeFiles/omega_h.dir/all] Error 2
make: *** [all] Error 2

[cwsmith]

You're welcome. What branch/commit of omega_h is this?

[zhangchonglin]

It's on the master branch at commit: 466e470

[zhangchonglin]

I could confirm this same issue on Centos7 system.

[cwsmith]

Hmm. Thank you. I'll give it a shot.

[zhangchonglin]

Different crash building on AiMOS with -DCMAKE_BUILD_TYPE=Release:

[ 79%] Building CUDA object src/CMakeFiles/omega_h.dir/Omega_h_coarsen_flip.cpp.o
nvcc_internal_extended_lambda_implementation: In instantiation of ‘struct __nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Reals (*)(Omega_h::Mesh*, Omega_h::Bytes, Omega_h::Bytes, double), Omega_h::compute_flip_normals_dim<3>, 1>, Omega_h::LOs, Omega_h::Adj, Omega_h::Bytes, Omega_h::LOs, Omega_h::Reals, double, Omega_h::Write<double> >’:
/gpfs/u/barn/MPFS/MPFScgzg/pumi-pic/install/kokkos/install/include/Cuda/Kokkos_Cuda_Parallel.hpp:577:19:   required from ‘class Kokkos::Impl::ParallelFor<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Reals (*)(Omega_h::Mesh*, Omega_h::Bytes, Omega_h::Bytes, double), Omega_h::compute_flip_normals_dim<3>, 1>, Omega_h::LOs, Omega_h::Adj, Omega_h::Bytes, Omega_h::LOs, Omega_h::Reals, double, Omega_h::Write<double> >, Kokkos::RangePolicy<Kokkos::Cuda, Kokkos::Schedule<Kokkos::Static>, int>, Kokkos::Cuda>’
/tmp/tmpxft_00244cb1_00000000-6_Omega_h_coarsen_flip.cudafe1.stub.c:21:538:   required from here
nvcc_internal_extended_lambda_implementation:168:46: error: static assertion failed: nvcc internal error: unexpected number of captures!
nvcc_internal_extended_lambda_implementation: In instantiation of ‘struct __nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Reals (*)(Omega_h::Mesh*, Omega_h::Bytes, Omega_h::Bytes, double), Omega_h::compute_flip_normals_dim<2>, 1>, Omega_h::LOs, Omega_h::Adj, Omega_h::Bytes, Omega_h::LOs, Omega_h::Reals, double, Omega_h::Write<double> >’:
/gpfs/u/barn/MPFS/MPFScgzg/pumi-pic/install/kokkos/install/include/Cuda/Kokkos_Cuda_Parallel.hpp:577:19:   required from ‘class Kokkos::Impl::ParallelFor<__nv_dl_wrapper_t<__nv_dl_tag<Omega_h::Reals (*)(Omega_h::Mesh*, Omega_h::Bytes, Omega_h::Bytes, double), Omega_h::compute_flip_normals_dim<2>, 1>, Omega_h::LOs, Omega_h::Adj, Omega_h::Bytes, Omega_h::LOs, Omega_h::Reals, double, Omega_h::Write<double> >, Kokkos::RangePolicy<Kokkos::Cuda, Kokkos::Schedule<Kokkos::Static>, int>, Kokkos::Cuda>’
/tmp/tmpxft_00244cb1_00000000-6_Omega_h_coarsen_flip.cudafe1.stub.c:29:538:   required from here
nvcc_internal_extended_lambda_implementation:168:46: error: static assertion failed: nvcc internal error: unexpected number of captures!
make[2]: *** [src/CMakeFiles/omega_h.dir/build.make:1473: src/CMakeFiles/omega_h.dir/Omega_h_coarsen_flip.cpp.o] Error 1
make[1]: *** [CMakeFiles/Makefile2:304: src/CMakeFiles/omega_h.dir/all] Error 2

Build script:

module use /gpfs/u/software/dcs-rhel8-spack-install/v0162gccSpectrum/lmod/linux-rhel8-ppc64le/Core/
module load spectrum-mpi/10.4-2ycgnlq
module load cmake/3.20.0/1
module load cuda/11.1-3vj4c72 \

export PATH=$cuda/bin:$PATH
export LD_LIBRARY_PATH=$cuda/lib64:$LD_LIBRARY_PATH
export installroot=$PWD
export kk=$installroot/kokkos/install # this is where kokkos will be installed
export srcroot=$installroot/../
export kksrc=$srcroot/kokkos

export oh=$installroot/omega_h/install # this is where omega_h will be installed
export ohsrc=$srcroot/omega_h

export OMPI_CXX=$kksrc/bin/nvcc_wrapper
export OMPI_CC=gcc

cd $installroot
mkdir -p omega_h/build
cd omega_h/build
cmake $ohsrc -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$oh -DBUILD_SHARED_LIBS=OFF \
             -DOmega_h_USE_Kokkos=ON -DOmega_h_CUDA_ARCH=70 \
             -DOmega_h_USE_CUDA=on -DOmega_h_USE_MPI=on \
             -DCMAKE_CXX_COMPILER=g++ \
             -DKokkos_PREFIX=$kk/lib64/cmake/

make -j4 install

[zhangchonglin]

Just an update, was building omega_h on Summit with RHEL8 update, still seeing the same issue.

[cwsmith]

@zhangchonglin Thanks for testing and the update.

[zhangchonglin]

Using main branch from: https://github.com/sandialabs/omega_h, with -DCMAKE_BUILD_TYPE=Release, it's working fine. It might be a good idea to merge that branch in to fix these build issues. @cwsmith.

[cwsmith]

Thanks for the update. Yes, the merge is a good idea.

[cwsmith]

The sandialabs main branch was merged into SCOREC master: #34 and should be fixed now.

The older SCOREC master OMEGA_H_CHECK wrapped assert around function calls that were required for correct execution and just skipped those calls when the code was built in release mode (NDEBUG defined). The fix is here:

omega_h/src/Omega_h_fail.hpp

Lines 39 to 84 in f58f1d9

	#if defined(OMEGA_H_USE_CUDA) && (defined(__clang__) || defined(_MSC_VER))
	#if defined(NDEBUG)
	# define OMEGA_H_CHECK(cond) static_cast<void>(cond)
	#else
	# define OMEGA_H_CHECK(cond) assert(static_cast<int>(static_cast<bool>(cond)))
	#endif
	#elif defined(__CUDA_ARCH__)
	#if defined(NDEBUG)
	# define OMEGA_H_CHECK(cond) static_cast<void>(cond)
	#else
	# define OMEGA_H_CHECK(cond) assert(static_cast<int>(static_cast<bool>(cond)))
	#endif
	#else
	# define OMEGA_H_CHECK(cond) \
	((cond) ? ((void)0) \
	: Omega_h::fail("assertion %s failed at %s +%d\n", #cond, __FILE__, __LINE__))
	#endif
	#ifndef NDEBUG
	# define OMEGA_H_CHECK_MSG(cond, a) do { \
	if (!(cond)) std::cout << "ERROR: " << a << std::endl; \
	OMEGA_H_CHECK(cond) ; \
	} while(0)
	# define OMEGA_H_CHECK_PRINTF(cond, format, ...) do { \
	if (!(cond)) { \
	std::printf("ERROR: " format "\n", __VA_ARGS__); \
	} \
	OMEGA_H_CHECK(cond) ; \
	} while(0)
	# define OMEGA_H_CHECK_OP(l,op,r) do { \
	if (!((l) op (r))) { \
	std::printf("ERROR: !( %s %s %s ) : ! ( %s %s %s )\n", #l, #op, #r, std::to_string(l).c_str(), #op, std::to_string(r).c_str()); \
	} \
	OMEGA_H_CHECK((l) op (r)); \
	} while(0)
	# ifdef OMEGA_H_ENABLE_DEMANGLED_STACKTRACE
	# define OMEGA_H_CHECK_OP_1(l,op,r) \
	((!((l) op (r))) ? ((void)0) \
	: Omega_h::fail("assertion %s %s %s [%d %s %d] failed at %s +%d\n%s\n", #l, #op, #r, l, #op, r, __FILE__, __LINE__))
	# endif
	#else // NDEBUG
	# define OMEGA_H_CHECK_MSG(cond, a) OMEGA_H_CHECK(cond)
	# define OMEGA_H_CHECK_PRINTF(cond, format, ...) OMEGA_H_CHECK(cond)
	# define OMEGA_H_CHECK_OP(l,op,r) OMEGA_H_CHECK((l) op (r))
	#endif // NDEBUG