diff --git a/.upstream-tests/test/cuda/bad_atomic_alignment.pass.cpp b/.upstream-tests/test/cuda/bad_atomic_alignment.pass.cpp new file mode 100644 index 0000000000..71ccb2d0b8 --- /dev/null +++ b/.upstream-tests/test/cuda/bad_atomic_alignment.pass.cpp @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// UNSUPPORTED: libcpp-has-no-threads, pre-sm-60 +// UNSUPPORTED: windows && pre-sm-70 + +// + +// cuda::atomic + +// Original test issue: +// https://github.com/NVIDIA/libcudacxx/issues/160 + +#include + +template +__host__ __device__ +constexpr bool unused(T &&) {return true;} + +int main(int argc, char ** argv) +{ + // Test default aligned user type + { + struct key { + int32_t a; + int32_t b; + }; + static_assert(alignof(key) == 4, ""); + cuda::atomic k; + auto r = k.load(); + k.store(r); + (void)k.exchange(r); + unused(r); + } + // Test forcibly aligned user type + { + struct alignas(8) key { + int32_t a; + int32_t b; + }; + static_assert(alignof(key) == 8, ""); + cuda::atomic k; + auto r = k.load(); + k.store(r); + (void)k.exchange(r); + unused(r); + } + return 0; +} diff --git a/.upstream-tests/test/cuda/pipeline_arrive_on.pass.cpp b/.upstream-tests/test/cuda/pipeline_arrive_on.pass.cpp index 4fab0fdf2b..74bfe96040 100644 --- a/.upstream-tests/test/cuda/pipeline_arrive_on.pass.cpp +++ b/.upstream-tests/test/cuda/pipeline_arrive_on.pass.cpp @@ -11,6 +11,8 @@ // Remove after bump to version 4 #define _LIBCUDACXX_CUDA_ABI_VERSION 3 +// TODO: Remove pointless comparison suppression when compiler fixes short-circuiting +#pragma nv_diag_suppress 186 #pragma nv_diag_suppress static_var_with_dynamic_init #pragma nv_diag_suppress declared_but_not_referenced diff --git a/.upstream-tests/test/cuda/pipeline_arrive_on_abi_v2.pass.cpp b/.upstream-tests/test/cuda/pipeline_arrive_on_abi_v2.pass.cpp index 686ff43de1..61ad5d8e56 100644 --- a/.upstream-tests/test/cuda/pipeline_arrive_on_abi_v2.pass.cpp +++ b/.upstream-tests/test/cuda/pipeline_arrive_on_abi_v2.pass.cpp @@ -11,6 +11,9 @@ #define _LIBCUDACXX_CUDA_ABI_VERSION 2 +// TODO: Remove pointless comparison suppression when compiler fixes short-circuiting +#pragma nv_diag_suppress 186 + #pragma nv_diag_suppress static_var_with_dynamic_init #pragma nv_diag_suppress declared_but_not_referenced diff --git a/.upstream-tests/test/cuda/pipeline_group_concept.h b/.upstream-tests/test/cuda/pipeline_group_concept.h index 9069bca1ac..db55fac7b2 100644 --- a/.upstream-tests/test/cuda/pipeline_group_concept.h +++ b/.upstream-tests/test/cuda/pipeline_group_concept.h @@ -9,6 +9,9 @@ // UNSUPPORTED: pre-sm-70 +// TODO: Remove pointless comparison suppression when compiler fixes short-circuiting +#pragma nv_diag_suppress 186 + #include template diff --git a/.upstream-tests/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp b/.upstream-tests/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp index 6eb6c342e0..c81421a758 100644 --- a/.upstream-tests/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp +++ b/.upstream-tests/test/std/atomics/atomics.types.operations/atomics.types.operations.req/atomic_is_lock_free.pass.cpp @@ -31,9 +31,9 @@ struct TestFn { __host__ __device__ void operator()() const { typedef cuda::std::atomic A; - A t; + A t{}; bool b1 = cuda::std::atomic_is_lock_free(static_cast(&t)); - volatile A vt; + volatile A vt{}; bool b2 = cuda::std::atomic_is_lock_free(static_cast(&vt)); assert(b1 == b2); } diff --git a/.upstream-tests/test/std/utilities/time/time.cal/euclidian.h b/.upstream-tests/test/std/utilities/time/time.cal/euclidian.h index eb8019fdf6..9233a04892 100644 --- a/.upstream-tests/test/std/utilities/time/time.cal/euclidian.h +++ b/.upstream-tests/test/std/utilities/time/time.cal/euclidian.h @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#pragma nv_diag_suppress 186 + #include diff --git a/codegen/codegen.cpp b/codegen/codegen.cpp index d724a2dd12..bcddbb334b 100644 --- a/codegen/codegen.cpp +++ b/codegen/codegen.cpp @@ -88,32 +88,35 @@ int main() { return "__cuda_fence_" + sem + "_" + scope; }; - out << "_LIBCUDACXX_BEGIN_NAMESPACE_CUDA\n"; - out << "namespace detail {\n"; - out << "\n"; - for(auto& s : scopes) { out << "static inline __device__ void __cuda_membar_" << s.first << "() { asm volatile(\"membar" << membar_scopes[s.first] << ";\":::\"memory\"); }\n"; for(auto& sem : fence_semantics) out << "static inline __device__ void " << fencename(sem.first, s.first) << "() { asm volatile(\"fence" << sem.second << s.second << ";\":::\"memory\"); }\n"; out << "static inline __device__ void __atomic_thread_fence_cuda(int __memorder, " << scopenametag(s.first) << ") {\n"; - out << " switch (__memorder) {\n"; - out << "#if __CUDA_ARCH__ >= 700\n"; - out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "(); break;\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE:\n"; - out << " case __ATOMIC_ACQ_REL:\n"; - out << " case __ATOMIC_RELEASE: " << fencename("acq_rel"s, s.first) << "(); break;\n"; - out << "#else\n"; - out << " case __ATOMIC_SEQ_CST:\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE:\n"; - out << " case __ATOMIC_ACQ_REL:\n"; - out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); break;\n"; - out << "#endif // __CUDA_ARCH__ >= 700\n"; - out << " case __ATOMIC_RELAXED: break;\n"; - out << " default: assert(0);\n"; - out << " }\n"; + out << " NV_DISPATCH_TARGET(\n"; + out << " NV_PROVIDES_SM_70, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "(); break;\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE:\n"; + out << " case __ATOMIC_ACQ_REL:\n"; + out << " case __ATOMIC_RELEASE: " << fencename("acq_rel"s, s.first) << "(); break;\n"; + out << " case __ATOMIC_RELAXED: break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " ),\n"; + out << " NV_IS_DEVICE, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST:\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE:\n"; + out << " case __ATOMIC_ACQ_REL:\n"; + out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); break;\n"; + out << " case __ATOMIC_RELAXED: break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " )\n"; + out << " )\n"; out << "}\n"; for(auto& sz : ld_sizes) { for(auto& sem : ld_semantics) { @@ -130,20 +133,26 @@ int main() { out << "template::type = 0>\n"; out << "__device__ void __atomic_load_cuda(const " << cv << "_Type *__ptr, _Type *__ret, int __memorder, " << scopenametag(s.first) << ") {\n"; out << " uint" << (registers[sz] == "r" ? 32 : sz) << "_t __tmp = 0;\n"; - out << " switch (__memorder) {\n"; - out << "#if __CUDA_ARCH__ >= 700\n"; - out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_load_acquire_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_load_relaxed_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; - out << "#else\n"; - out << " case __ATOMIC_SEQ_CST: __cuda_membar_" << s.first << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_load_volatile_" << sz << "_" << s.first << "(__ptr, __tmp); __cuda_membar_" << s.first << "(); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_load_volatile_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; - out << "#endif // __CUDA_ARCH__ >= 700\n"; - out << " default: assert(0);\n"; - out << " }\n"; + out << " NV_DISPATCH_TARGET(\n"; + out << " NV_PROVIDES_SM_70, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_load_acquire_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_load_relaxed_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " ),\n"; + out << " NV_IS_DEVICE, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST: __cuda_membar_" << s.first << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_load_volatile_" << sz << "_" << s.first << "(__ptr, __tmp); __cuda_membar_" << s.first << "(); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_load_volatile_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " )\n"; + out << " )\n"; out << " memcpy(__ret, &__tmp, " << sz/8 << ");\n"; out << "}\n"; } @@ -161,18 +170,24 @@ int main() { out << "__device__ void __atomic_store_cuda(" << cv << "_Type *__ptr, _Type *__val, int __memorder, " << scopenametag(s.first) << ") {\n"; out << " uint" << (registers[sz] == "r" ? 32 : sz) << "_t __tmp = 0;\n"; out << " memcpy(&__tmp, __val, " << sz/8 << ");\n"; - out << " switch (__memorder) {\n"; - out << "#if __CUDA_ARCH__ >= 700\n"; - out << " case __ATOMIC_RELEASE: __cuda_store_release_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; - out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; - out << " case __ATOMIC_RELAXED: __cuda_store_relaxed_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; - out << "#else\n"; - out << " case __ATOMIC_RELEASE:\n"; - out << " case __ATOMIC_SEQ_CST: __cuda_membar_" << s.first << "();\n"; - out << " case __ATOMIC_RELAXED: __cuda_store_volatile_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; - out << "#endif // __CUDA_ARCH__ >= 700\n"; - out << " default: assert(0);\n"; - out << " }\n"; + out << " NV_DISPATCH_TARGET(\n"; + out << " NV_PROVIDES_SM_70, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_RELEASE: __cuda_store_release_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; + out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; + out << " case __ATOMIC_RELAXED: __cuda_store_relaxed_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " ),\n"; + out << " NV_IS_DEVICE, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_RELEASE:\n"; + out << " case __ATOMIC_SEQ_CST: __cuda_membar_" << s.first << "();\n"; + out << " case __ATOMIC_RELAXED: __cuda_store_volatile_" << sz << "_" << s.first << "(__ptr, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " )\n"; + out << " )\n"; out << "}\n"; } } @@ -214,24 +229,30 @@ int main() { out << " memcpy(&__tmp, __desired, " << sz/8 << ");\n"; out << " memcpy(&__old, __expected, " << sz/8 << ");\n"; out << " __old_tmp = __old;\n"; - out << " switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) {\n"; - out << "#if __CUDA_ARCH__ >= 700\n"; - out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; - out << " case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELEASE: __cuda_compare_exchange_release_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; - out << "#else\n"; - out << " case __ATOMIC_SEQ_CST:\n"; - out << " case __ATOMIC_ACQ_REL: __cuda_membar_" << s.first << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); __cuda_membar_" << s.first << "(); break;\n"; - out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); __cuda_compare_exchange_volatile_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; - out << "#endif // __CUDA_ARCH__ >= 700\n"; - out << " default: assert(0);\n"; - out << " }\n"; + out << " NV_DISPATCH_TARGET(\n"; + out << " NV_PROVIDES_SM_70, (\n"; + out << " switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) {\n"; + out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; + out << " case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELEASE: __cuda_compare_exchange_release_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " ),\n"; + out << " NV_IS_DEVICE, (\n"; + out << " switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) {\n"; + out << " case __ATOMIC_SEQ_CST:\n"; + out << " case __ATOMIC_ACQ_REL: __cuda_membar_" << s.first << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); __cuda_membar_" << s.first << "(); break;\n"; + out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); __cuda_compare_exchange_volatile_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_" << sz << "_" << s.first << "(__ptr, __old, __old_tmp, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " )\n"; + out << " )\n"; out << " bool const __ret = __old == __old_tmp;\n"; out << " memcpy(__expected, &__old, " << sz/8 << ");\n"; out << " return __ret;\n"; @@ -250,24 +271,30 @@ int main() { out << " uint" << sz << "_t __tmp = 0;\n"; out << " memcpy(&__tmp, &__val, " << sz/8 << ");\n"; } - out << " switch (__memorder) {\n"; - out << "#if __CUDA_ARCH__ >= 700\n"; - out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_" << rmw.first << "_acquire_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_ACQ_REL: __cuda_" << rmw.first << "_acq_rel_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELEASE: __cuda_" << rmw.first << "_release_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_" << rmw.first << "_relaxed_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << "#else\n"; - out << " case __ATOMIC_SEQ_CST:\n"; - out << " case __ATOMIC_ACQ_REL: __cuda_membar_" << s.first << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_" << rmw.first << "_volatile_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); __cuda_membar_" << s.first << "(); break;\n"; - out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); __cuda_" << rmw.first << "_volatile_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_" << rmw.first << "_volatile_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << "#endif // __CUDA_ARCH__ >= 700\n"; - out << " default: assert(0);\n"; - out << " }\n"; + out << " NV_DISPATCH_TARGET(\n"; + out << " NV_PROVIDES_SM_70, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_" << rmw.first << "_acquire_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_ACQ_REL: __cuda_" << rmw.first << "_acq_rel_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELEASE: __cuda_" << rmw.first << "_release_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_" << rmw.first << "_relaxed_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " ),\n"; + out << " NV_IS_DEVICE, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST:\n"; + out << " case __ATOMIC_ACQ_REL: __cuda_membar_" << s.first << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_" << rmw.first << "_volatile_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); __cuda_membar_" << s.first << "(); break;\n"; + out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); __cuda_" << rmw.first << "_volatile_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_" << rmw.first << "_volatile_" << sz << "_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " )\n"; + out << " )\n"; if(rmw.first == "exchange") out << " memcpy(__ret, &__tmp, " << sz/8 << ");\n"; else { @@ -290,24 +317,29 @@ int main() { if(op == "sub") out << " __tmp = -__tmp;\n"; out << " __tmp *= sizeof(_Type);\n"; - out << " switch (__memorder) {\n"; - out << "#if __CUDA_ARCH__ >= 700\n"; - out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << "#else\n"; - out << " case __ATOMIC_SEQ_CST:\n"; - out << " case __ATOMIC_ACQ_REL: __cuda_membar_" << s.first << "();\n"; - out << " case __ATOMIC_CONSUME:\n"; - out << " case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_" << s.first << "(__ptr, __tmp, __tmp); __cuda_membar_" << s.first << "(); break;\n"; - out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); __cuda_fetch_add_volatile_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << " case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; - out << "#endif // __CUDA_ARCH__ >= 700\n"; - out << " default: assert(0);\n"; - out << " }\n"; + out << " NV_DISPATCH_TARGET(\n"; + out << " NV_PROVIDES_SM_70, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST: " << fencename("sc"s, s.first) << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " }\n"; + out << " ),\n"; + out << " NV_IS_DEVICE, (\n"; + out << " switch (__memorder) {\n"; + out << " case __ATOMIC_SEQ_CST:\n"; + out << " case __ATOMIC_ACQ_REL: __cuda_membar_" << s.first << "();\n"; + out << " case __ATOMIC_CONSUME:\n"; + out << " case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_" << s.first << "(__ptr, __tmp, __tmp); __cuda_membar_" << s.first << "(); break;\n"; + out << " case __ATOMIC_RELEASE: __cuda_membar_" << s.first << "(); __cuda_fetch_add_volatile_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_" << s.first << "(__ptr, __tmp, __tmp); break;\n"; + out << " default: assert(0);\n"; + out << " }\n"; + out << " )\n"; + out << " )\n"; out << " memcpy(&__ret, &__tmp, 8);\n"; out << " return __ret;\n"; out << "}\n"; @@ -315,9 +347,5 @@ int main() { } } - out << "\n"; - out << "}\n"; - out << "_LIBCUDACXX_END_NAMESPACE_CUDA\n"; - return 0; } diff --git a/docs/releases/changelog.md b/docs/releases/changelog.md index db5a93de28..77ff291c15 100644 --- a/docs/releases/changelog.md +++ b/docs/releases/changelog.md @@ -11,9 +11,34 @@ It pulls in the latest version of upstream libc++ and marks the beginning of !--> +## libcu++ 1.6.0 (CUDA Toolkit 11.5) + +libcu++ 1.6.0 is a major release. It adds two new functions to the `cuda::std::barrier` API and +uses `` as the primary dispatch mechanism for `cuda::std::atomic`. + +This release introduces ABI version 4, which is now the default. + +Supported ABI Versions: 4 (default), 3, and 2. + +Included in: CUDA Toolkit 11.5. + +### Issues Fixed + +- #179: Refactors the atomic layer to allow for layering the host device/host abstractions. +- #189: Changed pragmas for silencing chrono long double warnings. +- #186: Allows `` to be used under NVRTC. +- #177: Allows `` to build when compiled under C and C++98. + - Thanks to David Olsen for this contribution. +- #172: Introduces ABI version 4. + - Forces `cuda::std::complex` alignment for enhanced performance. + - Sets the internal representation of `cuda::std::chrono` literals to `double`. +- #165: For tests on some older distributions keep using Python 3, but downgrade lit. +- #164: Fixes testing issues related to Python 2/3 switch for lit. + - Thanks to Royil Damer for this contribution. + ## libcu++ 1.5.0 (CUDA Toolkit 11.4) -libcu++ 1.5.0 is a major release. It adds ``, +libcu++ 1.5.0 is a major release. It adds ``, the library support header for the new `if target` target specialization mechanism. diff --git a/include/cuda/std/atomic b/include/cuda/std/atomic index 3b07b21abd..c738d48fef 100644 --- a/include/cuda/std/atomic +++ b/include/cuda/std/atomic @@ -37,6 +37,7 @@ #undef ATOMIC_VAR_INIT #endif //__CUDACC_RTC__ +#include "cassert" #include "cstddef" #include "cstdint" #include "type_traits" @@ -46,16 +47,26 @@ #include "detail/__pragma_push" -#include "detail/__atomic" #include "detail/__threading_support" -#undef _LIBCUDACXX_HAS_GCC_ATOMIC_IMP -#undef _LIBCUDACXX_HAS_C_ATOMIC_IMP - #include "detail/libcxx/include/atomic" _LIBCUDACXX_BEGIN_NAMESPACE_CUDA +using std::__detail::thread_scope; +using std::__detail::thread_scope_system; +using std::__detail::thread_scope_device; +using std::__detail::thread_scope_block; +using std::__detail::thread_scope_thread; + +namespace __detail { +using std::__detail::__thread_scope_block_tag; +using std::__detail::__thread_scope_device_tag; +using std::__detail::__thread_scope_system_tag; +using std::__detail::__atomic_signal_fence_cuda; +using std::__detail::__atomic_thread_fence_cuda; +} + using memory_order = std::memory_order; constexpr memory_order memory_order_relaxed = std::memory_order_relaxed; @@ -67,7 +78,7 @@ constexpr memory_order memory_order_seq_cst = std::memory_order_seq_cst; // atomic -template +template struct atomic : public std::__atomic_base<_Tp, _Sco> { @@ -87,15 +98,15 @@ struct atomic __host__ __device__ _Tp fetch_max(const _Tp & __op, memory_order __m = memory_order_seq_cst) volatile noexcept { - return detail::__atomic_fetch_max_cuda(&this->__a_.__a_value, __op, - __m, detail::__scope_tag<_Sco>()); + return std::__detail::__atomic_fetch_max_cuda(&this->__a_.__a_value, __op, + __m, std::__detail::__scope_tag<_Sco>()); } __host__ __device__ _Tp fetch_min(const _Tp & __op, memory_order __m = memory_order_seq_cst) volatile noexcept { - return detail::__atomic_fetch_min_cuda(&this->__a_.__a_value, __op, - __m, detail::__scope_tag<_Sco>()); + return std::__detail::__atomic_fetch_min_cuda(&this->__a_.__a_value, __op, + __m, std::__detail::__scope_tag<_Sco>()); } }; @@ -159,31 +170,37 @@ struct atomic<_Tp*, _Sco> _Tp* operator-=(ptrdiff_t __op) noexcept {return fetch_sub(__op) - __op;} }; -inline __host__ __device__ void atomic_thread_fence(memory_order __m, thread_scope _Scope = thread_scope_system) { -#ifdef __CUDA_ARCH__ - switch(_Scope) { - case thread_scope_system: - detail::__atomic_thread_fence_cuda((int)__m, detail::__thread_scope_system_tag()); - break; - case thread_scope_device: - detail::__atomic_thread_fence_cuda((int)__m, detail::__thread_scope_device_tag()); - break; - case thread_scope_block: - detail::__atomic_thread_fence_cuda((int)__m, detail::__thread_scope_block_tag()); - break; - } -#else - (void) _Scope; - ::std::atomic_thread_fence((::std::memory_order)__m); -#endif +inline __host__ __device__ void atomic_thread_fence(memory_order __m, thread_scope _Scope = thread_scope::thread_scope_system) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + switch(_Scope) { + case thread_scope::thread_scope_system: + __detail::__atomic_thread_fence_cuda((int)__m, __detail::__thread_scope_system_tag()); + break; + case thread_scope::thread_scope_device: + __detail::__atomic_thread_fence_cuda((int)__m, __detail::__thread_scope_device_tag()); + break; + case thread_scope::thread_scope_block: + __detail::__atomic_thread_fence_cuda((int)__m, __detail::__thread_scope_block_tag()); + break; + } + ), + NV_IS_HOST, ( + (void) _Scope; + ::std::atomic_thread_fence((::std::memory_order)__m); + ) + ) } inline __host__ __device__ void atomic_signal_fence(memory_order __m) { -#ifdef __CUDA_ARCH__ - detail::__atomic_signal_fence_cuda((int)__m); -#else - ::std::atomic_signal_fence((::std::memory_order)__m); -#endif + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + __detail::__atomic_signal_fence_cuda((int)__m); + ), + NV_IS_HOST, ( + ::std::atomic_signal_fence((::std::memory_order)__m); + ) + ) } _LIBCUDACXX_END_NAMESPACE_CUDA diff --git a/include/cuda/std/detail/__atomic b/include/cuda/std/detail/__atomic deleted file mode 100644 index cdae5b1e50..0000000000 --- a/include/cuda/std/detail/__atomic +++ /dev/null @@ -1,436 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of libcu++, the C++ Standard Library for your entire system, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#if defined(__CUDA_ARCH__) && ((!defined(_MSC_VER) && __CUDA_ARCH__ < 600) || (defined(_MSC_VER) && __CUDA_ARCH__ < 700)) -# error "CUDA atomics are only supported for sm_60 and up on *nix and sm_70 and up on Windows." -#endif - -#ifndef __CUDACC_RTC__ -#include -#include -#endif // __CUDACC_RTC__ - -#if !defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) && !defined(__GCC_ATOMIC_BOOL_LOCK_FREE) -#define ATOMIC_BOOL_LOCK_FREE 2 -#define ATOMIC_CHAR_LOCK_FREE 2 -#define ATOMIC_CHAR16_T_LOCK_FREE 2 -#define ATOMIC_CHAR32_T_LOCK_FREE 2 -#define ATOMIC_WCHAR_T_LOCK_FREE 2 -#define ATOMIC_SHORT_LOCK_FREE 2 -#define ATOMIC_INT_LOCK_FREE 2 -#define ATOMIC_LONG_LOCK_FREE 2 -#define ATOMIC_LLONG_LOCK_FREE 2 -#define ATOMIC_POINTER_LOCK_FREE 2 -#endif //!defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) && !defined(__GCC_ATOMIC_BOOL_LOCK_FREE) - -#ifndef __ATOMIC_RELAXED -#define __ATOMIC_RELAXED 0 -#define __ATOMIC_CONSUME 1 -#define __ATOMIC_ACQUIRE 2 -#define __ATOMIC_RELEASE 3 -#define __ATOMIC_ACQ_REL 4 -#define __ATOMIC_SEQ_CST 5 -#endif //__ATOMIC_RELAXED - -#ifndef __ATOMIC_BLOCK -#define __ATOMIC_SYSTEM 0 // 0 indicates default -#define __ATOMIC_DEVICE 1 -#define __ATOMIC_BLOCK 2 -#define __ATOMIC_THREAD 10 -#endif //__ATOMIC_BLOCK - -_LIBCUDACXX_BEGIN_NAMESPACE_CUDA - -namespace detail { - - inline __host__ __device__ int __stronger_order_cuda(int __a, int __b) { - int const __max = __a > __b ? __a : __b; - if(__max != __ATOMIC_RELEASE) - return __max; - static int const __xform[] = { - __ATOMIC_RELEASE, - __ATOMIC_ACQ_REL, - __ATOMIC_ACQ_REL, - __ATOMIC_RELEASE }; - return __xform[__a < __b ? __a : __b]; - } -} - -enum thread_scope { - thread_scope_system = __ATOMIC_SYSTEM, - thread_scope_device = __ATOMIC_DEVICE, - thread_scope_block = __ATOMIC_BLOCK, - thread_scope_thread = __ATOMIC_THREAD -}; - -#define _LIBCUDACXX_ATOMIC_SCOPE_TYPE ::cuda::thread_scope -#define _LIBCUDACXX_ATOMIC_SCOPE_DEFAULT ::cuda::thread_scope::system - -namespace detail { - - struct __thread_scope_thread_tag { }; - struct __thread_scope_block_tag { }; - struct __thread_scope_device_tag { }; - struct __thread_scope_system_tag { }; - - template struct __scope_enum_to_tag { }; - /* This would be the implementation once an actual thread-scope backend exists. - template<> struct __scope_enum_to_tag<(int)thread_scope_thread> { - using type = __thread_scope_thread_tag; }; - Until then: */ - template<> struct __scope_enum_to_tag<(int)thread_scope_thread> { - using type = __thread_scope_block_tag; }; - template<> struct __scope_enum_to_tag<(int)thread_scope_block> { - using type = __thread_scope_block_tag; }; - template<> struct __scope_enum_to_tag<(int)thread_scope_device> { - using type = __thread_scope_device_tag; }; - template<> struct __scope_enum_to_tag<(int)thread_scope_system> { - using type = __thread_scope_system_tag; }; - - template - __host__ __device__ auto constexpr __scope_tag() -> - typename __scope_enum_to_tag<_Scope>::type { - return typename __scope_enum_to_tag<_Scope>::type(); - } -} - -_LIBCUDACXX_END_NAMESPACE_CUDA - -#if defined(_LIBCUDACXX_COMPILER_MSVC) - // Inject atomic intrinsics built from MSVC compiler intrinsics - #include "libcxx/include/support/win32/atomic_msvc.h" -#endif - -#include "__atomic_generated" -#include "__atomic_derived" - -_LIBCUDACXX_BEGIN_NAMESPACE_STD - -template -struct __skip_amt { enum {value = 1}; }; - -template -struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; - -// Forward-declare the function templates that are defined libcxx later. -template _LIBCUDACXX_INLINE_VISIBILITY -typename enable_if::value>::type -__cxx_atomic_assign_volatile(_Tp& __a_value, _Tv const& __val); - -template _LIBCUDACXX_INLINE_VISIBILITY -typename enable_if::value>::type -__cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val); - -__host__ __device__ inline bool __cxx_atomic_is_lock_free(size_t __x) { - return __x <= 8; -} -__host__ __device__ inline void __cxx_atomic_thread_fence(int __order) { -#ifdef __CUDA_ARCH__ - detail::__atomic_thread_fence_cuda(__order, detail::__thread_scope_system_tag()); -#else - __atomic_thread_fence(__order); -#endif -} -__host__ __device__ inline void __cxx_atomic_signal_fence(int __order) { -#ifdef __CUDA_ARCH__ - detail::__atomic_signal_fence_cuda(__order); -#else - __atomic_signal_fence(__order); -#endif -} - -template -struct __cxx_atomic_alignment_wrapper_impl; - -template -struct __cxx_atomic_alignment_wrapper_impl { - struct type { - using __wrapped_type = _Tp; - __host__ __device__ constexpr type() noexcept : __a_held() { - } - __host__ __device__ constexpr type(_Tp __held) noexcept : __a_held(__held) { - } - _ALIGNAS(sizeof(_Tp)) _Tp __a_held; - }; -}; - -template -struct __cxx_atomic_alignment_wrapper_impl<_Tp, typename enable_if<_LIBCUDACXX_ALIGNOF(_Tp) == sizeof(_Tp)>::type> { - using type = _Tp; -}; - -template -using __cxx_atomic_alignment_wrapper_t = typename __cxx_atomic_alignment_wrapper_impl<_Tp>::type; - -template -__host__ __device__ __cxx_atomic_alignment_wrapper_t<_Tp> __cxx_atomic_alignment_wrap(_Tp __value, true_type) { - return __value; -} -template -__host__ __device__ __cxx_atomic_alignment_wrapper_t<_Tp> __cxx_atomic_alignment_wrap(_Tp __value, false_type) { - return __cxx_atomic_alignment_wrapper_t<_Tp>(__value); -} -template -__host__ __device__ __cxx_atomic_alignment_wrapper_t<_Tp> __cxx_atomic_alignment_wrap(_Tp __value) { - return __cxx_atomic_alignment_wrap(__value, integral_constant{}); -} - -template -__host__ __device__ _Tp __cxx_atomic_alignment_unwrap(_Tp __value, true_type) { - return __value; -} -template -__host__ __device__ typename _Tp::__wrapped_type __cxx_atomic_alignment_unwrap(_Tp __value, false_type) { - return __value.__a_held; -} -template -__host__ __device__ auto __cxx_atomic_alignment_unwrap(_Tp __value) - -> decltype(__cxx_atomic_alignment_unwrap(__value, integral_constant{})) -{ - return __cxx_atomic_alignment_unwrap(__value, integral_constant{}); -} - -template -struct __cxx_atomic_base_impl_default { - constexpr __cxx_atomic_base_impl_default() noexcept = default; - __host__ __device__ constexpr explicit __cxx_atomic_base_impl_default(_Tp __value) noexcept : __a_value(__value) { - } - __cxx_atomic_alignment_wrapper_t<_Tp> __a_value; -}; - -template -__host__ __device__ inline void __cxx_atomic_init(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __val) { - auto __tmp = __cxx_atomic_alignment_wrap(__val); - __cxx_atomic_assign_volatile(__a->__a_value, __tmp); -} -template -__host__ __device__ inline void __cxx_atomic_store(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __val, int __order) { -#ifdef __CUDA_ARCH__ - detail::__atomic_store_n_cuda(&__a->__a_value, __cxx_atomic_alignment_wrap(__val), __order, detail::__scope_tag<_Sco>()); -#else - auto __t = __cxx_atomic_alignment_wrap(__val); - __atomic_store(&__a->__a_value, &__t, __order); -#endif -} -template -__host__ __device__ inline _Tp __cxx_atomic_load(__cxx_atomic_base_impl_default<_Tp, _Sco> const volatile* __a, int __order) { -#ifdef __CUDA_ARCH__ - return __cxx_atomic_alignment_unwrap(detail::__atomic_load_n_cuda(&__a->__a_value, __order, detail::__scope_tag<_Sco>())); -#else - alignas(_Tp) unsigned char __buf[sizeof(_Tp)]; - auto* __dest = reinterpret_cast<_Tp*>(__buf); - __atomic_load(&__a->__a_value, __dest, __order); - return __cxx_atomic_alignment_unwrap(*__dest); -#endif -} -template -__host__ __device__ inline _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __val, int __order) { -#ifdef __CUDA_ARCH__ - return __cxx_atomic_alignment_unwrap(detail::__atomic_exchange_n_cuda(&__a->__a_value, __cxx_atomic_alignment_wrap(__val), __order, detail::__scope_tag<_Sco>())); -#else - alignas(_Tp) unsigned char __buf[sizeof(_Tp)]; - auto* __dest = reinterpret_cast<_Tp*>(__buf); - auto __t = __cxx_atomic_alignment_wrap(__val); - __atomic_exchange(&__a->__a_value, &__t, __dest, __order); - return __cxx_atomic_alignment_unwrap(*__dest); -#endif -} -template -__host__ __device__ inline bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp* __expected, _Tp __val, int __success, int __failure) { - auto __tmp = __cxx_atomic_alignment_wrap(*__expected); -#ifdef __CUDA_ARCH__ - bool __result = detail::__atomic_compare_exchange_n_cuda(&__a->__a_value, &__tmp, __cxx_atomic_alignment_wrap(__val), false, __success, __failure, detail::__scope_tag<_Sco>()); -#else - bool __result = __atomic_compare_exchange(&__a->__a_value, &__tmp, &__val, false, __success, __failure); -#endif - *__expected = __cxx_atomic_alignment_unwrap(__tmp); - return __result; -} -template -__host__ __device__ inline bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp* __expected, _Tp __val, int __success, int __failure) { - auto __tmp = __cxx_atomic_alignment_wrap(*__expected); -#ifdef __CUDA_ARCH__ - bool __result = detail::__atomic_compare_exchange_n_cuda(&__a->__a_value, &__tmp, __cxx_atomic_alignment_wrap(__val), true, __success, __failure, detail::__scope_tag<_Sco>()); -#else - bool __result = __atomic_compare_exchange(&__a->__a_value, &__tmp, &__val, true, __success, __failure); -#endif - *__expected = __cxx_atomic_alignment_unwrap(__tmp); - return __result; -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __delta, int __order) { -#ifdef __CUDA_ARCH__ - return detail::__atomic_fetch_add_cuda(&__a->__a_value, __delta, __order, detail::__scope_tag<_Sco>()); -#else - return __atomic_fetch_add(&__a->__a_value, __delta, __order); -#endif -} -template -__host__ __device__ inline _Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl_default<_Tp*, _Sco> volatile* __a, ptrdiff_t __delta, int __order) { -#ifdef __CUDA_ARCH__ - return detail::__atomic_fetch_add_cuda(&__a->__a_value, __delta, __order, detail::__scope_tag<_Sco>()); -#else - return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp*>::value, __order); -#endif -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __delta, int __order) { -#ifdef __CUDA_ARCH__ - return detail::__atomic_fetch_sub_cuda(&__a->__a_value, __delta, __order, detail::__scope_tag<_Sco>()); -#else - return __atomic_fetch_sub(&__a->__a_value, __delta, __order); -#endif -} -template -__host__ __device__ inline _Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl_default<_Tp*, _Sco> volatile* __a, ptrdiff_t __delta, int __order) { -#ifdef __CUDA_ARCH__ - return detail::__atomic_fetch_sub_cuda(&__a->__a_value, __delta, __order, detail::__scope_tag<_Sco>()); -#else - return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp*>::value, __order); -#endif -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __pattern, int __order) { -#ifdef __CUDA_ARCH__ - return detail::__atomic_fetch_and_cuda(&__a->__a_value, __pattern, __order, detail::__scope_tag<_Sco>()); -#else - return __atomic_fetch_and(&__a->__a_value, __pattern, __order); -#endif -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __pattern, int __order) { -#ifdef __CUDA_ARCH__ - return detail::__atomic_fetch_or_cuda(&__a->__a_value, __pattern, __order, detail::__scope_tag<_Sco>()); -#else - return __atomic_fetch_or(&__a->__a_value, __pattern, __order); -#endif -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl_default<_Tp, _Sco> volatile* __a, _Tp __pattern, int __order) { -#ifdef __CUDA_ARCH__ - return detail::__atomic_fetch_xor_cuda(&__a->__a_value, __pattern, __order, detail::__scope_tag<_Sco>()); -#else - return __atomic_fetch_xor(&__a->__a_value, __pattern, __order); -#endif -} - -template -struct __cxx_atomic_base_impl_small { - - __cxx_atomic_base_impl_small() noexcept = default; - __host__ __device__ constexpr explicit __cxx_atomic_base_impl_small(_Tp __value) : __a_value(__value) { - } - - __cxx_atomic_base_impl_default __a_value; -}; - -template -using __cxx_small_proxy = typename conditional::type >::type; - -template -__host__ __device__ inline uint32_t __cxx_small_to_32(_Tp __val) { - __cxx_small_proxy<_Tp> __temp; - memcpy(&__temp, &__val, sizeof(_Tp)); - return __temp; -} - -template -__host__ __device__ inline _Tp __cxx_small_from_32(uint32_t __val) { - __cxx_small_proxy<_Tp> __temp = static_cast<__cxx_small_proxy<_Tp>>(__val); - _Tp __result; - memcpy(&__result, &__temp, sizeof(_Tp)); - return __result; -} - -template -__host__ __device__ inline void __cxx_atomic_init(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __val) { - __cxx_atomic_init(&__a->__a_value, __cxx_small_to_32(__val)); -} -template -__host__ __device__ inline void __cxx_atomic_store(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __val, int __order) { - __cxx_atomic_store(&__a->__a_value, __cxx_small_to_32(__val), __order); -} -template -__host__ __device__ inline _Tp __cxx_atomic_load(__cxx_atomic_base_impl_small<_Tp, _Sco> const volatile* __a, int __order) { - return __cxx_small_from_32<_Tp>(__cxx_atomic_load(&__a->__a_value, __order)); -} -template -__host__ __device__ inline _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __value, int __order) { - return __cxx_small_from_32<_Tp>(__cxx_atomic_exchange(&__a->__a_value, __cxx_small_to_32(__value), __order)); -} -__host__ __device__ -inline int __cuda_memcmp(void const * __lhs, void const * __rhs, size_t __count) { -#ifdef __CUDA_ARCH__ - auto __lhs_c = reinterpret_cast(__lhs); - auto __rhs_c = reinterpret_cast(__rhs); - while (__count--) { - auto const __lhs_v = *__lhs_c++; - auto const __rhs_v = *__rhs_c++; - if (__lhs_v < __rhs_v) { return -1; } - if (__lhs_v > __rhs_v) { return 1; } - } - return 0; -#else - return memcmp(__lhs, __rhs, __count); -#endif -} -template -__host__ __device__ inline bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp* __expected, _Tp __value, int __success, int __failure) { - auto __temp = __cxx_small_to_32(*__expected); - auto const __ret = __cxx_atomic_compare_exchange_weak(&__a->__a_value, &__temp, __cxx_small_to_32(__value), __success, __failure); - auto const __actual = __cxx_small_from_32<_Tp>(__temp); - if(!__ret) { - if(0 == __cuda_memcmp(&__actual, __expected, sizeof(_Tp))) - __cxx_atomic_fetch_and(&__a->__a_value, (1u << (8*sizeof(_Tp))) - 1, __ATOMIC_RELAXED); - else - *__expected = __actual; - } - return __ret; -} -template -__host__ __device__ inline bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp* __expected, _Tp __value, int __success, int __failure) { - auto const __old = *__expected; - while(1) { - if(__cxx_atomic_compare_exchange_weak(__a, __expected, __value, __success, __failure)) - return true; - if(0 != __cuda_memcmp(&__old, __expected, sizeof(_Tp))) - return false; - } -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __delta, int __order) { - return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_add(&__a->__a_value, __cxx_small_to_32(__delta), __order)); -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __delta, int __order) { - return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_sub(&__a->__a_value, __cxx_small_to_32(__delta), __order)); -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __pattern, int __order) { - return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_and(&__a->__a_value, __cxx_small_to_32(__pattern), __order)); -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __pattern, int __order) { - return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_or(&__a->__a_value, __cxx_small_to_32(__pattern), __order)); -} -template -__host__ __device__ inline _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl_small<_Tp, _Sco> volatile* __a, _Tp __pattern, int __order) { - return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_xor(&__a->__a_value, __cxx_small_to_32(__pattern), __order)); -} - -template -using __cxx_atomic_base_impl = typename conditional, - __cxx_atomic_base_impl_default<_Tp, _Sco> >::type; - -_LIBCUDACXX_END_NAMESPACE_STD diff --git a/include/cuda/std/detail/__config b/include/cuda/std/detail/__config index 4a2b5a1d1d..e735f7a053 100644 --- a/include/cuda/std/detail/__config +++ b/include/cuda/std/detail/__config @@ -97,7 +97,7 @@ # define _LIBCUDACXX_ABI_UNSTABLE #endif -#define _LIBCUDACXX_CUDA_API_VERSION 001005000 +#define _LIBCUDACXX_CUDA_API_VERSION 001006000 #define _LIBCUDACXX_CUDA_API_VERSION_MAJOR \ (_LIBCUDACXX_CUDA_API_VERSION / 1000000) diff --git a/libcxx/include/__config b/libcxx/include/__config index 2965dea640..70e1774c1a 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -28,6 +28,11 @@ #ifdef __cplusplus +// __config may be included in `extern "C"` contexts, switch back to include +extern "C++" { +#include +} + #ifdef __GNUC__ # define _GNUC_VER (__GNUC__ * 100 + __GNUC_MINOR__) // The _GNUC_VER_NEW macro better represents the new GCC versioning scheme @@ -1596,10 +1601,22 @@ _LIBCUDACXX_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container( #define _LIBCUDACXX_HAS_NO_THREAD_UNSAFE_C_FUNCTIONS #endif -#if __has_feature(cxx_atomic) || __has_extension(c_atomic) || __has_keyword(_Atomic) -# define _LIBCUDACXX_HAS_C_ATOMIC_IMP +// TODO: Support C11 Atomics? +// #if __has_feature(cxx_atomic) || __has_extension(c_atomic) || __has_keyword(_Atomic) +// # define _LIBCUDACXX_HAS_C_ATOMIC_IMP +#if defined(_LIBCUDACXX_COMPILER_CLANG) +# define _LIBCUDACXX_HAS_GCC_ATOMIC_IMP #elif defined(_LIBCUDACXX_COMPILER_GCC) # define _LIBCUDACXX_HAS_GCC_ATOMIC_IMP +#elif defined(_LIBCUDACXX_COMPILER_MSVC) +# define _LIBCUDACXX_HAS_MSVC_ATOMIC_IMPL +#endif + +#define _LIBCUDACXX_NO_RUNTIME_LOCK_FREE + +// CUDA Atomics supersede host atomics in order to insert the host/device dispatch layer +#if defined(_LIBCUDACXX_COMPILER_NVCC) || defined(_LIBCUDACXX_COMPILER_PGI) +# define _LIBCUDACXX_HAS_CUDA_ATOMIC_IMPL #endif #if (!defined(_LIBCUDACXX_HAS_C_ATOMIC_IMP) && \ diff --git a/libcxx/include/__threading_support b/libcxx/include/__threading_support index eb26e2c15b..9489f6a6bd 100644 --- a/libcxx/include/__threading_support +++ b/libcxx/include/__threading_support @@ -75,17 +75,19 @@ _LIBCUDACXX_BEGIN_NAMESPACE_STD _LIBCUDACXX_INLINE_VISIBILITY inline void __libcpp_thread_yield_processor() { -#if defined(__CUDA_ARCH__) - ; -#elif defined(__aarch64__) - asm volatile ("yield" :::); +#if defined(__aarch64__) +# define __LIBCUDACXX_ASM_THREAD_YIELD (asm volatile ("yield" :::);) #elif defined(__x86_64__) - asm volatile ("pause" :::); +# define __LIBCUDACXX_ASM_THREAD_YIELD (asm volatile ("pause" :::);) #elif defined (__powerpc__) - asm volatile ("or 27,27,27":::); +# define __LIBCUDACXX_ASM_THREAD_YIELD (asm volatile ("or 27,27,27":::);) #else - ; +# define __LIBCUDACXX_ASM_THREAD_YIELD (;) #endif + NV_IF_TARGET( + NV_IS_HOST, + __LIBCUDACXX_ASM_THREAD_YIELD + ) } _LIBCUDACXX_THREAD_ABI_VISIBILITY @@ -293,13 +295,13 @@ void __libcpp_thread_yield() {} _LIBCUDACXX_THREAD_ABI_VISIBILITY void __libcpp_thread_sleep_for(chrono::nanoseconds __ns) { -#if __CUDA_ARCH__ >= 700 - auto const __step = __ns.count(); - assert(__step < numeric_limits::max()); - asm volatile("nanosleep.u32 %0;"::"r"((unsigned)__step):); -#else - ; -#endif + NV_IF_TARGET( + NV_IS_DEVICE, ( + auto const __step = __ns.count(); + assert(__step < numeric_limits::max()); + asm volatile("nanosleep.u32 %0;"::"r"((unsigned)__step):); + ) + ) } #elif defined(_LIBCUDACXX_HAS_THREAD_API_PTHREAD) diff --git a/libcxx/include/atomic b/libcxx/include/atomic index be235a64dc..ceb679ae6f 100644 --- a/libcxx/include/atomic +++ b/libcxx/include/atomic @@ -556,10 +556,6 @@ void atomic_signal_fence(memory_order m) noexcept; #include #include <__pragma_push> -#if defined(_LIBCUDACXX_COMPILER_MSVC) -#include "support/win32/atomic_msvc.h" -#endif - #endif //__cuda_std__ #if defined(_LIBCUDACXX_USE_PRAGMA_GCC_SYSTEM_HEADER) @@ -592,6 +588,14 @@ void atomic_signal_fence(memory_order m) noexcept; __f == memory_order_acq_rel, \ "memory order argument to atomic operation is invalid") +#if defined(_LIBCUDACXX_HAS_MSVC_ATOMIC_IMPL) +# include +#endif + +#if !defined(_LIBCUDACXX_COMPILER_NVRTC) +# include +#endif + _LIBCUDACXX_BEGIN_NAMESPACE_STD // Figure out what the underlying type for `memory_order` would be if it were @@ -674,431 +678,34 @@ __cxx_atomic_assign_volatile(_Tp volatile& __a_value, _Tv volatile const& __val) #endif -#if defined(_LIBCUDACXX_HAS_GCC_ATOMIC_IMP) - -template -struct __cxx_atomic_base_impl { - - _LIBCUDACXX_INLINE_VISIBILITY -#ifndef _LIBCUDACXX_CXX03_LANG - __cxx_atomic_base_impl() _NOEXCEPT = default; -#else - __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} -#endif // _LIBCUDACXX_CXX03_LANG - _LIBCUDACXX_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT - : __a_value(value) {} - _ALIGNAS(sizeof(_Tp)) _Tp __a_value; -}; - -_LIBCUDACXX_INLINE_VISIBILITY inline _LIBCUDACXX_CONSTEXPR int __to_gcc_order(memory_order __order) { - // Avoid switch statement to make this a constexpr. - return __order == memory_order_relaxed ? __ATOMIC_RELAXED: - (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: - (__order == memory_order_release ? __ATOMIC_RELEASE: - (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: - (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: - __ATOMIC_CONSUME)))); -} - -_LIBCUDACXX_INLINE_VISIBILITY inline _LIBCUDACXX_CONSTEXPR int __to_gcc_failure_order(memory_order __order) { - // Avoid switch statement to make this a constexpr. - return __order == memory_order_relaxed ? __ATOMIC_RELAXED: - (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: - (__order == memory_order_release ? __ATOMIC_RELAXED: - (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: - (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: - __ATOMIC_CONSUME)))); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { - __cxx_atomic_assign_volatile(__a->__a_value, __val); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) { - __a->__a_value = __val; -} - -_LIBCUDACXX_INLINE_VISIBILITY inline -void __cxx_atomic_thread_fence(memory_order __order) { - __atomic_thread_fence(__to_gcc_order(__order)); -} - -_LIBCUDACXX_INLINE_VISIBILITY inline -void __cxx_atomic_signal_fence(memory_order __order) { - __atomic_signal_fence(__to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, - memory_order __order) { - __atomic_store(&__a->__a_value, &__val, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, - memory_order __order) { - __atomic_store(&__a->__a_value, &__val, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, - memory_order __order) { - _Tp __ret; - __atomic_load(&__a->__a_value, &__ret, - __to_gcc_order(__order)); - return __ret; -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) { - _Tp __ret; - __atomic_load(&__a->__a_value, &__ret, - __to_gcc_order(__order)); - return __ret; -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __value, memory_order __order) { - _Tp __ret; - __atomic_exchange(&__a->__a_value, &__value, &__ret, - __to_gcc_order(__order)); - return __ret; -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, - memory_order __order) { - _Tp __ret; - __atomic_exchange(&__a->__a_value, &__value, &__ret, - __to_gcc_order(__order)); - return __ret; -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_strong( - volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, - memory_order __success, memory_order __failure) { - return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, - false, - __to_gcc_order(__success), - __to_gcc_failure_order(__failure)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_strong( - __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, - memory_order __failure) { - return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, - false, - __to_gcc_order(__success), - __to_gcc_failure_order(__failure)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_weak( - volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, - memory_order __success, memory_order __failure) { - return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, - true, - __to_gcc_order(__success), - __to_gcc_failure_order(__failure)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_weak( - __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, - memory_order __failure) { - return __atomic_compare_exchange(&__a->__a_value, __expected, &__value, - true, - __to_gcc_order(__success), - __to_gcc_failure_order(__failure)); -} - -template -struct __skip_amt { enum {value = 1}; }; - -template -struct __skip_amt<_Tp*> { enum {value = sizeof(_Tp)}; }; - -// FIXME: Haven't figured out what the spec says about using arrays with -// atomic_fetch_add. Force a failure rather than creating bad behavior. -template -struct __skip_amt<_Tp[]> { }; -template -struct __skip_amt<_Tp[n]> { }; - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Td __delta, memory_order __order) { - return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, - memory_order __order) { - return __atomic_fetch_add(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Td __delta, memory_order __order) { - return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, - memory_order __order) { - return __atomic_fetch_sub(&__a->__a_value, __delta * __skip_amt<_Tp>::value, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { - return __atomic_fetch_and(&__a->__a_value, __pattern, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { - return __atomic_fetch_and(&__a->__a_value, __pattern, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { - return __atomic_fetch_or(&__a->__a_value, __pattern, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, - memory_order __order) { - return __atomic_fetch_or(&__a->__a_value, __pattern, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, - _Tp __pattern, memory_order __order) { - return __atomic_fetch_xor(&__a->__a_value, __pattern, - __to_gcc_order(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, - memory_order __order) { - return __atomic_fetch_xor(&__a->__a_value, __pattern, - __to_gcc_order(__order)); -} - -#define __cxx_atomic_is_lock_free(__s) __atomic_is_lock_free(__s, 0) - +// Headers are wrapped like so: (cuda::std::|std::)detail +namespace __detail { +#if defined(_LIBCUDACXX_HAS_CUDA_ATOMIC_IMPL) +# include "support/atomic/atomic_cuda.h" +#elif defined(_LIBCUDACXX_HAS_MSVC_ATOMIC_IMPL) +# include "support/atomic/atomic_msvc.h" +#elif defined(_LIBCUDACXX_HAS_GCC_ATOMIC_IMP) +# include "support/atomic/atomic_gcc.h" #elif defined(_LIBCUDACXX_HAS_C_ATOMIC_IMP) - -template -struct __cxx_atomic_base_impl { - - _LIBCUDACXX_INLINE_VISIBILITY -#ifndef _LIBCUDACXX_CXX03_LANG - __cxx_atomic_base_impl() _NOEXCEPT = default; -#else - __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} -#endif // _LIBCUDACXX_CXX03_LANG - _LIBCUDACXX_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT - : __a_value(value) {} - _LIBCUDACXX_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; -}; - -#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) - -_LIBCUDACXX_INLINE_VISIBILITY inline -void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { - __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); -} - -_LIBCUDACXX_INLINE_VISIBILITY inline -void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { - __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { - __c11_atomic_init(&__a->__a_value, __val); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { - __c11_atomic_init(&__a->__a_value, __val); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { - __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { - __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { - using __ptr_type = typename remove_const__a_value)>::type*; - return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { - using __ptr_type = typename remove_const__a_value)>::type*; - return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { - return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { - return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { - return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { - return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { - return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { - return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); -} - -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); -} -template -_LIBCUDACXX_INLINE_VISIBILITY -_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { - return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); +// TODO: Maybe support C11 atomics? +// #include "support/atomic/atomic_c11.h" +#endif // _LIBCUDACXX_HAS_GCC_ATOMIC_IMP, _LIBCUDACXX_HAS_C_ATOMIC_IMP } -#endif // _LIBCUDACXX_HAS_GCC_ATOMIC_IMP, _LIBCUDACXX_HAS_C_ATOMIC_IMP +using __detail::__cxx_atomic_base_impl; +using __detail::__cxx_atomic_thread_fence; +using __detail::__cxx_atomic_signal_fence; +using __detail::__cxx_atomic_load; +using __detail::__cxx_atomic_store; +using __detail::__cxx_atomic_exchange; +using __detail::__cxx_atomic_compare_exchange_weak; +using __detail::__cxx_atomic_compare_exchange_strong; +using __detail::__cxx_atomic_fetch_add; +using __detail::__cxx_atomic_fetch_sub; +using __detail::__cxx_atomic_fetch_or; +using __detail::__cxx_atomic_fetch_and; +using __detail::__cxx_atomic_fetch_xor; +using __detail::__cxx_atomic_is_lock_free; template _LIBCUDACXX_INLINE_VISIBILITY @@ -1440,7 +1047,7 @@ template >::type> #else template > + typename _Base = __cxx_atomic_base_impl<_Tp, _Sco> > #endif //_LIBCUDACXX_ATOMIC_ONLY_USE_BUILTINS struct __cxx_atomic_impl : public _Base { @@ -1591,7 +1198,7 @@ _LIBCUDACXX_INLINE_VISIBILITY void __cxx_atomic_try_wait_slow(__cxx_atomic_impl< template struct __atomic_wait_and_notify_supported -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 700 +#if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 700 : false_type #else : true_type @@ -2597,7 +2204,7 @@ typedef struct atomic_flag void clear(memory_order __m = memory_order_seq_cst) _NOEXCEPT {__cxx_atomic_store(&__a_, _LIBCUDACXX_ATOMIC_FLAG_TYPE(false), __m);} -#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700 +#if !defined(__CUDA_MINIMUM_ARCH__) || __CUDA_MINIMUM_ARCH__ >= 700 _LIBCUDACXX_INLINE_VISIBILITY void wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {__cxx_atomic_wait(&__a_, _LIBCUDACXX_ATOMIC_FLAG_TYPE(__v), __m);} @@ -2726,7 +2333,7 @@ atomic_flag_clear_explicit(atomic_flag* __o, memory_order __m) _NOEXCEPT __o->clear(__m); } -#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700 +#if !defined(__CUDA_MINIMUM_ARCH__) || __CUDA_MINIMUM_ARCH__ >= 700 inline _LIBCUDACXX_INLINE_VISIBILITY void diff --git a/libcxx/include/cmath b/libcxx/include/cmath index 99a9b055a5..0e44738eb6 100644 --- a/libcxx/include/cmath +++ b/libcxx/include/cmath @@ -596,13 +596,7 @@ _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR typename enable_if::value, bool>::type __libcpp_isnan_or_builtin(_A1 __lcpp_x) _NOEXCEPT { -#if defined(__CUDA_ARCH__) - return __isnan(__lcpp_x); -#elif __has_builtin(__builtin_isnan) - return __builtin_isnan(__lcpp_x); -#else - return isnan(__lcpp_x); -#endif + return isnan(static_cast(__lcpp_x)); } template @@ -618,13 +612,7 @@ _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR typename enable_if::value, bool>::type __libcpp_isinf_or_builtin(_A1 __lcpp_x) _NOEXCEPT { -#if defined(__CUDA_ARCH__) - return __isinf(__lcpp_x); -#elif __has_builtin(__builtin_isinf) - return __builtin_isinf(__lcpp_x); -#else - return isinf(__lcpp_x); -#endif + return isinf(static_cast(__lcpp_x)); } template @@ -640,13 +628,7 @@ _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR typename enable_if::value, bool>::type __libcpp_isfinite_or_builtin(_A1 __lcpp_x) _NOEXCEPT { -#if defined(__CUDA_ARCH__) - return __finite(__lcpp_x); -#elif __has_builtin(__builtin_isfinite) - return __builtin_isfinite(__lcpp_x); -#else - return isfinite(__lcpp_x); -#endif + return isfinite(static_cast(__lcpp_x)); } template diff --git a/libcxx/include/nv b/libcxx/include/nv new file mode 120000 index 0000000000..f10dec61cb --- /dev/null +++ b/libcxx/include/nv @@ -0,0 +1 @@ +../../include/nv \ No newline at end of file diff --git a/libcxx/include/support/atomic/atomic_base.h b/libcxx/include/support/atomic/atomic_base.h new file mode 100644 index 0000000000..548f636a40 --- /dev/null +++ b/libcxx/include/support/atomic/atomic_base.h @@ -0,0 +1,257 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX_ATOMIC_BASE_H +#define _LIBCUDACXX_ATOMIC_BASE_H + +template +struct __cxx_atomic_base_impl { + using __underlying_t = _Tp; + static constexpr int __sco = _Sco; + + _LIBCUDACXX_CONSTEXPR + __cxx_atomic_base_impl() _NOEXCEPT = default; + + _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR explicit + __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {} + + _ALIGNAS(sizeof(_Tp)) _Tp __a_value; +}; + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +_Tp* __cxx_get_underlying_atomic(__cxx_atomic_base_impl<_Tp, _Sco> * __a) _NOEXCEPT { + return &__a->__a_value; +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +volatile _Tp* __cxx_get_underlying_atomic(__cxx_atomic_base_impl<_Tp, _Sco> volatile* __a) _NOEXCEPT { + return &__a->__a_value; +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +const _Tp* __cxx_get_underlying_atomic(__cxx_atomic_base_impl<_Tp, _Sco> const* __a) _NOEXCEPT { + return &__a->__a_value; +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +const volatile _Tp* __cxx_get_underlying_atomic(__cxx_atomic_base_impl<_Tp, _Sco> const volatile* __a) _NOEXCEPT { + return &__a->__a_value; +} + +template +struct __cxx_atomic_ref_base_impl { + using __underlying_t = _Tp; + static constexpr int __sco = _Sco; + + _LIBCUDACXX_CONSTEXPR + __cxx_atomic_ref_base_impl() _NOEXCEPT = default; + + _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR explicit + __cxx_atomic_ref_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {} + + _Tp* __a_value; +}; + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +_Tp* __cxx_get_underlying_atomic(__cxx_atomic_ref_base_impl<_Tp, _Sco>* __a) _NOEXCEPT { + return __a->__a_value; +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +volatile _Tp* __cxx_get_underlying_atomic(__cxx_atomic_ref_base_impl<_Tp, _Sco> volatile* __a) _NOEXCEPT { + return __a->__a_value; +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +const _Tp* __cxx_get_underlying_atomic(__cxx_atomic_ref_base_impl<_Tp, _Sco> const* __a) _NOEXCEPT { + return __a->__a_value; +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +const volatile _Tp* __cxx_get_underlying_atomic(__cxx_atomic_ref_base_impl<_Tp, _Sco> const volatile* __a) _NOEXCEPT { + return __a->__a_value; +} + +template +_LIBCUDACXX_INLINE_VISIBILITY auto __cxx_atomic_base_unwrap(_Tp* __a) _NOEXCEPT -> decltype(__cxx_get_underlying_atomic(__a)) { + return __cxx_get_underlying_atomic(__a); +} + +template +using __cxx_atomic_underlying_t = typename _Tp::__underlying_t; + +_LIBCUDACXX_INLINE_VISIBILITY inline _LIBCUDACXX_CONSTEXPR int __cxx_atomic_order_to_int(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_relaxed ? __ATOMIC_RELAXED: + (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: + (__order == memory_order_release ? __ATOMIC_RELEASE: + (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: + (__order == memory_order_acq_rel ? __ATOMIC_ACQ_REL: + __ATOMIC_CONSUME)))); +} + +_LIBCUDACXX_INLINE_VISIBILITY inline _LIBCUDACXX_CONSTEXPR int __cxx_atomic_failure_order_to_int(memory_order __order) { + // Avoid switch statement to make this a constexpr. + return __order == memory_order_relaxed ? __ATOMIC_RELAXED: + (__order == memory_order_acquire ? __ATOMIC_ACQUIRE: + (__order == memory_order_release ? __ATOMIC_RELAXED: + (__order == memory_order_seq_cst ? __ATOMIC_SEQ_CST: + (__order == memory_order_acq_rel ? __ATOMIC_ACQUIRE: + __ATOMIC_CONSUME)))); +} + +template +inline void __cxx_atomic_init(volatile _Tp* __a, _Up __val) { + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + __cxx_atomic_assign_volatile(*__a_tmp, __val); +} + +template +inline void __cxx_atomic_init(_Tp* __a, _Up __val) { + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + *__a_tmp = __val; +} + +inline +void __cxx_atomic_thread_fence(memory_order __order) { + __atomic_thread_fence(__cxx_atomic_order_to_int(__order)); +} + +inline +void __cxx_atomic_signal_fence(memory_order __order) { + __atomic_signal_fence(__cxx_atomic_order_to_int(__order)); +} + +template +inline void __cxx_atomic_store(_Tp* __a, _Up __val, + memory_order __order) { + typename _CUDA_VSTD::remove_cv<_Tp>::type __v_temp(__val); + (void)__a; + __atomic_store(__a, &__v_temp, __cxx_atomic_order_to_int(__order)); +} + +template +inline auto __cxx_atomic_load(const _Tp* __a, + memory_order __order) -> __cxx_atomic_underlying_t<_Tp> { + typename _CUDA_VSTD::remove_cv<_Tp>::type __ret; + (void)__a; + __atomic_load(__a, &__ret, __cxx_atomic_order_to_int(__order)); + return __ret.__a_value; +} + +template +inline auto __cxx_atomic_exchange(_Tp* __a, _Up __value, + memory_order __order) -> __cxx_atomic_underlying_t<_Tp> { + typename _CUDA_VSTD::remove_cv<_Tp>::type __v_temp(__value); + typename _CUDA_VSTD::remove_cv<_Tp>::type __ret; + (void)__a; + __atomic_exchange(__a, &__v_temp, &__ret, __cxx_atomic_order_to_int(__order)); + return __ret.__a_value; +} + +template +inline bool __cxx_atomic_compare_exchange_strong( + _Tp* __a, _Up* __expected, _Up __value, memory_order __success, + memory_order __failure) { + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + (void)__a_tmp; + (void)__expected; + return __atomic_compare_exchange(__a_tmp, __expected, &__value, + false, + __cxx_atomic_order_to_int(__success), + __cxx_atomic_failure_order_to_int(__failure)); +} + +template +inline bool __cxx_atomic_compare_exchange_weak( + _Tp* __a, _Up* __expected, _Up __value, memory_order __success, + memory_order __failure) { + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + (void)__a_tmp; + (void)__expected; + return __atomic_compare_exchange(__a_tmp, __expected, &__value, + true, + __cxx_atomic_order_to_int(__success), + __cxx_atomic_failure_order_to_int(__failure)); +} + +template +struct __atomic_ptr_inc { enum {value = 1}; }; + +template +struct __atomic_ptr_inc<_Tp*> { enum {value = sizeof(_Tp)}; }; + +// FIXME: Haven't figured out what the spec says about using arrays with +// atomic_fetch_add. Force a failure rather than creating bad behavior. +template +struct __atomic_ptr_inc<_Tp[]> { }; +template +struct __atomic_ptr_inc<_Tp[n]> { }; + +template +inline auto __cxx_atomic_fetch_add(_Tp* __a, _Td __delta, + memory_order __order) -> __cxx_atomic_underlying_t<_Tp> { + constexpr auto __skip_v = __atomic_ptr_inc<__cxx_atomic_underlying_t<_Tp>>::value; + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + return __atomic_fetch_add(__a_tmp, __delta * __skip_v, + __cxx_atomic_order_to_int(__order)); +} + +template +inline auto __cxx_atomic_fetch_sub(_Tp* __a, _Td __delta, + memory_order __order) -> __cxx_atomic_underlying_t<_Tp> { + constexpr auto __skip_v = __atomic_ptr_inc<__cxx_atomic_underlying_t<_Tp>>::value; + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + return __atomic_fetch_sub(__a_tmp, __delta * __skip_v, + __cxx_atomic_order_to_int(__order)); +} + +template +inline auto __cxx_atomic_fetch_and(_Tp* __a, _Td __pattern, + memory_order __order) -> __cxx_atomic_underlying_t<_Tp> { + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + return __atomic_fetch_and(__a_tmp, __pattern, + __cxx_atomic_order_to_int(__order)); +} + +template +inline auto __cxx_atomic_fetch_or(_Tp* __a, _Td __pattern, + memory_order __order) -> __cxx_atomic_underlying_t<_Tp> { + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + return __atomic_fetch_or(__a_tmp, __pattern, + __cxx_atomic_order_to_int(__order)); +} + +template +inline auto __cxx_atomic_fetch_xor(_Tp* __a, _Td __pattern, + memory_order __order) -> __cxx_atomic_underlying_t<_Tp> { + auto __a_tmp = __cxx_atomic_base_unwrap(__a); + return __atomic_fetch_xor(__a_tmp, __pattern, + __cxx_atomic_order_to_int(__order)); +} + +inline constexpr + bool __cxx_atomic_is_lock_free(size_t __x) { + #if defined(_LIBCUDACXX_NO_RUNTIME_LOCK_FREE) + return __x <= 8; + #else + return __atomic_is_lock_free(__x, 0); + #endif +} + +#endif // _LIBCUDACXX_ATOMIC_BASE_H diff --git a/libcxx/include/support/atomic/atomic_c11.h b/libcxx/include/support/atomic/atomic_c11.h new file mode 100644 index 0000000000..dd6abbc6d4 --- /dev/null +++ b/libcxx/include/support/atomic/atomic_c11.h @@ -0,0 +1,181 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// Atomics for C11 + +template +struct __cxx_atomic_base_impl { + + _LIBCUDACXX_INLINE_VISIBILITY +#ifndef _LIBCUDACXX_CXX03_LANG + __cxx_atomic_base_impl() _NOEXCEPT = default; +#else + __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {} +#endif // _LIBCUDACXX_CXX03_LANG + _LIBCUDACXX_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT + : __a_value(value) {} + _LIBCUDACXX_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value; +}; + +#define __cxx_atomic_is_lock_free(__s) __c11_atomic_is_lock_free(__s) + +_LIBCUDACXX_INLINE_VISIBILITY inline +void __cxx_atomic_thread_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order)); +} + +_LIBCUDACXX_INLINE_VISIBILITY inline +void __cxx_atomic_signal_fence(memory_order __order) _NOEXCEPT { + __c11_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(&__a->__a_value, __val); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val) _NOEXCEPT { + __c11_atomic_init(&__a->__a_value, __val); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> * __a, _Tp __val, memory_order __order) _NOEXCEPT { + __c11_atomic_store(&__a->__a_value, __val, static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const volatile* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = typename remove_const__a_value)>::type*; + return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __order) _NOEXCEPT { + using __ptr_type = typename remove_const__a_value)>::type*; + return __c11_atomic_load(const_cast<__ptr_type>(&__a->__a_value), static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> * __a, _Tp __value, memory_order __order) _NOEXCEPT { + return __c11_atomic_exchange(&__a->__a_value, __value, static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_strong(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_impl<_Tp> * __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) _NOEXCEPT { + return __c11_atomic_compare_exchange_weak(&__a->__a_value, __expected, __value, static_cast<__memory_order_underlying_t>(__success), static_cast<__memory_order_underlying_t>(__failure)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_add(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp> * __a, _Tp __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> volatile* __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp*> * __a, ptrdiff_t __delta, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_sub(&__a->__a_value, __delta, static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_and(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_or(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); +} +template +_LIBCUDACXX_INLINE_VISIBILITY +_Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp> * __a, _Tp __pattern, memory_order __order) _NOEXCEPT { + return __c11_atomic_fetch_xor(&__a->__a_value, __pattern, static_cast<__memory_order_underlying_t>(__order)); +} diff --git a/libcxx/include/support/atomic/atomic_cuda.h b/libcxx/include/support/atomic/atomic_cuda.h new file mode 100644 index 0000000000..b0e17c5bd8 --- /dev/null +++ b/libcxx/include/support/atomic/atomic_cuda.h @@ -0,0 +1,479 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__CUDA_MINIMUM_ARCH__) && ((!defined(_MSC_VER) && __CUDA_MINIMUM_ARCH__ < 600) || (defined(_MSC_VER) && __CUDA_MINIMUM_ARCH__ < 700)) +# error "CUDA atomics are only supported for sm_60 and up on *nix and sm_70 and up on Windows." +#endif + +#if !defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) && !defined(__GCC_ATOMIC_BOOL_LOCK_FREE) +#define ATOMIC_BOOL_LOCK_FREE 2 +#define ATOMIC_CHAR_LOCK_FREE 2 +#define ATOMIC_CHAR16_T_LOCK_FREE 2 +#define ATOMIC_CHAR32_T_LOCK_FREE 2 +#define ATOMIC_WCHAR_T_LOCK_FREE 2 +#define ATOMIC_SHORT_LOCK_FREE 2 +#define ATOMIC_INT_LOCK_FREE 2 +#define ATOMIC_LONG_LOCK_FREE 2 +#define ATOMIC_LLONG_LOCK_FREE 2 +#define ATOMIC_POINTER_LOCK_FREE 2 +#endif //!defined(__CLANG_ATOMIC_BOOL_LOCK_FREE) && !defined(__GCC_ATOMIC_BOOL_LOCK_FREE) + +#ifndef __ATOMIC_RELAXED +#define __ATOMIC_RELAXED 0 +#define __ATOMIC_CONSUME 1 +#define __ATOMIC_ACQUIRE 2 +#define __ATOMIC_RELEASE 3 +#define __ATOMIC_ACQ_REL 4 +#define __ATOMIC_SEQ_CST 5 +#endif //__ATOMIC_RELAXED + +#ifndef __ATOMIC_BLOCK +#define __ATOMIC_SYSTEM 0 // 0 indicates default +#define __ATOMIC_DEVICE 1 +#define __ATOMIC_BLOCK 2 +#define __ATOMIC_THREAD 10 +#endif //__ATOMIC_BLOCK + +inline __host__ __device__ int __stronger_order_cuda(int __a, int __b) { + int const __max = __a > __b ? __a : __b; + if(__max != __ATOMIC_RELEASE) + return __max; + static int const __xform[] = { + __ATOMIC_RELEASE, + __ATOMIC_ACQ_REL, + __ATOMIC_ACQ_REL, + __ATOMIC_RELEASE }; + return __xform[__a < __b ? __a : __b]; +} + +enum thread_scope { + thread_scope_system = __ATOMIC_SYSTEM, + thread_scope_device = __ATOMIC_DEVICE, + thread_scope_block = __ATOMIC_BLOCK, + thread_scope_thread = __ATOMIC_THREAD +}; + +#define _LIBCUDACXX_ATOMIC_SCOPE_TYPE ::cuda::thread_scope +#define _LIBCUDACXX_ATOMIC_SCOPE_DEFAULT ::cuda::thread_scope::system + +struct __thread_scope_thread_tag { }; +struct __thread_scope_block_tag { }; +struct __thread_scope_device_tag { }; +struct __thread_scope_system_tag { }; + +template struct __scope_enum_to_tag { }; +/* This would be the implementation once an actual thread-scope backend exists. +template<> struct __scope_enum_to_tag<(int)thread_scope_thread> { + using type = __thread_scope_thread_tag; }; +Until then: */ +template<> struct __scope_enum_to_tag<(int)thread_scope_thread> { + using type = __thread_scope_block_tag; }; +template<> struct __scope_enum_to_tag<(int)thread_scope_block> { + using type = __thread_scope_block_tag; }; +template<> struct __scope_enum_to_tag<(int)thread_scope_device> { + using type = __thread_scope_device_tag; }; +template<> struct __scope_enum_to_tag<(int)thread_scope_system> { + using type = __thread_scope_system_tag; }; + +template +_LIBCUDACXX_INLINE_VISIBILITY auto constexpr __scope_tag() -> + typename __scope_enum_to_tag<_Scope>::type { + return typename __scope_enum_to_tag<_Scope>::type(); +} +// END TODO + +// Wrap host atomic implementations into a sub-namespace +namespace __host { +#if defined(_LIBCUDACXX_COMPILER_MSVC) +# include "atomic_msvc.h" +#elif defined (_LIBCUDACXX_HAS_GCC_ATOMIC_IMP) +# include "atomic_gcc.h" +#elif defined (_LIBCUDACXX_HAS_C11_ATOMIC_IMP) +//TODO +// # include "atomic_c11.h" +#endif +} + +#include "atomic_cuda_generated.h" +#include "atomic_cuda_derived.h" + +_LIBCUDACXX_INLINE_VISIBILITY + bool __cxx_atomic_is_lock_free(size_t __x) { + return __x <= 8; +} + +_LIBCUDACXX_INLINE_VISIBILITY + void __cxx_atomic_thread_fence(memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + __atomic_thread_fence_cuda(__order, __thread_scope_system_tag()); + ), + NV_IS_HOST, ( + __host::__cxx_atomic_thread_fence(__order); + ) + ) +} + +_LIBCUDACXX_INLINE_VISIBILITY + void __cxx_atomic_signal_fence(memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + __atomic_signal_fence_cuda(__order); + ), + NV_IS_HOST, ( + __host::__cxx_atomic_signal_fence(__order); + ) + ) +} + +template +using __cxx_atomic_base_heterogeneous_storage + = typename conditional<_Ref, + __host::__cxx_atomic_ref_base_impl<_Tp, _Sco>, + __host::__cxx_atomic_base_impl<_Tp, _Sco> >::type; + + +template +struct __cxx_atomic_base_heterogeneous_impl { + __cxx_atomic_base_heterogeneous_impl() noexcept = default; + _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR explicit + __cxx_atomic_base_heterogeneous_impl(_Tp __value) : __a_value(__value) { + } + + __cxx_atomic_base_heterogeneous_storage<_Tp, _Sco, _Ref> __a_value; +}; + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +_Tp* __cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> * __a) _NOEXCEPT { + return __cxx_atomic_base_unwrap(&__a->__a_value); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +volatile _Tp* __cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a) _NOEXCEPT { + return __cxx_atomic_base_unwrap(&__a->__a_value); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +const _Tp* __cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> const* __a) _NOEXCEPT { + return __cxx_atomic_base_unwrap(&__a->__a_value); +} + +template +_LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR +const volatile _Tp* __cxx_get_underlying_device_atomic(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> const volatile* __a) _NOEXCEPT { + return __cxx_atomic_base_unwrap(&__a->__a_value); +} + +template +struct __cxx_atomic_base_small_impl { + __cxx_atomic_base_small_impl() noexcept = default; + _LIBCUDACXX_INLINE_VISIBILITY _LIBCUDACXX_CONSTEXPR explicit + __cxx_atomic_base_small_impl(_Tp __value) : __a_value(__value) { + } + + __cxx_atomic_base_heterogeneous_impl __a_value; +}; + +template +using __cxx_small_proxy = typename conditional::type >::type; + +template +using __cxx_atomic_base_impl = typename conditional, + __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco> >::type; + + +template +using __cxx_atomic_ref_base_impl = __cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, true>; + +template +__host__ __device__ + void __cxx_atomic_init(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __val) { + alignas(_Tp) auto __tmp = __val; + __cxx_atomic_assign_volatile(*__cxx_get_underlying_device_atomic(__a), __tmp); +} + +template +__host__ __device__ + void __cxx_atomic_store(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __val, memory_order __order) { + alignas(_Tp) auto __tmp = __val; + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + __atomic_store_n_cuda(__cxx_get_underlying_device_atomic(__a), __tmp, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + __host::__cxx_atomic_store(&__a->__a_value, __tmp, __order); + ) + ) +} + +template +__host__ __device__ + _Tp __cxx_atomic_load(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> const volatile* __a, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_load_n_cuda(__cxx_get_underlying_device_atomic(__a), __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_load(&__a->__a_value, __order); + ) + ) +} + +template +__host__ __device__ + _Tp __cxx_atomic_exchange(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __val, memory_order __order) { + alignas(_Tp) auto __tmp = __val; + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_exchange_n_cuda(__cxx_get_underlying_device_atomic(__a), __tmp, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_exchange(&__a->__a_value, __tmp, __order); + ) + ) +} + +template +__host__ __device__ + bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp* __expected, _Tp __val, memory_order __success, memory_order __failure) { + alignas(_Tp) auto __tmp = *__expected; + bool __result = false; + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + alignas(_Tp) auto __tmp_v = __val; + __result = __atomic_compare_exchange_cuda(__cxx_get_underlying_device_atomic(__a), &__tmp, &__tmp_v, false, __success, __failure, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + __result = __host::__cxx_atomic_compare_exchange_strong(&__a->__a_value, &__tmp, __val, __success, __failure); + ) + ) + *__expected = __tmp; + return __result; +} + +template +__host__ __device__ + bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp* __expected, _Tp __val, memory_order __success, memory_order __failure) { + alignas(_Tp) auto __tmp = *__expected; + bool __result = false; + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + alignas(_Tp) auto __tmp_v = __val; + __result = __atomic_compare_exchange_cuda(__cxx_get_underlying_device_atomic(__a), &__tmp, &__tmp_v, true, __success, __failure, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + __result = __host::__cxx_atomic_compare_exchange_weak(&__a->__a_value, &__tmp, __val, __success, __failure); + ) + ) + *__expected = __tmp; + return __result; +} + +template +__host__ __device__ + _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __delta, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_fetch_add_cuda(__cxx_get_underlying_device_atomic(__a), __delta, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_fetch_add(&__a->__a_value, __delta, __order); + ) + ) +} + +template +__host__ __device__ + _Tp* __cxx_atomic_fetch_add(__cxx_atomic_base_heterogeneous_impl<_Tp*, _Sco, _Ref> volatile* __a, ptrdiff_t __delta, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_fetch_add_cuda(__cxx_get_underlying_device_atomic(__a), __delta, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_fetch_add(&__a->__a_value, __delta, __order); + ) + ) +} + +template +__host__ __device__ + _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __delta, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_fetch_sub_cuda(__cxx_get_underlying_device_atomic(__a), __delta, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_fetch_sub(&__a->__a_value, __delta, __order); + ) + ) +} + +template +__host__ __device__ + _Tp* __cxx_atomic_fetch_sub(__cxx_atomic_base_heterogeneous_impl<_Tp*, _Sco, _Ref> volatile* __a, ptrdiff_t __delta, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_fetch_sub_cuda(__cxx_get_underlying_device_atomic(__a), __delta, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_fetch_sub(&__a->__a_value, __delta, __order); + ) + ) +} + +template +__host__ __device__ + _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __pattern, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_fetch_and_cuda(__cxx_get_underlying_device_atomic(__a), __pattern, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_fetch_and(&__a->__a_value, __pattern, __order); + ) + ) +} + +template +__host__ __device__ + _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __pattern, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_fetch_or_cuda(__cxx_get_underlying_device_atomic(__a), __pattern, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_fetch_or(&__a->__a_value, __pattern, __order); + ) + ) +} + +template +__host__ __device__ + _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_heterogeneous_impl<_Tp, _Sco, _Ref> volatile* __a, _Tp __pattern, memory_order __order) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + return __atomic_fetch_xor_cuda(__cxx_get_underlying_device_atomic(__a), __pattern, __order, __scope_tag<_Sco>()); + ), + NV_IS_HOST, ( + return __host::__cxx_atomic_fetch_xor(&__a->__a_value, __pattern, __order); + ) + ) +} + +template +__host__ __device__ inline uint32_t __cxx_small_to_32(_Tp __val) { + __cxx_small_proxy<_Tp> __temp = 0; + memcpy(&__temp, &__val, sizeof(_Tp)); + return __temp; +} + +template +__host__ __device__ inline _Tp __cxx_small_from_32(uint32_t __val) { + __cxx_small_proxy<_Tp> __temp = static_cast<__cxx_small_proxy<_Tp>>(__val); + _Tp __result; + memcpy(&__result, &__temp, sizeof(_Tp)); + return __result; +} + +template +__host__ __device__ inline void __cxx_atomic_init(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __val) { + __cxx_atomic_init(&__a->__a_value, __cxx_small_to_32(__val)); +} + +template +__host__ __device__ inline void __cxx_atomic_store(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __val, memory_order __order) { + __cxx_atomic_store(&__a->__a_value, __cxx_small_to_32(__val), __order); +} + +template +__host__ __device__ inline _Tp __cxx_atomic_load(__cxx_atomic_base_small_impl<_Tp, _Sco> const volatile* __a, memory_order __order) { + return __cxx_small_from_32<_Tp>(__cxx_atomic_load(&__a->__a_value, __order)); +} + +template +__host__ __device__ inline _Tp __cxx_atomic_exchange(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __value, memory_order __order) { + return __cxx_small_from_32<_Tp>(__cxx_atomic_exchange(&__a->__a_value, __cxx_small_to_32(__value), __order)); +} +__host__ __device__ +inline int __cuda_memcmp(void const * __lhs, void const * __rhs, size_t __count) { + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + auto __lhs_c = reinterpret_cast(__lhs); + auto __rhs_c = reinterpret_cast(__rhs); + while (__count--) { + auto const __lhs_v = *__lhs_c++; + auto const __rhs_v = *__rhs_c++; + if (__lhs_v < __rhs_v) { return -1; } + if (__lhs_v > __rhs_v) { return 1; } + } + return 0; + ), + NV_IS_HOST, ( + return memcmp(__lhs, __rhs, __count); + ) + ) +} + +template +__host__ __device__ inline bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { + auto __temp = __cxx_small_to_32(*__expected); + auto const __ret = __cxx_atomic_compare_exchange_weak(&__a->__a_value, &__temp, __cxx_small_to_32(__value), __success, __failure); + auto const __actual = __cxx_small_from_32<_Tp>(__temp); + if(!__ret) { + if(0 == __cuda_memcmp(&__actual, __expected, sizeof(_Tp))) + __cxx_atomic_fetch_and(&__a->__a_value, (1u << (8*sizeof(_Tp))) - 1, memory_order::memory_order_relaxed); + else + *__expected = __actual; + } + return __ret; +} + +template +__host__ __device__ inline bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) { + auto const __old = *__expected; + while(1) { + if(__cxx_atomic_compare_exchange_weak(__a, __expected, __value, __success, __failure)) + return true; + if(0 != __cuda_memcmp(&__old, __expected, sizeof(_Tp))) + return false; + } +} + +template +__host__ __device__ inline _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __delta, memory_order __order) { + return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_add(&__a->__a_value, __cxx_small_to_32(__delta), __order)); +} + +template +__host__ __device__ inline _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __delta, memory_order __order) { + return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_sub(&__a->__a_value, __cxx_small_to_32(__delta), __order)); +} + +template +__host__ __device__ inline _Tp __cxx_atomic_fetch_and(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __pattern, memory_order __order) { + return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_and(&__a->__a_value, __cxx_small_to_32(__pattern), __order)); +} + +template +__host__ __device__ inline _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __pattern, memory_order __order) { + return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_or(&__a->__a_value, __cxx_small_to_32(__pattern), __order)); +} + +template +__host__ __device__ inline _Tp __cxx_atomic_fetch_xor(__cxx_atomic_base_small_impl<_Tp, _Sco> volatile* __a, _Tp __pattern, memory_order __order) { + return __cxx_small_from_32<_Tp>(__cxx_atomic_fetch_xor(&__a->__a_value, __cxx_small_to_32(__pattern), __order)); +} diff --git a/include/cuda/std/detail/__atomic_derived b/libcxx/include/support/atomic/atomic_cuda_derived.h similarity index 83% rename from include/cuda/std/detail/__atomic_derived rename to libcxx/include/support/atomic/atomic_cuda_derived.h index 204ebb9989..7c005e423e 100644 --- a/include/cuda/std/detail/__atomic_derived +++ b/libcxx/include/support/atomic/atomic_cuda_derived.h @@ -7,9 +7,6 @@ // //===----------------------------------------------------------------------===// -_LIBCUDACXX_BEGIN_NAMESPACE_CUDA -namespace detail { - template::type = 0> bool __device__ __atomic_compare_exchange_cuda(_Type volatile *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder, _Scope __s) { @@ -55,16 +52,22 @@ _Type __host__ __device__ __atomic_fetch_max_cuda(_Type volatile *__ptr, _Delta _Type __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, __s); _Type __desired = __expected > __val ? __expected : __val; -#ifdef __CUDA_ARCH__ - while(__desired == __val && - !__atomic_compare_exchange_cuda(__ptr, &__expected, &__desired, true, __memorder, __memorder, __s)) { -#else - while(__desired == __val && - !__atomic_compare_exchange(__ptr, &__expected, &__desired, true, __memorder, __memorder)) { -#endif - __desired = __expected > __val ? __expected : __val; - } - return __expected; + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + while(__desired == __val && + !__atomic_compare_exchange_cuda(__ptr, &__expected, &__desired, true, __memorder, __memorder, __s)) { + __desired = __expected > __val ? __expected : __val; + } + return __expected; + ), + NV_IS_HOST, ( + while(__desired == __val && + !__atomic_compare_exchange(__ptr, &__expected, &__desired, true, __memorder, __memorder)) { + __desired = __expected > __val ? __expected : __val; + } + return __expected; + ) + ) } template::type = 0> @@ -72,16 +75,22 @@ _Type __host__ __device__ __atomic_fetch_min_cuda(_Type volatile *__ptr, _Delta _Type __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, __s); _Type __desired = __expected < __val ? __expected : __val; -#ifdef __CUDA_ARCH__ - while(__desired != __val && - !__atomic_compare_exchange_cuda(__ptr, &__expected, &__desired, true, __memorder, __memorder, __s)) { -#else - while(__desired != __val && - !__atomic_compare_exchange(__ptr, &__expected, &__desired, true, __memorder, __memorder)) { -#endif - __desired = __expected < __val ? __expected : __val; - } - return __expected; + NV_DISPATCH_TARGET( + NV_IS_DEVICE, ( + while(__desired != __val && + !__atomic_compare_exchange_cuda(__ptr, &__expected, &__desired, true, __memorder, __memorder, __s)) { + __desired = __expected < __val ? __expected : __val; + } + return __expected; + ), + NV_IS_HOST, ( + while(__desired != __val && + !__atomic_compare_exchange(__ptr, &__expected, &__desired, true, __memorder, __memorder)) { + __desired = __expected < __val ? __expected : __val; + } + return __expected; + ) + ) } template::type = 0> @@ -152,6 +161,3 @@ static inline __device__ void __atomic_signal_fence_cuda(int) { asm volatile("":::"memory"); } -} -_LIBCUDACXX_END_NAMESPACE_CUDA - diff --git a/include/cuda/std/detail/__atomic_generated b/libcxx/include/support/atomic/atomic_cuda_generated.h similarity index 53% rename from include/cuda/std/detail/__atomic_generated rename to libcxx/include/support/atomic/atomic_cuda_generated.h index 596467d991..f4d8cd52c6 100644 --- a/include/cuda/std/detail/__atomic_generated +++ b/libcxx/include/support/atomic/atomic_cuda_generated.h @@ -8,30 +8,34 @@ //===----------------------------------------------------------------------===// -_LIBCUDACXX_BEGIN_NAMESPACE_CUDA -namespace detail { - static inline __device__ void __cuda_membar_block() { asm volatile("membar.cta;":::"memory"); } static inline __device__ void __cuda_fence_acq_rel_block() { asm volatile("fence.acq_rel.cta;":::"memory"); } static inline __device__ void __cuda_fence_sc_block() { asm volatile("fence.sc.cta;":::"memory"); } static inline __device__ void __atomic_thread_fence_cuda(int __memorder, __thread_scope_block_tag) { - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); break; - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - case __ATOMIC_ACQ_REL: - case __ATOMIC_RELEASE: __cuda_fence_acq_rel_block(); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - case __ATOMIC_ACQ_REL: - case __ATOMIC_RELEASE: __cuda_membar_block(); break; -#endif // __CUDA_ARCH__ >= 700 - case __ATOMIC_RELAXED: break; - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); break; + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: + case __ATOMIC_ACQ_REL: + case __ATOMIC_RELEASE: __cuda_fence_acq_rel_block(); break; + case __ATOMIC_RELAXED: break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: + case __ATOMIC_ACQ_REL: + case __ATOMIC_RELEASE: __cuda_membar_block(); break; + case __ATOMIC_RELAXED: break; + default: assert(0); + } + ) + ) } template static inline __device__ void __cuda_load_acquire_32_block(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline __device__ void __cuda_load_relaxed_32_block(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } @@ -39,20 +43,26 @@ template static inline __device__ void __cuda_load template::type = 0> __device__ void __atomic_load_cuda(const volatile _Type *__ptr, _Type *__ret, int __memorder, __thread_scope_block_tag) { uint32_t __tmp = 0; - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_acquire_32_block(__ptr, __tmp); break; - case __ATOMIC_RELAXED: __cuda_load_relaxed_32_block(__ptr, __tmp); break; -#else - case __ATOMIC_SEQ_CST: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_volatile_32_block(__ptr, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELAXED: __cuda_load_volatile_32_block(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_acquire_32_block(__ptr, __tmp); break; + case __ATOMIC_RELAXED: __cuda_load_relaxed_32_block(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_volatile_32_block(__ptr, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELAXED: __cuda_load_volatile_32_block(__ptr, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 4); } template static inline __device__ void __cuda_load_acquire_64_block(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } @@ -61,20 +71,26 @@ template static inline __device__ void __cuda_load template::type = 0> __device__ void __atomic_load_cuda(const volatile _Type *__ptr, _Type *__ret, int __memorder, __thread_scope_block_tag) { uint64_t __tmp = 0; - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_acquire_64_block(__ptr, __tmp); break; - case __ATOMIC_RELAXED: __cuda_load_relaxed_64_block(__ptr, __tmp); break; -#else - case __ATOMIC_SEQ_CST: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_volatile_64_block(__ptr, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELAXED: __cuda_load_volatile_64_block(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_acquire_64_block(__ptr, __tmp); break; + case __ATOMIC_RELAXED: __cuda_load_relaxed_64_block(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_volatile_64_block(__ptr, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELAXED: __cuda_load_volatile_64_block(__ptr, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 8); } template static inline __device__ void __cuda_store_relaxed_32_block(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.relaxed.cta.b32 [%0], %1;" :: "l"(__ptr),"r"(__src) : "memory"); } @@ -84,18 +100,24 @@ template::type __device__ void __atomic_store_cuda(volatile _Type *__ptr, _Type *__val, int __memorder, __thread_scope_block_tag) { uint32_t __tmp = 0; memcpy(&__tmp, __val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_RELEASE: __cuda_store_release_32_block(__ptr, __tmp); break; - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_RELAXED: __cuda_store_relaxed_32_block(__ptr, __tmp); break; -#else - case __ATOMIC_RELEASE: - case __ATOMIC_SEQ_CST: __cuda_membar_block(); - case __ATOMIC_RELAXED: __cuda_store_volatile_32_block(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_RELEASE: __cuda_store_release_32_block(__ptr, __tmp); break; + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_RELAXED: __cuda_store_relaxed_32_block(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_RELEASE: + case __ATOMIC_SEQ_CST: __cuda_membar_block(); + case __ATOMIC_RELAXED: __cuda_store_volatile_32_block(__ptr, __tmp); break; + default: assert(0); + } + ) + ) } template static inline __device__ void __cuda_store_relaxed_64_block(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.relaxed.cta.b64 [%0], %1;" :: "l"(__ptr),"l"(__src) : "memory"); } template static inline __device__ void __cuda_store_release_64_block(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.release.cta.b64 [%0], %1;" :: "l"(__ptr),"l"(__src) : "memory"); } @@ -104,48 +126,60 @@ template::type __device__ void __atomic_store_cuda(volatile _Type *__ptr, _Type *__val, int __memorder, __thread_scope_block_tag) { uint64_t __tmp = 0; memcpy(&__tmp, __val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_RELEASE: __cuda_store_release_64_block(__ptr, __tmp); break; - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_RELAXED: __cuda_store_relaxed_64_block(__ptr, __tmp); break; -#else - case __ATOMIC_RELEASE: - case __ATOMIC_SEQ_CST: __cuda_membar_block(); - case __ATOMIC_RELAXED: __cuda_store_volatile_64_block(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } -} -template static inline __device__ void __cuda_compare_exchange_acq_rel_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_acquire_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_relaxed_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_release_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_volatile_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_RELEASE: __cuda_store_release_64_block(__ptr, __tmp); break; + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_RELAXED: __cuda_store_relaxed_64_block(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_RELEASE: + case __ATOMIC_SEQ_CST: __cuda_membar_block(); + case __ATOMIC_RELAXED: __cuda_store_volatile_64_block(__ptr, __tmp); break; + default: assert(0); + } + ) + ) +} +template static inline __device__ void __cuda_compare_exchange_acq_rel_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acquire_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_relaxed_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_release_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_volatile_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } template::type = 0> __device__ bool __atomic_compare_exchange_cuda(volatile _Type *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder, __thread_scope_block_tag) { uint32_t __tmp = 0, __old = 0, __old_tmp; memcpy(&__tmp, __desired, 4); memcpy(&__old, __expected, 4); __old_tmp = __old; - switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_32_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_32_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_compare_exchange_release_32_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_32_block(__ptr, __old, __old_tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_32_block(__ptr, __old, __old_tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_compare_exchange_volatile_32_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_32_block(__ptr, __old, __old_tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_32_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_32_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_compare_exchange_release_32_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_32_block(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_32_block(__ptr, __old, __old_tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_compare_exchange_volatile_32_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_32_block(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ) + ) bool const __ret = __old == __old_tmp; memcpy(__expected, &__old, 4); return __ret; @@ -159,24 +193,30 @@ template::type __device__ void __atomic_exchange_cuda(volatile _Type *__ptr, _Type *__val, _Type *__ret, int __memorder, __thread_scope_block_tag) { uint32_t __tmp = 0; memcpy(&__tmp, __val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_exchange_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_exchange_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_exchange_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_exchange_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 4); } template static inline __device__ void __cuda_fetch_add_acq_rel_32_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __op) { asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr),"r"(__op) : "memory"); } @@ -189,24 +229,30 @@ __device__ _Type __atomic_fetch_add_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -220,24 +266,30 @@ __device__ _Type __atomic_fetch_and_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_and_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_and_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_and_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_and_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -251,24 +303,30 @@ __device__ _Type __atomic_fetch_max_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_max_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_max_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_max_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_max_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -282,24 +340,30 @@ __device__ _Type __atomic_fetch_min_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_min_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_min_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_min_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_min_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -313,24 +377,30 @@ __device__ _Type __atomic_fetch_or_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_or_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_or_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_or_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_or_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -349,24 +419,30 @@ __device__ _Type __atomic_fetch_sub_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_sub_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_sub_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_sub_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_sub_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -380,56 +456,68 @@ __device__ _Type __atomic_fetch_xor_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_xor_release_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_32_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_xor_volatile_32_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_32_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_xor_release_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_32_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_xor_volatile_32_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_32_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } -template static inline __device__ void __cuda_compare_exchange_acq_rel_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_acquire_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_relaxed_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_release_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_volatile_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acq_rel_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acquire_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_relaxed_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_release_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_volatile_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } template::type = 0> __device__ bool __atomic_compare_exchange_cuda(volatile _Type *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder, __thread_scope_block_tag) { uint64_t __tmp = 0, __old = 0, __old_tmp; memcpy(&__tmp, __desired, 8); memcpy(&__old, __expected, 8); __old_tmp = __old; - switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_64_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_64_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_compare_exchange_release_64_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_64_block(__ptr, __old, __old_tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_64_block(__ptr, __old, __old_tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_compare_exchange_volatile_64_block(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_64_block(__ptr, __old, __old_tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_64_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_64_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_compare_exchange_release_64_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_64_block(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_64_block(__ptr, __old, __old_tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_compare_exchange_volatile_64_block(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_64_block(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ) + ) bool const __ret = __old == __old_tmp; memcpy(__expected, &__old, 8); return __ret; @@ -443,24 +531,30 @@ template::type __device__ void __atomic_exchange_cuda(volatile _Type *__ptr, _Type *__val, _Type *__ret, int __memorder, __thread_scope_block_tag) { uint64_t __tmp = 0; memcpy(&__tmp, __val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_exchange_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_exchange_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_exchange_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_exchange_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 8); } template static inline __device__ void __cuda_fetch_add_acq_rel_64_block(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __op) { asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr),"l"(__op) : "memory"); } @@ -473,24 +567,30 @@ __device__ _Type __atomic_fetch_add_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -504,24 +604,30 @@ __device__ _Type __atomic_fetch_and_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_and_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_and_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_and_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_and_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -535,24 +641,30 @@ __device__ _Type __atomic_fetch_max_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_max_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_max_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_max_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_max_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -566,24 +678,30 @@ __device__ _Type __atomic_fetch_min_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_min_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_min_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_min_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_min_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -597,24 +715,30 @@ __device__ _Type __atomic_fetch_or_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_or_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_or_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_or_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_or_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -633,24 +757,30 @@ __device__ _Type __atomic_fetch_sub_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_sub_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_sub_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_sub_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_sub_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -664,24 +794,30 @@ __device__ _Type __atomic_fetch_xor_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_xor_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_xor_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_xor_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_xor_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -691,24 +827,29 @@ __device__ _Type* __atomic_fetch_add_cuda(_Type *volatile *__ptr, ptrdiff_t __va uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); __tmp *= sizeof(_Type); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_block(__ptr, __tmp, __tmp); break; + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -719,24 +860,29 @@ __device__ _Type* __atomic_fetch_sub_cuda(_Type *volatile *__ptr, ptrdiff_t __va memcpy(&__tmp, &__val, 8); __tmp = -__tmp; __tmp *= sizeof(_Type); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_block(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_block(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; - case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_block(__ptr, __tmp, __tmp); break; + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_block(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); __cuda_membar_block(); break; + case __ATOMIC_RELEASE: __cuda_membar_block(); __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_block(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -744,23 +890,30 @@ static inline __device__ void __cuda_membar_device() { asm volatile("membar.gl;" static inline __device__ void __cuda_fence_acq_rel_device() { asm volatile("fence.acq_rel.gpu;":::"memory"); } static inline __device__ void __cuda_fence_sc_device() { asm volatile("fence.sc.gpu;":::"memory"); } static inline __device__ void __atomic_thread_fence_cuda(int __memorder, __thread_scope_device_tag) { - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); break; - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - case __ATOMIC_ACQ_REL: - case __ATOMIC_RELEASE: __cuda_fence_acq_rel_device(); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - case __ATOMIC_ACQ_REL: - case __ATOMIC_RELEASE: __cuda_membar_device(); break; -#endif // __CUDA_ARCH__ >= 700 - case __ATOMIC_RELAXED: break; - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); break; + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: + case __ATOMIC_ACQ_REL: + case __ATOMIC_RELEASE: __cuda_fence_acq_rel_device(); break; + case __ATOMIC_RELAXED: break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: + case __ATOMIC_ACQ_REL: + case __ATOMIC_RELEASE: __cuda_membar_device(); break; + case __ATOMIC_RELAXED: break; + default: assert(0); + } + ) + ) } template static inline __device__ void __cuda_load_acquire_32_device(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline __device__ void __cuda_load_relaxed_32_device(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } @@ -768,20 +921,26 @@ template static inline __device__ void __cuda_load template::type = 0> __device__ void __atomic_load_cuda(const volatile _Type *__ptr, _Type *__ret, int __memorder, __thread_scope_device_tag) { uint32_t __tmp = 0; - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_acquire_32_device(__ptr, __tmp); break; - case __ATOMIC_RELAXED: __cuda_load_relaxed_32_device(__ptr, __tmp); break; -#else - case __ATOMIC_SEQ_CST: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_volatile_32_device(__ptr, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELAXED: __cuda_load_volatile_32_device(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_acquire_32_device(__ptr, __tmp); break; + case __ATOMIC_RELAXED: __cuda_load_relaxed_32_device(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_volatile_32_device(__ptr, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELAXED: __cuda_load_volatile_32_device(__ptr, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 4); } template static inline __device__ void __cuda_load_acquire_64_device(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } @@ -790,20 +949,26 @@ template static inline __device__ void __cuda_load template::type = 0> __device__ void __atomic_load_cuda(const volatile _Type *__ptr, _Type *__ret, int __memorder, __thread_scope_device_tag) { uint64_t __tmp = 0; - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_acquire_64_device(__ptr, __tmp); break; - case __ATOMIC_RELAXED: __cuda_load_relaxed_64_device(__ptr, __tmp); break; -#else - case __ATOMIC_SEQ_CST: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_volatile_64_device(__ptr, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELAXED: __cuda_load_volatile_64_device(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_acquire_64_device(__ptr, __tmp); break; + case __ATOMIC_RELAXED: __cuda_load_relaxed_64_device(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_volatile_64_device(__ptr, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELAXED: __cuda_load_volatile_64_device(__ptr, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 8); } template static inline __device__ void __cuda_store_relaxed_32_device(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.relaxed.gpu.b32 [%0], %1;" :: "l"(__ptr),"r"(__src) : "memory"); } @@ -813,18 +978,24 @@ template::type __device__ void __atomic_store_cuda(volatile _Type *__ptr, _Type *__val, int __memorder, __thread_scope_device_tag) { uint32_t __tmp = 0; memcpy(&__tmp, __val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_RELEASE: __cuda_store_release_32_device(__ptr, __tmp); break; - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_RELAXED: __cuda_store_relaxed_32_device(__ptr, __tmp); break; -#else - case __ATOMIC_RELEASE: - case __ATOMIC_SEQ_CST: __cuda_membar_device(); - case __ATOMIC_RELAXED: __cuda_store_volatile_32_device(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_RELEASE: __cuda_store_release_32_device(__ptr, __tmp); break; + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_RELAXED: __cuda_store_relaxed_32_device(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_RELEASE: + case __ATOMIC_SEQ_CST: __cuda_membar_device(); + case __ATOMIC_RELAXED: __cuda_store_volatile_32_device(__ptr, __tmp); break; + default: assert(0); + } + ) + ) } template static inline __device__ void __cuda_store_relaxed_64_device(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.relaxed.gpu.b64 [%0], %1;" :: "l"(__ptr),"l"(__src) : "memory"); } template static inline __device__ void __cuda_store_release_64_device(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.release.gpu.b64 [%0], %1;" :: "l"(__ptr),"l"(__src) : "memory"); } @@ -833,48 +1004,60 @@ template::type __device__ void __atomic_store_cuda(volatile _Type *__ptr, _Type *__val, int __memorder, __thread_scope_device_tag) { uint64_t __tmp = 0; memcpy(&__tmp, __val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_RELEASE: __cuda_store_release_64_device(__ptr, __tmp); break; - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_RELAXED: __cuda_store_relaxed_64_device(__ptr, __tmp); break; -#else - case __ATOMIC_RELEASE: - case __ATOMIC_SEQ_CST: __cuda_membar_device(); - case __ATOMIC_RELAXED: __cuda_store_volatile_64_device(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } -} -template static inline __device__ void __cuda_compare_exchange_acq_rel_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_acquire_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_relaxed_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_release_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_volatile_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_RELEASE: __cuda_store_release_64_device(__ptr, __tmp); break; + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_RELAXED: __cuda_store_relaxed_64_device(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_RELEASE: + case __ATOMIC_SEQ_CST: __cuda_membar_device(); + case __ATOMIC_RELAXED: __cuda_store_volatile_64_device(__ptr, __tmp); break; + default: assert(0); + } + ) + ) +} +template static inline __device__ void __cuda_compare_exchange_acq_rel_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acquire_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_relaxed_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_release_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_volatile_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } template::type = 0> __device__ bool __atomic_compare_exchange_cuda(volatile _Type *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder, __thread_scope_device_tag) { uint32_t __tmp = 0, __old = 0, __old_tmp; memcpy(&__tmp, __desired, 4); memcpy(&__old, __expected, 4); __old_tmp = __old; - switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_32_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_32_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_compare_exchange_release_32_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_32_device(__ptr, __old, __old_tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_32_device(__ptr, __old, __old_tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_compare_exchange_volatile_32_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_32_device(__ptr, __old, __old_tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_32_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_32_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_compare_exchange_release_32_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_32_device(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_32_device(__ptr, __old, __old_tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_compare_exchange_volatile_32_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_32_device(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ) + ) bool const __ret = __old == __old_tmp; memcpy(__expected, &__old, 4); return __ret; @@ -888,24 +1071,30 @@ template::type __device__ void __atomic_exchange_cuda(volatile _Type *__ptr, _Type *__val, _Type *__ret, int __memorder, __thread_scope_device_tag) { uint32_t __tmp = 0; memcpy(&__tmp, __val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_exchange_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_exchange_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_exchange_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_exchange_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 4); } template static inline __device__ void __cuda_fetch_add_acq_rel_32_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __op) { asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr),"r"(__op) : "memory"); } @@ -918,24 +1107,30 @@ __device__ _Type __atomic_fetch_add_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -949,24 +1144,30 @@ __device__ _Type __atomic_fetch_and_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_and_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_and_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_and_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_and_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -980,24 +1181,30 @@ __device__ _Type __atomic_fetch_max_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_max_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_max_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_max_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_max_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1011,24 +1218,30 @@ __device__ _Type __atomic_fetch_min_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_min_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_min_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_min_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_min_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1042,24 +1255,30 @@ __device__ _Type __atomic_fetch_or_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_or_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_or_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_or_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_or_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1078,24 +1297,30 @@ __device__ _Type __atomic_fetch_sub_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_sub_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_sub_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_sub_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_sub_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1109,56 +1334,68 @@ __device__ _Type __atomic_fetch_xor_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_xor_release_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_32_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_xor_volatile_32_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_32_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_xor_release_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_32_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_xor_volatile_32_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_32_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } -template static inline __device__ void __cuda_compare_exchange_acq_rel_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_acquire_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_relaxed_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_release_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_volatile_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acq_rel_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acquire_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_relaxed_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_release_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_volatile_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } template::type = 0> __device__ bool __atomic_compare_exchange_cuda(volatile _Type *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder, __thread_scope_device_tag) { uint64_t __tmp = 0, __old = 0, __old_tmp; memcpy(&__tmp, __desired, 8); memcpy(&__old, __expected, 8); __old_tmp = __old; - switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_64_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_64_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_compare_exchange_release_64_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_64_device(__ptr, __old, __old_tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_64_device(__ptr, __old, __old_tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_compare_exchange_volatile_64_device(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_64_device(__ptr, __old, __old_tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_64_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_64_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_compare_exchange_release_64_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_64_device(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_64_device(__ptr, __old, __old_tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_compare_exchange_volatile_64_device(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_64_device(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ) + ) bool const __ret = __old == __old_tmp; memcpy(__expected, &__old, 8); return __ret; @@ -1172,24 +1409,30 @@ template::type __device__ void __atomic_exchange_cuda(volatile _Type *__ptr, _Type *__val, _Type *__ret, int __memorder, __thread_scope_device_tag) { uint64_t __tmp = 0; memcpy(&__tmp, __val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_exchange_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_exchange_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_exchange_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_exchange_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 8); } template static inline __device__ void __cuda_fetch_add_acq_rel_64_device(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __op) { asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr),"l"(__op) : "memory"); } @@ -1202,24 +1445,30 @@ __device__ _Type __atomic_fetch_add_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1233,24 +1482,30 @@ __device__ _Type __atomic_fetch_and_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_and_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_and_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_and_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_and_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1264,24 +1519,30 @@ __device__ _Type __atomic_fetch_max_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_max_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_max_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_max_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_max_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1295,24 +1556,30 @@ __device__ _Type __atomic_fetch_min_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_min_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_min_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_min_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_min_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1326,24 +1593,30 @@ __device__ _Type __atomic_fetch_or_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_or_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_or_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_or_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_or_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1362,24 +1635,30 @@ __device__ _Type __atomic_fetch_sub_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_sub_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_sub_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_sub_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_sub_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1393,24 +1672,30 @@ __device__ _Type __atomic_fetch_xor_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_xor_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_xor_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_xor_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_xor_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1420,24 +1705,29 @@ __device__ _Type* __atomic_fetch_add_cuda(_Type *volatile *__ptr, ptrdiff_t __va uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); __tmp *= sizeof(_Type); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_device(__ptr, __tmp, __tmp); break; + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1448,24 +1738,29 @@ __device__ _Type* __atomic_fetch_sub_cuda(_Type *volatile *__ptr, ptrdiff_t __va memcpy(&__tmp, &__val, 8); __tmp = -__tmp; __tmp *= sizeof(_Type); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_device(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_device(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; - case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_device(__ptr, __tmp, __tmp); break; + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_device(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); __cuda_membar_device(); break; + case __ATOMIC_RELEASE: __cuda_membar_device(); __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_device(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1473,23 +1768,30 @@ static inline __device__ void __cuda_membar_system() { asm volatile("membar.sys; static inline __device__ void __cuda_fence_acq_rel_system() { asm volatile("fence.acq_rel.sys;":::"memory"); } static inline __device__ void __cuda_fence_sc_system() { asm volatile("fence.sc.sys;":::"memory"); } static inline __device__ void __atomic_thread_fence_cuda(int __memorder, __thread_scope_system_tag) { - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); break; - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - case __ATOMIC_ACQ_REL: - case __ATOMIC_RELEASE: __cuda_fence_acq_rel_system(); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: - case __ATOMIC_ACQ_REL: - case __ATOMIC_RELEASE: __cuda_membar_system(); break; -#endif // __CUDA_ARCH__ >= 700 - case __ATOMIC_RELAXED: break; - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); break; + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: + case __ATOMIC_ACQ_REL: + case __ATOMIC_RELEASE: __cuda_fence_acq_rel_system(); break; + case __ATOMIC_RELAXED: break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: + case __ATOMIC_ACQ_REL: + case __ATOMIC_RELEASE: __cuda_membar_system(); break; + case __ATOMIC_RELAXED: break; + default: assert(0); + } + ) + ) } template static inline __device__ void __cuda_load_acquire_32_system(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline __device__ void __cuda_load_relaxed_32_system(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } @@ -1497,20 +1799,26 @@ template static inline __device__ void __cuda_load template::type = 0> __device__ void __atomic_load_cuda(const volatile _Type *__ptr, _Type *__ret, int __memorder, __thread_scope_system_tag) { uint32_t __tmp = 0; - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_acquire_32_system(__ptr, __tmp); break; - case __ATOMIC_RELAXED: __cuda_load_relaxed_32_system(__ptr, __tmp); break; -#else - case __ATOMIC_SEQ_CST: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_volatile_32_system(__ptr, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELAXED: __cuda_load_volatile_32_system(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_acquire_32_system(__ptr, __tmp); break; + case __ATOMIC_RELAXED: __cuda_load_relaxed_32_system(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_volatile_32_system(__ptr, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELAXED: __cuda_load_volatile_32_system(__ptr, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 4); } template static inline __device__ void __cuda_load_acquire_64_system(_CUDA_A __ptr, _CUDA_B& __dst) {asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } @@ -1519,20 +1827,26 @@ template static inline __device__ void __cuda_load template::type = 0> __device__ void __atomic_load_cuda(const volatile _Type *__ptr, _Type *__ret, int __memorder, __thread_scope_system_tag) { uint64_t __tmp = 0; - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_acquire_64_system(__ptr, __tmp); break; - case __ATOMIC_RELAXED: __cuda_load_relaxed_64_system(__ptr, __tmp); break; -#else - case __ATOMIC_SEQ_CST: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_load_volatile_64_system(__ptr, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELAXED: __cuda_load_volatile_64_system(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_acquire_64_system(__ptr, __tmp); break; + case __ATOMIC_RELAXED: __cuda_load_relaxed_64_system(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_load_volatile_64_system(__ptr, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELAXED: __cuda_load_volatile_64_system(__ptr, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 8); } template static inline __device__ void __cuda_store_relaxed_32_system(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.relaxed.sys.b32 [%0], %1;" :: "l"(__ptr),"r"(__src) : "memory"); } @@ -1542,18 +1856,24 @@ template::type __device__ void __atomic_store_cuda(volatile _Type *__ptr, _Type *__val, int __memorder, __thread_scope_system_tag) { uint32_t __tmp = 0; memcpy(&__tmp, __val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_RELEASE: __cuda_store_release_32_system(__ptr, __tmp); break; - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_RELAXED: __cuda_store_relaxed_32_system(__ptr, __tmp); break; -#else - case __ATOMIC_RELEASE: - case __ATOMIC_SEQ_CST: __cuda_membar_system(); - case __ATOMIC_RELAXED: __cuda_store_volatile_32_system(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_RELEASE: __cuda_store_release_32_system(__ptr, __tmp); break; + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_RELAXED: __cuda_store_relaxed_32_system(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_RELEASE: + case __ATOMIC_SEQ_CST: __cuda_membar_system(); + case __ATOMIC_RELAXED: __cuda_store_volatile_32_system(__ptr, __tmp); break; + default: assert(0); + } + ) + ) } template static inline __device__ void __cuda_store_relaxed_64_system(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.relaxed.sys.b64 [%0], %1;" :: "l"(__ptr),"l"(__src) : "memory"); } template static inline __device__ void __cuda_store_release_64_system(_CUDA_A __ptr, _CUDA_B __src) { asm volatile("st.release.sys.b64 [%0], %1;" :: "l"(__ptr),"l"(__src) : "memory"); } @@ -1562,48 +1882,60 @@ template::type __device__ void __atomic_store_cuda(volatile _Type *__ptr, _Type *__val, int __memorder, __thread_scope_system_tag) { uint64_t __tmp = 0; memcpy(&__tmp, __val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_RELEASE: __cuda_store_release_64_system(__ptr, __tmp); break; - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_RELAXED: __cuda_store_relaxed_64_system(__ptr, __tmp); break; -#else - case __ATOMIC_RELEASE: - case __ATOMIC_SEQ_CST: __cuda_membar_system(); - case __ATOMIC_RELAXED: __cuda_store_volatile_64_system(__ptr, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } -} -template static inline __device__ void __cuda_compare_exchange_acq_rel_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_acquire_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_relaxed_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_release_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_volatile_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(_cmp),"r"(__op) : "memory"); } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_RELEASE: __cuda_store_release_64_system(__ptr, __tmp); break; + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_RELAXED: __cuda_store_relaxed_64_system(__ptr, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_RELEASE: + case __ATOMIC_SEQ_CST: __cuda_membar_system(); + case __ATOMIC_RELAXED: __cuda_store_volatile_64_system(__ptr, __tmp); break; + default: assert(0); + } + ) + ) +} +template static inline __device__ void __cuda_compare_exchange_acq_rel_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acquire_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_relaxed_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_release_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_volatile_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr),"r"(__cmp),"r"(__op) : "memory"); } template::type = 0> __device__ bool __atomic_compare_exchange_cuda(volatile _Type *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder, __thread_scope_system_tag) { uint32_t __tmp = 0, __old = 0, __old_tmp; memcpy(&__tmp, __desired, 4); memcpy(&__old, __expected, 4); __old_tmp = __old; - switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_32_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_32_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_compare_exchange_release_32_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_32_system(__ptr, __old, __old_tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_32_system(__ptr, __old, __old_tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_compare_exchange_volatile_32_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_32_system(__ptr, __old, __old_tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_32_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_32_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_compare_exchange_release_32_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_32_system(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_32_system(__ptr, __old, __old_tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_compare_exchange_volatile_32_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_32_system(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ) + ) bool const __ret = __old == __old_tmp; memcpy(__expected, &__old, 4); return __ret; @@ -1617,24 +1949,30 @@ template::type __device__ void __atomic_exchange_cuda(volatile _Type *__ptr, _Type *__val, _Type *__ret, int __memorder, __thread_scope_system_tag) { uint32_t __tmp = 0; memcpy(&__tmp, __val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_exchange_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_exchange_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_exchange_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_exchange_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 4); } template static inline __device__ void __cuda_fetch_add_acq_rel_32_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __op) { asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr),"r"(__op) : "memory"); } @@ -1647,24 +1985,30 @@ __device__ _Type __atomic_fetch_add_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1678,24 +2022,30 @@ __device__ _Type __atomic_fetch_and_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_and_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_and_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_and_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_and_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1709,24 +2059,30 @@ __device__ _Type __atomic_fetch_max_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_max_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_max_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_max_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_max_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1740,24 +2096,30 @@ __device__ _Type __atomic_fetch_min_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_min_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_min_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_min_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_min_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1771,24 +2133,30 @@ __device__ _Type __atomic_fetch_or_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_or_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_or_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_or_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_or_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1807,24 +2175,30 @@ __device__ _Type __atomic_fetch_sub_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_sub_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_sub_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_sub_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_sub_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } @@ -1838,56 +2212,68 @@ __device__ _Type __atomic_fetch_xor_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint32_t __tmp = 0; memcpy(&__tmp, &__val, 4); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_xor_release_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_32_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_xor_volatile_32_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_32_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_xor_release_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_32_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_xor_volatile_32_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_32_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 4); return __ret; } -template static inline __device__ void __cuda_compare_exchange_acq_rel_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_acquire_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_relaxed_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_release_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } -template static inline __device__ void __cuda_compare_exchange_volatile_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C _cmp, _CUDA_D __op) { asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(_cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acq_rel_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_acquire_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_relaxed_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_release_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } +template static inline __device__ void __cuda_compare_exchange_volatile_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __cmp, _CUDA_D __op) { asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr),"l"(__cmp),"l"(__op) : "memory"); } template::type = 0> __device__ bool __atomic_compare_exchange_cuda(volatile _Type *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder, __thread_scope_system_tag) { uint64_t __tmp = 0, __old = 0, __old_tmp; memcpy(&__tmp, __desired, 8); memcpy(&__old, __expected, 8); __old_tmp = __old; - switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_64_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_64_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_compare_exchange_release_64_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_64_system(__ptr, __old, __old_tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_64_system(__ptr, __old, __old_tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_compare_exchange_volatile_64_system(__ptr, __old, __old_tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_64_system(__ptr, __old, __old_tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_acquire_64_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_compare_exchange_acq_rel_64_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_compare_exchange_release_64_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_relaxed_64_system(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_compare_exchange_volatile_64_system(__ptr, __old, __old_tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_compare_exchange_volatile_64_system(__ptr, __old, __old_tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_compare_exchange_volatile_64_system(__ptr, __old, __old_tmp, __tmp); break; + default: assert(0); + } + ) + ) bool const __ret = __old == __old_tmp; memcpy(__expected, &__old, 8); return __ret; @@ -1901,24 +2287,30 @@ template::type __device__ void __atomic_exchange_cuda(volatile _Type *__ptr, _Type *__val, _Type *__ret, int __memorder, __thread_scope_system_tag) { uint64_t __tmp = 0; memcpy(&__tmp, __val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_exchange_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_exchange_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_exchange_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_exchange_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_exchange_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_exchange_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_exchange_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_exchange_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(__ret, &__tmp, 8); } template static inline __device__ void __cuda_fetch_add_acq_rel_64_system(_CUDA_A __ptr, _CUDA_B& __dst, _CUDA_C __op) { asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr),"l"(__op) : "memory"); } @@ -1931,24 +2323,30 @@ __device__ _Type __atomic_fetch_add_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1962,24 +2360,30 @@ __device__ _Type __atomic_fetch_and_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_and_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_and_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_and_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_and_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_and_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_and_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_and_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -1993,24 +2397,30 @@ __device__ _Type __atomic_fetch_max_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_max_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_max_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_max_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_max_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_max_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_max_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_max_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -2024,24 +2434,30 @@ __device__ _Type __atomic_fetch_min_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_min_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_min_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_min_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_min_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_min_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_min_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_min_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -2055,24 +2471,30 @@ __device__ _Type __atomic_fetch_or_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_or_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_or_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_or_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_or_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_or_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_or_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_or_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -2091,24 +2513,30 @@ __device__ _Type __atomic_fetch_sub_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_sub_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_sub_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_sub_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_sub_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_sub_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_sub_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_sub_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -2122,24 +2550,30 @@ __device__ _Type __atomic_fetch_xor_cuda(volatile _Type *__ptr, _Type __val, int _Type __ret; uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_xor_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_xor_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_xor_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_xor_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_relaxed_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_xor_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_xor_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_xor_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -2149,24 +2583,29 @@ __device__ _Type* __atomic_fetch_add_cuda(_Type *volatile *__ptr, ptrdiff_t __va uint64_t __tmp = 0; memcpy(&__tmp, &__val, 8); __tmp *= sizeof(_Type); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_system(__ptr, __tmp, __tmp); break; + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } @@ -2177,27 +2616,29 @@ __device__ _Type* __atomic_fetch_sub_cuda(_Type *volatile *__ptr, ptrdiff_t __va memcpy(&__tmp, &__val, 8); __tmp = -__tmp; __tmp *= sizeof(_Type); - switch (__memorder) { -#if __CUDA_ARCH__ >= 700 - case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_system(__ptr, __tmp, __tmp); break; -#else - case __ATOMIC_SEQ_CST: - case __ATOMIC_ACQ_REL: __cuda_membar_system(); - case __ATOMIC_CONSUME: - case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; - case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; - case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; -#endif // __CUDA_ARCH__ >= 700 - default: assert(0); - } + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: __cuda_fence_sc_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_acquire_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_ACQ_REL: __cuda_fetch_add_acq_rel_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELEASE: __cuda_fetch_add_release_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_relaxed_64_system(__ptr, __tmp, __tmp); break; + } + ), + NV_IS_DEVICE, ( + switch (__memorder) { + case __ATOMIC_SEQ_CST: + case __ATOMIC_ACQ_REL: __cuda_membar_system(); + case __ATOMIC_CONSUME: + case __ATOMIC_ACQUIRE: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); __cuda_membar_system(); break; + case __ATOMIC_RELEASE: __cuda_membar_system(); __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; + case __ATOMIC_RELAXED: __cuda_fetch_add_volatile_64_system(__ptr, __tmp, __tmp); break; + default: assert(0); + } + ) + ) memcpy(&__ret, &__tmp, 8); return __ret; } - -} -_LIBCUDACXX_END_NAMESPACE_CUDA diff --git a/libcxx/include/support/atomic/atomic_gcc.h b/libcxx/include/support/atomic/atomic_gcc.h new file mode 100644 index 0000000000..7f76e7e826 --- /dev/null +++ b/libcxx/include/support/atomic/atomic_gcc.h @@ -0,0 +1,16 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCUDACXX_ATOMIC_GCC_H +#define _LIBCUDACXX_ATOMIC_GCC_H + +#include "atomic_base.h" + +#endif // _LIBCUDACXX_ATOMIC_GCC_H \ No newline at end of file diff --git a/libcxx/include/support/win32/atomic_msvc.h b/libcxx/include/support/atomic/atomic_msvc.h similarity index 87% rename from libcxx/include/support/win32/atomic_msvc.h rename to libcxx/include/support/atomic/atomic_msvc.h index dc7d691446..9294a7fa3b 100644 --- a/libcxx/include/support/win32/atomic_msvc.h +++ b/libcxx/include/support/atomic/atomic_msvc.h @@ -12,8 +12,6 @@ #error "This file is only for CL.EXE's benefit" #endif -#include - #define _Compiler_barrier() _ReadWriteBarrier() #if defined(_M_ARM) || defined(_M_ARM64) @@ -27,7 +25,17 @@ #error Unsupported hardware #endif // hardware -_LIBCUDACXX_BEGIN_NAMESPACE_CUDA +inline int __stronger_order_msvc(int __a, int __b) { + int const __max = __a > __b ? __a : __b; + if(__max != __ATOMIC_RELEASE) + return __max; + static int const __xform[] = { + __ATOMIC_RELEASE, + __ATOMIC_ACQ_REL, + __ATOMIC_ACQ_REL, + __ATOMIC_RELEASE }; + return __xform[__a < __b ? __a : __b]; +} static inline void __atomic_signal_fence(int __memorder) { if (__memorder != __ATOMIC_RELAXED) @@ -39,12 +47,10 @@ static inline void __atomic_thread_fence(int __memorder) { _Memory_barrier(); } -namespace detail { - template - using _enable_if_sized_as = typename _CUDA_VSTD::enable_if::type; -} +template +using _enable_if_sized_as = typename enable_if::type; -template = 0> +template = 0> void __atomic_load_relaxed(const volatile _Type *__ptr, _Type *__ret) { #ifdef _LIBCUDACXX_MSVC_HAS_NO_ISO_INTRIN __int8 __tmp = *(const volatile __int8 *)__ptr; @@ -53,7 +59,7 @@ void __atomic_load_relaxed(const volatile _Type *__ptr, _Type *__ret) { #endif *__ret = reinterpret_cast<_Type&>(__tmp); } -template = 0> +template = 0> void __atomic_load_relaxed(const volatile _Type *__ptr, _Type *__ret) { #ifdef _LIBCUDACXX_MSVC_HAS_NO_ISO_INTRIN __int16 __tmp = *(const volatile __int16 *)__ptr; @@ -62,7 +68,7 @@ void __atomic_load_relaxed(const volatile _Type *__ptr, _Type *__ret) { #endif *__ret = reinterpret_cast<_Type&>(__tmp); } -template = 0> +template = 0> void __atomic_load_relaxed(const volatile _Type *__ptr, _Type *__ret) { #ifdef _LIBCUDACXX_MSVC_HAS_NO_ISO_INTRIN __int32 __tmp = *(const volatile __int32 *)__ptr; @@ -71,7 +77,7 @@ void __atomic_load_relaxed(const volatile _Type *__ptr, _Type *__ret) { #endif *__ret = reinterpret_cast<_Type&>(__tmp); } -template = 0> +template = 0> void __atomic_load_relaxed(const volatile _Type *__ptr, _Type *__ret) { #ifdef _LIBCUDACXX_MSVC_HAS_NO_ISO_INTRIN __int64 __tmp = *(const volatile __int64 *)__ptr; @@ -92,7 +98,7 @@ void __atomic_load(const volatile _Type *__ptr, _Type *__ret, int __memorder) { } } -template = 0> +template = 0> void __atomic_store_relaxed(volatile _Type *__ptr, _Type *__val) { auto __t = reinterpret_cast<__int8 *>(__val); auto __d = reinterpret_cast(__ptr); @@ -102,7 +108,7 @@ void __atomic_store_relaxed(volatile _Type *__ptr, _Type *__val) { __iso_volatile_store8(__d, *__t); #endif } -template = 0> +template = 0> void __atomic_store_relaxed(volatile _Type *__ptr, _Type *__val) { auto __t = reinterpret_cast<__int16 *>(__val); auto __d = reinterpret_cast(__ptr); @@ -112,7 +118,7 @@ void __atomic_store_relaxed(volatile _Type *__ptr, _Type *__val) { __iso_volatile_store16(__d, *__t); #endif } -template = 0> +template = 0> void __atomic_store_relaxed(volatile _Type *__ptr, _Type *__val) { auto __t = reinterpret_cast<__int32 *>(__val); auto __d = reinterpret_cast(__ptr); @@ -123,7 +129,7 @@ void __atomic_store_relaxed(volatile _Type *__ptr, _Type *__val) { __iso_volatile_store32(__d, *__t); #endif } -template = 0> +template = 0> void __atomic_store_relaxed(volatile _Type *__ptr, _Type *__val) { auto __t = reinterpret_cast<__int64 *>(__val); auto __d = reinterpret_cast(__ptr); @@ -144,7 +150,7 @@ void __atomic_store(volatile _Type *__ptr, _Type *__val, int __memorder) { } } -template = 0> +template = 0> bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__expected, const _Type *__desired) { auto __tmp_desired = reinterpret_cast(*__desired); auto __tmp_expected = reinterpret_cast(*__expected); @@ -154,7 +160,7 @@ bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__exp *__expected = reinterpret_cast(__old); return false; } -template = 0> +template = 0> bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__expected, const _Type *__desired) { auto __tmp_desired = reinterpret_cast(*__desired); auto __tmp_expected = reinterpret_cast(*__expected); @@ -164,7 +170,7 @@ bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__exp *__expected = reinterpret_cast(__old); return false; } -template = 0> +template = 0> bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__expected, const _Type *__desired) { auto __tmp_desired = reinterpret_cast(*__desired); auto __tmp_expected = reinterpret_cast(*__expected); @@ -174,7 +180,7 @@ bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__exp *__expected = reinterpret_cast(__old); return false; } -template = 0> +template = 0> bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__expected, const _Type *__desired) { auto __tmp_desired = reinterpret_cast(*__desired); auto __tmp_expected = reinterpret_cast<__int64&>(*__expected); @@ -187,7 +193,7 @@ bool __atomic_compare_exchange_relaxed(const volatile _Type *__ptr, _Type *__exp template bool __atomic_compare_exchange(_Type volatile *__ptr, _Type *__expected, const _Type *__desired, bool, int __success_memorder, int __failure_memorder) { bool success = false; - switch (detail::__stronger_order_cuda(__success_memorder, __failure_memorder)) { + switch (__stronger_order_msvc(__success_memorder, __failure_memorder)) { case __ATOMIC_RELEASE: _Compiler_or_memory_barrier(); success = __atomic_compare_exchange_relaxed(__ptr, __expected, __desired); break; case __ATOMIC_ACQ_REL: _Compiler_or_memory_barrier(); _LIBCUDACXX_FALLTHROUGH(); case __ATOMIC_CONSUME: @@ -199,22 +205,22 @@ bool __atomic_compare_exchange(_Type volatile *__ptr, _Type *__expected, const _ return success; } -template = 0> +template = 0> void __atomic_exchange_relaxed(const volatile _Type *__ptr, const _Type *__val, _Type *__ret) { auto const __old = _InterlockedExchange8((volatile char *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_exchange_relaxed(const volatile _Type *__ptr, const _Type *__val, _Type *__ret) { auto const __old = _InterlockedExchange16((volatile short *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_exchange_relaxed(const volatile _Type *__ptr, const _Type *__val, _Type *__ret) { auto const __old = _InterlockedExchange((volatile long *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_exchange_relaxed(const volatile _Type *__ptr, const _Type *__val, _Type *__ret) { auto const __old = _InterlockedExchange64((volatile __int64 *)__ptr, reinterpret_cast<__int64 const&>(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); @@ -232,22 +238,22 @@ void __atomic_exchange(_Type volatile *__ptr, const _Type *__val, _Type *__ret, } } -template = 0> +template = 0> void __atomic_fetch_add_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedExchangeAdd8((volatile char *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_add_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedExchangeAdd16((volatile short *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_add_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedExchangeAdd((volatile long *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_add_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedExchangeAdd64((volatile __int64 *)__ptr, reinterpret_cast<__int64 const&>(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); @@ -274,22 +280,22 @@ _Type __atomic_fetch_sub(_Type volatile *__ptr, _Delta __val, int __memorder) { } -template = 0> +template = 0> void __atomic_fetch_and_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedAnd8((volatile char *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_and_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedAnd16((volatile short *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_and_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedAnd((volatile long *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_and_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedAnd64((volatile __int64 *)__ptr, reinterpret_cast<__int64 const&>(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); @@ -311,22 +317,22 @@ _Type __atomic_fetch_and(_Type volatile *__ptr, _Delta __val, int __memorder) { return *__dest; } -template = 0> +template = 0> void __atomic_fetch_xor_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedXor8((volatile char *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_xor_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedXor16((volatile short *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_xor_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedXor((volatile long *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_xor_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedXor64((volatile __int64 *)__ptr, reinterpret_cast<__int64 const&>(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); @@ -348,22 +354,22 @@ _Type __atomic_fetch_xor(_Type volatile *__ptr, _Delta __val, int __memorder) { return *__dest; } -template = 0> +template = 0> void __atomic_fetch_or_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedOr8((volatile char *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_or_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedOr16((volatile short *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_or_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedOr((volatile long *)__ptr, reinterpret_cast(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); } -template = 0> +template = 0> void __atomic_fetch_or_relaxed(const volatile _Type *__ptr, const _Delta *__val, _Type *__ret) { auto const __old = _InterlockedOr64((volatile __int64 *)__ptr, reinterpret_cast<__int64 const&>(*__val)); *__ret = reinterpret_cast<_Type const&>(__old); @@ -414,7 +420,7 @@ _Type __atomic_exchange_n(_Type volatile *__ptr, _Type __val, int __memorder) { } template -_Type __host__ __atomic_fetch_max(_Type volatile *__ptr, _Delta __val, int __memorder) { +_Type __atomic_fetch_max(_Type volatile *__ptr, _Delta __val, int __memorder) { _Type __expected = __atomic_load_n(__ptr, __ATOMIC_RELAXED); _Type __desired = __expected < __val ? __expected : __val; while(__desired == __val && @@ -425,7 +431,7 @@ _Type __host__ __atomic_fetch_max(_Type volatile *__ptr, _Delta __val, int __mem } template -_Type __host__ __atomic_fetch_min(_Type volatile *__ptr, _Delta __val, int __memorder) { +_Type __atomic_fetch_min(_Type volatile *__ptr, _Delta __val, int __memorder) { _Type __expected = __atomic_load_n(__ptr, __ATOMIC_RELAXED); _Type __desired = __expected < __val ? __expected : __val; while(__desired != __val && @@ -435,4 +441,4 @@ _Type __host__ __atomic_fetch_min(_Type volatile *__ptr, _Delta __val, int __mem return __expected; } -_LIBCUDACXX_END_NAMESPACE_CUDA +#include "atomic_base.h"