diff --git a/libcudacxx/include/cuda/std/__simd/bit.h b/libcudacxx/include/cuda/std/__simd/bit.h index 5f76c8423f1..f23bf5ec0da 100644 --- a/libcudacxx/include/cuda/std/__simd/bit.h +++ b/libcudacxx/include/cuda/std/__simd/bit.h @@ -21,354 +21,6 @@ # pragma system_header #endif // no system header -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD - -//---------------------------------------------------------------------------------------------------------------------- -// [simd.bit] element-wise helpers - -template -struct __simd_byteswap_generator -{ - using __result_t = typename _Vp::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return ::cuda::std::byteswap(__v_[_Idx::value]); - } -}; - -template -struct __simd_bit_ceil_generator -{ - using __result_t = typename _Vp::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return ::cuda::std::bit_ceil(__v_[_Idx::value]); - } -}; - -template -struct __simd_bit_floor_generator -{ - using __result_t = typename _Vp::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return ::cuda::std::bit_floor(__v_[_Idx::value]); - } -}; - -template -struct __simd_has_single_bit_generator -{ - using __result_t = bool; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return ::cuda::std::has_single_bit(__v_[_Idx::value]); - } -}; - -template -struct __simd_rotl_generator -{ - using __result_t = typename _Vp0::value_type; - const _Vp0& __v0_; - const _Vp1& __v1_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - _CCCL_ASSERT(::cuda::std::in_range(__v1_[_Idx::value]), "rotl: count is out of range"); - return ::cuda::std::rotl(__v0_[_Idx::value], static_cast(__v1_[_Idx::value])); - } -}; - -template -struct __simd_rotr_generator -{ - using __result_t = typename _Vp0::value_type; - const _Vp0& __v0_; - const _Vp1& __v1_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - _CCCL_ASSERT(::cuda::std::in_range(__v1_[_Idx::value]), "rotr: count is out of range"); - return ::cuda::std::rotr(__v0_[_Idx::value], static_cast(__v1_[_Idx::value])); - } -}; - -template -struct __simd_rotl_scalar_generator -{ - using __result_t = typename _Vp::value_type; - const _Vp& __v_; - const int __s_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return ::cuda::std::rotl(__v_[_Idx::value], __s_); - } -}; - -template -struct __simd_rotr_scalar_generator -{ - using __result_t = typename _Vp::value_type; - const _Vp& __v_; - const int __s_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return ::cuda::std::rotr(__v_[_Idx::value], __s_); - } -}; - -template -struct __simd_bit_width_generator -{ - using __result_t = typename _Result::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return static_cast<__result_t>(::cuda::std::bit_width(__v_[_Idx::value])); - } -}; - -template -struct __simd_countl_zero_generator -{ - using __result_t = typename _Result::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return static_cast<__result_t>(::cuda::std::countl_zero(__v_[_Idx::value])); - } -}; - -template -struct __simd_countl_one_generator -{ - using __result_t = typename _Result::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return static_cast<__result_t>(::cuda::std::countl_one(__v_[_Idx::value])); - } -}; - -template -struct __simd_countr_zero_generator -{ - using __result_t = typename _Result::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return static_cast<__result_t>(::cuda::std::countr_zero(__v_[_Idx::value])); - } -}; - -template -struct __simd_countr_one_generator -{ - using __result_t = typename _Result::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return static_cast<__result_t>(::cuda::std::countr_one(__v_[_Idx::value])); - } -}; - -template -struct __simd_popcount_generator -{ - using __result_t = typename _Result::value_type; - const _Vp& __v_; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept - { - return static_cast<__result_t>(::cuda::std::popcount(__v_[_Idx::value])); - } -}; - -//---------------------------------------------------------------------------------------------------------------------- -// [simd.bit], bit manipulation - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(is_integral_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto byteswap(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - return __vec_t{__simd_byteswap_generator<__vec_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_ceil(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - return __vec_t{__simd_bit_ceil_generator<__vec_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_floor(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - return __vec_t{__simd_bit_floor_generator<__vec_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto has_single_bit(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - using __result_t = typename __vec_t::mask_type; - return __result_t{__simd_has_single_bit_generator<__vec_t>{__v}}; -} - -template -inline constexpr bool __simd_is_valid_rotate_v = - __cccl_is_unsigned_integer_v<_Tp0> // - && is_integral_v<_Tp1> // - && (__simd_size_v<_Tp0, _Abi0> == __simd_size_v<_Tp1, _Abi1>) // - &&(sizeof(_Tp0) == sizeof(_Tp1)); - -_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1) -_CCCL_REQUIRES(__simd_is_valid_rotate_v<_Tp0, _Abi0, _Tp1, _Abi1>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto -rotl(const basic_vec<_Tp0, _Abi0>& __v0, const basic_vec<_Tp1, _Abi1>& __v1) noexcept -{ - using __vec0_t = basic_vec<_Tp0, _Abi0>; - using __vec1_t = basic_vec<_Tp1, _Abi1>; - return __vec0_t{__simd_rotl_generator<__vec0_t, __vec1_t>{__v0, __v1}}; -} - -_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1) -_CCCL_REQUIRES(__simd_is_valid_rotate_v<_Tp0, _Abi0, _Tp1, _Abi1>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto -rotr(const basic_vec<_Tp0, _Abi0>& __v0, const basic_vec<_Tp1, _Abi1>& __v1) noexcept -{ - using __vec0_t = basic_vec<_Tp0, _Abi0>; - using __vec1_t = basic_vec<_Tp1, _Abi1>; - return __vec0_t{__simd_rotr_generator<__vec0_t, __vec1_t>{__v0, __v1}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rotl(const basic_vec<_Tp, _Abi>& __v, const int __s) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - return __vec_t{__simd_rotl_scalar_generator<__vec_t>{__v, __s}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rotr(const basic_vec<_Tp, _Abi>& __v, const int __s) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - return __vec_t{__simd_rotr_scalar_generator<__vec_t>{__v, __s}}; -} - -template -using __simd_bit_count_result_t = rebind_t, _Vp>; - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_width(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - using __result_t = __simd_bit_count_result_t<_Tp, __vec_t>; - return __result_t{__simd_bit_width_generator<__vec_t, __result_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countl_zero(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - using __result_t = __simd_bit_count_result_t<_Tp, __vec_t>; - return __result_t{__simd_countl_zero_generator<__vec_t, __result_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countl_one(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - using __result_t = __simd_bit_count_result_t<_Tp, __vec_t>; - return __result_t{__simd_countl_one_generator<__vec_t, __result_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countr_zero(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - using __result_t = __simd_bit_count_result_t<_Tp, __vec_t>; - return __result_t{__simd_countr_zero_generator<__vec_t, __result_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countr_one(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - using __result_t = __simd_bit_count_result_t<_Tp, __vec_t>; - return __result_t{__simd_countr_one_generator<__vec_t, __result_t>{__v}}; -} - -_CCCL_TEMPLATE(typename _Tp, typename _Abi) -_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) -[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto popcount(const basic_vec<_Tp, _Abi>& __v) noexcept -{ - using __vec_t = basic_vec<_Tp, _Abi>; - using __result_t = __simd_bit_count_result_t<_Tp, __vec_t>; - return __result_t{__simd_popcount_generator<__vec_t, __result_t>{__v}}; -} - -_CCCL_END_NAMESPACE_CUDA_STD_SIMD - -#include +#include #endif // _CUDA_STD___SIMD_BIT_H diff --git a/libcudacxx/include/cuda/std/__simd/bit/scalar.h b/libcudacxx/include/cuda/std/__simd/bit/scalar.h new file mode 100644 index 00000000000..4eac37d29e6 --- /dev/null +++ b/libcudacxx/include/cuda/std/__simd/bit/scalar.h @@ -0,0 +1,498 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++ in the CUDA C++ Core Libraries, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___SIMD_BIT_SCALAR_H +#define _CUDA_STD___SIMD_BIT_SCALAR_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD_SIMD + +//---------------------------------------------------------------------------------------------------------------------- +// [simd.bit] element-wise helpers + +template +struct __simd_byteswap_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::byteswap(__v_[_Idx::value]); + } +}; + +template +struct __simd_bit_reverse_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::bit_reverse(__v_[_Idx::value]); + } +}; + +template +struct __simd_bit_ceil_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::bit_ceil(__v_[_Idx::value]); + } +}; + +template +struct __simd_bit_floor_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::bit_floor(__v_[_Idx::value]); + } +}; + +template +struct __simd_has_single_bit_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = bool; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::has_single_bit(__v_[_Idx::value]); + } +}; + +template +struct __simd_shl_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type; + const _Vp0& __v0_; + const _Vp1& __v1_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::shl(__v0_[_Idx::value], __v1_[_Idx::value]); + } +}; + +template +struct __simd_shr_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type; + const _Vp0& __v0_; + const _Vp1& __v1_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::shr(__v0_[_Idx::value], __v1_[_Idx::value]); + } +}; + +template +struct __simd_shl_scalar_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + const _Sp __s_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::shl(__v_[_Idx::value], __s_); + } +}; + +template +struct __simd_shr_scalar_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + const _Sp __s_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::shr(__v_[_Idx::value], __s_); + } +}; + +template +struct __simd_rotl_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type; + const _Vp0& __v0_; + const _Vp1& __v1_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + _CCCL_ASSERT(::cuda::std::in_range(__v1_[_Idx::value]), "rotl: count is out of range"); + return ::cuda::std::rotl(__v0_[_Idx::value], static_cast(__v1_[_Idx::value])); + } +}; + +template +struct __simd_rotr_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp0::value_type; + const _Vp0& __v0_; + const _Vp1& __v1_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + _CCCL_ASSERT(::cuda::std::in_range(__v1_[_Idx::value]), "rotr: count is out of range"); + return ::cuda::std::rotr(__v0_[_Idx::value], static_cast(__v1_[_Idx::value])); + } +}; + +template +struct __simd_rotl_scalar_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + const int __s_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::rotl(__v_[_Idx::value], __s_); + } +}; + +template +struct __simd_rotr_scalar_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Vp::value_type; + const _Vp& __v_; + const int __s_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return ::cuda::std::rotr(__v_[_Idx::value], __s_); + } +}; + +template +struct __simd_bit_width_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return static_cast<__result_t>(::cuda::std::bit_width(__v_[_Idx::value])); + } +}; + +template +struct __simd_countl_zero_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return static_cast<__result_t>(::cuda::std::countl_zero(__v_[_Idx::value])); + } +}; + +template +struct __simd_countl_one_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return static_cast<__result_t>(::cuda::std::countl_one(__v_[_Idx::value])); + } +}; + +template +struct __simd_countr_zero_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return static_cast<__result_t>(::cuda::std::countr_zero(__v_[_Idx::value])); + } +}; + +template +struct __simd_countr_one_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return static_cast<__result_t>(::cuda::std::countr_one(__v_[_Idx::value])); + } +}; + +template +struct __simd_popcount_generator +{ + using __result_t _CCCL_NODEBUG_ALIAS = typename _Result::value_type; + const _Vp& __v_; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr __result_t operator()(_Idx) const noexcept + { + return static_cast<__result_t>(::cuda::std::popcount(__v_[_Idx::value])); + } +}; + +//---------------------------------------------------------------------------------------------------------------------- +// [simd.bit], bit manipulation + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(is_integral_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto byteswap(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_byteswap_generator<__vec_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_reverse(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_bit_reverse_generator<__vec_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_ceil(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_bit_ceil_generator<__vec_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_floor(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_bit_floor_generator<__vec_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto has_single_bit(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + using __result_t _CCCL_NODEBUG_ALIAS = typename __vec_t::mask_type; + return __result_t{__simd_has_single_bit_generator<__vec_t>{__v}}; +} + +template +inline constexpr bool __simd_is_valid_rotate_v = + (sizeof(_Tp0) == sizeof(_Tp1)) // + && __cccl_is_unsigned_integer_v<_Tp0> // + && is_integral_v<_Tp1> // + && (__simd_size_v<_Tp0, _Abi0> == __simd_size_v<_Tp1, _Abi1>); + +template +inline constexpr bool __simd_is_valid_shift_v = + (sizeof(_Tp0) == sizeof(_Tp1)) // + && __cccl_is_integer_v<_Tp0> // + && __cccl_is_integer_v<_Tp1> // + && (__simd_size_v<_Tp0, _Abi0> == __simd_size_v<_Tp1, _Abi1>); + +_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1) +_CCCL_REQUIRES(__simd_is_valid_shift_v<_Tp0, _Abi0, _Tp1, _Abi1>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +shl(const basic_vec<_Tp0, _Abi0>& __v, const basic_vec<_Tp1, _Abi1>& __s) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp0, _Abi0>; + using __shift_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>; + return __vec_t{__simd_shl_generator<__vec_t, __shift_t>{__v, __s}}; +} + +_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1) +_CCCL_REQUIRES(__simd_is_valid_shift_v<_Tp0, _Abi0, _Tp1, _Abi1>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +shr(const basic_vec<_Tp0, _Abi0>& __v, const basic_vec<_Tp1, _Abi1>& __s) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp0, _Abi0>; + using __shift_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>; + return __vec_t{__simd_shr_generator<__vec_t, __shift_t>{__v, __s}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Sp) +_CCCL_REQUIRES(__cccl_is_integer_v<_Tp> _CCCL_AND __cccl_is_integer_v<_Sp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto shl(const basic_vec<_Tp, _Abi>& __v, const _Sp __s) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_shl_scalar_generator<__vec_t, _Sp>{__v, __s}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi, typename _Sp) +_CCCL_REQUIRES(__cccl_is_integer_v<_Tp> _CCCL_AND __cccl_is_integer_v<_Sp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto shr(const basic_vec<_Tp, _Abi>& __v, const _Sp __s) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_shr_scalar_generator<__vec_t, _Sp>{__v, __s}}; +} + +_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1) +_CCCL_REQUIRES(__simd_is_valid_rotate_v<_Tp0, _Abi0, _Tp1, _Abi1>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +rotl(const basic_vec<_Tp0, _Abi0>& __v0, const basic_vec<_Tp1, _Abi1>& __v1) noexcept +{ + using __vec0_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp0, _Abi0>; + using __vec1_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>; + return __vec0_t{__simd_rotl_generator<__vec0_t, __vec1_t>{__v0, __v1}}; +} + +_CCCL_TEMPLATE(typename _Tp0, typename _Abi0, typename _Tp1, typename _Abi1) +_CCCL_REQUIRES(__simd_is_valid_rotate_v<_Tp0, _Abi0, _Tp1, _Abi1>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +rotr(const basic_vec<_Tp0, _Abi0>& __v0, const basic_vec<_Tp1, _Abi1>& __v1) noexcept +{ + using __vec0_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp0, _Abi0>; + using __vec1_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp1, _Abi1>; + return __vec0_t{__simd_rotr_generator<__vec0_t, __vec1_t>{__v0, __v1}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rotl(const basic_vec<_Tp, _Abi>& __v, const int __s) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_rotl_scalar_generator<__vec_t>{__v, __s}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto rotr(const basic_vec<_Tp, _Abi>& __v, const int __s) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + return __vec_t{__simd_rotr_scalar_generator<__vec_t>{__v, __s}}; +} + +template +using __simd_bit_count_result_t _CCCL_NODEBUG_ALIAS = rebind_t, _Vp>; + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto bit_width(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>; + return __result_t{__simd_bit_width_generator<__vec_t, __result_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countl_zero(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>; + return __result_t{__simd_countl_zero_generator<__vec_t, __result_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countl_one(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>; + return __result_t{__simd_countl_one_generator<__vec_t, __result_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countr_zero(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>; + return __result_t{__simd_countr_zero_generator<__vec_t, __result_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto countr_one(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>; + return __result_t{__simd_countr_one_generator<__vec_t, __result_t>{__v}}; +} + +_CCCL_TEMPLATE(typename _Tp, typename _Abi) +_CCCL_REQUIRES(__cccl_is_unsigned_integer_v<_Tp>) +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto popcount(const basic_vec<_Tp, _Abi>& __v) noexcept +{ + using __vec_t _CCCL_NODEBUG_ALIAS = basic_vec<_Tp, _Abi>; + using __result_t _CCCL_NODEBUG_ALIAS = __simd_bit_count_result_t<_Tp, __vec_t>; + return __result_t{__simd_popcount_generator<__vec_t, __result_t>{__v}}; +} + +_CCCL_END_NAMESPACE_CUDA_STD_SIMD + +#include + +#endif // _CUDA_STD___SIMD_BIT_SCALAR_H diff --git a/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/bit_reverse.pass.cpp b/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/bit_reverse.pass.cpp new file mode 100644 index 00000000000..8cf5dde47ed --- /dev/null +++ b/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/bit_reverse.pass.cpp @@ -0,0 +1,74 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++ in the CUDA C++ Core Libraries, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// + +// [simd.bit], bit_reverse + +#include +#include +#include +#include +#include + +#include "../simd_test_utils.h" + +template +struct test_bit_reverse +{ + TEST_FUNC constexpr void operator()() const + { + using Vec = simd::basic_vec>; + + Vec vec(bit_values{}); + static_assert(cuda::std::is_same_v); + static_assert(noexcept(simd::bit_reverse(vec))); + + Vec result = simd::bit_reverse(vec); + for (int i = 0; i < N; ++i) + { + assert(result[i] == cuda::bit_reverse(vec[i])); + } + + constexpr T all_bits = cuda::std::numeric_limits::max(); + assert(simd::bit_reverse(Vec{T{0}})[0] == T{0}); + assert(simd::bit_reverse(Vec{T{1}})[0] == static_cast(T{1} << (cuda::std::numeric_limits::digits - 1))); + assert(simd::bit_reverse(Vec{all_bits})[0] == all_bits); + } +}; + +template +struct has_simd_bit_reverse : cuda::std::false_type +{}; + +template +struct has_simd_bit_reverse()))>> + : cuda::std::true_type +{}; + +TEST_FUNC constexpr void test_constraints() +{ + using int_vec = simd::basic_vec>; + using uint_vec = simd::basic_vec>; + using float_vec = simd::basic_vec>; + + static_assert(has_simd_bit_reverse::value); + static_assert(!has_simd_bit_reverse::value); + static_assert(!has_simd_bit_reverse::value); +} + +DEFINE_SIMD_BIT_UNSIGNED_TEST(test_bit_reverse) + +int main(int, char**) +{ + assert(test()); + static_assert(test()); + return 0; +} diff --git a/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/shl.pass.cpp b/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/shl.pass.cpp new file mode 100644 index 00000000000..e1a5714fbc3 --- /dev/null +++ b/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/shl.pass.cpp @@ -0,0 +1,152 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++ in the CUDA C++ Core Libraries, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// + +// [simd.bit], shl + +#include +#include +#include +#include +#include +#include + +#include "../simd_test_utils.h" + +template +struct shift_input_values_gen +{ + template + TEST_FUNC constexpr T operator()(I) const + { + if constexpr (cuda::std::is_signed_v) + { + return I::value % 2 == 0 ? T{-7} : T{5}; + } + else + { + return I::value % 2 == 0 ? T{7} : T{5}; + } + } +}; + +template +struct shift_count_values_gen +{ + template + TEST_FUNC constexpr T operator()(I) const + { + constexpr int digits = cuda::std::numeric_limits>::digits; + constexpr int values[] = {-1, 0, digits, digits + 1}; + return static_cast(values[I::value % 4]); + } +}; + +template +struct test_shl_vec +{ + TEST_FUNC constexpr void operator()() const + { + using vec_t = simd::basic_vec>; + using shift_vec = simd::rebind_t, vec_t>; + + vec_t vec(shift_input_values_gen{}); + shift_vec shifts(shift_count_values_gen{}); + + static_assert(cuda::std::is_same_v); + static_assert(noexcept(simd::shl(vec, shifts))); + + vec_t result = simd::shl(vec, shifts); + for (int i = 0; i < N; ++i) + { + assert(result[i] == cuda::std::shl(vec[i], shifts[i])); + } + } +}; + +template +struct test_shl_scalar +{ + TEST_FUNC constexpr void operator()() const + { + using vec_t = simd::basic_vec>; + vec_t vec(shift_input_values_gen{}); + + static_assert(cuda::std::is_same_v); + static_assert(noexcept(simd::shl(vec, 1))); + + constexpr int digits = cuda::std::numeric_limits>::digits; + const int shifts[] = {-digits - 1, -1, 0, 1, digits, digits + 1}; + for (int shift : shifts) + { + vec_t result = simd::shl(vec, shift); + for (int i = 0; i < N; ++i) + { + assert(result[i] == cuda::std::shl(vec[i], shift)); + } + } + } +}; + +template +struct has_simd_shl_vec : cuda::std::false_type +{}; + +template +struct has_simd_shl_vec(), cuda::std::declval()))>> + : cuda::std::true_type +{}; + +template +struct has_simd_shl_scalar : cuda::std::false_type +{}; + +template +struct has_simd_shl_scalar(), cuda::std::declval()))>> + : cuda::std::true_type +{}; + +TEST_FUNC constexpr void test_constraints() +{ + using int32_vec4 = simd::basic_vec>; + using uint32_vec4 = simd::basic_vec>; + using int16_vec4 = simd::basic_vec>; + using int32_vec2 = simd::basic_vec>; + using float_vec4 = simd::basic_vec>; + + static_assert(has_simd_shl_vec::value); + static_assert(!has_simd_shl_vec::value); + static_assert(!has_simd_shl_vec::value); + static_assert(!has_simd_shl_vec::value); + static_assert(!has_simd_shl_vec::value); + + static_assert(has_simd_shl_scalar::value); + static_assert(has_simd_shl_scalar::value); + static_assert(!has_simd_shl_scalar::value); + static_assert(!has_simd_shl_scalar::value); +} + +TEST_FUNC constexpr bool test() +{ + _SIMD_BIT_TEST_SIGNED_TYPES(test_shl_vec) + _SIMD_BIT_TEST_UNSIGNED_TYPES(test_shl_vec) + _SIMD_BIT_TEST_SIGNED_TYPES(test_shl_scalar) + _SIMD_BIT_TEST_UNSIGNED_TYPES(test_shl_scalar) + test_constraints(); + return true; +} + +int main(int, char**) +{ + assert(test()); + static_assert(test()); + return 0; +} diff --git a/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/shr.pass.cpp b/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/shr.pass.cpp new file mode 100644 index 00000000000..f74975ea2e4 --- /dev/null +++ b/libcudacxx/test/libcudacxx/std/numerics/simd/simd.bit/shr.pass.cpp @@ -0,0 +1,152 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++ in the CUDA C++ Core Libraries, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +// + +// [simd.bit], shr + +#include +#include +#include +#include +#include +#include + +#include "../simd_test_utils.h" + +template +struct shift_input_values_gen +{ + template + TEST_FUNC constexpr T operator()(I) const + { + if constexpr (cuda::std::is_signed_v) + { + return I::value % 2 == 0 ? T{-7} : T{5}; + } + else + { + return I::value % 2 == 0 ? T{7} : T{5}; + } + } +}; + +template +struct shift_count_values_gen +{ + template + TEST_FUNC constexpr T operator()(I) const + { + constexpr int digits = cuda::std::numeric_limits>::digits; + constexpr int values[] = {-1, 0, digits, digits + 1}; + return static_cast(values[I::value % 4]); + } +}; + +template +struct test_shr_vec +{ + TEST_FUNC constexpr void operator()() const + { + using vec_t = simd::basic_vec>; + using shift_vec_t = simd::rebind_t, vec_t>; + + vec_t vec(shift_input_values_gen{}); + shift_vec_t shifts(shift_count_values_gen{}); + + static_assert(cuda::std::is_same_v); + static_assert(noexcept(simd::shr(vec, shifts))); + + vec_t result = simd::shr(vec, shifts); + for (int i = 0; i < N; ++i) + { + assert(result[i] == cuda::std::shr(vec[i], shifts[i])); + } + } +}; + +template +struct test_shr_scalar +{ + TEST_FUNC constexpr void operator()() const + { + using vec_t = simd::basic_vec>; + vec_t vec(shift_input_values_gen{}); + + static_assert(cuda::std::is_same_v); + static_assert(noexcept(simd::shr(vec, 1))); + + constexpr int digits = cuda::std::numeric_limits>::digits; + const int shifts[] = {-digits - 1, -1, 0, 1, digits, digits + 1}; + for (int shift : shifts) + { + vec_t result = simd::shr(vec, shift); + for (int i = 0; i < N; ++i) + { + assert(result[i] == cuda::std::shr(vec[i], shift)); + } + } + } +}; + +template +struct has_simd_shr_vec : cuda::std::false_type +{}; + +template +struct has_simd_shr_vec(), cuda::std::declval()))>> + : cuda::std::true_type +{}; + +template +struct has_simd_shr_scalar : cuda::std::false_type +{}; + +template +struct has_simd_shr_scalar(), cuda::std::declval()))>> + : cuda::std::true_type +{}; + +TEST_FUNC constexpr void test_constraints() +{ + using int32_vec4 = simd::basic_vec>; + using uint32_vec4 = simd::basic_vec>; + using int16_vec4 = simd::basic_vec>; + using int32_vec2 = simd::basic_vec>; + using float_vec4 = simd::basic_vec>; + + static_assert(has_simd_shr_vec::value); + static_assert(!has_simd_shr_vec::value); + static_assert(!has_simd_shr_vec::value); + static_assert(!has_simd_shr_vec::value); + static_assert(!has_simd_shr_vec::value); + + static_assert(has_simd_shr_scalar::value); + static_assert(has_simd_shr_scalar::value); + static_assert(!has_simd_shr_scalar::value); + static_assert(!has_simd_shr_scalar::value); +} + +TEST_FUNC constexpr bool test() +{ + _SIMD_BIT_TEST_SIGNED_TYPES(test_shr_vec) + _SIMD_BIT_TEST_UNSIGNED_TYPES(test_shr_vec) + _SIMD_BIT_TEST_SIGNED_TYPES(test_shr_scalar) + _SIMD_BIT_TEST_UNSIGNED_TYPES(test_shr_scalar) + test_constraints(); + return true; +} + +int main(int, char**) +{ + assert(test()); + static_assert(test()); + return 0; +}