30 #ifndef _GLIBCXX_ATOMIC_WAIT_H
31 #define _GLIBCXX_ATOMIC_WAIT_H 1
33 #ifdef _GLIBCXX_SYSHDR
34 #pragma GCC system_header
39 #if __glibcxx_atomic_wait
42 #include <bits/gthr.h>
45 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
55 namespace std _GLIBCXX_VISIBILITY(default)
57 _GLIBCXX_BEGIN_NAMESPACE_VERSION
60 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
61 #define _GLIBCXX_HAVE_PLATFORM_WAIT 1
62 using __platform_wait_t = int;
63 inline constexpr
size_t __platform_wait_alignment = 4;
69 # if ATOMIC_LONG_LOCK_FREE == 2
70 using __platform_wait_t =
unsigned long;
72 using __platform_wait_t =
unsigned int;
74 inline constexpr
size_t __platform_wait_alignment
75 = __alignof__(__platform_wait_t);
79 template<
typename _Tp>
80 inline constexpr
bool __platform_wait_uses_type
81 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
83 && ((
sizeof(_Tp) ==
sizeof(__detail::__platform_wait_t))
84 && (
alignof(_Tp*) >= __detail::__platform_wait_alignment));
91 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
92 enum class __futex_wait_flags : int
94 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE
103 __wait_private = __wait | __private_flag,
104 __wake_private = __wake | __private_flag,
105 __wait_bitset_private = __wait_bitset | __private_flag,
106 __wake_bitset_private = __wake_bitset | __private_flag,
107 __bitset_match_any = -1
110 template<
typename _Tp>
112 __platform_wait(
const _Tp* __addr, __platform_wait_t __val) noexcept
114 auto __e = syscall (SYS_futex,
static_cast<const void*
>(__addr),
115 static_cast<int>(__futex_wait_flags::__wait_private),
117 if (!__e || errno == EAGAIN)
120 __throw_system_error(errno);
123 template<
typename _Tp>
125 __platform_notify(
const _Tp* __addr,
bool __all) noexcept
127 syscall (SYS_futex,
static_cast<const void*
>(__addr),
128 static_cast<int>(__futex_wait_flags::__wake_private),
129 __all ? INT_MAX : 1);
134 __thread_yield() noexcept
136 #if defined _GLIBCXX_HAS_GTHREADS && defined _GLIBCXX_USE_SCHED_YIELD
142 __thread_relax() noexcept
144 #if defined __i386__ || defined __x86_64__
145 __builtin_ia32_pause();
151 inline constexpr
auto __atomic_spin_count_relax = 12;
152 inline constexpr
auto __atomic_spin_count = 16;
154 struct __default_spin_policy
157 operator()() const noexcept
161 template<
typename _Pred,
162 typename _Spin = __default_spin_policy>
164 __atomic_spin(_Pred& __pred, _Spin __spin = _Spin{ }) noexcept
166 for (
auto __i = 0; __i < __atomic_spin_count; ++__i)
171 if (__i < __atomic_spin_count_relax)
172 __detail::__thread_relax();
174 __detail::__thread_yield();
187 template<
typename _Tp>
188 bool __atomic_compare(
const _Tp& __a,
const _Tp& __b)
191 return __builtin_memcmp(&__a, &__b,
sizeof(_Tp)) == 0;
194 struct __waiter_pool_base
198 static constexpr
auto _S_align = 64;
200 alignas(_S_align) __platform_wait_t _M_wait = 0;
202 #ifndef _GLIBCXX_HAVE_PLATFORM_WAIT
206 alignas(_S_align) __platform_wait_t _M_ver = 0;
208 #ifndef _GLIBCXX_HAVE_PLATFORM_WAIT
211 __waiter_pool_base() =
default;
214 _M_enter_wait() noexcept
215 { __atomic_fetch_add(&_M_wait, 1, __ATOMIC_SEQ_CST); }
218 _M_leave_wait() noexcept
219 { __atomic_fetch_sub(&_M_wait, 1, __ATOMIC_RELEASE); }
222 _M_waiting() const noexcept
224 __platform_wait_t __res;
225 __atomic_load(&_M_wait, &__res, __ATOMIC_SEQ_CST);
230 _M_notify(__platform_wait_t* __addr, [[maybe_unused]]
bool __all,
231 bool __bare) noexcept
233 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
234 if (__addr == &_M_ver)
236 __atomic_fetch_add(__addr, 1, __ATOMIC_SEQ_CST);
240 if (__bare || _M_waiting())
241 __platform_notify(__addr, __all);
244 lock_guard<mutex> __l(_M_mtx);
245 __atomic_fetch_add(__addr, 1, __ATOMIC_RELAXED);
247 if (__bare || _M_waiting())
252 static __waiter_pool_base&
253 _S_for(
const void* __addr) noexcept
255 constexpr __UINTPTR_TYPE__ __ct = 16;
256 static __waiter_pool_base __w[__ct];
257 auto __key = ((__UINTPTR_TYPE__)__addr >> 2) % __ct;
262 struct __waiter_pool : __waiter_pool_base
265 _M_do_wait(
const __platform_wait_t* __addr, __platform_wait_t __old) noexcept
267 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
268 __platform_wait(__addr, __old);
270 __platform_wait_t __val;
271 __atomic_load(__addr, &__val, __ATOMIC_SEQ_CST);
274 lock_guard<mutex> __l(_M_mtx);
275 __atomic_load(__addr, &__val, __ATOMIC_RELAXED);
283 template<
typename _Tp>
286 using __waiter_type = _Tp;
289 __platform_wait_t* _M_addr;
291 template<
typename _Up>
292 static __platform_wait_t*
293 _S_wait_addr(
const _Up* __a, __platform_wait_t* __b)
295 if constexpr (__platform_wait_uses_type<_Up>)
296 return reinterpret_cast<__platform_wait_t*
>(
const_cast<_Up*
>(__a));
301 static __waiter_type&
302 _S_for(
const void* __addr) noexcept
304 static_assert(
sizeof(__waiter_type) ==
sizeof(__waiter_pool_base));
305 auto& res = __waiter_pool_base::_S_for(__addr);
306 return reinterpret_cast<__waiter_type&
>(res);
309 template<
typename _Up>
310 explicit __waiter_base(
const _Up* __addr) noexcept
311 : _M_w(_S_for(__addr))
312 , _M_addr(_S_wait_addr(__addr, &_M_w._M_ver))
316 _M_notify(
bool __all,
bool __bare =
false) noexcept
317 { _M_w._M_notify(_M_addr, __all, __bare); }
319 template<
typename _Up,
typename _ValFn,
320 typename _Spin = __default_spin_policy>
322 _S_do_spin_v(__platform_wait_t* __addr,
323 const _Up& __old, _ValFn __vfn,
324 __platform_wait_t& __val,
325 _Spin __spin = _Spin{ })
327 auto const __pred = [=]
328 {
return !__detail::__atomic_compare(__old, __vfn()); };
330 if constexpr (__platform_wait_uses_type<_Up>)
332 __builtin_memcpy(&__val, &__old,
sizeof(__val));
336 __atomic_load(__addr, &__val, __ATOMIC_ACQUIRE);
338 return __atomic_spin(__pred, __spin);
341 template<
typename _Up,
typename _ValFn,
342 typename _Spin = __default_spin_policy>
344 _M_do_spin_v(
const _Up& __old, _ValFn __vfn,
345 __platform_wait_t& __val,
346 _Spin __spin = _Spin{ })
347 {
return _S_do_spin_v(_M_addr, __old, __vfn, __val, __spin); }
349 template<
typename _Pred,
350 typename _Spin = __default_spin_policy>
352 _S_do_spin(
const __platform_wait_t* __addr,
354 __platform_wait_t& __val,
355 _Spin __spin = _Spin{ })
357 __atomic_load(__addr, &__val, __ATOMIC_ACQUIRE);
358 return __atomic_spin(__pred, __spin);
361 template<
typename _Pred,
362 typename _Spin = __default_spin_policy>
364 _M_do_spin(_Pred __pred, __platform_wait_t& __val,
365 _Spin __spin = _Spin{ })
366 {
return _S_do_spin(_M_addr, __pred, __val, __spin); }
369 template<
typename _EntersWait>
370 struct __waiter : __waiter_base<__waiter_pool>
372 using __base_type = __waiter_base<__waiter_pool>;
374 template<
typename _Tp>
375 explicit __waiter(
const _Tp* __addr) noexcept
376 : __base_type(__addr)
378 if constexpr (_EntersWait::value)
379 _M_w._M_enter_wait();
384 if constexpr (_EntersWait::value)
385 _M_w._M_leave_wait();
388 template<
typename _Tp,
typename _ValFn>
390 _M_do_wait_v(_Tp __old, _ValFn __vfn)
394 __platform_wait_t __val;
395 if (__base_type::_M_do_spin_v(__old, __vfn, __val))
397 __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val);
399 while (__detail::__atomic_compare(__old, __vfn()));
402 template<
typename _Pred>
404 _M_do_wait(_Pred __pred) noexcept
408 __platform_wait_t __val;
409 if (__base_type::_M_do_spin(__pred, __val))
411 __base_type::_M_w._M_do_wait(__base_type::_M_addr, __val);
417 using __enters_wait = __waiter<std::true_type>;
418 using __bare_wait = __waiter<std::false_type>;
421 template<
typename _Tp,
typename _ValFn>
423 __atomic_wait_address_v(
const _Tp* __addr, _Tp __old,
424 _ValFn __vfn) noexcept
426 __detail::__enters_wait __w(__addr);
427 __w._M_do_wait_v(__old, __vfn);
430 template<
typename _Tp,
typename _Pred>
432 __atomic_wait_address(
const _Tp* __addr, _Pred __pred) noexcept
434 __detail::__enters_wait __w(__addr);
435 __w._M_do_wait(__pred);
439 template<
typename _Pred>
441 __atomic_wait_address_bare(
const __detail::__platform_wait_t* __addr,
442 _Pred __pred) noexcept
444 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
447 __detail::__platform_wait_t __val;
448 if (__detail::__bare_wait::_S_do_spin(__addr, __pred, __val))
450 __detail::__platform_wait(__addr, __val);
454 __detail::__bare_wait __w(__addr);
455 __w._M_do_wait(__pred);
459 template<
typename _Tp>
461 __atomic_notify_address(
const _Tp* __addr,
bool __all) noexcept
463 __detail::__bare_wait __w(__addr);
464 __w._M_notify(__all);
469 __atomic_notify_address_bare(
const __detail::__platform_wait_t* __addr,
472 #ifdef _GLIBCXX_HAVE_PLATFORM_WAIT
473 __detail::__platform_notify(__addr, __all);
475 __detail::__bare_wait __w(__addr);
476 __w._M_notify(__all,
true);
479 _GLIBCXX_END_NAMESPACE_VERSION
ISO C++ entities toplevel namespace is std.