21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
25 #include <sys/syscall.h>
43 void __kmp_validate_locks(
void) {
48 x = ~((kmp_uint32)0) - 2;
51 for (i = 0; i < 8; ++i, ++x, ++y) {
52 kmp_uint32 z = (x - y);
56 KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
70 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
71 return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1;
74 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
75 return lck->lk.depth_locked != -1;
78 __forceinline
static int
79 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
82 #ifdef USE_LOCK_PROFILE
83 kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll);
84 if ((curr != 0) && (curr != gtid + 1))
85 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
89 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
90 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
92 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
93 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
94 KMP_FSYNC_ACQUIRED(lck);
95 return KMP_LOCK_ACQUIRED_FIRST;
99 KMP_FSYNC_PREPARE(lck);
100 KMP_INIT_YIELD(spins);
101 kmp_backoff_t backoff = __kmp_spin_backoff_params;
103 __kmp_spin_backoff(&backoff);
104 KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
105 }
while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
106 !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy));
107 KMP_FSYNC_ACQUIRED(lck);
108 return KMP_LOCK_ACQUIRED_FIRST;
111 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
112 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
116 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
118 char const *
const func =
"omp_set_lock";
119 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
120 __kmp_is_tas_lock_nestable(lck)) {
121 KMP_FATAL(LockNestableUsedAsSimple, func);
123 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
124 KMP_FATAL(LockIsAlreadyOwned, func);
126 return __kmp_acquire_tas_lock(lck, gtid);
129 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
130 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
131 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
132 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
133 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
134 KMP_FSYNC_ACQUIRED(lck);
140 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
142 char const *
const func =
"omp_test_lock";
143 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
144 __kmp_is_tas_lock_nestable(lck)) {
145 KMP_FATAL(LockNestableUsedAsSimple, func);
147 return __kmp_test_tas_lock(lck, gtid);
150 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
153 KMP_FSYNC_RELEASING(lck);
154 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas));
158 return KMP_LOCK_RELEASED;
161 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
163 char const *
const func =
"omp_unset_lock";
165 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
166 __kmp_is_tas_lock_nestable(lck)) {
167 KMP_FATAL(LockNestableUsedAsSimple, func);
169 if (__kmp_get_tas_lock_owner(lck) == -1) {
170 KMP_FATAL(LockUnsettingFree, func);
172 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
173 (__kmp_get_tas_lock_owner(lck) != gtid)) {
174 KMP_FATAL(LockUnsettingSetByAnother, func);
176 return __kmp_release_tas_lock(lck, gtid);
179 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
180 lck->lk.poll = KMP_LOCK_FREE(tas);
183 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
185 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
186 char const *
const func =
"omp_destroy_lock";
187 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
188 __kmp_is_tas_lock_nestable(lck)) {
189 KMP_FATAL(LockNestableUsedAsSimple, func);
191 if (__kmp_get_tas_lock_owner(lck) != -1) {
192 KMP_FATAL(LockStillOwned, func);
194 __kmp_destroy_tas_lock(lck);
199 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
200 KMP_DEBUG_ASSERT(gtid >= 0);
202 if (__kmp_get_tas_lock_owner(lck) == gtid) {
203 lck->lk.depth_locked += 1;
204 return KMP_LOCK_ACQUIRED_NEXT;
206 __kmp_acquire_tas_lock_timed_template(lck, gtid);
207 lck->lk.depth_locked = 1;
208 return KMP_LOCK_ACQUIRED_FIRST;
212 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
214 char const *
const func =
"omp_set_nest_lock";
215 if (!__kmp_is_tas_lock_nestable(lck)) {
216 KMP_FATAL(LockSimpleUsedAsNestable, func);
218 return __kmp_acquire_nested_tas_lock(lck, gtid);
221 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
224 KMP_DEBUG_ASSERT(gtid >= 0);
226 if (__kmp_get_tas_lock_owner(lck) == gtid) {
227 retval = ++lck->lk.depth_locked;
228 }
else if (!__kmp_test_tas_lock(lck, gtid)) {
232 retval = lck->lk.depth_locked = 1;
237 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
239 char const *
const func =
"omp_test_nest_lock";
240 if (!__kmp_is_tas_lock_nestable(lck)) {
241 KMP_FATAL(LockSimpleUsedAsNestable, func);
243 return __kmp_test_nested_tas_lock(lck, gtid);
246 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
247 KMP_DEBUG_ASSERT(gtid >= 0);
250 if (--(lck->lk.depth_locked) == 0) {
251 __kmp_release_tas_lock(lck, gtid);
252 return KMP_LOCK_RELEASED;
254 return KMP_LOCK_STILL_HELD;
257 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
259 char const *
const func =
"omp_unset_nest_lock";
261 if (!__kmp_is_tas_lock_nestable(lck)) {
262 KMP_FATAL(LockSimpleUsedAsNestable, func);
264 if (__kmp_get_tas_lock_owner(lck) == -1) {
265 KMP_FATAL(LockUnsettingFree, func);
267 if (__kmp_get_tas_lock_owner(lck) != gtid) {
268 KMP_FATAL(LockUnsettingSetByAnother, func);
270 return __kmp_release_nested_tas_lock(lck, gtid);
273 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
274 __kmp_init_tas_lock(lck);
275 lck->lk.depth_locked = 0;
278 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
279 __kmp_destroy_tas_lock(lck);
280 lck->lk.depth_locked = 0;
283 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
284 char const *
const func =
"omp_destroy_nest_lock";
285 if (!__kmp_is_tas_lock_nestable(lck)) {
286 KMP_FATAL(LockSimpleUsedAsNestable, func);
288 if (__kmp_get_tas_lock_owner(lck) != -1) {
289 KMP_FATAL(LockStillOwned, func);
291 __kmp_destroy_nested_tas_lock(lck);
304 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
305 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
308 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
309 return lck->lk.depth_locked != -1;
312 __forceinline
static int
313 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
314 kmp_int32 gtid_code = (gtid + 1) << 1;
318 #ifdef USE_LOCK_PROFILE
319 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
320 if ((curr != 0) && (curr != gtid_code))
321 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
325 KMP_FSYNC_PREPARE(lck);
326 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
327 lck, lck->lk.poll, gtid));
331 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(
332 &(lck->lk.poll), KMP_LOCK_FREE(futex),
333 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) {
335 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
338 (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
339 lck, gtid, poll_val, cond));
350 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
351 poll_val | KMP_LOCK_BUSY(1, futex))) {
354 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
355 lck, lck->lk.poll, gtid));
358 poll_val |= KMP_LOCK_BUSY(1, futex);
361 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
362 lck->lk.poll, gtid));
367 (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
368 lck, gtid, poll_val));
371 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
373 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) "
374 "failed (rc=%ld errno=%d)\n",
375 lck, gtid, poll_val, rc, errno));
380 (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
381 lck, gtid, poll_val));
388 KMP_FSYNC_ACQUIRED(lck);
389 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
390 lck->lk.poll, gtid));
391 return KMP_LOCK_ACQUIRED_FIRST;
394 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
395 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
399 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
401 char const *
const func =
"omp_set_lock";
402 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
403 __kmp_is_futex_lock_nestable(lck)) {
404 KMP_FATAL(LockNestableUsedAsSimple, func);
406 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
407 KMP_FATAL(LockIsAlreadyOwned, func);
409 return __kmp_acquire_futex_lock(lck, gtid);
412 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
413 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
414 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) {
415 KMP_FSYNC_ACQUIRED(lck);
421 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
423 char const *
const func =
"omp_test_lock";
424 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
425 __kmp_is_futex_lock_nestable(lck)) {
426 KMP_FATAL(LockNestableUsedAsSimple, func);
428 return __kmp_test_futex_lock(lck, gtid);
431 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
434 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
435 lck, lck->lk.poll, gtid));
437 KMP_FSYNC_RELEASING(lck);
439 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
442 (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
443 lck, gtid, poll_val));
445 if (KMP_LOCK_STRIP(poll_val) & 1) {
447 (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
449 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
455 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
456 lck->lk.poll, gtid));
459 return KMP_LOCK_RELEASED;
462 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
464 char const *
const func =
"omp_unset_lock";
466 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
467 __kmp_is_futex_lock_nestable(lck)) {
468 KMP_FATAL(LockNestableUsedAsSimple, func);
470 if (__kmp_get_futex_lock_owner(lck) == -1) {
471 KMP_FATAL(LockUnsettingFree, func);
473 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
474 (__kmp_get_futex_lock_owner(lck) != gtid)) {
475 KMP_FATAL(LockUnsettingSetByAnother, func);
477 return __kmp_release_futex_lock(lck, gtid);
480 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
481 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
484 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
486 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
487 char const *
const func =
"omp_destroy_lock";
488 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
489 __kmp_is_futex_lock_nestable(lck)) {
490 KMP_FATAL(LockNestableUsedAsSimple, func);
492 if (__kmp_get_futex_lock_owner(lck) != -1) {
493 KMP_FATAL(LockStillOwned, func);
495 __kmp_destroy_futex_lock(lck);
500 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
501 KMP_DEBUG_ASSERT(gtid >= 0);
503 if (__kmp_get_futex_lock_owner(lck) == gtid) {
504 lck->lk.depth_locked += 1;
505 return KMP_LOCK_ACQUIRED_NEXT;
507 __kmp_acquire_futex_lock_timed_template(lck, gtid);
508 lck->lk.depth_locked = 1;
509 return KMP_LOCK_ACQUIRED_FIRST;
513 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
515 char const *
const func =
"omp_set_nest_lock";
516 if (!__kmp_is_futex_lock_nestable(lck)) {
517 KMP_FATAL(LockSimpleUsedAsNestable, func);
519 return __kmp_acquire_nested_futex_lock(lck, gtid);
522 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
525 KMP_DEBUG_ASSERT(gtid >= 0);
527 if (__kmp_get_futex_lock_owner(lck) == gtid) {
528 retval = ++lck->lk.depth_locked;
529 }
else if (!__kmp_test_futex_lock(lck, gtid)) {
533 retval = lck->lk.depth_locked = 1;
538 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
540 char const *
const func =
"omp_test_nest_lock";
541 if (!__kmp_is_futex_lock_nestable(lck)) {
542 KMP_FATAL(LockSimpleUsedAsNestable, func);
544 return __kmp_test_nested_futex_lock(lck, gtid);
547 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
548 KMP_DEBUG_ASSERT(gtid >= 0);
551 if (--(lck->lk.depth_locked) == 0) {
552 __kmp_release_futex_lock(lck, gtid);
553 return KMP_LOCK_RELEASED;
555 return KMP_LOCK_STILL_HELD;
558 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
560 char const *
const func =
"omp_unset_nest_lock";
562 if (!__kmp_is_futex_lock_nestable(lck)) {
563 KMP_FATAL(LockSimpleUsedAsNestable, func);
565 if (__kmp_get_futex_lock_owner(lck) == -1) {
566 KMP_FATAL(LockUnsettingFree, func);
568 if (__kmp_get_futex_lock_owner(lck) != gtid) {
569 KMP_FATAL(LockUnsettingSetByAnother, func);
571 return __kmp_release_nested_futex_lock(lck, gtid);
574 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
575 __kmp_init_futex_lock(lck);
576 lck->lk.depth_locked = 0;
579 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
580 __kmp_destroy_futex_lock(lck);
581 lck->lk.depth_locked = 0;
584 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
585 char const *
const func =
"omp_destroy_nest_lock";
586 if (!__kmp_is_futex_lock_nestable(lck)) {
587 KMP_FATAL(LockSimpleUsedAsNestable, func);
589 if (__kmp_get_futex_lock_owner(lck) != -1) {
590 KMP_FATAL(LockStillOwned, func);
592 __kmp_destroy_nested_futex_lock(lck);
600 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
601 return std::atomic_load_explicit(&lck->lk.owner_id,
602 std::memory_order_relaxed) -
606 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
607 return std::atomic_load_explicit(&lck->lk.depth_locked,
608 std::memory_order_relaxed) != -1;
611 static kmp_uint32 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket) {
612 return std::atomic_load_explicit((std::atomic<unsigned> *)now_serving,
613 std::memory_order_acquire) == my_ticket;
616 __forceinline
static int
617 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
619 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit(
620 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
622 #ifdef USE_LOCK_PROFILE
623 if (std::atomic_load_explicit(&lck->lk.now_serving,
624 std::memory_order_relaxed) != my_ticket)
625 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
629 if (std::atomic_load_explicit(&lck->lk.now_serving,
630 std::memory_order_acquire) == my_ticket) {
631 return KMP_LOCK_ACQUIRED_FIRST;
633 KMP_WAIT_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
634 return KMP_LOCK_ACQUIRED_FIRST;
637 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
638 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
642 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
644 char const *
const func =
"omp_set_lock";
646 if (!std::atomic_load_explicit(&lck->lk.initialized,
647 std::memory_order_relaxed)) {
648 KMP_FATAL(LockIsUninitialized, func);
650 if (lck->lk.self != lck) {
651 KMP_FATAL(LockIsUninitialized, func);
653 if (__kmp_is_ticket_lock_nestable(lck)) {
654 KMP_FATAL(LockNestableUsedAsSimple, func);
656 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
657 KMP_FATAL(LockIsAlreadyOwned, func);
660 __kmp_acquire_ticket_lock(lck, gtid);
662 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
663 std::memory_order_relaxed);
664 return KMP_LOCK_ACQUIRED_FIRST;
667 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
668 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
669 std::memory_order_relaxed);
671 if (std::atomic_load_explicit(&lck->lk.now_serving,
672 std::memory_order_relaxed) == my_ticket) {
673 kmp_uint32 next_ticket = my_ticket + 1;
674 if (std::atomic_compare_exchange_strong_explicit(
675 &lck->lk.next_ticket, &my_ticket, next_ticket,
676 std::memory_order_acquire, std::memory_order_acquire)) {
683 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
685 char const *
const func =
"omp_test_lock";
687 if (!std::atomic_load_explicit(&lck->lk.initialized,
688 std::memory_order_relaxed)) {
689 KMP_FATAL(LockIsUninitialized, func);
691 if (lck->lk.self != lck) {
692 KMP_FATAL(LockIsUninitialized, func);
694 if (__kmp_is_ticket_lock_nestable(lck)) {
695 KMP_FATAL(LockNestableUsedAsSimple, func);
698 int retval = __kmp_test_ticket_lock(lck, gtid);
701 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
702 std::memory_order_relaxed);
707 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
708 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
709 std::memory_order_relaxed) -
710 std::atomic_load_explicit(&lck->lk.now_serving,
711 std::memory_order_relaxed);
713 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
714 std::memory_order_release);
717 (kmp_uint32)(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
718 return KMP_LOCK_RELEASED;
721 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
723 char const *
const func =
"omp_unset_lock";
725 if (!std::atomic_load_explicit(&lck->lk.initialized,
726 std::memory_order_relaxed)) {
727 KMP_FATAL(LockIsUninitialized, func);
729 if (lck->lk.self != lck) {
730 KMP_FATAL(LockIsUninitialized, func);
732 if (__kmp_is_ticket_lock_nestable(lck)) {
733 KMP_FATAL(LockNestableUsedAsSimple, func);
735 if (__kmp_get_ticket_lock_owner(lck) == -1) {
736 KMP_FATAL(LockUnsettingFree, func);
738 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
739 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
740 KMP_FATAL(LockUnsettingSetByAnother, func);
742 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
743 return __kmp_release_ticket_lock(lck, gtid);
746 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
747 lck->lk.location = NULL;
749 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
750 std::memory_order_relaxed);
751 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
752 std::memory_order_relaxed);
753 std::atomic_store_explicit(
754 &lck->lk.owner_id, 0,
755 std::memory_order_relaxed);
756 std::atomic_store_explicit(
757 &lck->lk.depth_locked, -1,
758 std::memory_order_relaxed);
759 std::atomic_store_explicit(&lck->lk.initialized,
true,
760 std::memory_order_release);
763 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
764 std::atomic_store_explicit(&lck->lk.initialized,
false,
765 std::memory_order_release);
767 lck->lk.location = NULL;
768 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
769 std::memory_order_relaxed);
770 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
771 std::memory_order_relaxed);
772 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
773 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
774 std::memory_order_relaxed);
777 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
778 char const *
const func =
"omp_destroy_lock";
780 if (!std::atomic_load_explicit(&lck->lk.initialized,
781 std::memory_order_relaxed)) {
782 KMP_FATAL(LockIsUninitialized, func);
784 if (lck->lk.self != lck) {
785 KMP_FATAL(LockIsUninitialized, func);
787 if (__kmp_is_ticket_lock_nestable(lck)) {
788 KMP_FATAL(LockNestableUsedAsSimple, func);
790 if (__kmp_get_ticket_lock_owner(lck) != -1) {
791 KMP_FATAL(LockStillOwned, func);
793 __kmp_destroy_ticket_lock(lck);
798 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
799 KMP_DEBUG_ASSERT(gtid >= 0);
801 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
802 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
803 std::memory_order_relaxed);
804 return KMP_LOCK_ACQUIRED_NEXT;
806 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
807 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
808 std::memory_order_relaxed);
809 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
810 std::memory_order_relaxed);
811 return KMP_LOCK_ACQUIRED_FIRST;
815 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
817 char const *
const func =
"omp_set_nest_lock";
819 if (!std::atomic_load_explicit(&lck->lk.initialized,
820 std::memory_order_relaxed)) {
821 KMP_FATAL(LockIsUninitialized, func);
823 if (lck->lk.self != lck) {
824 KMP_FATAL(LockIsUninitialized, func);
826 if (!__kmp_is_ticket_lock_nestable(lck)) {
827 KMP_FATAL(LockSimpleUsedAsNestable, func);
829 return __kmp_acquire_nested_ticket_lock(lck, gtid);
832 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
835 KMP_DEBUG_ASSERT(gtid >= 0);
837 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
838 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
839 std::memory_order_relaxed) +
841 }
else if (!__kmp_test_ticket_lock(lck, gtid)) {
844 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
845 std::memory_order_relaxed);
846 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
847 std::memory_order_relaxed);
853 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
855 char const *
const func =
"omp_test_nest_lock";
857 if (!std::atomic_load_explicit(&lck->lk.initialized,
858 std::memory_order_relaxed)) {
859 KMP_FATAL(LockIsUninitialized, func);
861 if (lck->lk.self != lck) {
862 KMP_FATAL(LockIsUninitialized, func);
864 if (!__kmp_is_ticket_lock_nestable(lck)) {
865 KMP_FATAL(LockSimpleUsedAsNestable, func);
867 return __kmp_test_nested_ticket_lock(lck, gtid);
870 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
871 KMP_DEBUG_ASSERT(gtid >= 0);
873 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
874 std::memory_order_relaxed) -
876 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
877 __kmp_release_ticket_lock(lck, gtid);
878 return KMP_LOCK_RELEASED;
880 return KMP_LOCK_STILL_HELD;
883 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
885 char const *
const func =
"omp_unset_nest_lock";
887 if (!std::atomic_load_explicit(&lck->lk.initialized,
888 std::memory_order_relaxed)) {
889 KMP_FATAL(LockIsUninitialized, func);
891 if (lck->lk.self != lck) {
892 KMP_FATAL(LockIsUninitialized, func);
894 if (!__kmp_is_ticket_lock_nestable(lck)) {
895 KMP_FATAL(LockSimpleUsedAsNestable, func);
897 if (__kmp_get_ticket_lock_owner(lck) == -1) {
898 KMP_FATAL(LockUnsettingFree, func);
900 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
901 KMP_FATAL(LockUnsettingSetByAnother, func);
903 return __kmp_release_nested_ticket_lock(lck, gtid);
906 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
907 __kmp_init_ticket_lock(lck);
908 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
909 std::memory_order_relaxed);
913 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
914 __kmp_destroy_ticket_lock(lck);
915 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
916 std::memory_order_relaxed);
920 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
921 char const *
const func =
"omp_destroy_nest_lock";
923 if (!std::atomic_load_explicit(&lck->lk.initialized,
924 std::memory_order_relaxed)) {
925 KMP_FATAL(LockIsUninitialized, func);
927 if (lck->lk.self != lck) {
928 KMP_FATAL(LockIsUninitialized, func);
930 if (!__kmp_is_ticket_lock_nestable(lck)) {
931 KMP_FATAL(LockSimpleUsedAsNestable, func);
933 if (__kmp_get_ticket_lock_owner(lck) != -1) {
934 KMP_FATAL(LockStillOwned, func);
936 __kmp_destroy_nested_ticket_lock(lck);
941 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
942 return lck->lk.location;
945 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
947 lck->lk.location = loc;
950 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
951 return lck->lk.flags;
954 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
955 kmp_lock_flags_t flags) {
956 lck->lk.flags = flags;
1014 #ifdef DEBUG_QUEUING_LOCKS
1017 #define TRACE_BUF_ELE 1024
1018 static char traces[TRACE_BUF_ELE][128] = {0};
1020 #define TRACE_LOCK(X, Y) \
1021 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y);
1022 #define TRACE_LOCK_T(X, Y, Z) \
1023 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X, Y, Z);
1024 #define TRACE_LOCK_HT(X, Y, Z, Q) \
1025 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, \
1028 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid,
1029 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1030 kmp_int32 tail_id) {
1033 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n");
1035 i = tc % TRACE_BUF_ELE;
1036 __kmp_printf_no_lock(
"%s\n", traces[i]);
1037 i = (i + 1) % TRACE_BUF_ELE;
1038 while (i != (tc % TRACE_BUF_ELE)) {
1039 __kmp_printf_no_lock(
"%s", traces[i]);
1040 i = (i + 1) % TRACE_BUF_ELE;
1042 __kmp_printf_no_lock(
"\n");
1044 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, "
1045 "next_wait:%d, head_id:%d, tail_id:%d\n",
1046 gtid + 1, this_thr->th.th_spin_here,
1047 this_thr->th.th_next_waiting, head_id, tail_id);
1049 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id);
1051 if (lck->lk.head_id >= 1) {
1052 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1054 __kmp_printf_no_lock(
"-> %d ", t);
1055 t = __kmp_threads[t - 1]->th.th_next_waiting;
1058 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id);
1059 __kmp_printf_no_lock(
"\n\n");
1064 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1065 return TCR_4(lck->lk.owner_id) - 1;
1068 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1069 return lck->lk.depth_locked != -1;
1073 template <
bool takeTime>
1076 __forceinline
static int
1077 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1079 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1080 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1081 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1082 volatile kmp_uint32 *spin_here_p;
1085 ompt_state_t prev_state = ompt_state_undefined;
1089 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1091 KMP_FSYNC_PREPARE(lck);
1092 KMP_DEBUG_ASSERT(this_thr != NULL);
1093 spin_here_p = &this_thr->th.th_spin_here;
1095 #ifdef DEBUG_QUEUING_LOCKS
1096 TRACE_LOCK(gtid + 1,
"acq ent");
1098 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1099 if (this_thr->th.th_next_waiting != 0)
1100 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1102 KMP_DEBUG_ASSERT(!*spin_here_p);
1103 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1112 *spin_here_p = TRUE;
1124 #ifdef DEBUG_QUEUING_LOCKS
1126 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1134 enqueued = KMP_COMPARE_AND_STORE_ACQ64((
volatile kmp_int64 *)tail_id_p,
1136 KMP_PACK_64(gtid + 1, gtid + 1));
1137 #ifdef DEBUG_QUEUING_LOCKS
1139 TRACE_LOCK(gtid + 1,
"acq enq: (-1,0)->(tid,tid)");
1145 KMP_DEBUG_ASSERT(tail != gtid + 1);
1147 #ifdef DEBUG_QUEUING_LOCKS
1148 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1155 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1);
1157 #ifdef DEBUG_QUEUING_LOCKS
1159 TRACE_LOCK(gtid + 1,
"acq enq: (h,t)->(h,tid)");
1166 kmp_int32 grabbed_lock;
1168 #ifdef DEBUG_QUEUING_LOCKS
1170 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1176 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1);
1180 *spin_here_p = FALSE;
1184 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1186 #ifdef DEBUG_QUEUING_LOCKS
1187 TRACE_LOCK_HT(gtid + 1,
"acq exit: ", head, 0);
1191 if (ompt_enabled.enabled && prev_state != ompt_state_undefined) {
1193 this_thr->th.ompt_thread_info.state = prev_state;
1194 this_thr->th.ompt_thread_info.wait_id = 0;
1198 KMP_FSYNC_ACQUIRED(lck);
1199 return KMP_LOCK_ACQUIRED_FIRST;
1206 if (ompt_enabled.enabled && prev_state == ompt_state_undefined) {
1208 prev_state = this_thr->th.ompt_thread_info.state;
1209 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1210 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1216 kmp_info_t *tail_thr = __kmp_thread_from_gtid(tail - 1);
1217 KMP_ASSERT(tail_thr != NULL);
1218 tail_thr->th.th_next_waiting = gtid + 1;
1222 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1227 KMP_WAIT(spin_here_p, FALSE, KMP_EQ, lck);
1232 #ifdef DEBUG_QUEUING_LOCKS
1233 TRACE_LOCK(gtid + 1,
"acq spin");
1235 if (this_thr->th.th_next_waiting != 0)
1236 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1238 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1239 KA_TRACE(1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after "
1240 "waiting on queue\n",
1243 #ifdef DEBUG_QUEUING_LOCKS
1244 TRACE_LOCK(gtid + 1,
"acq exit 2");
1249 this_thr->th.ompt_thread_info.state = prev_state;
1250 this_thr->th.ompt_thread_info.wait_id = 0;
1254 return KMP_LOCK_ACQUIRED_FIRST;
1260 KMP_YIELD_OVERSUB();
1262 #ifdef DEBUG_QUEUING_LOCKS
1263 TRACE_LOCK(gtid + 1,
"acq retry");
1266 KMP_ASSERT2(0,
"should not get here");
1267 return KMP_LOCK_ACQUIRED_FIRST;
1270 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1271 KMP_DEBUG_ASSERT(gtid >= 0);
1273 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1277 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1279 char const *
const func =
"omp_set_lock";
1280 if (lck->lk.initialized != lck) {
1281 KMP_FATAL(LockIsUninitialized, func);
1283 if (__kmp_is_queuing_lock_nestable(lck)) {
1284 KMP_FATAL(LockNestableUsedAsSimple, func);
1286 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1287 KMP_FATAL(LockIsAlreadyOwned, func);
1290 __kmp_acquire_queuing_lock(lck, gtid);
1292 lck->lk.owner_id = gtid + 1;
1293 return KMP_LOCK_ACQUIRED_FIRST;
1296 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1297 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1300 kmp_info_t *this_thr;
1303 KA_TRACE(1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid));
1304 KMP_DEBUG_ASSERT(gtid >= 0);
1306 this_thr = __kmp_thread_from_gtid(gtid);
1307 KMP_DEBUG_ASSERT(this_thr != NULL);
1308 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1315 if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
1317 (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
1318 KMP_FSYNC_ACQUIRED(lck);
1324 (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
1328 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1330 char const *
const func =
"omp_test_lock";
1331 if (lck->lk.initialized != lck) {
1332 KMP_FATAL(LockIsUninitialized, func);
1334 if (__kmp_is_queuing_lock_nestable(lck)) {
1335 KMP_FATAL(LockNestableUsedAsSimple, func);
1338 int retval = __kmp_test_queuing_lock(lck, gtid);
1341 lck->lk.owner_id = gtid + 1;
1346 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1347 kmp_info_t *this_thr;
1348 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1349 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1352 (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1353 KMP_DEBUG_ASSERT(gtid >= 0);
1354 this_thr = __kmp_thread_from_gtid(gtid);
1355 KMP_DEBUG_ASSERT(this_thr != NULL);
1356 #ifdef DEBUG_QUEUING_LOCKS
1357 TRACE_LOCK(gtid + 1,
"rel ent");
1359 if (this_thr->th.th_spin_here)
1360 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1361 if (this_thr->th.th_next_waiting != 0)
1362 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1364 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1365 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1367 KMP_FSYNC_RELEASING(lck);
1376 #ifdef DEBUG_QUEUING_LOCKS
1378 TRACE_LOCK_HT(gtid + 1,
"rel read: ", head, tail);
1380 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1382 KMP_DEBUG_ASSERT(head !=
1387 if (KMP_COMPARE_AND_STORE_REL32(head_id_p, -1, 0)) {
1390 (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1392 #ifdef DEBUG_QUEUING_LOCKS
1393 TRACE_LOCK_HT(gtid + 1,
"rel exit: ", 0, 0);
1399 return KMP_LOCK_RELEASED;
1406 #ifdef DEBUG_QUEUING_LOCKS
1408 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1410 KMP_DEBUG_ASSERT(head > 0);
1413 dequeued = KMP_COMPARE_AND_STORE_REL64(
1414 RCAST(
volatile kmp_int64 *, tail_id_p), KMP_PACK_64(head, head),
1415 KMP_PACK_64(-1, 0));
1416 #ifdef DEBUG_QUEUING_LOCKS
1417 TRACE_LOCK(gtid + 1,
"rel deq: (h,h)->(-1,0)");
1421 volatile kmp_int32 *waiting_id_p;
1422 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1423 KMP_DEBUG_ASSERT(head_thr != NULL);
1424 waiting_id_p = &head_thr->th.th_next_waiting;
1427 #ifdef DEBUG_QUEUING_LOCKS
1428 if (head <= 0 || tail <= 0)
1429 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1431 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1438 KMP_WAIT((
volatile kmp_uint32 *)waiting_id_p, 0, KMP_NEQ, NULL);
1439 #ifdef DEBUG_QUEUING_LOCKS
1440 TRACE_LOCK(gtid + 1,
"rel deq: (h,t)->(h',t)");
1447 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1448 KMP_DEBUG_ASSERT(head_thr != NULL);
1451 #ifdef DEBUG_QUEUING_LOCKS
1452 if (head <= 0 || tail <= 0)
1453 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1455 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1459 head_thr->th.th_next_waiting = 0;
1460 #ifdef DEBUG_QUEUING_LOCKS
1461 TRACE_LOCK_T(gtid + 1,
"rel nw=0 for t=", head);
1466 head_thr->th.th_spin_here = FALSE;
1468 KA_TRACE(1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after "
1471 #ifdef DEBUG_QUEUING_LOCKS
1472 TRACE_LOCK(gtid + 1,
"rel exit 2");
1474 return KMP_LOCK_RELEASED;
1479 #ifdef DEBUG_QUEUING_LOCKS
1480 TRACE_LOCK(gtid + 1,
"rel retry");
1484 KMP_ASSERT2(0,
"should not get here");
1485 return KMP_LOCK_RELEASED;
1488 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1490 char const *
const func =
"omp_unset_lock";
1492 if (lck->lk.initialized != lck) {
1493 KMP_FATAL(LockIsUninitialized, func);
1495 if (__kmp_is_queuing_lock_nestable(lck)) {
1496 KMP_FATAL(LockNestableUsedAsSimple, func);
1498 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1499 KMP_FATAL(LockUnsettingFree, func);
1501 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1502 KMP_FATAL(LockUnsettingSetByAnother, func);
1504 lck->lk.owner_id = 0;
1505 return __kmp_release_queuing_lock(lck, gtid);
1508 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1509 lck->lk.location = NULL;
1510 lck->lk.head_id = 0;
1511 lck->lk.tail_id = 0;
1512 lck->lk.next_ticket = 0;
1513 lck->lk.now_serving = 0;
1514 lck->lk.owner_id = 0;
1515 lck->lk.depth_locked = -1;
1516 lck->lk.initialized = lck;
1518 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1521 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1522 lck->lk.initialized = NULL;
1523 lck->lk.location = NULL;
1524 lck->lk.head_id = 0;
1525 lck->lk.tail_id = 0;
1526 lck->lk.next_ticket = 0;
1527 lck->lk.now_serving = 0;
1528 lck->lk.owner_id = 0;
1529 lck->lk.depth_locked = -1;
1532 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1533 char const *
const func =
"omp_destroy_lock";
1534 if (lck->lk.initialized != lck) {
1535 KMP_FATAL(LockIsUninitialized, func);
1537 if (__kmp_is_queuing_lock_nestable(lck)) {
1538 KMP_FATAL(LockNestableUsedAsSimple, func);
1540 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1541 KMP_FATAL(LockStillOwned, func);
1543 __kmp_destroy_queuing_lock(lck);
1548 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1549 KMP_DEBUG_ASSERT(gtid >= 0);
1551 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1552 lck->lk.depth_locked += 1;
1553 return KMP_LOCK_ACQUIRED_NEXT;
1555 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1557 lck->lk.depth_locked = 1;
1559 lck->lk.owner_id = gtid + 1;
1560 return KMP_LOCK_ACQUIRED_FIRST;
1565 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1567 char const *
const func =
"omp_set_nest_lock";
1568 if (lck->lk.initialized != lck) {
1569 KMP_FATAL(LockIsUninitialized, func);
1571 if (!__kmp_is_queuing_lock_nestable(lck)) {
1572 KMP_FATAL(LockSimpleUsedAsNestable, func);
1574 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1577 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1580 KMP_DEBUG_ASSERT(gtid >= 0);
1582 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1583 retval = ++lck->lk.depth_locked;
1584 }
else if (!__kmp_test_queuing_lock(lck, gtid)) {
1588 retval = lck->lk.depth_locked = 1;
1590 lck->lk.owner_id = gtid + 1;
1595 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1597 char const *
const func =
"omp_test_nest_lock";
1598 if (lck->lk.initialized != lck) {
1599 KMP_FATAL(LockIsUninitialized, func);
1601 if (!__kmp_is_queuing_lock_nestable(lck)) {
1602 KMP_FATAL(LockSimpleUsedAsNestable, func);
1604 return __kmp_test_nested_queuing_lock(lck, gtid);
1607 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1608 KMP_DEBUG_ASSERT(gtid >= 0);
1611 if (--(lck->lk.depth_locked) == 0) {
1613 lck->lk.owner_id = 0;
1614 __kmp_release_queuing_lock(lck, gtid);
1615 return KMP_LOCK_RELEASED;
1617 return KMP_LOCK_STILL_HELD;
1621 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1623 char const *
const func =
"omp_unset_nest_lock";
1625 if (lck->lk.initialized != lck) {
1626 KMP_FATAL(LockIsUninitialized, func);
1628 if (!__kmp_is_queuing_lock_nestable(lck)) {
1629 KMP_FATAL(LockSimpleUsedAsNestable, func);
1631 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1632 KMP_FATAL(LockUnsettingFree, func);
1634 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1635 KMP_FATAL(LockUnsettingSetByAnother, func);
1637 return __kmp_release_nested_queuing_lock(lck, gtid);
1640 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1641 __kmp_init_queuing_lock(lck);
1642 lck->lk.depth_locked = 0;
1645 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1646 __kmp_destroy_queuing_lock(lck);
1647 lck->lk.depth_locked = 0;
1651 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1652 char const *
const func =
"omp_destroy_nest_lock";
1653 if (lck->lk.initialized != lck) {
1654 KMP_FATAL(LockIsUninitialized, func);
1656 if (!__kmp_is_queuing_lock_nestable(lck)) {
1657 KMP_FATAL(LockSimpleUsedAsNestable, func);
1659 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1660 KMP_FATAL(LockStillOwned, func);
1662 __kmp_destroy_nested_queuing_lock(lck);
1667 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1668 return lck->lk.location;
1671 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1673 lck->lk.location = loc;
1676 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1677 return lck->lk.flags;
1680 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1681 kmp_lock_flags_t flags) {
1682 lck->lk.flags = flags;
1685 #if KMP_USE_ADAPTIVE_LOCKS
1689 #if KMP_HAVE_RTM_INTRINSICS
1690 #include <immintrin.h>
1691 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1696 #define _XBEGIN_STARTED (~0u)
1697 #define _XABORT_EXPLICIT (1 << 0)
1698 #define _XABORT_RETRY (1 << 1)
1699 #define _XABORT_CONFLICT (1 << 2)
1700 #define _XABORT_CAPACITY (1 << 3)
1701 #define _XABORT_DEBUG (1 << 4)
1702 #define _XABORT_NESTED (1 << 5)
1703 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1706 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1708 #define STRINGIZE_INTERNAL(arg) #arg
1709 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1715 static __inline
int _xbegin() {
1752 __asm__
volatile(
"1: .byte 0xC7; .byte 0xF8;\n"
1755 "1: movl %%eax,%0\n"
1757 :
"+r"(res)::
"memory",
"%eax");
1763 static __inline
void _xend() {
1771 __asm__
volatile(
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1780 #define _xabort(ARG) _asm _emit 0xc6 _asm _emit 0xf8 _asm _emit ARG
1782 #define _xabort(ARG) \
1783 __asm__ volatile(".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG):::"memory");
1789 #if KMP_DEBUG_ADAPTIVE_LOCKS
1794 static kmp_adaptive_lock_statistics_t destroyedStats;
1797 static kmp_adaptive_lock_info_t liveLocks;
1800 static kmp_bootstrap_lock_t chain_lock =
1801 KMP_BOOTSTRAP_LOCK_INITIALIZER(chain_lock);
1804 void __kmp_init_speculative_stats() {
1805 kmp_adaptive_lock_info_t *lck = &liveLocks;
1807 memset(CCAST(kmp_adaptive_lock_statistics_t *, &(lck->stats)), 0,
1808 sizeof(lck->stats));
1809 lck->stats.next = lck;
1810 lck->stats.prev = lck;
1812 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1813 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1815 __kmp_init_bootstrap_lock(&chain_lock);
1819 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1820 __kmp_acquire_bootstrap_lock(&chain_lock);
1822 lck->stats.next = liveLocks.stats.next;
1823 lck->stats.prev = &liveLocks;
1825 liveLocks.stats.next = lck;
1826 lck->stats.next->stats.prev = lck;
1828 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1829 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1831 __kmp_release_bootstrap_lock(&chain_lock);
1834 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1835 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1836 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1838 kmp_adaptive_lock_info_t *n = lck->stats.next;
1839 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1845 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1846 memset(CCAST(kmp_adaptive_lock_statistics_t *, &lck->stats), 0,
1847 sizeof(lck->stats));
1848 __kmp_remember_lock(lck);
1851 static void __kmp_add_stats(kmp_adaptive_lock_statistics_t *t,
1852 kmp_adaptive_lock_info_t *lck) {
1853 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
1855 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1856 t->successfulSpeculations += s->successfulSpeculations;
1857 t->hardFailedSpeculations += s->hardFailedSpeculations;
1858 t->softFailedSpeculations += s->softFailedSpeculations;
1859 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
1860 t->lemmingYields += s->lemmingYields;
1863 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1864 __kmp_acquire_bootstrap_lock(&chain_lock);
1866 __kmp_add_stats(&destroyedStats, lck);
1867 __kmp_forget_lock(lck);
1869 __kmp_release_bootstrap_lock(&chain_lock);
1872 static float percent(kmp_uint32 count, kmp_uint32 total) {
1873 return (total == 0) ? 0.0 : (100.0 * count) / total;
1876 void __kmp_print_speculative_stats() {
1877 kmp_adaptive_lock_statistics_t total = destroyedStats;
1878 kmp_adaptive_lock_info_t *lck;
1880 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1881 __kmp_add_stats(&total, lck);
1883 kmp_adaptive_lock_statistics_t *t = &total;
1884 kmp_uint32 totalSections =
1885 t->nonSpeculativeAcquires + t->successfulSpeculations;
1886 kmp_uint32 totalSpeculations = t->successfulSpeculations +
1887 t->hardFailedSpeculations +
1888 t->softFailedSpeculations;
1889 if (totalSections <= 0)
1893 if (strcmp(__kmp_speculative_statsfile,
"-") == 0) {
1896 size_t buffLen = KMP_STRLEN(__kmp_speculative_statsfile) + 20;
1897 char buffer[buffLen];
1898 KMP_SNPRINTF(&buffer[0], buffLen, __kmp_speculative_statsfile,
1899 (kmp_int32)getpid());
1900 statsFile.
open(buffer,
"w");
1903 fprintf(statsFile,
"Speculative lock statistics (all approximate!)\n");
1905 " Lock parameters: \n"
1906 " max_soft_retries : %10d\n"
1907 " max_badness : %10d\n",
1908 __kmp_adaptive_backoff_params.max_soft_retries,
1909 __kmp_adaptive_backoff_params.max_badness);
1910 fprintf(statsFile,
" Non-speculative acquire attempts : %10d\n",
1911 t->nonSpeculativeAcquireAttempts);
1912 fprintf(statsFile,
" Total critical sections : %10d\n",
1914 fprintf(statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
1915 t->successfulSpeculations,
1916 percent(t->successfulSpeculations, totalSections));
1917 fprintf(statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
1918 t->nonSpeculativeAcquires,
1919 percent(t->nonSpeculativeAcquires, totalSections));
1920 fprintf(statsFile,
" Lemming yields : %10d\n\n",
1923 fprintf(statsFile,
" Speculative acquire attempts : %10d\n",
1925 fprintf(statsFile,
" Successes : %10d (%5.1f%%)\n",
1926 t->successfulSpeculations,
1927 percent(t->successfulSpeculations, totalSpeculations));
1928 fprintf(statsFile,
" Soft failures : %10d (%5.1f%%)\n",
1929 t->softFailedSpeculations,
1930 percent(t->softFailedSpeculations, totalSpeculations));
1931 fprintf(statsFile,
" Hard failures : %10d (%5.1f%%)\n",
1932 t->hardFailedSpeculations,
1933 percent(t->hardFailedSpeculations, totalSpeculations));
1936 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++)
1938 #define KMP_INC_STAT(lck, stat)
1942 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
1945 bool res = lck->lk.head_id == 0;
1949 #if KMP_COMPILER_ICC
1952 __sync_synchronize();
1959 static __inline
void
1960 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
1962 lck->lk.adaptive.badness = 0;
1963 KMP_INC_STAT(lck, successfulSpeculations);
1967 static __inline
void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
1968 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
1969 if (newBadness > lck->lk.adaptive.max_badness) {
1972 lck->lk.adaptive.badness = newBadness;
1977 KMP_ATTRIBUTE_TARGET_RTM
1978 static __inline
int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
1980 kmp_uint32 badness = lck->lk.adaptive.badness;
1981 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
1982 int res = (attempts & badness) == 0;
1988 KMP_ATTRIBUTE_TARGET_RTM
1989 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
1991 int retries = lck->lk.adaptive.max_soft_retries;
1998 kmp_uint32 status = _xbegin();
2003 if (status == _XBEGIN_STARTED) {
2008 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2012 KMP_ASSERT2(0,
"should not get here");
2017 if (status & SOFT_ABORT_MASK) {
2018 KMP_INC_STAT(lck, softFailedSpeculations);
2021 KMP_INC_STAT(lck, hardFailedSpeculations);
2026 }
while (retries--);
2030 __kmp_step_badness(lck);
2037 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2039 if (__kmp_should_speculate(lck, gtid) &&
2040 __kmp_test_adaptive_lock_only(lck, gtid))
2045 lck->lk.adaptive.acquire_attempts++;
2048 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2049 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2056 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2058 char const *
const func =
"omp_test_lock";
2059 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2060 KMP_FATAL(LockIsUninitialized, func);
2063 int retval = __kmp_test_adaptive_lock(lck, gtid);
2066 lck->lk.qlk.owner_id = gtid + 1;
2082 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2084 if (__kmp_should_speculate(lck, gtid)) {
2085 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2086 if (__kmp_test_adaptive_lock_only(lck, gtid))
2095 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2096 KMP_INC_STAT(lck, lemmingYields);
2100 if (__kmp_test_adaptive_lock_only(lck, gtid))
2107 lck->lk.adaptive.acquire_attempts++;
2109 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2111 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2114 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2116 char const *
const func =
"omp_set_lock";
2117 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2118 KMP_FATAL(LockIsUninitialized, func);
2120 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2121 KMP_FATAL(LockIsAlreadyOwned, func);
2124 __kmp_acquire_adaptive_lock(lck, gtid);
2126 lck->lk.qlk.owner_id = gtid + 1;
2129 KMP_ATTRIBUTE_TARGET_RTM
2130 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2132 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
2137 __kmp_update_badness_after_success(lck);
2140 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2142 return KMP_LOCK_RELEASED;
2145 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2147 char const *
const func =
"omp_unset_lock";
2149 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2150 KMP_FATAL(LockIsUninitialized, func);
2152 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2153 KMP_FATAL(LockUnsettingFree, func);
2155 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2156 KMP_FATAL(LockUnsettingSetByAnother, func);
2158 lck->lk.qlk.owner_id = 0;
2159 __kmp_release_adaptive_lock(lck, gtid);
2160 return KMP_LOCK_RELEASED;
2163 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2164 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2165 lck->lk.adaptive.badness = 0;
2166 lck->lk.adaptive.acquire_attempts = 0;
2167 lck->lk.adaptive.max_soft_retries =
2168 __kmp_adaptive_backoff_params.max_soft_retries;
2169 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2170 #if KMP_DEBUG_ADAPTIVE_LOCKS
2171 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2173 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2176 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2177 #if KMP_DEBUG_ADAPTIVE_LOCKS
2178 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2180 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2184 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2185 char const *
const func =
"omp_destroy_lock";
2186 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2187 KMP_FATAL(LockIsUninitialized, func);
2189 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2190 KMP_FATAL(LockStillOwned, func);
2192 __kmp_destroy_adaptive_lock(lck);
2201 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2202 return lck->lk.owner_id - 1;
2205 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2206 return lck->lk.depth_locked != -1;
2209 __forceinline
static int
2210 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2211 kmp_uint64 ticket = KMP_ATOMIC_INC(&lck->lk.next_ticket);
2212 kmp_uint64 mask = lck->lk.mask;
2213 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2215 #ifdef USE_LOCK_PROFILE
2216 if (polls[ticket & mask] != ticket)
2217 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2229 KMP_FSYNC_PREPARE(lck);
2230 KMP_INIT_YIELD(spins);
2231 while (polls[ticket & mask] < ticket) {
2232 KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
2240 mask = lck->lk.mask;
2241 polls = lck->lk.polls;
2245 KMP_FSYNC_ACQUIRED(lck);
2246 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2248 lck->lk.now_serving = ticket;
2255 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2256 __kmp_free(lck->lk.old_polls);
2257 lck->lk.old_polls = NULL;
2258 lck->lk.cleanup_ticket = 0;
2264 if (lck->lk.old_polls == NULL) {
2265 bool reconfigure =
false;
2266 std::atomic<kmp_uint64> *old_polls = polls;
2267 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2269 if (TCR_4(__kmp_nth) >
2270 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2273 if (num_polls > 1) {
2275 num_polls = TCR_4(lck->lk.num_polls);
2278 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2286 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2287 if (num_waiting > num_polls) {
2288 kmp_uint32 old_num_polls = num_polls;
2291 mask = (mask << 1) | 1;
2293 }
while (num_polls <= num_waiting);
2299 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2302 for (i = 0; i < old_num_polls; i++) {
2303 polls[i].store(old_polls[i]);
2318 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring "
2319 "lock %p to %d polls\n",
2320 ticket, lck, num_polls));
2322 lck->lk.old_polls = old_polls;
2323 lck->lk.polls = polls;
2327 lck->lk.num_polls = num_polls;
2328 lck->lk.mask = mask;
2336 lck->lk.cleanup_ticket = lck->lk.next_ticket;
2339 return KMP_LOCK_ACQUIRED_FIRST;
2342 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2343 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2347 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2349 char const *
const func =
"omp_set_lock";
2350 if (lck->lk.initialized != lck) {
2351 KMP_FATAL(LockIsUninitialized, func);
2353 if (__kmp_is_drdpa_lock_nestable(lck)) {
2354 KMP_FATAL(LockNestableUsedAsSimple, func);
2356 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2357 KMP_FATAL(LockIsAlreadyOwned, func);
2360 __kmp_acquire_drdpa_lock(lck, gtid);
2362 lck->lk.owner_id = gtid + 1;
2363 return KMP_LOCK_ACQUIRED_FIRST;
2366 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2369 kmp_uint64 ticket = lck->lk.next_ticket;
2370 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2371 kmp_uint64 mask = lck->lk.mask;
2372 if (polls[ticket & mask] == ticket) {
2373 kmp_uint64 next_ticket = ticket + 1;
2374 if (__kmp_atomic_compare_store_acq(&lck->lk.next_ticket, ticket,
2376 KMP_FSYNC_ACQUIRED(lck);
2377 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2379 lck->lk.now_serving = ticket;
2393 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2395 char const *
const func =
"omp_test_lock";
2396 if (lck->lk.initialized != lck) {
2397 KMP_FATAL(LockIsUninitialized, func);
2399 if (__kmp_is_drdpa_lock_nestable(lck)) {
2400 KMP_FATAL(LockNestableUsedAsSimple, func);
2403 int retval = __kmp_test_drdpa_lock(lck, gtid);
2406 lck->lk.owner_id = gtid + 1;
2411 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2414 kmp_uint64 ticket = lck->lk.now_serving + 1;
2415 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2416 kmp_uint64 mask = lck->lk.mask;
2417 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2419 KMP_FSYNC_RELEASING(lck);
2420 polls[ticket & mask] = ticket;
2421 return KMP_LOCK_RELEASED;
2424 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2426 char const *
const func =
"omp_unset_lock";
2428 if (lck->lk.initialized != lck) {
2429 KMP_FATAL(LockIsUninitialized, func);
2431 if (__kmp_is_drdpa_lock_nestable(lck)) {
2432 KMP_FATAL(LockNestableUsedAsSimple, func);
2434 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2435 KMP_FATAL(LockUnsettingFree, func);
2437 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2438 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2439 KMP_FATAL(LockUnsettingSetByAnother, func);
2441 lck->lk.owner_id = 0;
2442 return __kmp_release_drdpa_lock(lck, gtid);
2445 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2446 lck->lk.location = NULL;
2448 lck->lk.num_polls = 1;
2449 lck->lk.polls = (std::atomic<kmp_uint64> *)__kmp_allocate(
2450 lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2451 lck->lk.cleanup_ticket = 0;
2452 lck->lk.old_polls = NULL;
2453 lck->lk.next_ticket = 0;
2454 lck->lk.now_serving = 0;
2455 lck->lk.owner_id = 0;
2456 lck->lk.depth_locked = -1;
2457 lck->lk.initialized = lck;
2459 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2462 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2463 lck->lk.initialized = NULL;
2464 lck->lk.location = NULL;
2465 if (lck->lk.polls.load() != NULL) {
2466 __kmp_free(lck->lk.polls.load());
2467 lck->lk.polls = NULL;
2469 if (lck->lk.old_polls != NULL) {
2470 __kmp_free(lck->lk.old_polls);
2471 lck->lk.old_polls = NULL;
2474 lck->lk.num_polls = 0;
2475 lck->lk.cleanup_ticket = 0;
2476 lck->lk.next_ticket = 0;
2477 lck->lk.now_serving = 0;
2478 lck->lk.owner_id = 0;
2479 lck->lk.depth_locked = -1;
2482 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2483 char const *
const func =
"omp_destroy_lock";
2484 if (lck->lk.initialized != lck) {
2485 KMP_FATAL(LockIsUninitialized, func);
2487 if (__kmp_is_drdpa_lock_nestable(lck)) {
2488 KMP_FATAL(LockNestableUsedAsSimple, func);
2490 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2491 KMP_FATAL(LockStillOwned, func);
2493 __kmp_destroy_drdpa_lock(lck);
2498 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2499 KMP_DEBUG_ASSERT(gtid >= 0);
2501 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2502 lck->lk.depth_locked += 1;
2503 return KMP_LOCK_ACQUIRED_NEXT;
2505 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2507 lck->lk.depth_locked = 1;
2509 lck->lk.owner_id = gtid + 1;
2510 return KMP_LOCK_ACQUIRED_FIRST;
2514 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2516 char const *
const func =
"omp_set_nest_lock";
2517 if (lck->lk.initialized != lck) {
2518 KMP_FATAL(LockIsUninitialized, func);
2520 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2521 KMP_FATAL(LockSimpleUsedAsNestable, func);
2523 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2526 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2529 KMP_DEBUG_ASSERT(gtid >= 0);
2531 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2532 retval = ++lck->lk.depth_locked;
2533 }
else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2537 retval = lck->lk.depth_locked = 1;
2539 lck->lk.owner_id = gtid + 1;
2544 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2546 char const *
const func =
"omp_test_nest_lock";
2547 if (lck->lk.initialized != lck) {
2548 KMP_FATAL(LockIsUninitialized, func);
2550 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2551 KMP_FATAL(LockSimpleUsedAsNestable, func);
2553 return __kmp_test_nested_drdpa_lock(lck, gtid);
2556 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2557 KMP_DEBUG_ASSERT(gtid >= 0);
2560 if (--(lck->lk.depth_locked) == 0) {
2562 lck->lk.owner_id = 0;
2563 __kmp_release_drdpa_lock(lck, gtid);
2564 return KMP_LOCK_RELEASED;
2566 return KMP_LOCK_STILL_HELD;
2569 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2571 char const *
const func =
"omp_unset_nest_lock";
2573 if (lck->lk.initialized != lck) {
2574 KMP_FATAL(LockIsUninitialized, func);
2576 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2577 KMP_FATAL(LockSimpleUsedAsNestable, func);
2579 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2580 KMP_FATAL(LockUnsettingFree, func);
2582 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2583 KMP_FATAL(LockUnsettingSetByAnother, func);
2585 return __kmp_release_nested_drdpa_lock(lck, gtid);
2588 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2589 __kmp_init_drdpa_lock(lck);
2590 lck->lk.depth_locked = 0;
2593 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2594 __kmp_destroy_drdpa_lock(lck);
2595 lck->lk.depth_locked = 0;
2598 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2599 char const *
const func =
"omp_destroy_nest_lock";
2600 if (lck->lk.initialized != lck) {
2601 KMP_FATAL(LockIsUninitialized, func);
2603 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2604 KMP_FATAL(LockSimpleUsedAsNestable, func);
2606 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2607 KMP_FATAL(LockStillOwned, func);
2609 __kmp_destroy_nested_drdpa_lock(lck);
2614 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2615 return lck->lk.location;
2618 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2620 lck->lk.location = loc;
2623 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2624 return lck->lk.flags;
2627 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2628 kmp_lock_flags_t flags) {
2629 lck->lk.flags = flags;
2633 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2634 #define __kmp_tsc() __kmp_hardware_timestamp()
2636 kmp_backoff_t __kmp_spin_backoff_params = {1, 4096, 100};
2639 extern kmp_uint64 __kmp_now_nsec();
2640 kmp_backoff_t __kmp_spin_backoff_params = {1, 256, 100};
2641 #define __kmp_tsc() __kmp_now_nsec()
2651 static inline bool before(kmp_uint64 a, kmp_uint64 b) {
2652 return ((kmp_int64)b - (kmp_int64)a) > 0;
2656 void __kmp_spin_backoff(kmp_backoff_t *boff) {
2659 for (i = boff->step; i > 0; i--) {
2660 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
2663 }
while (before(__kmp_tsc(), goal));
2665 boff->step = (boff->step << 1 | 1) & (boff->max_backoff - 1);
2668 #if KMP_USE_DYNAMIC_LOCK
2672 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2673 kmp_dyna_lockseq_t seq) {
2674 TCW_4(*lck, KMP_GET_D_TAG(seq));
2677 (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
2683 #define HLE_ACQUIRE ".byte 0xf2;"
2684 #define HLE_RELEASE ".byte 0xf3;"
2686 static inline kmp_uint32 swap4(kmp_uint32
volatile *p, kmp_uint32 v) {
2687 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" :
"+r"(v),
"+m"(*p) : :
"memory");
2691 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2693 static void __kmp_destroy_hle_lock_with_checks(kmp_dyna_lock_t *lck) {
2697 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2699 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2702 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
2703 for (
int i = delay; i != 0; --i)
2705 delay = ((delay << 1) | 1) & 7;
2707 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2711 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2713 __kmp_acquire_hle_lock(lck, gtid);
2716 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2717 __asm__
volatile(HLE_RELEASE
"movl %1,%0"
2719 :
"r"(KMP_LOCK_FREE(hle))
2721 return KMP_LOCK_RELEASED;
2724 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2726 return __kmp_release_hle_lock(lck, gtid);
2729 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2730 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2733 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2735 return __kmp_test_hle_lock(lck, gtid);
2738 static void __kmp_init_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2739 __kmp_init_queuing_lock(lck);
2742 static void __kmp_destroy_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2743 __kmp_destroy_queuing_lock(lck);
2747 __kmp_destroy_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
2748 __kmp_destroy_queuing_lock_with_checks(lck);
2751 KMP_ATTRIBUTE_TARGET_RTM
2752 static void __kmp_acquire_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2754 unsigned retries = 3, status;
2757 if (status == _XBEGIN_STARTED) {
2758 if (__kmp_is_unlocked_queuing_lock(lck))
2762 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2764 while (!__kmp_is_unlocked_queuing_lock(lck)) {
2767 }
else if (!(status & _XABORT_RETRY))
2769 }
while (retries--);
2772 __kmp_acquire_queuing_lock(lck, gtid);
2775 static void __kmp_acquire_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2777 __kmp_acquire_rtm_queuing_lock(lck, gtid);
2780 KMP_ATTRIBUTE_TARGET_RTM
2781 static int __kmp_release_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2783 if (__kmp_is_unlocked_queuing_lock(lck)) {
2788 __kmp_release_queuing_lock(lck, gtid);
2790 return KMP_LOCK_RELEASED;
2793 static int __kmp_release_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2795 return __kmp_release_rtm_queuing_lock(lck, gtid);
2798 KMP_ATTRIBUTE_TARGET_RTM
2799 static int __kmp_test_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2801 unsigned retries = 3, status;
2804 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2807 if (!(status & _XABORT_RETRY))
2809 }
while (retries--);
2811 return __kmp_test_queuing_lock(lck, gtid);
2814 static int __kmp_test_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2816 return __kmp_test_rtm_queuing_lock(lck, gtid);
2820 typedef kmp_tas_lock_t kmp_rtm_spin_lock_t;
2822 static void __kmp_destroy_rtm_spin_lock(kmp_rtm_spin_lock_t *lck) {
2823 KMP_ATOMIC_ST_REL(&lck->lk.poll, 0);
2826 static void __kmp_destroy_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck) {
2827 __kmp_destroy_rtm_spin_lock(lck);
2830 KMP_ATTRIBUTE_TARGET_RTM
2831 static int __kmp_acquire_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2833 unsigned retries = 3, status;
2834 kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
2835 kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
2838 if (status == _XBEGIN_STARTED) {
2839 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free)
2840 return KMP_LOCK_ACQUIRED_FIRST;
2843 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2845 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free) {
2848 }
else if (!(status & _XABORT_RETRY))
2850 }
while (retries--);
2853 KMP_FSYNC_PREPARE(lck);
2854 kmp_backoff_t backoff = __kmp_spin_backoff_params;
2855 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free ||
2856 !__kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2857 __kmp_spin_backoff(&backoff);
2859 KMP_FSYNC_ACQUIRED(lck);
2860 return KMP_LOCK_ACQUIRED_FIRST;
2863 static int __kmp_acquire_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2865 return __kmp_acquire_rtm_spin_lock(lck, gtid);
2868 KMP_ATTRIBUTE_TARGET_RTM
2869 static int __kmp_release_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2871 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == KMP_LOCK_FREE(rtm_spin)) {
2876 KMP_FSYNC_RELEASING(lck);
2877 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(rtm_spin));
2879 return KMP_LOCK_RELEASED;
2882 static int __kmp_release_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2884 return __kmp_release_rtm_spin_lock(lck, gtid);
2887 KMP_ATTRIBUTE_TARGET_RTM
2888 static int __kmp_test_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, kmp_int32 gtid) {
2889 unsigned retries = 3, status;
2890 kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
2891 kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
2894 if (status == _XBEGIN_STARTED &&
2895 KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free) {
2898 if (!(status & _XABORT_RETRY))
2900 }
while (retries--);
2902 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free &&
2903 __kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2904 KMP_FSYNC_ACQUIRED(lck);
2910 static int __kmp_test_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2912 return __kmp_test_rtm_spin_lock(lck, gtid);
2918 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *l,
2919 kmp_dyna_lockseq_t tag);
2920 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock);
2921 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2922 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2923 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2924 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2926 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2928 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2932 #define KMP_FOREACH_LOCK_KIND(m, a) m(ticket, a) m(queuing, a) m(drdpa, a)
2934 #define expand1(lk, op) \
2935 static void __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock) { \
2936 __kmp_##op##_##lk##_##lock(&lock->lk); \
2938 #define expand2(lk, op) \
2939 static int __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock, \
2941 return __kmp_##op##_##lk##_##lock(&lock->lk, gtid); \
2943 #define expand3(lk, op) \
2944 static void __kmp_set_##lk##_##lock_flags(kmp_user_lock_p lock, \
2945 kmp_lock_flags_t flags) { \
2946 __kmp_set_##lk##_lock_flags(&lock->lk, flags); \
2948 #define expand4(lk, op) \
2949 static void __kmp_set_##lk##_##lock_location(kmp_user_lock_p lock, \
2950 const ident_t *loc) { \
2951 __kmp_set_##lk##_lock_location(&lock->lk, loc); \
2954 KMP_FOREACH_LOCK_KIND(expand1, init)
2955 KMP_FOREACH_LOCK_KIND(expand1, init_nested)
2956 KMP_FOREACH_LOCK_KIND(expand1, destroy)
2957 KMP_FOREACH_LOCK_KIND(expand1, destroy_nested)
2958 KMP_FOREACH_LOCK_KIND(expand2, acquire)
2959 KMP_FOREACH_LOCK_KIND(expand2, acquire_nested)
2960 KMP_FOREACH_LOCK_KIND(expand2, release)
2961 KMP_FOREACH_LOCK_KIND(expand2, release_nested)
2962 KMP_FOREACH_LOCK_KIND(expand2, test)
2963 KMP_FOREACH_LOCK_KIND(expand2, test_nested)
2964 KMP_FOREACH_LOCK_KIND(expand3, )
2965 KMP_FOREACH_LOCK_KIND(expand4, )
2976 #define expand(l, op) 0, __kmp_init_direct_lock,
2977 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t) = {
2978 __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init)};
2982 #define expand(l, op) 0, (void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock,
2983 static void (*direct_destroy[])(kmp_dyna_lock_t *) = {
2984 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2986 #define expand(l, op) \
2987 0, (void (*)(kmp_dyna_lock_t *))__kmp_destroy_##l##_lock_with_checks,
2988 static void (*direct_destroy_check[])(kmp_dyna_lock_t *) = {
2989 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2993 #define expand(l, op) \
2994 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
2995 static int (*direct_set[])(kmp_dyna_lock_t *, kmp_int32) = {
2996 __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire)};
2998 #define expand(l, op) \
2999 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
3000 static int (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3001 __kmp_set_indirect_lock_with_checks, 0,
3002 KMP_FOREACH_D_LOCK(expand, acquire)};
3006 #define expand(l, op) \
3007 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
3008 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32) = {
3009 __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release)};
3010 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32) = {
3011 __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test)};
3013 #define expand(l, op) \
3014 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
3015 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3016 __kmp_unset_indirect_lock_with_checks, 0,
3017 KMP_FOREACH_D_LOCK(expand, release)};
3018 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3019 __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test)};
3023 void (**__kmp_direct_destroy)(kmp_dyna_lock_t *) = 0;
3024 int (**__kmp_direct_set)(kmp_dyna_lock_t *, kmp_int32) = 0;
3025 int (**__kmp_direct_unset)(kmp_dyna_lock_t *, kmp_int32) = 0;
3026 int (**__kmp_direct_test)(kmp_dyna_lock_t *, kmp_int32) = 0;
3029 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
3030 void (*__kmp_indirect_init[])(kmp_user_lock_p) = {
3031 KMP_FOREACH_I_LOCK(expand, init)};
3034 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
3035 static void (*indirect_destroy[])(kmp_user_lock_p) = {
3036 KMP_FOREACH_I_LOCK(expand, destroy)};
3038 #define expand(l, op) \
3039 (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock_with_checks,
3040 static void (*indirect_destroy_check[])(kmp_user_lock_p) = {
3041 KMP_FOREACH_I_LOCK(expand, destroy)};
3045 #define expand(l, op) \
3046 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
3047 static int (*indirect_set[])(kmp_user_lock_p,
3048 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, acquire)};
3050 #define expand(l, op) \
3051 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
3052 static int (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = {
3053 KMP_FOREACH_I_LOCK(expand, acquire)};
3057 #define expand(l, op) \
3058 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
3059 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = {
3060 KMP_FOREACH_I_LOCK(expand, release)};
3061 static int (*indirect_test[])(kmp_user_lock_p,
3062 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, test)};
3064 #define expand(l, op) \
3065 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
3066 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = {
3067 KMP_FOREACH_I_LOCK(expand, release)};
3068 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = {
3069 KMP_FOREACH_I_LOCK(expand, test)};
3073 void (**__kmp_indirect_destroy)(kmp_user_lock_p) = 0;
3074 int (**__kmp_indirect_set)(kmp_user_lock_p, kmp_int32) = 0;
3075 int (**__kmp_indirect_unset)(kmp_user_lock_p, kmp_int32) = 0;
3076 int (**__kmp_indirect_test)(kmp_user_lock_p, kmp_int32) = 0;
3079 kmp_indirect_lock_table_t __kmp_i_lock_table;
3082 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {0};
3085 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3087 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3088 kmp_lock_flags_t) = {0};
3089 const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
3090 kmp_user_lock_p) = {0};
3091 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
3092 kmp_user_lock_p) = {0};
3095 static kmp_indirect_lock_t *__kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = {0};
3102 kmp_indirect_lock_t *__kmp_allocate_indirect_lock(
void **user_lock,
3104 kmp_indirect_locktag_t tag) {
3105 kmp_indirect_lock_t *lck;
3106 kmp_lock_index_t idx;
3108 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3110 if (__kmp_indirect_lock_pool[tag] != NULL) {
3112 lck = __kmp_indirect_lock_pool[tag];
3113 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3114 idx = lck->lock->pool.index;
3115 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3116 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n",
3119 idx = __kmp_i_lock_table.next;
3121 if (idx == __kmp_i_lock_table.size) {
3123 int row = __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK;
3124 kmp_indirect_lock_t **new_table = (kmp_indirect_lock_t **)__kmp_allocate(
3125 2 * row *
sizeof(kmp_indirect_lock_t *));
3126 KMP_MEMCPY(new_table, __kmp_i_lock_table.table,
3127 row *
sizeof(kmp_indirect_lock_t *));
3128 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3129 __kmp_i_lock_table.table = new_table;
3130 __kmp_free(old_table);
3132 for (
int i = row; i < 2 * row; ++i)
3133 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)__kmp_allocate(
3134 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3135 __kmp_i_lock_table.size = 2 * idx;
3137 __kmp_i_lock_table.next++;
3138 lck = KMP_GET_I_LOCK(idx);
3140 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3142 (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3145 __kmp_release_lock(&__kmp_global_lock, gtid);
3149 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3150 *((kmp_lock_index_t *)user_lock) = idx
3153 *((kmp_indirect_lock_t **)user_lock) = lck;
3160 static __forceinline kmp_indirect_lock_t *
3161 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func) {
3162 if (__kmp_env_consistency_check) {
3163 kmp_indirect_lock_t *lck = NULL;
3164 if (user_lock == NULL) {
3165 KMP_FATAL(LockIsUninitialized, func);
3167 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3168 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3169 if (idx >= __kmp_i_lock_table.size) {
3170 KMP_FATAL(LockIsUninitialized, func);
3172 lck = KMP_GET_I_LOCK(idx);
3174 lck = *((kmp_indirect_lock_t **)user_lock);
3177 KMP_FATAL(LockIsUninitialized, func);
3181 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3182 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3184 return *((kmp_indirect_lock_t **)user_lock);
3189 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
3190 kmp_dyna_lockseq_t seq) {
3191 #if KMP_USE_ADAPTIVE_LOCKS
3192 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3193 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3194 seq = lockseq_queuing;
3198 if (seq == lockseq_rtm_queuing && !__kmp_cpuinfo.rtm) {
3199 seq = lockseq_queuing;
3202 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3203 kmp_indirect_lock_t *l =
3204 __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3205 KMP_I_LOCK_FUNC(l, init)(l->lock);
3207 20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n",
3211 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
3212 kmp_uint32 gtid = __kmp_entry_gtid();
3213 kmp_indirect_lock_t *l =
3214 __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3215 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3216 kmp_indirect_locktag_t tag = l->type;
3218 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3221 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3222 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3223 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3225 __kmp_indirect_lock_pool[tag] = l;
3227 __kmp_release_lock(&__kmp_global_lock, gtid);
3230 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3231 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3232 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3235 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3236 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3237 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3240 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3241 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3242 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3245 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3247 kmp_indirect_lock_t *l =
3248 __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3249 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3252 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3254 kmp_indirect_lock_t *l =
3255 __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3256 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3259 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3261 kmp_indirect_lock_t *l =
3262 __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3263 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3266 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3269 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3272 case lockseq_nested_tas:
3273 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3276 case lockseq_nested_futex:
3277 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3279 case lockseq_ticket:
3280 case lockseq_nested_ticket:
3281 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3282 case lockseq_queuing:
3283 case lockseq_nested_queuing:
3284 #if KMP_USE_ADAPTIVE_LOCKS
3285 case lockseq_adaptive:
3287 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3289 case lockseq_nested_drdpa:
3290 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3297 void __kmp_init_dynamic_user_locks() {
3299 if (__kmp_env_consistency_check) {
3300 __kmp_direct_set = direct_set_check;
3301 __kmp_direct_unset = direct_unset_check;
3302 __kmp_direct_test = direct_test_check;
3303 __kmp_direct_destroy = direct_destroy_check;
3304 __kmp_indirect_set = indirect_set_check;
3305 __kmp_indirect_unset = indirect_unset_check;
3306 __kmp_indirect_test = indirect_test_check;
3307 __kmp_indirect_destroy = indirect_destroy_check;
3309 __kmp_direct_set = direct_set;
3310 __kmp_direct_unset = direct_unset;
3311 __kmp_direct_test = direct_test;
3312 __kmp_direct_destroy = direct_destroy;
3313 __kmp_indirect_set = indirect_set;
3314 __kmp_indirect_unset = indirect_unset;
3315 __kmp_indirect_test = indirect_test;
3316 __kmp_indirect_destroy = indirect_destroy;
3321 if (__kmp_init_user_locks)
3325 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3326 __kmp_i_lock_table.table =
3327 (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3328 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)__kmp_allocate(
3329 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3330 __kmp_i_lock_table.next = 0;
3333 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3334 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3335 #if KMP_USE_ADAPTIVE_LOCKS
3336 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3338 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3340 __kmp_indirect_lock_size[locktag_rtm_queuing] =
sizeof(kmp_queuing_lock_t);
3342 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3344 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3346 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3347 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3348 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3351 #define fill_jumps(table, expand, sep) \
3353 table[locktag##sep##ticket] = expand(ticket); \
3354 table[locktag##sep##queuing] = expand(queuing); \
3355 table[locktag##sep##drdpa] = expand(drdpa); \
3358 #if KMP_USE_ADAPTIVE_LOCKS
3359 #define fill_table(table, expand) \
3361 fill_jumps(table, expand, _); \
3362 table[locktag_adaptive] = expand(queuing); \
3363 fill_jumps(table, expand, _nested_); \
3366 #define fill_table(table, expand) \
3368 fill_jumps(table, expand, _); \
3369 fill_jumps(table, expand, _nested_); \
3374 (void (*)(kmp_user_lock_p, const ident_t *)) __kmp_set_##l##_lock_location
3375 fill_table(__kmp_indirect_set_location, expand);
3378 (void (*)(kmp_user_lock_p, kmp_lock_flags_t)) __kmp_set_##l##_lock_flags
3379 fill_table(__kmp_indirect_set_flags, expand);
3382 (const ident_t *(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_location
3383 fill_table(__kmp_indirect_get_location, expand);
3386 (kmp_lock_flags_t(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_flags
3387 fill_table(__kmp_indirect_get_flags, expand);
3390 __kmp_init_user_locks = TRUE;
3394 void __kmp_cleanup_indirect_user_locks() {
3400 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3401 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3403 kmp_indirect_lock_t *ll = l;
3404 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3405 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n",
3407 __kmp_free(ll->lock);
3410 __kmp_indirect_lock_pool[k] = NULL;
3413 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3414 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3415 if (l->lock != NULL) {
3417 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3420 (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n",
3422 __kmp_free(l->lock);
3426 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3427 __kmp_free(__kmp_i_lock_table.table[i]);
3428 __kmp_free(__kmp_i_lock_table.table);
3430 __kmp_init_user_locks = FALSE;
3433 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3434 int __kmp_num_locks_in_block = 1;
3438 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3439 __kmp_init_tas_lock(lck);
3442 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3443 __kmp_init_nested_tas_lock(lck);
3447 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3448 __kmp_init_futex_lock(lck);
3451 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3452 __kmp_init_nested_futex_lock(lck);
3456 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
3457 return lck == lck->lk.self;
3460 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3461 __kmp_init_ticket_lock(lck);
3464 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3465 __kmp_init_nested_ticket_lock(lck);
3468 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
3469 return lck == lck->lk.initialized;
3472 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3473 __kmp_init_queuing_lock(lck);
3477 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3478 __kmp_init_nested_queuing_lock(lck);
3481 #if KMP_USE_ADAPTIVE_LOCKS
3482 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
3483 __kmp_init_adaptive_lock(lck);
3487 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
3488 return lck == lck->lk.initialized;
3491 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3492 __kmp_init_drdpa_lock(lck);
3495 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3496 __kmp_init_nested_drdpa_lock(lck);
3503 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3505 size_t __kmp_base_user_lock_size = 0;
3506 size_t __kmp_user_lock_size = 0;
3508 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3509 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3510 kmp_int32 gtid) = NULL;
3512 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3513 kmp_int32 gtid) = NULL;
3514 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3515 kmp_int32 gtid) = NULL;
3516 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3517 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3518 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3519 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3520 kmp_int32 gtid) = NULL;
3522 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3523 kmp_int32 gtid) = NULL;
3524 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3525 kmp_int32 gtid) = NULL;
3526 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3527 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3529 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3530 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3531 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3533 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3534 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3535 kmp_lock_flags_t flags) = NULL;
3537 void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind) {
3538 switch (user_lock_kind) {
3544 __kmp_base_user_lock_size =
sizeof(kmp_base_tas_lock_t);
3545 __kmp_user_lock_size =
sizeof(kmp_tas_lock_t);
3547 __kmp_get_user_lock_owner_ =
3548 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_tas_lock_owner);
3550 if (__kmp_env_consistency_check) {
3551 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3552 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3554 KMP_BIND_USER_LOCK(tas);
3555 KMP_BIND_NESTED_USER_LOCK(tas);
3558 __kmp_destroy_user_lock_ =
3559 (void (*)(kmp_user_lock_p))(&__kmp_destroy_tas_lock);
3561 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3563 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3565 __kmp_set_user_lock_location_ =
3566 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3568 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3570 __kmp_set_user_lock_flags_ =
3571 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3577 __kmp_base_user_lock_size =
sizeof(kmp_base_futex_lock_t);
3578 __kmp_user_lock_size =
sizeof(kmp_futex_lock_t);
3580 __kmp_get_user_lock_owner_ =
3581 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_futex_lock_owner);
3583 if (__kmp_env_consistency_check) {
3584 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3585 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3587 KMP_BIND_USER_LOCK(futex);
3588 KMP_BIND_NESTED_USER_LOCK(futex);
3591 __kmp_destroy_user_lock_ =
3592 (void (*)(kmp_user_lock_p))(&__kmp_destroy_futex_lock);
3594 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3596 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3598 __kmp_set_user_lock_location_ =
3599 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3601 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3603 __kmp_set_user_lock_flags_ =
3604 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3610 __kmp_base_user_lock_size =
sizeof(kmp_base_ticket_lock_t);
3611 __kmp_user_lock_size =
sizeof(kmp_ticket_lock_t);
3613 __kmp_get_user_lock_owner_ =
3614 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_owner);
3616 if (__kmp_env_consistency_check) {
3617 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3618 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3620 KMP_BIND_USER_LOCK(ticket);
3621 KMP_BIND_NESTED_USER_LOCK(ticket);
3624 __kmp_destroy_user_lock_ =
3625 (void (*)(kmp_user_lock_p))(&__kmp_destroy_ticket_lock);
3627 __kmp_is_user_lock_initialized_ =
3628 (int (*)(kmp_user_lock_p))(&__kmp_is_ticket_lock_initialized);
3630 __kmp_get_user_lock_location_ =
3631 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_location);
3633 __kmp_set_user_lock_location_ = (void (*)(
3634 kmp_user_lock_p,
const ident_t *))(&__kmp_set_ticket_lock_location);
3636 __kmp_get_user_lock_flags_ =
3637 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_flags);
3639 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3640 &__kmp_set_ticket_lock_flags);
3644 __kmp_base_user_lock_size =
sizeof(kmp_base_queuing_lock_t);
3645 __kmp_user_lock_size =
sizeof(kmp_queuing_lock_t);
3647 __kmp_get_user_lock_owner_ =
3648 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3650 if (__kmp_env_consistency_check) {
3651 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3652 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3654 KMP_BIND_USER_LOCK(queuing);
3655 KMP_BIND_NESTED_USER_LOCK(queuing);
3658 __kmp_destroy_user_lock_ =
3659 (void (*)(kmp_user_lock_p))(&__kmp_destroy_queuing_lock);
3661 __kmp_is_user_lock_initialized_ =
3662 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3664 __kmp_get_user_lock_location_ =
3665 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3667 __kmp_set_user_lock_location_ = (void (*)(
3668 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3670 __kmp_get_user_lock_flags_ =
3671 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3673 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3674 &__kmp_set_queuing_lock_flags);
3677 #if KMP_USE_ADAPTIVE_LOCKS
3679 __kmp_base_user_lock_size =
sizeof(kmp_base_adaptive_lock_t);
3680 __kmp_user_lock_size =
sizeof(kmp_adaptive_lock_t);
3682 __kmp_get_user_lock_owner_ =
3683 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3685 if (__kmp_env_consistency_check) {
3686 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3688 KMP_BIND_USER_LOCK(adaptive);
3691 __kmp_destroy_user_lock_ =
3692 (void (*)(kmp_user_lock_p))(&__kmp_destroy_adaptive_lock);
3694 __kmp_is_user_lock_initialized_ =
3695 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3697 __kmp_get_user_lock_location_ =
3698 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3700 __kmp_set_user_lock_location_ = (void (*)(
3701 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3703 __kmp_get_user_lock_flags_ =
3704 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3706 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3707 &__kmp_set_queuing_lock_flags);
3713 __kmp_base_user_lock_size =
sizeof(kmp_base_drdpa_lock_t);
3714 __kmp_user_lock_size =
sizeof(kmp_drdpa_lock_t);
3716 __kmp_get_user_lock_owner_ =
3717 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_owner);
3719 if (__kmp_env_consistency_check) {
3720 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3721 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3723 KMP_BIND_USER_LOCK(drdpa);
3724 KMP_BIND_NESTED_USER_LOCK(drdpa);
3727 __kmp_destroy_user_lock_ =
3728 (void (*)(kmp_user_lock_p))(&__kmp_destroy_drdpa_lock);
3730 __kmp_is_user_lock_initialized_ =
3731 (int (*)(kmp_user_lock_p))(&__kmp_is_drdpa_lock_initialized);
3733 __kmp_get_user_lock_location_ =
3734 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_location);
3736 __kmp_set_user_lock_location_ = (void (*)(
3737 kmp_user_lock_p,
const ident_t *))(&__kmp_set_drdpa_lock_location);
3739 __kmp_get_user_lock_flags_ =
3740 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_flags);
3742 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3743 &__kmp_set_drdpa_lock_flags);
3751 kmp_lock_table_t __kmp_user_lock_table = {1, 0, NULL};
3752 kmp_user_lock_p __kmp_lock_pool = NULL;
3755 kmp_block_of_locks *__kmp_lock_blocks = NULL;
3756 int __kmp_num_locks_in_block = 1;
3758 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3760 kmp_lock_index_t index;
3761 if (__kmp_user_lock_table.used >= __kmp_user_lock_table.allocated) {
3762 kmp_lock_index_t size;
3763 kmp_user_lock_p *table;
3765 if (__kmp_user_lock_table.allocated == 0) {
3768 size = __kmp_user_lock_table.allocated * 2;
3770 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof(kmp_user_lock_p) * size);
3771 KMP_MEMCPY(table + 1, __kmp_user_lock_table.table + 1,
3772 sizeof(kmp_user_lock_p) * (__kmp_user_lock_table.used - 1));
3773 table[0] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3778 __kmp_user_lock_table.table = table;
3779 __kmp_user_lock_table.allocated = size;
3781 KMP_DEBUG_ASSERT(__kmp_user_lock_table.used <
3782 __kmp_user_lock_table.allocated);
3783 index = __kmp_user_lock_table.used;
3784 __kmp_user_lock_table.table[index] = lck;
3785 ++__kmp_user_lock_table.used;
3789 static kmp_user_lock_p __kmp_lock_block_allocate() {
3791 static int last_index = 0;
3792 if ((last_index >= __kmp_num_locks_in_block) || (__kmp_lock_blocks == NULL)) {
3796 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3797 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3799 (
char *)__kmp_allocate(space_for_locks +
sizeof(kmp_block_of_locks));
3801 kmp_block_of_locks *new_block =
3802 (kmp_block_of_locks *)(&buffer[space_for_locks]);
3803 new_block->next_block = __kmp_lock_blocks;
3804 new_block->locks = (
void *)buffer;
3807 __kmp_lock_blocks = new_block;
3809 kmp_user_lock_p ret = (kmp_user_lock_p)(&(
3810 ((
char *)(__kmp_lock_blocks->locks))[last_index * __kmp_user_lock_size]));
3817 kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3818 kmp_lock_flags_t flags) {
3819 kmp_user_lock_p lck;
3820 kmp_lock_index_t index;
3821 KMP_DEBUG_ASSERT(user_lock);
3823 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3825 if (__kmp_lock_pool == NULL) {
3828 if (__kmp_num_locks_in_block <= 1) {
3829 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3831 lck = __kmp_lock_block_allocate();
3836 index = __kmp_lock_table_insert(lck);
3839 lck = __kmp_lock_pool;
3840 index = __kmp_lock_pool->pool.index;
3841 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3846 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3847 *((kmp_lock_index_t *)user_lock) = index;
3849 *((kmp_user_lock_p *)user_lock) = lck;
3853 __kmp_set_user_lock_flags(lck, flags);
3855 __kmp_release_lock(&__kmp_global_lock, gtid);
3861 void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
3862 kmp_user_lock_p lck) {
3863 KMP_DEBUG_ASSERT(user_lock != NULL);
3864 KMP_DEBUG_ASSERT(lck != NULL);
3866 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3868 lck->pool.next = __kmp_lock_pool;
3869 __kmp_lock_pool = lck;
3870 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3871 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3872 KMP_DEBUG_ASSERT(0 < index && index <= __kmp_user_lock_table.used);
3873 lck->pool.index = index;
3876 __kmp_release_lock(&__kmp_global_lock, gtid);
3879 kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func) {
3880 kmp_user_lock_p lck = NULL;
3882 if (__kmp_env_consistency_check) {
3883 if (user_lock == NULL) {
3884 KMP_FATAL(LockIsUninitialized, func);
3888 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3889 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3890 if (__kmp_env_consistency_check) {
3891 if (!(0 < index && index < __kmp_user_lock_table.used)) {
3892 KMP_FATAL(LockIsUninitialized, func);
3895 KMP_DEBUG_ASSERT(0 < index && index < __kmp_user_lock_table.used);
3896 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3897 lck = __kmp_user_lock_table.table[index];
3899 lck = *((kmp_user_lock_p *)user_lock);
3902 if (__kmp_env_consistency_check) {
3904 KMP_FATAL(LockIsUninitialized, func);
3911 void __kmp_cleanup_user_locks(
void) {
3914 __kmp_lock_pool = NULL;
3916 #define IS_CRITICAL(lck) \
3917 ((__kmp_get_user_lock_flags_ != NULL) && \
3918 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section))
3943 while (__kmp_user_lock_table.used > 1) {
3948 kmp_user_lock_p lck =
3949 __kmp_user_lock_table.table[--__kmp_user_lock_table.used];
3951 if ((__kmp_is_user_lock_initialized_ != NULL) &&
3952 (*__kmp_is_user_lock_initialized_)(lck)) {
3956 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3957 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
3959 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->
psource,
false);
3960 KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
3961 __kmp_str_loc_free(&str_loc);
3965 if (IS_CRITICAL(lck)) {
3968 (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n",
3969 lck, *(
void **)lck));
3971 KA_TRACE(20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
3977 __kmp_destroy_user_lock(lck);
3981 if (__kmp_lock_blocks == NULL) {
3989 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
3990 __kmp_user_lock_table.table = NULL;
3991 __kmp_user_lock_table.allocated = 0;
3993 while (table_ptr != NULL) {
3996 kmp_user_lock_p *next = (kmp_user_lock_p *)(table_ptr[0]);
3997 __kmp_free(table_ptr);
4002 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4003 __kmp_lock_blocks = NULL;
4005 while (block_ptr != NULL) {
4006 kmp_block_of_locks_t *next = block_ptr->next_block;
4007 __kmp_free(block_ptr->locks);
4012 TCW_4(__kmp_init_user_locks, FALSE);
void open(const char *filename, const char *mode, const char *env_var=nullptr)