25 #include "tsan_annotations.h" 28 #include <sys/syscall.h> 46 void __kmp_validate_locks(
void) {
51 x = ~((kmp_uint32)0) - 2;
54 for (i = 0; i < 8; ++i, ++x, ++y) {
55 kmp_uint32 z = (x - y);
59 KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
73 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
74 return KMP_LOCK_STRIP(TCR_4(lck->lk.poll)) - 1;
77 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
78 return lck->lk.depth_locked != -1;
81 __forceinline
static int 82 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
85 #ifdef USE_LOCK_PROFILE 86 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
87 if ((curr != 0) && (curr != gtid + 1))
88 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
92 if ((lck->lk.poll == KMP_LOCK_FREE(tas)) &&
93 KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(tas),
94 KMP_LOCK_BUSY(gtid + 1, tas))) {
95 KMP_FSYNC_ACQUIRED(lck);
96 return KMP_LOCK_ACQUIRED_FIRST;
100 KMP_FSYNC_PREPARE(lck);
101 KMP_INIT_YIELD(spins);
102 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
105 KMP_YIELD_SPIN(spins);
108 kmp_backoff_t backoff = __kmp_spin_backoff_params;
109 while ((lck->lk.poll != KMP_LOCK_FREE(tas)) ||
110 (!KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(tas),
111 KMP_LOCK_BUSY(gtid + 1, tas)))) {
113 __kmp_spin_backoff(&backoff);
114 if (TCR_4(__kmp_nth) >
115 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
118 KMP_YIELD_SPIN(spins);
121 KMP_FSYNC_ACQUIRED(lck);
122 return KMP_LOCK_ACQUIRED_FIRST;
125 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
126 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
127 ANNOTATE_TAS_ACQUIRED(lck);
131 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
133 char const *
const func =
"omp_set_lock";
134 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
135 __kmp_is_tas_lock_nestable(lck)) {
136 KMP_FATAL(LockNestableUsedAsSimple, func);
138 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
139 KMP_FATAL(LockIsAlreadyOwned, func);
141 return __kmp_acquire_tas_lock(lck, gtid);
144 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
145 if ((lck->lk.poll == KMP_LOCK_FREE(tas)) &&
146 KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(tas),
147 KMP_LOCK_BUSY(gtid + 1, tas))) {
148 KMP_FSYNC_ACQUIRED(lck);
154 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
156 char const *
const func =
"omp_test_lock";
157 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
158 __kmp_is_tas_lock_nestable(lck)) {
159 KMP_FATAL(LockNestableUsedAsSimple, func);
161 return __kmp_test_tas_lock(lck, gtid);
164 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
167 KMP_FSYNC_RELEASING(lck);
168 ANNOTATE_TAS_RELEASED(lck);
169 KMP_ST_REL32(&(lck->lk.poll), KMP_LOCK_FREE(tas));
172 KMP_YIELD(TCR_4(__kmp_nth) >
173 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
174 return KMP_LOCK_RELEASED;
177 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
179 char const *
const func =
"omp_unset_lock";
181 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
182 __kmp_is_tas_lock_nestable(lck)) {
183 KMP_FATAL(LockNestableUsedAsSimple, func);
185 if (__kmp_get_tas_lock_owner(lck) == -1) {
186 KMP_FATAL(LockUnsettingFree, func);
188 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
189 (__kmp_get_tas_lock_owner(lck) != gtid)) {
190 KMP_FATAL(LockUnsettingSetByAnother, func);
192 return __kmp_release_tas_lock(lck, gtid);
195 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
196 TCW_4(lck->lk.poll, KMP_LOCK_FREE(tas));
199 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
200 __kmp_init_tas_lock(lck);
203 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
205 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
206 char const *
const func =
"omp_destroy_lock";
207 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
208 __kmp_is_tas_lock_nestable(lck)) {
209 KMP_FATAL(LockNestableUsedAsSimple, func);
211 if (__kmp_get_tas_lock_owner(lck) != -1) {
212 KMP_FATAL(LockStillOwned, func);
214 __kmp_destroy_tas_lock(lck);
219 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
220 KMP_DEBUG_ASSERT(gtid >= 0);
222 if (__kmp_get_tas_lock_owner(lck) == gtid) {
223 lck->lk.depth_locked += 1;
224 return KMP_LOCK_ACQUIRED_NEXT;
226 __kmp_acquire_tas_lock_timed_template(lck, gtid);
227 ANNOTATE_TAS_ACQUIRED(lck);
228 lck->lk.depth_locked = 1;
229 return KMP_LOCK_ACQUIRED_FIRST;
233 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
235 char const *
const func =
"omp_set_nest_lock";
236 if (!__kmp_is_tas_lock_nestable(lck)) {
237 KMP_FATAL(LockSimpleUsedAsNestable, func);
239 return __kmp_acquire_nested_tas_lock(lck, gtid);
242 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
245 KMP_DEBUG_ASSERT(gtid >= 0);
247 if (__kmp_get_tas_lock_owner(lck) == gtid) {
248 retval = ++lck->lk.depth_locked;
249 }
else if (!__kmp_test_tas_lock(lck, gtid)) {
253 retval = lck->lk.depth_locked = 1;
258 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
260 char const *
const func =
"omp_test_nest_lock";
261 if (!__kmp_is_tas_lock_nestable(lck)) {
262 KMP_FATAL(LockSimpleUsedAsNestable, func);
264 return __kmp_test_nested_tas_lock(lck, gtid);
267 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
268 KMP_DEBUG_ASSERT(gtid >= 0);
271 if (--(lck->lk.depth_locked) == 0) {
272 __kmp_release_tas_lock(lck, gtid);
273 return KMP_LOCK_RELEASED;
275 return KMP_LOCK_STILL_HELD;
278 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
280 char const *
const func =
"omp_unset_nest_lock";
282 if (!__kmp_is_tas_lock_nestable(lck)) {
283 KMP_FATAL(LockSimpleUsedAsNestable, func);
285 if (__kmp_get_tas_lock_owner(lck) == -1) {
286 KMP_FATAL(LockUnsettingFree, func);
288 if (__kmp_get_tas_lock_owner(lck) != gtid) {
289 KMP_FATAL(LockUnsettingSetByAnother, func);
291 return __kmp_release_nested_tas_lock(lck, gtid);
294 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
295 __kmp_init_tas_lock(lck);
296 lck->lk.depth_locked = 0;
299 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
300 __kmp_init_nested_tas_lock(lck);
303 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
304 __kmp_destroy_tas_lock(lck);
305 lck->lk.depth_locked = 0;
308 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
309 char const *
const func =
"omp_destroy_nest_lock";
310 if (!__kmp_is_tas_lock_nestable(lck)) {
311 KMP_FATAL(LockSimpleUsedAsNestable, func);
313 if (__kmp_get_tas_lock_owner(lck) != -1) {
314 KMP_FATAL(LockStillOwned, func);
316 __kmp_destroy_nested_tas_lock(lck);
329 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
330 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
333 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
334 return lck->lk.depth_locked != -1;
337 __forceinline
static int 338 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
339 kmp_int32 gtid_code = (gtid + 1) << 1;
343 #ifdef USE_LOCK_PROFILE 344 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
345 if ((curr != 0) && (curr != gtid_code))
346 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
350 KMP_FSYNC_PREPARE(lck);
351 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
352 lck, lck->lk.poll, gtid));
356 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(
357 &(lck->lk.poll), KMP_LOCK_FREE(futex),
358 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) {
360 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
363 (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
364 lck, gtid, poll_val, cond));
375 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
376 poll_val | KMP_LOCK_BUSY(1, futex))) {
379 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
380 lck, lck->lk.poll, gtid));
383 poll_val |= KMP_LOCK_BUSY(1, futex);
386 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
387 lck->lk.poll, gtid));
392 (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
393 lck, gtid, poll_val));
396 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
398 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) " 399 "failed (rc=%d errno=%d)\n",
400 lck, gtid, poll_val, rc, errno));
405 (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
406 lck, gtid, poll_val));
413 KMP_FSYNC_ACQUIRED(lck);
414 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
415 lck->lk.poll, gtid));
416 return KMP_LOCK_ACQUIRED_FIRST;
419 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
420 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
421 ANNOTATE_FUTEX_ACQUIRED(lck);
425 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
427 char const *
const func =
"omp_set_lock";
428 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
429 __kmp_is_futex_lock_nestable(lck)) {
430 KMP_FATAL(LockNestableUsedAsSimple, func);
432 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
433 KMP_FATAL(LockIsAlreadyOwned, func);
435 return __kmp_acquire_futex_lock(lck, gtid);
438 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
439 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
440 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) {
441 KMP_FSYNC_ACQUIRED(lck);
447 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
449 char const *
const func =
"omp_test_lock";
450 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
451 __kmp_is_futex_lock_nestable(lck)) {
452 KMP_FATAL(LockNestableUsedAsSimple, func);
454 return __kmp_test_futex_lock(lck, gtid);
457 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
460 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
461 lck, lck->lk.poll, gtid));
463 KMP_FSYNC_RELEASING(lck);
464 ANNOTATE_FUTEX_RELEASED(lck);
466 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
469 (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
470 lck, gtid, poll_val));
472 if (KMP_LOCK_STRIP(poll_val) & 1) {
474 (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
476 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
482 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
483 lck->lk.poll, gtid));
485 KMP_YIELD(TCR_4(__kmp_nth) >
486 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
487 return KMP_LOCK_RELEASED;
490 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
492 char const *
const func =
"omp_unset_lock";
494 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
495 __kmp_is_futex_lock_nestable(lck)) {
496 KMP_FATAL(LockNestableUsedAsSimple, func);
498 if (__kmp_get_futex_lock_owner(lck) == -1) {
499 KMP_FATAL(LockUnsettingFree, func);
501 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
502 (__kmp_get_futex_lock_owner(lck) != gtid)) {
503 KMP_FATAL(LockUnsettingSetByAnother, func);
505 return __kmp_release_futex_lock(lck, gtid);
508 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
509 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
512 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
513 __kmp_init_futex_lock(lck);
516 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
518 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
519 char const *
const func =
"omp_destroy_lock";
520 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
521 __kmp_is_futex_lock_nestable(lck)) {
522 KMP_FATAL(LockNestableUsedAsSimple, func);
524 if (__kmp_get_futex_lock_owner(lck) != -1) {
525 KMP_FATAL(LockStillOwned, func);
527 __kmp_destroy_futex_lock(lck);
532 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
533 KMP_DEBUG_ASSERT(gtid >= 0);
535 if (__kmp_get_futex_lock_owner(lck) == gtid) {
536 lck->lk.depth_locked += 1;
537 return KMP_LOCK_ACQUIRED_NEXT;
539 __kmp_acquire_futex_lock_timed_template(lck, gtid);
540 ANNOTATE_FUTEX_ACQUIRED(lck);
541 lck->lk.depth_locked = 1;
542 return KMP_LOCK_ACQUIRED_FIRST;
546 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
548 char const *
const func =
"omp_set_nest_lock";
549 if (!__kmp_is_futex_lock_nestable(lck)) {
550 KMP_FATAL(LockSimpleUsedAsNestable, func);
552 return __kmp_acquire_nested_futex_lock(lck, gtid);
555 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
558 KMP_DEBUG_ASSERT(gtid >= 0);
560 if (__kmp_get_futex_lock_owner(lck) == gtid) {
561 retval = ++lck->lk.depth_locked;
562 }
else if (!__kmp_test_futex_lock(lck, gtid)) {
566 retval = lck->lk.depth_locked = 1;
571 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
573 char const *
const func =
"omp_test_nest_lock";
574 if (!__kmp_is_futex_lock_nestable(lck)) {
575 KMP_FATAL(LockSimpleUsedAsNestable, func);
577 return __kmp_test_nested_futex_lock(lck, gtid);
580 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
581 KMP_DEBUG_ASSERT(gtid >= 0);
584 if (--(lck->lk.depth_locked) == 0) {
585 __kmp_release_futex_lock(lck, gtid);
586 return KMP_LOCK_RELEASED;
588 return KMP_LOCK_STILL_HELD;
591 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
593 char const *
const func =
"omp_unset_nest_lock";
595 if (!__kmp_is_futex_lock_nestable(lck)) {
596 KMP_FATAL(LockSimpleUsedAsNestable, func);
598 if (__kmp_get_futex_lock_owner(lck) == -1) {
599 KMP_FATAL(LockUnsettingFree, func);
601 if (__kmp_get_futex_lock_owner(lck) != gtid) {
602 KMP_FATAL(LockUnsettingSetByAnother, func);
604 return __kmp_release_nested_futex_lock(lck, gtid);
607 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
608 __kmp_init_futex_lock(lck);
609 lck->lk.depth_locked = 0;
612 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
613 __kmp_init_nested_futex_lock(lck);
616 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
617 __kmp_destroy_futex_lock(lck);
618 lck->lk.depth_locked = 0;
621 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
622 char const *
const func =
"omp_destroy_nest_lock";
623 if (!__kmp_is_futex_lock_nestable(lck)) {
624 KMP_FATAL(LockSimpleUsedAsNestable, func);
626 if (__kmp_get_futex_lock_owner(lck) != -1) {
627 KMP_FATAL(LockStillOwned, func);
629 __kmp_destroy_nested_futex_lock(lck);
632 #endif // KMP_USE_FUTEX 637 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
638 return std::atomic_load_explicit(&lck->lk.owner_id,
639 std::memory_order_relaxed) -
643 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
644 return std::atomic_load_explicit(&lck->lk.depth_locked,
645 std::memory_order_relaxed) != -1;
648 static kmp_uint32 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket) {
649 return std::atomic_load_explicit((std::atomic<unsigned> *)now_serving,
650 std::memory_order_acquire) == my_ticket;
653 __forceinline
static int 654 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
656 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit(
657 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
659 #ifdef USE_LOCK_PROFILE 660 if (std::atomic_load_explicit(&lck->lk.now_serving,
661 std::memory_order_relaxed) != my_ticket)
662 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
666 if (std::atomic_load_explicit(&lck->lk.now_serving,
667 std::memory_order_acquire) == my_ticket) {
668 return KMP_LOCK_ACQUIRED_FIRST;
670 KMP_WAIT_YIELD_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
671 return KMP_LOCK_ACQUIRED_FIRST;
674 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
675 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
676 ANNOTATE_TICKET_ACQUIRED(lck);
680 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
682 char const *
const func =
"omp_set_lock";
684 if (!std::atomic_load_explicit(&lck->lk.initialized,
685 std::memory_order_relaxed)) {
686 KMP_FATAL(LockIsUninitialized, func);
688 if (lck->lk.self != lck) {
689 KMP_FATAL(LockIsUninitialized, func);
691 if (__kmp_is_ticket_lock_nestable(lck)) {
692 KMP_FATAL(LockNestableUsedAsSimple, func);
694 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
695 KMP_FATAL(LockIsAlreadyOwned, func);
698 __kmp_acquire_ticket_lock(lck, gtid);
700 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
701 std::memory_order_relaxed);
702 return KMP_LOCK_ACQUIRED_FIRST;
705 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
706 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
707 std::memory_order_relaxed);
709 if (std::atomic_load_explicit(&lck->lk.now_serving,
710 std::memory_order_relaxed) == my_ticket) {
711 kmp_uint32 next_ticket = my_ticket + 1;
712 if (std::atomic_compare_exchange_strong_explicit(
713 &lck->lk.next_ticket, &my_ticket, next_ticket,
714 std::memory_order_acquire, std::memory_order_acquire)) {
721 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
723 char const *
const func =
"omp_test_lock";
725 if (!std::atomic_load_explicit(&lck->lk.initialized,
726 std::memory_order_relaxed)) {
727 KMP_FATAL(LockIsUninitialized, func);
729 if (lck->lk.self != lck) {
730 KMP_FATAL(LockIsUninitialized, func);
732 if (__kmp_is_ticket_lock_nestable(lck)) {
733 KMP_FATAL(LockNestableUsedAsSimple, func);
736 int retval = __kmp_test_ticket_lock(lck, gtid);
739 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
740 std::memory_order_relaxed);
745 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
746 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
747 std::memory_order_relaxed) -
748 std::atomic_load_explicit(&lck->lk.now_serving,
749 std::memory_order_relaxed);
751 ANNOTATE_TICKET_RELEASED(lck);
752 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
753 std::memory_order_release);
756 (kmp_uint32)(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
757 return KMP_LOCK_RELEASED;
760 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
762 char const *
const func =
"omp_unset_lock";
764 if (!std::atomic_load_explicit(&lck->lk.initialized,
765 std::memory_order_relaxed)) {
766 KMP_FATAL(LockIsUninitialized, func);
768 if (lck->lk.self != lck) {
769 KMP_FATAL(LockIsUninitialized, func);
771 if (__kmp_is_ticket_lock_nestable(lck)) {
772 KMP_FATAL(LockNestableUsedAsSimple, func);
774 if (__kmp_get_ticket_lock_owner(lck) == -1) {
775 KMP_FATAL(LockUnsettingFree, func);
777 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
778 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
779 KMP_FATAL(LockUnsettingSetByAnother, func);
781 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
782 return __kmp_release_ticket_lock(lck, gtid);
785 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
786 lck->lk.location = NULL;
788 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
789 std::memory_order_relaxed);
790 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
791 std::memory_order_relaxed);
792 std::atomic_store_explicit(
793 &lck->lk.owner_id, 0,
794 std::memory_order_relaxed);
795 std::atomic_store_explicit(
796 &lck->lk.depth_locked, -1,
797 std::memory_order_relaxed);
798 std::atomic_store_explicit(&lck->lk.initialized,
true,
799 std::memory_order_release);
802 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
803 __kmp_init_ticket_lock(lck);
806 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
807 std::atomic_store_explicit(&lck->lk.initialized,
false,
808 std::memory_order_release);
810 lck->lk.location = NULL;
811 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
812 std::memory_order_relaxed);
813 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
814 std::memory_order_relaxed);
815 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
816 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
817 std::memory_order_relaxed);
820 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
821 char const *
const func =
"omp_destroy_lock";
823 if (!std::atomic_load_explicit(&lck->lk.initialized,
824 std::memory_order_relaxed)) {
825 KMP_FATAL(LockIsUninitialized, func);
827 if (lck->lk.self != lck) {
828 KMP_FATAL(LockIsUninitialized, func);
830 if (__kmp_is_ticket_lock_nestable(lck)) {
831 KMP_FATAL(LockNestableUsedAsSimple, func);
833 if (__kmp_get_ticket_lock_owner(lck) != -1) {
834 KMP_FATAL(LockStillOwned, func);
836 __kmp_destroy_ticket_lock(lck);
841 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
842 KMP_DEBUG_ASSERT(gtid >= 0);
844 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
845 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
846 std::memory_order_relaxed);
847 return KMP_LOCK_ACQUIRED_NEXT;
849 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
850 ANNOTATE_TICKET_ACQUIRED(lck);
851 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
852 std::memory_order_relaxed);
853 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
854 std::memory_order_relaxed);
855 return KMP_LOCK_ACQUIRED_FIRST;
859 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
861 char const *
const func =
"omp_set_nest_lock";
863 if (!std::atomic_load_explicit(&lck->lk.initialized,
864 std::memory_order_relaxed)) {
865 KMP_FATAL(LockIsUninitialized, func);
867 if (lck->lk.self != lck) {
868 KMP_FATAL(LockIsUninitialized, func);
870 if (!__kmp_is_ticket_lock_nestable(lck)) {
871 KMP_FATAL(LockSimpleUsedAsNestable, func);
873 return __kmp_acquire_nested_ticket_lock(lck, gtid);
876 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
879 KMP_DEBUG_ASSERT(gtid >= 0);
881 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
882 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
883 std::memory_order_relaxed) +
885 }
else if (!__kmp_test_ticket_lock(lck, gtid)) {
888 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
889 std::memory_order_relaxed);
890 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
891 std::memory_order_relaxed);
897 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
899 char const *
const func =
"omp_test_nest_lock";
901 if (!std::atomic_load_explicit(&lck->lk.initialized,
902 std::memory_order_relaxed)) {
903 KMP_FATAL(LockIsUninitialized, func);
905 if (lck->lk.self != lck) {
906 KMP_FATAL(LockIsUninitialized, func);
908 if (!__kmp_is_ticket_lock_nestable(lck)) {
909 KMP_FATAL(LockSimpleUsedAsNestable, func);
911 return __kmp_test_nested_ticket_lock(lck, gtid);
914 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
915 KMP_DEBUG_ASSERT(gtid >= 0);
917 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
918 std::memory_order_relaxed) -
920 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
921 __kmp_release_ticket_lock(lck, gtid);
922 return KMP_LOCK_RELEASED;
924 return KMP_LOCK_STILL_HELD;
927 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
929 char const *
const func =
"omp_unset_nest_lock";
931 if (!std::atomic_load_explicit(&lck->lk.initialized,
932 std::memory_order_relaxed)) {
933 KMP_FATAL(LockIsUninitialized, func);
935 if (lck->lk.self != lck) {
936 KMP_FATAL(LockIsUninitialized, func);
938 if (!__kmp_is_ticket_lock_nestable(lck)) {
939 KMP_FATAL(LockSimpleUsedAsNestable, func);
941 if (__kmp_get_ticket_lock_owner(lck) == -1) {
942 KMP_FATAL(LockUnsettingFree, func);
944 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
945 KMP_FATAL(LockUnsettingSetByAnother, func);
947 return __kmp_release_nested_ticket_lock(lck, gtid);
950 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
951 __kmp_init_ticket_lock(lck);
952 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
953 std::memory_order_relaxed);
957 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
958 __kmp_init_nested_ticket_lock(lck);
961 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
962 __kmp_destroy_ticket_lock(lck);
963 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
964 std::memory_order_relaxed);
968 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
969 char const *
const func =
"omp_destroy_nest_lock";
971 if (!std::atomic_load_explicit(&lck->lk.initialized,
972 std::memory_order_relaxed)) {
973 KMP_FATAL(LockIsUninitialized, func);
975 if (lck->lk.self != lck) {
976 KMP_FATAL(LockIsUninitialized, func);
978 if (!__kmp_is_ticket_lock_nestable(lck)) {
979 KMP_FATAL(LockSimpleUsedAsNestable, func);
981 if (__kmp_get_ticket_lock_owner(lck) != -1) {
982 KMP_FATAL(LockStillOwned, func);
984 __kmp_destroy_nested_ticket_lock(lck);
989 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
990 return std::atomic_load_explicit(&lck->lk.initialized,
991 std::memory_order_relaxed) &&
992 (lck->lk.self == lck);
995 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
996 return lck->lk.location;
999 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
1001 lck->lk.location = loc;
1004 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
1005 return lck->lk.flags;
1008 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
1009 kmp_lock_flags_t flags) {
1010 lck->lk.flags = flags;
1068 #ifdef DEBUG_QUEUING_LOCKS 1071 #define TRACE_BUF_ELE 1024 1072 static char traces[TRACE_BUF_ELE][128] = {0};
1074 #define TRACE_LOCK(X, Y) \ 1075 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y); 1076 #define TRACE_LOCK_T(X, Y, Z) \ 1077 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X, Y, Z); 1078 #define TRACE_LOCK_HT(X, Y, Z, Q) \ 1079 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, \ 1082 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid,
1083 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1084 kmp_int32 tail_id) {
1087 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n");
1089 i = tc % TRACE_BUF_ELE;
1090 __kmp_printf_no_lock(
"%s\n", traces[i]);
1091 i = (i + 1) % TRACE_BUF_ELE;
1092 while (i != (tc % TRACE_BUF_ELE)) {
1093 __kmp_printf_no_lock(
"%s", traces[i]);
1094 i = (i + 1) % TRACE_BUF_ELE;
1096 __kmp_printf_no_lock(
"\n");
1098 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, " 1099 "next_wait:%d, head_id:%d, tail_id:%d\n",
1100 gtid + 1, this_thr->th.th_spin_here,
1101 this_thr->th.th_next_waiting, head_id, tail_id);
1103 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id);
1105 if (lck->lk.head_id >= 1) {
1106 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1108 __kmp_printf_no_lock(
"-> %d ", t);
1109 t = __kmp_threads[t - 1]->th.th_next_waiting;
1112 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id);
1113 __kmp_printf_no_lock(
"\n\n");
1118 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1119 return TCR_4(lck->lk.owner_id) - 1;
1122 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1123 return lck->lk.depth_locked != -1;
1127 template <
bool takeTime>
1130 __forceinline
static int 1131 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1133 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1134 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1135 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1136 volatile kmp_uint32 *spin_here_p;
1137 kmp_int32 need_mf = 1;
1140 ompt_state_t prev_state = ompt_state_undefined;
1144 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1146 KMP_FSYNC_PREPARE(lck);
1147 KMP_DEBUG_ASSERT(this_thr != NULL);
1148 spin_here_p = &this_thr->th.th_spin_here;
1150 #ifdef DEBUG_QUEUING_LOCKS 1151 TRACE_LOCK(gtid + 1,
"acq ent");
1153 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1154 if (this_thr->th.th_next_waiting != 0)
1155 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1157 KMP_DEBUG_ASSERT(!*spin_here_p);
1158 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1167 *spin_here_p = TRUE;
1179 #ifdef DEBUG_QUEUING_LOCKS 1181 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1190 enqueued = KMP_COMPARE_AND_STORE_ACQ64((
volatile kmp_int64 *)tail_id_p,
1192 KMP_PACK_64(gtid + 1, gtid + 1));
1193 #ifdef DEBUG_QUEUING_LOCKS 1195 TRACE_LOCK(gtid + 1,
"acq enq: (-1,0)->(tid,tid)");
1201 KMP_DEBUG_ASSERT(tail != gtid + 1);
1203 #ifdef DEBUG_QUEUING_LOCKS 1204 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1212 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1);
1214 #ifdef DEBUG_QUEUING_LOCKS 1216 TRACE_LOCK(gtid + 1,
"acq enq: (h,t)->(h,tid)");
1223 kmp_int32 grabbed_lock;
1225 #ifdef DEBUG_QUEUING_LOCKS 1227 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1233 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1);
1237 *spin_here_p = FALSE;
1241 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1243 #ifdef DEBUG_QUEUING_LOCKS 1244 TRACE_LOCK_HT(gtid + 1,
"acq exit: ", head, 0);
1248 if (ompt_enabled && prev_state != ompt_state_undefined) {
1250 this_thr->th.ompt_thread_info.state = prev_state;
1251 this_thr->th.ompt_thread_info.wait_id = 0;
1255 KMP_FSYNC_ACQUIRED(lck);
1256 return KMP_LOCK_ACQUIRED_FIRST;
1263 if (ompt_enabled && prev_state == ompt_state_undefined) {
1265 prev_state = this_thr->th.ompt_thread_info.state;
1266 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1267 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1273 kmp_info_t *tail_thr = __kmp_thread_from_gtid(tail - 1);
1274 KMP_ASSERT(tail_thr != NULL);
1275 tail_thr->th.th_next_waiting = gtid + 1;
1279 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1285 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1287 #ifdef DEBUG_QUEUING_LOCKS 1288 TRACE_LOCK(gtid + 1,
"acq spin");
1290 if (this_thr->th.th_next_waiting != 0)
1291 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1293 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1294 KA_TRACE(1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after " 1295 "waiting on queue\n",
1298 #ifdef DEBUG_QUEUING_LOCKS 1299 TRACE_LOCK(gtid + 1,
"acq exit 2");
1304 this_thr->th.ompt_thread_info.state = prev_state;
1305 this_thr->th.ompt_thread_info.wait_id = 0;
1309 return KMP_LOCK_ACQUIRED_FIRST;
1315 KMP_YIELD(TCR_4(__kmp_nth) >
1316 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
1317 #ifdef DEBUG_QUEUING_LOCKS 1318 TRACE_LOCK(gtid + 1,
"acq retry");
1321 KMP_ASSERT2(0,
"should not get here");
1322 return KMP_LOCK_ACQUIRED_FIRST;
1325 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1326 KMP_DEBUG_ASSERT(gtid >= 0);
1328 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1329 ANNOTATE_QUEUING_ACQUIRED(lck);
1333 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1335 char const *
const func =
"omp_set_lock";
1336 if (lck->lk.initialized != lck) {
1337 KMP_FATAL(LockIsUninitialized, func);
1339 if (__kmp_is_queuing_lock_nestable(lck)) {
1340 KMP_FATAL(LockNestableUsedAsSimple, func);
1342 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1343 KMP_FATAL(LockIsAlreadyOwned, func);
1346 __kmp_acquire_queuing_lock(lck, gtid);
1348 lck->lk.owner_id = gtid + 1;
1349 return KMP_LOCK_ACQUIRED_FIRST;
1352 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1353 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1356 kmp_info_t *this_thr;
1359 KA_TRACE(1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid));
1360 KMP_DEBUG_ASSERT(gtid >= 0);
1362 this_thr = __kmp_thread_from_gtid(gtid);
1363 KMP_DEBUG_ASSERT(this_thr != NULL);
1364 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1371 if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
1373 (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
1374 KMP_FSYNC_ACQUIRED(lck);
1375 ANNOTATE_QUEUING_ACQUIRED(lck);
1381 (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
1385 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1387 char const *
const func =
"omp_test_lock";
1388 if (lck->lk.initialized != lck) {
1389 KMP_FATAL(LockIsUninitialized, func);
1391 if (__kmp_is_queuing_lock_nestable(lck)) {
1392 KMP_FATAL(LockNestableUsedAsSimple, func);
1395 int retval = __kmp_test_queuing_lock(lck, gtid);
1398 lck->lk.owner_id = gtid + 1;
1403 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1404 kmp_info_t *this_thr;
1405 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1406 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1409 (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1410 KMP_DEBUG_ASSERT(gtid >= 0);
1411 this_thr = __kmp_thread_from_gtid(gtid);
1412 KMP_DEBUG_ASSERT(this_thr != NULL);
1413 #ifdef DEBUG_QUEUING_LOCKS 1414 TRACE_LOCK(gtid + 1,
"rel ent");
1416 if (this_thr->th.th_spin_here)
1417 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1418 if (this_thr->th.th_next_waiting != 0)
1419 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1421 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1422 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1424 KMP_FSYNC_RELEASING(lck);
1425 ANNOTATE_QUEUING_RELEASED(lck);
1434 #ifdef DEBUG_QUEUING_LOCKS 1436 TRACE_LOCK_HT(gtid + 1,
"rel read: ", head, tail);
1438 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1440 KMP_DEBUG_ASSERT(head !=
1445 if (KMP_COMPARE_AND_STORE_REL32(head_id_p, -1, 0)) {
1448 (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1450 #ifdef DEBUG_QUEUING_LOCKS 1451 TRACE_LOCK_HT(gtid + 1,
"rel exit: ", 0, 0);
1457 return KMP_LOCK_RELEASED;
1463 #ifdef DEBUG_QUEUING_LOCKS 1465 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1467 KMP_DEBUG_ASSERT(head > 0);
1470 dequeued = KMP_COMPARE_AND_STORE_REL64(
1471 RCAST(
volatile kmp_int64 *, tail_id_p), KMP_PACK_64(head, head),
1472 KMP_PACK_64(-1, 0));
1473 #ifdef DEBUG_QUEUING_LOCKS 1474 TRACE_LOCK(gtid + 1,
"rel deq: (h,h)->(-1,0)");
1478 volatile kmp_int32 *waiting_id_p;
1479 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1480 KMP_DEBUG_ASSERT(head_thr != NULL);
1481 waiting_id_p = &head_thr->th.th_next_waiting;
1484 #ifdef DEBUG_QUEUING_LOCKS 1485 if (head <= 0 || tail <= 0)
1486 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1488 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1494 *head_id_p = KMP_WAIT_YIELD((
volatile kmp_uint32 *)waiting_id_p, 0,
1496 #ifdef DEBUG_QUEUING_LOCKS 1497 TRACE_LOCK(gtid + 1,
"rel deq: (h,t)->(h',t)");
1504 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1505 KMP_DEBUG_ASSERT(head_thr != NULL);
1508 #ifdef DEBUG_QUEUING_LOCKS 1509 if (head <= 0 || tail <= 0)
1510 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1512 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1516 head_thr->th.th_next_waiting = 0;
1517 #ifdef DEBUG_QUEUING_LOCKS 1518 TRACE_LOCK_T(gtid + 1,
"rel nw=0 for t=", head);
1523 head_thr->th.th_spin_here = FALSE;
1525 KA_TRACE(1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after " 1528 #ifdef DEBUG_QUEUING_LOCKS 1529 TRACE_LOCK(gtid + 1,
"rel exit 2");
1531 return KMP_LOCK_RELEASED;
1536 #ifdef DEBUG_QUEUING_LOCKS 1537 TRACE_LOCK(gtid + 1,
"rel retry");
1541 KMP_ASSERT2(0,
"should not get here");
1542 return KMP_LOCK_RELEASED;
1545 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1547 char const *
const func =
"omp_unset_lock";
1549 if (lck->lk.initialized != lck) {
1550 KMP_FATAL(LockIsUninitialized, func);
1552 if (__kmp_is_queuing_lock_nestable(lck)) {
1553 KMP_FATAL(LockNestableUsedAsSimple, func);
1555 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1556 KMP_FATAL(LockUnsettingFree, func);
1558 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1559 KMP_FATAL(LockUnsettingSetByAnother, func);
1561 lck->lk.owner_id = 0;
1562 return __kmp_release_queuing_lock(lck, gtid);
1565 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1566 lck->lk.location = NULL;
1567 lck->lk.head_id = 0;
1568 lck->lk.tail_id = 0;
1569 lck->lk.next_ticket = 0;
1570 lck->lk.now_serving = 0;
1571 lck->lk.owner_id = 0;
1572 lck->lk.depth_locked = -1;
1573 lck->lk.initialized = lck;
1575 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1578 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1579 __kmp_init_queuing_lock(lck);
1582 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1583 lck->lk.initialized = NULL;
1584 lck->lk.location = NULL;
1585 lck->lk.head_id = 0;
1586 lck->lk.tail_id = 0;
1587 lck->lk.next_ticket = 0;
1588 lck->lk.now_serving = 0;
1589 lck->lk.owner_id = 0;
1590 lck->lk.depth_locked = -1;
1593 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1594 char const *
const func =
"omp_destroy_lock";
1595 if (lck->lk.initialized != lck) {
1596 KMP_FATAL(LockIsUninitialized, func);
1598 if (__kmp_is_queuing_lock_nestable(lck)) {
1599 KMP_FATAL(LockNestableUsedAsSimple, func);
1601 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1602 KMP_FATAL(LockStillOwned, func);
1604 __kmp_destroy_queuing_lock(lck);
1609 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1610 KMP_DEBUG_ASSERT(gtid >= 0);
1612 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1613 lck->lk.depth_locked += 1;
1614 return KMP_LOCK_ACQUIRED_NEXT;
1616 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1617 ANNOTATE_QUEUING_ACQUIRED(lck);
1619 lck->lk.depth_locked = 1;
1621 lck->lk.owner_id = gtid + 1;
1622 return KMP_LOCK_ACQUIRED_FIRST;
1627 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1629 char const *
const func =
"omp_set_nest_lock";
1630 if (lck->lk.initialized != lck) {
1631 KMP_FATAL(LockIsUninitialized, func);
1633 if (!__kmp_is_queuing_lock_nestable(lck)) {
1634 KMP_FATAL(LockSimpleUsedAsNestable, func);
1636 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1639 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1642 KMP_DEBUG_ASSERT(gtid >= 0);
1644 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1645 retval = ++lck->lk.depth_locked;
1646 }
else if (!__kmp_test_queuing_lock(lck, gtid)) {
1650 retval = lck->lk.depth_locked = 1;
1652 lck->lk.owner_id = gtid + 1;
1657 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1659 char const *
const func =
"omp_test_nest_lock";
1660 if (lck->lk.initialized != lck) {
1661 KMP_FATAL(LockIsUninitialized, func);
1663 if (!__kmp_is_queuing_lock_nestable(lck)) {
1664 KMP_FATAL(LockSimpleUsedAsNestable, func);
1666 return __kmp_test_nested_queuing_lock(lck, gtid);
1669 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1670 KMP_DEBUG_ASSERT(gtid >= 0);
1673 if (--(lck->lk.depth_locked) == 0) {
1675 lck->lk.owner_id = 0;
1676 __kmp_release_queuing_lock(lck, gtid);
1677 return KMP_LOCK_RELEASED;
1679 return KMP_LOCK_STILL_HELD;
1683 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1685 char const *
const func =
"omp_unset_nest_lock";
1687 if (lck->lk.initialized != lck) {
1688 KMP_FATAL(LockIsUninitialized, func);
1690 if (!__kmp_is_queuing_lock_nestable(lck)) {
1691 KMP_FATAL(LockSimpleUsedAsNestable, func);
1693 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1694 KMP_FATAL(LockUnsettingFree, func);
1696 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1697 KMP_FATAL(LockUnsettingSetByAnother, func);
1699 return __kmp_release_nested_queuing_lock(lck, gtid);
1702 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1703 __kmp_init_queuing_lock(lck);
1704 lck->lk.depth_locked = 0;
1708 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1709 __kmp_init_nested_queuing_lock(lck);
1712 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1713 __kmp_destroy_queuing_lock(lck);
1714 lck->lk.depth_locked = 0;
1718 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1719 char const *
const func =
"omp_destroy_nest_lock";
1720 if (lck->lk.initialized != lck) {
1721 KMP_FATAL(LockIsUninitialized, func);
1723 if (!__kmp_is_queuing_lock_nestable(lck)) {
1724 KMP_FATAL(LockSimpleUsedAsNestable, func);
1726 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1727 KMP_FATAL(LockStillOwned, func);
1729 __kmp_destroy_nested_queuing_lock(lck);
1734 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
1735 return lck == lck->lk.initialized;
1738 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1739 return lck->lk.location;
1742 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1744 lck->lk.location = loc;
1747 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1748 return lck->lk.flags;
1751 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1752 kmp_lock_flags_t flags) {
1753 lck->lk.flags = flags;
1756 #if KMP_USE_ADAPTIVE_LOCKS 1760 #if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1762 #include <immintrin.h> 1763 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1768 #define _XBEGIN_STARTED (~0u) 1769 #define _XABORT_EXPLICIT (1 << 0) 1770 #define _XABORT_RETRY (1 << 1) 1771 #define _XABORT_CONFLICT (1 << 2) 1772 #define _XABORT_CAPACITY (1 << 3) 1773 #define _XABORT_DEBUG (1 << 4) 1774 #define _XABORT_NESTED (1 << 5) 1775 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF)) 1778 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1780 #define STRINGIZE_INTERNAL(arg) #arg 1781 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg) 1787 static __inline
int _xbegin() {
1815 #endif // KMP_ARCH_X86_64 1824 __asm__
volatile(
"1: .byte 0xC7; .byte 0xF8;\n" 1827 "1: movl %%eax,%0\n" 1829 :
"+r"(res)::
"memory",
"%eax");
1830 #endif // KMP_OS_WINDOWS 1835 static __inline
void _xend() {
1843 __asm__
volatile(
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1852 #define _xabort(ARG) _asm _emit 0xc6 _asm _emit 0xf8 _asm _emit ARG 1854 #define _xabort(ARG) \ 1855 __asm__ volatile(".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG):::"memory"); 1858 #endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1861 #if KMP_DEBUG_ADAPTIVE_LOCKS 1866 static kmp_adaptive_lock_statistics_t destroyedStats;
1869 static kmp_adaptive_lock_info_t liveLocks;
1872 static kmp_bootstrap_lock_t chain_lock;
1875 void __kmp_init_speculative_stats() {
1876 kmp_adaptive_lock_info_t *lck = &liveLocks;
1878 memset((
void *)&(lck->stats), 0,
sizeof(lck->stats));
1879 lck->stats.next = lck;
1880 lck->stats.prev = lck;
1882 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1883 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1885 __kmp_init_bootstrap_lock(&chain_lock);
1889 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1890 __kmp_acquire_bootstrap_lock(&chain_lock);
1892 lck->stats.next = liveLocks.stats.next;
1893 lck->stats.prev = &liveLocks;
1895 liveLocks.stats.next = lck;
1896 lck->stats.next->stats.prev = lck;
1898 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1899 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1901 __kmp_release_bootstrap_lock(&chain_lock);
1904 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1905 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1906 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1908 kmp_adaptive_lock_info_t *n = lck->stats.next;
1909 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1915 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1916 memset((
void *)&lck->stats, 0,
sizeof(lck->stats));
1917 __kmp_remember_lock(lck);
1920 static void __kmp_add_stats(kmp_adaptive_lock_statistics_t *t,
1921 kmp_adaptive_lock_info_t *lck) {
1922 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
1924 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1925 t->successfulSpeculations += s->successfulSpeculations;
1926 t->hardFailedSpeculations += s->hardFailedSpeculations;
1927 t->softFailedSpeculations += s->softFailedSpeculations;
1928 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
1929 t->lemmingYields += s->lemmingYields;
1932 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1933 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
1935 __kmp_acquire_bootstrap_lock(&chain_lock);
1937 __kmp_add_stats(&destroyedStats, lck);
1938 __kmp_forget_lock(lck);
1940 __kmp_release_bootstrap_lock(&chain_lock);
1943 static float percent(kmp_uint32 count, kmp_uint32 total) {
1944 return (total == 0) ? 0.0 : (100.0 * count) / total;
1947 static FILE *__kmp_open_stats_file() {
1948 if (strcmp(__kmp_speculative_statsfile,
"-") == 0)
1951 size_t buffLen = KMP_STRLEN(__kmp_speculative_statsfile) + 20;
1952 char buffer[buffLen];
1953 KMP_SNPRINTF(&buffer[0], buffLen, __kmp_speculative_statsfile,
1954 (kmp_int32)getpid());
1955 FILE *result = fopen(&buffer[0],
"w");
1958 return result ? result : stdout;
1961 void __kmp_print_speculative_stats() {
1962 if (__kmp_user_lock_kind != lk_adaptive)
1965 FILE *statsFile = __kmp_open_stats_file();
1967 kmp_adaptive_lock_statistics_t total = destroyedStats;
1968 kmp_adaptive_lock_info_t *lck;
1970 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1971 __kmp_add_stats(&total, lck);
1973 kmp_adaptive_lock_statistics_t *t = &total;
1974 kmp_uint32 totalSections =
1975 t->nonSpeculativeAcquires + t->successfulSpeculations;
1976 kmp_uint32 totalSpeculations = t->successfulSpeculations +
1977 t->hardFailedSpeculations +
1978 t->softFailedSpeculations;
1980 fprintf(statsFile,
"Speculative lock statistics (all approximate!)\n");
1981 fprintf(statsFile,
" Lock parameters: \n" 1982 " max_soft_retries : %10d\n" 1983 " max_badness : %10d\n",
1984 __kmp_adaptive_backoff_params.max_soft_retries,
1985 __kmp_adaptive_backoff_params.max_badness);
1986 fprintf(statsFile,
" Non-speculative acquire attempts : %10d\n",
1987 t->nonSpeculativeAcquireAttempts);
1988 fprintf(statsFile,
" Total critical sections : %10d\n",
1990 fprintf(statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
1991 t->successfulSpeculations,
1992 percent(t->successfulSpeculations, totalSections));
1993 fprintf(statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
1994 t->nonSpeculativeAcquires,
1995 percent(t->nonSpeculativeAcquires, totalSections));
1996 fprintf(statsFile,
" Lemming yields : %10d\n\n",
1999 fprintf(statsFile,
" Speculative acquire attempts : %10d\n",
2001 fprintf(statsFile,
" Successes : %10d (%5.1f%%)\n",
2002 t->successfulSpeculations,
2003 percent(t->successfulSpeculations, totalSpeculations));
2004 fprintf(statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2005 t->softFailedSpeculations,
2006 percent(t->softFailedSpeculations, totalSpeculations));
2007 fprintf(statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2008 t->hardFailedSpeculations,
2009 percent(t->hardFailedSpeculations, totalSpeculations));
2011 if (statsFile != stdout)
2015 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++) 2017 #define KMP_INC_STAT(lck, stat) 2019 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 2021 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
2024 bool res = lck->lk.head_id == 0;
2028 #if KMP_COMPILER_ICC 2031 __sync_synchronize();
2038 static __inline
void 2039 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
2041 lck->lk.adaptive.badness = 0;
2042 KMP_INC_STAT(lck, successfulSpeculations);
2046 static __inline
void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
2047 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
2048 if (newBadness > lck->lk.adaptive.max_badness) {
2051 lck->lk.adaptive.badness = newBadness;
2056 static __inline
int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
2058 kmp_uint32 badness = lck->lk.adaptive.badness;
2059 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
2060 int res = (attempts & badness) == 0;
2066 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
2068 int retries = lck->lk.adaptive.max_soft_retries;
2075 kmp_uint32 status = _xbegin();
2080 if (status == _XBEGIN_STARTED) {
2085 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2089 KMP_ASSERT2(0,
"should not get here");
2094 if (status & SOFT_ABORT_MASK) {
2095 KMP_INC_STAT(lck, softFailedSpeculations);
2098 KMP_INC_STAT(lck, hardFailedSpeculations);
2103 }
while (retries--);
2107 __kmp_step_badness(lck);
2114 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2116 if (__kmp_should_speculate(lck, gtid) &&
2117 __kmp_test_adaptive_lock_only(lck, gtid))
2122 lck->lk.adaptive.acquire_attempts++;
2125 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2126 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2133 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2135 char const *
const func =
"omp_test_lock";
2136 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2137 KMP_FATAL(LockIsUninitialized, func);
2140 int retval = __kmp_test_adaptive_lock(lck, gtid);
2143 lck->lk.qlk.owner_id = gtid + 1;
2159 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2161 if (__kmp_should_speculate(lck, gtid)) {
2162 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2163 if (__kmp_test_adaptive_lock_only(lck, gtid))
2172 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2173 KMP_INC_STAT(lck, lemmingYields);
2177 if (__kmp_test_adaptive_lock_only(lck, gtid))
2184 lck->lk.adaptive.acquire_attempts++;
2186 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2188 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2189 ANNOTATE_QUEUING_ACQUIRED(lck);
2192 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2194 char const *
const func =
"omp_set_lock";
2195 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2196 KMP_FATAL(LockIsUninitialized, func);
2198 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2199 KMP_FATAL(LockIsAlreadyOwned, func);
2202 __kmp_acquire_adaptive_lock(lck, gtid);
2204 lck->lk.qlk.owner_id = gtid + 1;
2207 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2209 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
2214 __kmp_update_badness_after_success(lck);
2217 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2219 return KMP_LOCK_RELEASED;
2222 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2224 char const *
const func =
"omp_unset_lock";
2226 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2227 KMP_FATAL(LockIsUninitialized, func);
2229 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2230 KMP_FATAL(LockUnsettingFree, func);
2232 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2233 KMP_FATAL(LockUnsettingSetByAnother, func);
2235 lck->lk.qlk.owner_id = 0;
2236 __kmp_release_adaptive_lock(lck, gtid);
2237 return KMP_LOCK_RELEASED;
2240 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2241 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2242 lck->lk.adaptive.badness = 0;
2243 lck->lk.adaptive.acquire_attempts = 0;
2244 lck->lk.adaptive.max_soft_retries =
2245 __kmp_adaptive_backoff_params.max_soft_retries;
2246 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2247 #if KMP_DEBUG_ADAPTIVE_LOCKS 2248 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2250 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2253 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2254 __kmp_init_adaptive_lock(lck);
2257 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2258 #if KMP_DEBUG_ADAPTIVE_LOCKS 2259 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2261 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2265 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2266 char const *
const func =
"omp_destroy_lock";
2267 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2268 KMP_FATAL(LockIsUninitialized, func);
2270 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2271 KMP_FATAL(LockStillOwned, func);
2273 __kmp_destroy_adaptive_lock(lck);
2276 #endif // KMP_USE_ADAPTIVE_LOCKS 2282 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2283 return TCR_4(lck->lk.owner_id) - 1;
2286 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2287 return lck->lk.depth_locked != -1;
2290 __forceinline
static int 2291 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2293 KMP_TEST_THEN_INC64(RCAST(
volatile kmp_int64 *, &lck->lk.next_ticket));
2294 kmp_uint64 mask = TCR_8(lck->lk.mask);
2295 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls = lck->lk.polls;
2297 #ifdef USE_LOCK_PROFILE 2298 if (TCR_8(polls[ticket & mask].poll) != ticket)
2299 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2313 KMP_FSYNC_PREPARE(lck);
2314 KMP_INIT_YIELD(spins);
2315 while (TCR_8(polls[ticket & mask].poll) < ticket) {
2320 KMP_YIELD(TCR_4(__kmp_nth) >
2321 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2322 KMP_YIELD_SPIN(spins);
2331 mask = TCR_8(lck->lk.mask);
2332 polls = lck->lk.polls;
2336 KMP_FSYNC_ACQUIRED(lck);
2337 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2339 lck->lk.now_serving = ticket;
2346 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2347 __kmp_free(CCAST(kmp_base_drdpa_lock::kmp_lock_poll *, lck->lk.old_polls));
2348 lck->lk.old_polls = NULL;
2349 lck->lk.cleanup_ticket = 0;
2355 if (lck->lk.old_polls == NULL) {
2356 bool reconfigure =
false;
2357 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2358 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2360 if (TCR_4(__kmp_nth) >
2361 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2364 if (num_polls > 1) {
2366 num_polls = TCR_4(lck->lk.num_polls);
2369 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2370 __kmp_allocate(num_polls *
sizeof(*polls));
2371 polls[0].poll = ticket;
2377 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2378 if (num_waiting > num_polls) {
2379 kmp_uint32 old_num_polls = num_polls;
2382 mask = (mask << 1) | 1;
2384 }
while (num_polls <= num_waiting);
2390 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2391 __kmp_allocate(num_polls *
sizeof(*polls));
2393 for (i = 0; i < old_num_polls; i++) {
2394 polls[i].poll = old_polls[i].poll;
2409 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring " 2410 "lock %p to %d polls\n",
2411 ticket, lck, num_polls));
2413 lck->lk.old_polls = old_polls;
2414 lck->lk.polls = polls;
2418 lck->lk.num_polls = num_polls;
2419 lck->lk.mask = mask;
2427 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2430 return KMP_LOCK_ACQUIRED_FIRST;
2433 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2434 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2435 ANNOTATE_DRDPA_ACQUIRED(lck);
2439 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2441 char const *
const func =
"omp_set_lock";
2442 if (lck->lk.initialized != lck) {
2443 KMP_FATAL(LockIsUninitialized, func);
2445 if (__kmp_is_drdpa_lock_nestable(lck)) {
2446 KMP_FATAL(LockNestableUsedAsSimple, func);
2448 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2449 KMP_FATAL(LockIsAlreadyOwned, func);
2452 __kmp_acquire_drdpa_lock(lck, gtid);
2454 lck->lk.owner_id = gtid + 1;
2455 return KMP_LOCK_ACQUIRED_FIRST;
2458 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2461 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2462 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls = lck->lk.polls;
2463 kmp_uint64 mask = TCR_8(lck->lk.mask);
2464 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2465 kmp_uint64 next_ticket = ticket + 1;
2466 if (KMP_COMPARE_AND_STORE_ACQ64(&lck->lk.next_ticket, ticket,
2468 KMP_FSYNC_ACQUIRED(lck);
2469 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2471 lck->lk.now_serving = ticket;
2485 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2487 char const *
const func =
"omp_test_lock";
2488 if (lck->lk.initialized != lck) {
2489 KMP_FATAL(LockIsUninitialized, func);
2491 if (__kmp_is_drdpa_lock_nestable(lck)) {
2492 KMP_FATAL(LockNestableUsedAsSimple, func);
2495 int retval = __kmp_test_drdpa_lock(lck, gtid);
2498 lck->lk.owner_id = gtid + 1;
2503 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2506 kmp_uint64 ticket = lck->lk.now_serving + 1;
2507 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls = lck->lk.polls;
2508 kmp_uint64 mask = TCR_8(lck->lk.mask);
2509 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2511 KMP_FSYNC_RELEASING(lck);
2512 ANNOTATE_DRDPA_RELEASED(lck);
2513 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2514 return KMP_LOCK_RELEASED;
2517 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2519 char const *
const func =
"omp_unset_lock";
2521 if (lck->lk.initialized != lck) {
2522 KMP_FATAL(LockIsUninitialized, func);
2524 if (__kmp_is_drdpa_lock_nestable(lck)) {
2525 KMP_FATAL(LockNestableUsedAsSimple, func);
2527 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2528 KMP_FATAL(LockUnsettingFree, func);
2530 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2531 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2532 KMP_FATAL(LockUnsettingSetByAnother, func);
2534 lck->lk.owner_id = 0;
2535 return __kmp_release_drdpa_lock(lck, gtid);
2538 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2539 lck->lk.location = NULL;
2541 lck->lk.num_polls = 1;
2543 (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)__kmp_allocate(
2544 lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2545 lck->lk.cleanup_ticket = 0;
2546 lck->lk.old_polls = NULL;
2547 lck->lk.next_ticket = 0;
2548 lck->lk.now_serving = 0;
2549 lck->lk.owner_id = 0;
2550 lck->lk.depth_locked = -1;
2551 lck->lk.initialized = lck;
2553 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2556 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2557 __kmp_init_drdpa_lock(lck);
2560 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2561 lck->lk.initialized = NULL;
2562 lck->lk.location = NULL;
2563 if (lck->lk.polls != NULL) {
2564 __kmp_free(CCAST(kmp_base_drdpa_lock::kmp_lock_poll *, lck->lk.polls));
2565 lck->lk.polls = NULL;
2567 if (lck->lk.old_polls != NULL) {
2568 __kmp_free(CCAST(kmp_base_drdpa_lock::kmp_lock_poll *, lck->lk.old_polls));
2569 lck->lk.old_polls = NULL;
2572 lck->lk.num_polls = 0;
2573 lck->lk.cleanup_ticket = 0;
2574 lck->lk.next_ticket = 0;
2575 lck->lk.now_serving = 0;
2576 lck->lk.owner_id = 0;
2577 lck->lk.depth_locked = -1;
2580 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2581 char const *
const func =
"omp_destroy_lock";
2582 if (lck->lk.initialized != lck) {
2583 KMP_FATAL(LockIsUninitialized, func);
2585 if (__kmp_is_drdpa_lock_nestable(lck)) {
2586 KMP_FATAL(LockNestableUsedAsSimple, func);
2588 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2589 KMP_FATAL(LockStillOwned, func);
2591 __kmp_destroy_drdpa_lock(lck);
2596 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2597 KMP_DEBUG_ASSERT(gtid >= 0);
2599 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2600 lck->lk.depth_locked += 1;
2601 return KMP_LOCK_ACQUIRED_NEXT;
2603 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2604 ANNOTATE_DRDPA_ACQUIRED(lck);
2606 lck->lk.depth_locked = 1;
2608 lck->lk.owner_id = gtid + 1;
2609 return KMP_LOCK_ACQUIRED_FIRST;
2613 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2615 char const *
const func =
"omp_set_nest_lock";
2616 if (lck->lk.initialized != lck) {
2617 KMP_FATAL(LockIsUninitialized, func);
2619 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2620 KMP_FATAL(LockSimpleUsedAsNestable, func);
2622 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2625 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2628 KMP_DEBUG_ASSERT(gtid >= 0);
2630 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2631 retval = ++lck->lk.depth_locked;
2632 }
else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2636 retval = lck->lk.depth_locked = 1;
2638 lck->lk.owner_id = gtid + 1;
2643 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2645 char const *
const func =
"omp_test_nest_lock";
2646 if (lck->lk.initialized != lck) {
2647 KMP_FATAL(LockIsUninitialized, func);
2649 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2650 KMP_FATAL(LockSimpleUsedAsNestable, func);
2652 return __kmp_test_nested_drdpa_lock(lck, gtid);
2655 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2656 KMP_DEBUG_ASSERT(gtid >= 0);
2659 if (--(lck->lk.depth_locked) == 0) {
2661 lck->lk.owner_id = 0;
2662 __kmp_release_drdpa_lock(lck, gtid);
2663 return KMP_LOCK_RELEASED;
2665 return KMP_LOCK_STILL_HELD;
2668 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2670 char const *
const func =
"omp_unset_nest_lock";
2672 if (lck->lk.initialized != lck) {
2673 KMP_FATAL(LockIsUninitialized, func);
2675 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2676 KMP_FATAL(LockSimpleUsedAsNestable, func);
2678 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2679 KMP_FATAL(LockUnsettingFree, func);
2681 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2682 KMP_FATAL(LockUnsettingSetByAnother, func);
2684 return __kmp_release_nested_drdpa_lock(lck, gtid);
2687 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2688 __kmp_init_drdpa_lock(lck);
2689 lck->lk.depth_locked = 0;
2692 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2693 __kmp_init_nested_drdpa_lock(lck);
2696 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2697 __kmp_destroy_drdpa_lock(lck);
2698 lck->lk.depth_locked = 0;
2701 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2702 char const *
const func =
"omp_destroy_nest_lock";
2703 if (lck->lk.initialized != lck) {
2704 KMP_FATAL(LockIsUninitialized, func);
2706 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2707 KMP_FATAL(LockSimpleUsedAsNestable, func);
2709 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2710 KMP_FATAL(LockStillOwned, func);
2712 __kmp_destroy_nested_drdpa_lock(lck);
2717 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
2718 return lck == lck->lk.initialized;
2721 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2722 return lck->lk.location;
2725 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2727 lck->lk.location = loc;
2730 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2731 return lck->lk.flags;
2734 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2735 kmp_lock_flags_t flags) {
2736 lck->lk.flags = flags;
2740 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 2741 #define __kmp_tsc() __kmp_hardware_timestamp() 2743 kmp_backoff_t __kmp_spin_backoff_params = {1, 4096, 100};
2746 extern kmp_uint64 __kmp_now_nsec();
2747 kmp_backoff_t __kmp_spin_backoff_params = {1, 256, 100};
2748 #define __kmp_tsc() __kmp_now_nsec() 2758 static inline bool before(kmp_uint64 a, kmp_uint64 b) {
2759 return ((kmp_int64)b - (kmp_int64)a) > 0;
2763 void __kmp_spin_backoff(kmp_backoff_t *boff) {
2766 for (i = boff->step; i > 0; i--) {
2767 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
2770 }
while (before(__kmp_tsc(), goal));
2772 boff->step = (boff->step << 1 | 1) & (boff->max_backoff - 1);
2775 #if KMP_USE_DYNAMIC_LOCK 2779 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2780 kmp_dyna_lockseq_t seq) {
2781 TCW_4(*lck, KMP_GET_D_TAG(seq));
2784 (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
2790 #define HLE_ACQUIRE ".byte 0xf2;" 2791 #define HLE_RELEASE ".byte 0xf3;" 2793 static inline kmp_uint32 swap4(kmp_uint32
volatile *p, kmp_uint32 v) {
2794 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" :
"+r"(v),
"+m"(*p) : :
"memory");
2798 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2800 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2802 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2805 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
2806 for (
int i = delay; i != 0; --i)
2808 delay = ((delay << 1) | 1) & 7;
2810 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2814 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2816 __kmp_acquire_hle_lock(lck, gtid);
2819 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2820 __asm__
volatile(HLE_RELEASE
"movl %1,%0" 2822 :
"r"(KMP_LOCK_FREE(hle))
2824 return KMP_LOCK_RELEASED;
2827 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2829 return __kmp_release_hle_lock(lck, gtid);
2832 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2833 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2836 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2838 return __kmp_test_hle_lock(lck, gtid);
2841 static void __kmp_init_rtm_lock(kmp_queuing_lock_t *lck) {
2842 __kmp_init_queuing_lock(lck);
2845 static void __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck) {
2846 __kmp_destroy_queuing_lock(lck);
2849 static void __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2850 unsigned retries = 3, status;
2853 if (status == _XBEGIN_STARTED) {
2854 if (__kmp_is_unlocked_queuing_lock(lck))
2858 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2860 while (!__kmp_is_unlocked_queuing_lock(lck))
2862 }
else if (!(status & _XABORT_RETRY))
2864 }
while (retries--);
2867 __kmp_acquire_queuing_lock(lck, gtid);
2870 static void __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2872 __kmp_acquire_rtm_lock(lck, gtid);
2875 static int __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2876 if (__kmp_is_unlocked_queuing_lock(lck)) {
2881 __kmp_release_queuing_lock(lck, gtid);
2883 return KMP_LOCK_RELEASED;
2886 static int __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2888 return __kmp_release_rtm_lock(lck, gtid);
2891 static int __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2892 unsigned retries = 3, status;
2895 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2898 if (!(status & _XABORT_RETRY))
2900 }
while (retries--);
2902 return (__kmp_is_unlocked_queuing_lock(lck)) ? 1 : 0;
2905 static int __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2907 return __kmp_test_rtm_lock(lck, gtid);
2910 #endif // KMP_USE_TSX 2913 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *l,
2914 kmp_dyna_lockseq_t tag);
2915 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock);
2916 static void __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2917 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2918 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2919 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2921 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2923 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2930 #define expand(l, op) 0, __kmp_init_direct_lock, 2931 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t) = {
2932 __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init)};
2936 #define expand(l, op) 0, (void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock, 2937 void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *) = {
2938 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2942 #define expand(l, op) \ 2943 0, (void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 2944 static void (*direct_set[])(kmp_dyna_lock_t *, kmp_int32) = {
2945 __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire)};
2947 #define expand(l, op) \ 2948 0, (void (*)(kmp_dyna_lock_t *, \ 2949 kmp_int32))__kmp_##op##_##l##_lock_with_checks, 2950 static void (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2951 __kmp_set_indirect_lock_with_checks, 0,
2952 KMP_FOREACH_D_LOCK(expand, acquire)};
2956 #define expand(l, op) \ 2957 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 2958 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32) = {
2959 __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release)};
2960 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32) = {
2961 __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test)};
2963 #define expand(l, op) \ 2964 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 2965 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2966 __kmp_unset_indirect_lock_with_checks, 0,
2967 KMP_FOREACH_D_LOCK(expand, release)};
2968 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2969 __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test)};
2973 void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
2974 int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
2975 int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
2978 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock, 2979 void (*__kmp_indirect_init[])(kmp_user_lock_p) = {
2980 KMP_FOREACH_I_LOCK(expand, init)};
2981 void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = {
2982 KMP_FOREACH_I_LOCK(expand, destroy)};
2986 #define expand(l, op) \ 2987 (void (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock, 2988 static void (*indirect_set[])(kmp_user_lock_p, kmp_int32) = {
2989 KMP_FOREACH_I_LOCK(expand, acquire)};
2991 #define expand(l, op) \ 2992 (void (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks, 2993 static void (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = {
2994 KMP_FOREACH_I_LOCK(expand, acquire)};
2998 #define expand(l, op) \ 2999 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock, 3000 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = {
3001 KMP_FOREACH_I_LOCK(expand, release)};
3002 static int (*indirect_test[])(kmp_user_lock_p,
3003 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, test)};
3005 #define expand(l, op) \ 3006 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks, 3007 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = {
3008 KMP_FOREACH_I_LOCK(expand, release)};
3009 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = {
3010 KMP_FOREACH_I_LOCK(expand, test)};
3014 void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
3015 int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
3016 int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
3019 kmp_indirect_lock_table_t __kmp_i_lock_table;
3022 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {0};
3025 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3027 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3028 kmp_lock_flags_t) = {0};
3029 const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
3030 kmp_user_lock_p) = {0};
3031 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
3032 kmp_user_lock_p) = {0};
3035 static kmp_indirect_lock_t *__kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = {0};
3042 kmp_indirect_lock_t *__kmp_allocate_indirect_lock(
void **user_lock,
3044 kmp_indirect_locktag_t tag) {
3045 kmp_indirect_lock_t *lck;
3046 kmp_lock_index_t idx;
3048 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3050 if (__kmp_indirect_lock_pool[tag] != NULL) {
3052 lck = __kmp_indirect_lock_pool[tag];
3053 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3054 idx = lck->lock->pool.index;
3055 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3056 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n",
3059 idx = __kmp_i_lock_table.next;
3061 if (idx == __kmp_i_lock_table.size) {
3063 int row = __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK;
3064 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3065 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(
3066 2 * row *
sizeof(kmp_indirect_lock_t *));
3067 KMP_MEMCPY(__kmp_i_lock_table.table, old_table,
3068 row *
sizeof(kmp_indirect_lock_t *));
3069 __kmp_free(old_table);
3071 for (
int i = row; i < 2 * row; ++i)
3072 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)__kmp_allocate(
3073 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3074 __kmp_i_lock_table.size = 2 * idx;
3076 __kmp_i_lock_table.next++;
3077 lck = KMP_GET_I_LOCK(idx);
3079 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3081 (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3084 __kmp_release_lock(&__kmp_global_lock, gtid);
3088 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3089 *((kmp_lock_index_t *)user_lock) = idx
3092 *((kmp_indirect_lock_t **)user_lock) = lck;
3099 static __forceinline kmp_indirect_lock_t *
3100 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func) {
3101 if (__kmp_env_consistency_check) {
3102 kmp_indirect_lock_t *lck = NULL;
3103 if (user_lock == NULL) {
3104 KMP_FATAL(LockIsUninitialized, func);
3106 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3107 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3108 if (idx >= __kmp_i_lock_table.size) {
3109 KMP_FATAL(LockIsUninitialized, func);
3111 lck = KMP_GET_I_LOCK(idx);
3113 lck = *((kmp_indirect_lock_t **)user_lock);
3116 KMP_FATAL(LockIsUninitialized, func);
3120 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3121 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3123 return *((kmp_indirect_lock_t **)user_lock);
3128 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
3129 kmp_dyna_lockseq_t seq) {
3130 #if KMP_USE_ADAPTIVE_LOCKS 3131 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3132 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3133 seq = lockseq_queuing;
3137 if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
3138 seq = lockseq_queuing;
3141 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3142 kmp_indirect_lock_t *l =
3143 __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3144 KMP_I_LOCK_FUNC(l, init)(l->lock);
3146 20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n",
3150 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
3151 kmp_uint32 gtid = __kmp_entry_gtid();
3152 kmp_indirect_lock_t *l =
3153 __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3154 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3155 kmp_indirect_locktag_t tag = l->type;
3157 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3160 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3161 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3162 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3164 __kmp_indirect_lock_pool[tag] = l;
3166 __kmp_release_lock(&__kmp_global_lock, gtid);
3169 static void __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3170 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3171 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3174 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3175 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3176 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3179 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3180 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3181 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3184 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3186 kmp_indirect_lock_t *l =
3187 __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3188 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3191 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3193 kmp_indirect_lock_t *l =
3194 __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3195 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3198 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3200 kmp_indirect_lock_t *l =
3201 __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3202 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3205 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3208 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3211 case lockseq_nested_tas:
3212 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3215 case lockseq_nested_futex:
3216 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3218 case lockseq_ticket:
3219 case lockseq_nested_ticket:
3220 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3221 case lockseq_queuing:
3222 case lockseq_nested_queuing:
3223 #if KMP_USE_ADAPTIVE_LOCKS 3224 case lockseq_adaptive:
3226 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3228 case lockseq_nested_drdpa:
3229 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3236 void __kmp_init_dynamic_user_locks() {
3238 if (__kmp_env_consistency_check) {
3239 __kmp_direct_set = direct_set_check;
3240 __kmp_direct_unset = direct_unset_check;
3241 __kmp_direct_test = direct_test_check;
3242 __kmp_indirect_set = indirect_set_check;
3243 __kmp_indirect_unset = indirect_unset_check;
3244 __kmp_indirect_test = indirect_test_check;
3246 __kmp_direct_set = direct_set;
3247 __kmp_direct_unset = direct_unset;
3248 __kmp_direct_test = direct_test;
3249 __kmp_indirect_set = indirect_set;
3250 __kmp_indirect_unset = indirect_unset;
3251 __kmp_indirect_test = indirect_test;
3256 if (__kmp_init_user_locks)
3260 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3261 __kmp_i_lock_table.table =
3262 (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3263 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)__kmp_allocate(
3264 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3265 __kmp_i_lock_table.next = 0;
3268 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3269 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3270 #if KMP_USE_ADAPTIVE_LOCKS 3271 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3273 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3275 __kmp_indirect_lock_size[locktag_rtm] =
sizeof(kmp_queuing_lock_t);
3277 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3279 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3281 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3282 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3283 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3286 #define fill_jumps(table, expand, sep) \ 3288 table[locktag##sep##ticket] = expand(ticket); \ 3289 table[locktag##sep##queuing] = expand(queuing); \ 3290 table[locktag##sep##drdpa] = expand(drdpa); \ 3293 #if KMP_USE_ADAPTIVE_LOCKS 3294 #define fill_table(table, expand) \ 3296 fill_jumps(table, expand, _); \ 3297 table[locktag_adaptive] = expand(queuing); \ 3298 fill_jumps(table, expand, _nested_); \ 3301 #define fill_table(table, expand) \ 3303 fill_jumps(table, expand, _); \ 3304 fill_jumps(table, expand, _nested_); \ 3306 #endif // KMP_USE_ADAPTIVE_LOCKS 3309 (void (*)(kmp_user_lock_p, const ident_t *)) __kmp_set_##l##_lock_location 3310 fill_table(__kmp_indirect_set_location, expand);
3313 (void (*)(kmp_user_lock_p, kmp_lock_flags_t)) __kmp_set_##l##_lock_flags 3314 fill_table(__kmp_indirect_set_flags, expand);
3317 (const ident_t *(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_location 3318 fill_table(__kmp_indirect_get_location, expand);
3321 (kmp_lock_flags_t(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_flags 3322 fill_table(__kmp_indirect_get_flags, expand);
3325 __kmp_init_user_locks = TRUE;
3329 void __kmp_cleanup_indirect_user_locks() {
3335 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3336 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3338 kmp_indirect_lock_t *ll = l;
3339 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3340 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n",
3342 __kmp_free(ll->lock);
3345 __kmp_indirect_lock_pool[k] = NULL;
3348 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3349 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3350 if (l->lock != NULL) {
3352 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3355 (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n",
3357 __kmp_free(l->lock);
3361 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3362 __kmp_free(__kmp_i_lock_table.table[i]);
3363 __kmp_free(__kmp_i_lock_table.table);
3365 __kmp_init_user_locks = FALSE;
3368 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3369 int __kmp_num_locks_in_block = 1;
3371 #else // KMP_USE_DYNAMIC_LOCK 3377 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3379 size_t __kmp_base_user_lock_size = 0;
3380 size_t __kmp_user_lock_size = 0;
3382 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3383 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3384 kmp_int32 gtid) = NULL;
3386 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3387 kmp_int32 gtid) = NULL;
3388 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3389 kmp_int32 gtid) = NULL;
3390 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3391 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3392 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3393 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3394 kmp_int32 gtid) = NULL;
3396 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3397 kmp_int32 gtid) = NULL;
3398 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3399 kmp_int32 gtid) = NULL;
3400 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3401 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3403 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3404 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3405 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3407 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3408 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3409 kmp_lock_flags_t flags) = NULL;
3411 void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind) {
3412 switch (user_lock_kind) {
3418 __kmp_base_user_lock_size =
sizeof(kmp_base_tas_lock_t);
3419 __kmp_user_lock_size =
sizeof(kmp_tas_lock_t);
3421 __kmp_get_user_lock_owner_ =
3422 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_tas_lock_owner);
3424 if (__kmp_env_consistency_check) {
3425 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3426 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3428 KMP_BIND_USER_LOCK(tas);
3429 KMP_BIND_NESTED_USER_LOCK(tas);
3432 __kmp_destroy_user_lock_ =
3433 (void (*)(kmp_user_lock_p))(&__kmp_destroy_tas_lock);
3435 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3437 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3439 __kmp_set_user_lock_location_ =
3440 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3442 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3444 __kmp_set_user_lock_flags_ =
3445 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3451 __kmp_base_user_lock_size =
sizeof(kmp_base_futex_lock_t);
3452 __kmp_user_lock_size =
sizeof(kmp_futex_lock_t);
3454 __kmp_get_user_lock_owner_ =
3455 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_futex_lock_owner);
3457 if (__kmp_env_consistency_check) {
3458 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3459 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3461 KMP_BIND_USER_LOCK(futex);
3462 KMP_BIND_NESTED_USER_LOCK(futex);
3465 __kmp_destroy_user_lock_ =
3466 (void (*)(kmp_user_lock_p))(&__kmp_destroy_futex_lock);
3468 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3470 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3472 __kmp_set_user_lock_location_ =
3473 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3475 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3477 __kmp_set_user_lock_flags_ =
3478 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3481 #endif // KMP_USE_FUTEX 3484 __kmp_base_user_lock_size =
sizeof(kmp_base_ticket_lock_t);
3485 __kmp_user_lock_size =
sizeof(kmp_ticket_lock_t);
3487 __kmp_get_user_lock_owner_ =
3488 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_owner);
3490 if (__kmp_env_consistency_check) {
3491 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3492 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3494 KMP_BIND_USER_LOCK(ticket);
3495 KMP_BIND_NESTED_USER_LOCK(ticket);
3498 __kmp_destroy_user_lock_ =
3499 (void (*)(kmp_user_lock_p))(&__kmp_destroy_ticket_lock);
3501 __kmp_is_user_lock_initialized_ =
3502 (int (*)(kmp_user_lock_p))(&__kmp_is_ticket_lock_initialized);
3504 __kmp_get_user_lock_location_ =
3505 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_location);
3507 __kmp_set_user_lock_location_ = (void (*)(
3508 kmp_user_lock_p,
const ident_t *))(&__kmp_set_ticket_lock_location);
3510 __kmp_get_user_lock_flags_ =
3511 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_flags);
3513 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3514 &__kmp_set_ticket_lock_flags);
3518 __kmp_base_user_lock_size =
sizeof(kmp_base_queuing_lock_t);
3519 __kmp_user_lock_size =
sizeof(kmp_queuing_lock_t);
3521 __kmp_get_user_lock_owner_ =
3522 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3524 if (__kmp_env_consistency_check) {
3525 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3526 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3528 KMP_BIND_USER_LOCK(queuing);
3529 KMP_BIND_NESTED_USER_LOCK(queuing);
3532 __kmp_destroy_user_lock_ =
3533 (void (*)(kmp_user_lock_p))(&__kmp_destroy_queuing_lock);
3535 __kmp_is_user_lock_initialized_ =
3536 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3538 __kmp_get_user_lock_location_ =
3539 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3541 __kmp_set_user_lock_location_ = (void (*)(
3542 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3544 __kmp_get_user_lock_flags_ =
3545 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3547 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3548 &__kmp_set_queuing_lock_flags);
3551 #if KMP_USE_ADAPTIVE_LOCKS 3553 __kmp_base_user_lock_size =
sizeof(kmp_base_adaptive_lock_t);
3554 __kmp_user_lock_size =
sizeof(kmp_adaptive_lock_t);
3556 __kmp_get_user_lock_owner_ =
3557 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3559 if (__kmp_env_consistency_check) {
3560 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3562 KMP_BIND_USER_LOCK(adaptive);
3565 __kmp_destroy_user_lock_ =
3566 (void (*)(kmp_user_lock_p))(&__kmp_destroy_adaptive_lock);
3568 __kmp_is_user_lock_initialized_ =
3569 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3571 __kmp_get_user_lock_location_ =
3572 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3574 __kmp_set_user_lock_location_ = (void (*)(
3575 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3577 __kmp_get_user_lock_flags_ =
3578 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3580 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3581 &__kmp_set_queuing_lock_flags);
3584 #endif // KMP_USE_ADAPTIVE_LOCKS 3587 __kmp_base_user_lock_size =
sizeof(kmp_base_drdpa_lock_t);
3588 __kmp_user_lock_size =
sizeof(kmp_drdpa_lock_t);
3590 __kmp_get_user_lock_owner_ =
3591 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_owner);
3593 if (__kmp_env_consistency_check) {
3594 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3595 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3597 KMP_BIND_USER_LOCK(drdpa);
3598 KMP_BIND_NESTED_USER_LOCK(drdpa);
3601 __kmp_destroy_user_lock_ =
3602 (void (*)(kmp_user_lock_p))(&__kmp_destroy_drdpa_lock);
3604 __kmp_is_user_lock_initialized_ =
3605 (int (*)(kmp_user_lock_p))(&__kmp_is_drdpa_lock_initialized);
3607 __kmp_get_user_lock_location_ =
3608 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_location);
3610 __kmp_set_user_lock_location_ = (void (*)(
3611 kmp_user_lock_p,
const ident_t *))(&__kmp_set_drdpa_lock_location);
3613 __kmp_get_user_lock_flags_ =
3614 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_flags);
3616 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3617 &__kmp_set_drdpa_lock_flags);
3625 kmp_lock_table_t __kmp_user_lock_table = {1, 0, NULL};
3626 kmp_user_lock_p __kmp_lock_pool = NULL;
3629 kmp_block_of_locks *__kmp_lock_blocks = NULL;
3630 int __kmp_num_locks_in_block = 1;
3632 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3634 kmp_lock_index_t index;
3635 if (__kmp_user_lock_table.used >= __kmp_user_lock_table.allocated) {
3636 kmp_lock_index_t size;
3637 kmp_user_lock_p *table;
3639 if (__kmp_user_lock_table.allocated == 0) {
3642 size = __kmp_user_lock_table.allocated * 2;
3644 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof(kmp_user_lock_p) * size);
3645 KMP_MEMCPY(table + 1, __kmp_user_lock_table.table + 1,
3646 sizeof(kmp_user_lock_p) * (__kmp_user_lock_table.used - 1));
3647 table[0] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3652 __kmp_user_lock_table.table = table;
3653 __kmp_user_lock_table.allocated = size;
3655 KMP_DEBUG_ASSERT(__kmp_user_lock_table.used <
3656 __kmp_user_lock_table.allocated);
3657 index = __kmp_user_lock_table.used;
3658 __kmp_user_lock_table.table[index] = lck;
3659 ++__kmp_user_lock_table.used;
3663 static kmp_user_lock_p __kmp_lock_block_allocate() {
3665 static int last_index = 0;
3666 if ((last_index >= __kmp_num_locks_in_block) || (__kmp_lock_blocks == NULL)) {
3670 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3671 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3673 (
char *)__kmp_allocate(space_for_locks +
sizeof(kmp_block_of_locks));
3675 kmp_block_of_locks *new_block =
3676 (kmp_block_of_locks *)(&buffer[space_for_locks]);
3677 new_block->next_block = __kmp_lock_blocks;
3678 new_block->locks = (
void *)buffer;
3681 __kmp_lock_blocks = new_block;
3683 kmp_user_lock_p ret = (kmp_user_lock_p)(&(
3684 ((
char *)(__kmp_lock_blocks->locks))[last_index * __kmp_user_lock_size]));
3691 kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3692 kmp_lock_flags_t flags) {
3693 kmp_user_lock_p lck;
3694 kmp_lock_index_t index;
3695 KMP_DEBUG_ASSERT(user_lock);
3697 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3699 if (__kmp_lock_pool == NULL) {
3704 ANNOTATE_IGNORE_WRITES_BEGIN();
3705 if (__kmp_num_locks_in_block <= 1) {
3706 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3708 lck = __kmp_lock_block_allocate();
3710 ANNOTATE_IGNORE_WRITES_END();
3714 index = __kmp_lock_table_insert(lck);
3717 lck = __kmp_lock_pool;
3718 index = __kmp_lock_pool->pool.index;
3719 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3724 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3725 *((kmp_lock_index_t *)user_lock) = index;
3727 *((kmp_user_lock_p *)user_lock) = lck;
3731 __kmp_set_user_lock_flags(lck, flags);
3733 __kmp_release_lock(&__kmp_global_lock, gtid);
3739 void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
3740 kmp_user_lock_p lck) {
3741 KMP_DEBUG_ASSERT(user_lock != NULL);
3742 KMP_DEBUG_ASSERT(lck != NULL);
3744 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3746 lck->pool.next = __kmp_lock_pool;
3747 __kmp_lock_pool = lck;
3748 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3749 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3750 KMP_DEBUG_ASSERT(0 < index && index <= __kmp_user_lock_table.used);
3751 lck->pool.index = index;
3754 __kmp_release_lock(&__kmp_global_lock, gtid);
3757 kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func) {
3758 kmp_user_lock_p lck = NULL;
3760 if (__kmp_env_consistency_check) {
3761 if (user_lock == NULL) {
3762 KMP_FATAL(LockIsUninitialized, func);
3766 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3767 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3768 if (__kmp_env_consistency_check) {
3769 if (!(0 < index && index < __kmp_user_lock_table.used)) {
3770 KMP_FATAL(LockIsUninitialized, func);
3773 KMP_DEBUG_ASSERT(0 < index && index < __kmp_user_lock_table.used);
3774 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3775 lck = __kmp_user_lock_table.table[index];
3777 lck = *((kmp_user_lock_p *)user_lock);
3780 if (__kmp_env_consistency_check) {
3782 KMP_FATAL(LockIsUninitialized, func);
3789 void __kmp_cleanup_user_locks(
void) {
3792 __kmp_lock_pool = NULL;
3794 #define IS_CRITICAL(lck) \ 3795 ((__kmp_get_user_lock_flags_ != NULL) && \ 3796 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section)) 3821 while (__kmp_user_lock_table.used > 1) {
3826 kmp_user_lock_p lck =
3827 __kmp_user_lock_table.table[--__kmp_user_lock_table.used];
3829 if ((__kmp_is_user_lock_initialized_ != NULL) &&
3830 (*__kmp_is_user_lock_initialized_)(lck)) {
3834 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3835 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
3837 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->
psource, 0);
3838 KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
3839 __kmp_str_loc_free(&str_loc);
3843 if (IS_CRITICAL(lck)) {
3846 (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n",
3847 lck, *(
void **)lck));
3849 KA_TRACE(20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
3855 __kmp_destroy_user_lock(lck);
3859 if (__kmp_lock_blocks == NULL) {
3867 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
3868 __kmp_user_lock_table.table = NULL;
3869 __kmp_user_lock_table.allocated = 0;
3871 while (table_ptr != NULL) {
3874 kmp_user_lock_p *next = (kmp_user_lock_p *)(table_ptr[0]);
3875 __kmp_free(table_ptr);
3880 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
3881 __kmp_lock_blocks = NULL;
3883 while (block_ptr != NULL) {
3884 kmp_block_of_locks_t *next = block_ptr->next_block;
3885 __kmp_free(block_ptr->locks);
3890 TCW_4(__kmp_init_user_locks, FALSE);
3893 #endif // KMP_USE_DYNAMIC_LOCK