43 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
45 # include <sys/syscall.h>
63 __kmp_validate_locks(
void )
69 x = ~((kmp_uint32) 0) - 2;
72 for (i = 0; i < 8; ++i, ++x, ++y) {
73 kmp_uint32 z = (x - y);
77 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
95 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
97 return DYNA_LOCK_STRIP(TCR_4( lck->lk.poll )) - 1;
101 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
103 return lck->lk.depth_locked != -1;
106 __forceinline
static void
107 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
111 #ifdef USE_LOCK_PROFILE
112 kmp_uint32 curr = TCR_4( lck->lk.poll );
113 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
114 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
118 if ( ( lck->lk.poll == DYNA_LOCK_FREE(tas) )
119 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) {
120 KMP_FSYNC_ACQUIRED(lck);
125 KMP_FSYNC_PREPARE( lck );
126 KMP_INIT_YIELD( spins );
127 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
132 KMP_YIELD_SPIN( spins );
135 while ( ( lck->lk.poll != DYNA_LOCK_FREE(tas) ) ||
136 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) ) {
140 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
145 KMP_YIELD_SPIN( spins );
148 KMP_FSYNC_ACQUIRED( lck );
152 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
154 __kmp_acquire_tas_lock_timed_template( lck, gtid );
158 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
160 char const *
const func =
"omp_set_lock";
161 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
162 && __kmp_is_tas_lock_nestable( lck ) ) {
163 KMP_FATAL( LockNestableUsedAsSimple, func );
165 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
166 KMP_FATAL( LockIsAlreadyOwned, func );
168 __kmp_acquire_tas_lock( lck, gtid );
172 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
174 if ( ( lck->lk.poll == DYNA_LOCK_FREE(tas) )
175 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(tas), DYNA_LOCK_BUSY(gtid+1, tas) ) ) {
176 KMP_FSYNC_ACQUIRED( lck );
183 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
185 char const *
const func =
"omp_test_lock";
186 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
187 && __kmp_is_tas_lock_nestable( lck ) ) {
188 KMP_FATAL( LockNestableUsedAsSimple, func );
190 return __kmp_test_tas_lock( lck, gtid );
194 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
198 KMP_FSYNC_RELEASING(lck);
199 KMP_ST_REL32( &(lck->lk.poll), DYNA_LOCK_FREE(tas) );
202 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
207 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
209 char const *
const func =
"omp_unset_lock";
211 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
212 && __kmp_is_tas_lock_nestable( lck ) ) {
213 KMP_FATAL( LockNestableUsedAsSimple, func );
215 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
216 KMP_FATAL( LockUnsettingFree, func );
218 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
219 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
220 KMP_FATAL( LockUnsettingSetByAnother, func );
222 __kmp_release_tas_lock( lck, gtid );
226 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
228 TCW_4( lck->lk.poll, DYNA_LOCK_FREE(tas) );
232 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
234 __kmp_init_tas_lock( lck );
238 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
244 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
246 char const *
const func =
"omp_destroy_lock";
247 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
248 && __kmp_is_tas_lock_nestable( lck ) ) {
249 KMP_FATAL( LockNestableUsedAsSimple, func );
251 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
252 KMP_FATAL( LockStillOwned, func );
254 __kmp_destroy_tas_lock( lck );
263 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
265 KMP_DEBUG_ASSERT( gtid >= 0 );
267 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
268 lck->lk.depth_locked += 1;
271 __kmp_acquire_tas_lock_timed_template( lck, gtid );
272 lck->lk.depth_locked = 1;
277 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
279 char const *
const func =
"omp_set_nest_lock";
280 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
281 KMP_FATAL( LockSimpleUsedAsNestable, func );
283 __kmp_acquire_nested_tas_lock( lck, gtid );
287 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
291 KMP_DEBUG_ASSERT( gtid >= 0 );
293 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
294 retval = ++lck->lk.depth_locked;
296 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
301 retval = lck->lk.depth_locked = 1;
307 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
309 char const *
const func =
"omp_test_nest_lock";
310 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
311 KMP_FATAL( LockSimpleUsedAsNestable, func );
313 return __kmp_test_nested_tas_lock( lck, gtid );
317 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
319 KMP_DEBUG_ASSERT( gtid >= 0 );
322 if ( --(lck->lk.depth_locked) == 0 ) {
323 __kmp_release_tas_lock( lck, gtid );
328 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
330 char const *
const func =
"omp_unset_nest_lock";
332 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
333 KMP_FATAL( LockSimpleUsedAsNestable, func );
335 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
336 KMP_FATAL( LockUnsettingFree, func );
338 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
339 KMP_FATAL( LockUnsettingSetByAnother, func );
341 __kmp_release_nested_tas_lock( lck, gtid );
345 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
347 __kmp_init_tas_lock( lck );
348 lck->lk.depth_locked = 0;
352 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
354 __kmp_init_nested_tas_lock( lck );
358 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
360 __kmp_destroy_tas_lock( lck );
361 lck->lk.depth_locked = 0;
365 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
367 char const *
const func =
"omp_destroy_nest_lock";
368 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
369 KMP_FATAL( LockSimpleUsedAsNestable, func );
371 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
372 KMP_FATAL( LockStillOwned, func );
374 __kmp_destroy_nested_tas_lock( lck );
378 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
389 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
391 return DYNA_LOCK_STRIP(( TCR_4( lck->lk.poll ) >> 1 )) - 1;
395 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
397 return lck->lk.depth_locked != -1;
400 __forceinline
static void
401 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
403 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
407 #ifdef USE_LOCK_PROFILE
408 kmp_uint32 curr = TCR_4( lck->lk.poll );
409 if ( ( curr != 0 ) && ( curr != gtid_code ) )
410 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
414 KMP_FSYNC_PREPARE( lck );
415 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
416 lck, lck->lk.poll, gtid ) );
420 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex),
421 DYNA_LOCK_BUSY(gtid_code, futex) ) ) != DYNA_LOCK_FREE(futex) ) {
423 kmp_int32 cond = DYNA_LOCK_STRIP(poll_val) & 1;
424 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
425 lck, gtid, poll_val, cond ) );
440 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ), poll_val, poll_val | DYNA_LOCK_BUSY(1, futex) ) ) {
441 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
442 lck, lck->lk.poll, gtid ) );
445 poll_val |= DYNA_LOCK_BUSY(1, futex);
447 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
448 lck, lck->lk.poll, gtid ) );
451 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
452 lck, gtid, poll_val ) );
455 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
456 poll_val, NULL, NULL, 0 ) ) != 0 ) {
457 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
458 lck, gtid, poll_val, rc, errno ) );
462 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
463 lck, gtid, poll_val ) );
473 KMP_FSYNC_ACQUIRED( lck );
474 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
475 lck, lck->lk.poll, gtid ) );
479 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
481 __kmp_acquire_futex_lock_timed_template( lck, gtid );
485 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
487 char const *
const func =
"omp_set_lock";
488 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
489 && __kmp_is_futex_lock_nestable( lck ) ) {
490 KMP_FATAL( LockNestableUsedAsSimple, func );
492 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
493 KMP_FATAL( LockIsAlreadyOwned, func );
495 __kmp_acquire_futex_lock( lck, gtid );
499 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
501 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex), DYNA_LOCK_BUSY(gtid+1, futex) << 1 ) ) {
502 KMP_FSYNC_ACQUIRED( lck );
509 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
511 char const *
const func =
"omp_test_lock";
512 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
513 && __kmp_is_futex_lock_nestable( lck ) ) {
514 KMP_FATAL( LockNestableUsedAsSimple, func );
516 return __kmp_test_futex_lock( lck, gtid );
520 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
524 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
525 lck, lck->lk.poll, gtid ) );
527 KMP_FSYNC_RELEASING(lck);
529 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), DYNA_LOCK_FREE(futex) );
531 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
532 lck, gtid, poll_val ) );
534 if ( DYNA_LOCK_STRIP(poll_val) & 1 ) {
535 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
537 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, DYNA_LOCK_BUSY(1, futex), NULL, NULL, 0 );
542 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
543 lck, lck->lk.poll, gtid ) );
545 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
550 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
552 char const *
const func =
"omp_unset_lock";
554 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
555 && __kmp_is_futex_lock_nestable( lck ) ) {
556 KMP_FATAL( LockNestableUsedAsSimple, func );
558 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
559 KMP_FATAL( LockUnsettingFree, func );
561 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
562 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
563 KMP_FATAL( LockUnsettingSetByAnother, func );
565 __kmp_release_futex_lock( lck, gtid );
569 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
571 TCW_4( lck->lk.poll, DYNA_LOCK_FREE(futex) );
575 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
577 __kmp_init_futex_lock( lck );
581 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
587 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
589 char const *
const func =
"omp_destroy_lock";
590 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
591 && __kmp_is_futex_lock_nestable( lck ) ) {
592 KMP_FATAL( LockNestableUsedAsSimple, func );
594 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
595 KMP_FATAL( LockStillOwned, func );
597 __kmp_destroy_futex_lock( lck );
606 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
608 KMP_DEBUG_ASSERT( gtid >= 0 );
610 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
611 lck->lk.depth_locked += 1;
614 __kmp_acquire_futex_lock_timed_template( lck, gtid );
615 lck->lk.depth_locked = 1;
620 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
622 char const *
const func =
"omp_set_nest_lock";
623 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
624 KMP_FATAL( LockSimpleUsedAsNestable, func );
626 __kmp_acquire_nested_futex_lock( lck, gtid );
630 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
634 KMP_DEBUG_ASSERT( gtid >= 0 );
636 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
637 retval = ++lck->lk.depth_locked;
639 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
644 retval = lck->lk.depth_locked = 1;
650 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
652 char const *
const func =
"omp_test_nest_lock";
653 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
654 KMP_FATAL( LockSimpleUsedAsNestable, func );
656 return __kmp_test_nested_futex_lock( lck, gtid );
660 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
662 KMP_DEBUG_ASSERT( gtid >= 0 );
665 if ( --(lck->lk.depth_locked) == 0 ) {
666 __kmp_release_futex_lock( lck, gtid );
671 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
673 char const *
const func =
"omp_unset_nest_lock";
675 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
676 KMP_FATAL( LockSimpleUsedAsNestable, func );
678 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
679 KMP_FATAL( LockUnsettingFree, func );
681 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
682 KMP_FATAL( LockUnsettingSetByAnother, func );
684 __kmp_release_nested_futex_lock( lck, gtid );
688 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
690 __kmp_init_futex_lock( lck );
691 lck->lk.depth_locked = 0;
695 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
697 __kmp_init_nested_futex_lock( lck );
701 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
703 __kmp_destroy_futex_lock( lck );
704 lck->lk.depth_locked = 0;
708 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
710 char const *
const func =
"omp_destroy_nest_lock";
711 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
712 KMP_FATAL( LockSimpleUsedAsNestable, func );
714 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
715 KMP_FATAL( LockStillOwned, func );
717 __kmp_destroy_nested_futex_lock( lck );
720 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
727 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
729 return TCR_4( lck->lk.owner_id ) - 1;
733 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
735 return lck->lk.depth_locked != -1;
739 __kmp_bakery_check(kmp_uint value, kmp_uint checker)
741 register kmp_uint32 pause;
743 if (value == checker) {
746 for (pause = checker - value; pause != 0; --pause);
750 __forceinline
static void
751 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
753 kmp_uint32 my_ticket;
756 my_ticket = KMP_TEST_THEN_INC32( (kmp_int32 *) &lck->lk.next_ticket );
758 #ifdef USE_LOCK_PROFILE
759 if ( TCR_4( lck->lk.now_serving ) != my_ticket )
760 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
764 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
765 KMP_FSYNC_ACQUIRED(lck);
768 KMP_WAIT_YIELD( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
769 KMP_FSYNC_ACQUIRED(lck);
773 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
775 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
779 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
781 char const *
const func =
"omp_set_lock";
782 if ( lck->lk.initialized != lck ) {
783 KMP_FATAL( LockIsUninitialized, func );
785 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
786 KMP_FATAL( LockNestableUsedAsSimple, func );
788 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
789 KMP_FATAL( LockIsAlreadyOwned, func );
792 __kmp_acquire_ticket_lock( lck, gtid );
794 lck->lk.owner_id = gtid + 1;
798 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
800 kmp_uint32 my_ticket = TCR_4( lck->lk.next_ticket );
801 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
802 kmp_uint32 next_ticket = my_ticket + 1;
803 if ( KMP_COMPARE_AND_STORE_ACQ32( (kmp_int32 *) &lck->lk.next_ticket,
804 my_ticket, next_ticket ) ) {
805 KMP_FSYNC_ACQUIRED( lck );
813 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
815 char const *
const func =
"omp_test_lock";
816 if ( lck->lk.initialized != lck ) {
817 KMP_FATAL( LockIsUninitialized, func );
819 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
820 KMP_FATAL( LockNestableUsedAsSimple, func );
823 int retval = __kmp_test_ticket_lock( lck, gtid );
826 lck->lk.owner_id = gtid + 1;
832 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
838 KMP_FSYNC_RELEASING(lck);
839 distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) );
841 KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 );
846 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
850 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
852 char const *
const func =
"omp_unset_lock";
854 if ( lck->lk.initialized != lck ) {
855 KMP_FATAL( LockIsUninitialized, func );
857 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
858 KMP_FATAL( LockNestableUsedAsSimple, func );
860 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
861 KMP_FATAL( LockUnsettingFree, func );
863 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
864 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
865 KMP_FATAL( LockUnsettingSetByAnother, func );
867 lck->lk.owner_id = 0;
868 __kmp_release_ticket_lock( lck, gtid );
872 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
874 lck->lk.location = NULL;
875 TCW_4( lck->lk.next_ticket, 0 );
876 TCW_4( lck->lk.now_serving, 0 );
877 lck->lk.owner_id = 0;
878 lck->lk.depth_locked = -1;
879 lck->lk.initialized = (kmp_ticket_lock *)lck;
883 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
885 __kmp_init_ticket_lock( lck );
889 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
891 lck->lk.initialized = NULL;
892 lck->lk.location = NULL;
893 lck->lk.next_ticket = 0;
894 lck->lk.now_serving = 0;
895 lck->lk.owner_id = 0;
896 lck->lk.depth_locked = -1;
900 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
902 char const *
const func =
"omp_destroy_lock";
903 if ( lck->lk.initialized != lck ) {
904 KMP_FATAL( LockIsUninitialized, func );
906 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
907 KMP_FATAL( LockNestableUsedAsSimple, func );
909 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
910 KMP_FATAL( LockStillOwned, func );
912 __kmp_destroy_ticket_lock( lck );
921 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
923 KMP_DEBUG_ASSERT( gtid >= 0 );
925 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
926 lck->lk.depth_locked += 1;
929 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
931 lck->lk.depth_locked = 1;
933 lck->lk.owner_id = gtid + 1;
938 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
940 char const *
const func =
"omp_set_nest_lock";
941 if ( lck->lk.initialized != lck ) {
942 KMP_FATAL( LockIsUninitialized, func );
944 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
945 KMP_FATAL( LockSimpleUsedAsNestable, func );
947 __kmp_acquire_nested_ticket_lock( lck, gtid );
951 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
955 KMP_DEBUG_ASSERT( gtid >= 0 );
957 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
958 retval = ++lck->lk.depth_locked;
960 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
965 retval = lck->lk.depth_locked = 1;
967 lck->lk.owner_id = gtid + 1;
973 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
976 char const *
const func =
"omp_test_nest_lock";
977 if ( lck->lk.initialized != lck ) {
978 KMP_FATAL( LockIsUninitialized, func );
980 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
981 KMP_FATAL( LockSimpleUsedAsNestable, func );
983 return __kmp_test_nested_ticket_lock( lck, gtid );
987 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
989 KMP_DEBUG_ASSERT( gtid >= 0 );
992 if ( --(lck->lk.depth_locked) == 0 ) {
994 lck->lk.owner_id = 0;
995 __kmp_release_ticket_lock( lck, gtid );
1000 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1002 char const *
const func =
"omp_unset_nest_lock";
1004 if ( lck->lk.initialized != lck ) {
1005 KMP_FATAL( LockIsUninitialized, func );
1007 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1008 KMP_FATAL( LockSimpleUsedAsNestable, func );
1010 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1011 KMP_FATAL( LockUnsettingFree, func );
1013 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1014 KMP_FATAL( LockUnsettingSetByAnother, func );
1016 __kmp_release_nested_ticket_lock( lck, gtid );
1020 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1022 __kmp_init_ticket_lock( lck );
1023 lck->lk.depth_locked = 0;
1027 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1029 __kmp_init_nested_ticket_lock( lck );
1033 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1035 __kmp_destroy_ticket_lock( lck );
1036 lck->lk.depth_locked = 0;
1040 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1042 char const *
const func =
"omp_destroy_nest_lock";
1043 if ( lck->lk.initialized != lck ) {
1044 KMP_FATAL( LockIsUninitialized, func );
1046 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1047 KMP_FATAL( LockSimpleUsedAsNestable, func );
1049 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1050 KMP_FATAL( LockStillOwned, func );
1052 __kmp_destroy_nested_ticket_lock( lck );
1061 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1063 return lck == lck->lk.initialized;
1067 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1069 return lck->lk.location;
1073 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1075 lck->lk.location = loc;
1078 static kmp_lock_flags_t
1079 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1081 return lck->lk.flags;
1085 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1087 lck->lk.flags = flags;
1146 #ifdef DEBUG_QUEUING_LOCKS
1149 #define TRACE_BUF_ELE 1024
1150 static char traces[TRACE_BUF_ELE][128] = { 0 }
1152 #define TRACE_LOCK(X,Y) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y );
1153 #define TRACE_LOCK_T(X,Y,Z) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X,Y,Z );
1154 #define TRACE_LOCK_HT(X,Y,Z,Q) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, Z, Q );
1157 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1158 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1162 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1164 i = tc % TRACE_BUF_ELE;
1165 __kmp_printf_no_lock(
"%s\n", traces[i] );
1166 i = (i+1) % TRACE_BUF_ELE;
1167 while ( i != (tc % TRACE_BUF_ELE) ) {
1168 __kmp_printf_no_lock(
"%s", traces[i] );
1169 i = (i+1) % TRACE_BUF_ELE;
1171 __kmp_printf_no_lock(
"\n" );
1173 __kmp_printf_no_lock(
1174 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1175 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1178 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1180 if ( lck->lk.head_id >= 1 ) {
1181 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1183 __kmp_printf_no_lock(
"-> %d ", t );
1184 t = __kmp_threads[t-1]->th.th_next_waiting;
1187 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1188 __kmp_printf_no_lock(
"\n\n" );
1194 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1196 return TCR_4( lck->lk.owner_id ) - 1;
1200 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1202 return lck->lk.depth_locked != -1;
1206 template <
bool takeTime>
1209 __forceinline
static void
1210 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1213 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1214 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1215 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1216 volatile kmp_uint32 *spin_here_p;
1217 kmp_int32 need_mf = 1;
1219 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1221 KMP_FSYNC_PREPARE( lck );
1222 KMP_DEBUG_ASSERT( this_thr != NULL );
1223 spin_here_p = & this_thr->th.th_spin_here;
1225 #ifdef DEBUG_QUEUING_LOCKS
1226 TRACE_LOCK( gtid+1,
"acq ent" );
1228 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1229 if ( this_thr->th.th_next_waiting != 0 )
1230 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1232 KMP_DEBUG_ASSERT( !*spin_here_p );
1233 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1243 *spin_here_p = TRUE;
1256 #ifdef DEBUG_QUEUING_LOCKS
1258 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1267 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1268 KMP_PACK_64( -1, 0 ),
1269 KMP_PACK_64( gtid+1, gtid+1 ) );
1270 #ifdef DEBUG_QUEUING_LOCKS
1271 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1279 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1281 #ifdef DEBUG_QUEUING_LOCKS
1282 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1291 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1293 #ifdef DEBUG_QUEUING_LOCKS
1294 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1302 kmp_int32 grabbed_lock;
1304 #ifdef DEBUG_QUEUING_LOCKS
1306 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1311 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1313 if ( grabbed_lock ) {
1315 *spin_here_p = FALSE;
1317 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1319 #ifdef DEBUG_QUEUING_LOCKS
1320 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1322 KMP_FSYNC_ACQUIRED( lck );
1332 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1333 KMP_ASSERT( tail_thr != NULL );
1334 tail_thr->th.th_next_waiting = gtid+1;
1337 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1344 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1346 #ifdef DEBUG_QUEUING_LOCKS
1347 TRACE_LOCK( gtid+1,
"acq spin" );
1349 if ( this_thr->th.th_next_waiting != 0 )
1350 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1352 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1353 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1356 #ifdef DEBUG_QUEUING_LOCKS
1357 TRACE_LOCK( gtid+1,
"acq exit 2" );
1366 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1368 #ifdef DEBUG_QUEUING_LOCKS
1369 TRACE_LOCK( gtid+1,
"acq retry" );
1373 KMP_ASSERT2( 0,
"should not get here" );
1377 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1379 KMP_DEBUG_ASSERT( gtid >= 0 );
1381 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1385 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1388 char const *
const func =
"omp_set_lock";
1389 if ( lck->lk.initialized != lck ) {
1390 KMP_FATAL( LockIsUninitialized, func );
1392 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1393 KMP_FATAL( LockNestableUsedAsSimple, func );
1395 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1396 KMP_FATAL( LockIsAlreadyOwned, func );
1399 __kmp_acquire_queuing_lock( lck, gtid );
1401 lck->lk.owner_id = gtid + 1;
1405 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1407 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1410 kmp_info_t *this_thr;
1413 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1414 KMP_DEBUG_ASSERT( gtid >= 0 );
1416 this_thr = __kmp_thread_from_gtid( gtid );
1417 KMP_DEBUG_ASSERT( this_thr != NULL );
1418 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1427 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1428 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1429 KMP_FSYNC_ACQUIRED(lck);
1434 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1439 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1441 char const *
const func =
"omp_test_lock";
1442 if ( lck->lk.initialized != lck ) {
1443 KMP_FATAL( LockIsUninitialized, func );
1445 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1446 KMP_FATAL( LockNestableUsedAsSimple, func );
1449 int retval = __kmp_test_queuing_lock( lck, gtid );
1452 lck->lk.owner_id = gtid + 1;
1458 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1460 register kmp_info_t *this_thr;
1461 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1462 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1464 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1465 KMP_DEBUG_ASSERT( gtid >= 0 );
1466 this_thr = __kmp_thread_from_gtid( gtid );
1467 KMP_DEBUG_ASSERT( this_thr != NULL );
1468 #ifdef DEBUG_QUEUING_LOCKS
1469 TRACE_LOCK( gtid+1,
"rel ent" );
1471 if ( this_thr->th.th_spin_here )
1472 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1473 if ( this_thr->th.th_next_waiting != 0 )
1474 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1476 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1477 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1479 KMP_FSYNC_RELEASING(lck);
1488 #ifdef DEBUG_QUEUING_LOCKS
1490 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1491 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1493 KMP_DEBUG_ASSERT( head != 0 );
1498 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1499 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1501 #ifdef DEBUG_QUEUING_LOCKS
1502 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1512 if ( head == tail ) {
1514 #ifdef DEBUG_QUEUING_LOCKS
1515 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1517 KMP_DEBUG_ASSERT( head > 0 );
1520 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1521 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1522 #ifdef DEBUG_QUEUING_LOCKS
1523 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1528 volatile kmp_int32 *waiting_id_p;
1529 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1530 KMP_DEBUG_ASSERT( head_thr != NULL );
1531 waiting_id_p = & head_thr->th.th_next_waiting;
1534 #ifdef DEBUG_QUEUING_LOCKS
1535 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1537 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1543 *head_id_p = (kmp_int32) KMP_WAIT_YIELD((
volatile kmp_uint*) waiting_id_p, 0, KMP_NEQ, NULL);
1544 #ifdef DEBUG_QUEUING_LOCKS
1545 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1552 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1553 KMP_DEBUG_ASSERT( head_thr != NULL );
1556 #ifdef DEBUG_QUEUING_LOCKS
1557 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1559 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1564 head_thr->th.th_next_waiting = 0;
1565 #ifdef DEBUG_QUEUING_LOCKS
1566 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1571 head_thr->th.th_spin_here = FALSE;
1573 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1575 #ifdef DEBUG_QUEUING_LOCKS
1576 TRACE_LOCK( gtid+1,
"rel exit 2" );
1582 #ifdef DEBUG_QUEUING_LOCKS
1583 TRACE_LOCK( gtid+1,
"rel retry" );
1587 KMP_ASSERT2( 0,
"should not get here" );
1591 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1594 char const *
const func =
"omp_unset_lock";
1596 if ( lck->lk.initialized != lck ) {
1597 KMP_FATAL( LockIsUninitialized, func );
1599 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1600 KMP_FATAL( LockNestableUsedAsSimple, func );
1602 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1603 KMP_FATAL( LockUnsettingFree, func );
1605 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1606 KMP_FATAL( LockUnsettingSetByAnother, func );
1608 lck->lk.owner_id = 0;
1609 __kmp_release_queuing_lock( lck, gtid );
1613 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1615 lck->lk.location = NULL;
1616 lck->lk.head_id = 0;
1617 lck->lk.tail_id = 0;
1618 lck->lk.next_ticket = 0;
1619 lck->lk.now_serving = 0;
1620 lck->lk.owner_id = 0;
1621 lck->lk.depth_locked = -1;
1622 lck->lk.initialized = lck;
1624 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1628 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1630 __kmp_init_queuing_lock( lck );
1634 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1636 lck->lk.initialized = NULL;
1637 lck->lk.location = NULL;
1638 lck->lk.head_id = 0;
1639 lck->lk.tail_id = 0;
1640 lck->lk.next_ticket = 0;
1641 lck->lk.now_serving = 0;
1642 lck->lk.owner_id = 0;
1643 lck->lk.depth_locked = -1;
1647 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1649 char const *
const func =
"omp_destroy_lock";
1650 if ( lck->lk.initialized != lck ) {
1651 KMP_FATAL( LockIsUninitialized, func );
1653 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1654 KMP_FATAL( LockNestableUsedAsSimple, func );
1656 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1657 KMP_FATAL( LockStillOwned, func );
1659 __kmp_destroy_queuing_lock( lck );
1668 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1670 KMP_DEBUG_ASSERT( gtid >= 0 );
1672 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1673 lck->lk.depth_locked += 1;
1676 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1678 lck->lk.depth_locked = 1;
1680 lck->lk.owner_id = gtid + 1;
1685 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1687 char const *
const func =
"omp_set_nest_lock";
1688 if ( lck->lk.initialized != lck ) {
1689 KMP_FATAL( LockIsUninitialized, func );
1691 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1692 KMP_FATAL( LockSimpleUsedAsNestable, func );
1694 __kmp_acquire_nested_queuing_lock( lck, gtid );
1698 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1702 KMP_DEBUG_ASSERT( gtid >= 0 );
1704 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1705 retval = ++lck->lk.depth_locked;
1707 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1712 retval = lck->lk.depth_locked = 1;
1714 lck->lk.owner_id = gtid + 1;
1720 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1723 char const *
const func =
"omp_test_nest_lock";
1724 if ( lck->lk.initialized != lck ) {
1725 KMP_FATAL( LockIsUninitialized, func );
1727 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1728 KMP_FATAL( LockSimpleUsedAsNestable, func );
1730 return __kmp_test_nested_queuing_lock( lck, gtid );
1734 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1736 KMP_DEBUG_ASSERT( gtid >= 0 );
1739 if ( --(lck->lk.depth_locked) == 0 ) {
1741 lck->lk.owner_id = 0;
1742 __kmp_release_queuing_lock( lck, gtid );
1747 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1749 char const *
const func =
"omp_unset_nest_lock";
1751 if ( lck->lk.initialized != lck ) {
1752 KMP_FATAL( LockIsUninitialized, func );
1754 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1755 KMP_FATAL( LockSimpleUsedAsNestable, func );
1757 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1758 KMP_FATAL( LockUnsettingFree, func );
1760 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1761 KMP_FATAL( LockUnsettingSetByAnother, func );
1763 __kmp_release_nested_queuing_lock( lck, gtid );
1767 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1769 __kmp_init_queuing_lock( lck );
1770 lck->lk.depth_locked = 0;
1774 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1776 __kmp_init_nested_queuing_lock( lck );
1780 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1782 __kmp_destroy_queuing_lock( lck );
1783 lck->lk.depth_locked = 0;
1787 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1789 char const *
const func =
"omp_destroy_nest_lock";
1790 if ( lck->lk.initialized != lck ) {
1791 KMP_FATAL( LockIsUninitialized, func );
1793 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1794 KMP_FATAL( LockSimpleUsedAsNestable, func );
1796 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1797 KMP_FATAL( LockStillOwned, func );
1799 __kmp_destroy_nested_queuing_lock( lck );
1808 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1810 return lck == lck->lk.initialized;
1814 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1816 return lck->lk.location;
1820 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1822 lck->lk.location = loc;
1825 static kmp_lock_flags_t
1826 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1828 return lck->lk.flags;
1832 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1834 lck->lk.flags = flags;
1837 #if KMP_USE_ADAPTIVE_LOCKS
1847 #define _XBEGIN_STARTED (~0u)
1848 #define _XABORT_EXPLICIT (1 << 0)
1849 #define _XABORT_RETRY (1 << 1)
1850 #define _XABORT_CONFLICT (1 << 2)
1851 #define _XABORT_CAPACITY (1 << 3)
1852 #define _XABORT_DEBUG (1 << 4)
1853 #define _XABORT_NESTED (1 << 5)
1854 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1857 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1859 #define STRINGIZE_INTERNAL(arg) #arg
1860 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1868 static __inline
int _xbegin()
1897 #endif // KMP_ARCH_X86_64
1907 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n"
1910 "1: movl %%eax,%0\n"
1912 :
"+r"(res)::
"memory",
"%eax");
1913 #endif // KMP_OS_WINDOWS
1920 static __inline
void _xend()
1929 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1939 #define _xabort(ARG) \
1944 #define _xabort(ARG) \
1945 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory");
1951 #if KMP_DEBUG_ADAPTIVE_LOCKS
1956 static kmp_adaptive_lock_statistics_t destroyedStats;
1959 static kmp_adaptive_lock_info_t liveLocks;
1962 static kmp_bootstrap_lock_t chain_lock;
1966 __kmp_init_speculative_stats()
1968 kmp_adaptive_lock_info_t *lck = &liveLocks;
1970 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
1971 lck->stats.next = lck;
1972 lck->stats.prev = lck;
1974 KMP_ASSERT( lck->stats.next->stats.prev == lck );
1975 KMP_ASSERT( lck->stats.prev->stats.next == lck );
1977 __kmp_init_bootstrap_lock( &chain_lock );
1983 __kmp_remember_lock( kmp_adaptive_lock_info_t * lck )
1985 __kmp_acquire_bootstrap_lock( &chain_lock );
1987 lck->stats.next = liveLocks.stats.next;
1988 lck->stats.prev = &liveLocks;
1990 liveLocks.stats.next = lck;
1991 lck->stats.next->stats.prev = lck;
1993 KMP_ASSERT( lck->stats.next->stats.prev == lck );
1994 KMP_ASSERT( lck->stats.prev->stats.next == lck );
1996 __kmp_release_bootstrap_lock( &chain_lock );
2000 __kmp_forget_lock( kmp_adaptive_lock_info_t * lck )
2002 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2003 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2005 kmp_adaptive_lock_info_t * n = lck->stats.next;
2006 kmp_adaptive_lock_info_t * p = lck->stats.prev;
2013 __kmp_zero_speculative_stats( kmp_adaptive_lock_info_t * lck )
2015 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2016 __kmp_remember_lock( lck );
2020 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_info_t * lck )
2022 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2024 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2025 t->successfulSpeculations += s->successfulSpeculations;
2026 t->hardFailedSpeculations += s->hardFailedSpeculations;
2027 t->softFailedSpeculations += s->softFailedSpeculations;
2028 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2029 t->lemmingYields += s->lemmingYields;
2033 __kmp_accumulate_speculative_stats( kmp_adaptive_lock_info_t * lck)
2035 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2037 __kmp_acquire_bootstrap_lock( &chain_lock );
2039 __kmp_add_stats( &destroyedStats, lck );
2040 __kmp_forget_lock( lck );
2042 __kmp_release_bootstrap_lock( &chain_lock );
2046 percent (kmp_uint32 count, kmp_uint32 total)
2048 return (total == 0) ? 0.0: (100.0 * count)/total;
2052 FILE * __kmp_open_stats_file()
2054 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2057 size_t buffLen = KMP_STRLEN( __kmp_speculative_statsfile ) + 20;
2058 char buffer[buffLen];
2059 KMP_SNPRINTF (&buffer[0], buffLen, __kmp_speculative_statsfile,
2060 (kmp_int32)getpid());
2061 FILE * result = fopen(&buffer[0],
"w");
2064 return result ? result : stdout;
2068 __kmp_print_speculative_stats()
2070 if (__kmp_user_lock_kind != lk_adaptive)
2073 FILE * statsFile = __kmp_open_stats_file();
2075 kmp_adaptive_lock_statistics_t total = destroyedStats;
2076 kmp_adaptive_lock_info_t *lck;
2078 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2079 __kmp_add_stats( &total, lck );
2081 kmp_adaptive_lock_statistics_t *t = &total;
2082 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2083 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2084 t->softFailedSpeculations;
2086 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2087 fprintf ( statsFile,
" Lock parameters: \n"
2088 " max_soft_retries : %10d\n"
2089 " max_badness : %10d\n",
2090 __kmp_adaptive_backoff_params.max_soft_retries,
2091 __kmp_adaptive_backoff_params.max_badness);
2092 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2093 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2094 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2095 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2096 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2097 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2098 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2100 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2101 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2102 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2103 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2104 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2105 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2106 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2108 if (statsFile != stdout)
2109 fclose( statsFile );
2112 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ )
2114 # define KMP_INC_STAT(lck,stat)
2116 #endif // KMP_DEBUG_ADAPTIVE_LOCKS
2119 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2123 bool res = lck->lk.head_id == 0;
2127 #if KMP_COMPILER_ICC
2130 __sync_synchronize();
2137 static __inline
void
2138 __kmp_update_badness_after_success( kmp_adaptive_lock_t *lck )
2141 lck->lk.adaptive.badness = 0;
2142 KMP_INC_STAT(lck,successfulSpeculations);
2146 static __inline
void
2147 __kmp_step_badness( kmp_adaptive_lock_t *lck )
2149 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2150 if ( newBadness > lck->lk.adaptive.max_badness) {
2153 lck->lk.adaptive.badness = newBadness;
2159 __kmp_should_speculate( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2161 kmp_uint32 badness = lck->lk.adaptive.badness;
2162 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2163 int res = (attempts & badness) == 0;
2171 __kmp_test_adaptive_lock_only( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2173 int retries = lck->lk.adaptive.max_soft_retries;
2181 kmp_uint32 status = _xbegin();
2186 if (status == _XBEGIN_STARTED )
2192 if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2198 KMP_ASSERT2( 0, "should not get here" );
2203 if ( status & SOFT_ABORT_MASK)
2205 KMP_INC_STAT(lck,softFailedSpeculations);
2210 KMP_INC_STAT(lck,hardFailedSpeculations);
2215 }
while( retries-- );
2219 __kmp_step_badness( lck );
2227 __kmp_test_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2230 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2235 lck->lk.adaptive.acquire_attempts++;
2238 if ( __kmp_test_queuing_lock( GET_QLK_PTR(lck), gtid ) )
2240 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2250 __kmp_test_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2252 char const *
const func =
"omp_test_lock";
2253 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2254 KMP_FATAL( LockIsUninitialized, func );
2257 int retval = __kmp_test_adaptive_lock( lck, gtid );
2260 lck->lk.qlk.owner_id = gtid + 1;
2280 __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2282 if ( __kmp_should_speculate( lck, gtid ) )
2284 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2286 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2298 while ( ! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2300 KMP_INC_STAT(lck,lemmingYields);
2304 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2311 lck->lk.adaptive.acquire_attempts++;
2313 __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
2315 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2319 __kmp_acquire_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2321 char const *
const func =
"omp_set_lock";
2322 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2323 KMP_FATAL( LockIsUninitialized, func );
2325 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == gtid ) {
2326 KMP_FATAL( LockIsAlreadyOwned, func );
2329 __kmp_acquire_adaptive_lock( lck, gtid );
2331 lck->lk.qlk.owner_id = gtid + 1;
2335 __kmp_release_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2337 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2342 __kmp_update_badness_after_success( lck );
2347 __kmp_release_queuing_lock( GET_QLK_PTR(lck), gtid );
2352 __kmp_release_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2354 char const *
const func =
"omp_unset_lock";
2356 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2357 KMP_FATAL( LockIsUninitialized, func );
2359 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == -1 ) {
2360 KMP_FATAL( LockUnsettingFree, func );
2362 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != gtid ) {
2363 KMP_FATAL( LockUnsettingSetByAnother, func );
2365 lck->lk.qlk.owner_id = 0;
2366 __kmp_release_adaptive_lock( lck, gtid );
2370 __kmp_init_adaptive_lock( kmp_adaptive_lock_t *lck )
2372 __kmp_init_queuing_lock( GET_QLK_PTR(lck) );
2373 lck->lk.adaptive.badness = 0;
2374 lck->lk.adaptive.acquire_attempts = 0;
2375 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2376 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2377 #if KMP_DEBUG_ADAPTIVE_LOCKS
2378 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2380 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2384 __kmp_init_adaptive_lock_with_checks( kmp_adaptive_lock_t * lck )
2386 __kmp_init_adaptive_lock( lck );
2390 __kmp_destroy_adaptive_lock( kmp_adaptive_lock_t *lck )
2392 #if KMP_DEBUG_ADAPTIVE_LOCKS
2393 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2395 __kmp_destroy_queuing_lock (GET_QLK_PTR(lck));
2400 __kmp_destroy_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck )
2402 char const *
const func =
"omp_destroy_lock";
2403 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2404 KMP_FATAL( LockIsUninitialized, func );
2406 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != -1 ) {
2407 KMP_FATAL( LockStillOwned, func );
2409 __kmp_destroy_adaptive_lock( lck );
2413 #endif // KMP_USE_ADAPTIVE_LOCKS
2421 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2423 return TCR_4( lck->lk.owner_id ) - 1;
2427 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2429 return lck->lk.depth_locked != -1;
2432 __forceinline
static void
2433 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2435 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2436 kmp_uint64 mask = TCR_8(lck->lk.mask);
2437 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2438 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2439 TCR_PTR(lck->lk.polls);
2441 #ifdef USE_LOCK_PROFILE
2442 if (TCR_8(polls[ticket & mask].poll) != ticket)
2443 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2459 KMP_FSYNC_PREPARE(lck);
2460 KMP_INIT_YIELD(spins);
2461 while (TCR_8(polls[ticket & mask]).poll < ticket) {
2466 KMP_YIELD(TCR_4(__kmp_nth)
2467 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2468 KMP_YIELD_SPIN(spins);
2479 mask = TCR_8(lck->lk.mask);
2480 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2481 TCR_PTR(lck->lk.polls);
2487 KMP_FSYNC_ACQUIRED(lck);
2488 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2490 lck->lk.now_serving = ticket;
2499 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2500 __kmp_free((
void *)lck->lk.old_polls);
2501 lck->lk.old_polls = NULL;
2502 lck->lk.cleanup_ticket = 0;
2510 if (lck->lk.old_polls == NULL) {
2511 bool reconfigure =
false;
2512 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2513 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2515 if (TCR_4(__kmp_nth)
2516 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2521 if (num_polls > 1) {
2523 num_polls = TCR_4(lck->lk.num_polls);
2526 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2527 __kmp_allocate(num_polls *
sizeof(*polls));
2528 polls[0].poll = ticket;
2537 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2538 if (num_waiting > num_polls) {
2539 kmp_uint32 old_num_polls = num_polls;
2542 mask = (mask << 1) | 1;
2544 }
while (num_polls <= num_waiting);
2552 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2553 __kmp_allocate(num_polls *
sizeof(*polls));
2555 for (i = 0; i < old_num_polls; i++) {
2556 polls[i].poll = old_polls[i].poll;
2573 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2574 ticket, lck, num_polls));
2576 lck->lk.old_polls = old_polls;
2577 lck->lk.polls = polls;
2581 lck->lk.num_polls = num_polls;
2582 lck->lk.mask = mask;
2592 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2598 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2600 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2604 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2606 char const *
const func =
"omp_set_lock";
2607 if ( lck->lk.initialized != lck ) {
2608 KMP_FATAL( LockIsUninitialized, func );
2610 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2611 KMP_FATAL( LockNestableUsedAsSimple, func );
2613 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2614 KMP_FATAL( LockIsAlreadyOwned, func );
2617 __kmp_acquire_drdpa_lock( lck, gtid );
2619 lck->lk.owner_id = gtid + 1;
2623 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2629 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2630 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2631 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2632 TCR_PTR(lck->lk.polls);
2633 kmp_uint64 mask = TCR_8(lck->lk.mask);
2634 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2635 kmp_uint64 next_ticket = ticket + 1;
2636 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2637 ticket, next_ticket)) {
2638 KMP_FSYNC_ACQUIRED(lck);
2639 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2641 lck->lk.now_serving = ticket;
2659 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2661 char const *
const func =
"omp_test_lock";
2662 if ( lck->lk.initialized != lck ) {
2663 KMP_FATAL( LockIsUninitialized, func );
2665 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2666 KMP_FATAL( LockNestableUsedAsSimple, func );
2669 int retval = __kmp_test_drdpa_lock( lck, gtid );
2672 lck->lk.owner_id = gtid + 1;
2678 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2685 kmp_uint64 ticket = lck->lk.now_serving + 1;
2686 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2687 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2688 TCR_PTR(lck->lk.polls);
2689 kmp_uint64 mask = TCR_8(lck->lk.mask);
2690 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2692 KMP_FSYNC_RELEASING(lck);
2693 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2697 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2699 char const *
const func =
"omp_unset_lock";
2701 if ( lck->lk.initialized != lck ) {
2702 KMP_FATAL( LockIsUninitialized, func );
2704 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2705 KMP_FATAL( LockNestableUsedAsSimple, func );
2707 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2708 KMP_FATAL( LockUnsettingFree, func );
2710 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2711 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2712 KMP_FATAL( LockUnsettingSetByAnother, func );
2714 lck->lk.owner_id = 0;
2715 __kmp_release_drdpa_lock( lck, gtid );
2719 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2721 lck->lk.location = NULL;
2723 lck->lk.num_polls = 1;
2724 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2725 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2726 lck->lk.cleanup_ticket = 0;
2727 lck->lk.old_polls = NULL;
2728 lck->lk.next_ticket = 0;
2729 lck->lk.now_serving = 0;
2730 lck->lk.owner_id = 0;
2731 lck->lk.depth_locked = -1;
2732 lck->lk.initialized = lck;
2734 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2738 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2740 __kmp_init_drdpa_lock( lck );
2744 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2746 lck->lk.initialized = NULL;
2747 lck->lk.location = NULL;
2748 if (lck->lk.polls != NULL) {
2749 __kmp_free((
void *)lck->lk.polls);
2750 lck->lk.polls = NULL;
2752 if (lck->lk.old_polls != NULL) {
2753 __kmp_free((
void *)lck->lk.old_polls);
2754 lck->lk.old_polls = NULL;
2757 lck->lk.num_polls = 0;
2758 lck->lk.cleanup_ticket = 0;
2759 lck->lk.next_ticket = 0;
2760 lck->lk.now_serving = 0;
2761 lck->lk.owner_id = 0;
2762 lck->lk.depth_locked = -1;
2766 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2768 char const *
const func =
"omp_destroy_lock";
2769 if ( lck->lk.initialized != lck ) {
2770 KMP_FATAL( LockIsUninitialized, func );
2772 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2773 KMP_FATAL( LockNestableUsedAsSimple, func );
2775 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2776 KMP_FATAL( LockStillOwned, func );
2778 __kmp_destroy_drdpa_lock( lck );
2787 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2789 KMP_DEBUG_ASSERT( gtid >= 0 );
2791 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2792 lck->lk.depth_locked += 1;
2795 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2797 lck->lk.depth_locked = 1;
2799 lck->lk.owner_id = gtid + 1;
2804 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2806 char const *
const func =
"omp_set_nest_lock";
2807 if ( lck->lk.initialized != lck ) {
2808 KMP_FATAL( LockIsUninitialized, func );
2810 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2811 KMP_FATAL( LockSimpleUsedAsNestable, func );
2813 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2817 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2821 KMP_DEBUG_ASSERT( gtid >= 0 );
2823 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2824 retval = ++lck->lk.depth_locked;
2826 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2831 retval = lck->lk.depth_locked = 1;
2833 lck->lk.owner_id = gtid + 1;
2839 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2841 char const *
const func =
"omp_test_nest_lock";
2842 if ( lck->lk.initialized != lck ) {
2843 KMP_FATAL( LockIsUninitialized, func );
2845 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2846 KMP_FATAL( LockSimpleUsedAsNestable, func );
2848 return __kmp_test_nested_drdpa_lock( lck, gtid );
2852 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2854 KMP_DEBUG_ASSERT( gtid >= 0 );
2857 if ( --(lck->lk.depth_locked) == 0 ) {
2859 lck->lk.owner_id = 0;
2860 __kmp_release_drdpa_lock( lck, gtid );
2865 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2867 char const *
const func =
"omp_unset_nest_lock";
2869 if ( lck->lk.initialized != lck ) {
2870 KMP_FATAL( LockIsUninitialized, func );
2872 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2873 KMP_FATAL( LockSimpleUsedAsNestable, func );
2875 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2876 KMP_FATAL( LockUnsettingFree, func );
2878 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
2879 KMP_FATAL( LockUnsettingSetByAnother, func );
2881 __kmp_release_nested_drdpa_lock( lck, gtid );
2885 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
2887 __kmp_init_drdpa_lock( lck );
2888 lck->lk.depth_locked = 0;
2892 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2894 __kmp_init_nested_drdpa_lock( lck );
2898 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
2900 __kmp_destroy_drdpa_lock( lck );
2901 lck->lk.depth_locked = 0;
2905 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2907 char const *
const func =
"omp_destroy_nest_lock";
2908 if ( lck->lk.initialized != lck ) {
2909 KMP_FATAL( LockIsUninitialized, func );
2911 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2912 KMP_FATAL( LockSimpleUsedAsNestable, func );
2914 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2915 KMP_FATAL( LockStillOwned, func );
2917 __kmp_destroy_nested_drdpa_lock( lck );
2926 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
2928 return lck == lck->lk.initialized;
2932 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
2934 return lck->lk.location;
2938 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
2940 lck->lk.location = loc;
2943 static kmp_lock_flags_t
2944 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
2946 return lck->lk.flags;
2950 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
2952 lck->lk.flags = flags;
2955 #if KMP_USE_DYNAMIC_LOCK
2959 typedef enum kmp_lock_hint_t {
2960 kmp_lock_hint_none = 0,
2961 kmp_lock_hint_contended,
2962 kmp_lock_hint_uncontended,
2963 kmp_lock_hint_nonspeculative,
2964 kmp_lock_hint_speculative,
2965 kmp_lock_hint_adaptive,
2970 #define expand_init_lock(l, a) \
2971 static void init_##l##_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq) { \
2972 *lck = DYNA_LOCK_FREE(l); \
2973 KA_TRACE(20, ("Initialized direct lock, tag = %x\n", *lck)); \
2975 FOREACH_D_LOCK(expand_init_lock, 0)
2976 #undef expand_init_lock
2982 # define machine_pause() _mm_delay_32(10) // TODO: find the right argument
2984 # define machine_pause() _mm_pause()
2986 #define HLE_ACQUIRE ".byte 0xf2;"
2987 #define HLE_RELEASE ".byte 0xf3;"
2989 static inline kmp_uint32
2990 swap4(kmp_uint32
volatile *p, kmp_uint32 v)
2992 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0"
3000 __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
3006 __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3009 if (swap4(lck, DYNA_LOCK_BUSY(1, hle)) != DYNA_LOCK_FREE(hle)) {
3012 while (*(kmp_uint32
volatile *)lck != DYNA_LOCK_FREE(hle)) {
3013 for (
int i = delay; i != 0; --i)
3015 delay = ((delay << 1) | 1) & 7;
3017 }
while (swap4(lck, DYNA_LOCK_BUSY(1, hle)) != DYNA_LOCK_FREE(hle));
3022 __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3024 __kmp_acquire_hle_lock(lck, gtid);
3028 __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3030 __asm__
volatile(HLE_RELEASE
"movl %1,%0"
3032 :
"r"(DYNA_LOCK_FREE(hle))
3037 __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3039 __kmp_release_hle_lock(lck, gtid);
3043 __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3045 return swap4(lck, DYNA_LOCK_BUSY(1, hle)) == DYNA_LOCK_FREE(hle);
3049 __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3051 return __kmp_test_hle_lock(lck, gtid);
3054 #endif // DYNA_HAS_HLE
3057 static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
3058 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
3059 static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3060 static void __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3061 static int __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3062 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3063 static void __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3064 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3070 #define expand_func0(l, op) 0,op##_##l##_##lock,
3071 void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
3072 = { __kmp_init_indirect_lock, 0, FOREACH_D_LOCK(expand_func0, init) };
3074 #define expand_func1(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_##lock,
3075 void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *)
3076 = { __kmp_destroy_indirect_lock, 0, FOREACH_D_LOCK(expand_func1, destroy) };
3079 #define expand_func2(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
3080 #define expand_func2c(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3081 static void (*direct_set_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3082 = { { __kmp_set_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, acquire) },
3083 { __kmp_set_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, acquire) } };
3084 static void (*direct_unset_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3085 = { { __kmp_unset_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, release) },
3086 { __kmp_unset_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, release) } };
3088 #define expand_func3(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
3089 #define expand_func3c(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3090 static int (*direct_test_tab[][DYNA_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
3091 = { { __kmp_test_indirect_lock, 0, FOREACH_D_LOCK(expand_func3, test) },
3092 { __kmp_test_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func3c, test) } };
3095 void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3096 void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3097 int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
3102 #define expand_func4(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock,
3103 void (*__kmp_indirect_init_ops[])(kmp_user_lock_p)
3104 = { FOREACH_I_LOCK(expand_func4, init) };
3105 void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p)
3106 = { FOREACH_I_LOCK(expand_func4, destroy) };
3109 #define expand_func5(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
3110 #define expand_func5c(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3111 static void (*indirect_set_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3112 = { { FOREACH_I_LOCK(expand_func5, acquire) },
3113 { FOREACH_I_LOCK(expand_func5c, acquire) } };
3114 static void (*indirect_unset_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3115 = { { FOREACH_I_LOCK(expand_func5, release) },
3116 { FOREACH_I_LOCK(expand_func5c, release) } };
3118 #define expand_func6(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
3119 #define expand_func6c(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
3120 static int (*indirect_test_tab[][DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
3121 = { { FOREACH_I_LOCK(expand_func6, test) },
3122 { FOREACH_I_LOCK(expand_func6c, test) } };
3125 void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32) = 0;
3126 void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32) = 0;
3127 int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32) = 0;
3130 kmp_indirect_lock_t **__kmp_indirect_lock_table;
3131 kmp_lock_index_t __kmp_indirect_lock_table_size;
3132 kmp_lock_index_t __kmp_indirect_lock_table_next;
3135 static kmp_uint32 __kmp_indirect_lock_size[DYNA_NUM_I_LOCKS] = {
3136 sizeof(kmp_ticket_lock_t),
sizeof(kmp_queuing_lock_t),
3137 #if KMP_USE_ADAPTIVE_LOCKS
3138 sizeof(kmp_adaptive_lock_t),
3140 sizeof(kmp_drdpa_lock_t),
3141 sizeof(kmp_tas_lock_t),
3143 sizeof(kmp_futex_lock_t),
3145 sizeof(kmp_ticket_lock_t),
sizeof(kmp_queuing_lock_t),
3146 sizeof(kmp_drdpa_lock_t)
3150 void (*__kmp_indirect_set_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *) = { 0 };
3151 void (*__kmp_indirect_set_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t) = { 0 };
3152 const ident_t * (*__kmp_indirect_get_location[DYNA_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3153 kmp_lock_flags_t (*__kmp_indirect_get_flags[DYNA_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3156 static kmp_indirect_lock_t * __kmp_indirect_lock_pool[DYNA_NUM_I_LOCKS] = { 0 };
3160 __kmp_insert_indirect_lock(kmp_indirect_lock_t *lck)
3162 kmp_lock_index_t next = __kmp_indirect_lock_table_next;
3164 if (next >= __kmp_indirect_lock_table_size) {
3166 kmp_lock_index_t size = __kmp_indirect_lock_table_size;
3167 kmp_indirect_lock_t **old_table = __kmp_indirect_lock_table;
3168 __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(2*next*
sizeof(kmp_indirect_lock_t *));
3169 KMP_MEMCPY(__kmp_indirect_lock_table, old_table, next*
sizeof(kmp_indirect_lock_t *));
3170 __kmp_free(old_table);
3171 __kmp_indirect_lock_table_size = 2*next;
3174 __kmp_indirect_lock_table[next] = lck;
3175 __kmp_indirect_lock_table_next++;
3180 kmp_indirect_lock_t *
3181 __kmp_allocate_indirect_lock(
void **user_lock, kmp_int32 gtid, kmp_indirect_locktag_t tag)
3183 kmp_indirect_lock_t *lck;
3184 kmp_lock_index_t idx;
3186 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3188 if (__kmp_indirect_lock_pool[tag] != NULL) {
3189 lck = __kmp_indirect_lock_pool[tag];
3190 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3191 idx = lck->lock->pool.index;
3192 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3194 lck = (kmp_indirect_lock_t *)__kmp_allocate(
sizeof(kmp_indirect_lock_t));
3195 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3196 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3197 idx = __kmp_insert_indirect_lock(lck);
3200 __kmp_release_lock(&__kmp_global_lock, gtid);
3204 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3205 *((kmp_lock_index_t *)user_lock) = idx << 1;
3207 *((kmp_indirect_lock_t **)user_lock) = lck;
3214 static __forceinline
3215 kmp_indirect_lock_t *
3216 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func)
3218 if (__kmp_env_consistency_check) {
3219 kmp_indirect_lock_t *lck = NULL;
3220 if (user_lock == NULL) {
3221 KMP_FATAL(LockIsUninitialized, func);
3223 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3224 kmp_lock_index_t idx = DYNA_EXTRACT_I_INDEX(user_lock);
3225 if (idx < 0 || idx >= __kmp_indirect_lock_table_size) {
3226 KMP_FATAL(LockIsUninitialized, func);
3228 lck = __kmp_indirect_lock_table[idx];
3230 lck = *((kmp_indirect_lock_t **)user_lock);
3233 KMP_FATAL(LockIsUninitialized, func);
3237 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3238 return __kmp_indirect_lock_table[DYNA_EXTRACT_I_INDEX(user_lock)];
3240 return *((kmp_indirect_lock_t **)user_lock);
3246 __kmp_init_indirect_lock(kmp_dyna_lock_t * lock, kmp_dyna_lockseq_t seq)
3248 #if KMP_USE_ADAPTIVE_LOCKS
3249 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3250 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3251 seq = lockseq_queuing;
3254 kmp_indirect_locktag_t tag = DYNA_GET_I_TAG(seq);
3255 kmp_indirect_lock_t *l = __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3256 DYNA_I_LOCK_FUNC(l, init)(l->lock);
3257 KA_TRACE(20, (
"__kmp_init_indirect_lock: initialized indirect lock, tag = %x\n", l->type));
3261 __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock)
3263 kmp_uint32 gtid = __kmp_entry_gtid();
3264 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3265 DYNA_I_LOCK_FUNC(l, destroy)(l->lock);
3266 kmp_indirect_locktag_t tag = l->type;
3268 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3271 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3272 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3273 l->lock->pool.index = DYNA_EXTRACT_I_INDEX(lock);
3275 __kmp_indirect_lock_pool[tag] = l;
3277 __kmp_release_lock(&__kmp_global_lock, gtid);
3281 __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3283 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3284 DYNA_I_LOCK_FUNC(l, set)(l->lock, gtid);
3288 __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3290 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3291 DYNA_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3295 __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3297 kmp_indirect_lock_t *l = DYNA_LOOKUP_I_LOCK(lock);
3298 return DYNA_I_LOCK_FUNC(l, test)(l->lock, gtid);
3302 __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3304 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3305 DYNA_I_LOCK_FUNC(l, set)(l->lock, gtid);
3309 __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3311 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3312 DYNA_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3316 __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3318 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3319 return DYNA_I_LOCK_FUNC(l, test)(l->lock, gtid);
3322 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3326 __kmp_init_lock_hinted(
void **lock,
int hint)
3328 kmp_dyna_lockseq_t seq;
3330 case kmp_lock_hint_uncontended:
3333 case kmp_lock_hint_speculative:
3340 case kmp_lock_hint_adaptive:
3341 #if KMP_USE_ADAPTIVE_LOCKS
3342 seq = lockseq_adaptive;
3344 seq = lockseq_queuing;
3348 case kmp_lock_hint_contended:
3349 case kmp_lock_hint_nonspeculative:
3351 seq = lockseq_queuing;
3354 if (DYNA_IS_D_LOCK(seq)) {
3355 DYNA_INIT_D_LOCK(lock, seq);
3357 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
3360 DYNA_INIT_I_LOCK(lock, seq);
3362 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(lock);
3363 __kmp_itt_lock_creating(ilk->lock, NULL);
3370 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
3374 case lockseq_nested_tas:
3375 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3378 case lockseq_nested_futex:
3379 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3381 case lockseq_ticket:
3382 case lockseq_nested_ticket:
3383 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3384 case lockseq_queuing:
3385 case lockseq_nested_queuing:
3386 #if KMP_USE_ADAPTIVE_LOCKS
3387 case lockseq_adaptive:
3388 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3391 case lockseq_nested_drdpa:
3392 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3401 __kmp_init_nest_lock_hinted(
void **lock,
int hint)
3403 kmp_dyna_lockseq_t seq;
3405 case kmp_lock_hint_uncontended:
3406 seq = lockseq_nested_tas;
3409 case kmp_lock_hint_contended:
3410 case kmp_lock_hint_nonspeculative:
3412 seq = lockseq_nested_queuing;
3415 DYNA_INIT_I_LOCK(lock, seq);
3417 kmp_indirect_lock_t *ilk = DYNA_LOOKUP_I_LOCK(lock);
3418 __kmp_itt_lock_creating(ilk->lock, NULL);
3424 __kmp_init_indirect_lock_table()
3426 __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *)*1024);
3427 __kmp_indirect_lock_table_size = 1024;
3428 __kmp_indirect_lock_table_next = 0;
3431 #if KMP_USE_ADAPTIVE_LOCKS
3432 # define init_lock_func(table, expand) { \
3433 table[locktag_ticket] = expand(ticket); \
3434 table[locktag_queuing] = expand(queuing); \
3435 table[locktag_adaptive] = expand(queuing); \
3436 table[locktag_drdpa] = expand(drdpa); \
3437 table[locktag_nested_ticket] = expand(ticket); \
3438 table[locktag_nested_queuing] = expand(queuing); \
3439 table[locktag_nested_drdpa] = expand(drdpa); \
3442 # define init_lock_func(table, expand) { \
3443 table[locktag_ticket] = expand(ticket); \
3444 table[locktag_queuing] = expand(queuing); \
3445 table[locktag_drdpa] = expand(drdpa); \
3446 table[locktag_nested_ticket] = expand(ticket); \
3447 table[locktag_nested_queuing] = expand(queuing); \
3448 table[locktag_nested_drdpa] = expand(drdpa); \
3450 #endif // KMP_USE_ADAPTIVE_LOCKS
3454 __kmp_init_dynamic_user_locks()
3457 int offset = (__kmp_env_consistency_check)? 1: 0;
3458 __kmp_direct_set_ops = direct_set_tab[offset];
3459 __kmp_direct_unset_ops = direct_unset_tab[offset];
3460 __kmp_direct_test_ops = direct_test_tab[offset];
3461 __kmp_indirect_set_ops = indirect_set_tab[offset];
3462 __kmp_indirect_unset_ops = indirect_unset_tab[offset];
3463 __kmp_indirect_test_ops = indirect_test_tab[offset];
3464 __kmp_init_indirect_lock_table();
3468 #define expand_func(l) (void (*)(kmp_user_lock_p, const ident_t *))__kmp_set_##l##_lock_location
3469 init_lock_func(__kmp_indirect_set_location, expand_func);
3471 #define expand_func(l) (void (*)(kmp_user_lock_p, kmp_lock_flags_t))__kmp_set_##l##_lock_flags
3472 init_lock_func(__kmp_indirect_set_flags, expand_func);
3474 #define expand_func(l) (const ident_t * (*)(kmp_user_lock_p))__kmp_get_##l##_lock_location
3475 init_lock_func(__kmp_indirect_get_location, expand_func);
3477 #define expand_func(l) (kmp_lock_flags_t (*)(kmp_user_lock_p))__kmp_get_##l##_lock_flags
3478 init_lock_func(__kmp_indirect_get_flags, expand_func);
3481 __kmp_init_user_locks = TRUE;
3486 __kmp_cleanup_indirect_user_locks()
3492 for (k = 0; k < DYNA_NUM_I_LOCKS; ++k) {
3493 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3495 kmp_indirect_lock_t *ll = l;
3496 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3497 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3498 __kmp_indirect_lock_table[ll->lock->pool.index] = NULL;
3500 __kmp_free(ll->lock);
3505 for (i = 0; i < __kmp_indirect_lock_table_next; i++) {
3506 kmp_indirect_lock_t *l = __kmp_indirect_lock_table[i];
3509 DYNA_I_LOCK_FUNC(l, destroy)(l->lock);
3510 __kmp_free(l->lock);
3515 __kmp_free(__kmp_indirect_lock_table);
3517 __kmp_init_user_locks = FALSE;
3520 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3521 int __kmp_num_locks_in_block = 1;
3523 #else // KMP_USE_DYNAMIC_LOCK
3532 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3534 size_t __kmp_base_user_lock_size = 0;
3535 size_t __kmp_user_lock_size = 0;
3537 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
3538 void ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3540 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3541 void ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3542 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3543 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
3544 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3545 void ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3547 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3548 void ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3549 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3550 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3552 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
3553 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
3554 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
3555 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
3556 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
3558 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
3560 switch ( user_lock_kind ) {
3566 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
3567 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3569 __kmp_get_user_lock_owner_ =
3570 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3571 ( &__kmp_get_tas_lock_owner );
3573 if ( __kmp_env_consistency_check ) {
3574 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3575 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3578 KMP_BIND_USER_LOCK(tas);
3579 KMP_BIND_NESTED_USER_LOCK(tas);
3582 __kmp_destroy_user_lock_ =
3583 ( void ( * )( kmp_user_lock_p ) )
3584 ( &__kmp_destroy_tas_lock );
3586 __kmp_is_user_lock_initialized_ =
3587 ( int ( * )( kmp_user_lock_p ) ) NULL;
3589 __kmp_get_user_lock_location_ =
3590 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3592 __kmp_set_user_lock_location_ =
3593 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3595 __kmp_get_user_lock_flags_ =
3596 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3598 __kmp_set_user_lock_flags_ =
3599 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3603 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
3606 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3607 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3609 __kmp_get_user_lock_owner_ =
3610 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3611 ( &__kmp_get_futex_lock_owner );
3613 if ( __kmp_env_consistency_check ) {
3614 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3615 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3618 KMP_BIND_USER_LOCK(futex);
3619 KMP_BIND_NESTED_USER_LOCK(futex);
3622 __kmp_destroy_user_lock_ =
3623 ( void ( * )( kmp_user_lock_p ) )
3624 ( &__kmp_destroy_futex_lock );
3626 __kmp_is_user_lock_initialized_ =
3627 ( int ( * )( kmp_user_lock_p ) ) NULL;
3629 __kmp_get_user_lock_location_ =
3630 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3632 __kmp_set_user_lock_location_ =
3633 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3635 __kmp_get_user_lock_flags_ =
3636 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3638 __kmp_set_user_lock_flags_ =
3639 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3643 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
3646 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3647 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3649 __kmp_get_user_lock_owner_ =
3650 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3651 ( &__kmp_get_ticket_lock_owner );
3653 if ( __kmp_env_consistency_check ) {
3654 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3655 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3658 KMP_BIND_USER_LOCK(ticket);
3659 KMP_BIND_NESTED_USER_LOCK(ticket);
3662 __kmp_destroy_user_lock_ =
3663 ( void ( * )( kmp_user_lock_p ) )
3664 ( &__kmp_destroy_ticket_lock );
3666 __kmp_is_user_lock_initialized_ =
3667 ( int ( * )( kmp_user_lock_p ) )
3668 ( &__kmp_is_ticket_lock_initialized );
3670 __kmp_get_user_lock_location_ =
3671 (
const ident_t * ( * )( kmp_user_lock_p ) )
3672 ( &__kmp_get_ticket_lock_location );
3674 __kmp_set_user_lock_location_ =
3675 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3676 ( &__kmp_set_ticket_lock_location );
3678 __kmp_get_user_lock_flags_ =
3679 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3680 ( &__kmp_get_ticket_lock_flags );
3682 __kmp_set_user_lock_flags_ =
3683 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3684 ( &__kmp_set_ticket_lock_flags );
3689 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3690 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3692 __kmp_get_user_lock_owner_ =
3693 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3694 ( &__kmp_get_queuing_lock_owner );
3696 if ( __kmp_env_consistency_check ) {
3697 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3698 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3701 KMP_BIND_USER_LOCK(queuing);
3702 KMP_BIND_NESTED_USER_LOCK(queuing);
3705 __kmp_destroy_user_lock_ =
3706 ( void ( * )( kmp_user_lock_p ) )
3707 ( &__kmp_destroy_queuing_lock );
3709 __kmp_is_user_lock_initialized_ =
3710 ( int ( * )( kmp_user_lock_p ) )
3711 ( &__kmp_is_queuing_lock_initialized );
3713 __kmp_get_user_lock_location_ =
3714 (
const ident_t * ( * )( kmp_user_lock_p ) )
3715 ( &__kmp_get_queuing_lock_location );
3717 __kmp_set_user_lock_location_ =
3718 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3719 ( &__kmp_set_queuing_lock_location );
3721 __kmp_get_user_lock_flags_ =
3722 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3723 ( &__kmp_get_queuing_lock_flags );
3725 __kmp_set_user_lock_flags_ =
3726 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3727 ( &__kmp_set_queuing_lock_flags );
3731 #if KMP_USE_ADAPTIVE_LOCKS
3733 __kmp_base_user_lock_size =
sizeof( kmp_base_adaptive_lock_t );
3734 __kmp_user_lock_size =
sizeof( kmp_adaptive_lock_t );
3736 __kmp_get_user_lock_owner_ =
3737 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3738 ( &__kmp_get_queuing_lock_owner );
3740 if ( __kmp_env_consistency_check ) {
3741 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3744 KMP_BIND_USER_LOCK(adaptive);
3747 __kmp_destroy_user_lock_ =
3748 ( void ( * )( kmp_user_lock_p ) )
3749 ( &__kmp_destroy_adaptive_lock );
3751 __kmp_is_user_lock_initialized_ =
3752 ( int ( * )( kmp_user_lock_p ) )
3753 ( &__kmp_is_queuing_lock_initialized );
3755 __kmp_get_user_lock_location_ =
3756 (
const ident_t * ( * )( kmp_user_lock_p ) )
3757 ( &__kmp_get_queuing_lock_location );
3759 __kmp_set_user_lock_location_ =
3760 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3761 ( &__kmp_set_queuing_lock_location );
3763 __kmp_get_user_lock_flags_ =
3764 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3765 ( &__kmp_get_queuing_lock_flags );
3767 __kmp_set_user_lock_flags_ =
3768 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3769 ( &__kmp_set_queuing_lock_flags );
3773 #endif // KMP_USE_ADAPTIVE_LOCKS
3776 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3777 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3779 __kmp_get_user_lock_owner_ =
3780 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3781 ( &__kmp_get_drdpa_lock_owner );
3783 if ( __kmp_env_consistency_check ) {
3784 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3785 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3788 KMP_BIND_USER_LOCK(drdpa);
3789 KMP_BIND_NESTED_USER_LOCK(drdpa);
3792 __kmp_destroy_user_lock_ =
3793 ( void ( * )( kmp_user_lock_p ) )
3794 ( &__kmp_destroy_drdpa_lock );
3796 __kmp_is_user_lock_initialized_ =
3797 ( int ( * )( kmp_user_lock_p ) )
3798 ( &__kmp_is_drdpa_lock_initialized );
3800 __kmp_get_user_lock_location_ =
3801 (
const ident_t * ( * )( kmp_user_lock_p ) )
3802 ( &__kmp_get_drdpa_lock_location );
3804 __kmp_set_user_lock_location_ =
3805 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3806 ( &__kmp_set_drdpa_lock_location );
3808 __kmp_get_user_lock_flags_ =
3809 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3810 ( &__kmp_get_drdpa_lock_flags );
3812 __kmp_set_user_lock_flags_ =
3813 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3814 ( &__kmp_set_drdpa_lock_flags );
3824 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3825 kmp_user_lock_p __kmp_lock_pool = NULL;
3828 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3829 int __kmp_num_locks_in_block = 1;
3831 static kmp_lock_index_t
3832 __kmp_lock_table_insert( kmp_user_lock_p lck )
3835 kmp_lock_index_t index;
3836 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3837 kmp_lock_index_t size;
3838 kmp_user_lock_p *table;
3841 if ( __kmp_user_lock_table.allocated == 0 ) {
3845 size = __kmp_user_lock_table.allocated * 2;
3847 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
3848 KMP_MEMCPY( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
3849 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3854 __kmp_user_lock_table.table = table;
3855 __kmp_user_lock_table.allocated = size;
3857 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
3858 index = __kmp_user_lock_table.used;
3859 __kmp_user_lock_table.table[ index ] = lck;
3860 ++ __kmp_user_lock_table.used;
3864 static kmp_user_lock_p
3865 __kmp_lock_block_allocate()
3868 static int last_index = 0;
3869 if ( ( last_index >= __kmp_num_locks_in_block )
3870 || ( __kmp_lock_blocks == NULL ) ) {
3874 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3875 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3876 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
3878 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
3879 new_block->next_block = __kmp_lock_blocks;
3880 new_block->locks = (
void *)buffer;
3883 __kmp_lock_blocks = new_block;
3885 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
3886 [ last_index * __kmp_user_lock_size ] ) );
3896 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3897 kmp_lock_flags_t flags )
3899 kmp_user_lock_p lck;
3900 kmp_lock_index_t index;
3901 KMP_DEBUG_ASSERT( user_lock );
3903 __kmp_acquire_lock( &__kmp_global_lock, gtid );
3905 if ( __kmp_lock_pool == NULL ) {
3907 if ( __kmp_num_locks_in_block <= 1 ) {
3908 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
3911 lck = __kmp_lock_block_allocate();
3916 index = __kmp_lock_table_insert( lck );
3920 lck = __kmp_lock_pool;
3921 index = __kmp_lock_pool->pool.index;
3922 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3929 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3930 * ( (kmp_lock_index_t *) user_lock ) = index;
3933 * ( (kmp_user_lock_p *) user_lock ) = lck;
3937 __kmp_set_user_lock_flags( lck, flags );
3939 __kmp_release_lock( & __kmp_global_lock, gtid );
3946 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
3948 kmp_lock_pool_t * lock_pool;
3950 KMP_DEBUG_ASSERT( user_lock != NULL );
3951 KMP_DEBUG_ASSERT( lck != NULL );
3953 __kmp_acquire_lock( & __kmp_global_lock, gtid );
3955 lck->pool.next = __kmp_lock_pool;
3956 __kmp_lock_pool = lck;
3957 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3958 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
3959 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
3960 lck->pool.index = index;
3963 __kmp_release_lock( & __kmp_global_lock, gtid );
3967 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
3969 kmp_user_lock_p lck = NULL;
3971 if ( __kmp_env_consistency_check ) {
3972 if ( user_lock == NULL ) {
3973 KMP_FATAL( LockIsUninitialized, func );
3977 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3978 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
3979 if ( __kmp_env_consistency_check ) {
3980 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
3981 KMP_FATAL( LockIsUninitialized, func );
3984 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
3985 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3986 lck = __kmp_user_lock_table.table[index];
3989 lck = *( (kmp_user_lock_p *)user_lock );
3992 if ( __kmp_env_consistency_check ) {
3993 if ( lck == NULL ) {
3994 KMP_FATAL( LockIsUninitialized, func );
4002 __kmp_cleanup_user_locks(
void )
4009 __kmp_lock_pool = NULL;
4011 #define IS_CRITICAL(lck) \
4012 ( ( __kmp_get_user_lock_flags_ != NULL ) && \
4013 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) )
4043 while ( __kmp_user_lock_table.used > 1 ) {
4050 kmp_user_lock_p lck = __kmp_user_lock_table.table[
4051 --__kmp_user_lock_table.used ];
4053 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
4054 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
4061 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
4062 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
4064 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
4065 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.line );
4066 __kmp_str_loc_free( &str_loc);
4070 if ( IS_CRITICAL( lck ) ) {
4071 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
4074 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
4082 __kmp_destroy_user_lock( lck );
4088 if ( __kmp_lock_blocks == NULL ) {
4098 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4099 __kmp_user_lock_table.table = NULL;
4100 __kmp_user_lock_table.allocated = 0;
4102 while ( table_ptr != NULL ) {
4107 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
4108 __kmp_free( table_ptr );
4115 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4116 __kmp_lock_blocks = NULL;
4118 while ( block_ptr != NULL ) {
4119 kmp_block_of_locks_t *next = block_ptr->next_block;
4120 __kmp_free( block_ptr->locks );
4127 TCW_4(__kmp_init_user_locks, FALSE);
4130 #endif // KMP_USE_DYNAMIC_LOCK