45 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
47 # include <sys/syscall.h>
65 __kmp_validate_locks(
void )
71 x = ~((kmp_uint32) 0) - 2;
74 for (i = 0; i < 8; ++i, ++x, ++y) {
75 kmp_uint32 z = (x - y);
79 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
97 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
99 return TCR_4( lck->lk.poll ) - 1;
103 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
105 return lck->lk.depth_locked != -1;
108 __forceinline
static void
109 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
113 #ifdef USE_LOCK_PROFILE
114 kmp_uint32 curr = TCR_4( lck->lk.poll );
115 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
116 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
120 if ( ( lck->lk.poll == 0 )
121 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, gtid + 1 ) ) {
122 KMP_FSYNC_ACQUIRED(lck);
127 KMP_FSYNC_PREPARE( lck );
128 KMP_INIT_YIELD( spins );
129 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
134 KMP_YIELD_SPIN( spins );
137 while ( ( lck->lk.poll != 0 ) ||
138 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, gtid + 1 ) ) ) {
142 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
147 KMP_YIELD_SPIN( spins );
150 KMP_FSYNC_ACQUIRED( lck );
154 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
156 __kmp_acquire_tas_lock_timed_template( lck, gtid );
160 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
162 char const *
const func =
"omp_set_lock";
163 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
164 && __kmp_is_tas_lock_nestable( lck ) ) {
165 KMP_FATAL( LockNestableUsedAsSimple, func );
167 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
168 KMP_FATAL( LockIsAlreadyOwned, func );
170 __kmp_acquire_tas_lock( lck, gtid );
174 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
176 if ( ( lck->lk.poll == 0 )
177 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, gtid + 1 ) ) {
178 KMP_FSYNC_ACQUIRED( lck );
185 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
187 char const *
const func =
"omp_test_lock";
188 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
189 && __kmp_is_tas_lock_nestable( lck ) ) {
190 KMP_FATAL( LockNestableUsedAsSimple, func );
192 return __kmp_test_tas_lock( lck, gtid );
196 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
200 KMP_FSYNC_RELEASING(lck);
201 KMP_ST_REL32( &(lck->lk.poll), 0 );
205 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
210 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
212 char const *
const func =
"omp_unset_lock";
214 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
215 && __kmp_is_tas_lock_nestable( lck ) ) {
216 KMP_FATAL( LockNestableUsedAsSimple, func );
218 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
219 KMP_FATAL( LockUnsettingFree, func );
221 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
222 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
223 KMP_FATAL( LockUnsettingSetByAnother, func );
225 __kmp_release_tas_lock( lck, gtid );
229 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
231 TCW_4( lck->lk.poll, 0 );
235 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
237 __kmp_init_tas_lock( lck );
241 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
247 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
249 char const *
const func =
"omp_destroy_lock";
250 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
251 && __kmp_is_tas_lock_nestable( lck ) ) {
252 KMP_FATAL( LockNestableUsedAsSimple, func );
254 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
255 KMP_FATAL( LockStillOwned, func );
257 __kmp_destroy_tas_lock( lck );
266 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
268 KMP_DEBUG_ASSERT( gtid >= 0 );
270 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
271 lck->lk.depth_locked += 1;
274 __kmp_acquire_tas_lock_timed_template( lck, gtid );
275 lck->lk.depth_locked = 1;
280 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
282 char const *
const func =
"omp_set_nest_lock";
283 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
284 KMP_FATAL( LockSimpleUsedAsNestable, func );
286 __kmp_acquire_nested_tas_lock( lck, gtid );
290 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
294 KMP_DEBUG_ASSERT( gtid >= 0 );
296 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
297 retval = ++lck->lk.depth_locked;
299 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
304 retval = lck->lk.depth_locked = 1;
310 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
312 char const *
const func =
"omp_test_nest_lock";
313 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
314 KMP_FATAL( LockSimpleUsedAsNestable, func );
316 return __kmp_test_nested_tas_lock( lck, gtid );
320 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
322 KMP_DEBUG_ASSERT( gtid >= 0 );
325 if ( --(lck->lk.depth_locked) == 0 ) {
326 __kmp_release_tas_lock( lck, gtid );
331 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
333 char const *
const func =
"omp_unset_nest_lock";
335 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
336 KMP_FATAL( LockSimpleUsedAsNestable, func );
338 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
339 KMP_FATAL( LockUnsettingFree, func );
341 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
342 KMP_FATAL( LockUnsettingSetByAnother, func );
344 __kmp_release_nested_tas_lock( lck, gtid );
348 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
350 __kmp_init_tas_lock( lck );
351 lck->lk.depth_locked = 0;
355 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
357 __kmp_init_nested_tas_lock( lck );
361 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
363 __kmp_destroy_tas_lock( lck );
364 lck->lk.depth_locked = 0;
368 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
370 char const *
const func =
"omp_destroy_nest_lock";
371 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
372 KMP_FATAL( LockSimpleUsedAsNestable, func );
374 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
375 KMP_FATAL( LockStillOwned, func );
377 __kmp_destroy_nested_tas_lock( lck );
381 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
392 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
394 return ( TCR_4( lck->lk.poll ) >> 1 ) - 1;
398 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
400 return lck->lk.depth_locked != -1;
403 __forceinline
static void
404 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
406 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
410 #ifdef USE_LOCK_PROFILE
411 kmp_uint32 curr = TCR_4( lck->lk.poll );
412 if ( ( curr != 0 ) && ( curr != gtid_code ) )
413 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
417 KMP_FSYNC_PREPARE( lck );
418 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
419 lck, lck->lk.poll, gtid ) );
422 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), 0,
423 gtid_code ) ) != 0 ) {
424 kmp_int32 cond = poll_val & 1;
425 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
426 lck, gtid, poll_val, cond ) );
441 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ),
442 poll_val, poll_val | 1 ) ) {
443 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
444 lck, lck->lk.poll, gtid ) );
449 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
450 lck, lck->lk.poll, gtid ) );
453 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
454 lck, gtid, poll_val ) );
457 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
458 poll_val, NULL, NULL, 0 ) ) != 0 ) {
459 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
460 lck, gtid, poll_val, rc, errno ) );
464 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
465 lck, gtid, poll_val ) );
475 KMP_FSYNC_ACQUIRED( lck );
476 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
477 lck, lck->lk.poll, gtid ) );
481 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
483 __kmp_acquire_futex_lock_timed_template( lck, gtid );
487 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
489 char const *
const func =
"omp_set_lock";
490 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
491 && __kmp_is_futex_lock_nestable( lck ) ) {
492 KMP_FATAL( LockNestableUsedAsSimple, func );
494 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
495 KMP_FATAL( LockIsAlreadyOwned, func );
497 __kmp_acquire_futex_lock( lck, gtid );
501 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
503 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), 0, ( gtid + 1 ) << 1 ) ) {
504 KMP_FSYNC_ACQUIRED( lck );
511 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
513 char const *
const func =
"omp_test_lock";
514 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
515 && __kmp_is_futex_lock_nestable( lck ) ) {
516 KMP_FATAL( LockNestableUsedAsSimple, func );
518 return __kmp_test_futex_lock( lck, gtid );
522 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
526 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
527 lck, lck->lk.poll, gtid ) );
529 KMP_FSYNC_RELEASING(lck);
531 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), 0 );
533 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
534 lck, gtid, poll_val ) );
536 if ( poll_val & 1 ) {
537 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
539 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, 1, NULL, NULL, 0 );
544 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
545 lck, lck->lk.poll, gtid ) );
547 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
552 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
554 char const *
const func =
"omp_unset_lock";
556 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
557 && __kmp_is_futex_lock_nestable( lck ) ) {
558 KMP_FATAL( LockNestableUsedAsSimple, func );
560 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
561 KMP_FATAL( LockUnsettingFree, func );
563 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
564 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
565 KMP_FATAL( LockUnsettingSetByAnother, func );
567 __kmp_release_futex_lock( lck, gtid );
571 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
573 TCW_4( lck->lk.poll, 0 );
577 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
579 __kmp_init_futex_lock( lck );
583 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
589 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
591 char const *
const func =
"omp_destroy_lock";
592 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
593 && __kmp_is_futex_lock_nestable( lck ) ) {
594 KMP_FATAL( LockNestableUsedAsSimple, func );
596 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
597 KMP_FATAL( LockStillOwned, func );
599 __kmp_destroy_futex_lock( lck );
608 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
610 KMP_DEBUG_ASSERT( gtid >= 0 );
612 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
613 lck->lk.depth_locked += 1;
616 __kmp_acquire_futex_lock_timed_template( lck, gtid );
617 lck->lk.depth_locked = 1;
622 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
624 char const *
const func =
"omp_set_nest_lock";
625 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
626 KMP_FATAL( LockSimpleUsedAsNestable, func );
628 __kmp_acquire_nested_futex_lock( lck, gtid );
632 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
636 KMP_DEBUG_ASSERT( gtid >= 0 );
638 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
639 retval = ++lck->lk.depth_locked;
641 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
646 retval = lck->lk.depth_locked = 1;
652 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
654 char const *
const func =
"omp_test_nest_lock";
655 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
656 KMP_FATAL( LockSimpleUsedAsNestable, func );
658 return __kmp_test_nested_futex_lock( lck, gtid );
662 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
664 KMP_DEBUG_ASSERT( gtid >= 0 );
667 if ( --(lck->lk.depth_locked) == 0 ) {
668 __kmp_release_futex_lock( lck, gtid );
673 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
675 char const *
const func =
"omp_unset_nest_lock";
677 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
678 KMP_FATAL( LockSimpleUsedAsNestable, func );
680 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
681 KMP_FATAL( LockUnsettingFree, func );
683 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
684 KMP_FATAL( LockUnsettingSetByAnother, func );
686 __kmp_release_nested_futex_lock( lck, gtid );
690 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
692 __kmp_init_futex_lock( lck );
693 lck->lk.depth_locked = 0;
697 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
699 __kmp_init_nested_futex_lock( lck );
703 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
705 __kmp_destroy_futex_lock( lck );
706 lck->lk.depth_locked = 0;
710 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
712 char const *
const func =
"omp_destroy_nest_lock";
713 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
714 KMP_FATAL( LockSimpleUsedAsNestable, func );
716 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
717 KMP_FATAL( LockStillOwned, func );
719 __kmp_destroy_nested_futex_lock( lck );
722 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
729 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
731 return TCR_4( lck->lk.owner_id ) - 1;
735 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
737 return lck->lk.depth_locked != -1;
741 __kmp_bakery_check(kmp_uint value, kmp_uint checker)
743 register kmp_uint32 pause;
745 if (value == checker) {
748 for (pause = checker - value; pause != 0; --pause);
752 __forceinline
static void
753 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
755 kmp_uint32 my_ticket;
758 my_ticket = KMP_TEST_THEN_INC32( (kmp_int32 *) &lck->lk.next_ticket );
760 #ifdef USE_LOCK_PROFILE
761 if ( TCR_4( lck->lk.now_serving ) != my_ticket )
762 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
766 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
767 KMP_FSYNC_ACQUIRED(lck);
770 KMP_WAIT_YIELD( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
771 KMP_FSYNC_ACQUIRED(lck);
775 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
777 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
781 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
783 char const *
const func =
"omp_set_lock";
784 if ( lck->lk.initialized != lck ) {
785 KMP_FATAL( LockIsUninitialized, func );
787 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
788 KMP_FATAL( LockNestableUsedAsSimple, func );
790 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
791 KMP_FATAL( LockIsAlreadyOwned, func );
794 __kmp_acquire_ticket_lock( lck, gtid );
796 lck->lk.owner_id = gtid + 1;
800 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
802 kmp_uint32 my_ticket = TCR_4( lck->lk.next_ticket );
803 if ( TCR_4( lck->lk.now_serving ) == my_ticket ) {
804 kmp_uint32 next_ticket = my_ticket + 1;
805 if ( KMP_COMPARE_AND_STORE_ACQ32( (kmp_int32 *) &lck->lk.next_ticket,
806 my_ticket, next_ticket ) ) {
807 KMP_FSYNC_ACQUIRED( lck );
815 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
817 char const *
const func =
"omp_test_lock";
818 if ( lck->lk.initialized != lck ) {
819 KMP_FATAL( LockIsUninitialized, func );
821 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
822 KMP_FATAL( LockNestableUsedAsSimple, func );
825 int retval = __kmp_test_ticket_lock( lck, gtid );
828 lck->lk.owner_id = gtid + 1;
834 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
840 KMP_FSYNC_RELEASING(lck);
841 distance = ( TCR_4( lck->lk.next_ticket ) - TCR_4( lck->lk.now_serving ) );
843 KMP_ST_REL32( &(lck->lk.now_serving), lck->lk.now_serving + 1 );
848 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
852 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
854 char const *
const func =
"omp_unset_lock";
856 if ( lck->lk.initialized != lck ) {
857 KMP_FATAL( LockIsUninitialized, func );
859 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
860 KMP_FATAL( LockNestableUsedAsSimple, func );
862 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
863 KMP_FATAL( LockUnsettingFree, func );
865 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
866 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
867 KMP_FATAL( LockUnsettingSetByAnother, func );
869 lck->lk.owner_id = 0;
870 __kmp_release_ticket_lock( lck, gtid );
874 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
876 lck->lk.location = NULL;
877 TCW_4( lck->lk.next_ticket, 0 );
878 TCW_4( lck->lk.now_serving, 0 );
879 lck->lk.owner_id = 0;
880 lck->lk.depth_locked = -1;
881 lck->lk.initialized = (kmp_ticket_lock *)lck;
885 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
887 __kmp_init_ticket_lock( lck );
891 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
893 lck->lk.initialized = NULL;
894 lck->lk.location = NULL;
895 lck->lk.next_ticket = 0;
896 lck->lk.now_serving = 0;
897 lck->lk.owner_id = 0;
898 lck->lk.depth_locked = -1;
902 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
904 char const *
const func =
"omp_destroy_lock";
905 if ( lck->lk.initialized != lck ) {
906 KMP_FATAL( LockIsUninitialized, func );
908 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
909 KMP_FATAL( LockNestableUsedAsSimple, func );
911 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
912 KMP_FATAL( LockStillOwned, func );
914 __kmp_destroy_ticket_lock( lck );
923 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
925 KMP_DEBUG_ASSERT( gtid >= 0 );
927 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
928 lck->lk.depth_locked += 1;
931 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
933 lck->lk.depth_locked = 1;
935 lck->lk.owner_id = gtid + 1;
940 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
942 char const *
const func =
"omp_set_nest_lock";
943 if ( lck->lk.initialized != lck ) {
944 KMP_FATAL( LockIsUninitialized, func );
946 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
947 KMP_FATAL( LockSimpleUsedAsNestable, func );
949 __kmp_acquire_nested_ticket_lock( lck, gtid );
953 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
957 KMP_DEBUG_ASSERT( gtid >= 0 );
959 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
960 retval = ++lck->lk.depth_locked;
962 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
967 retval = lck->lk.depth_locked = 1;
969 lck->lk.owner_id = gtid + 1;
975 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
978 char const *
const func =
"omp_test_nest_lock";
979 if ( lck->lk.initialized != lck ) {
980 KMP_FATAL( LockIsUninitialized, func );
982 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
983 KMP_FATAL( LockSimpleUsedAsNestable, func );
985 return __kmp_test_nested_ticket_lock( lck, gtid );
989 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
991 KMP_DEBUG_ASSERT( gtid >= 0 );
994 if ( --(lck->lk.depth_locked) == 0 ) {
996 lck->lk.owner_id = 0;
997 __kmp_release_ticket_lock( lck, gtid );
1002 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1004 char const *
const func =
"omp_unset_nest_lock";
1006 if ( lck->lk.initialized != lck ) {
1007 KMP_FATAL( LockIsUninitialized, func );
1009 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1010 KMP_FATAL( LockSimpleUsedAsNestable, func );
1012 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1013 KMP_FATAL( LockUnsettingFree, func );
1015 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1016 KMP_FATAL( LockUnsettingSetByAnother, func );
1018 __kmp_release_nested_ticket_lock( lck, gtid );
1022 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1024 __kmp_init_ticket_lock( lck );
1025 lck->lk.depth_locked = 0;
1029 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1031 __kmp_init_nested_ticket_lock( lck );
1035 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1037 __kmp_destroy_ticket_lock( lck );
1038 lck->lk.depth_locked = 0;
1042 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1044 char const *
const func =
"omp_destroy_nest_lock";
1045 if ( lck->lk.initialized != lck ) {
1046 KMP_FATAL( LockIsUninitialized, func );
1048 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1049 KMP_FATAL( LockSimpleUsedAsNestable, func );
1051 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1052 KMP_FATAL( LockStillOwned, func );
1054 __kmp_destroy_nested_ticket_lock( lck );
1063 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1065 return lck == lck->lk.initialized;
1069 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1071 return lck->lk.location;
1075 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1077 lck->lk.location = loc;
1080 static kmp_lock_flags_t
1081 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1083 return lck->lk.flags;
1087 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1089 lck->lk.flags = flags;
1148 #ifdef DEBUG_QUEUING_LOCKS
1151 #define TRACE_BUF_ELE 1024
1152 static char traces[TRACE_BUF_ELE][128] = { 0 }
1154 #define TRACE_LOCK(X,Y) sprintf( traces[tc++ % TRACE_BUF_ELE], "t%d at %s\n", X, Y );
1155 #define TRACE_LOCK_T(X,Y,Z) sprintf( traces[tc++ % TRACE_BUF_ELE], "t%d at %s%d\n", X,Y,Z );
1156 #define TRACE_LOCK_HT(X,Y,Z,Q) sprintf( traces[tc++ % TRACE_BUF_ELE], "t%d at %s %d,%d\n", X, Y, Z, Q );
1159 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1160 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1164 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1166 i = tc % TRACE_BUF_ELE;
1167 __kmp_printf_no_lock(
"%s\n", traces[i] );
1168 i = (i+1) % TRACE_BUF_ELE;
1169 while ( i != (tc % TRACE_BUF_ELE) ) {
1170 __kmp_printf_no_lock(
"%s", traces[i] );
1171 i = (i+1) % TRACE_BUF_ELE;
1173 __kmp_printf_no_lock(
"\n" );
1175 __kmp_printf_no_lock(
1176 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1177 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1180 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1182 if ( lck->lk.head_id >= 1 ) {
1183 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1185 __kmp_printf_no_lock(
"-> %d ", t );
1186 t = __kmp_threads[t-1]->th.th_next_waiting;
1189 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1190 __kmp_printf_no_lock(
"\n\n" );
1196 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1198 return TCR_4( lck->lk.owner_id ) - 1;
1202 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1204 return lck->lk.depth_locked != -1;
1208 template <
bool takeTime>
1211 __forceinline
static void
1212 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1215 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1216 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1217 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1218 volatile kmp_uint32 *spin_here_p;
1219 kmp_int32 need_mf = 1;
1221 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1223 KMP_FSYNC_PREPARE( lck );
1224 KMP_DEBUG_ASSERT( this_thr != NULL );
1225 spin_here_p = & this_thr->th.th_spin_here;
1227 #ifdef DEBUG_QUEUING_LOCKS
1228 TRACE_LOCK( gtid+1,
"acq ent" );
1230 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1231 if ( this_thr->th.th_next_waiting != 0 )
1232 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1234 KMP_DEBUG_ASSERT( !*spin_here_p );
1235 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1245 *spin_here_p = TRUE;
1258 #ifdef DEBUG_QUEUING_LOCKS
1260 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1269 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1270 KMP_PACK_64( -1, 0 ),
1271 KMP_PACK_64( gtid+1, gtid+1 ) );
1272 #ifdef DEBUG_QUEUING_LOCKS
1273 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1281 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1283 #ifdef DEBUG_QUEUING_LOCKS
1284 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1293 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1295 #ifdef DEBUG_QUEUING_LOCKS
1296 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1304 kmp_int32 grabbed_lock;
1306 #ifdef DEBUG_QUEUING_LOCKS
1308 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1313 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1315 if ( grabbed_lock ) {
1317 *spin_here_p = FALSE;
1319 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1321 #ifdef DEBUG_QUEUING_LOCKS
1322 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1324 KMP_FSYNC_ACQUIRED( lck );
1334 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1335 KMP_ASSERT( tail_thr != NULL );
1336 tail_thr->th.th_next_waiting = gtid+1;
1339 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1346 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1348 #ifdef DEBUG_QUEUING_LOCKS
1349 TRACE_LOCK( gtid+1,
"acq spin" );
1351 if ( this_thr->th.th_next_waiting != 0 )
1352 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1354 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1355 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1358 #ifdef DEBUG_QUEUING_LOCKS
1359 TRACE_LOCK( gtid+1,
"acq exit 2" );
1368 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1370 #ifdef DEBUG_QUEUING_LOCKS
1371 TRACE_LOCK( gtid+1,
"acq retry" );
1375 KMP_ASSERT2( 0,
"should not get here" );
1379 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1381 KMP_DEBUG_ASSERT( gtid >= 0 );
1383 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1387 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1390 char const *
const func =
"omp_set_lock";
1391 if ( lck->lk.initialized != lck ) {
1392 KMP_FATAL( LockIsUninitialized, func );
1394 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1395 KMP_FATAL( LockNestableUsedAsSimple, func );
1397 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1398 KMP_FATAL( LockIsAlreadyOwned, func );
1401 __kmp_acquire_queuing_lock( lck, gtid );
1403 lck->lk.owner_id = gtid + 1;
1407 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1409 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1412 kmp_info_t *this_thr;
1415 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1416 KMP_DEBUG_ASSERT( gtid >= 0 );
1418 this_thr = __kmp_thread_from_gtid( gtid );
1419 KMP_DEBUG_ASSERT( this_thr != NULL );
1420 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1429 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1430 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1431 KMP_FSYNC_ACQUIRED(lck);
1436 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1441 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1443 char const *
const func =
"omp_test_lock";
1444 if ( lck->lk.initialized != lck ) {
1445 KMP_FATAL( LockIsUninitialized, func );
1447 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1448 KMP_FATAL( LockNestableUsedAsSimple, func );
1451 int retval = __kmp_test_queuing_lock( lck, gtid );
1454 lck->lk.owner_id = gtid + 1;
1460 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1462 register kmp_info_t *this_thr;
1463 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1464 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1466 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1467 KMP_DEBUG_ASSERT( gtid >= 0 );
1468 this_thr = __kmp_thread_from_gtid( gtid );
1469 KMP_DEBUG_ASSERT( this_thr != NULL );
1470 #ifdef DEBUG_QUEUING_LOCKS
1471 TRACE_LOCK( gtid+1,
"rel ent" );
1473 if ( this_thr->th.th_spin_here )
1474 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1475 if ( this_thr->th.th_next_waiting != 0 )
1476 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1478 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1479 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1481 KMP_FSYNC_RELEASING(lck);
1490 #ifdef DEBUG_QUEUING_LOCKS
1492 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1493 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1495 KMP_DEBUG_ASSERT( head != 0 );
1500 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1501 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1503 #ifdef DEBUG_QUEUING_LOCKS
1504 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1514 if ( head == tail ) {
1516 #ifdef DEBUG_QUEUING_LOCKS
1517 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1519 KMP_DEBUG_ASSERT( head > 0 );
1522 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1523 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1524 #ifdef DEBUG_QUEUING_LOCKS
1525 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1530 volatile kmp_int32 *waiting_id_p;
1531 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1532 KMP_DEBUG_ASSERT( head_thr != NULL );
1533 waiting_id_p = & head_thr->th.th_next_waiting;
1536 #ifdef DEBUG_QUEUING_LOCKS
1537 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1539 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1545 *head_id_p = (kmp_int32) KMP_WAIT_YIELD((
volatile kmp_uint*) waiting_id_p, 0, KMP_NEQ, NULL);
1546 #ifdef DEBUG_QUEUING_LOCKS
1547 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1554 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1555 KMP_DEBUG_ASSERT( head_thr != NULL );
1558 #ifdef DEBUG_QUEUING_LOCKS
1559 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1561 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1566 head_thr->th.th_next_waiting = 0;
1567 #ifdef DEBUG_QUEUING_LOCKS
1568 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1573 head_thr->th.th_spin_here = FALSE;
1575 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1577 #ifdef DEBUG_QUEUING_LOCKS
1578 TRACE_LOCK( gtid+1,
"rel exit 2" );
1584 #ifdef DEBUG_QUEUING_LOCKS
1585 TRACE_LOCK( gtid+1,
"rel retry" );
1589 KMP_ASSERT2( 0,
"should not get here" );
1593 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1596 char const *
const func =
"omp_unset_lock";
1598 if ( lck->lk.initialized != lck ) {
1599 KMP_FATAL( LockIsUninitialized, func );
1601 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1602 KMP_FATAL( LockNestableUsedAsSimple, func );
1604 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1605 KMP_FATAL( LockUnsettingFree, func );
1607 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1608 KMP_FATAL( LockUnsettingSetByAnother, func );
1610 lck->lk.owner_id = 0;
1611 __kmp_release_queuing_lock( lck, gtid );
1615 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1617 lck->lk.location = NULL;
1618 lck->lk.head_id = 0;
1619 lck->lk.tail_id = 0;
1620 lck->lk.next_ticket = 0;
1621 lck->lk.now_serving = 0;
1622 lck->lk.owner_id = 0;
1623 lck->lk.depth_locked = -1;
1624 lck->lk.initialized = lck;
1626 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1630 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1632 __kmp_init_queuing_lock( lck );
1636 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1638 lck->lk.initialized = NULL;
1639 lck->lk.location = NULL;
1640 lck->lk.head_id = 0;
1641 lck->lk.tail_id = 0;
1642 lck->lk.next_ticket = 0;
1643 lck->lk.now_serving = 0;
1644 lck->lk.owner_id = 0;
1645 lck->lk.depth_locked = -1;
1649 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1651 char const *
const func =
"omp_destroy_lock";
1652 if ( lck->lk.initialized != lck ) {
1653 KMP_FATAL( LockIsUninitialized, func );
1655 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1656 KMP_FATAL( LockNestableUsedAsSimple, func );
1658 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1659 KMP_FATAL( LockStillOwned, func );
1661 __kmp_destroy_queuing_lock( lck );
1670 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1672 KMP_DEBUG_ASSERT( gtid >= 0 );
1674 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1675 lck->lk.depth_locked += 1;
1678 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1680 lck->lk.depth_locked = 1;
1682 lck->lk.owner_id = gtid + 1;
1687 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1689 char const *
const func =
"omp_set_nest_lock";
1690 if ( lck->lk.initialized != lck ) {
1691 KMP_FATAL( LockIsUninitialized, func );
1693 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1694 KMP_FATAL( LockSimpleUsedAsNestable, func );
1696 __kmp_acquire_nested_queuing_lock( lck, gtid );
1700 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1704 KMP_DEBUG_ASSERT( gtid >= 0 );
1706 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1707 retval = ++lck->lk.depth_locked;
1709 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1714 retval = lck->lk.depth_locked = 1;
1716 lck->lk.owner_id = gtid + 1;
1722 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1725 char const *
const func =
"omp_test_nest_lock";
1726 if ( lck->lk.initialized != lck ) {
1727 KMP_FATAL( LockIsUninitialized, func );
1729 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1730 KMP_FATAL( LockSimpleUsedAsNestable, func );
1732 return __kmp_test_nested_queuing_lock( lck, gtid );
1736 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1738 KMP_DEBUG_ASSERT( gtid >= 0 );
1741 if ( --(lck->lk.depth_locked) == 0 ) {
1743 lck->lk.owner_id = 0;
1744 __kmp_release_queuing_lock( lck, gtid );
1749 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1751 char const *
const func =
"omp_unset_nest_lock";
1753 if ( lck->lk.initialized != lck ) {
1754 KMP_FATAL( LockIsUninitialized, func );
1756 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1757 KMP_FATAL( LockSimpleUsedAsNestable, func );
1759 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1760 KMP_FATAL( LockUnsettingFree, func );
1762 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1763 KMP_FATAL( LockUnsettingSetByAnother, func );
1765 __kmp_release_nested_queuing_lock( lck, gtid );
1769 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1771 __kmp_init_queuing_lock( lck );
1772 lck->lk.depth_locked = 0;
1776 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1778 __kmp_init_nested_queuing_lock( lck );
1782 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1784 __kmp_destroy_queuing_lock( lck );
1785 lck->lk.depth_locked = 0;
1789 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1791 char const *
const func =
"omp_destroy_nest_lock";
1792 if ( lck->lk.initialized != lck ) {
1793 KMP_FATAL( LockIsUninitialized, func );
1795 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1796 KMP_FATAL( LockSimpleUsedAsNestable, func );
1798 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1799 KMP_FATAL( LockStillOwned, func );
1801 __kmp_destroy_nested_queuing_lock( lck );
1810 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1812 return lck == lck->lk.initialized;
1816 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1818 return lck->lk.location;
1822 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1824 lck->lk.location = loc;
1827 static kmp_lock_flags_t
1828 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1830 return lck->lk.flags;
1834 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1836 lck->lk.flags = flags;
1839 #if KMP_USE_ADAPTIVE_LOCKS
1849 #define _XBEGIN_STARTED (~0u)
1850 #define _XABORT_EXPLICIT (1 << 0)
1851 #define _XABORT_RETRY (1 << 1)
1852 #define _XABORT_CONFLICT (1 << 2)
1853 #define _XABORT_CAPACITY (1 << 3)
1854 #define _XABORT_DEBUG (1 << 4)
1855 #define _XABORT_NESTED (1 << 5)
1856 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1859 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1861 #define STRINGIZE_INTERNAL(arg) #arg
1862 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1870 static __inline
int _xbegin()
1899 #endif // KMP_ARCH_X86_64
1909 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n"
1912 "1: movl %%eax,%0\n"
1914 :
"+r"(res)::
"memory",
"%eax");
1915 #endif // KMP_OS_WINDOWS
1922 static __inline
void _xend()
1931 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1941 #define _xabort(ARG) \
1946 #define _xabort(ARG) \
1947 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory");
1953 #if KMP_DEBUG_ADAPTIVE_LOCKS
1958 static kmp_adaptive_lock_statistics_t destroyedStats;
1961 static kmp_adaptive_lock_info_t liveLocks;
1964 static kmp_bootstrap_lock_t chain_lock;
1968 __kmp_init_speculative_stats()
1970 kmp_adaptive_lock_info_t *lck = &liveLocks;
1972 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
1973 lck->stats.next = lck;
1974 lck->stats.prev = lck;
1976 KMP_ASSERT( lck->stats.next->stats.prev == lck );
1977 KMP_ASSERT( lck->stats.prev->stats.next == lck );
1979 __kmp_init_bootstrap_lock( &chain_lock );
1985 __kmp_remember_lock( kmp_adaptive_lock_info_t * lck )
1987 __kmp_acquire_bootstrap_lock( &chain_lock );
1989 lck->stats.next = liveLocks.stats.next;
1990 lck->stats.prev = &liveLocks;
1992 liveLocks.stats.next = lck;
1993 lck->stats.next->stats.prev = lck;
1995 KMP_ASSERT( lck->stats.next->stats.prev == lck );
1996 KMP_ASSERT( lck->stats.prev->stats.next == lck );
1998 __kmp_release_bootstrap_lock( &chain_lock );
2002 __kmp_forget_lock( kmp_adaptive_lock_info_t * lck )
2004 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2005 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2007 kmp_adaptive_lock_info_t * n = lck->stats.next;
2008 kmp_adaptive_lock_info_t * p = lck->stats.prev;
2015 __kmp_zero_speculative_stats( kmp_adaptive_lock_info_t * lck )
2017 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2018 __kmp_remember_lock( lck );
2022 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_info_t * lck )
2024 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2026 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2027 t->successfulSpeculations += s->successfulSpeculations;
2028 t->hardFailedSpeculations += s->hardFailedSpeculations;
2029 t->softFailedSpeculations += s->softFailedSpeculations;
2030 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2031 t->lemmingYields += s->lemmingYields;
2035 __kmp_accumulate_speculative_stats( kmp_adaptive_lock_info_t * lck)
2037 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2039 __kmp_acquire_bootstrap_lock( &chain_lock );
2041 __kmp_add_stats( &destroyedStats, lck );
2042 __kmp_forget_lock( lck );
2044 __kmp_release_bootstrap_lock( &chain_lock );
2048 percent (kmp_uint32 count, kmp_uint32 total)
2050 return (total == 0) ? 0.0: (100.0 * count)/total;
2054 FILE * __kmp_open_stats_file()
2056 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2059 size_t buffLen = strlen( __kmp_speculative_statsfile ) + 20;
2060 char buffer[buffLen];
2061 snprintf (&buffer[0], buffLen, __kmp_speculative_statsfile,
2062 (kmp_int32)getpid());
2063 FILE * result = fopen(&buffer[0],
"w");
2066 return result ? result : stdout;
2070 __kmp_print_speculative_stats()
2072 if (__kmp_user_lock_kind != lk_adaptive)
2075 FILE * statsFile = __kmp_open_stats_file();
2077 kmp_adaptive_lock_statistics_t total = destroyedStats;
2078 kmp_adaptive_lock_info_t *lck;
2080 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2081 __kmp_add_stats( &total, lck );
2083 kmp_adaptive_lock_statistics_t *t = &total;
2084 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2085 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2086 t->softFailedSpeculations;
2088 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2089 fprintf ( statsFile,
" Lock parameters: \n"
2090 " max_soft_retries : %10d\n"
2091 " max_badness : %10d\n",
2092 __kmp_adaptive_backoff_params.max_soft_retries,
2093 __kmp_adaptive_backoff_params.max_badness);
2094 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2095 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2096 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2097 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2098 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2099 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2100 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2102 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2103 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2104 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2105 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2106 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2107 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2108 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2110 if (statsFile != stdout)
2111 fclose( statsFile );
2114 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ )
2116 # define KMP_INC_STAT(lck,stat)
2118 #endif // KMP_DEBUG_ADAPTIVE_LOCKS
2121 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2125 bool res = lck->lk.head_id == 0;
2129 #if KMP_COMPILER_ICC
2132 __sync_synchronize();
2139 static __inline
void
2140 __kmp_update_badness_after_success( kmp_adaptive_lock_t *lck )
2143 lck->lk.adaptive.badness = 0;
2144 KMP_INC_STAT(lck,successfulSpeculations);
2148 static __inline
void
2149 __kmp_step_badness( kmp_adaptive_lock_t *lck )
2151 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2152 if ( newBadness > lck->lk.adaptive.max_badness) {
2155 lck->lk.adaptive.badness = newBadness;
2161 __kmp_should_speculate( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2163 kmp_uint32 badness = lck->lk.adaptive.badness;
2164 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2165 int res = (attempts & badness) == 0;
2173 __kmp_test_adaptive_lock_only( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2175 int retries = lck->lk.adaptive.max_soft_retries;
2183 kmp_uint32 status = _xbegin();
2188 if (status == _XBEGIN_STARTED )
2194 if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2200 KMP_ASSERT2( 0, "should not get here" );
2205 if ( status & SOFT_ABORT_MASK)
2207 KMP_INC_STAT(lck,softFailedSpeculations);
2212 KMP_INC_STAT(lck,hardFailedSpeculations);
2217 }
while( retries-- );
2221 __kmp_step_badness( lck );
2229 __kmp_test_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2232 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2237 lck->lk.adaptive.acquire_attempts++;
2240 if ( __kmp_test_queuing_lock( GET_QLK_PTR(lck), gtid ) )
2242 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2252 __kmp_test_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2254 char const *
const func =
"omp_test_lock";
2255 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2256 KMP_FATAL( LockIsUninitialized, func );
2259 int retval = __kmp_test_adaptive_lock( lck, gtid );
2262 lck->lk.qlk.owner_id = gtid + 1;
2282 __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2284 if ( __kmp_should_speculate( lck, gtid ) )
2286 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2288 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2300 while ( ! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2302 KMP_INC_STAT(lck,lemmingYields);
2306 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2313 lck->lk.adaptive.acquire_attempts++;
2315 __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
2317 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2321 __kmp_acquire_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2323 char const *
const func =
"omp_set_lock";
2324 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2325 KMP_FATAL( LockIsUninitialized, func );
2327 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == gtid ) {
2328 KMP_FATAL( LockIsAlreadyOwned, func );
2331 __kmp_acquire_adaptive_lock( lck, gtid );
2333 lck->lk.qlk.owner_id = gtid + 1;
2337 __kmp_release_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2339 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2344 __kmp_update_badness_after_success( lck );
2349 __kmp_release_queuing_lock( GET_QLK_PTR(lck), gtid );
2354 __kmp_release_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2356 char const *
const func =
"omp_unset_lock";
2358 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2359 KMP_FATAL( LockIsUninitialized, func );
2361 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == -1 ) {
2362 KMP_FATAL( LockUnsettingFree, func );
2364 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != gtid ) {
2365 KMP_FATAL( LockUnsettingSetByAnother, func );
2367 lck->lk.qlk.owner_id = 0;
2368 __kmp_release_adaptive_lock( lck, gtid );
2372 __kmp_init_adaptive_lock( kmp_adaptive_lock_t *lck )
2374 __kmp_init_queuing_lock( GET_QLK_PTR(lck) );
2375 lck->lk.adaptive.badness = 0;
2376 lck->lk.adaptive.acquire_attempts = 0;
2377 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2378 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2379 #if KMP_DEBUG_ADAPTIVE_LOCKS
2380 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2382 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2386 __kmp_init_adaptive_lock_with_checks( kmp_adaptive_lock_t * lck )
2388 __kmp_init_adaptive_lock( lck );
2392 __kmp_destroy_adaptive_lock( kmp_adaptive_lock_t *lck )
2394 #if KMP_DEBUG_ADAPTIVE_LOCKS
2395 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2397 __kmp_destroy_queuing_lock (GET_QLK_PTR(lck));
2402 __kmp_destroy_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck )
2404 char const *
const func =
"omp_destroy_lock";
2405 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2406 KMP_FATAL( LockIsUninitialized, func );
2408 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != -1 ) {
2409 KMP_FATAL( LockStillOwned, func );
2411 __kmp_destroy_adaptive_lock( lck );
2415 #endif // KMP_USE_ADAPTIVE_LOCKS
2423 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2425 return TCR_4( lck->lk.owner_id ) - 1;
2429 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2431 return lck->lk.depth_locked != -1;
2434 __forceinline
static void
2435 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2437 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2438 kmp_uint64 mask = TCR_8(lck->lk.mask);
2439 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2440 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2441 TCR_PTR(lck->lk.polls);
2443 #ifdef USE_LOCK_PROFILE
2444 if (TCR_8(polls[ticket & mask].poll) != ticket)
2445 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2461 KMP_FSYNC_PREPARE(lck);
2462 KMP_INIT_YIELD(spins);
2463 while (TCR_8(polls[ticket & mask]).poll < ticket) {
2468 KMP_YIELD(TCR_4(__kmp_nth)
2469 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2470 KMP_YIELD_SPIN(spins);
2481 mask = TCR_8(lck->lk.mask);
2482 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2483 TCR_PTR(lck->lk.polls);
2489 KMP_FSYNC_ACQUIRED(lck);
2490 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2492 lck->lk.now_serving = ticket;
2501 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2502 __kmp_free((
void *)lck->lk.old_polls);
2503 lck->lk.old_polls = NULL;
2504 lck->lk.cleanup_ticket = 0;
2512 if (lck->lk.old_polls == NULL) {
2513 bool reconfigure =
false;
2514 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2515 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2517 if (TCR_4(__kmp_nth)
2518 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2523 if (num_polls > 1) {
2525 num_polls = TCR_4(lck->lk.num_polls);
2528 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2529 __kmp_allocate(num_polls *
sizeof(*polls));
2530 polls[0].poll = ticket;
2539 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2540 if (num_waiting > num_polls) {
2541 kmp_uint32 old_num_polls = num_polls;
2544 mask = (mask << 1) | 1;
2546 }
while (num_polls <= num_waiting);
2554 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2555 __kmp_allocate(num_polls *
sizeof(*polls));
2557 for (i = 0; i < old_num_polls; i++) {
2558 polls[i].poll = old_polls[i].poll;
2575 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2576 ticket, lck, num_polls));
2578 lck->lk.old_polls = old_polls;
2579 lck->lk.polls = polls;
2583 lck->lk.num_polls = num_polls;
2584 lck->lk.mask = mask;
2594 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2600 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2602 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2606 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2608 char const *
const func =
"omp_set_lock";
2609 if ( lck->lk.initialized != lck ) {
2610 KMP_FATAL( LockIsUninitialized, func );
2612 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2613 KMP_FATAL( LockNestableUsedAsSimple, func );
2615 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2616 KMP_FATAL( LockIsAlreadyOwned, func );
2619 __kmp_acquire_drdpa_lock( lck, gtid );
2621 lck->lk.owner_id = gtid + 1;
2625 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2631 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2632 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2633 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2634 TCR_PTR(lck->lk.polls);
2635 kmp_uint64 mask = TCR_8(lck->lk.mask);
2636 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2637 kmp_uint64 next_ticket = ticket + 1;
2638 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2639 ticket, next_ticket)) {
2640 KMP_FSYNC_ACQUIRED(lck);
2641 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2643 lck->lk.now_serving = ticket;
2661 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2663 char const *
const func =
"omp_test_lock";
2664 if ( lck->lk.initialized != lck ) {
2665 KMP_FATAL( LockIsUninitialized, func );
2667 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2668 KMP_FATAL( LockNestableUsedAsSimple, func );
2671 int retval = __kmp_test_drdpa_lock( lck, gtid );
2674 lck->lk.owner_id = gtid + 1;
2680 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2687 kmp_uint64 ticket = lck->lk.now_serving + 1;
2688 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2689 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2690 TCR_PTR(lck->lk.polls);
2691 kmp_uint64 mask = TCR_8(lck->lk.mask);
2692 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2694 KMP_FSYNC_RELEASING(lck);
2695 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2699 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2701 char const *
const func =
"omp_unset_lock";
2703 if ( lck->lk.initialized != lck ) {
2704 KMP_FATAL( LockIsUninitialized, func );
2706 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2707 KMP_FATAL( LockNestableUsedAsSimple, func );
2709 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2710 KMP_FATAL( LockUnsettingFree, func );
2712 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2713 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2714 KMP_FATAL( LockUnsettingSetByAnother, func );
2716 lck->lk.owner_id = 0;
2717 __kmp_release_drdpa_lock( lck, gtid );
2721 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2723 lck->lk.location = NULL;
2725 lck->lk.num_polls = 1;
2726 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2727 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2728 lck->lk.cleanup_ticket = 0;
2729 lck->lk.old_polls = NULL;
2730 lck->lk.next_ticket = 0;
2731 lck->lk.now_serving = 0;
2732 lck->lk.owner_id = 0;
2733 lck->lk.depth_locked = -1;
2734 lck->lk.initialized = lck;
2736 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2740 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2742 __kmp_init_drdpa_lock( lck );
2746 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2748 lck->lk.initialized = NULL;
2749 lck->lk.location = NULL;
2750 if (lck->lk.polls != NULL) {
2751 __kmp_free((
void *)lck->lk.polls);
2752 lck->lk.polls = NULL;
2754 if (lck->lk.old_polls != NULL) {
2755 __kmp_free((
void *)lck->lk.old_polls);
2756 lck->lk.old_polls = NULL;
2759 lck->lk.num_polls = 0;
2760 lck->lk.cleanup_ticket = 0;
2761 lck->lk.next_ticket = 0;
2762 lck->lk.now_serving = 0;
2763 lck->lk.owner_id = 0;
2764 lck->lk.depth_locked = -1;
2768 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2770 char const *
const func =
"omp_destroy_lock";
2771 if ( lck->lk.initialized != lck ) {
2772 KMP_FATAL( LockIsUninitialized, func );
2774 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2775 KMP_FATAL( LockNestableUsedAsSimple, func );
2777 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2778 KMP_FATAL( LockStillOwned, func );
2780 __kmp_destroy_drdpa_lock( lck );
2789 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2791 KMP_DEBUG_ASSERT( gtid >= 0 );
2793 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2794 lck->lk.depth_locked += 1;
2797 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2799 lck->lk.depth_locked = 1;
2801 lck->lk.owner_id = gtid + 1;
2806 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2808 char const *
const func =
"omp_set_nest_lock";
2809 if ( lck->lk.initialized != lck ) {
2810 KMP_FATAL( LockIsUninitialized, func );
2812 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2813 KMP_FATAL( LockSimpleUsedAsNestable, func );
2815 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2819 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2823 KMP_DEBUG_ASSERT( gtid >= 0 );
2825 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2826 retval = ++lck->lk.depth_locked;
2828 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2833 retval = lck->lk.depth_locked = 1;
2835 lck->lk.owner_id = gtid + 1;
2841 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2843 char const *
const func =
"omp_test_nest_lock";
2844 if ( lck->lk.initialized != lck ) {
2845 KMP_FATAL( LockIsUninitialized, func );
2847 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2848 KMP_FATAL( LockSimpleUsedAsNestable, func );
2850 return __kmp_test_nested_drdpa_lock( lck, gtid );
2854 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2856 KMP_DEBUG_ASSERT( gtid >= 0 );
2859 if ( --(lck->lk.depth_locked) == 0 ) {
2861 lck->lk.owner_id = 0;
2862 __kmp_release_drdpa_lock( lck, gtid );
2867 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2869 char const *
const func =
"omp_unset_nest_lock";
2871 if ( lck->lk.initialized != lck ) {
2872 KMP_FATAL( LockIsUninitialized, func );
2874 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2875 KMP_FATAL( LockSimpleUsedAsNestable, func );
2877 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2878 KMP_FATAL( LockUnsettingFree, func );
2880 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
2881 KMP_FATAL( LockUnsettingSetByAnother, func );
2883 __kmp_release_nested_drdpa_lock( lck, gtid );
2887 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
2889 __kmp_init_drdpa_lock( lck );
2890 lck->lk.depth_locked = 0;
2894 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2896 __kmp_init_nested_drdpa_lock( lck );
2900 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
2902 __kmp_destroy_drdpa_lock( lck );
2903 lck->lk.depth_locked = 0;
2907 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2909 char const *
const func =
"omp_destroy_nest_lock";
2910 if ( lck->lk.initialized != lck ) {
2911 KMP_FATAL( LockIsUninitialized, func );
2913 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2914 KMP_FATAL( LockSimpleUsedAsNestable, func );
2916 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2917 KMP_FATAL( LockStillOwned, func );
2919 __kmp_destroy_nested_drdpa_lock( lck );
2928 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
2930 return lck == lck->lk.initialized;
2934 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
2936 return lck->lk.location;
2940 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
2942 lck->lk.location = loc;
2945 static kmp_lock_flags_t
2946 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
2948 return lck->lk.flags;
2952 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
2954 lck->lk.flags = flags;
2964 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
2966 size_t __kmp_base_user_lock_size = 0;
2967 size_t __kmp_user_lock_size = 0;
2969 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
2970 void ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
2972 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
2973 void ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
2974 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
2975 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
2976 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
2977 void ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
2979 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
2980 void ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
2981 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
2982 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
2984 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
2985 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
2986 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
2987 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
2988 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
2990 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
2992 switch ( user_lock_kind ) {
2998 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
2999 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3001 __kmp_get_user_lock_owner_ =
3002 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3003 ( &__kmp_get_tas_lock_owner );
3005 if ( __kmp_env_consistency_check ) {
3006 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3007 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3010 KMP_BIND_USER_LOCK(tas);
3011 KMP_BIND_NESTED_USER_LOCK(tas);
3014 __kmp_destroy_user_lock_ =
3015 ( void ( * )( kmp_user_lock_p ) )
3016 ( &__kmp_destroy_tas_lock );
3018 __kmp_is_user_lock_initialized_ =
3019 ( int ( * )( kmp_user_lock_p ) ) NULL;
3021 __kmp_get_user_lock_location_ =
3022 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3024 __kmp_set_user_lock_location_ =
3025 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3027 __kmp_get_user_lock_flags_ =
3028 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3030 __kmp_set_user_lock_flags_ =
3031 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3035 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
3038 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3039 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3041 __kmp_get_user_lock_owner_ =
3042 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3043 ( &__kmp_get_futex_lock_owner );
3045 if ( __kmp_env_consistency_check ) {
3046 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3047 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3050 KMP_BIND_USER_LOCK(futex);
3051 KMP_BIND_NESTED_USER_LOCK(futex);
3054 __kmp_destroy_user_lock_ =
3055 ( void ( * )( kmp_user_lock_p ) )
3056 ( &__kmp_destroy_futex_lock );
3058 __kmp_is_user_lock_initialized_ =
3059 ( int ( * )( kmp_user_lock_p ) ) NULL;
3061 __kmp_get_user_lock_location_ =
3062 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3064 __kmp_set_user_lock_location_ =
3065 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3067 __kmp_get_user_lock_flags_ =
3068 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3070 __kmp_set_user_lock_flags_ =
3071 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3075 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
3078 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3079 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3081 __kmp_get_user_lock_owner_ =
3082 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3083 ( &__kmp_get_ticket_lock_owner );
3085 if ( __kmp_env_consistency_check ) {
3086 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3087 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3090 KMP_BIND_USER_LOCK(ticket);
3091 KMP_BIND_NESTED_USER_LOCK(ticket);
3094 __kmp_destroy_user_lock_ =
3095 ( void ( * )( kmp_user_lock_p ) )
3096 ( &__kmp_destroy_ticket_lock );
3098 __kmp_is_user_lock_initialized_ =
3099 ( int ( * )( kmp_user_lock_p ) )
3100 ( &__kmp_is_ticket_lock_initialized );
3102 __kmp_get_user_lock_location_ =
3103 (
const ident_t * ( * )( kmp_user_lock_p ) )
3104 ( &__kmp_get_ticket_lock_location );
3106 __kmp_set_user_lock_location_ =
3107 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3108 ( &__kmp_set_ticket_lock_location );
3110 __kmp_get_user_lock_flags_ =
3111 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3112 ( &__kmp_get_ticket_lock_flags );
3114 __kmp_set_user_lock_flags_ =
3115 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3116 ( &__kmp_set_ticket_lock_flags );
3121 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3122 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3124 __kmp_get_user_lock_owner_ =
3125 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3126 ( &__kmp_get_queuing_lock_owner );
3128 if ( __kmp_env_consistency_check ) {
3129 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3130 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3133 KMP_BIND_USER_LOCK(queuing);
3134 KMP_BIND_NESTED_USER_LOCK(queuing);
3137 __kmp_destroy_user_lock_ =
3138 ( void ( * )( kmp_user_lock_p ) )
3139 ( &__kmp_destroy_queuing_lock );
3141 __kmp_is_user_lock_initialized_ =
3142 ( int ( * )( kmp_user_lock_p ) )
3143 ( &__kmp_is_queuing_lock_initialized );
3145 __kmp_get_user_lock_location_ =
3146 (
const ident_t * ( * )( kmp_user_lock_p ) )
3147 ( &__kmp_get_queuing_lock_location );
3149 __kmp_set_user_lock_location_ =
3150 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3151 ( &__kmp_set_queuing_lock_location );
3153 __kmp_get_user_lock_flags_ =
3154 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3155 ( &__kmp_get_queuing_lock_flags );
3157 __kmp_set_user_lock_flags_ =
3158 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3159 ( &__kmp_set_queuing_lock_flags );
3163 #if KMP_USE_ADAPTIVE_LOCKS
3165 __kmp_base_user_lock_size =
sizeof( kmp_base_adaptive_lock_t );
3166 __kmp_user_lock_size =
sizeof( kmp_adaptive_lock_t );
3168 __kmp_get_user_lock_owner_ =
3169 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3170 ( &__kmp_get_queuing_lock_owner );
3172 if ( __kmp_env_consistency_check ) {
3173 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3176 KMP_BIND_USER_LOCK(adaptive);
3179 __kmp_destroy_user_lock_ =
3180 ( void ( * )( kmp_user_lock_p ) )
3181 ( &__kmp_destroy_adaptive_lock );
3183 __kmp_is_user_lock_initialized_ =
3184 ( int ( * )( kmp_user_lock_p ) )
3185 ( &__kmp_is_queuing_lock_initialized );
3187 __kmp_get_user_lock_location_ =
3188 (
const ident_t * ( * )( kmp_user_lock_p ) )
3189 ( &__kmp_get_queuing_lock_location );
3191 __kmp_set_user_lock_location_ =
3192 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3193 ( &__kmp_set_queuing_lock_location );
3195 __kmp_get_user_lock_flags_ =
3196 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3197 ( &__kmp_get_queuing_lock_flags );
3199 __kmp_set_user_lock_flags_ =
3200 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3201 ( &__kmp_set_queuing_lock_flags );
3205 #endif // KMP_USE_ADAPTIVE_LOCKS
3208 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3209 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3211 __kmp_get_user_lock_owner_ =
3212 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3213 ( &__kmp_get_drdpa_lock_owner );
3215 if ( __kmp_env_consistency_check ) {
3216 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3217 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3220 KMP_BIND_USER_LOCK(drdpa);
3221 KMP_BIND_NESTED_USER_LOCK(drdpa);
3224 __kmp_destroy_user_lock_ =
3225 ( void ( * )( kmp_user_lock_p ) )
3226 ( &__kmp_destroy_drdpa_lock );
3228 __kmp_is_user_lock_initialized_ =
3229 ( int ( * )( kmp_user_lock_p ) )
3230 ( &__kmp_is_drdpa_lock_initialized );
3232 __kmp_get_user_lock_location_ =
3233 (
const ident_t * ( * )( kmp_user_lock_p ) )
3234 ( &__kmp_get_drdpa_lock_location );
3236 __kmp_set_user_lock_location_ =
3237 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3238 ( &__kmp_set_drdpa_lock_location );
3240 __kmp_get_user_lock_flags_ =
3241 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3242 ( &__kmp_get_drdpa_lock_flags );
3244 __kmp_set_user_lock_flags_ =
3245 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3246 ( &__kmp_set_drdpa_lock_flags );
3256 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3257 kmp_user_lock_p __kmp_lock_pool = NULL;
3260 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3261 int __kmp_num_locks_in_block = 1;
3263 static kmp_lock_index_t
3264 __kmp_lock_table_insert( kmp_user_lock_p lck )
3267 kmp_lock_index_t index;
3268 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3269 kmp_lock_index_t size;
3270 kmp_user_lock_p *table;
3273 if ( __kmp_user_lock_table.allocated == 0 ) {
3277 size = __kmp_user_lock_table.allocated * 2;
3279 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
3280 memcpy( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
3281 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3286 __kmp_user_lock_table.table = table;
3287 __kmp_user_lock_table.allocated = size;
3289 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
3290 index = __kmp_user_lock_table.used;
3291 __kmp_user_lock_table.table[ index ] = lck;
3292 ++ __kmp_user_lock_table.used;
3296 static kmp_user_lock_p
3297 __kmp_lock_block_allocate()
3300 static int last_index = 0;
3301 if ( ( last_index >= __kmp_num_locks_in_block )
3302 || ( __kmp_lock_blocks == NULL ) ) {
3306 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3307 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3308 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
3310 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
3311 new_block->next_block = __kmp_lock_blocks;
3312 new_block->locks = (
void *)buffer;
3315 __kmp_lock_blocks = new_block;
3317 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
3318 [ last_index * __kmp_user_lock_size ] ) );
3328 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3329 kmp_lock_flags_t flags )
3331 kmp_user_lock_p lck;
3332 kmp_lock_index_t index;
3333 KMP_DEBUG_ASSERT( user_lock );
3335 __kmp_acquire_lock( &__kmp_global_lock, gtid );
3337 if ( __kmp_lock_pool == NULL ) {
3339 if ( __kmp_num_locks_in_block <= 1 ) {
3340 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
3343 lck = __kmp_lock_block_allocate();
3348 index = __kmp_lock_table_insert( lck );
3352 lck = __kmp_lock_pool;
3353 index = __kmp_lock_pool->pool.index;
3354 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3361 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3362 * ( (kmp_lock_index_t *) user_lock ) = index;
3365 * ( (kmp_user_lock_p *) user_lock ) = lck;
3369 __kmp_set_user_lock_flags( lck, flags );
3371 __kmp_release_lock( & __kmp_global_lock, gtid );
3378 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
3380 kmp_lock_pool_t * lock_pool;
3382 KMP_DEBUG_ASSERT( user_lock != NULL );
3383 KMP_DEBUG_ASSERT( lck != NULL );
3385 __kmp_acquire_lock( & __kmp_global_lock, gtid );
3387 lck->pool.next = __kmp_lock_pool;
3388 __kmp_lock_pool = lck;
3389 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3390 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
3391 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
3392 lck->pool.index = index;
3395 __kmp_release_lock( & __kmp_global_lock, gtid );
3399 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
3401 kmp_user_lock_p lck = NULL;
3403 if ( __kmp_env_consistency_check ) {
3404 if ( user_lock == NULL ) {
3405 KMP_FATAL( LockIsUninitialized, func );
3409 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
3410 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
3411 if ( __kmp_env_consistency_check ) {
3412 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
3413 KMP_FATAL( LockIsUninitialized, func );
3416 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
3417 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
3418 lck = __kmp_user_lock_table.table[index];
3421 lck = *( (kmp_user_lock_p *)user_lock );
3424 if ( __kmp_env_consistency_check ) {
3425 if ( lck == NULL ) {
3426 KMP_FATAL( LockIsUninitialized, func );
3434 __kmp_cleanup_user_locks(
void )
3441 __kmp_lock_pool = NULL;
3443 #define IS_CRITICAL(lck) \
3444 ( ( __kmp_get_user_lock_flags_ != NULL ) && \
3445 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) )
3475 while ( __kmp_user_lock_table.used > 1 ) {
3482 kmp_user_lock_p lck = __kmp_user_lock_table.table[
3483 --__kmp_user_lock_table.used ];
3485 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
3486 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
3493 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
3494 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
3496 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
3497 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.func,
3498 str_loc.line, str_loc.col );
3499 __kmp_str_loc_free( &str_loc);
3503 if ( IS_CRITICAL( lck ) ) {
3504 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
3507 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
3515 __kmp_destroy_user_lock( lck );
3521 if ( __kmp_lock_blocks == NULL ) {
3531 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
3532 __kmp_user_lock_table.table = NULL;
3533 __kmp_user_lock_table.allocated = 0;
3535 while ( table_ptr != NULL ) {
3540 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
3541 __kmp_free( table_ptr );
3548 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
3549 __kmp_lock_blocks = NULL;
3551 while ( block_ptr != NULL ) {
3552 kmp_block_of_locks_t *next = block_ptr->next_block;
3553 __kmp_free( block_ptr->locks );
3560 TCW_4(__kmp_init_user_locks, FALSE);