38 #include "kmp_wrapper_malloc.h"
46 typedef int (*bget_compact_t)(size_t, int);
47 typedef void *(*bget_acquire_t)(size_t);
48 typedef void (*bget_release_t)(
void *);
53 # if KMP_ARCH_X86 || KMP_ARCH_ARM
54 typedef kmp_int32 bufsize;
56 typedef kmp_int64 bufsize;
59 typedef ssize_t bufsize;
64 typedef enum bget_mode {
71 static void bpool( kmp_info_t *th,
void *buffer, bufsize len);
72 static void *bget( kmp_info_t *th, bufsize size);
73 static void *bgetz( kmp_info_t *th, bufsize size);
74 static void *bgetr( kmp_info_t *th,
void *buffer, bufsize newsize);
75 static void brel( kmp_info_t *th,
void *buf);
76 static void bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr );
79 static void bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
long *nget,
long *nrel);
80 static void bstatse( kmp_info_t *th, bufsize *pool_incr,
long *npool,
long *npget,
long *nprel,
long *ndget,
long *ndrel);
81 static void bufdump( kmp_info_t *th,
void *buf);
82 static void bpoold( kmp_info_t *th,
void *pool,
int dumpalloc,
int dumpfree);
83 static int bpoolv( kmp_info_t *th,
void *pool);
96 #if KMP_ARCH_X86 || !KMP_HAVE_QUAD
99 #define AlignType double
104 #define AlignType _Quad
145 static bufsize bget_bin_size[ ] = {
169 #define MAX_BGET_BINS (int)(sizeof(bget_bin_size) / sizeof(bufsize))
178 typedef struct qlinks {
179 struct bfhead *flink;
180 struct bfhead *blink;
185 typedef struct bhead2 {
196 typedef union bhead {
197 KMP_ALIGN( SizeQuant )
199 char b_pad[ sizeof(bhead2_t) + (SizeQuant - (sizeof(bhead2_t) % SizeQuant)) ];
202 #define BH(p) ((bhead_t *) (p))
206 typedef struct bdhead
211 #define BDH(p) ((bdhead_t *) (p))
215 typedef struct bfhead {
219 #define BFH(p) ((bfhead_t *) (p))
221 typedef struct thr_data {
222 bfhead_t freelist[ MAX_BGET_BINS ];
227 long numpget, numprel;
228 long numdget, numdrel;
232 bget_compact_t compfcn;
233 bget_acquire_t acqfcn;
234 bget_release_t relfcn;
245 bfhead_t * last_pool;
250 #define QLSize (sizeof(qlinks_t))
251 #define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize)
252 #define MaxSize (bufsize)( ~ ( ( (bufsize)( 1 ) << ( sizeof( bufsize ) * CHAR_BIT - 1 ) ) | ( SizeQuant - 1 ) ) )
259 #define ESent ((bufsize) (-(((((bufsize)1)<<((int)sizeof(bufsize)*8-2))-1)*2)-2))
266 bget_get_bin( bufsize size )
269 int lo = 0, hi = MAX_BGET_BINS - 1;
271 KMP_DEBUG_ASSERT( size > 0 );
273 while ( (hi - lo) > 1 ) {
274 int mid = (lo + hi) >> 1;
275 if (size < bget_bin_size[ mid ])
281 KMP_DEBUG_ASSERT( (lo >= 0) && (lo < MAX_BGET_BINS) );
287 set_thr_data( kmp_info_t *th )
294 ( ! th->th.th_local.bget_data ) ? __kmp_allocate(
sizeof( *data ) ) : th->th.th_local.bget_data
297 memset( data,
'\0',
sizeof( *data ) );
299 for (i = 0; i < MAX_BGET_BINS; ++i) {
300 data->freelist[ i ].ql.flink = & data->freelist[ i ];
301 data->freelist[ i ].ql.blink = & data->freelist[ i ];
304 th->th.th_local.bget_data = data;
305 th->th.th_local.bget_list = 0;
306 #if ! USE_CMP_XCHG_FOR_BGET
307 #ifdef USE_QUEUING_LOCK_FOR_BGET
308 __kmp_init_lock( & th->th.th_local.bget_lock );
310 __kmp_init_bootstrap_lock( & th->th.th_local.bget_lock );
316 get_thr_data( kmp_info_t *th )
320 data = (thr_data_t *) th->th.th_local.bget_data;
322 KMP_DEBUG_ASSERT( data != 0 );
331 __kmp_bget_validate_queue( kmp_info_t *th )
335 void *p = (
void *) th->th.th_local.bget_list;
338 bfhead_t *b = BFH(((
char *) p) -
sizeof(bhead_t));
340 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
341 p = (
void *) b->ql.flink;
350 __kmp_bget_dequeue( kmp_info_t *th )
352 void *p = TCR_SYNC_PTR(th->th.th_local.bget_list);
355 #if USE_CMP_XCHG_FOR_BGET
357 volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
358 while ( ! KMP_COMPARE_AND_STORE_PTR(
359 & th->th.th_local.bget_list, old_value, NULL ) )
362 old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
364 p = (
void *) old_value;
367 #ifdef USE_QUEUING_LOCK_FOR_BGET
368 __kmp_acquire_lock( & th->th.th_local.bget_lock,
369 __kmp_gtid_from_thread(th) );
371 __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
374 p = (
void *) th->th.th_local.bget_list;
375 th->th.th_local.bget_list = 0;
377 #ifdef USE_QUEUING_LOCK_FOR_BGET
378 __kmp_release_lock( & th->th.th_local.bget_lock,
379 __kmp_gtid_from_thread(th) );
381 __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
389 bfhead_t *b = BFH(((
char *) p) -
sizeof(bhead_t));
391 KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
392 KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
394 KMP_DEBUG_ASSERT( b->ql.blink == 0 );
396 p = (
void *) b->ql.flink;
406 __kmp_bget_enqueue( kmp_info_t *th,
void *buf
407 #ifdef USE_QUEUING_LOCK_FOR_BGET
412 bfhead_t *b = BFH(((
char *) buf) -
sizeof(bhead_t));
414 KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
415 KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
420 KC_TRACE( 10, (
"__kmp_bget_enqueue: moving buffer to T#%d list\n",
421 __kmp_gtid_from_thread( th ) ) );
423 #if USE_CMP_XCHG_FOR_BGET
425 volatile void *old_value = TCR_PTR(th->th.th_local.bget_list);
428 b->ql.flink = BFH( old_value );
430 while ( ! KMP_COMPARE_AND_STORE_PTR(
431 & th->th.th_local.bget_list, old_value, buf ) )
434 old_value = TCR_PTR(th->th.th_local.bget_list);
437 b->ql.flink = BFH( old_value );
441 # ifdef USE_QUEUING_LOCK_FOR_BGET
442 __kmp_acquire_lock( & th->th.th_local.bget_lock, rel_gtid );
444 __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
447 b->ql.flink = BFH( th->th.th_local.bget_list );
448 th->th.th_local.bget_list = (
void *) buf;
450 # ifdef USE_QUEUING_LOCK_FOR_BGET
451 __kmp_release_lock( & th->th.th_local.bget_lock, rel_gtid );
453 __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
461 __kmp_bget_insert_into_freelist( thr_data_t *thr, bfhead_t *b )
465 KMP_DEBUG_ASSERT( ((
size_t)b ) % SizeQuant == 0 );
466 KMP_DEBUG_ASSERT( b->bh.bb.bsize % SizeQuant == 0 );
468 bin = bget_get_bin( b->bh.bb.bsize );
470 KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.blink->ql.flink == &thr->freelist[ bin ]);
471 KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.flink->ql.blink == &thr->freelist[ bin ]);
473 b->ql.flink = &thr->freelist[ bin ];
474 b->ql.blink = thr->freelist[ bin ].ql.blink;
476 thr->freelist[ bin ].ql.blink = b;
477 b->ql.blink->ql.flink = b;
483 __kmp_bget_remove_from_freelist( bfhead_t *b )
485 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b);
486 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b);
488 b->ql.blink->ql.flink = b->ql.flink;
489 b->ql.flink->ql.blink = b->ql.blink;
497 bcheck( kmp_info_t *th, bufsize *max_free, bufsize *total_free )
499 thr_data_t *thr = get_thr_data( th );
502 *total_free = *max_free = 0;
504 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
507 best = &thr->freelist[ bin ];
510 while (b != &thr->freelist[ bin ]) {
511 *total_free += (b->bh.bb.bsize -
sizeof( bhead_t ));
512 if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize))
519 if (*max_free < best->bh.bb.bsize)
520 *max_free = best->bh.bb.bsize;
523 if (*max_free > (bufsize)
sizeof( bhead_t ))
524 *max_free -=
sizeof( bhead_t );
532 bget( kmp_info_t *th, bufsize requested_size )
534 thr_data_t *thr = get_thr_data( th );
535 bufsize size = requested_size;
543 if ( size < 0 || size +
sizeof( bhead_t ) > MaxSize ) {
547 __kmp_bget_dequeue( th );
549 if (size < (bufsize)SizeQ) {
552 #if defined( SizeQuant ) && ( SizeQuant > 1 )
553 size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1));
556 size +=
sizeof(bhead_t);
558 KMP_DEBUG_ASSERT( size >= 0 );
559 KMP_DEBUG_ASSERT( size % SizeQuant == 0 );
561 use_blink = ( thr->mode == bget_mode_lifo );
570 for (bin = bget_get_bin( size ); bin < MAX_BGET_BINS; ++bin) {
572 b = ( use_blink ? thr->freelist[ bin ].ql.blink : thr->freelist[ bin ].ql.flink );
574 if (thr->mode == bget_mode_best) {
575 best = &thr->freelist[ bin ];
580 while (b != &thr->freelist[ bin ]) {
581 if (b->bh.bb.bsize >= (bufsize) size) {
582 if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize)) {
588 b = ( use_blink ? b->ql.blink : b->ql.flink );
593 while (b != &thr->freelist[ bin ]) {
594 if ((bufsize) b->bh.bb.bsize >= (bufsize) size) {
605 if ((b->bh.bb.bsize - (bufsize) size) > (bufsize)(SizeQ + (
sizeof(bhead_t)))) {
608 ba = BH(((
char *) b) + (b->bh.bb.bsize - (bufsize) size));
609 bn = BH(((
char *) ba) + size);
611 KMP_DEBUG_ASSERT(bn->bb.prevfree == b->bh.bb.bsize);
614 b->bh.bb.bsize -= (bufsize) size;
617 ba->bb.prevfree = b->bh.bb.bsize;
620 ba->bb.bsize = -size;
623 TCW_PTR(ba->bb.bthr, th);
628 __kmp_bget_remove_from_freelist( b );
629 __kmp_bget_insert_into_freelist( thr, b );
631 thr->totalloc += (size_t) size;
634 buf = (
void *) ((((
char *) ba) +
sizeof(bhead_t)));
635 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
640 ba = BH(((
char *) b) + b->bh.bb.bsize);
642 KMP_DEBUG_ASSERT(ba->bb.prevfree == b->bh.bb.bsize);
647 __kmp_bget_remove_from_freelist( b );
649 thr->totalloc += (size_t) b->bh.bb.bsize;
653 b->bh.bb.bsize = -(b->bh.bb.bsize);
656 TCW_PTR(ba->bb.bthr, th);
662 buf = (
void *) &(b->ql);
663 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
669 b = ( use_blink ? b->ql.blink : b->ql.flink );
677 if ((thr->compfcn == 0) || (!(*thr->compfcn)(size, ++compactseq))) {
686 if (thr->acqfcn != 0) {
687 if (size > (bufsize) (thr->exp_incr -
sizeof(bhead_t))) {
694 size +=
sizeof(bdhead_t) -
sizeof(bhead_t);
696 KE_TRACE( 10, (
"%%%%%% MALLOC( %d )\n", (
int) size ) );
699 bdh = BDH((*thr->acqfcn)((bufsize) size));
704 bdh->bh.bb.bsize = 0;
707 TCW_PTR(bdh->bh.bb.bthr, th);
709 bdh->bh.bb.prevfree = 0;
712 thr->totalloc += (size_t) size;
716 buf = (
void *) (bdh + 1);
717 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
727 KE_TRACE( 10, (
"%%%%%% MALLOCB( %d )\n", (
int) thr->exp_incr ) );
730 newpool = (*thr->acqfcn)((bufsize) thr->exp_incr);
731 KMP_DEBUG_ASSERT( ((
size_t)newpool) % SizeQuant == 0 );
732 if (newpool != NULL) {
733 bpool( th, newpool, thr->exp_incr);
734 buf = bget( th, requested_size);
750 bgetz( kmp_info_t *th, bufsize size )
752 char *buf = (
char *) bget( th, size);
758 b = BH(buf -
sizeof(bhead_t));
759 rsize = -(b->bb.bsize);
763 bd = BDH(buf -
sizeof(bdhead_t));
764 rsize = bd->tsize - (bufsize)
sizeof(bdhead_t);
766 rsize -=
sizeof(bhead_t);
769 KMP_DEBUG_ASSERT(rsize >= size);
771 (void) memset(buf, 0, (bufsize) rsize);
773 return ((
void *) buf);
782 bgetr( kmp_info_t *th,
void *buf, bufsize size)
788 nbuf = bget( th, size );
789 if ( nbuf == NULL ) {
795 b = BH(((
char *) buf) -
sizeof(bhead_t));
796 osize = -b->bb.bsize;
801 bd = BDH(((
char *) buf) -
sizeof(bdhead_t));
802 osize = bd->tsize - (bufsize)
sizeof(bdhead_t);
804 osize -=
sizeof(bhead_t);
807 KMP_DEBUG_ASSERT(osize > 0);
809 (void) memcpy((
char *) nbuf, (
char *) buf,
810 (
size_t) ((size < osize) ? size : osize));
819 brel( kmp_info_t *th,
void *buf )
821 thr_data_t *thr = get_thr_data( th );
825 KMP_DEBUG_ASSERT(buf != NULL);
826 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
828 b = BFH(((
char *) buf) -
sizeof(bhead_t));
830 if (b->bh.bb.bsize == 0) {
833 bdh = BDH(((
char *) buf) -
sizeof(bdhead_t));
834 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
836 thr->totalloc -= (size_t) bdh->tsize;
841 (
void) memset((
char *) buf, 0x55,
842 (
size_t) (bdh->tsize -
sizeof(bdhead_t)));
845 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) bdh ) );
847 KMP_DEBUG_ASSERT( thr->relfcn != 0 );
848 (*thr->relfcn)((
void *) bdh);
852 bth = (kmp_info_t *)( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 );
855 __kmp_bget_enqueue( bth, buf
856 #ifdef USE_QUEUING_LOCK_FOR_BGET
857 , __kmp_gtid_from_thread( th )
866 if (b->bh.bb.bsize >= 0) {
869 KMP_DEBUG_ASSERT(b->bh.bb.bsize < 0);
874 KMP_DEBUG_ASSERT(BH((
char *) b - b->bh.bb.bsize)->bb.prevfree == 0);
878 thr->totalloc += (size_t) b->bh.bb.bsize;
883 if (b->bh.bb.prevfree != 0) {
890 register bufsize size = b->bh.bb.bsize;
893 KMP_DEBUG_ASSERT(BH((
char *) b - b->bh.bb.prevfree)->bb.bsize == b->bh.bb.prevfree);
894 b = BFH(((
char *) b) - b->bh.bb.prevfree);
895 b->bh.bb.bsize -= size;
898 __kmp_bget_remove_from_freelist( b );
905 b->bh.bb.bsize = -b->bh.bb.bsize;
909 __kmp_bget_insert_into_freelist( thr, b );
917 bn = BFH(((
char *) b) + b->bh.bb.bsize);
918 if (bn->bh.bb.bsize > 0) {
923 KMP_DEBUG_ASSERT(BH((
char *) bn + bn->bh.bb.bsize)->bb.prevfree == bn->bh.bb.bsize);
925 __kmp_bget_remove_from_freelist( bn );
927 b->bh.bb.bsize += bn->bh.bb.bsize;
931 __kmp_bget_remove_from_freelist( b );
932 __kmp_bget_insert_into_freelist( thr, b );
941 bn = BFH(((
char *) b) + b->bh.bb.bsize);
944 (void) memset(((
char *) b) +
sizeof(bfhead_t), 0x55,
945 (
size_t) (b->bh.bb.bsize -
sizeof(bfhead_t)));
947 KMP_DEBUG_ASSERT(bn->bh.bb.bsize < 0);
952 bn->bh.bb.prevfree = b->bh.bb.bsize;
959 if (thr->relfcn != 0 &&
960 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t)))
963 if (thr->numpblk != 1) {
966 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
967 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
968 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
971 __kmp_bget_remove_from_freelist( b );
973 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) b ) );
979 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
982 if (thr->last_pool == b) thr->last_pool = 0;
994 bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr)
996 thr_data_t *thr = get_thr_data( th );
998 thr->compfcn = compact;
999 thr->acqfcn = acquire;
1000 thr->relfcn = release;
1001 thr->exp_incr = pool_incr;
1007 bpool( kmp_info_t *th,
void *buf, bufsize len)
1010 thr_data_t *thr = get_thr_data( th );
1011 bfhead_t *b = BFH(buf);
1014 __kmp_bget_dequeue( th );
1017 len &= ~(SizeQuant - 1);
1019 if (thr->pool_len == 0) {
1020 thr->pool_len = len;
1021 }
else if (len != thr->pool_len) {
1027 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
1034 KMP_DEBUG_ASSERT(len -
sizeof(bhead_t) <= -((bufsize) ESent + 1));
1040 b->bh.bb.prevfree = 0;
1050 len -=
sizeof(bhead_t);
1051 b->bh.bb.bsize = (bufsize) len;
1053 TCW_PTR( b->bh.bb.bthr, (kmp_info_t*)((kmp_uintptr_t)th | 1) );
1056 __kmp_bget_insert_into_freelist( thr, b );
1059 (void) memset(((
char *) b) +
sizeof(bfhead_t), 0x55,
1060 (
size_t) (len -
sizeof(bfhead_t)));
1062 bn = BH(((
char *) b) + len);
1063 bn->bb.prevfree = (bufsize) len;
1065 KMP_DEBUG_ASSERT( (~0) == -1 && (bn != 0) );
1067 bn->bb.bsize = ESent;
1075 bfreed( kmp_info_t *th )
1077 int bin = 0, count = 0;
1078 int gtid = __kmp_gtid_from_thread( th );
1079 thr_data_t *thr = get_thr_data( th );
1082 __kmp_printf_no_lock(
"__kmp_printpool: T#%d total=%" KMP_UINT64_SPEC
" get=%" KMP_INT64_SPEC
" rel=%" \
1083 KMP_INT64_SPEC
" pblk=%" KMP_INT64_SPEC
" pget=%" KMP_INT64_SPEC
" prel=%" KMP_INT64_SPEC \
1084 " dget=%" KMP_INT64_SPEC
" drel=%" KMP_INT64_SPEC
"\n",
1085 gtid, (kmp_uint64) thr->totalloc,
1086 (kmp_int64) thr->numget, (kmp_int64) thr->numrel,
1087 (kmp_int64) thr->numpblk,
1088 (kmp_int64) thr->numpget, (kmp_int64) thr->numprel,
1089 (kmp_int64) thr->numdget, (kmp_int64) thr->numdrel );
1092 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1095 for (b = thr->freelist[ bin ].ql.flink; b != &thr->freelist[ bin ]; b = b->ql.flink) {
1096 bufsize bs = b->bh.bb.bsize;
1098 KMP_DEBUG_ASSERT( b->ql.blink->ql.flink == b );
1099 KMP_DEBUG_ASSERT( b->ql.flink->ql.blink == b );
1100 KMP_DEBUG_ASSERT( bs > 0 );
1104 __kmp_printf_no_lock(
"__kmp_printpool: T#%d Free block: 0x%p size %6ld bytes.\n", gtid, b, (
long) bs );
1107 char *lerr = ((
char *) b) +
sizeof(bfhead_t);
1108 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) || (memcmp(lerr, lerr + 1, (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1109 __kmp_printf_no_lock(
"__kmp_printpool: T#%d (Contents of above free block have been overstored.)\n", gtid );
1117 __kmp_printf_no_lock(
"__kmp_printpool: T#%d No free blocks\n", gtid );
1129 bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
long *nget,
long *nrel)
1132 thr_data_t *thr = get_thr_data( th );
1134 *nget = thr->numget;
1135 *nrel = thr->numrel;
1136 *curalloc = (bufsize) thr->totalloc;
1140 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1141 bfhead_t *b = thr->freelist[ bin ].ql.flink;
1143 while (b != &thr->freelist[ bin ]) {
1144 KMP_DEBUG_ASSERT(b->bh.bb.bsize > 0);
1145 *totfree += b->bh.bb.bsize;
1146 if (b->bh.bb.bsize > *maxfree) {
1147 *maxfree = b->bh.bb.bsize;
1157 bstatse( kmp_info_t *th, bufsize *pool_incr,
long *npool,
long *npget,
long *nprel,
long *ndget,
long *ndrel)
1159 thr_data_t *thr = get_thr_data( th );
1161 *pool_incr = (thr->pool_len < 0) ? -thr->exp_incr : thr->exp_incr;
1162 *npool = thr->numpblk;
1163 *npget = thr->numpget;
1164 *nprel = thr->numprel;
1165 *ndget = thr->numdget;
1166 *ndrel = thr->numdrel;
1176 bufdump( kmp_info_t *th,
void *buf )
1179 unsigned char *bdump;
1182 b = BFH(((
char *) buf) -
sizeof(bhead_t));
1183 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
1184 if (b->bh.bb.bsize < 0) {
1185 bdump = (
unsigned char *) buf;
1186 bdlen = (-b->bh.bb.bsize) - (bufsize)
sizeof(bhead_t);
1188 bdump = (
unsigned char *) (((
char *) b) +
sizeof(bfhead_t));
1189 bdlen = b->bh.bb.bsize - (bufsize)
sizeof(bfhead_t);
1195 char bhex[50], bascii[20];
1201 for (i = 0; i < l; i++) {
1202 (void) sprintf(bhex + i * 3,
"%02X ", bdump[i]);
1203 if (bdump[i] > 0x20 && bdump[i] < 0x7F)
1204 bascii[ i ] = bdump[ i ];
1209 (void) __kmp_printf_no_lock(
"%-48s %s\n", bhex, bascii);
1212 while ((bdlen > 16) && (memcmp((
char *) (bdump - 16),
1213 (
char *) bdump, 16) == 0)) {
1219 (void) __kmp_printf_no_lock(
1220 " (%d lines [%d bytes] identical to above line skipped)\n",
1222 }
else if (dupes == 1) {
1236 bpoold( kmp_info_t *th,
void *buf,
int dumpalloc,
int dumpfree)
1238 bfhead_t *b = BFH( (
char*)buf -
sizeof(bhead_t));
1240 while (b->bh.bb.bsize != ESent) {
1241 bufsize bs = b->bh.bb.bsize;
1245 (void) __kmp_printf_no_lock(
"Allocated buffer: size %6ld bytes.\n", (
long) bs);
1247 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1250 const char *lerr =
"";
1252 KMP_DEBUG_ASSERT(bs > 0);
1253 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1254 lerr =
" (Bad free list links)";
1256 (void) __kmp_printf_no_lock(
"Free block: size %6ld bytes.%s\n",
1259 lerr = ((
char *) b) +
sizeof(bfhead_t);
1260 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) ||
1261 (memcmp(lerr, lerr + 1,
1262 (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1263 (void) __kmp_printf_no_lock(
1264 "(Contents of above free block have been overstored.)\n");
1265 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1269 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1272 b = BFH(((
char *) b) + bs);
1279 bpoolv( kmp_info_t *th,
void *buf )
1281 bfhead_t *b = BFH(buf);
1283 while (b->bh.bb.bsize != ESent) {
1284 bufsize bs = b->bh.bb.bsize;
1293 KMP_DEBUG_ASSERT(bs > 0);
1297 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1298 (void) __kmp_printf_no_lock(
"Free block: size %6ld bytes. (Bad free list links)\n",
1300 KMP_DEBUG_ASSERT(0);
1304 lerr = ((
char *) b) +
sizeof(bfhead_t);
1305 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) ||
1306 (memcmp(lerr, lerr + 1,
1307 (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1308 (void) __kmp_printf_no_lock(
1309 "(Contents of above free block have been overstored.)\n");
1310 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1311 KMP_DEBUG_ASSERT(0);
1316 b = BFH(((
char *) b) + bs);
1326 __kmp_initialize_bget( kmp_info_t *th )
1328 KMP_DEBUG_ASSERT( SizeQuant >=
sizeof(
void * ) && (th != 0) );
1332 bectl( th, (bget_compact_t) 0, (bget_acquire_t) malloc, (bget_release_t) free,
1333 (bufsize) __kmp_malloc_pool_incr );
1337 __kmp_finalize_bget( kmp_info_t *th )
1342 KMP_DEBUG_ASSERT( th != 0 );
1345 thr = (thr_data_t *) th->th.th_local.bget_data;
1346 KMP_DEBUG_ASSERT( thr != NULL );
1355 if (thr->relfcn != 0 && b != 0 && thr->numpblk != 0 &&
1356 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t)))
1358 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
1359 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
1360 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
1363 __kmp_bget_remove_from_freelist( b );
1365 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) b ) );
1370 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
1375 if ( th->th.th_local.bget_data != NULL ) {
1376 __kmp_free( th->th.th_local.bget_data );
1377 th->th.th_local.bget_data = NULL;
1382 kmpc_set_poolsize(
size_t size )
1384 bectl( __kmp_get_thread(), (bget_compact_t) 0, (bget_acquire_t) malloc,
1385 (bget_release_t) free, (bufsize) size );
1389 kmpc_get_poolsize(
void )
1393 p = get_thr_data( __kmp_get_thread() );
1399 kmpc_set_poolmode(
int mode )
1403 if (mode == bget_mode_fifo || mode == bget_mode_lifo || mode == bget_mode_best) {
1404 p = get_thr_data( __kmp_get_thread() );
1405 p->mode = (bget_mode_t) mode;
1410 kmpc_get_poolmode(
void )
1414 p = get_thr_data( __kmp_get_thread() );
1420 kmpc_get_poolstat(
size_t *maxmem,
size_t *allmem )
1422 kmp_info_t *th = __kmp_get_thread();
1425 __kmp_bget_dequeue( th );
1427 bcheck( th, &a, &b );
1434 kmpc_poolprint(
void )
1436 kmp_info_t *th = __kmp_get_thread();
1438 __kmp_bget_dequeue( th );
1443 #endif // #if KMP_USE_BGET
1448 kmpc_malloc(
size_t size )
1451 ptr = bget( __kmp_entry_thread(), (bufsize) size );
1457 kmpc_calloc(
size_t nelem,
size_t elsize )
1460 ptr = bgetz( __kmp_entry_thread(), (bufsize) (nelem * elsize) );
1466 kmpc_realloc(
void * ptr,
size_t size )
1468 void * result = NULL;
1470 if ( ptr == NULL ) {
1472 result = bget( __kmp_entry_thread(), (bufsize) size );
1473 }
else if ( size == 0 ) {
1477 brel( __kmp_get_thread(), ptr );
1479 result = bgetr( __kmp_entry_thread(), ptr, (bufsize) size );
1488 kmpc_free(
void * ptr )
1490 if ( ! __kmp_init_serial ) {
1493 if ( ptr != NULL ) {
1494 kmp_info_t *th = __kmp_get_thread();
1495 __kmp_bget_dequeue( th );
1504 ___kmp_thread_malloc( kmp_info_t *th,
size_t size KMP_SRC_LOC_DECL )
1508 "-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n",
1513 ptr = bget( th, (bufsize) size );
1514 KE_TRACE( 30, (
"<- __kmp_thread_malloc() returns %p\n", ptr ) );
1519 ___kmp_thread_calloc( kmp_info_t *th,
size_t nelem,
size_t elsize KMP_SRC_LOC_DECL )
1523 "-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n",
1529 ptr = bgetz( th, (bufsize) (nelem * elsize) );
1530 KE_TRACE( 30, (
"<- __kmp_thread_calloc() returns %p\n", ptr ) );
1535 ___kmp_thread_realloc( kmp_info_t *th,
void *ptr,
size_t size KMP_SRC_LOC_DECL )
1538 "-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n",
1544 ptr = bgetr( th, ptr, (bufsize) size );
1545 KE_TRACE( 30, (
"<- __kmp_thread_realloc() returns %p\n", ptr ) );
1550 ___kmp_thread_free( kmp_info_t *th,
void *ptr KMP_SRC_LOC_DECL )
1553 "-> __kmp_thread_free( %p, %p ) called from %s:%d\n",
1558 if ( ptr != NULL ) {
1559 __kmp_bget_dequeue( th );
1562 KE_TRACE( 30, (
"<- __kmp_thread_free()\n" ) );
1573 struct kmp_mem_descr {
1574 void * ptr_allocated;
1575 size_t size_allocated;
1577 size_t size_aligned;
1579 typedef struct kmp_mem_descr kmp_mem_descr_t;
1588 ___kmp_allocate_align(
size_t size,
size_t alignment KMP_SRC_LOC_DECL )
1607 kmp_mem_descr_t descr;
1608 kmp_uintptr_t addr_allocated;
1609 kmp_uintptr_t addr_aligned;
1610 kmp_uintptr_t addr_descr;
1613 "-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n",
1619 KMP_DEBUG_ASSERT( alignment < 32 * 1024 );
1620 KMP_DEBUG_ASSERT(
sizeof(
void * ) <=
sizeof( kmp_uintptr_t ) );
1623 descr.size_aligned = size;
1624 descr.size_allocated = descr.size_aligned +
sizeof( kmp_mem_descr_t ) + alignment;
1627 descr.ptr_allocated = _malloc_src_loc( descr.size_allocated, _file_, _line_ );
1629 descr.ptr_allocated = malloc_src_loc( descr.size_allocated KMP_SRC_LOC_PARM );
1632 " malloc( %d ) returned %p\n",
1633 (
int) descr.size_allocated,
1636 if ( descr.ptr_allocated == NULL ) {
1637 KMP_FATAL( OutOfHeapMemory );
1640 addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
1642 ( addr_allocated +
sizeof( kmp_mem_descr_t ) + alignment )
1643 & ~ ( alignment - 1 );
1644 addr_descr = addr_aligned -
sizeof( kmp_mem_descr_t );
1646 descr.ptr_aligned = (
void *) addr_aligned;
1649 " ___kmp_allocate_align: "
1650 "ptr_allocated=%p, size_allocated=%d, "
1651 "ptr_aligned=%p, size_aligned=%d\n",
1652 descr.ptr_allocated,
1653 (
int) descr.size_allocated,
1655 (
int) descr.size_aligned
1658 KMP_DEBUG_ASSERT( addr_allocated <= addr_descr );
1659 KMP_DEBUG_ASSERT( addr_descr +
sizeof( kmp_mem_descr_t ) == addr_aligned );
1660 KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
1661 KMP_DEBUG_ASSERT( addr_aligned % alignment == 0 );
1664 memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
1667 memset( descr.ptr_aligned, 0x00, descr.size_aligned );
1671 * ( (kmp_mem_descr_t *) addr_descr ) = descr;
1675 KE_TRACE( 25, (
"<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned ) );
1676 return descr.ptr_aligned;
1688 ___kmp_allocate(
size_t size KMP_SRC_LOC_DECL )
1692 KE_TRACE( 25, (
"-> __kmp_allocate( %d ) called from %s:%d\n", (
int) size KMP_SRC_LOC_PARM ) );
1693 ptr = ___kmp_allocate_align( size, __kmp_align_alloc KMP_SRC_LOC_PARM );
1694 KE_TRACE( 25, (
"<- __kmp_allocate() returns %p\n", ptr ) );
1699 #if (BUILD_MEMORY==FIRST_TOUCH)
1701 __kmp_ft_page_allocate(
size_t size)
1706 int page_size = getpagesize();
1709 int page_size = PAGE_SIZE;
1712 adr = (
void *) __kmp_thread_malloc( __kmp_get_thread(),
1713 size + page_size + KMP_PTR_SKIP);
1715 KMP_FATAL( OutOfHeapMemory );
1718 if ( ( (kmp_uintptr_t) adr & (page_size - 1)) == 0)
1723 aadr = (
void *) ( ( (kmp_uintptr_t) adr + page_size) & ~(page_size - 1) );
1726 *((
void**)aadr) = adr;
1729 return (
void*)((
char*)aadr + KMP_PTR_SKIP);
1740 ___kmp_page_allocate(
size_t size KMP_SRC_LOC_DECL )
1742 int page_size = 8 * 1024;
1746 "-> __kmp_page_allocate( %d ) called from %s:%d\n",
1750 ptr = ___kmp_allocate_align( size, page_size KMP_SRC_LOC_PARM );
1751 KE_TRACE( 25, (
"<- __kmp_page_allocate( %d ) returns %p\n", (
int) size, ptr ) );
1760 ___kmp_free(
void * ptr KMP_SRC_LOC_DECL )
1763 kmp_mem_descr_t descr;
1764 kmp_uintptr_t addr_allocated;
1765 kmp_uintptr_t addr_aligned;
1767 KE_TRACE( 25, (
"-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM ) );
1768 KMP_ASSERT( ptr != NULL );
1770 descr = * ( kmp_mem_descr_t *) ( (kmp_uintptr_t) ptr -
sizeof( kmp_mem_descr_t ) );
1772 KE_TRACE( 26, (
" __kmp_free: "
1773 "ptr_allocated=%p, size_allocated=%d, "
1774 "ptr_aligned=%p, size_aligned=%d\n",
1775 descr.ptr_allocated, (
int) descr.size_allocated,
1776 descr.ptr_aligned, (
int) descr.size_aligned ));
1778 addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
1779 addr_aligned = (kmp_uintptr_t) descr.ptr_aligned;
1781 KMP_DEBUG_ASSERT( addr_aligned % CACHE_LINE == 0 );
1782 KMP_DEBUG_ASSERT( descr.ptr_aligned == ptr );
1783 KMP_DEBUG_ASSERT( addr_allocated +
sizeof( kmp_mem_descr_t ) <= addr_aligned );
1784 KMP_DEBUG_ASSERT( descr.size_aligned < descr.size_allocated );
1785 KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
1788 memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
1793 KE_TRACE( 10, (
" free( %p )\n", descr.ptr_allocated ) );
1795 _free_src_loc( descr.ptr_allocated, _file_, _line_ );
1797 free_src_loc( descr.ptr_allocated KMP_SRC_LOC_PARM );
1803 KE_TRACE( 25, (
"<- __kmp_free() returns\n" ) );
1810 #if USE_FAST_MEMORY == 3
1816 #define KMP_FREE_LIST_LIMIT 16
1819 #define DCACHE_LINE 128
1822 ___kmp_fast_allocate( kmp_info_t *this_thr,
size_t size KMP_SRC_LOC_DECL )
1830 kmp_mem_descr_t * descr;
1832 KE_TRACE( 25, (
"-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n",
1833 __kmp_gtid_from_thread(this_thr), (
int) size KMP_SRC_LOC_PARM ) );
1835 num_lines = ( size + DCACHE_LINE - 1 ) / DCACHE_LINE;
1836 idx = num_lines - 1;
1837 KMP_DEBUG_ASSERT( idx >= 0 );
1841 }
else if ( ( idx >>= 2 ) == 0 ) {
1844 }
else if ( ( idx >>= 2 ) == 0 ) {
1847 }
else if ( ( idx >>= 2 ) == 0 ) {
1854 ptr = this_thr->th.th_free_lists[index].th_free_list_self;
1855 if ( ptr != NULL ) {
1857 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1858 KMP_DEBUG_ASSERT( this_thr ==
1859 ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t) ))->ptr_aligned );
1862 ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
1863 if ( ptr != NULL ) {
1866 while ( ! KMP_COMPARE_AND_STORE_PTR(
1867 &this_thr->th.th_free_lists[index].th_free_list_sync, ptr, NULL ) )
1870 ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
1873 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1874 KMP_DEBUG_ASSERT( this_thr ==
1875 ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t) ))->ptr_aligned );
1881 size = num_lines * DCACHE_LINE;
1883 alloc_size = size +
sizeof( kmp_mem_descr_t ) + DCACHE_LINE;
1884 KE_TRACE( 25, (
"__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with alloc_size %d\n",
1885 __kmp_gtid_from_thread( this_thr ), alloc_size ) );
1886 alloc_ptr = bget( this_thr, (bufsize) alloc_size );
1889 ptr = (
void *)(( ((kmp_uintptr_t)alloc_ptr) +
sizeof(kmp_mem_descr_t) + DCACHE_LINE ) & ~( DCACHE_LINE - 1 ));
1890 descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t) );
1892 descr->ptr_allocated = alloc_ptr;
1894 descr->ptr_aligned = (
void *)this_thr;
1897 descr->size_aligned = size;
1900 KE_TRACE( 25, (
"<- __kmp_fast_allocate( T#%d ) returns %p\n",
1901 __kmp_gtid_from_thread( this_thr ), ptr ) );
1908 ___kmp_fast_free( kmp_info_t *this_thr,
void * ptr KMP_SRC_LOC_DECL )
1910 kmp_mem_descr_t * descr;
1911 kmp_info_t * alloc_thr;
1916 KE_TRACE( 25, (
"-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n",
1917 __kmp_gtid_from_thread(this_thr), ptr KMP_SRC_LOC_PARM ) );
1918 KMP_ASSERT( ptr != NULL );
1920 descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t) );
1922 KE_TRACE(26, (
" __kmp_fast_free: size_aligned=%d\n",
1923 (
int) descr->size_aligned ) );
1925 size = descr->size_aligned;
1927 idx = DCACHE_LINE * 2;
1928 if ( idx == size ) {
1930 }
else if ( ( idx <<= 1 ) == size ) {
1932 }
else if ( ( idx <<= 2 ) == size ) {
1934 }
else if ( ( idx <<= 2 ) == size ) {
1937 KMP_DEBUG_ASSERT( size > DCACHE_LINE * 64 );
1941 alloc_thr = (kmp_info_t *)descr->ptr_aligned;
1942 if ( alloc_thr == this_thr ) {
1944 *((
void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self;
1945 this_thr->th.th_free_lists[index].th_free_list_self = ptr;
1947 void * head = this_thr->th.th_free_lists[index].th_free_list_other;
1948 if ( head == NULL ) {
1950 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1951 *((
void **)ptr) = NULL;
1952 descr->size_allocated = (size_t)1;
1955 kmp_mem_descr_t * dsc = (kmp_mem_descr_t *)( (
char*)head -
sizeof(kmp_mem_descr_t) );
1956 kmp_info_t * q_th = (kmp_info_t *)(dsc->ptr_aligned);
1957 size_t q_sz = dsc->size_allocated + 1;
1958 if ( q_th == alloc_thr && q_sz <= KMP_FREE_LIST_LIMIT ) {
1960 *((
void **)ptr) = head;
1961 descr->size_allocated = q_sz;
1962 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1969 void * next = *((
void **)head);
1970 while ( next != NULL ) {
1973 ((kmp_mem_descr_t*)((
char*)next -
sizeof(kmp_mem_descr_t)))->size_allocated + 1 ==
1974 ((kmp_mem_descr_t*)((
char*)tail -
sizeof(kmp_mem_descr_t)))->size_allocated );
1976 next = *((
void **)next);
1978 KMP_DEBUG_ASSERT( q_th != NULL );
1980 old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
1983 *((
void **)tail) = old_ptr;
1985 while ( ! KMP_COMPARE_AND_STORE_PTR(
1986 &q_th->th.th_free_lists[index].th_free_list_sync,
1991 old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
1992 *((
void **)tail) = old_ptr;
1996 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1997 *((
void **)ptr) = NULL;
1998 descr->size_allocated = (size_t)1;
2005 KE_TRACE(25, (
"__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n",
2006 __kmp_gtid_from_thread( this_thr), size ) );
2007 __kmp_bget_dequeue( this_thr );
2008 brel( this_thr, descr->ptr_allocated );
2011 KE_TRACE( 25, (
"<- __kmp_fast_free() returns\n" ) );
2019 __kmp_initialize_fast_memory( kmp_info_t *this_thr )
2021 KE_TRACE(10, (
"__kmp_initialize_fast_memory: Called from th %p\n", this_thr ) );
2023 memset ( this_thr->th.th_free_lists, 0, NUM_LISTS *
sizeof( kmp_free_list_t ) );
2029 __kmp_free_fast_memory( kmp_info_t *th )
2033 thr_data_t * thr = get_thr_data( th );
2036 KE_TRACE(5, (
"__kmp_free_fast_memory: Called T#%d\n",
2037 __kmp_gtid_from_thread( th ) ) );
2039 __kmp_bget_dequeue( th );
2042 for ( bin = 0; bin < MAX_BGET_BINS; ++bin ) {
2043 bfhead_t * b = thr->freelist[ bin ].ql.flink;
2044 while ( b != &thr->freelist[ bin ] ) {
2045 if ( (kmp_uintptr_t)b->bh.bb.bthr & 1 ) {
2052 while ( lst != NULL ) {
2054 KE_TRACE(10, (
"__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n",
2055 lst, next, th, __kmp_gtid_from_thread( th ) ) );
2056 (*thr->relfcn)(lst);
2065 KE_TRACE(5, (
"__kmp_free_fast_memory: Freed T#%d\n",
2066 __kmp_gtid_from_thread( th ) ) );
2069 #endif // USE_FAST_MEMORY