36 #include "kmp_wrapper_malloc.h"
44 typedef int (*bget_compact_t)(size_t, int);
45 typedef void *(*bget_acquire_t)(size_t);
46 typedef void (*bget_release_t)(
void *);
51 # if KMP_ARCH_X86 || KMP_ARCH_ARM
52 typedef kmp_int32 bufsize;
54 typedef kmp_int64 bufsize;
57 typedef ssize_t bufsize;
62 typedef enum bget_mode {
69 static void bpool( kmp_info_t *th,
void *buffer, bufsize len);
70 static void *bget( kmp_info_t *th, bufsize size);
71 static void *bgetz( kmp_info_t *th, bufsize size);
72 static void *bgetr( kmp_info_t *th,
void *buffer, bufsize newsize);
73 static void brel( kmp_info_t *th,
void *buf);
74 static void bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr );
77 static void bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
long *nget,
long *nrel);
78 static void bstatse( kmp_info_t *th, bufsize *pool_incr,
long *npool,
long *npget,
long *nprel,
long *ndget,
long *ndrel);
79 static void bufdump( kmp_info_t *th,
void *buf);
80 static void bpoold( kmp_info_t *th,
void *pool,
int dumpalloc,
int dumpfree);
81 static int bpoolv( kmp_info_t *th,
void *pool);
94 #if KMP_ARCH_X86 || !KMP_HAVE_QUAD
97 #define AlignType double
102 #define AlignType _Quad
143 static bufsize bget_bin_size[ ] = {
167 #define MAX_BGET_BINS (int)(sizeof(bget_bin_size) / sizeof(bufsize))
176 typedef struct qlinks {
177 struct bfhead *flink;
178 struct bfhead *blink;
183 typedef struct bhead2 {
194 typedef union bhead {
195 KMP_ALIGN( SizeQuant )
197 char b_pad[ sizeof(bhead2_t) + (SizeQuant - (sizeof(bhead2_t) % SizeQuant)) ];
200 #define BH(p) ((bhead_t *) (p))
204 typedef struct bdhead
209 #define BDH(p) ((bdhead_t *) (p))
213 typedef struct bfhead {
217 #define BFH(p) ((bfhead_t *) (p))
219 typedef struct thr_data {
220 bfhead_t freelist[ MAX_BGET_BINS ];
225 long numpget, numprel;
226 long numdget, numdrel;
230 bget_compact_t compfcn;
231 bget_acquire_t acqfcn;
232 bget_release_t relfcn;
243 bfhead_t * last_pool;
248 #define QLSize (sizeof(qlinks_t))
249 #define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize)
250 #define MaxSize (bufsize)( ~ ( ( (bufsize)( 1 ) << ( sizeof( bufsize ) * CHAR_BIT - 1 ) ) | ( SizeQuant - 1 ) ) )
257 #define ESent ((bufsize) (-(((((bufsize)1)<<((int)sizeof(bufsize)*8-2))-1)*2)-2))
264 bget_get_bin( bufsize size )
267 int lo = 0, hi = MAX_BGET_BINS - 1;
269 KMP_DEBUG_ASSERT( size > 0 );
271 while ( (hi - lo) > 1 ) {
272 int mid = (lo + hi) >> 1;
273 if (size < bget_bin_size[ mid ])
279 KMP_DEBUG_ASSERT( (lo >= 0) && (lo < MAX_BGET_BINS) );
285 set_thr_data( kmp_info_t *th )
292 ( ! th->th.th_local.bget_data ) ? __kmp_allocate(
sizeof( *data ) ) : th->th.th_local.bget_data
295 memset( data,
'\0',
sizeof( *data ) );
297 for (i = 0; i < MAX_BGET_BINS; ++i) {
298 data->freelist[ i ].ql.flink = & data->freelist[ i ];
299 data->freelist[ i ].ql.blink = & data->freelist[ i ];
302 th->th.th_local.bget_data = data;
303 th->th.th_local.bget_list = 0;
304 #if ! USE_CMP_XCHG_FOR_BGET
305 #ifdef USE_QUEUING_LOCK_FOR_BGET
306 __kmp_init_lock( & th->th.th_local.bget_lock );
308 __kmp_init_bootstrap_lock( & th->th.th_local.bget_lock );
314 get_thr_data( kmp_info_t *th )
318 data = (thr_data_t *) th->th.th_local.bget_data;
320 KMP_DEBUG_ASSERT( data != 0 );
329 __kmp_bget_validate_queue( kmp_info_t *th )
333 void *p = (
void *) th->th.th_local.bget_list;
336 bfhead_t *b = BFH(((
char *) p) -
sizeof(bhead_t));
338 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
339 p = (
void *) b->ql.flink;
348 __kmp_bget_dequeue( kmp_info_t *th )
350 void *p = TCR_SYNC_PTR(th->th.th_local.bget_list);
353 #if USE_CMP_XCHG_FOR_BGET
355 volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
356 while ( ! KMP_COMPARE_AND_STORE_PTR(
357 & th->th.th_local.bget_list, old_value, NULL ) )
360 old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
362 p = (
void *) old_value;
365 #ifdef USE_QUEUING_LOCK_FOR_BGET
366 __kmp_acquire_lock( & th->th.th_local.bget_lock,
367 __kmp_gtid_from_thread(th) );
369 __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
372 p = (
void *) th->th.th_local.bget_list;
373 th->th.th_local.bget_list = 0;
375 #ifdef USE_QUEUING_LOCK_FOR_BGET
376 __kmp_release_lock( & th->th.th_local.bget_lock,
377 __kmp_gtid_from_thread(th) );
379 __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
387 bfhead_t *b = BFH(((
char *) p) -
sizeof(bhead_t));
389 KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
390 KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
392 KMP_DEBUG_ASSERT( b->ql.blink == 0 );
394 p = (
void *) b->ql.flink;
404 __kmp_bget_enqueue( kmp_info_t *th,
void *buf
405 #ifdef USE_QUEUING_LOCK_FOR_BGET
410 bfhead_t *b = BFH(((
char *) buf) -
sizeof(bhead_t));
412 KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
413 KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
418 KC_TRACE( 10, (
"__kmp_bget_enqueue: moving buffer to T#%d list\n",
419 __kmp_gtid_from_thread( th ) ) );
421 #if USE_CMP_XCHG_FOR_BGET
423 volatile void *old_value = TCR_PTR(th->th.th_local.bget_list);
426 b->ql.flink = BFH( old_value );
428 while ( ! KMP_COMPARE_AND_STORE_PTR(
429 & th->th.th_local.bget_list, old_value, buf ) )
432 old_value = TCR_PTR(th->th.th_local.bget_list);
435 b->ql.flink = BFH( old_value );
439 # ifdef USE_QUEUING_LOCK_FOR_BGET
440 __kmp_acquire_lock( & th->th.th_local.bget_lock, rel_gtid );
442 __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
445 b->ql.flink = BFH( th->th.th_local.bget_list );
446 th->th.th_local.bget_list = (
void *) buf;
448 # ifdef USE_QUEUING_LOCK_FOR_BGET
449 __kmp_release_lock( & th->th.th_local.bget_lock, rel_gtid );
451 __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
459 __kmp_bget_insert_into_freelist( thr_data_t *thr, bfhead_t *b )
463 KMP_DEBUG_ASSERT( ((
size_t)b ) % SizeQuant == 0 );
464 KMP_DEBUG_ASSERT( b->bh.bb.bsize % SizeQuant == 0 );
466 bin = bget_get_bin( b->bh.bb.bsize );
468 KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.blink->ql.flink == &thr->freelist[ bin ]);
469 KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.flink->ql.blink == &thr->freelist[ bin ]);
471 b->ql.flink = &thr->freelist[ bin ];
472 b->ql.blink = thr->freelist[ bin ].ql.blink;
474 thr->freelist[ bin ].ql.blink = b;
475 b->ql.blink->ql.flink = b;
481 __kmp_bget_remove_from_freelist( bfhead_t *b )
483 KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b);
484 KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b);
486 b->ql.blink->ql.flink = b->ql.flink;
487 b->ql.flink->ql.blink = b->ql.blink;
495 bcheck( kmp_info_t *th, bufsize *max_free, bufsize *total_free )
497 thr_data_t *thr = get_thr_data( th );
500 *total_free = *max_free = 0;
502 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
505 best = &thr->freelist[ bin ];
508 while (b != &thr->freelist[ bin ]) {
509 *total_free += (b->bh.bb.bsize -
sizeof( bhead_t ));
510 if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize))
517 if (*max_free < best->bh.bb.bsize)
518 *max_free = best->bh.bb.bsize;
521 if (*max_free > (bufsize)
sizeof( bhead_t ))
522 *max_free -=
sizeof( bhead_t );
530 bget( kmp_info_t *th, bufsize requested_size )
532 thr_data_t *thr = get_thr_data( th );
533 bufsize size = requested_size;
541 if ( size < 0 || size +
sizeof( bhead_t ) > MaxSize ) {
545 __kmp_bget_dequeue( th );
547 if (size < (bufsize)SizeQ) {
550 #if defined( SizeQuant ) && ( SizeQuant > 1 )
551 size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1));
554 size +=
sizeof(bhead_t);
556 KMP_DEBUG_ASSERT( size >= 0 );
557 KMP_DEBUG_ASSERT( size % SizeQuant == 0 );
559 use_blink = ( thr->mode == bget_mode_lifo );
568 for (bin = bget_get_bin( size ); bin < MAX_BGET_BINS; ++bin) {
570 b = ( use_blink ? thr->freelist[ bin ].ql.blink : thr->freelist[ bin ].ql.flink );
572 if (thr->mode == bget_mode_best) {
573 best = &thr->freelist[ bin ];
578 while (b != &thr->freelist[ bin ]) {
579 if (b->bh.bb.bsize >= (bufsize) size) {
580 if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize)) {
586 b = ( use_blink ? b->ql.blink : b->ql.flink );
591 while (b != &thr->freelist[ bin ]) {
592 if ((bufsize) b->bh.bb.bsize >= (bufsize) size) {
603 if ((b->bh.bb.bsize - (bufsize) size) > (bufsize)(SizeQ + (
sizeof(bhead_t)))) {
606 ba = BH(((
char *) b) + (b->bh.bb.bsize - (bufsize) size));
607 bn = BH(((
char *) ba) + size);
609 KMP_DEBUG_ASSERT(bn->bb.prevfree == b->bh.bb.bsize);
612 b->bh.bb.bsize -= (bufsize) size;
615 ba->bb.prevfree = b->bh.bb.bsize;
618 ba->bb.bsize = -size;
621 TCW_PTR(ba->bb.bthr, th);
626 __kmp_bget_remove_from_freelist( b );
627 __kmp_bget_insert_into_freelist( thr, b );
629 thr->totalloc += (size_t) size;
632 buf = (
void *) ((((
char *) ba) +
sizeof(bhead_t)));
633 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
638 ba = BH(((
char *) b) + b->bh.bb.bsize);
640 KMP_DEBUG_ASSERT(ba->bb.prevfree == b->bh.bb.bsize);
645 __kmp_bget_remove_from_freelist( b );
647 thr->totalloc += (size_t) b->bh.bb.bsize;
651 b->bh.bb.bsize = -(b->bh.bb.bsize);
654 TCW_PTR(ba->bb.bthr, th);
660 buf = (
void *) &(b->ql);
661 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
667 b = ( use_blink ? b->ql.blink : b->ql.flink );
675 if ((thr->compfcn == 0) || (!(*thr->compfcn)(size, ++compactseq))) {
684 if (thr->acqfcn != 0) {
685 if (size > (bufsize) (thr->exp_incr -
sizeof(bhead_t))) {
692 size +=
sizeof(bdhead_t) -
sizeof(bhead_t);
694 KE_TRACE( 10, (
"%%%%%% MALLOC( %d )\n", (
int) size ) );
697 bdh = BDH((*thr->acqfcn)((bufsize) size));
702 bdh->bh.bb.bsize = 0;
705 TCW_PTR(bdh->bh.bb.bthr, th);
707 bdh->bh.bb.prevfree = 0;
710 thr->totalloc += (size_t) size;
714 buf = (
void *) (bdh + 1);
715 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
725 KE_TRACE( 10, (
"%%%%%% MALLOCB( %d )\n", (
int) thr->exp_incr ) );
728 newpool = (*thr->acqfcn)((bufsize) thr->exp_incr);
729 KMP_DEBUG_ASSERT( ((
size_t)newpool) % SizeQuant == 0 );
730 if (newpool != NULL) {
731 bpool( th, newpool, thr->exp_incr);
732 buf = bget( th, requested_size);
748 bgetz( kmp_info_t *th, bufsize size )
750 char *buf = (
char *) bget( th, size);
756 b = BH(buf -
sizeof(bhead_t));
757 rsize = -(b->bb.bsize);
761 bd = BDH(buf -
sizeof(bdhead_t));
762 rsize = bd->tsize - (bufsize)
sizeof(bdhead_t);
764 rsize -=
sizeof(bhead_t);
767 KMP_DEBUG_ASSERT(rsize >= size);
769 (void) memset(buf, 0, (bufsize) rsize);
771 return ((
void *) buf);
780 bgetr( kmp_info_t *th,
void *buf, bufsize size)
786 nbuf = bget( th, size );
787 if ( nbuf == NULL ) {
793 b = BH(((
char *) buf) -
sizeof(bhead_t));
794 osize = -b->bb.bsize;
799 bd = BDH(((
char *) buf) -
sizeof(bdhead_t));
800 osize = bd->tsize - (bufsize)
sizeof(bdhead_t);
802 osize -=
sizeof(bhead_t);
805 KMP_DEBUG_ASSERT(osize > 0);
807 (void) KMP_MEMCPY((
char *) nbuf, (
char *) buf,
808 (
size_t) ((size < osize) ? size : osize));
817 brel( kmp_info_t *th,
void *buf )
819 thr_data_t *thr = get_thr_data( th );
823 KMP_DEBUG_ASSERT(buf != NULL);
824 KMP_DEBUG_ASSERT( ((
size_t)buf) % SizeQuant == 0 );
826 b = BFH(((
char *) buf) -
sizeof(bhead_t));
828 if (b->bh.bb.bsize == 0) {
831 bdh = BDH(((
char *) buf) -
sizeof(bdhead_t));
832 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
834 thr->totalloc -= (size_t) bdh->tsize;
839 (
void) memset((
char *) buf, 0x55,
840 (
size_t) (bdh->tsize -
sizeof(bdhead_t)));
843 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) bdh ) );
845 KMP_DEBUG_ASSERT( thr->relfcn != 0 );
846 (*thr->relfcn)((
void *) bdh);
850 bth = (kmp_info_t *)( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 );
853 __kmp_bget_enqueue( bth, buf
854 #ifdef USE_QUEUING_LOCK_FOR_BGET
855 , __kmp_gtid_from_thread( th )
864 if (b->bh.bb.bsize >= 0) {
867 KMP_DEBUG_ASSERT(b->bh.bb.bsize < 0);
872 KMP_DEBUG_ASSERT(BH((
char *) b - b->bh.bb.bsize)->bb.prevfree == 0);
876 thr->totalloc += (size_t) b->bh.bb.bsize;
881 if (b->bh.bb.prevfree != 0) {
888 register bufsize size = b->bh.bb.bsize;
891 KMP_DEBUG_ASSERT(BH((
char *) b - b->bh.bb.prevfree)->bb.bsize == b->bh.bb.prevfree);
892 b = BFH(((
char *) b) - b->bh.bb.prevfree);
893 b->bh.bb.bsize -= size;
896 __kmp_bget_remove_from_freelist( b );
903 b->bh.bb.bsize = -b->bh.bb.bsize;
907 __kmp_bget_insert_into_freelist( thr, b );
915 bn = BFH(((
char *) b) + b->bh.bb.bsize);
916 if (bn->bh.bb.bsize > 0) {
921 KMP_DEBUG_ASSERT(BH((
char *) bn + bn->bh.bb.bsize)->bb.prevfree == bn->bh.bb.bsize);
923 __kmp_bget_remove_from_freelist( bn );
925 b->bh.bb.bsize += bn->bh.bb.bsize;
929 __kmp_bget_remove_from_freelist( b );
930 __kmp_bget_insert_into_freelist( thr, b );
939 bn = BFH(((
char *) b) + b->bh.bb.bsize);
942 (void) memset(((
char *) b) +
sizeof(bfhead_t), 0x55,
943 (
size_t) (b->bh.bb.bsize -
sizeof(bfhead_t)));
945 KMP_DEBUG_ASSERT(bn->bh.bb.bsize < 0);
950 bn->bh.bb.prevfree = b->bh.bb.bsize;
957 if (thr->relfcn != 0 &&
958 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t)))
961 if (thr->numpblk != 1) {
964 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
965 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
966 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
969 __kmp_bget_remove_from_freelist( b );
971 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) b ) );
977 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
980 if (thr->last_pool == b) thr->last_pool = 0;
992 bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr)
994 thr_data_t *thr = get_thr_data( th );
996 thr->compfcn = compact;
997 thr->acqfcn = acquire;
998 thr->relfcn = release;
999 thr->exp_incr = pool_incr;
1005 bpool( kmp_info_t *th,
void *buf, bufsize len)
1008 thr_data_t *thr = get_thr_data( th );
1009 bfhead_t *b = BFH(buf);
1012 __kmp_bget_dequeue( th );
1015 len &= ~(SizeQuant - 1);
1017 if (thr->pool_len == 0) {
1018 thr->pool_len = len;
1019 }
else if (len != thr->pool_len) {
1025 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
1032 KMP_DEBUG_ASSERT(len -
sizeof(bhead_t) <= -((bufsize) ESent + 1));
1038 b->bh.bb.prevfree = 0;
1048 len -=
sizeof(bhead_t);
1049 b->bh.bb.bsize = (bufsize) len;
1051 TCW_PTR( b->bh.bb.bthr, (kmp_info_t*)((kmp_uintptr_t)th | 1) );
1054 __kmp_bget_insert_into_freelist( thr, b );
1057 (void) memset(((
char *) b) +
sizeof(bfhead_t), 0x55,
1058 (
size_t) (len -
sizeof(bfhead_t)));
1060 bn = BH(((
char *) b) + len);
1061 bn->bb.prevfree = (bufsize) len;
1063 KMP_DEBUG_ASSERT( (~0) == -1 && (bn != 0) );
1065 bn->bb.bsize = ESent;
1073 bfreed( kmp_info_t *th )
1075 int bin = 0, count = 0;
1076 int gtid = __kmp_gtid_from_thread( th );
1077 thr_data_t *thr = get_thr_data( th );
1080 __kmp_printf_no_lock(
"__kmp_printpool: T#%d total=%" KMP_UINT64_SPEC
" get=%" KMP_INT64_SPEC
" rel=%" \
1081 KMP_INT64_SPEC
" pblk=%" KMP_INT64_SPEC
" pget=%" KMP_INT64_SPEC
" prel=%" KMP_INT64_SPEC \
1082 " dget=%" KMP_INT64_SPEC
" drel=%" KMP_INT64_SPEC
"\n",
1083 gtid, (kmp_uint64) thr->totalloc,
1084 (kmp_int64) thr->numget, (kmp_int64) thr->numrel,
1085 (kmp_int64) thr->numpblk,
1086 (kmp_int64) thr->numpget, (kmp_int64) thr->numprel,
1087 (kmp_int64) thr->numdget, (kmp_int64) thr->numdrel );
1090 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1093 for (b = thr->freelist[ bin ].ql.flink; b != &thr->freelist[ bin ]; b = b->ql.flink) {
1094 bufsize bs = b->bh.bb.bsize;
1096 KMP_DEBUG_ASSERT( b->ql.blink->ql.flink == b );
1097 KMP_DEBUG_ASSERT( b->ql.flink->ql.blink == b );
1098 KMP_DEBUG_ASSERT( bs > 0 );
1102 __kmp_printf_no_lock(
"__kmp_printpool: T#%d Free block: 0x%p size %6ld bytes.\n", gtid, b, (
long) bs );
1105 char *lerr = ((
char *) b) +
sizeof(bfhead_t);
1106 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) || (memcmp(lerr, lerr + 1, (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1107 __kmp_printf_no_lock(
"__kmp_printpool: T#%d (Contents of above free block have been overstored.)\n", gtid );
1115 __kmp_printf_no_lock(
"__kmp_printpool: T#%d No free blocks\n", gtid );
1127 bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
long *nget,
long *nrel)
1130 thr_data_t *thr = get_thr_data( th );
1132 *nget = thr->numget;
1133 *nrel = thr->numrel;
1134 *curalloc = (bufsize) thr->totalloc;
1138 for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
1139 bfhead_t *b = thr->freelist[ bin ].ql.flink;
1141 while (b != &thr->freelist[ bin ]) {
1142 KMP_DEBUG_ASSERT(b->bh.bb.bsize > 0);
1143 *totfree += b->bh.bb.bsize;
1144 if (b->bh.bb.bsize > *maxfree) {
1145 *maxfree = b->bh.bb.bsize;
1155 bstatse( kmp_info_t *th, bufsize *pool_incr,
long *npool,
long *npget,
long *nprel,
long *ndget,
long *ndrel)
1157 thr_data_t *thr = get_thr_data( th );
1159 *pool_incr = (thr->pool_len < 0) ? -thr->exp_incr : thr->exp_incr;
1160 *npool = thr->numpblk;
1161 *npget = thr->numpget;
1162 *nprel = thr->numprel;
1163 *ndget = thr->numdget;
1164 *ndrel = thr->numdrel;
1174 bufdump( kmp_info_t *th,
void *buf )
1177 unsigned char *bdump;
1180 b = BFH(((
char *) buf) -
sizeof(bhead_t));
1181 KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
1182 if (b->bh.bb.bsize < 0) {
1183 bdump = (
unsigned char *) buf;
1184 bdlen = (-b->bh.bb.bsize) - (bufsize)
sizeof(bhead_t);
1186 bdump = (
unsigned char *) (((
char *) b) +
sizeof(bfhead_t));
1187 bdlen = b->bh.bb.bsize - (bufsize)
sizeof(bfhead_t);
1193 char bhex[50], bascii[20];
1199 for (i = 0; i < l; i++) {
1200 (void) KMP_SNPRINTF(bhex + i * 3,
sizeof(bhex),
"%02X ", bdump[i]);
1201 if (bdump[i] > 0x20 && bdump[i] < 0x7F)
1202 bascii[ i ] = bdump[ i ];
1207 (void) __kmp_printf_no_lock(
"%-48s %s\n", bhex, bascii);
1210 while ((bdlen > 16) && (memcmp((
char *) (bdump - 16),
1211 (
char *) bdump, 16) == 0)) {
1217 (void) __kmp_printf_no_lock(
1218 " (%d lines [%d bytes] identical to above line skipped)\n",
1220 }
else if (dupes == 1) {
1234 bpoold( kmp_info_t *th,
void *buf,
int dumpalloc,
int dumpfree)
1236 bfhead_t *b = BFH( (
char*)buf -
sizeof(bhead_t));
1238 while (b->bh.bb.bsize != ESent) {
1239 bufsize bs = b->bh.bb.bsize;
1243 (void) __kmp_printf_no_lock(
"Allocated buffer: size %6ld bytes.\n", (
long) bs);
1245 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1248 const char *lerr =
"";
1250 KMP_DEBUG_ASSERT(bs > 0);
1251 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1252 lerr =
" (Bad free list links)";
1254 (void) __kmp_printf_no_lock(
"Free block: size %6ld bytes.%s\n",
1257 lerr = ((
char *) b) +
sizeof(bfhead_t);
1258 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) ||
1259 (memcmp(lerr, lerr + 1,
1260 (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1261 (void) __kmp_printf_no_lock(
1262 "(Contents of above free block have been overstored.)\n");
1263 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1267 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1270 b = BFH(((
char *) b) + bs);
1277 bpoolv( kmp_info_t *th,
void *buf )
1279 bfhead_t *b = BFH(buf);
1281 while (b->bh.bb.bsize != ESent) {
1282 bufsize bs = b->bh.bb.bsize;
1291 KMP_DEBUG_ASSERT(bs > 0);
1295 if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
1296 (void) __kmp_printf_no_lock(
"Free block: size %6ld bytes. (Bad free list links)\n",
1298 KMP_DEBUG_ASSERT(0);
1302 lerr = ((
char *) b) +
sizeof(bfhead_t);
1303 if ((bs >
sizeof(bfhead_t)) && ((*lerr != 0x55) ||
1304 (memcmp(lerr, lerr + 1,
1305 (
size_t) (bs - (
sizeof(bfhead_t) + 1))) != 0))) {
1306 (void) __kmp_printf_no_lock(
1307 "(Contents of above free block have been overstored.)\n");
1308 bufdump( th, (
void *) (((
char *) b) +
sizeof(bhead_t)));
1309 KMP_DEBUG_ASSERT(0);
1314 b = BFH(((
char *) b) + bs);
1324 __kmp_initialize_bget( kmp_info_t *th )
1326 KMP_DEBUG_ASSERT( SizeQuant >=
sizeof(
void * ) && (th != 0) );
1330 bectl( th, (bget_compact_t) 0, (bget_acquire_t) malloc, (bget_release_t) free,
1331 (bufsize) __kmp_malloc_pool_incr );
1335 __kmp_finalize_bget( kmp_info_t *th )
1340 KMP_DEBUG_ASSERT( th != 0 );
1343 thr = (thr_data_t *) th->th.th_local.bget_data;
1344 KMP_DEBUG_ASSERT( thr != NULL );
1353 if (thr->relfcn != 0 && b != 0 && thr->numpblk != 0 &&
1354 b->bh.bb.bsize == (bufsize)(thr->pool_len -
sizeof(bhead_t)))
1356 KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
1357 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
1358 KMP_DEBUG_ASSERT(BH((
char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
1361 __kmp_bget_remove_from_freelist( b );
1363 KE_TRACE( 10, (
"%%%%%% FREE( %p )\n", (
void *) b ) );
1368 KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
1373 if ( th->th.th_local.bget_data != NULL ) {
1374 __kmp_free( th->th.th_local.bget_data );
1375 th->th.th_local.bget_data = NULL;
1380 kmpc_set_poolsize(
size_t size )
1382 bectl( __kmp_get_thread(), (bget_compact_t) 0, (bget_acquire_t) malloc,
1383 (bget_release_t) free, (bufsize) size );
1387 kmpc_get_poolsize(
void )
1391 p = get_thr_data( __kmp_get_thread() );
1397 kmpc_set_poolmode(
int mode )
1401 if (mode == bget_mode_fifo || mode == bget_mode_lifo || mode == bget_mode_best) {
1402 p = get_thr_data( __kmp_get_thread() );
1403 p->mode = (bget_mode_t) mode;
1408 kmpc_get_poolmode(
void )
1412 p = get_thr_data( __kmp_get_thread() );
1418 kmpc_get_poolstat(
size_t *maxmem,
size_t *allmem )
1420 kmp_info_t *th = __kmp_get_thread();
1423 __kmp_bget_dequeue( th );
1425 bcheck( th, &a, &b );
1432 kmpc_poolprint(
void )
1434 kmp_info_t *th = __kmp_get_thread();
1436 __kmp_bget_dequeue( th );
1441 #endif // #if KMP_USE_BGET
1446 kmpc_malloc(
size_t size )
1449 ptr = bget( __kmp_entry_thread(), (bufsize) size );
1455 kmpc_calloc(
size_t nelem,
size_t elsize )
1458 ptr = bgetz( __kmp_entry_thread(), (bufsize) (nelem * elsize) );
1464 kmpc_realloc(
void * ptr,
size_t size )
1466 void * result = NULL;
1468 if ( ptr == NULL ) {
1470 result = bget( __kmp_entry_thread(), (bufsize) size );
1471 }
else if ( size == 0 ) {
1475 brel( __kmp_get_thread(), ptr );
1477 result = bgetr( __kmp_entry_thread(), ptr, (bufsize) size );
1486 kmpc_free(
void * ptr )
1488 if ( ! __kmp_init_serial ) {
1491 if ( ptr != NULL ) {
1492 kmp_info_t *th = __kmp_get_thread();
1493 __kmp_bget_dequeue( th );
1502 ___kmp_thread_malloc( kmp_info_t *th,
size_t size KMP_SRC_LOC_DECL )
1506 "-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n",
1511 ptr = bget( th, (bufsize) size );
1512 KE_TRACE( 30, (
"<- __kmp_thread_malloc() returns %p\n", ptr ) );
1517 ___kmp_thread_calloc( kmp_info_t *th,
size_t nelem,
size_t elsize KMP_SRC_LOC_DECL )
1521 "-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n",
1527 ptr = bgetz( th, (bufsize) (nelem * elsize) );
1528 KE_TRACE( 30, (
"<- __kmp_thread_calloc() returns %p\n", ptr ) );
1533 ___kmp_thread_realloc( kmp_info_t *th,
void *ptr,
size_t size KMP_SRC_LOC_DECL )
1536 "-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n",
1542 ptr = bgetr( th, ptr, (bufsize) size );
1543 KE_TRACE( 30, (
"<- __kmp_thread_realloc() returns %p\n", ptr ) );
1548 ___kmp_thread_free( kmp_info_t *th,
void *ptr KMP_SRC_LOC_DECL )
1551 "-> __kmp_thread_free( %p, %p ) called from %s:%d\n",
1556 if ( ptr != NULL ) {
1557 __kmp_bget_dequeue( th );
1560 KE_TRACE( 30, (
"<- __kmp_thread_free()\n" ) );
1571 struct kmp_mem_descr {
1572 void * ptr_allocated;
1573 size_t size_allocated;
1575 size_t size_aligned;
1577 typedef struct kmp_mem_descr kmp_mem_descr_t;
1586 ___kmp_allocate_align(
size_t size,
size_t alignment KMP_SRC_LOC_DECL )
1605 kmp_mem_descr_t descr;
1606 kmp_uintptr_t addr_allocated;
1607 kmp_uintptr_t addr_aligned;
1608 kmp_uintptr_t addr_descr;
1611 "-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n",
1617 KMP_DEBUG_ASSERT( alignment < 32 * 1024 );
1618 KMP_DEBUG_ASSERT(
sizeof(
void * ) <=
sizeof( kmp_uintptr_t ) );
1621 descr.size_aligned = size;
1622 descr.size_allocated = descr.size_aligned +
sizeof( kmp_mem_descr_t ) + alignment;
1625 descr.ptr_allocated = _malloc_src_loc( descr.size_allocated, _file_, _line_ );
1627 descr.ptr_allocated = malloc_src_loc( descr.size_allocated KMP_SRC_LOC_PARM );
1630 " malloc( %d ) returned %p\n",
1631 (
int) descr.size_allocated,
1634 if ( descr.ptr_allocated == NULL ) {
1635 KMP_FATAL( OutOfHeapMemory );
1638 addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
1640 ( addr_allocated +
sizeof( kmp_mem_descr_t ) + alignment )
1641 & ~ ( alignment - 1 );
1642 addr_descr = addr_aligned -
sizeof( kmp_mem_descr_t );
1644 descr.ptr_aligned = (
void *) addr_aligned;
1647 " ___kmp_allocate_align: "
1648 "ptr_allocated=%p, size_allocated=%d, "
1649 "ptr_aligned=%p, size_aligned=%d\n",
1650 descr.ptr_allocated,
1651 (
int) descr.size_allocated,
1653 (
int) descr.size_aligned
1656 KMP_DEBUG_ASSERT( addr_allocated <= addr_descr );
1657 KMP_DEBUG_ASSERT( addr_descr +
sizeof( kmp_mem_descr_t ) == addr_aligned );
1658 KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
1659 KMP_DEBUG_ASSERT( addr_aligned % alignment == 0 );
1662 memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
1665 memset( descr.ptr_aligned, 0x00, descr.size_aligned );
1669 * ( (kmp_mem_descr_t *) addr_descr ) = descr;
1673 KE_TRACE( 25, (
"<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned ) );
1674 return descr.ptr_aligned;
1686 ___kmp_allocate(
size_t size KMP_SRC_LOC_DECL )
1690 KE_TRACE( 25, (
"-> __kmp_allocate( %d ) called from %s:%d\n", (
int) size KMP_SRC_LOC_PARM ) );
1691 ptr = ___kmp_allocate_align( size, __kmp_align_alloc KMP_SRC_LOC_PARM );
1692 KE_TRACE( 25, (
"<- __kmp_allocate() returns %p\n", ptr ) );
1697 #if (BUILD_MEMORY==FIRST_TOUCH)
1699 __kmp_ft_page_allocate(
size_t size)
1704 int page_size = getpagesize();
1707 int page_size = PAGE_SIZE;
1710 adr = (
void *) __kmp_thread_malloc( __kmp_get_thread(),
1711 size + page_size + KMP_PTR_SKIP);
1713 KMP_FATAL( OutOfHeapMemory );
1716 if ( ( (kmp_uintptr_t) adr & (page_size - 1)) == 0)
1721 aadr = (
void *) ( ( (kmp_uintptr_t) adr + page_size) & ~(page_size - 1) );
1724 *((
void**)aadr) = adr;
1727 return (
void*)((
char*)aadr + KMP_PTR_SKIP);
1738 ___kmp_page_allocate(
size_t size KMP_SRC_LOC_DECL )
1740 int page_size = 8 * 1024;
1744 "-> __kmp_page_allocate( %d ) called from %s:%d\n",
1748 ptr = ___kmp_allocate_align( size, page_size KMP_SRC_LOC_PARM );
1749 KE_TRACE( 25, (
"<- __kmp_page_allocate( %d ) returns %p\n", (
int) size, ptr ) );
1758 ___kmp_free(
void * ptr KMP_SRC_LOC_DECL )
1761 kmp_mem_descr_t descr;
1762 kmp_uintptr_t addr_allocated;
1763 kmp_uintptr_t addr_aligned;
1765 KE_TRACE( 25, (
"-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM ) );
1766 KMP_ASSERT( ptr != NULL );
1768 descr = * ( kmp_mem_descr_t *) ( (kmp_uintptr_t) ptr -
sizeof( kmp_mem_descr_t ) );
1770 KE_TRACE( 26, (
" __kmp_free: "
1771 "ptr_allocated=%p, size_allocated=%d, "
1772 "ptr_aligned=%p, size_aligned=%d\n",
1773 descr.ptr_allocated, (
int) descr.size_allocated,
1774 descr.ptr_aligned, (
int) descr.size_aligned ));
1776 addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
1777 addr_aligned = (kmp_uintptr_t) descr.ptr_aligned;
1779 KMP_DEBUG_ASSERT( addr_aligned % CACHE_LINE == 0 );
1780 KMP_DEBUG_ASSERT( descr.ptr_aligned == ptr );
1781 KMP_DEBUG_ASSERT( addr_allocated +
sizeof( kmp_mem_descr_t ) <= addr_aligned );
1782 KMP_DEBUG_ASSERT( descr.size_aligned < descr.size_allocated );
1783 KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
1786 memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
1791 KE_TRACE( 10, (
" free( %p )\n", descr.ptr_allocated ) );
1793 _free_src_loc( descr.ptr_allocated, _file_, _line_ );
1795 free_src_loc( descr.ptr_allocated KMP_SRC_LOC_PARM );
1801 KE_TRACE( 25, (
"<- __kmp_free() returns\n" ) );
1808 #if USE_FAST_MEMORY == 3
1814 #define KMP_FREE_LIST_LIMIT 16
1817 #define DCACHE_LINE 128
1820 ___kmp_fast_allocate( kmp_info_t *this_thr,
size_t size KMP_SRC_LOC_DECL )
1828 kmp_mem_descr_t * descr;
1830 KE_TRACE( 25, (
"-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n",
1831 __kmp_gtid_from_thread(this_thr), (
int) size KMP_SRC_LOC_PARM ) );
1833 num_lines = ( size + DCACHE_LINE - 1 ) / DCACHE_LINE;
1834 idx = num_lines - 1;
1835 KMP_DEBUG_ASSERT( idx >= 0 );
1839 }
else if ( ( idx >>= 2 ) == 0 ) {
1842 }
else if ( ( idx >>= 2 ) == 0 ) {
1845 }
else if ( ( idx >>= 2 ) == 0 ) {
1852 ptr = this_thr->th.th_free_lists[index].th_free_list_self;
1853 if ( ptr != NULL ) {
1855 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1856 KMP_DEBUG_ASSERT( this_thr ==
1857 ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t) ))->ptr_aligned );
1860 ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
1861 if ( ptr != NULL ) {
1864 while ( ! KMP_COMPARE_AND_STORE_PTR(
1865 &this_thr->th.th_free_lists[index].th_free_list_sync, ptr, NULL ) )
1868 ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
1871 this_thr->th.th_free_lists[index].th_free_list_self = *((
void **)ptr);
1872 KMP_DEBUG_ASSERT( this_thr ==
1873 ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr -
sizeof(kmp_mem_descr_t) ))->ptr_aligned );
1879 size = num_lines * DCACHE_LINE;
1881 alloc_size = size +
sizeof( kmp_mem_descr_t ) + DCACHE_LINE;
1882 KE_TRACE( 25, (
"__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with alloc_size %d\n",
1883 __kmp_gtid_from_thread( this_thr ), alloc_size ) );
1884 alloc_ptr = bget( this_thr, (bufsize) alloc_size );
1887 ptr = (
void *)(( ((kmp_uintptr_t)alloc_ptr) +
sizeof(kmp_mem_descr_t) + DCACHE_LINE ) & ~( DCACHE_LINE - 1 ));
1888 descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t) );
1890 descr->ptr_allocated = alloc_ptr;
1892 descr->ptr_aligned = (
void *)this_thr;
1895 descr->size_aligned = size;
1898 KE_TRACE( 25, (
"<- __kmp_fast_allocate( T#%d ) returns %p\n",
1899 __kmp_gtid_from_thread( this_thr ), ptr ) );
1906 ___kmp_fast_free( kmp_info_t *this_thr,
void * ptr KMP_SRC_LOC_DECL )
1908 kmp_mem_descr_t * descr;
1909 kmp_info_t * alloc_thr;
1914 KE_TRACE( 25, (
"-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n",
1915 __kmp_gtid_from_thread(this_thr), ptr KMP_SRC_LOC_PARM ) );
1916 KMP_ASSERT( ptr != NULL );
1918 descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) -
sizeof(kmp_mem_descr_t) );
1920 KE_TRACE(26, (
" __kmp_fast_free: size_aligned=%d\n",
1921 (
int) descr->size_aligned ) );
1923 size = descr->size_aligned;
1925 idx = DCACHE_LINE * 2;
1926 if ( idx == size ) {
1928 }
else if ( ( idx <<= 1 ) == size ) {
1930 }
else if ( ( idx <<= 2 ) == size ) {
1932 }
else if ( ( idx <<= 2 ) == size ) {
1935 KMP_DEBUG_ASSERT( size > DCACHE_LINE * 64 );
1939 alloc_thr = (kmp_info_t *)descr->ptr_aligned;
1940 if ( alloc_thr == this_thr ) {
1942 *((
void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self;
1943 this_thr->th.th_free_lists[index].th_free_list_self = ptr;
1945 void * head = this_thr->th.th_free_lists[index].th_free_list_other;
1946 if ( head == NULL ) {
1948 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1949 *((
void **)ptr) = NULL;
1950 descr->size_allocated = (size_t)1;
1953 kmp_mem_descr_t * dsc = (kmp_mem_descr_t *)( (
char*)head -
sizeof(kmp_mem_descr_t) );
1954 kmp_info_t * q_th = (kmp_info_t *)(dsc->ptr_aligned);
1955 size_t q_sz = dsc->size_allocated + 1;
1956 if ( q_th == alloc_thr && q_sz <= KMP_FREE_LIST_LIMIT ) {
1958 *((
void **)ptr) = head;
1959 descr->size_allocated = q_sz;
1960 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1967 void * next = *((
void **)head);
1968 while ( next != NULL ) {
1971 ((kmp_mem_descr_t*)((
char*)next -
sizeof(kmp_mem_descr_t)))->size_allocated + 1 ==
1972 ((kmp_mem_descr_t*)((
char*)tail -
sizeof(kmp_mem_descr_t)))->size_allocated );
1974 next = *((
void **)next);
1976 KMP_DEBUG_ASSERT( q_th != NULL );
1978 old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
1981 *((
void **)tail) = old_ptr;
1983 while ( ! KMP_COMPARE_AND_STORE_PTR(
1984 &q_th->th.th_free_lists[index].th_free_list_sync,
1989 old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
1990 *((
void **)tail) = old_ptr;
1994 this_thr->th.th_free_lists[index].th_free_list_other = ptr;
1995 *((
void **)ptr) = NULL;
1996 descr->size_allocated = (size_t)1;
2003 KE_TRACE(25, (
"__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n",
2004 __kmp_gtid_from_thread( this_thr), size ) );
2005 __kmp_bget_dequeue( this_thr );
2006 brel( this_thr, descr->ptr_allocated );
2009 KE_TRACE( 25, (
"<- __kmp_fast_free() returns\n" ) );
2017 __kmp_initialize_fast_memory( kmp_info_t *this_thr )
2019 KE_TRACE(10, (
"__kmp_initialize_fast_memory: Called from th %p\n", this_thr ) );
2021 memset ( this_thr->th.th_free_lists, 0, NUM_LISTS *
sizeof( kmp_free_list_t ) );
2027 __kmp_free_fast_memory( kmp_info_t *th )
2031 thr_data_t * thr = get_thr_data( th );
2034 KE_TRACE(5, (
"__kmp_free_fast_memory: Called T#%d\n",
2035 __kmp_gtid_from_thread( th ) ) );
2037 __kmp_bget_dequeue( th );
2040 for ( bin = 0; bin < MAX_BGET_BINS; ++bin ) {
2041 bfhead_t * b = thr->freelist[ bin ].ql.flink;
2042 while ( b != &thr->freelist[ bin ] ) {
2043 if ( (kmp_uintptr_t)b->bh.bb.bthr & 1 ) {
2050 while ( lst != NULL ) {
2052 KE_TRACE(10, (
"__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n",
2053 lst, next, th, __kmp_gtid_from_thread( th ) ) );
2054 (*thr->relfcn)(lst);
2063 KE_TRACE(5, (
"__kmp_free_fast_memory: Freed T#%d\n",
2064 __kmp_gtid_from_thread( th ) ) );
2067 #endif // USE_FAST_MEMORY