38 #include "kmp_wrapper_getpid.h"
43 #include "kmp_stats.h"
44 #include "kmp_wait_release.h"
52 #include <sys/times.h>
53 #include <sys/resource.h>
54 #include <sys/syscall.h>
56 #if KMP_OS_LINUX && !KMP_OS_CNK
57 # include <sys/sysinfo.h>
58 # if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)
72 # include <sys/sysctl.h>
73 # include <mach/mach.h>
75 # include <sys/sysctl.h>
76 # include <pthread_np.h>
85 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64)
93 struct kmp_sys_timer {
94 struct timespec start;
98 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
100 static struct kmp_sys_timer __kmp_sys_timer_data;
102 #if KMP_HANDLE_SIGNALS
103 typedef void (* sig_func_t )( int );
104 STATIC_EFI2_WORKAROUND
struct sigaction __kmp_sighldrs[ NSIG ];
105 static sigset_t __kmp_sigset;
108 static int __kmp_init_runtime = FALSE;
110 static int __kmp_fork_count = 0;
112 static pthread_condattr_t __kmp_suspend_cond_attr;
113 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
115 static kmp_cond_align_t __kmp_wait_cv;
116 static kmp_mutex_align_t __kmp_wait_mx;
123 __kmp_print_cond(
char *buffer, kmp_cond_align_t *cond )
125 sprintf( buffer,
"(cond (lock (%ld, %d)), (descr (%p)))",
126 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
127 cond->c_cond.__c_waiting );
134 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
147 # if KMP_ARCH_X86 || KMP_ARCH_ARM
148 # ifndef __NR_sched_setaffinity
149 # define __NR_sched_setaffinity 241
150 # elif __NR_sched_setaffinity != 241
151 # error Wrong code for setaffinity system call.
153 # ifndef __NR_sched_getaffinity
154 # define __NR_sched_getaffinity 242
155 # elif __NR_sched_getaffinity != 242
156 # error Wrong code for getaffinity system call.
159 # elif KMP_ARCH_X86_64
160 # ifndef __NR_sched_setaffinity
161 # define __NR_sched_setaffinity 203
162 # elif __NR_sched_setaffinity != 203
163 # error Wrong code for setaffinity system call.
165 # ifndef __NR_sched_getaffinity
166 # define __NR_sched_getaffinity 204
167 # elif __NR_sched_getaffinity != 204
168 # error Wrong code for getaffinity system call.
171 # elif KMP_ARCH_PPC64
172 # ifndef __NR_sched_setaffinity
173 # define __NR_sched_setaffinity 222
174 # elif __NR_sched_setaffinity != 222
175 # error Wrong code for setaffinity system call.
177 # ifndef __NR_sched_getaffinity
178 # define __NR_sched_getaffinity 223
179 # elif __NR_sched_getaffinity != 223
180 # error Wrong code for getaffinity system call.
185 # error Unknown or unsupported architecture
190 __kmp_set_system_affinity( kmp_affin_mask_t
const *mask,
int abort_on_error )
192 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
193 "Illegal set affinity operation when not capable");
195 int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
200 if (abort_on_error) {
203 KMP_MSG( FatalSysError ),
212 __kmp_get_system_affinity( kmp_affin_mask_t *mask,
int abort_on_error )
214 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
215 "Illegal get affinity operation when not capable");
217 int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
222 if (abort_on_error) {
225 KMP_MSG( FatalSysError ),
234 __kmp_affinity_bind_thread(
int which )
236 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
237 "Illegal set affinity operation when not capable");
239 kmp_affin_mask_t *mask = (kmp_affin_mask_t *)alloca(__kmp_affin_mask_size);
241 KMP_CPU_SET(which, mask);
242 __kmp_set_system_affinity(mask, TRUE);
251 __kmp_affinity_determine_capable(
const char *env_var)
257 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
261 kmp_affin_mask_t *buf;
262 buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
267 gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
268 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
269 "initial getaffinity call returned %d errno = %d\n",
277 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
278 && (__kmp_affinity_type != affinity_none)
279 && (__kmp_affinity_type != affinity_default)
280 && (__kmp_affinity_type != affinity_disabled))) {
284 KMP_MSG( GetAffSysCallNotSupported, env_var ),
289 __kmp_affin_mask_size = 0;
290 KMP_INTERNAL_FREE(buf);
299 sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
300 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
301 "setaffinity for mask size %d returned %d errno = %d\n",
302 gCode, sCode, errno));
304 if (errno == ENOSYS) {
305 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
306 && (__kmp_affinity_type != affinity_none)
307 && (__kmp_affinity_type != affinity_default)
308 && (__kmp_affinity_type != affinity_disabled))) {
312 KMP_MSG( SetAffSysCallNotSupported, env_var ),
317 __kmp_affin_mask_size = 0;
318 KMP_INTERNAL_FREE(buf);
320 if (errno == EFAULT) {
321 __kmp_affin_mask_size = gCode;
322 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
323 "affinity supported (mask size %d)\n",
324 (
int)__kmp_affin_mask_size));
325 KMP_INTERNAL_FREE(buf);
335 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
336 "searching for proper set size\n"));
338 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
339 gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
340 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
341 "getaffinity for mask size %d returned %d errno = %d\n", size,
345 if ( errno == ENOSYS )
350 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
351 "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
353 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
354 && (__kmp_affinity_type != affinity_none)
355 && (__kmp_affinity_type != affinity_default)
356 && (__kmp_affinity_type != affinity_disabled))) {
360 KMP_MSG( GetAffSysCallNotSupported, env_var ),
365 __kmp_affin_mask_size = 0;
366 KMP_INTERNAL_FREE(buf);
372 sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
373 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
374 "setaffinity for mask size %d returned %d errno = %d\n",
375 gCode, sCode, errno));
377 if (errno == ENOSYS) {
381 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
382 "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
384 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
385 && (__kmp_affinity_type != affinity_none)
386 && (__kmp_affinity_type != affinity_default)
387 && (__kmp_affinity_type != affinity_disabled))) {
391 KMP_MSG( SetAffSysCallNotSupported, env_var ),
396 __kmp_affin_mask_size = 0;
397 KMP_INTERNAL_FREE(buf);
400 if (errno == EFAULT) {
401 __kmp_affin_mask_size = gCode;
402 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
403 "affinity supported (mask size %d)\n",
404 (
int)__kmp_affin_mask_size));
405 KMP_INTERNAL_FREE(buf);
411 KMP_INTERNAL_FREE(buf);
417 __kmp_affin_mask_size = 0;
418 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
419 "cannot determine mask size - affinity not supported\n"));
420 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
421 && (__kmp_affinity_type != affinity_none)
422 && (__kmp_affinity_type != affinity_default)
423 && (__kmp_affinity_type != affinity_disabled))) {
424 KMP_WARNING( AffCantGetMaskSize, env_var );
436 __kmp_change_thread_affinity_mask(
int gtid, kmp_affin_mask_t *new_mask,
437 kmp_affin_mask_t *old_mask )
439 KMP_DEBUG_ASSERT( gtid == __kmp_get_gtid() );
440 if ( KMP_AFFINITY_CAPABLE() ) {
442 kmp_info_t *th = __kmp_threads[ gtid ];
444 KMP_DEBUG_ASSERT( new_mask != NULL );
446 if ( old_mask != NULL ) {
447 status = __kmp_get_system_affinity( old_mask, TRUE );
452 KMP_MSG( ChangeThreadAffMaskError ),
459 __kmp_set_system_affinity( new_mask, TRUE );
461 if (__kmp_affinity_verbose) {
462 char old_buf[KMP_AFFIN_MASK_PRINT_LEN];
463 char new_buf[KMP_AFFIN_MASK_PRINT_LEN];
464 __kmp_affinity_print_mask(old_buf, KMP_AFFIN_MASK_PRINT_LEN, old_mask);
465 __kmp_affinity_print_mask(new_buf, KMP_AFFIN_MASK_PRINT_LEN, new_mask);
466 KMP_INFORM( ChangeAffMask,
"KMP_AFFINITY (Bind)", gtid, old_buf, new_buf );
471 KMP_DEBUG_ASSERT( old_mask != NULL && (memcmp(old_mask,
472 th->th.th_affin_mask, __kmp_affin_mask_size) == 0) );
473 KMP_CPU_COPY( th->th.th_affin_mask, new_mask );
477 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
482 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
485 __kmp_futex_determine_capable()
488 int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
489 int retval = ( rc == 0 ) || ( errno != ENOSYS );
491 KA_TRACE(10, (
"__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
493 KA_TRACE(10, (
"__kmp_futex_determine_capable: futex syscall%s supported\n",
494 retval ?
"" :
" not" ) );
499 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
504 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
511 __kmp_test_then_or32(
volatile kmp_int32 *p, kmp_int32 d )
513 kmp_int32 old_value, new_value;
515 old_value = TCR_4( *p );
516 new_value = old_value | d;
518 while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
521 old_value = TCR_4( *p );
522 new_value = old_value | d;
528 __kmp_test_then_and32(
volatile kmp_int32 *p, kmp_int32 d )
530 kmp_int32 old_value, new_value;
532 old_value = TCR_4( *p );
533 new_value = old_value & d;
535 while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
538 old_value = TCR_4( *p );
539 new_value = old_value & d;
544 # if KMP_ARCH_X86 || KMP_ARCH_PPC64
546 __kmp_test_then_add64(
volatile kmp_int64 *p, kmp_int64 d )
548 kmp_int64 old_value, new_value;
550 old_value = TCR_8( *p );
551 new_value = old_value + d;
553 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
556 old_value = TCR_8( *p );
557 new_value = old_value + d;
564 __kmp_test_then_or64(
volatile kmp_int64 *p, kmp_int64 d )
566 kmp_int64 old_value, new_value;
568 old_value = TCR_8( *p );
569 new_value = old_value | d;
570 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
573 old_value = TCR_8( *p );
574 new_value = old_value | d;
580 __kmp_test_then_and64(
volatile kmp_int64 *p, kmp_int64 d )
582 kmp_int64 old_value, new_value;
584 old_value = TCR_8( *p );
585 new_value = old_value & d;
586 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
589 old_value = TCR_8( *p );
590 new_value = old_value & d;
598 __kmp_terminate_thread(
int gtid )
601 kmp_info_t *th = __kmp_threads[ gtid ];
605 #ifdef KMP_CANCEL_THREADS
606 KA_TRACE( 10, (
"__kmp_terminate_thread: kill (%d)\n", gtid ) );
607 status = pthread_cancel( th->th.th_info.ds.ds_thread );
608 if ( status != 0 && status != ESRCH ) {
611 KMP_MSG( CantTerminateWorkerThread ),
635 __kmp_set_stack_info(
int gtid, kmp_info_t *th )
638 #if KMP_OS_LINUX || KMP_OS_FREEBSD
648 if ( ! KMP_UBER_GTID(gtid) ) {
651 status = pthread_attr_init( &attr );
652 KMP_CHECK_SYSFAIL(
"pthread_attr_init", status );
654 status = pthread_attr_get_np( pthread_self(), &attr );
655 KMP_CHECK_SYSFAIL(
"pthread_attr_get_np", status );
657 status = pthread_getattr_np( pthread_self(), &attr );
658 KMP_CHECK_SYSFAIL(
"pthread_getattr_np", status );
660 status = pthread_attr_getstack( &attr, &addr, &size );
661 KMP_CHECK_SYSFAIL(
"pthread_attr_getstack", status );
662 KA_TRACE( 60, (
"__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
666 status = pthread_attr_destroy( &attr );
667 KMP_CHECK_SYSFAIL(
"pthread_attr_destroy", status );
670 if ( size != 0 && addr != 0 ) {
672 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((
char *)addr) + size));
673 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
674 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
679 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
680 TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
681 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
686 __kmp_launch_worker(
void *thr )
688 int status, old_type, old_state;
689 #ifdef KMP_BLOCK_SIGNALS
690 sigset_t new_set, old_set;
697 gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
698 __kmp_gtid_set_specific( gtid );
699 #ifdef KMP_TDATA_GTID
702 #if KMP_STATS_ENABLED
704 __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
708 __kmp_itt_thread_name( gtid );
711 #if KMP_AFFINITY_SUPPORTED
712 __kmp_affinity_set_init_mask( gtid, FALSE );
715 #ifdef KMP_CANCEL_THREADS
716 status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
717 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status );
719 status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
720 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
723 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
728 __kmp_clear_x87_fpu_status_word();
729 __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
730 __kmp_load_mxcsr( &__kmp_init_mxcsr );
733 #ifdef KMP_BLOCK_SIGNALS
734 status = sigfillset( & new_set );
735 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status );
736 status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
737 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
740 #if KMP_OS_LINUX || KMP_OS_FREEBSD
741 if ( __kmp_stkoffset > 0 && gtid > 0 ) {
742 padding = alloca( gtid * __kmp_stkoffset );
747 __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
749 __kmp_check_stack_overlap( (kmp_info_t*)thr );
751 exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
753 #ifdef KMP_BLOCK_SIGNALS
754 status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
755 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
765 __kmp_launch_monitor(
void *thr )
767 int status, old_type, old_state;
768 #ifdef KMP_BLOCK_SIGNALS
771 struct timespec interval;
773 int yield_cycles = 0;
778 KA_TRACE( 10, (
"__kmp_launch_monitor: #1 launched\n" ) );
781 __kmp_gtid_set_specific( KMP_GTID_MONITOR );
782 #ifdef KMP_TDATA_GTID
783 __kmp_gtid = KMP_GTID_MONITOR;
789 __kmp_itt_thread_ignore();
792 __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
794 __kmp_check_stack_overlap( (kmp_info_t*)thr );
796 #ifdef KMP_CANCEL_THREADS
797 status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
798 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status );
800 status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
801 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
804 #if KMP_REAL_TIME_FIX
808 int sched = sched_getscheduler( 0 );
809 if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
812 struct sched_param param;
813 int max_priority = sched_get_priority_max( sched );
815 KMP_WARNING( RealTimeSchedNotSupported );
816 sched_getparam( 0, & param );
817 if ( param.sched_priority < max_priority ) {
818 param.sched_priority += 1;
819 rc = sched_setscheduler( 0, sched, & param );
824 KMP_MSG( CantChangeMonitorPriority ),
826 KMP_MSG( MonitorWillStarve ),
835 KMP_MSG( RunningAtMaxPriority ),
836 KMP_MSG( MonitorWillStarve ),
837 KMP_HNT( RunningAtMaxPriority ),
842 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
844 #endif // KMP_REAL_TIME_FIX
848 if ( __kmp_monitor_wakeups == 1 ) {
850 interval.tv_nsec = 0;
853 interval.tv_nsec = (NSEC_PER_SEC / __kmp_monitor_wakeups);
856 KA_TRACE( 10, (
"__kmp_launch_monitor: #2 monitor\n" ) );
858 if (__kmp_yield_cycle) {
859 __kmp_yielding_on = 0;
860 yield_count = __kmp_yield_off_count;
862 __kmp_yielding_on = 1;
865 while( ! TCR_4( __kmp_global.g.g_done ) ) {
871 KA_TRACE( 15, (
"__kmp_launch_monitor: update\n" ) );
873 status = gettimeofday( &tval, NULL );
874 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
875 TIMEVAL_TO_TIMESPEC( &tval, &now );
877 now.tv_sec += interval.tv_sec;
878 now.tv_nsec += interval.tv_nsec;
880 if (now.tv_nsec >= NSEC_PER_SEC) {
882 now.tv_nsec -= NSEC_PER_SEC;
885 status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
886 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
888 if ( !TCR_4(__kmp_global.g.g_done) ) {
889 status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
891 if ( status != ETIMEDOUT && status != EINTR ) {
892 KMP_SYSFAIL(
"pthread_cond_timedwait", status );
896 status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
897 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
899 if (__kmp_yield_cycle) {
901 if ( (yield_cycles % yield_count) == 0 ) {
902 if (__kmp_yielding_on) {
903 __kmp_yielding_on = 0;
904 yield_count = __kmp_yield_off_count;
906 __kmp_yielding_on = 1;
907 yield_count = __kmp_yield_on_count;
912 __kmp_yielding_on = 1;
915 TCW_4( __kmp_global.g.g_time.dt.t_value,
916 TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
921 KA_TRACE( 10, (
"__kmp_launch_monitor: #3 cleanup\n" ) );
923 #ifdef KMP_BLOCK_SIGNALS
924 status = sigfillset( & new_set );
925 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status );
926 status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
927 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
930 KA_TRACE( 10, (
"__kmp_launch_monitor: #4 finished\n" ) );
932 if( __kmp_global.g.g_abort != 0 ) {
938 KA_TRACE( 10, (
"__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
943 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
944 __kmp_terminate_thread( gtid );
948 KA_TRACE( 10, (
"__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
950 if (__kmp_global.g.g_abort > 0)
951 raise( __kmp_global.g.g_abort );
955 KA_TRACE( 10, (
"__kmp_launch_monitor: #7 exit\n" ) );
961 __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size )
964 pthread_attr_t thread_attr;
968 th->th.th_info.ds.ds_gtid = gtid;
970 #if KMP_STATS_ENABLED
972 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
977 th->th.th_stats = __kmp_stats_list.push_back(gtid);
978 if(KMP_UBER_GTID(gtid)) {
979 __kmp_stats_start_time = tsc_tick_count::now();
980 __kmp_stats_thread_ptr = th->th.th_stats;
985 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
987 #endif // KMP_STATS_ENABLED
989 if ( KMP_UBER_GTID(gtid) ) {
990 KA_TRACE( 10, (
"__kmp_create_worker: uber thread (%d)\n", gtid ) );
991 th -> th.th_info.ds.ds_thread = pthread_self();
992 __kmp_set_stack_info( gtid, th );
993 __kmp_check_stack_overlap( th );
997 KA_TRACE( 10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1001 #ifdef KMP_THREAD_ATTR
1003 status = pthread_attr_init( &thread_attr );
1004 if ( status != 0 ) {
1007 KMP_MSG( CantInitThreadAttrs ),
1012 status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1013 if ( status != 0 ) {
1016 KMP_MSG( CantSetWorkerState ),
1023 stack_size += gtid * __kmp_stkoffset;
1025 KA_TRACE( 10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1026 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1027 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1029 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
1030 status = pthread_attr_setstacksize( & thread_attr, stack_size );
1031 # ifdef KMP_BACKUP_STKSIZE
1032 if ( status != 0 ) {
1033 if ( ! __kmp_env_stksize ) {
1034 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1035 __kmp_stksize = KMP_BACKUP_STKSIZE;
1036 KA_TRACE( 10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1037 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
1039 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1041 status = pthread_attr_setstacksize( &thread_attr, stack_size );
1045 if ( status != 0 ) {
1048 KMP_MSG( CantSetWorkerStackSize, stack_size ),
1050 KMP_HNT( ChangeWorkerStackSize ),
1059 status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (
void *) th );
1060 if ( status != 0 || ! handle ) {
1061 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1062 if ( status == EINVAL ) {
1065 KMP_MSG( CantSetWorkerStackSize, stack_size ),
1067 KMP_HNT( IncreaseWorkerStackSize ),
1071 if ( status == ENOMEM ) {
1074 KMP_MSG( CantSetWorkerStackSize, stack_size ),
1076 KMP_HNT( DecreaseWorkerStackSize ),
1081 if ( status == EAGAIN ) {
1084 KMP_MSG( NoResourcesForWorkerThread ),
1086 KMP_HNT( Decrease_NUM_THREADS ),
1090 KMP_SYSFAIL(
"pthread_create", status );
1093 th->th.th_info.ds.ds_thread = handle;
1096 #ifdef KMP_THREAD_ATTR
1098 status = pthread_attr_destroy( & thread_attr );
1102 KMP_MSG( CantDestroyThreadAttrs ),
1112 KA_TRACE( 10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1118 __kmp_create_monitor( kmp_info_t *th )
1121 pthread_attr_t thread_attr;
1124 int caller_gtid = __kmp_get_gtid();
1125 int auto_adj_size = FALSE;
1127 KA_TRACE( 10, (
"__kmp_create_monitor: try to create monitor\n" ) );
1131 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1132 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1133 #if KMP_REAL_TIME_FIX
1134 TCW_4( __kmp_global.g.g_time.dt.t_value, -1 );
1136 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1137 #endif // KMP_REAL_TIME_FIX
1139 #ifdef KMP_THREAD_ATTR
1140 if ( __kmp_monitor_stksize == 0 ) {
1141 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1142 auto_adj_size = TRUE;
1144 status = pthread_attr_init( &thread_attr );
1145 if ( status != 0 ) {
1148 KMP_MSG( CantInitThreadAttrs ),
1153 status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1154 if ( status != 0 ) {
1157 KMP_MSG( CantSetMonitorState ),
1163 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1164 status = pthread_attr_getstacksize( & thread_attr, & size );
1165 KMP_CHECK_SYSFAIL(
"pthread_attr_getstacksize", status );
1167 size = __kmp_sys_min_stksize;
1171 if ( __kmp_monitor_stksize == 0 ) {
1172 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1174 if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1175 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1178 KA_TRACE( 10, (
"__kmp_create_monitor: default stacksize = %lu bytes,"
1179 "requested stacksize = %lu bytes\n",
1180 size, __kmp_monitor_stksize ) );
1186 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1187 KA_TRACE( 10, (
"__kmp_create_monitor: setting stacksize = %lu bytes,",
1188 __kmp_monitor_stksize ) );
1189 status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1190 if ( status != 0 ) {
1191 if ( auto_adj_size ) {
1192 __kmp_monitor_stksize *= 2;
1197 KMP_MSG( CantSetMonitorStackSize, (
long int) __kmp_monitor_stksize ),
1199 KMP_HNT( ChangeMonitorStackSize ),
1205 status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (
void *) th );
1207 if ( status != 0 ) {
1208 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1209 if ( status == EINVAL ) {
1210 if ( auto_adj_size && ( __kmp_monitor_stksize < (
size_t)0x40000000 ) ) {
1211 __kmp_monitor_stksize *= 2;
1216 KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1218 KMP_HNT( IncreaseMonitorStackSize ),
1222 if ( status == ENOMEM ) {
1225 KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1227 KMP_HNT( DecreaseMonitorStackSize ),
1232 if ( status == EAGAIN ) {
1235 KMP_MSG( NoResourcesForMonitorThread ),
1237 KMP_HNT( DecreaseNumberOfThreadsInUse ),
1241 KMP_SYSFAIL(
"pthread_create", status );
1244 th->th.th_info.ds.ds_thread = handle;
1246 #if KMP_REAL_TIME_FIX
1248 KMP_DEBUG_ASSERT(
sizeof( kmp_uint32 ) ==
sizeof( __kmp_global.g.g_time.dt.t_value ) );
1250 (kmp_uint32
volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1252 #endif // KMP_REAL_TIME_FIX
1254 #ifdef KMP_THREAD_ATTR
1255 status = pthread_attr_destroy( & thread_attr );
1256 if ( status != 0 ) {
1259 KMP_MSG( CantDestroyThreadAttrs ),
1268 KA_TRACE( 10, (
"__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1276 pthread_exit( (
void *)(intptr_t) exit_status );
1279 void __kmp_resume_monitor();
1282 __kmp_reap_monitor( kmp_info_t *th )
1287 KA_TRACE( 10, (
"__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1288 th->th.th_info.ds.ds_thread ) );
1293 KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1294 if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1305 status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1306 if (status == ESRCH) {
1308 KA_TRACE( 10, (
"__kmp_reap_monitor: monitor does not exist, returning\n") );
1312 __kmp_resume_monitor();
1313 status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1314 if (exit_val != th) {
1317 KMP_MSG( ReapMonitorError ),
1324 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1325 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1327 KA_TRACE( 10, (
"__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1328 th->th.th_info.ds.ds_thread ) );
1335 __kmp_reap_worker( kmp_info_t *th )
1342 KA_TRACE( 10, (
"__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1349 status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1350 if (status == ESRCH) {
1351 KA_TRACE( 10, (
"__kmp_reap_worker: worker T#%d does not exist, returning\n",
1352 th->th.th_info.ds.ds_gtid ) );
1355 KA_TRACE( 10, (
"__kmp_reap_worker: try to join with worker T#%d\n",
1356 th->th.th_info.ds.ds_gtid ) );
1358 status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1361 if ( status != 0 ) {
1364 KMP_MSG( ReapWorkerError ),
1369 if ( exit_val != th ) {
1370 KA_TRACE( 10, (
"__kmp_reap_worker: worker T#%d did not reap properly, "
1372 th->th.th_info.ds.ds_gtid, exit_val ) );
1378 KA_TRACE( 10, (
"__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1387 #if KMP_HANDLE_SIGNALS
1391 __kmp_null_handler(
int signo )
1398 __kmp_team_handler(
int signo )
1400 if ( __kmp_global.g.g_abort == 0 ) {
1403 __kmp_debug_printf(
"__kmp_team_handler: caught signal = %d\n", signo );
1418 if ( __kmp_debug_buf ) {
1419 __kmp_dump_debug_buffer( );
1422 TCW_4( __kmp_global.g.g_abort, signo );
1424 TCW_4( __kmp_global.g.g_done, TRUE );
1429 __kmp_debug_printf(
"__kmp_team_handler: unknown signal type" );
1438 void __kmp_sigaction(
int signum,
const struct sigaction * act,
struct sigaction * oldact ) {
1439 int rc = sigaction( signum, act, oldact );
1440 KMP_CHECK_SYSFAIL_ERRNO(
"sigaction", rc );
1445 __kmp_install_one_handler(
int sig, sig_func_t handler_func,
int parallel_init )
1448 KB_TRACE( 60, (
"__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1449 if ( parallel_init ) {
1450 struct sigaction new_action;
1451 struct sigaction old_action;
1452 new_action.sa_handler = handler_func;
1453 new_action.sa_flags = 0;
1454 sigfillset( & new_action.sa_mask );
1455 __kmp_sigaction( sig, & new_action, & old_action );
1456 if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1457 sigaddset( & __kmp_sigset, sig );
1460 __kmp_sigaction( sig, & old_action, NULL );
1464 __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1471 __kmp_remove_one_handler(
int sig )
1473 KB_TRACE( 60, (
"__kmp_remove_one_handler( %d )\n", sig ) );
1474 if ( sigismember( & __kmp_sigset, sig ) ) {
1475 struct sigaction old;
1477 __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1478 if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1480 KB_TRACE( 10, (
"__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1481 __kmp_sigaction( sig, & old, NULL );
1483 sigdelset( & __kmp_sigset, sig );
1490 __kmp_install_signals(
int parallel_init )
1492 KB_TRACE( 10, (
"__kmp_install_signals( %d )\n", parallel_init ) );
1493 if ( __kmp_handle_signals || ! parallel_init ) {
1496 sigemptyset( & __kmp_sigset );
1497 __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1498 __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1499 __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1500 __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1501 __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1502 __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1503 __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1504 __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1506 __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1508 __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1510 __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1517 __kmp_remove_signals(
void )
1520 KB_TRACE( 10, (
"__kmp_remove_signals()\n" ) );
1521 for ( sig = 1; sig < NSIG; ++ sig ) {
1522 __kmp_remove_one_handler( sig );
1527 #endif // KMP_HANDLE_SIGNALS
1533 __kmp_enable(
int new_state )
1535 #ifdef KMP_CANCEL_THREADS
1536 int status, old_state;
1537 status = pthread_setcancelstate( new_state, & old_state );
1538 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
1539 KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1544 __kmp_disable(
int * old_state )
1546 #ifdef KMP_CANCEL_THREADS
1548 status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1549 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
1557 __kmp_atfork_prepare (
void)
1563 __kmp_atfork_parent (
void)
1574 __kmp_atfork_child (
void)
1582 __kmp_init_runtime = FALSE;
1583 __kmp_init_monitor = 0;
1584 __kmp_init_parallel = FALSE;
1585 __kmp_init_middle = FALSE;
1586 __kmp_init_serial = FALSE;
1587 TCW_4(__kmp_init_gtid, FALSE);
1588 __kmp_init_common = FALSE;
1590 TCW_4(__kmp_init_user_locks, FALSE);
1591 __kmp_user_lock_table.used = 1;
1592 __kmp_user_lock_table.allocated = 0;
1593 __kmp_user_lock_table.table = NULL;
1594 __kmp_lock_blocks = NULL;
1597 TCW_4(__kmp_nth, 0);
1601 KA_TRACE( 10, (
"__kmp_atfork_child: checking cache address list %p\n",
1602 __kmp_threadpriv_cache_list ) );
1604 while ( __kmp_threadpriv_cache_list != NULL ) {
1606 if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1607 KC_TRACE( 50, (
"__kmp_atfork_child: zeroing cache at address %p\n",
1608 &(*__kmp_threadpriv_cache_list -> addr) ) );
1610 *__kmp_threadpriv_cache_list -> addr = NULL;
1612 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1615 __kmp_init_runtime = FALSE;
1618 __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1619 __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1620 __kmp_init_bootstrap_lock( &__kmp_console_lock );
1635 __kmp_register_atfork(
void) {
1636 if ( __kmp_need_register_atfork ) {
1637 int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1638 KMP_CHECK_SYSFAIL(
"pthread_atfork", status );
1639 __kmp_need_register_atfork = FALSE;
1644 __kmp_suspend_initialize(
void )
1647 status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1648 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status );
1649 status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1650 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status );
1654 __kmp_suspend_initialize_thread( kmp_info_t *th )
1656 if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1660 status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1661 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status );
1662 status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1663 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status );
1664 *(
volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1669 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1671 if(th->th.th_suspend_init_count > __kmp_fork_count) {
1676 status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1677 if ( status != 0 && status != EBUSY ) {
1678 KMP_SYSFAIL(
"pthread_cond_destroy", status );
1680 status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1681 if ( status != 0 && status != EBUSY ) {
1682 KMP_SYSFAIL(
"pthread_mutex_destroy", status );
1684 --th->th.th_suspend_init_count;
1685 KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1693 static inline void __kmp_suspend_template(
int th_gtid, C *flag )
1696 kmp_info_t *th = __kmp_threads[th_gtid];
1698 typename C::flag_t old_spin;
1700 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1702 __kmp_suspend_initialize_thread( th );
1704 status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1705 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1707 KF_TRACE( 10, (
"__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1708 th_gtid, flag->get() ) );
1713 old_spin = flag->set_sleeping();
1715 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%d\n",
1716 th_gtid, flag->get(), *(flag->get()) ) );
1718 if ( flag->done_check_val(old_spin) ) {
1719 old_spin = flag->unset_sleeping();
1720 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1721 th_gtid, flag->get()) );
1727 int deactivated = FALSE;
1728 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
1729 while ( flag->is_sleeping() ) {
1730 #ifdef DEBUG_SUSPEND
1732 __kmp_suspend_count++;
1733 __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1734 __kmp_printf(
"__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1737 if ( ! deactivated ) {
1738 th->th.th_active = FALSE;
1739 if ( th->th.th_active_in_pool ) {
1740 th->th.th_active_in_pool = FALSE;
1741 KMP_TEST_THEN_DEC32(
1742 (kmp_int32 *) &__kmp_thread_pool_active_nth );
1743 KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1750 #if USE_SUSPEND_TIMEOUT
1751 struct timespec now;
1752 struct timeval tval;
1755 status = gettimeofday( &tval, NULL );
1756 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
1757 TIMEVAL_TO_TIMESPEC( &tval, &now );
1759 msecs = (4*__kmp_dflt_blocktime) + 200;
1760 now.tv_sec += msecs / 1000;
1761 now.tv_nsec += (msecs % 1000)*1000;
1763 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1765 status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1767 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1770 status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1773 if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1774 KMP_SYSFAIL(
"pthread_cond_wait", status );
1777 if (status == ETIMEDOUT) {
1778 if ( flag->is_sleeping() ) {
1779 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1781 KF_TRACE( 2, (
"__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1784 }
else if ( flag->is_sleeping() ) {
1785 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1791 if ( deactivated ) {
1792 th->th.th_active = TRUE;
1793 if ( TCR_4(th->th.th_in_pool) ) {
1794 KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1795 th->th.th_active_in_pool = TRUE;
1800 #ifdef DEBUG_SUSPEND
1803 __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1804 __kmp_printf(
"__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1809 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1810 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1812 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1815 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
1816 __kmp_suspend_template(th_gtid, flag);
1818 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
1819 __kmp_suspend_template(th_gtid, flag);
1821 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
1822 __kmp_suspend_template(th_gtid, flag);
1831 static inline void __kmp_resume_template(
int target_gtid, C *flag )
1833 kmp_info_t *th = __kmp_threads[target_gtid];
1837 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1840 KF_TRACE( 30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1841 KMP_DEBUG_ASSERT( gtid != target_gtid );
1843 __kmp_suspend_initialize_thread( th );
1845 status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1846 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1849 flag = (C *)th->th.th_sleep_loc;
1853 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1854 gtid, target_gtid, NULL ) );
1855 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1856 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1860 typename C::flag_t old_spin = flag->unset_sleeping();
1861 if ( ! flag->is_sleeping_val(old_spin) ) {
1862 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1864 gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1866 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1867 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1870 KF_TRACE( 5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1872 gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1874 TCW_PTR(th->th.th_sleep_loc, NULL);
1877 #ifdef DEBUG_SUSPEND
1880 __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1881 __kmp_printf(
"__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1886 status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1887 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status );
1888 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1889 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1890 KF_TRACE( 30, (
"__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1891 gtid, target_gtid ) );
1894 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
1895 __kmp_resume_template(target_gtid, flag);
1897 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
1898 __kmp_resume_template(target_gtid, flag);
1900 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
1901 __kmp_resume_template(target_gtid, flag);
1905 __kmp_resume_monitor()
1910 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1911 KF_TRACE( 30, (
"__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1912 gtid, KMP_GTID_MONITOR ) );
1913 KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1915 status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1916 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1917 #ifdef DEBUG_SUSPEND
1920 __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1921 __kmp_printf(
"__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1924 status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1925 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status );
1926 status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1927 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1928 KF_TRACE( 30, (
"__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1929 gtid, KMP_GTID_MONITOR ) );
1936 __kmp_yield(
int cond )
1938 if (cond && __kmp_yielding_on) {
1947 __kmp_gtid_set_specific(
int gtid )
1950 KMP_ASSERT( __kmp_init_runtime );
1951 status = pthread_setspecific( __kmp_gtid_threadprivate_key, (
void*)(intptr_t)(gtid+1) );
1952 KMP_CHECK_SYSFAIL(
"pthread_setspecific", status );
1956 __kmp_gtid_get_specific()
1959 if ( !__kmp_init_runtime ) {
1960 KA_TRACE( 50, (
"__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1961 return KMP_GTID_SHUTDOWN;
1963 gtid = (int)(
size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1965 gtid = KMP_GTID_DNE;
1970 KA_TRACE( 50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
1971 __kmp_gtid_threadprivate_key, gtid ));
1979 __kmp_read_cpu_time(
void )
1986 return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
1990 __kmp_read_system_info(
struct kmp_sys_info *info )
1993 struct rusage r_usage;
1995 memset( info, 0,
sizeof( *info ) );
1997 status = getrusage( RUSAGE_SELF, &r_usage);
1998 KMP_CHECK_SYSFAIL_ERRNO(
"getrusage", status );
2000 info->maxrss = r_usage.ru_maxrss;
2001 info->minflt = r_usage.ru_minflt;
2002 info->majflt = r_usage.ru_majflt;
2003 info->nswap = r_usage.ru_nswap;
2004 info->inblock = r_usage.ru_inblock;
2005 info->oublock = r_usage.ru_oublock;
2006 info->nvcsw = r_usage.ru_nvcsw;
2007 info->nivcsw = r_usage.ru_nivcsw;
2009 return (status != 0);
2017 __kmp_read_system_time(
double *delta )
2020 struct timeval tval;
2021 struct timespec stop;
2024 status = gettimeofday( &tval, NULL );
2025 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2026 TIMEVAL_TO_TIMESPEC( &tval, &stop );
2027 t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
2028 *delta = (t_ns * 1e-9);
2032 __kmp_clear_system_time(
void )
2034 struct timeval tval;
2036 status = gettimeofday( &tval, NULL );
2037 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2038 TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2047 __kmp_tv_threadprivate_store( kmp_info_t *th,
void *global_addr,
void *thread_addr )
2051 p = (
struct tv_data *) __kmp_allocate(
sizeof( *p ) );
2053 p->u.tp.global_addr = global_addr;
2054 p->u.tp.thread_addr = thread_addr;
2056 p->type = (
void *) 1;
2058 p->next = th->th.th_local.tv_data;
2059 th->th.th_local.tv_data = p;
2061 if ( p->next == 0 ) {
2062 int rc = pthread_setspecific( __kmp_tv_key, p );
2063 KMP_CHECK_SYSFAIL(
"pthread_setspecific", rc );
2073 __kmp_get_xproc(
void ) {
2079 r = sysconf( _SC_NPROCESSORS_ONLN );
2087 host_basic_info_data_t info;
2088 mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2089 rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2090 if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2093 r = info.avail_cpus;
2095 KMP_WARNING( CantGetNumAvailCPU );
2096 KMP_INFORM( AssumedNumCPU );
2099 #elif KMP_OS_FREEBSD
2101 int mib[] = { CTL_HW, HW_NCPU };
2102 size_t len =
sizeof( r );
2103 if ( sysctl( mib, 2, &r, &len, NULL, 0 ) < 0 ) {
2105 KMP_WARNING( CantGetNumAvailCPU );
2106 KMP_INFORM( AssumedNumCPU );
2111 #error "Unknown or unsupported OS."
2115 return r > 0 ? r : 2;
2120 __kmp_read_from_file(
char const *path,
char const *format, ... )
2125 va_start(args, format);
2126 FILE *f = fopen(path,
"rb");
2129 result = vfscanf(f, format, args);
2136 __kmp_runtime_initialize(
void )
2139 pthread_mutexattr_t mutex_attr;
2140 pthread_condattr_t cond_attr;
2142 if ( __kmp_init_runtime ) {
2146 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2147 if ( ! __kmp_cpuinfo.initialized ) {
2148 __kmp_query_cpuid( &__kmp_cpuinfo );
2152 __kmp_xproc = __kmp_get_xproc();
2154 if ( sysconf( _SC_THREADS ) ) {
2157 __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2158 if ( __kmp_sys_max_nth == -1 ) {
2160 __kmp_sys_max_nth = INT_MAX;
2162 else if ( __kmp_sys_max_nth <= 1 ) {
2164 __kmp_sys_max_nth = KMP_MAX_NTH;
2168 __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2169 if ( __kmp_sys_min_stksize <= 1 ) {
2170 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2175 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2180 int rc = pthread_key_create( & __kmp_tv_key, 0 );
2181 KMP_CHECK_SYSFAIL(
"pthread_key_create", rc );
2185 status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2186 KMP_CHECK_SYSFAIL(
"pthread_key_create", status );
2187 status = pthread_mutexattr_init( & mutex_attr );
2188 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status );
2189 status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2190 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status );
2191 status = pthread_condattr_init( & cond_attr );
2192 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status );
2193 status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2194 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status );
2196 __kmp_itt_initialize();
2199 __kmp_init_runtime = TRUE;
2203 __kmp_runtime_destroy(
void )
2207 if ( ! __kmp_init_runtime ) {
2212 __kmp_itt_destroy();
2215 status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2216 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status );
2218 status = pthread_key_delete( __kmp_tv_key );
2219 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status );
2222 status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2223 if ( status != 0 && status != EBUSY ) {
2224 KMP_SYSFAIL(
"pthread_mutex_destroy", status );
2226 status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2227 if ( status != 0 && status != EBUSY ) {
2228 KMP_SYSFAIL(
"pthread_cond_destroy", status );
2230 #if KMP_AFFINITY_SUPPORTED
2231 __kmp_affinity_uninitialize();
2234 __kmp_init_runtime = FALSE;
2241 __kmp_thread_sleep(
int millis )
2243 sleep( ( millis + 500 ) / 1000 );
2248 __kmp_elapsed(
double *t )
2251 # ifdef FIX_SGI_CLOCK
2254 status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2255 KMP_CHECK_SYSFAIL_ERRNO(
"clock_gettime", status );
2256 *t = (double) ts.tv_nsec * (1.0 / (
double) NSEC_PER_SEC) +
2261 status = gettimeofday( & tv, NULL );
2262 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2263 *t = (double) tv.tv_usec * (1.0 / (
double) USEC_PER_SEC) +
2270 __kmp_elapsed_tick(
double *t )
2272 *t = 1 / (double) CLOCKS_PER_SEC;
2280 __kmp_is_address_mapped(
void * addr ) {
2292 char * name = __kmp_str_format(
"/proc/%d/maps", getpid() );
2295 file = fopen( name,
"r" );
2296 KMP_ASSERT( file != NULL );
2300 void * beginning = NULL;
2301 void * ending = NULL;
2304 rc = fscanf( file,
"%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2308 KMP_ASSERT( rc == 3 && strlen( perms ) == 4 );
2311 if ( ( addr >= beginning ) && ( addr < ending ) ) {
2313 if ( strcmp( perms,
"rw" ) == 0 ) {
2324 KMP_INTERNAL_FREE( name );
2338 (vm_address_t)( addr ),
2340 (vm_address_t)( & buffer ),
2348 #elif KMP_OS_FREEBSD
2355 #error "Unknown or unsupported OS"
2363 #ifdef USE_LOAD_BALANCE
2374 __kmp_get_load_balance(
int max )
2379 int res = getloadavg( averages, 3 );
2384 if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2385 ret_avg = averages[0];
2386 }
else if ( ( __kmp_load_balance_interval >= 180
2387 && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2388 ret_avg = averages[1];
2389 }
else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2390 ret_avg = averages[2];
2404 __kmp_get_load_balance(
int max )
2406 static int permanent_error = 0;
2408 static int glb_running_threads = 0;
2409 static double glb_call_time = 0;
2411 int running_threads = 0;
2413 DIR * proc_dir = NULL;
2414 struct dirent * proc_entry = NULL;
2416 kmp_str_buf_t task_path;
2417 DIR * task_dir = NULL;
2418 struct dirent * task_entry = NULL;
2419 int task_path_fixed_len;
2421 kmp_str_buf_t stat_path;
2423 int stat_path_fixed_len;
2425 int total_processes = 0;
2426 int total_threads = 0;
2428 double call_time = 0.0;
2430 __kmp_str_buf_init( & task_path );
2431 __kmp_str_buf_init( & stat_path );
2433 __kmp_elapsed( & call_time );
2435 if ( glb_call_time &&
2436 ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2437 running_threads = glb_running_threads;
2441 glb_call_time = call_time;
2444 if ( permanent_error ) {
2445 running_threads = -1;
2454 proc_dir = opendir(
"/proc" );
2455 if ( proc_dir == NULL ) {
2458 running_threads = -1;
2459 permanent_error = 1;
2464 __kmp_str_buf_cat( & task_path,
"/proc/", 6 );
2465 task_path_fixed_len = task_path.used;
2467 proc_entry = readdir( proc_dir );
2468 while ( proc_entry != NULL ) {
2471 if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2479 KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name,
"1" ) == 0 );
2482 task_path.used = task_path_fixed_len;
2483 __kmp_str_buf_cat( & task_path, proc_entry->d_name, strlen( proc_entry->d_name ) );
2484 __kmp_str_buf_cat( & task_path,
"/task", 5 );
2486 task_dir = opendir( task_path.str );
2487 if ( task_dir == NULL ) {
2495 if ( strcmp( proc_entry->d_name,
"1" ) == 0 ) {
2496 running_threads = -1;
2497 permanent_error = 1;
2502 __kmp_str_buf_clear( & stat_path );
2503 __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2504 __kmp_str_buf_cat( & stat_path,
"/", 1 );
2505 stat_path_fixed_len = stat_path.used;
2507 task_entry = readdir( task_dir );
2508 while ( task_entry != NULL ) {
2510 if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2517 stat_path.used = stat_path_fixed_len;
2518 __kmp_str_buf_cat( & stat_path, task_entry->d_name, strlen( task_entry->d_name ) );
2519 __kmp_str_buf_cat( & stat_path,
"/stat", 5 );
2523 stat_file = open( stat_path.str, O_RDONLY );
2524 if ( stat_file == -1 ) {
2557 len = read( stat_file, buffer,
sizeof( buffer ) - 1 );
2564 char * close_parent = strstr( buffer,
") " );
2565 if ( close_parent != NULL ) {
2566 char state = * ( close_parent + 2 );
2567 if ( state ==
'R' ) {
2569 if ( running_threads >= max ) {
2579 task_entry = readdir( task_dir );
2581 closedir( task_dir );
2585 proc_entry = readdir( proc_dir );
2593 KMP_DEBUG_ASSERT( running_threads > 0 );
2594 if ( running_threads <= 0 ) {
2595 running_threads = 1;
2599 if ( proc_dir != NULL ) {
2600 closedir( proc_dir );
2602 __kmp_str_buf_free( & task_path );
2603 if ( task_dir != NULL ) {
2604 closedir( task_dir );
2606 __kmp_str_buf_free( & stat_path );
2607 if ( stat_file != -1 ) {
2611 glb_running_threads = running_threads;
2613 return running_threads;
2617 # endif // KMP_OS_DARWIN
2619 #endif // USE_LOAD_BALANCE
2622 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64)
2624 int __kmp_invoke_microtask( microtask_t pkfn,
int gtid,
int tid,
int argc,
2627 int argc_full = argc + 2;
2630 ffi_type *types[argc_full];
2631 void *args[argc_full];
2635 for (i = 0; i < argc_full; i++)
2636 types[i] = &ffi_type_pointer;
2644 for (i = 0; i < argc; i++)
2645 args[2 + i] = &p_argv[i];
2647 if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, argc_full,
2648 &ffi_type_void, types) != FFI_OK)
2651 ffi_call(&cif, (
void (*)(
void))pkfn, NULL, args);
2656 #endif // KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64)
2663 __kmp_invoke_microtask( microtask_t pkfn,
2665 int argc,
void *p_argv[] ) {
2668 fprintf(stderr,
"Too many args to microtask: %d!\n", argc);
2672 (*pkfn)(>id, &tid);
2675 (*pkfn)(>id, &tid, p_argv[0]);
2678 (*pkfn)(>id, &tid, p_argv[0], p_argv[1]);
2681 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]);
2684 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2687 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2690 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2694 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2695 p_argv[5], p_argv[6]);
2698 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2699 p_argv[5], p_argv[6], p_argv[7]);
2702 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2703 p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2706 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2707 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2710 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2711 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2714 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2715 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2719 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2720 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2721 p_argv[11], p_argv[12]);
2724 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2725 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2726 p_argv[11], p_argv[12], p_argv[13]);
2729 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2730 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2731 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.