41 #include "kmp_wait_release.h"
55 enum SYSTEM_INFORMATION_CLASS {
56 SystemProcessInformation = 5
76 SIZE_T PeakVirtualSize;
79 SIZE_T PeakWorkingSetSize;
80 SIZE_T WorkingSetSize;
81 SIZE_T QuotaPeakPagedPoolUsage;
82 SIZE_T QuotaPagedPoolUsage;
83 SIZE_T QuotaPeakNonPagedPoolUsage;
84 SIZE_T QuotaNonPagedPoolUsage;
86 SIZE_T PeakPagefileUsage;
87 SIZE_T PrivatePageCount;
90 struct SYSTEM_THREAD {
91 LARGE_INTEGER KernelTime;
92 LARGE_INTEGER UserTime;
93 LARGE_INTEGER CreateTime;
99 ULONG ContextSwitchCount;
104 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
106 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
107 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 );
109 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
110 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 );
113 struct SYSTEM_PROCESS_INFORMATION {
114 ULONG NextEntryOffset;
115 ULONG NumberOfThreads;
116 LARGE_INTEGER Reserved[ 3 ];
117 LARGE_INTEGER CreateTime;
118 LARGE_INTEGER UserTime;
119 LARGE_INTEGER KernelTime;
120 UNICODE_STRING ImageName;
123 HANDLE ParentProcessId;
125 ULONG Reserved2[ 2 ];
126 VM_COUNTERS VMCounters;
127 IO_COUNTERS IOCounters;
128 SYSTEM_THREAD Threads[ 1 ];
130 typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;
132 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 );
133 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 );
134 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 );
136 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 );
137 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 );
138 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 );
139 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 );
140 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 );
142 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 );
143 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 );
144 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 );
145 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 );
146 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 );
149 typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
150 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
152 HMODULE ntdll = NULL;
157 static HMODULE kernel32 = NULL;
163 #if KMP_HANDLE_SIGNALS
164 typedef void (* sig_func_t )( int );
165 static sig_func_t __kmp_sighldrs[ NSIG ];
166 static int __kmp_siginstalled[ NSIG ];
169 static HANDLE __kmp_monitor_ev;
170 static kmp_int64 __kmp_win32_time;
171 double __kmp_win32_tick;
173 int __kmp_init_runtime = FALSE;
174 CRITICAL_SECTION __kmp_win32_section;
177 __kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
179 InitializeCriticalSection( & mx->cs );
181 __kmp_itt_system_object_created( & mx->cs,
"Critical Section" );
186 __kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
188 DeleteCriticalSection( & mx->cs );
192 __kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
194 EnterCriticalSection( & mx->cs );
198 __kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
200 LeaveCriticalSection( & mx->cs );
204 __kmp_win32_cond_init( kmp_win32_cond_t *cv )
206 cv->waiters_count_ = 0;
207 cv->wait_generation_count_ = 0;
208 cv->release_count_ = 0;
211 __kmp_win32_mutex_init( & cv->waiters_count_lock_ );
214 cv->event_ = CreateEvent( NULL,
219 __kmp_itt_system_object_created( cv->event_,
"Event" );
224 __kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
226 __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
227 __kmp_free_handle( cv->event_ );
228 memset( cv,
'\0',
sizeof( *cv ) );
235 __kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th,
int need_decrease_load )
241 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
244 cv->waiters_count_++;
247 my_generation = cv->wait_generation_count_;
249 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
250 __kmp_win32_mutex_unlock( mx );
257 WaitForSingleObject( cv->event_, INFINITE );
259 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
264 wait_done = ( cv->release_count_ > 0 ) &&
265 ( cv->wait_generation_count_ != my_generation );
267 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_);
275 __kmp_win32_mutex_lock( mx );
276 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
278 cv->waiters_count_--;
279 cv->release_count_--;
281 last_waiter = ( cv->release_count_ == 0 );
283 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
287 ResetEvent( cv->event_ );
292 __kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
294 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
296 if( cv->waiters_count_ > 0 ) {
297 SetEvent( cv->event_ );
300 cv->release_count_ = cv->waiters_count_;
303 cv->wait_generation_count_++;
306 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
310 __kmp_win32_cond_signal( kmp_win32_cond_t *cv )
312 __kmp_win32_cond_broadcast( cv );
319 __kmp_enable(
int new_state )
321 if (__kmp_init_runtime)
322 LeaveCriticalSection( & __kmp_win32_section );
326 __kmp_disable(
int *old_state )
330 if (__kmp_init_runtime)
331 EnterCriticalSection( & __kmp_win32_section );
335 __kmp_suspend_initialize(
void )
341 __kmp_suspend_initialize_thread( kmp_info_t *th )
343 if ( ! TCR_4( th->th.th_suspend_init ) ) {
346 __kmp_win32_cond_init( &th->th.th_suspend_cv );
347 __kmp_win32_mutex_init( &th->th.th_suspend_mx );
348 TCW_4( th->th.th_suspend_init, TRUE );
353 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
355 if ( TCR_4( th->th.th_suspend_init ) ) {
358 __kmp_win32_cond_destroy( & th->th.th_suspend_cv );
359 __kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
360 TCW_4( th->th.th_suspend_init, FALSE );
368 static inline void __kmp_suspend_template(
int th_gtid, C *flag )
370 kmp_info_t *th = __kmp_threads[th_gtid];
372 typename C::flag_t old_spin;
374 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) );
376 __kmp_suspend_initialize_thread( th );
377 __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
379 KF_TRACE( 10, (
"__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n",
380 th_gtid, flag->get() ) );
385 old_spin = flag->set_sleeping();
387 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n",
388 th_gtid, flag->get(), *(flag->get()) ) );
390 if ( flag->done_check_val(old_spin) ) {
391 old_spin = flag->unset_sleeping();
392 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n",
393 th_gtid, flag->get()) );
396 __kmp_suspend_count++;
402 int deactivated = FALSE;
403 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
404 while ( flag->is_sleeping() ) {
405 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n",
408 if ( ! deactivated ) {
409 th->th.th_active = FALSE;
410 if ( th->th.th_active_in_pool ) {
411 th->th.th_active_in_pool = FALSE;
413 (kmp_int32 *) &__kmp_thread_pool_active_nth );
414 KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
419 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
422 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
426 if( flag->is_sleeping() ) {
427 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ));
435 th->th.th_active = TRUE;
436 if ( TCR_4(th->th.th_in_pool) ) {
438 (kmp_int32 *) &__kmp_thread_pool_active_nth );
439 th->th.th_active_in_pool = TRUE;
445 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
447 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid ) );
450 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
451 __kmp_suspend_template(th_gtid, flag);
453 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
454 __kmp_suspend_template(th_gtid, flag);
456 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
457 __kmp_suspend_template(th_gtid, flag);
465 static inline void __kmp_resume_template(
int target_gtid, C *flag )
467 kmp_info_t *th = __kmp_threads[target_gtid];
471 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
474 KF_TRACE( 30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
476 __kmp_suspend_initialize_thread( th );
477 __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
480 flag = (C *)th->th.th_sleep_loc;
484 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n",
485 gtid, target_gtid, NULL ) );
486 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
490 typename C::flag_t old_spin = flag->unset_sleeping();
491 if ( !flag->is_sleeping_val(old_spin) ) {
492 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): "
494 gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) );
495 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
499 TCW_PTR(th->th.th_sleep_loc, NULL);
501 KF_TRACE( 5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n",
502 gtid, target_gtid, flag->get() ) );
505 __kmp_win32_cond_signal( &th->th.th_suspend_cv );
506 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
508 KF_TRACE( 30, (
"__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
509 gtid, target_gtid ) );
512 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
513 __kmp_resume_template(target_gtid, flag);
515 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
516 __kmp_resume_template(target_gtid, flag);
518 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
519 __kmp_resume_template(target_gtid, flag);
527 __kmp_yield(
int cond )
537 __kmp_gtid_set_specific(
int gtid )
539 KA_TRACE( 50, (
"__kmp_gtid_set_specific: T#%d key:%d\n",
540 gtid, __kmp_gtid_threadprivate_key ));
541 KMP_ASSERT( __kmp_init_runtime );
542 if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
543 KMP_FATAL( TLSSetValueFailed );
547 __kmp_gtid_get_specific()
550 if( !__kmp_init_runtime ) {
551 KA_TRACE( 50, (
"__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
552 return KMP_GTID_SHUTDOWN;
554 gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
561 KA_TRACE( 50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
562 __kmp_gtid_threadprivate_key, gtid ));
576 __kmp_get_proc_group( kmp_affin_mask_t
const *mask )
580 for (i = 0; i < __kmp_num_proc_groups; i++) {
595 __kmp_set_system_affinity( kmp_affin_mask_t
const *mask,
int abort_on_error )
600 if (__kmp_num_proc_groups > 1) {
605 int group = __kmp_get_proc_group( mask );
607 if (abort_on_error) {
608 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
618 ga.Mask = mask[group];
619 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
621 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
622 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
623 DWORD error = GetLastError();
624 if (abort_on_error) {
627 KMP_MSG( CantSetThreadAffMask ),
640 if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
641 DWORD error = GetLastError();
642 if (abort_on_error) {
645 KMP_MSG( CantSetThreadAffMask ),
657 __kmp_get_system_affinity( kmp_affin_mask_t *mask,
int abort_on_error )
662 if (__kmp_num_proc_groups > 1) {
665 KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
667 if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
668 DWORD error = GetLastError();
669 if (abort_on_error) {
672 KMP_MSG(FunctionError,
"GetThreadGroupAffinity()"),
680 if ((ga.Group < 0) || (ga.Group > __kmp_num_proc_groups)
685 mask[ga.Group] = ga.Mask;
692 kmp_affin_mask_t newMask, sysMask, retval;
694 if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
695 DWORD error = GetLastError();
696 if (abort_on_error) {
699 KMP_MSG(FunctionError,
"GetProcessAffinityMask()"),
706 retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
708 DWORD error = GetLastError();
709 if (abort_on_error) {
712 KMP_MSG(FunctionError,
"SetThreadAffinityMask()"),
719 newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
721 DWORD error = GetLastError();
722 if (abort_on_error) {
725 KMP_MSG(FunctionError,
"SetThreadAffinityMask()"),
737 __kmp_affinity_bind_thread(
int proc )
742 if (__kmp_num_proc_groups > 1) {
748 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
749 * CHAR_BIT *
sizeof(DWORD_PTR))));
750 ga.Group = proc / (CHAR_BIT *
sizeof(DWORD_PTR));
751 ga.Mask = (
unsigned long long)1 << (proc % (CHAR_BIT *
sizeof(DWORD_PTR)));
752 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
754 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
755 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
756 DWORD error = GetLastError();
757 if (__kmp_affinity_verbose) {
760 KMP_MSG( CantSetThreadAffMask ),
772 kmp_affin_mask_t mask;
774 KMP_CPU_SET(proc, &mask);
775 __kmp_set_system_affinity(&mask, TRUE);
780 __kmp_affinity_determine_capable(
const char *env_var )
787 __kmp_affin_mask_size = __kmp_num_proc_groups *
sizeof(kmp_affin_mask_t);
789 __kmp_affin_mask_size =
sizeof(kmp_affin_mask_t);
793 "__kmp_affinity_determine_capable: "
794 "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC
").\n",
795 __kmp_affin_mask_size
800 __kmp_read_cpu_time(
void )
802 FILETIME CreationTime, ExitTime, KernelTime, UserTime;
808 status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
809 &ExitTime, &KernelTime, &UserTime );
814 sec += KernelTime.dwHighDateTime;
815 sec += UserTime.dwHighDateTime;
818 sec *= (double) (1 << 16) * (double) (1 << 16);
820 sec += KernelTime.dwLowDateTime;
821 sec += UserTime.dwLowDateTime;
823 cpu_time += (sec * 100.0) / NSEC_PER_SEC;
830 __kmp_read_system_info(
struct kmp_sys_info *info )
849 __kmp_runtime_initialize(
void )
855 if ( __kmp_init_runtime ) {
859 InitializeCriticalSection( & __kmp_win32_section );
861 __kmp_itt_system_object_created( & __kmp_win32_section,
"Critical Section" );
863 __kmp_initialize_system_tick();
865 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
866 if ( ! __kmp_cpuinfo.initialized ) {
867 __kmp_query_cpuid( & __kmp_cpuinfo );
872 #if KMP_OS_WINDOWS && ! defined GUIDEDLL_EXPORTS
887 __kmp_tls_gtid_min = 0;
889 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
893 if ( !__kmp_gtid_threadprivate_key ) {
894 __kmp_gtid_threadprivate_key = TlsAlloc();
895 if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
896 KMP_FATAL( TLSOutOfIndexes );
911 __kmp_str_buf_init( & path );
912 path_size = GetSystemDirectory( path.str, path.size );
913 KMP_DEBUG_ASSERT( path_size > 0 );
914 if ( path_size >= path.size ) {
918 __kmp_str_buf_reserve( & path, path_size );
919 path_size = GetSystemDirectory( path.str, path.size );
920 KMP_DEBUG_ASSERT( path_size > 0 );
922 if ( path_size > 0 && path_size < path.size ) {
927 path.used = path_size;
928 __kmp_str_buf_print( & path,
"\\%s",
"ntdll.dll" );
933 ntdll = GetModuleHandle( path.str );
936 KMP_DEBUG_ASSERT( ntdll != NULL );
937 if ( ntdll != NULL ) {
938 NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll,
"NtQuerySystemInformation" );
940 KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );
947 if ( path_size > 0 && path_size < path.size ) {
952 path.used = path_size;
953 __kmp_str_buf_print( & path,
"\\%s",
"kernel32.dll" );
958 kernel32 = GetModuleHandle( path.str );
964 if ( kernel32 != NULL ) {
965 __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32,
"GetActiveProcessorCount" );
966 __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32,
"GetActiveProcessorGroupCount" );
967 __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32,
"GetThreadGroupAffinity" );
968 __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32,
"SetThreadGroupAffinity" );
977 if ( ( __kmp_GetActiveProcessorCount != NULL )
978 && ( __kmp_GetActiveProcessorGroupCount != NULL )
979 && ( __kmp_GetThreadGroupAffinity != NULL )
980 && ( __kmp_SetThreadGroupAffinity != NULL )
981 && ( ( __kmp_num_proc_groups
982 = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
988 KA_TRACE( 10, (
"__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
992 for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
993 DWORD size = __kmp_GetActiveProcessorCount( i );
995 KA_TRACE( 20, (
"__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
1000 if ( __kmp_num_proc_groups <= 1 ) {
1001 GetSystemInfo( & info );
1002 __kmp_xproc = info.dwNumberOfProcessors;
1005 GetSystemInfo( & info );
1006 __kmp_xproc = info.dwNumberOfProcessors;
1007 #endif // KMP_ARCH_X86_64
1013 if ( __kmp_xproc <= 0 ) {
1017 KA_TRACE( 5, (
"__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );
1019 __kmp_str_buf_free( & path );
1022 __kmp_itt_initialize();
1025 __kmp_init_runtime = TRUE;
1029 __kmp_runtime_destroy(
void )
1031 if ( ! __kmp_init_runtime ) {
1036 __kmp_itt_destroy();
1041 KA_TRACE( 40, (
"__kmp_runtime_destroy\n" ));
1043 if( __kmp_gtid_threadprivate_key ) {
1044 TlsFree( __kmp_gtid_threadprivate_key );
1045 __kmp_gtid_threadprivate_key = 0;
1048 __kmp_affinity_uninitialize();
1049 DeleteCriticalSection( & __kmp_win32_section );
1052 NtQuerySystemInformation = NULL;
1056 __kmp_GetActiveProcessorCount = NULL;
1057 __kmp_GetActiveProcessorGroupCount = NULL;
1058 __kmp_GetThreadGroupAffinity = NULL;
1059 __kmp_SetThreadGroupAffinity = NULL;
1060 #endif // KMP_ARCH_X86_64
1062 __kmp_init_runtime = FALSE;
1067 __kmp_terminate_thread(
int gtid )
1069 kmp_info_t *th = __kmp_threads[ gtid ];
1073 KA_TRACE( 10, (
"__kmp_terminate_thread: kill (%d)\n", gtid ) );
1075 if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
1078 __kmp_free_handle( th->th.th_info.ds.ds_thread );
1085 __kmp_clear_system_time(
void )
1089 status = QueryPerformanceCounter( & time );
1090 __kmp_win32_time = (kmp_int64) time.QuadPart;
1094 __kmp_initialize_system_tick(
void )
1100 status = QueryPerformanceFrequency( & freq );
1102 DWORD error = GetLastError();
1105 KMP_MSG( FunctionError,
"QueryPerformanceFrequency()" ),
1112 __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
1120 __kmp_elapsed(
double *t )
1124 status = QueryPerformanceCounter( & now );
1125 *t = ((double) now.QuadPart) * __kmp_win32_tick;
1131 __kmp_elapsed_tick(
double *t )
1133 *t = __kmp_win32_tick;
1137 __kmp_read_system_time(
double *delta )
1140 if (delta != NULL) {
1144 status = QueryPerformanceCounter( & now );
1146 *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
1161 __kmp_change_thread_affinity_mask(
int gtid, kmp_affin_mask_t *new_mask,
1162 kmp_affin_mask_t *old_mask )
1164 kmp_info_t *th = __kmp_threads[ gtid ];
1166 KMP_DEBUG_ASSERT( *new_mask != 0 );
1168 if ( old_mask != NULL ) {
1169 *old_mask = SetThreadAffinityMask( th -> th.th_info.ds.ds_thread, *new_mask );
1172 DWORD error = GetLastError();
1175 KMP_MSG( CantSetThreadAffMask ),
1181 if (__kmp_affinity_verbose)
1182 KMP_INFORM( ChangeAffMask,
"KMP_AFFINITY (Bind)", gtid, *old_mask, *new_mask );
1185 KMP_DEBUG_ASSERT( old_mask != NULL && *old_mask == *(th -> th.th_affin_mask ));
1187 KMP_CPU_COPY(th -> th.th_affin_mask, new_mask);
1195 __kmp_launch_worker(
void *arg )
1197 volatile void *stack_data;
1200 kmp_info_t *this_thr = (kmp_info_t *) arg;
1203 gtid = this_thr->th.th_info.ds.ds_gtid;
1204 __kmp_gtid_set_specific( gtid );
1205 #ifdef KMP_TDATA_GTID
1206 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1207 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1208 "reference: http://support.microsoft.com/kb/118816"
1213 __kmp_itt_thread_name( gtid );
1216 __kmp_affinity_set_init_mask( gtid, FALSE );
1218 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1223 __kmp_clear_x87_fpu_status_word();
1224 __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
1225 __kmp_load_mxcsr( &__kmp_init_mxcsr );
1228 if ( __kmp_stkoffset > 0 && gtid > 0 ) {
1229 padding = _alloca( gtid * __kmp_stkoffset );
1232 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1233 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1234 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1236 if ( TCR_4(__kmp_gtid_mode) < 2 ) {
1237 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
1238 KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
1239 __kmp_check_stack_overlap( this_thr );
1242 exit_val = __kmp_launch_thread( this_thr );
1243 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1244 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1253 __kmp_launch_monitor(
void *arg )
1256 kmp_thread_t monitor;
1259 kmp_info_t *this_thr = (kmp_info_t *) arg;
1261 KMP_DEBUG_ASSERT(__kmp_init_monitor);
1262 TCW_4( __kmp_init_monitor, 2 );
1264 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1265 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1268 KA_TRACE( 10, (
"__kmp_launch_monitor: launched\n" ) );
1270 monitor = GetCurrentThread();
1273 status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
1275 DWORD error = GetLastError();
1278 KMP_MSG( CantSetThreadPriority ),
1285 __kmp_gtid_set_specific( KMP_GTID_MONITOR );
1286 #ifdef KMP_TDATA_GTID
1287 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1288 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1289 "reference: http://support.microsoft.com/kb/118816"
1294 __kmp_itt_thread_ignore();
1299 interval = ( 1000 / __kmp_monitor_wakeups );
1301 while (! TCR_4(__kmp_global.g.g_done)) {
1304 KA_TRACE( 15, (
"__kmp_launch_monitor: update\n" ) );
1306 wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );
1308 if (wait_status == WAIT_TIMEOUT) {
1309 TCW_4( __kmp_global.g.g_time.dt.t_value,
1310 TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
1316 KA_TRACE( 10, (
"__kmp_launch_monitor: finished\n" ) );
1318 status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
1320 DWORD error = GetLastError();
1323 KMP_MSG( CantSetThreadPriority ),
1329 if (__kmp_global.g.g_abort != 0) {
1335 KA_TRACE( 10, (
"__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );
1340 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1341 __kmp_terminate_thread( gtid );
1347 KA_TRACE( 10, (
"__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );
1349 if (__kmp_global.g.g_abort > 0) {
1350 raise( __kmp_global.g.g_abort );
1354 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1361 __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size )
1363 kmp_thread_t handle;
1366 KA_TRACE( 10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1368 th->th.th_info.ds.ds_gtid = gtid;
1370 if ( KMP_UBER_GTID(gtid) ) {
1379 rc = DuplicateHandle(
1380 GetCurrentProcess(),
1382 GetCurrentProcess(),
1383 &th->th.th_info.ds.ds_thread,
1386 DUPLICATE_SAME_ACCESS
1389 KA_TRACE( 10, (
" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC
"\n",
1391 th->th.th_info.ds.ds_thread ) );
1392 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1394 if ( TCR_4(__kmp_gtid_mode) < 2 ) {
1396 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1397 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1398 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1399 __kmp_check_stack_overlap( th );
1406 KA_TRACE( 10, (
"__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
1407 " bytes\n", stack_size ) );
1409 stack_size += gtid * __kmp_stkoffset;
1411 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1412 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1414 KA_TRACE( 10, (
"__kmp_create_worker: (before) stack_size = %"
1416 " bytes, &__kmp_launch_worker = %p, th = %p, "
1418 (SIZE_T) stack_size,
1419 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1420 (LPVOID) th, &idThread ) );
1423 handle = CreateThread( NULL, (SIZE_T) stack_size,
1424 (LPTHREAD_START_ROUTINE) __kmp_launch_worker,
1425 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1428 KA_TRACE( 10, (
"__kmp_create_worker: (after) stack_size = %"
1430 " bytes, &__kmp_launch_worker = %p, th = %p, "
1431 "idThread = %u, handle = %" KMP_UINTPTR_SPEC
"\n",
1432 (SIZE_T) stack_size,
1433 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1434 (LPVOID) th, idThread, handle ) );
1437 if ( handle == 0 ) {
1438 DWORD error = GetLastError();
1441 KMP_MSG( CantCreateThread ),
1446 th->th.th_info.ds.ds_thread = handle;
1452 KA_TRACE( 10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1456 __kmp_still_running(kmp_info_t *th) {
1457 return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
1461 __kmp_create_monitor( kmp_info_t *th )
1463 kmp_thread_t handle;
1465 int ideal, new_ideal;
1466 int caller_gtid = __kmp_get_gtid();
1468 KA_TRACE( 10, (
"__kmp_create_monitor: try to create monitor\n" ) );
1472 __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
1473 if ( __kmp_monitor_ev == NULL ) {
1474 DWORD error = GetLastError();
1477 KMP_MSG( CantCreateEvent ),
1483 __kmp_itt_system_object_created( __kmp_monitor_ev,
"Event" );
1486 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1487 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1491 if ( __kmp_monitor_stksize == 0 ) {
1492 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1494 if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1495 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1498 KA_TRACE( 10, (
"__kmp_create_monitor: requested stacksize = %d bytes\n",
1499 (
int) __kmp_monitor_stksize ) );
1501 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1503 handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
1504 (LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
1505 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1507 DWORD error = GetLastError();
1510 KMP_MSG( CantCreateThread ),
1516 th->th.th_info.ds.ds_thread = handle;
1520 KA_TRACE( 10, (
"__kmp_create_monitor: monitor created %p\n",
1521 (
void *) th->th.th_info.ds.ds_thread ) );
1534 __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
1537 rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
1539 DWORD error = GetLastError();
1542 KMP_MSG( FunctionError,
"GetExitCodeThread()" ),
1547 return ( *exit_val == STILL_ACTIVE );
1555 ExitThread( exit_status );
1562 __kmp_reap_common( kmp_info_t * th )
1568 KA_TRACE( 10, (
"__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );
1588 register kmp_uint32 spins;
1590 KMP_FSYNC_SPIN_INIT( obj, (
void*) & th->th.th_info.ds.ds_alive );
1592 KMP_INIT_YIELD( spins );
1595 KMP_FSYNC_SPIN_PREPARE( obj );
1597 __kmp_is_thread_alive( th, &exit_val );
1598 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
1599 KMP_YIELD_SPIN( spins );
1600 }
while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
1602 if ( exit_val == STILL_ACTIVE ) {
1603 KMP_FSYNC_CANCEL( obj );
1605 KMP_FSYNC_SPIN_ACQUIRED( obj );
1610 __kmp_free_handle( th->th.th_info.ds.ds_thread );
1617 if ( exit_val == STILL_ACTIVE ) {
1618 KA_TRACE( 1, (
"__kmp_reap_common: thread still active.\n" ) );
1619 }
else if ( (
void *) exit_val != (
void *) th) {
1620 KA_TRACE( 1, (
"__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
1625 "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
"\n",
1626 th->th.th_info.ds.ds_gtid,
1627 th->th.th_info.ds.ds_thread
1631 th->th.th_info.ds.ds_thread = 0;
1632 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1633 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1634 th->th.th_info.ds.ds_thread_id = 0;
1640 __kmp_reap_monitor( kmp_info_t *th )
1644 KA_TRACE( 10, (
"__kmp_reap_monitor: try to reap %p\n",
1645 (
void *) th->th.th_info.ds.ds_thread ) );
1650 KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1651 if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1657 status = SetEvent( __kmp_monitor_ev );
1658 if ( status == FALSE ) {
1659 DWORD error = GetLastError();
1662 KMP_MSG( CantSetEvent ),
1667 KA_TRACE( 10, (
"__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1668 __kmp_reap_common( th );
1670 __kmp_free_handle( __kmp_monitor_ev );
1676 __kmp_reap_worker( kmp_info_t * th )
1678 KA_TRACE( 10, (
"__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1679 __kmp_reap_common( th );
1685 #if KMP_HANDLE_SIGNALS
1689 __kmp_team_handler(
int signo )
1691 if ( __kmp_global.g.g_abort == 0 ) {
1693 if ( __kmp_debug_buf ) {
1694 __kmp_dump_debug_buffer();
1697 TCW_4( __kmp_global.g.g_abort, signo );
1699 TCW_4( __kmp_global.g.g_done, TRUE );
1707 sig_func_t __kmp_signal(
int signum, sig_func_t handler ) {
1708 sig_func_t old = signal( signum, handler );
1709 if ( old == SIG_ERR ) {
1711 __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError,
"signal" ), KMP_ERR( error ), __kmp_msg_null );
1717 __kmp_install_one_handler(
1724 KB_TRACE( 60, (
"__kmp_install_one_handler: called: sig=%d\n", sig ) );
1725 if ( parallel_init ) {
1726 old = __kmp_signal( sig, handler );
1728 if ( old == __kmp_sighldrs[ sig ] ) {
1729 __kmp_siginstalled[ sig ] = 1;
1732 old = __kmp_signal( sig, old );
1738 old = __kmp_signal( sig, SIG_DFL );
1739 __kmp_sighldrs[ sig ] = old;
1740 __kmp_signal( sig, old );
1746 __kmp_remove_one_handler(
int sig ) {
1747 if ( __kmp_siginstalled[ sig ] ) {
1750 KB_TRACE( 60, (
"__kmp_remove_one_handler: called: sig=%d\n", sig ) );
1751 old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
1752 if ( old != __kmp_team_handler ) {
1753 KB_TRACE( 10, (
"__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1754 old = __kmp_signal( sig, old );
1756 __kmp_sighldrs[ sig ] = NULL;
1757 __kmp_siginstalled[ sig ] = 0;
1764 __kmp_install_signals(
int parallel_init )
1766 KB_TRACE( 10, (
"__kmp_install_signals: called\n" ) );
1767 if ( ! __kmp_handle_signals ) {
1768 KB_TRACE( 10, (
"__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
1771 __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1772 __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1773 __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1774 __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1775 __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1776 __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1781 __kmp_remove_signals(
void )
1784 KB_TRACE( 10, (
"__kmp_remove_signals: called\n" ) );
1785 for ( sig = 1; sig < NSIG; ++ sig ) {
1786 __kmp_remove_one_handler( sig );
1791 #endif // KMP_HANDLE_SIGNALS
1795 __kmp_thread_sleep(
int millis )
1799 status = SleepEx( (DWORD) millis, FALSE );
1801 DWORD error = GetLastError();
1804 KMP_MSG( FunctionError,
"SleepEx()" ),
1813 __kmp_is_address_mapped(
void * addr )
1816 MEMORY_BASIC_INFORMATION lpBuffer;
1819 dwLength =
sizeof(MEMORY_BASIC_INFORMATION);
1821 status = VirtualQuery( addr, &lpBuffer, dwLength );
1823 return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
1824 (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
1828 __kmp_hardware_timestamp(
void)
1832 QueryPerformanceCounter((LARGE_INTEGER*) &r);
1838 __kmp_free_handle( kmp_thread_t tHandle )
1842 rc = CloseHandle( tHandle );
1844 DWORD error = GetLastError();
1847 KMP_MSG( CantCloseHandle ),
1855 __kmp_get_load_balance(
int max ) {
1857 static ULONG glb_buff_size = 100 * 1024;
1859 static int glb_running_threads = 0;
1860 static double glb_call_time = 0;
1862 int running_threads = 0;
1863 NTSTATUS status = 0;
1864 ULONG buff_size = 0;
1865 ULONG info_size = 0;
1866 void * buffer = NULL;
1867 PSYSTEM_PROCESS_INFORMATION spi = NULL;
1870 double call_time = 0.0;
1872 __kmp_elapsed( & call_time );
1874 if ( glb_call_time &&
1875 ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
1876 running_threads = glb_running_threads;
1879 glb_call_time = call_time;
1882 if ( NtQuerySystemInformation == NULL ) {
1883 running_threads = -1;
1894 buff_size = glb_buff_size;
1896 buff_size = 2 * buff_size;
1899 buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
1900 if ( buffer == NULL ) {
1901 running_threads = -1;
1904 status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
1907 }
while ( status == STATUS_INFO_LENGTH_MISMATCH );
1908 glb_buff_size = buff_size;
1910 #define CHECK( cond ) \
1912 KMP_DEBUG_ASSERT( cond ); \
1913 if ( ! ( cond ) ) { \
1914 running_threads = -1; \
1919 CHECK( buff_size >= info_size );
1920 spi = PSYSTEM_PROCESS_INFORMATION( buffer );
1922 ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
1923 CHECK( 0 <= offset && offset +
sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
1924 HANDLE pid = spi->ProcessId;
1925 ULONG num = spi->NumberOfThreads;
1927 size_t spi_size =
sizeof( SYSTEM_PROCESS_INFORMATION ) +
sizeof( SYSTEM_THREAD ) * ( num - 1 );
1928 CHECK( offset + spi_size < info_size );
1929 if ( spi->NextEntryOffset != 0 ) {
1930 CHECK( spi_size <= spi->NextEntryOffset );
1935 for (
int i = 0; i < num; ++ i ) {
1936 THREAD_STATE state = spi->Threads[ i ].State;
1939 if ( state == StateRunning ) {
1943 if ( running_threads >= max ) {
1949 if ( spi->NextEntryOffset == 0 ) {
1952 spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
1959 if ( buffer != NULL ) {
1960 KMP_INTERNAL_FREE( buffer );
1963 glb_running_threads = running_threads;
1965 return running_threads;