Intel® OpenMP* Runtime Library
 All Classes Functions Variables Typedefs Enumerations Enumerator Groups Pages
z_Windows_NT_util.c
1 /*
2  * z_Windows_NT_util.c -- platform specific routines.
3  * $Revision: 42816 $
4  * $Date: 2013-11-11 15:33:37 -0600 (Mon, 11 Nov 2013) $
5  */
6 
7 /* <copyright>
8  Copyright (c) 1997-2013 Intel Corporation. All Rights Reserved.
9 
10  Redistribution and use in source and binary forms, with or without
11  modification, are permitted provided that the following conditions
12  are met:
13 
14  * Redistributions of source code must retain the above copyright
15  notice, this list of conditions and the following disclaimer.
16  * Redistributions in binary form must reproduce the above copyright
17  notice, this list of conditions and the following disclaimer in the
18  documentation and/or other materials provided with the distribution.
19  * Neither the name of Intel Corporation nor the names of its
20  contributors may be used to endorse or promote products derived
21  from this software without specific prior written permission.
22 
23  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 
35 </copyright> */
36 
37 #include "kmp.h"
38 #include "kmp_itt.h"
39 #include "kmp_i18n.h"
40 #include "kmp_io.h"
41 
42 
43 
44 /* ----------------------------------------------------------------------------------- */
45 /* ----------------------------------------------------------------------------------- */
46 
47 /* This code is related to NtQuerySystemInformation() function. This function
48  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
49  number of running threads in the system. */
50 
51 #include <ntstatus.h>
52 #include <ntsecapi.h> // UNICODE_STRING
53 
54 enum SYSTEM_INFORMATION_CLASS {
55  SystemProcessInformation = 5
56 }; // SYSTEM_INFORMATION_CLASS
57 
58 struct CLIENT_ID {
59  HANDLE UniqueProcess;
60  HANDLE UniqueThread;
61 }; // struct CLIENT_ID
62 
63 enum THREAD_STATE {
64  StateInitialized,
65  StateReady,
66  StateRunning,
67  StateStandby,
68  StateTerminated,
69  StateWait,
70  StateTransition,
71  StateUnknown
72 }; // enum THREAD_STATE
73 
74 struct VM_COUNTERS {
75  SIZE_T PeakVirtualSize;
76  SIZE_T VirtualSize;
77  ULONG PageFaultCount;
78  SIZE_T PeakWorkingSetSize;
79  SIZE_T WorkingSetSize;
80  SIZE_T QuotaPeakPagedPoolUsage;
81  SIZE_T QuotaPagedPoolUsage;
82  SIZE_T QuotaPeakNonPagedPoolUsage;
83  SIZE_T QuotaNonPagedPoolUsage;
84  SIZE_T PagefileUsage;
85  SIZE_T PeakPagefileUsage;
86  SIZE_T PrivatePageCount;
87 }; // struct VM_COUNTERS
88 
89 struct SYSTEM_THREAD {
90  LARGE_INTEGER KernelTime;
91  LARGE_INTEGER UserTime;
92  LARGE_INTEGER CreateTime;
93  ULONG WaitTime;
94  LPVOID StartAddress;
95  CLIENT_ID ClientId;
96  DWORD Priority;
97  LONG BasePriority;
98  ULONG ContextSwitchCount;
99  THREAD_STATE State;
100  ULONG WaitReason;
101 }; // SYSTEM_THREAD
102 
103 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
104 #if KMP_ARCH_X86
105  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
106  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 );
107 #else
108  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
109  KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 );
110 #endif
111 
112 struct SYSTEM_PROCESS_INFORMATION {
113  ULONG NextEntryOffset;
114  ULONG NumberOfThreads;
115  LARGE_INTEGER Reserved[ 3 ];
116  LARGE_INTEGER CreateTime;
117  LARGE_INTEGER UserTime;
118  LARGE_INTEGER KernelTime;
119  UNICODE_STRING ImageName;
120  DWORD BasePriority;
121  HANDLE ProcessId;
122  HANDLE ParentProcessId;
123  ULONG HandleCount;
124  ULONG Reserved2[ 2 ];
125  VM_COUNTERS VMCounters;
126  IO_COUNTERS IOCounters;
127  SYSTEM_THREAD Threads[ 1 ];
128 }; // SYSTEM_PROCESS_INFORMATION
129 typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;
130 
131 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 );
132 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 );
133 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 );
134 #if KMP_ARCH_X86
135  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 );
136  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 );
137  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 );
138  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 );
139  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 );
140 #else
141  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 );
142  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 );
143  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 );
144  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 );
145  KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 );
146 #endif
147 
148 typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
149 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
150 
151 HMODULE ntdll = NULL;
152 
153 /* End of NtQuerySystemInformation()-related code */
154 
155 #if KMP_ARCH_X86_64
156 static HMODULE kernel32 = NULL;
157 #endif /* KMP_ARCH_X86_64 */
158 
159 /* ----------------------------------------------------------------------------------- */
160 /* ----------------------------------------------------------------------------------- */
161 
162 
163 // Why do we have multiple copies of __kmp_static_delay() and __kmp_static_yield() in many files?
164 #ifdef KMP_DEBUG
165 
166 static void
167 __kmp_static_delay( int arg ) {
168  /* Work around weird code-gen bug that causes assert to trip */
169  #if KMP_ARCH_X86_64 && KMP_OS_LINUX
170  KMP_ASSERT( arg != 0 );
171  #else
172  KMP_ASSERT( arg >= 0 );
173  #endif
174 }
175 
176 #else
177 
178  #define __kmp_static_delay( arg ) /* nothing to do */
179 
180 #endif /* KMP_DEBUG */
181 
182 static void
183 __kmp_static_yield( int arg )
184 {
185  __kmp_yield( arg );
186 }
187 
188 #if KMP_HANDLE_SIGNALS
189  typedef void (* sig_func_t )( int );
190  static sig_func_t __kmp_sighldrs[ NSIG ];
191  static int __kmp_siginstalled[ NSIG ];
192 #endif
193 
194 static HANDLE __kmp_monitor_ev;
195 static kmp_int64 __kmp_win32_time;
196 double __kmp_win32_tick;
197 
198 int __kmp_init_runtime = FALSE;
199 CRITICAL_SECTION __kmp_win32_section;
200 
201 void
202 __kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
203 {
204  InitializeCriticalSection( & mx->cs );
205 #if USE_ITT_BUILD
206  __kmp_itt_system_object_created( & mx->cs, "Critical Section" );
207 #endif /* USE_ITT_BUILD */
208 }
209 
210 void
211 __kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
212 {
213  DeleteCriticalSection( & mx->cs );
214 }
215 
216 void
217 __kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
218 {
219  EnterCriticalSection( & mx->cs );
220 }
221 
222 void
223 __kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
224 {
225  LeaveCriticalSection( & mx->cs );
226 }
227 
228 void
229 __kmp_win32_cond_init( kmp_win32_cond_t *cv )
230 {
231  cv->waiters_count_ = 0;
232  cv->wait_generation_count_ = 0;
233  cv->release_count_ = 0;
234 
235  /* Initialize the critical section */
236  __kmp_win32_mutex_init( & cv->waiters_count_lock_ );
237 
238  /* Create a manual-reset event. */
239  cv->event_ = CreateEvent( NULL, // no security
240  TRUE, // manual-reset
241  FALSE, // non-signaled initially
242  NULL ); // unnamed
243 #if USE_ITT_BUILD
244  __kmp_itt_system_object_created( cv->event_, "Event" );
245 #endif /* USE_ITT_BUILD */
246 }
247 
248 void
249 __kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
250 {
251  __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
252  __kmp_free_handle( cv->event_ );
253  memset( cv, '\0', sizeof( *cv ) );
254 }
255 
256 /* TODO associate cv with a team instead of a thread so as to optimize
257  * the case where we wake up a whole team */
258 
259 void
260 __kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th, int need_decrease_load )
261 {
262  int my_generation;
263  int last_waiter;
264 
265  /* Avoid race conditions */
266  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
267 
268  /* Increment count of waiters */
269  cv->waiters_count_++;
270 
271  /* Store current generation in our activation record. */
272  my_generation = cv->wait_generation_count_;
273 
274  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
275  __kmp_win32_mutex_unlock( mx );
276 
277 
278  for (;;) {
279  int wait_done;
280 
281  /* Wait until the event is signaled */
282  WaitForSingleObject( cv->event_, INFINITE );
283 
284  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
285 
286  /* Exit the loop when the <cv->event_> is signaled and
287  * there are still waiting threads from this <wait_generation>
288  * that haven't been released from this wait yet. */
289  wait_done = ( cv->release_count_ > 0 ) &&
290  ( cv->wait_generation_count_ != my_generation );
291 
292  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_);
293 
294  /* there used to be a semicolon after the if statement,
295  * it looked like a bug, so i removed it */
296  if( wait_done )
297  break;
298  }
299 
300  __kmp_win32_mutex_lock( mx );
301  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
302 
303  cv->waiters_count_--;
304  cv->release_count_--;
305 
306  last_waiter = ( cv->release_count_ == 0 );
307 
308  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
309 
310  if( last_waiter ) {
311  /* We're the last waiter to be notified, so reset the manual event. */
312  ResetEvent( cv->event_ );
313  }
314 }
315 
316 void
317 __kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
318 {
319  __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
320 
321  if( cv->waiters_count_ > 0 ) {
322  SetEvent( cv->event_ );
323  /* Release all the threads in this generation. */
324 
325  cv->release_count_ = cv->waiters_count_;
326 
327  /* Start a new generation. */
328  cv->wait_generation_count_++;
329  }
330 
331  __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
332 }
333 
334 void
335 __kmp_win32_cond_signal( kmp_win32_cond_t *cv )
336 {
337  __kmp_win32_cond_broadcast( cv );
338 }
339 
340 /* ------------------------------------------------------------------------ */
341 /* ------------------------------------------------------------------------ */
342 
343 void
344 __kmp_enable( int new_state )
345 {
346  if (__kmp_init_runtime)
347  LeaveCriticalSection( & __kmp_win32_section );
348 }
349 
350 void
351 __kmp_disable( int *old_state )
352 {
353  *old_state = 0;
354 
355  if (__kmp_init_runtime)
356  EnterCriticalSection( & __kmp_win32_section );
357 }
358 
359 void
360 __kmp_suspend_initialize( void )
361 {
362  /* do nothing */
363 }
364 
365 static void
366 __kmp_suspend_initialize_thread( kmp_info_t *th )
367 {
368  if ( ! TCR_4( th->th.th_suspend_init ) ) {
369  /* this means we haven't initialized the suspension pthread objects for this thread
370  in this instance of the process */
371  __kmp_win32_cond_init( &th->th.th_suspend_cv );
372  __kmp_win32_mutex_init( &th->th.th_suspend_mx );
373  TCW_4( th->th.th_suspend_init, TRUE );
374  }
375 }
376 
377 void
378 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
379 {
380  if ( TCR_4( th->th.th_suspend_init ) ) {
381  /* this means we have initialize the suspension pthread objects for this thread
382  in this instance of the process */
383  __kmp_win32_cond_destroy( & th->th.th_suspend_cv );
384  __kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
385  TCW_4( th->th.th_suspend_init, FALSE );
386  }
387 }
388 
389 /*
390  * This routine puts the calling thread to sleep after setting the
391  * sleep bit for the indicated spin variable to true.
392  */
393 
394 void
395 __kmp_suspend( int th_gtid, volatile kmp_uint *spinner, kmp_uint checker )
396 {
397  kmp_info_t *th = __kmp_threads[th_gtid];
398  int status;
399  kmp_uint old_spin;
400 
401  KF_TRACE( 30, ("__kmp_suspend: T#%d enter for spin = %p\n", th_gtid, spinner ) );
402 
403  __kmp_suspend_initialize_thread( th );
404 
405  __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
406 
407  KF_TRACE( 10, ( "__kmp_suspend: T#%d setting sleep bit for spin(%p)\n",
408  th_gtid, spinner ) );
409 
410  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
411  gets called first?
412  */
413  old_spin = KMP_TEST_THEN_OR32( (volatile kmp_int32 *) spinner,
414  KMP_BARRIER_SLEEP_STATE );
415 
416  KF_TRACE( 5, ( "__kmp_suspend: T#%d set sleep bit for spin(%p)==%d\n",
417  th_gtid, spinner, *spinner ) );
418 
419  if ( old_spin == checker ) {
420  KMP_TEST_THEN_AND32( (volatile kmp_int32 *) spinner, ~(KMP_BARRIER_SLEEP_STATE) );
421 
422  KF_TRACE( 5, ( "__kmp_suspend: T#%d false alarm, reset sleep bit for spin(%p)\n",
423  th_gtid, spinner) );
424  } else {
425 #ifdef DEBUG_SUSPEND
426  __kmp_suspend_count++;
427 #endif
428 
429  /* Encapsulate in a loop as the documentation states that this may
430  * "with low probability" return when the condition variable has
431  * not been signaled or broadcast
432  */
433  int deactivated = FALSE;
434  TCW_PTR(th->th.th_sleep_loc, spinner);
435  while ( TCR_4( *spinner ) & KMP_BARRIER_SLEEP_STATE ) {
436 
437  KF_TRACE( 15, ("__kmp_suspend: T#%d about to perform kmp_win32_cond_wait()\n",
438  th_gtid ) );
439 
440 
441  //
442  // Mark the thread as no longer active
443  // (only in the first iteration of the loop).
444  //
445  if ( ! deactivated ) {
446  th->th.th_active = FALSE;
447  if ( th->th.th_active_in_pool ) {
448  th->th.th_active_in_pool = FALSE;
449  KMP_TEST_THEN_DEC32(
450  (kmp_int32 *) &__kmp_thread_pool_active_nth );
451  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
452  }
453  deactivated = TRUE;
454 
455 
456  __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
457  }
458  else {
459  __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
460  }
461 
462 #ifdef KMP_DEBUG
463  if( (*spinner) & KMP_BARRIER_SLEEP_STATE ) {
464  KF_TRACE( 100, ("__kmp_suspend: T#%d spurious wakeup\n", th_gtid ));
465  }
466 #endif /* KMP_DEBUG */
467 
468  } // while
469 
470  //
471  // Mark the thread as active again
472  // (if it was previous marked as inactive)
473  //
474  if ( deactivated ) {
475  th->th.th_active = TRUE;
476  if ( TCR_4(th->th.th_in_pool) ) {
477  KMP_TEST_THEN_INC32(
478  (kmp_int32 *) &__kmp_thread_pool_active_nth );
479  th->th.th_active_in_pool = TRUE;
480  }
481  }
482  }
483 
484 
485  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
486 
487  KF_TRACE( 30, ("__kmp_suspend: T#%d exit\n", th_gtid ) );
488 }
489 
490 /* This routine signals the thread specified by target_gtid to wake up
491  * after setting the sleep bit indicated by the spin argument to FALSE
492  */
493 void
494 __kmp_resume( int target_gtid, volatile kmp_uint *spin )
495 {
496  kmp_info_t *th = __kmp_threads[target_gtid];
497  int status;
498  kmp_uint32 old_spin;
499 
500 #ifdef KMP_DEBUG
501  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
502 #endif
503 
504  KF_TRACE( 30, ( "__kmp_resume: T#%d wants to wakeup T#%d enter\n",
505  gtid, target_gtid ) );
506 
507  __kmp_suspend_initialize_thread( th );
508 
509  __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
510 
511  if ( spin == NULL ) {
512  spin = (volatile kmp_uint *)TCR_PTR(th->th.th_sleep_loc);
513  if ( spin == NULL ) {
514  KF_TRACE( 5, ( "__kmp_resume: T#%d exiting, thread T#%d already awake - spin(%p)\n",
515  gtid, target_gtid, spin ) );
516 
517  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
518  return;
519  }
520  }
521 
522  TCW_PTR(th->th.th_sleep_loc, NULL);
523  old_spin = KMP_TEST_THEN_AND32( (kmp_int32 volatile *) spin, ~( KMP_BARRIER_SLEEP_STATE ) );
524 
525  if ( ( old_spin & KMP_BARRIER_SLEEP_STATE ) == 0 ) {
526  KF_TRACE( 5, ( "__kmp_resume: T#%d exiting, thread T#%d already awake - spin(%p): "
527  "%u => %u\n",
528  gtid, target_gtid, spin, old_spin, *spin ) );
529 
530  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
531  return;
532  }
533  TCW_PTR(th->th.th_sleep_loc, NULL);
534 
535  KF_TRACE( 5, ( "__kmp_resume: T#%d about to wakeup T#%d, reset sleep bit for spin(%p)\n",
536  gtid, target_gtid, spin) );
537 
538 
539  __kmp_win32_cond_signal( &th->th.th_suspend_cv );
540 
541  __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
542 
543  KF_TRACE( 30, ( "__kmp_resume: T#%d exiting after signaling wake up for T#%d\n",
544  gtid, target_gtid ) );
545 }
546 
547 /* ------------------------------------------------------------------------ */
548 /* ------------------------------------------------------------------------ */
549 
550 void
551 __kmp_yield( int cond )
552 {
553  if (cond)
554  Sleep(0);
555 }
556 
557 /* ------------------------------------------------------------------------ */
558 /* ------------------------------------------------------------------------ */
559 
560 void
561 __kmp_gtid_set_specific( int gtid )
562 {
563  KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n",
564  gtid, __kmp_gtid_threadprivate_key ));
565  KMP_ASSERT( __kmp_init_runtime );
566  if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
567  KMP_FATAL( TLSSetValueFailed );
568 }
569 
570 int
571 __kmp_gtid_get_specific()
572 {
573  int gtid;
574  if( !__kmp_init_runtime ) {
575  KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
576  return KMP_GTID_SHUTDOWN;
577  }
578  gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
579  if ( gtid == 0 ) {
580  gtid = KMP_GTID_DNE;
581  }
582  else {
583  gtid--;
584  }
585  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
586  __kmp_gtid_threadprivate_key, gtid ));
587  return gtid;
588 }
589 
590 /* ------------------------------------------------------------------------ */
591 /* ------------------------------------------------------------------------ */
592 
593 #if KMP_ARCH_X86_64
594 
595 //
596 // Only 1 DWORD in the mask should have any procs set.
597 // Return the appropriate index, or -1 for an invalid mask.
598 //
599 int
600 __kmp_get_proc_group( kmp_affin_mask_t const *mask )
601 {
602  int i;
603  int group = -1;
604  struct GROUP_AFFINITY new_ga, prev_ga;
605  for (i = 0; i < __kmp_num_proc_groups; i++) {
606  if (mask[i] == 0) {
607  continue;
608  }
609  if (group >= 0) {
610  return -1;
611  }
612  group = i;
613  }
614  return group;
615 }
616 
617 #endif /* KMP_ARCH_X86_64 */
618 
619 int
620 __kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
621 {
622 
623 #if KMP_ARCH_X86_64
624 
625  if (__kmp_num_proc_groups > 1) {
626  //
627  // Check for a valid mask.
628  //
629  struct GROUP_AFFINITY ga;
630  int group = __kmp_get_proc_group( mask );
631  if (group < 0) {
632  if (abort_on_error) {
633  KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
634  }
635  return -1;
636  }
637 
638  //
639  // Transform the bit vector into a GROUP_AFFINITY struct
640  // and make the system call to set affinity.
641  //
642  ga.group = group;
643  ga.mask = mask[group];
644  ga.reserved[0] = ga.reserved[1] = ga.reserved[2] = 0;
645 
646  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
647  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
648  DWORD error = GetLastError();
649  if (abort_on_error) {
650  __kmp_msg(
651  kmp_ms_fatal,
652  KMP_MSG( CantSetThreadAffMask ),
653  KMP_ERR( error ),
654  __kmp_msg_null
655  );
656  }
657  return error;
658  }
659  }
660  else
661 
662 #endif /* KMP_ARCH_X86_64 */
663 
664  {
665  if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
666  DWORD error = GetLastError();
667  if (abort_on_error) {
668  __kmp_msg(
669  kmp_ms_fatal,
670  KMP_MSG( CantSetThreadAffMask ),
671  KMP_ERR( error ),
672  __kmp_msg_null
673  );
674  }
675  return error;
676  }
677  }
678  return 0;
679 }
680 
681 int
682 __kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
683 {
684 
685 #if KMP_ARCH_X86_64
686 
687  if (__kmp_num_proc_groups > 1) {
688  KMP_CPU_ZERO(mask);
689  struct GROUP_AFFINITY ga;
690  KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
691 
692  if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
693  DWORD error = GetLastError();
694  if (abort_on_error) {
695  __kmp_msg(
696  kmp_ms_fatal,
697  KMP_MSG(FunctionError, "GetThreadGroupAffinity()"),
698  KMP_ERR(error),
699  __kmp_msg_null
700  );
701  }
702  return error;
703  }
704 
705  if ((ga.group < 0) || (ga.group > __kmp_num_proc_groups)
706  || (ga.mask == 0)) {
707  return -1;
708  }
709 
710  mask[ga.group] = ga.mask;
711  }
712  else
713 
714 #endif /* KMP_ARCH_X86_64 */
715 
716  {
717  kmp_affin_mask_t newMask, sysMask, retval;
718 
719  if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
720  DWORD error = GetLastError();
721  if (abort_on_error) {
722  __kmp_msg(
723  kmp_ms_fatal,
724  KMP_MSG(FunctionError, "GetProcessAffinityMask()"),
725  KMP_ERR(error),
726  __kmp_msg_null
727  );
728  }
729  return error;
730  }
731  retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
732  if (! retval) {
733  DWORD error = GetLastError();
734  if (abort_on_error) {
735  __kmp_msg(
736  kmp_ms_fatal,
737  KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
738  KMP_ERR(error),
739  __kmp_msg_null
740  );
741  }
742  return error;
743  }
744  newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
745  if (! newMask) {
746  DWORD error = GetLastError();
747  if (abort_on_error) {
748  __kmp_msg(
749  kmp_ms_fatal,
750  KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
751  KMP_ERR(error),
752  __kmp_msg_null
753  );
754  }
755  }
756  *mask = retval;
757  }
758  return 0;
759 }
760 
761 void
762 __kmp_affinity_bind_thread( int proc )
763 {
764 
765 #if KMP_ARCH_X86_64
766 
767  if (__kmp_num_proc_groups > 1) {
768  //
769  // Form the GROUP_AFFINITY struct directly, rather than filling
770  // out a bit vector and calling __kmp_set_system_affinity().
771  //
772  struct GROUP_AFFINITY ga;
773  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
774  * CHAR_BIT * sizeof(DWORD_PTR))));
775  ga.group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
776  ga.mask = 1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
777  ga.reserved[0] = ga.reserved[1] = ga.reserved[2] = 0;
778 
779  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
780  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
781  DWORD error = GetLastError();
782  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
783  __kmp_msg(
784  kmp_ms_warning,
785  KMP_MSG( CantSetThreadAffMask ),
786  KMP_ERR( error ),
787  __kmp_msg_null
788  );
789  }
790  }
791  }
792  else
793 
794 #endif /* KMP_ARCH_X86_64 */
795 
796  {
797  kmp_affin_mask_t mask;
798  KMP_CPU_ZERO(&mask);
799  KMP_CPU_SET(proc, &mask);
800  __kmp_set_system_affinity(&mask, TRUE);
801  }
802 }
803 
804 void
805 __kmp_affinity_determine_capable( const char *env_var )
806 {
807  //
808  // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
809  //
810 
811 #if KMP_ARCH_X86_64
812  __kmp_affin_mask_size = __kmp_num_proc_groups * sizeof(kmp_affin_mask_t);
813 #else
814  __kmp_affin_mask_size = sizeof(kmp_affin_mask_t);
815 #endif
816 
817  KA_TRACE( 10, (
818  "__kmp_affinity_determine_capable: "
819  "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC ").\n",
820  __kmp_affin_mask_size
821  ) );
822 }
823 
824 double
825 __kmp_read_cpu_time( void )
826 {
827  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
828  int status;
829  double cpu_time;
830 
831  cpu_time = 0;
832 
833  status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
834  &ExitTime, &KernelTime, &UserTime );
835 
836  if (status) {
837  double sec = 0;
838 
839  sec += KernelTime.dwHighDateTime;
840  sec += UserTime.dwHighDateTime;
841 
842  /* Shift left by 32 bits */
843  sec *= (double) (1 << 16) * (double) (1 << 16);
844 
845  sec += KernelTime.dwLowDateTime;
846  sec += UserTime.dwLowDateTime;
847 
848  cpu_time += (sec * 100.0) / NSEC_PER_SEC;
849  }
850 
851  return cpu_time;
852 }
853 
854 int
855 __kmp_read_system_info( struct kmp_sys_info *info )
856 {
857  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
858  info->minflt = 0; /* the number of page faults serviced without any I/O */
859  info->majflt = 0; /* the number of page faults serviced that required I/O */
860  info->nswap = 0; /* the number of times a process was "swapped" out of memory */
861  info->inblock = 0; /* the number of times the file system had to perform input */
862  info->oublock = 0; /* the number of times the file system had to perform output */
863  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
864  info->nivcsw = 0; /* the number of times a context switch was forced */
865 
866  return 1;
867 }
868 
869 /* ------------------------------------------------------------------------ */
870 /* ------------------------------------------------------------------------ */
871 
872 
873 void
874 __kmp_runtime_initialize( void )
875 {
876  SYSTEM_INFO info;
877  kmp_str_buf_t path;
878  UINT path_size;
879 
880  if ( __kmp_init_runtime ) {
881  return;
882  };
883 
884  InitializeCriticalSection( & __kmp_win32_section );
885 #if USE_ITT_BUILD
886  __kmp_itt_system_object_created( & __kmp_win32_section, "Critical Section" );
887 #endif /* USE_ITT_BUILD */
888  __kmp_initialize_system_tick();
889 
890  #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
891  if ( ! __kmp_cpuinfo.initialized ) {
892  __kmp_query_cpuid( & __kmp_cpuinfo );
893  }; // if
894  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
895 
896  /* Set up minimum number of threads to switch to TLS gtid */
897  #if KMP_OS_WINDOWS && ! defined GUIDEDLL_EXPORTS
898  // Windows* OS, static library.
899  /*
900  New thread may use stack space previously used by another thread, currently terminated.
901  On Windows* OS, in case of static linking, we do not know the moment of thread termination,
902  and our structures (__kmp_threads and __kmp_root arrays) are still keep info about dead
903  threads. This leads to problem in __kmp_get_global_thread_id() function: it wrongly
904  finds gtid (by searching through stack addresses of all known threads) for unregistered
905  foreign tread.
906 
907  Setting __kmp_tls_gtid_min to 0 workarounds this problem: __kmp_get_global_thread_id()
908  does not search through stacks, but get gtid from TLS immediatelly.
909 
910  --ln
911  */
912  __kmp_tls_gtid_min = 0;
913  #else
914  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
915  #endif
916 
917  /* for the static library */
918  if ( !__kmp_gtid_threadprivate_key ) {
919  __kmp_gtid_threadprivate_key = TlsAlloc();
920  if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
921  KMP_FATAL( TLSOutOfIndexes );
922  }
923  }
924 
925 
926  //
927  // Load ntdll.dll.
928  //
929  /*
930  Simple
931  GetModuleHandle( "ntdll.dl" )
932  is not suitable due to security issue (see
933  http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full
934  path to the library.
935  */
936  __kmp_str_buf_init( & path );
937  path_size = GetSystemDirectory( path.str, path.size );
938  KMP_DEBUG_ASSERT( path_size > 0 );
939  if ( path_size >= path.size ) {
940  //
941  // Buffer is too short. Expand the buffer and try again.
942  //
943  __kmp_str_buf_reserve( & path, path_size );
944  path_size = GetSystemDirectory( path.str, path.size );
945  KMP_DEBUG_ASSERT( path_size > 0 );
946  }; // if
947  if ( path_size > 0 && path_size < path.size ) {
948  //
949  // Now we have system directory name in the buffer.
950  // Append backslash and name of dll to form full path,
951  //
952  path.used = path_size;
953  __kmp_str_buf_print( & path, "\\%s", "ntdll.dll" );
954 
955  //
956  // Now load ntdll using full path.
957  //
958  ntdll = GetModuleHandle( path.str );
959  }
960 
961  KMP_DEBUG_ASSERT( ntdll != NULL );
962  if ( ntdll != NULL ) {
963  NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll, "NtQuerySystemInformation" );
964  }
965  KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );
966 
967 #if KMP_ARCH_X86_64
968  //
969  // Load kernel32.dll.
970  // Same caveat - must use full system path name.
971  //
972  if ( path_size > 0 && path_size < path.size ) {
973  //
974  // Truncate the buffer back to just the system path length,
975  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
976  //
977  path.used = path_size;
978  __kmp_str_buf_print( & path, "\\%s", "kernel32.dll" );
979 
980  //
981  // Load kernel32.dll using full path.
982  //
983  kernel32 = GetModuleHandle( path.str );
984 
985  //
986  // Load the function pointers to kernel32.dll routines
987  // that may or may not exist on this system.
988  //
989  if ( kernel32 != NULL ) {
990  __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32, "GetActiveProcessorCount" );
991  __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32, "GetActiveProcessorGroupCount" );
992  __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32, "GetThreadGroupAffinity" );
993  __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32, "SetThreadGroupAffinity" );
994 
995  //
996  // See if group affinity is supported on this system.
997  // If so, calculate the #groups and #procs.
998  //
999  // Group affinity was introduced with Windows* 7 OS and
1000  // Windows* Server 2008 R2 OS.
1001  //
1002  if ( ( __kmp_GetActiveProcessorCount != NULL )
1003  && ( __kmp_GetActiveProcessorGroupCount != NULL )
1004  && ( __kmp_GetThreadGroupAffinity != NULL )
1005  && ( __kmp_SetThreadGroupAffinity != NULL )
1006  && ( ( __kmp_num_proc_groups
1007  = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
1008  //
1009  // Calculate the total number of active OS procs.
1010  //
1011  int i;
1012 
1013  KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
1014 
1015  __kmp_xproc = 0;
1016 
1017  for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
1018  DWORD size = __kmp_GetActiveProcessorCount( i );
1019  __kmp_xproc += size;
1020  KA_TRACE( 20, ("__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
1021  }
1022  }
1023  }
1024  }
1025  if ( __kmp_num_proc_groups <= 1 ) {
1026  GetSystemInfo( & info );
1027  __kmp_xproc = info.dwNumberOfProcessors;
1028  }
1029 #else
1030  GetSystemInfo( & info );
1031  __kmp_xproc = info.dwNumberOfProcessors;
1032 #endif // KMP_ARCH_X86_64
1033 
1034  //
1035  // If the OS said there were 0 procs, take a guess and use a value of 2.
1036  // This is done for Linux* OS, also. Do we need error / warning?
1037  //
1038  if ( __kmp_xproc <= 0 ) {
1039  __kmp_xproc = 2;
1040  }
1041 
1042  KA_TRACE( 5, ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );
1043 
1044  __kmp_str_buf_free( & path );
1045 
1046 #if USE_ITT_BUILD
1047  __kmp_itt_initialize();
1048 #endif /* USE_ITT_BUILD */
1049 
1050  __kmp_init_runtime = TRUE;
1051 } // __kmp_runtime_initialize
1052 
1053 void
1054 __kmp_runtime_destroy( void )
1055 {
1056  if ( ! __kmp_init_runtime ) {
1057  return;
1058  }
1059 
1060 #if USE_ITT_BUILD
1061  __kmp_itt_destroy();
1062 #endif /* USE_ITT_BUILD */
1063 
1064  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
1065  /* due to the KX_TRACE() commands */
1066  KA_TRACE( 40, ("__kmp_runtime_destroy\n" ));
1067 
1068  if( __kmp_gtid_threadprivate_key ) {
1069  TlsFree( __kmp_gtid_threadprivate_key );
1070  __kmp_gtid_threadprivate_key = 0;
1071  }
1072 
1073  __kmp_affinity_uninitialize();
1074  DeleteCriticalSection( & __kmp_win32_section );
1075 
1076  ntdll = NULL;
1077  NtQuerySystemInformation = NULL;
1078 
1079 #if KMP_ARCH_X86_64
1080  kernel32 = NULL;
1081  __kmp_GetActiveProcessorCount = NULL;
1082  __kmp_GetActiveProcessorGroupCount = NULL;
1083  __kmp_GetThreadGroupAffinity = NULL;
1084  __kmp_SetThreadGroupAffinity = NULL;
1085 #endif // KMP_ARCH_X86_64
1086 
1087  __kmp_init_runtime = FALSE;
1088 }
1089 
1090 
1091 void
1092 __kmp_terminate_thread( int gtid )
1093 {
1094  kmp_info_t *th = __kmp_threads[ gtid ];
1095 
1096  if( !th ) return;
1097 
1098  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
1099 
1100  if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
1101  /* It's OK, the thread may have exited already */
1102  }
1103  __kmp_free_handle( th->th.th_info.ds.ds_thread );
1104 }
1105 
1106 /* ------------------------------------------------------------------------ */
1107 /* ------------------------------------------------------------------------ */
1108 
1109 void
1110 __kmp_clear_system_time( void )
1111 {
1112  BOOL status;
1113  LARGE_INTEGER time;
1114  status = QueryPerformanceCounter( & time );
1115  __kmp_win32_time = (kmp_int64) time.QuadPart;
1116 }
1117 
1118 void
1119 __kmp_initialize_system_tick( void )
1120 {
1121  {
1122  BOOL status;
1123  LARGE_INTEGER freq;
1124 
1125  status = QueryPerformanceFrequency( & freq );
1126  if (! status) {
1127  DWORD error = GetLastError();
1128  __kmp_msg(
1129  kmp_ms_fatal,
1130  KMP_MSG( FunctionError, "QueryPerformanceFrequency()" ),
1131  KMP_ERR( error ),
1132  __kmp_msg_null
1133  );
1134 
1135  }
1136  else {
1137  __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
1138  }
1139  }
1140 }
1141 
1142 /* Calculate the elapsed wall clock time for the user */
1143 
1144 void
1145 __kmp_elapsed( double *t )
1146 {
1147  BOOL status;
1148  LARGE_INTEGER now;
1149  status = QueryPerformanceCounter( & now );
1150  *t = ((double) now.QuadPart) * __kmp_win32_tick;
1151 }
1152 
1153 /* Calculate the elapsed wall clock tick for the user */
1154 
1155 void
1156 __kmp_elapsed_tick( double *t )
1157 {
1158  *t = __kmp_win32_tick;
1159 }
1160 
1161 void
1162 __kmp_read_system_time( double *delta )
1163 {
1164 
1165  if (delta != NULL) {
1166  BOOL status;
1167  LARGE_INTEGER now;
1168 
1169  status = QueryPerformanceCounter( & now );
1170 
1171  *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
1172  * __kmp_win32_tick;
1173  }
1174 }
1175 
1176 /* ------------------------------------------------------------------------ */
1177 /* ------------------------------------------------------------------------ */
1178 
1179 /*
1180  * Change thread to the affinity mask pointed to by affin_mask argument
1181  * and return a pointer to the old value in the old_mask argument, if argument
1182  * is non-NULL.
1183  */
1184 
1185 void
1186 __kmp_change_thread_affinity_mask( int gtid, kmp_affin_mask_t *new_mask,
1187  kmp_affin_mask_t *old_mask )
1188 {
1189  kmp_info_t *th = __kmp_threads[ gtid ];
1190 
1191  KMP_DEBUG_ASSERT( *new_mask != 0 );
1192 
1193  if ( old_mask != NULL ) {
1194  *old_mask = SetThreadAffinityMask( th -> th.th_info.ds.ds_thread, *new_mask );
1195 
1196  if (! *old_mask ) {
1197  DWORD error = GetLastError();
1198  __kmp_msg(
1199  kmp_ms_fatal,
1200  KMP_MSG( CantSetThreadAffMask ),
1201  KMP_ERR( error ),
1202  __kmp_msg_null
1203  );
1204  }
1205  }
1206  if (__kmp_affinity_verbose)
1207  KMP_INFORM( ChangeAffMask, "KMP_AFFINITY (Bind)", gtid, *old_mask, *new_mask );
1208 
1209  /* Make sure old value is correct in thread data structures */
1210  KMP_DEBUG_ASSERT( old_mask != NULL && *old_mask == *(th -> th.th_affin_mask ));
1211 
1212  KMP_CPU_COPY(th -> th.th_affin_mask, new_mask);
1213 }
1214 
1215 
1216 /* ------------------------------------------------------------------------ */
1217 /* ------------------------------------------------------------------------ */
1218 
1219 void * __stdcall
1220 __kmp_launch_worker( void *arg )
1221 {
1222  volatile void *stack_data;
1223  void *exit_val;
1224  void *padding = 0;
1225  kmp_info_t *this_thr = (kmp_info_t *) arg;
1226  int gtid;
1227 
1228  gtid = this_thr->th.th_info.ds.ds_gtid;
1229  __kmp_gtid_set_specific( gtid );
1230 #ifdef KMP_TDATA_GTID
1231  #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1232  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1233  "reference: http://support.microsoft.com/kb/118816"
1234  //__kmp_gtid = gtid;
1235 #endif
1236 
1237 #if USE_ITT_BUILD
1238  __kmp_itt_thread_name( gtid );
1239 #endif /* USE_ITT_BUILD */
1240 
1241  __kmp_affinity_set_init_mask( gtid, FALSE );
1242 
1243 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1244  //
1245  // Set the FP control regs to be a copy of
1246  // the parallel initialization thread's.
1247  //
1248  __kmp_clear_x87_fpu_status_word();
1249  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
1250  __kmp_load_mxcsr( &__kmp_init_mxcsr );
1251 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1252 
1253  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
1254  padding = _alloca( gtid * __kmp_stkoffset );
1255  }
1256 
1257  KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1258  this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1259  TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1260 
1261  if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
1262  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
1263  KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
1264  __kmp_check_stack_overlap( this_thr );
1265  }
1266  KMP_MB();
1267  exit_val = __kmp_launch_thread( this_thr );
1268  KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1269  TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1270  KMP_MB();
1271  return exit_val;
1272 }
1273 
1274 
1275 /* The monitor thread controls all of the threads in the complex */
1276 
1277 void * __stdcall
1278 __kmp_launch_monitor( void *arg )
1279 {
1280  DWORD wait_status;
1281  kmp_thread_t monitor;
1282  int status;
1283  int interval;
1284  kmp_info_t *this_thr = (kmp_info_t *) arg;
1285 
1286  KMP_DEBUG_ASSERT(__kmp_init_monitor);
1287  TCW_4( __kmp_init_monitor, 2 ); // AC: Signal the library that monitor has started
1288  // TODO: hide "2" in enum (like {true,false,started})
1289  this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1290  TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1291 
1292  KMP_MB(); /* Flush all pending memory write invalidates. */
1293  KA_TRACE( 10, ("__kmp_launch_monitor: launched\n" ) );
1294 
1295  monitor = GetCurrentThread();
1296 
1297  /* set thread priority */
1298  status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
1299  if (! status) {
1300  DWORD error = GetLastError();
1301  __kmp_msg(
1302  kmp_ms_fatal,
1303  KMP_MSG( CantSetThreadPriority ),
1304  KMP_ERR( error ),
1305  __kmp_msg_null
1306  );
1307  }
1308 
1309  /* register us as monitor */
1310  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
1311 #ifdef KMP_TDATA_GTID
1312  #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1313  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1314  "reference: http://support.microsoft.com/kb/118816"
1315  //__kmp_gtid = KMP_GTID_MONITOR;
1316 #endif
1317 
1318 #if USE_ITT_BUILD
1319  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
1320 #endif /* USE_ITT_BUILD */
1321 
1322  KMP_MB(); /* Flush all pending memory write invalidates. */
1323 
1324  interval = ( 1000 / __kmp_monitor_wakeups ); /* in milliseconds */
1325 
1326  while (! TCR_4(__kmp_global.g.g_done)) {
1327  /* This thread monitors the state of the system */
1328 
1329  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
1330 
1331  wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );
1332 
1333  if (wait_status == WAIT_TIMEOUT) {
1334  TCW_4( __kmp_global.g.g_time.dt.t_value,
1335  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
1336  }
1337 
1338  KMP_MB(); /* Flush all pending memory write invalidates. */
1339  }
1340 
1341  KA_TRACE( 10, ("__kmp_launch_monitor: finished\n" ) );
1342 
1343  status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
1344  if (! status) {
1345  DWORD error = GetLastError();
1346  __kmp_msg(
1347  kmp_ms_fatal,
1348  KMP_MSG( CantSetThreadPriority ),
1349  KMP_ERR( error ),
1350  __kmp_msg_null
1351  );
1352  }
1353 
1354  if (__kmp_global.g.g_abort != 0) {
1355  /* now we need to terminate the worker threads */
1356  /* the value of t_abort is the signal we caught */
1357 
1358  int gtid;
1359 
1360  KA_TRACE( 10, ("__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );
1361 
1362  /* terminate the OpenMP worker threads */
1363  /* TODO this is not valid for sibling threads!!
1364  * the uber master might not be 0 anymore.. */
1365  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1366  __kmp_terminate_thread( gtid );
1367 
1368  __kmp_cleanup();
1369 
1370  Sleep( 0 );
1371 
1372  KA_TRACE( 10, ("__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );
1373 
1374  if (__kmp_global.g.g_abort > 0) {
1375  raise( __kmp_global.g.g_abort );
1376  }
1377  }
1378 
1379  TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1380 
1381  KMP_MB();
1382  return arg;
1383 }
1384 
1385 void
1386 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
1387 {
1388  kmp_thread_t handle;
1389  DWORD idThread;
1390 
1391  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1392 
1393  th->th.th_info.ds.ds_gtid = gtid;
1394 
1395  if ( KMP_UBER_GTID(gtid) ) {
1396  int stack_data;
1397 
1398  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for other threads to use.
1399  Is it appropriate to just use GetCurrentThread? When should we close this handle? When
1400  unregistering the root?
1401  */
1402  {
1403  BOOL rc;
1404  rc = DuplicateHandle(
1405  GetCurrentProcess(),
1406  GetCurrentThread(),
1407  GetCurrentProcess(),
1408  &th->th.th_info.ds.ds_thread,
1409  0,
1410  FALSE,
1411  DUPLICATE_SAME_ACCESS
1412  );
1413  KMP_ASSERT( rc );
1414  KA_TRACE( 10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC "\n",
1415  (LPVOID)th,
1416  th->th.th_info.ds.ds_thread ) );
1417  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1418  }
1419  if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
1420  /* we will dynamically update the stack range if gtid_mode == 1 */
1421  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1422  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1423  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1424  __kmp_check_stack_overlap( th );
1425  }
1426  }
1427  else {
1428  KMP_MB(); /* Flush all pending memory write invalidates. */
1429 
1430  /* Set stack size for this thread now. */
1431  KA_TRACE( 10, ( "__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
1432  " bytes\n", stack_size ) );
1433 
1434  stack_size += gtid * __kmp_stkoffset;
1435 
1436  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1437  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1438 
1439  KA_TRACE( 10, ( "__kmp_create_worker: (before) stack_size = %"
1440  KMP_SIZE_T_SPEC
1441  " bytes, &__kmp_launch_worker = %p, th = %p, "
1442  "&idThread = %p\n",
1443  (SIZE_T) stack_size,
1444  (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1445  (LPVOID) th, &idThread ) );
1446 
1447  {
1448  handle = CreateThread( NULL, (SIZE_T) stack_size,
1449  (LPTHREAD_START_ROUTINE) __kmp_launch_worker,
1450  (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1451  }
1452 
1453  KA_TRACE( 10, ( "__kmp_create_worker: (after) stack_size = %"
1454  KMP_SIZE_T_SPEC
1455  " bytes, &__kmp_launch_worker = %p, th = %p, "
1456  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1457  (SIZE_T) stack_size,
1458  (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1459  (LPVOID) th, idThread, handle ) );
1460 
1461  {
1462  if ( handle == 0 ) {
1463  DWORD error = GetLastError();
1464  __kmp_msg(
1465  kmp_ms_fatal,
1466  KMP_MSG( CantCreateThread ),
1467  KMP_ERR( error ),
1468  __kmp_msg_null
1469  );
1470  } else {
1471  th->th.th_info.ds.ds_thread = handle;
1472  }
1473  }
1474  KMP_MB(); /* Flush all pending memory write invalidates. */
1475  }
1476 
1477  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1478 }
1479 
1480 int
1481 __kmp_still_running(kmp_info_t *th) {
1482  return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
1483 }
1484 
1485 void
1486 __kmp_create_monitor( kmp_info_t *th )
1487 {
1488  kmp_thread_t handle;
1489  DWORD idThread;
1490  int ideal, new_ideal;
1491  int caller_gtid = __kmp_get_gtid();
1492 
1493  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1494 
1495  KMP_MB(); /* Flush all pending memory write invalidates. */
1496 
1497  __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
1498  if ( __kmp_monitor_ev == NULL ) {
1499  DWORD error = GetLastError();
1500  __kmp_msg(
1501  kmp_ms_fatal,
1502  KMP_MSG( CantCreateEvent ),
1503  KMP_ERR( error ),
1504  __kmp_msg_null
1505  );
1506  }; // if
1507 #if USE_ITT_BUILD
1508  __kmp_itt_system_object_created( __kmp_monitor_ev, "Event" );
1509 #endif /* USE_ITT_BUILD */
1510 
1511  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1512  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1513 
1514  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1515  // to automatically expand stacksize based on CreateThread error code.
1516  if ( __kmp_monitor_stksize == 0 ) {
1517  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1518  }
1519  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1520  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1521  }
1522 
1523  KA_TRACE( 10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1524  (int) __kmp_monitor_stksize ) );
1525 
1526  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1527 
1528  handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
1529  (LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
1530  (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1531  if (handle == 0) {
1532  DWORD error = GetLastError();
1533  __kmp_msg(
1534  kmp_ms_fatal,
1535  KMP_MSG( CantCreateThread ),
1536  KMP_ERR( error ),
1537  __kmp_msg_null
1538  );
1539  }
1540  else
1541  th->th.th_info.ds.ds_thread = handle;
1542 
1543  KMP_MB(); /* Flush all pending memory write invalidates. */
1544 
1545  KA_TRACE( 10, ("__kmp_create_monitor: monitor created %p\n",
1546  (void *) th->th.th_info.ds.ds_thread ) );
1547 }
1548 
1549 /*
1550  Check to see if thread is still alive.
1551 
1552  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1553  with a exit_val = code. Because of this we can not rely on
1554  exit_val having any particular value. So this routine may
1555  return STILL_ALIVE in exit_val even after the thread is dead.
1556 */
1557 
1558 int
1559 __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
1560 {
1561  DWORD rc;
1562  rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
1563  if ( rc == 0 ) {
1564  DWORD error = GetLastError();
1565  __kmp_msg(
1566  kmp_ms_fatal,
1567  KMP_MSG( FunctionError, "GetExitCodeThread()" ),
1568  KMP_ERR( error ),
1569  __kmp_msg_null
1570  );
1571  }; // if
1572  return ( *exit_val == STILL_ACTIVE );
1573 }
1574 
1575 
1576 void
1577 __kmp_exit_thread(
1578  int exit_status
1579 ) {
1580  ExitThread( exit_status );
1581 } // __kmp_exit_thread
1582 
1583 /*
1584  This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1585 */
1586 static void
1587 __kmp_reap_common( kmp_info_t * th )
1588 {
1589  DWORD exit_val;
1590 
1591  KMP_MB(); /* Flush all pending memory write invalidates. */
1592 
1593  KA_TRACE( 10, ( "__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );
1594 
1595  /*
1596  2006-10-19:
1597 
1598  There are two opposite situations:
1599 
1600  1. Windows* OS keep thread alive after it resets ds_alive flag and exits from thread
1601  function. (For example, see C70770/Q394281 "unloading of dll based on OMP is very
1602  slow".)
1603  2. Windows* OS may kill thread before it resets ds_alive flag.
1604 
1605  Right solution seems to be waiting for *either* thread termination *or* ds_alive resetting.
1606 
1607  */
1608 
1609  {
1610  // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize KMP_WAIT_YIELD to
1611  // cover this usage also.
1612  void * obj = NULL;
1613  register kmp_uint32 spins;
1614 #if USE_ITT_BUILD
1615  KMP_FSYNC_SPIN_INIT( obj, (void*) & th->th.th_info.ds.ds_alive );
1616 #endif /* USE_ITT_BUILD */
1617  KMP_INIT_YIELD( spins );
1618  do {
1619 #if USE_ITT_BUILD
1620  KMP_FSYNC_SPIN_PREPARE( obj );
1621 #endif /* USE_ITT_BUILD */
1622  __kmp_is_thread_alive( th, &exit_val );
1623  __kmp_static_delay( TRUE );
1624  KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
1625  KMP_YIELD_SPIN( spins );
1626  } while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
1627 #if USE_ITT_BUILD
1628  if ( exit_val == STILL_ACTIVE ) {
1629  KMP_FSYNC_CANCEL( obj );
1630  } else {
1631  KMP_FSYNC_SPIN_ACQUIRED( obj );
1632  }; // if
1633 #endif /* USE_ITT_BUILD */
1634  }
1635 
1636  __kmp_free_handle( th->th.th_info.ds.ds_thread );
1637 
1638  /*
1639  * NOTE: The ExitProcess(code) system call causes all threads to Terminate
1640  * with a exit_val = code. Because of this we can not rely on
1641  * exit_val having any particular value.
1642  */
1643  if ( exit_val == STILL_ACTIVE ) {
1644  KA_TRACE( 1, ( "__kmp_reap_common: thread still active.\n" ) );
1645  } else if ( (void *) exit_val != (void *) th) {
1646  KA_TRACE( 1, ( "__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
1647  }; // if
1648 
1649  KA_TRACE( 10,
1650  (
1651  "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC "\n",
1652  th->th.th_info.ds.ds_gtid,
1653  th->th.th_info.ds.ds_thread
1654  )
1655  );
1656 
1657  th->th.th_info.ds.ds_thread = 0;
1658  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1659  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1660  th->th.th_info.ds.ds_thread_id = 0;
1661 
1662  KMP_MB(); /* Flush all pending memory write invalidates. */
1663 }
1664 
1665 void
1666 __kmp_reap_monitor( kmp_info_t *th )
1667 {
1668  int status;
1669 
1670  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap %p\n",
1671  (void *) th->th.th_info.ds.ds_thread ) );
1672 
1673  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1674  // If both tid and gtid are 0, it means the monitor did not ever start.
1675  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1676  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1677  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1678  return;
1679  }; // if
1680 
1681  KMP_MB(); /* Flush all pending memory write invalidates. */
1682 
1683  status = SetEvent( __kmp_monitor_ev );
1684  if ( status == FALSE ) {
1685  DWORD error = GetLastError();
1686  __kmp_msg(
1687  kmp_ms_fatal,
1688  KMP_MSG( CantSetEvent ),
1689  KMP_ERR( error ),
1690  __kmp_msg_null
1691  );
1692  }
1693  KA_TRACE( 10, ( "__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1694  __kmp_reap_common( th );
1695 
1696  __kmp_free_handle( __kmp_monitor_ev );
1697 
1698  KMP_MB(); /* Flush all pending memory write invalidates. */
1699 }
1700 
1701 void
1702 __kmp_reap_worker( kmp_info_t * th )
1703 {
1704  KA_TRACE( 10, ( "__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1705  __kmp_reap_common( th );
1706 }
1707 
1708 /* ------------------------------------------------------------------------ */
1709 /* ------------------------------------------------------------------------ */
1710 
1711 #if KMP_HANDLE_SIGNALS
1712 
1713 
1714 static void
1715 __kmp_team_handler( int signo )
1716 {
1717  if ( __kmp_global.g.g_abort == 0 ) {
1718  // Stage 1 signal handler, let's shut down all of the threads.
1719  if ( __kmp_debug_buf ) {
1720  __kmp_dump_debug_buffer();
1721  }; // if
1722  KMP_MB(); // Flush all pending memory write invalidates.
1723  TCW_4( __kmp_global.g.g_abort, signo );
1724  KMP_MB(); // Flush all pending memory write invalidates.
1725  TCW_4( __kmp_global.g.g_done, TRUE );
1726  KMP_MB(); // Flush all pending memory write invalidates.
1727  }
1728 } // __kmp_team_handler
1729 
1730 
1731 
1732 static
1733 sig_func_t __kmp_signal( int signum, sig_func_t handler ) {
1734  sig_func_t old = signal( signum, handler );
1735  if ( old == SIG_ERR ) {
1736  int error = errno;
1737  __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "signal" ), KMP_ERR( error ), __kmp_msg_null );
1738  }; // if
1739  return old;
1740 }
1741 
1742 static void
1743 __kmp_install_one_handler(
1744  int sig,
1745  sig_func_t handler,
1746  int parallel_init
1747 ) {
1748  sig_func_t old;
1749  KMP_MB(); /* Flush all pending memory write invalidates. */
1750  KB_TRACE( 60, ("__kmp_install_one_handler: called: sig=%d\n", sig ) );
1751  if ( parallel_init ) {
1752  old = __kmp_signal( sig, handler );
1753  // SIG_DFL on Windows* OS in NULL or 0.
1754  if ( old == __kmp_sighldrs[ sig ] ) {
1755  __kmp_siginstalled[ sig ] = 1;
1756  } else {
1757  // Restore/keep user's handler if one previously installed.
1758  old = __kmp_signal( sig, old );
1759  }; // if
1760  } else {
1761  // Save initial/system signal handlers to see if user handlers installed.
1762  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals called once with
1763  // parallel_init == TRUE.
1764  old = __kmp_signal( sig, SIG_DFL );
1765  __kmp_sighldrs[ sig ] = old;
1766  __kmp_signal( sig, old );
1767  }; // if
1768  KMP_MB(); /* Flush all pending memory write invalidates. */
1769 } // __kmp_install_one_handler
1770 
1771 static void
1772 __kmp_remove_one_handler( int sig ) {
1773  if ( __kmp_siginstalled[ sig ] ) {
1774  sig_func_t old;
1775  KMP_MB(); // Flush all pending memory write invalidates.
1776  KB_TRACE( 60, ( "__kmp_remove_one_handler: called: sig=%d\n", sig ) );
1777  old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
1778  if ( old != __kmp_team_handler ) {
1779  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1780  old = __kmp_signal( sig, old );
1781  }; // if
1782  __kmp_sighldrs[ sig ] = NULL;
1783  __kmp_siginstalled[ sig ] = 0;
1784  KMP_MB(); // Flush all pending memory write invalidates.
1785  }; // if
1786 } // __kmp_remove_one_handler
1787 
1788 
1789 void
1790 __kmp_install_signals( int parallel_init )
1791 {
1792  KB_TRACE( 10, ( "__kmp_install_signals: called\n" ) );
1793  if ( ! __kmp_handle_signals ) {
1794  KB_TRACE( 10, ( "__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
1795  return;
1796  }; // if
1797  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1798  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1799  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1800  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1801  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1802  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1803 } // __kmp_install_signals
1804 
1805 
1806 void
1807 __kmp_remove_signals( void )
1808 {
1809  int sig;
1810  KB_TRACE( 10, ("__kmp_remove_signals: called\n" ) );
1811  for ( sig = 1; sig < NSIG; ++ sig ) {
1812  __kmp_remove_one_handler( sig );
1813  }; // for sig
1814 } // __kmp_remove_signals
1815 
1816 
1817 #endif // KMP_HANDLE_SIGNALS
1818 
1819 /* Put the thread to sleep for a time period */
1820 void
1821 __kmp_thread_sleep( int millis )
1822 {
1823  DWORD status;
1824 
1825  status = SleepEx( (DWORD) millis, FALSE );
1826  if ( status ) {
1827  DWORD error = GetLastError();
1828  __kmp_msg(
1829  kmp_ms_fatal,
1830  KMP_MSG( FunctionError, "SleepEx()" ),
1831  KMP_ERR( error ),
1832  __kmp_msg_null
1833  );
1834  }
1835 }
1836 
1837 /* Determine whether the given address is mapped into the current address space. */
1838 int
1839 __kmp_is_address_mapped( void * addr )
1840 {
1841  DWORD status;
1842  MEMORY_BASIC_INFORMATION lpBuffer;
1843  SIZE_T dwLength;
1844 
1845  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1846 
1847  status = VirtualQuery( addr, &lpBuffer, dwLength );
1848 
1849  return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
1850  (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
1851 }
1852 
1853 kmp_uint64
1854 __kmp_hardware_timestamp(void)
1855 {
1856  kmp_uint64 r = 0;
1857 
1858  QueryPerformanceCounter((LARGE_INTEGER*) &r);
1859  return r;
1860 }
1861 
1862 /* Free handle and check the error code */
1863 void
1864 __kmp_free_handle( kmp_thread_t tHandle )
1865 {
1866 /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined as HANDLE */
1867  BOOL rc;
1868  rc = CloseHandle( tHandle );
1869  if ( !rc ) {
1870  DWORD error = GetLastError();
1871  __kmp_msg(
1872  kmp_ms_fatal,
1873  KMP_MSG( CantCloseHandle ),
1874  KMP_ERR( error ),
1875  __kmp_msg_null
1876  );
1877  }
1878 }
1879 
1880 int
1881 __kmp_get_load_balance( int max ) {
1882 
1883  static ULONG glb_buff_size = 100 * 1024;
1884 
1885  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
1886  static double glb_call_time = 0; /* Thread balance algorithm call time */
1887 
1888  int running_threads = 0; // Number of running threads in the system.
1889  NTSTATUS status = 0;
1890  ULONG buff_size = 0;
1891  ULONG info_size = 0;
1892  void * buffer = NULL;
1893  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1894  int first_time = 1;
1895 
1896  double call_time = 0.0; //start, finish;
1897 
1898  __kmp_elapsed( & call_time );
1899 
1900  if ( glb_call_time &&
1901  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
1902  running_threads = glb_running_threads;
1903  goto finish;
1904  }
1905  glb_call_time = call_time;
1906 
1907  // Do not spend time on running algorithm if we have a permanent error.
1908  if ( NtQuerySystemInformation == NULL ) {
1909  running_threads = -1;
1910  goto finish;
1911  }; // if
1912 
1913  if ( max <= 0 ) {
1914  max = INT_MAX;
1915  }; // if
1916 
1917  do {
1918 
1919  if ( first_time ) {
1920  buff_size = glb_buff_size;
1921  } else {
1922  buff_size = 2 * buff_size;
1923  }
1924 
1925  buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
1926  if ( buffer == NULL ) {
1927  running_threads = -1;
1928  goto finish;
1929  }; // if
1930  status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
1931  first_time = 0;
1932 
1933  } while ( status == STATUS_INFO_LENGTH_MISMATCH );
1934  glb_buff_size = buff_size;
1935 
1936  #define CHECK( cond ) \
1937  { \
1938  KMP_DEBUG_ASSERT( cond ); \
1939  if ( ! ( cond ) ) { \
1940  running_threads = -1; \
1941  goto finish; \
1942  } \
1943  }
1944 
1945  CHECK( buff_size >= info_size );
1946  spi = PSYSTEM_PROCESS_INFORMATION( buffer );
1947  for ( ; ; ) {
1948  ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
1949  CHECK( 0 <= offset && offset + sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
1950  HANDLE pid = spi->ProcessId;
1951  ULONG num = spi->NumberOfThreads;
1952  CHECK( num >= 1 );
1953  size_t spi_size = sizeof( SYSTEM_PROCESS_INFORMATION ) + sizeof( SYSTEM_THREAD ) * ( num - 1 );
1954  CHECK( offset + spi_size < info_size ); // Make sure process info record fits the buffer.
1955  if ( spi->NextEntryOffset != 0 ) {
1956  CHECK( spi_size <= spi->NextEntryOffset ); // And do not overlap with the next record.
1957  }; // if
1958  // pid == 0 corresponds to the System Idle Process. It always has running threads
1959  // on all cores. So, we don't consider the running threads of this process.
1960  if ( pid != 0 ) {
1961  for ( int i = 0; i < num; ++ i ) {
1962  THREAD_STATE state = spi->Threads[ i ].State;
1963  // Count threads that have Ready or Running state.
1964  // !!! TODO: Why comment does not match the code???
1965  if ( state == StateRunning ) {
1966  ++ running_threads;
1967  // Stop counting running threads if the number is already greater than
1968  // the number of available cores
1969  if ( running_threads >= max ) {
1970  goto finish;
1971  }
1972  } // if
1973  }; // for i
1974  } // if
1975  if ( spi->NextEntryOffset == 0 ) {
1976  break;
1977  }; // if
1978  spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
1979  }; // forever
1980 
1981  #undef CHECK
1982 
1983  finish: // Clean up and exit.
1984 
1985  if ( buffer != NULL ) {
1986  KMP_INTERNAL_FREE( buffer );
1987  }; // if
1988 
1989  glb_running_threads = running_threads;
1990 
1991  return running_threads;
1992 
1993 } //__kmp_get_load_balance()
1994