Intel® OpenMP* Runtime Library
 All Classes Functions Variables Typedefs Enumerations Enumerator Modules Pages
z_Linux_util.c
1 /*
2  * z_Linux_util.c -- platform specific routines.
3  */
4 
5 /* <copyright>
6  Copyright (c) 1997-2015 Intel Corporation. All Rights Reserved.
7 
8  Redistribution and use in source and binary forms, with or without
9  modification, are permitted provided that the following conditions
10  are met:
11 
12  * Redistributions of source code must retain the above copyright
13  notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  notice, this list of conditions and the following disclaimer in the
16  documentation and/or other materials provided with the distribution.
17  * Neither the name of Intel Corporation nor the names of its
18  contributors may be used to endorse or promote products derived
19  from this software without specific prior written permission.
20 
21  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 
33 </copyright> */
34 
35 #include "kmp.h"
36 #include "kmp_wrapper_getpid.h"
37 #include "kmp_itt.h"
38 #include "kmp_str.h"
39 #include "kmp_i18n.h"
40 #include "kmp_io.h"
41 #include "kmp_stats.h"
42 #include "kmp_wait_release.h"
43 
44 #if !KMP_OS_FREEBSD
45 # include <alloca.h>
46 #endif
47 #include <unistd.h>
48 #include <math.h> // HUGE_VAL.
49 #include <sys/time.h>
50 #include <sys/times.h>
51 #include <sys/resource.h>
52 #include <sys/syscall.h>
53 
54 #if KMP_OS_LINUX && !KMP_OS_CNK
55 # include <sys/sysinfo.h>
56 # if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
57 // We should really include <futex.h>, but that causes compatibility problems on different
58 // Linux* OS distributions that either require that you include (or break when you try to include)
59 // <pci/types.h>.
60 // Since all we need is the two macros below (which are part of the kernel ABI, so can't change)
61 // we just define the constants here and don't include <futex.h>
62 # ifndef FUTEX_WAIT
63 # define FUTEX_WAIT 0
64 # endif
65 # ifndef FUTEX_WAKE
66 # define FUTEX_WAKE 1
67 # endif
68 # endif
69 #elif KMP_OS_DARWIN
70 # include <sys/sysctl.h>
71 # include <mach/mach.h>
72 #elif KMP_OS_FREEBSD
73 # include <sys/sysctl.h>
74 # include <pthread_np.h>
75 #endif
76 
77 
78 #include <dirent.h>
79 #include <ctype.h>
80 #include <fcntl.h>
81 
82 // For non-x86 architecture
83 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
84 # include <stdbool.h>
85 # include <ffi.h>
86 #endif
87 
88 /* ------------------------------------------------------------------------ */
89 /* ------------------------------------------------------------------------ */
90 
91 struct kmp_sys_timer {
92  struct timespec start;
93 };
94 
95 // Convert timespec to nanoseconds.
96 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec)
97 
98 static struct kmp_sys_timer __kmp_sys_timer_data;
99 
100 #if KMP_HANDLE_SIGNALS
101  typedef void (* sig_func_t )( int );
102  STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[ NSIG ];
103  static sigset_t __kmp_sigset;
104 #endif
105 
106 static int __kmp_init_runtime = FALSE;
107 
108 static int __kmp_fork_count = 0;
109 
110 static pthread_condattr_t __kmp_suspend_cond_attr;
111 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
112 
113 static kmp_cond_align_t __kmp_wait_cv;
114 static kmp_mutex_align_t __kmp_wait_mx;
115 
116 /* ------------------------------------------------------------------------ */
117 /* ------------------------------------------------------------------------ */
118 
119 #ifdef DEBUG_SUSPEND
120 static void
121 __kmp_print_cond( char *buffer, kmp_cond_align_t *cond )
122 {
123  KMP_SNPRINTF( buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
124  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
125  cond->c_cond.__c_waiting );
126 }
127 #endif
128 
129 /* ------------------------------------------------------------------------ */
130 /* ------------------------------------------------------------------------ */
131 
132 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED)
133 
134 /*
135  * Affinity support
136  */
137 
138 /*
139  * On some of the older OS's that we build on, these constants aren't present
140  * in <asm/unistd.h> #included from <sys.syscall.h>. They must be the same on
141  * all systems of the same arch where they are defined, and they cannot change.
142  * stone forever.
143  */
144 
145 # if KMP_ARCH_X86 || KMP_ARCH_ARM
146 # ifndef __NR_sched_setaffinity
147 # define __NR_sched_setaffinity 241
148 # elif __NR_sched_setaffinity != 241
149 # error Wrong code for setaffinity system call.
150 # endif /* __NR_sched_setaffinity */
151 # ifndef __NR_sched_getaffinity
152 # define __NR_sched_getaffinity 242
153 # elif __NR_sched_getaffinity != 242
154 # error Wrong code for getaffinity system call.
155 # endif /* __NR_sched_getaffinity */
156 
157 # elif KMP_ARCH_AARCH64
158 # ifndef __NR_sched_setaffinity
159 # define __NR_sched_setaffinity 122
160 # elif __NR_sched_setaffinity != 122
161 # error Wrong code for setaffinity system call.
162 # endif /* __NR_sched_setaffinity */
163 # ifndef __NR_sched_getaffinity
164 # define __NR_sched_getaffinity 123
165 # elif __NR_sched_getaffinity != 123
166 # error Wrong code for getaffinity system call.
167 # endif /* __NR_sched_getaffinity */
168 
169 # elif KMP_ARCH_X86_64
170 # ifndef __NR_sched_setaffinity
171 # define __NR_sched_setaffinity 203
172 # elif __NR_sched_setaffinity != 203
173 # error Wrong code for setaffinity system call.
174 # endif /* __NR_sched_setaffinity */
175 # ifndef __NR_sched_getaffinity
176 # define __NR_sched_getaffinity 204
177 # elif __NR_sched_getaffinity != 204
178 # error Wrong code for getaffinity system call.
179 # endif /* __NR_sched_getaffinity */
180 
181 # elif KMP_ARCH_PPC64
182 # ifndef __NR_sched_setaffinity
183 # define __NR_sched_setaffinity 222
184 # elif __NR_sched_setaffinity != 222
185 # error Wrong code for setaffinity system call.
186 # endif /* __NR_sched_setaffinity */
187 # ifndef __NR_sched_getaffinity
188 # define __NR_sched_getaffinity 223
189 # elif __NR_sched_getaffinity != 223
190 # error Wrong code for getaffinity system call.
191 # endif /* __NR_sched_getaffinity */
192 
193 
194 # else
195 # error Unknown or unsupported architecture
196 
197 # endif /* KMP_ARCH_* */
198 
199 int
200 __kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
201 {
202  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
203  "Illegal set affinity operation when not capable");
204 
205  int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
206  if (retval >= 0) {
207  return 0;
208  }
209  int error = errno;
210  if (abort_on_error) {
211  __kmp_msg(
212  kmp_ms_fatal,
213  KMP_MSG( FatalSysError ),
214  KMP_ERR( error ),
215  __kmp_msg_null
216  );
217  }
218  return error;
219 }
220 
221 int
222 __kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
223 {
224  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
225  "Illegal get affinity operation when not capable");
226 
227  int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
228  if (retval >= 0) {
229  return 0;
230  }
231  int error = errno;
232  if (abort_on_error) {
233  __kmp_msg(
234  kmp_ms_fatal,
235  KMP_MSG( FatalSysError ),
236  KMP_ERR( error ),
237  __kmp_msg_null
238  );
239  }
240  return error;
241 }
242 
243 void
244 __kmp_affinity_bind_thread( int which )
245 {
246  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
247  "Illegal set affinity operation when not capable");
248 
249  kmp_affin_mask_t *mask = (kmp_affin_mask_t *)KMP_ALLOCA(__kmp_affin_mask_size);
250  KMP_CPU_ZERO(mask);
251  KMP_CPU_SET(which, mask);
252  __kmp_set_system_affinity(mask, TRUE);
253 }
254 
255 /*
256  * Determine if we can access affinity functionality on this version of
257  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
258  * __kmp_affin_mask_size to the appropriate value (0 means not capable).
259  */
260 void
261 __kmp_affinity_determine_capable(const char *env_var)
262 {
263  //
264  // Check and see if the OS supports thread affinity.
265  //
266 
267 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024)
268 
269  int gCode;
270  int sCode;
271  kmp_affin_mask_t *buf;
272  buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
273 
274  // If Linux* OS:
275  // If the syscall fails or returns a suggestion for the size,
276  // then we don't have to search for an appropriate size.
277  gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
278  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
279  "initial getaffinity call returned %d errno = %d\n",
280  gCode, errno));
281 
282  //if ((gCode < 0) && (errno == ENOSYS))
283  if (gCode < 0) {
284  //
285  // System call not supported
286  //
287  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
288  && (__kmp_affinity_type != affinity_none)
289  && (__kmp_affinity_type != affinity_default)
290  && (__kmp_affinity_type != affinity_disabled))) {
291  int error = errno;
292  __kmp_msg(
293  kmp_ms_warning,
294  KMP_MSG( GetAffSysCallNotSupported, env_var ),
295  KMP_ERR( error ),
296  __kmp_msg_null
297  );
298  }
299  KMP_AFFINITY_DISABLE();
300  KMP_INTERNAL_FREE(buf);
301  return;
302  }
303  if (gCode > 0) { // Linux* OS only
304  // The optimal situation: the OS returns the size of the buffer
305  // it expects.
306  //
307  // A verification of correct behavior is that Isetaffinity on a NULL
308  // buffer with the same size fails with errno set to EFAULT.
309  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
310  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
311  "setaffinity for mask size %d returned %d errno = %d\n",
312  gCode, sCode, errno));
313  if (sCode < 0) {
314  if (errno == ENOSYS) {
315  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
316  && (__kmp_affinity_type != affinity_none)
317  && (__kmp_affinity_type != affinity_default)
318  && (__kmp_affinity_type != affinity_disabled))) {
319  int error = errno;
320  __kmp_msg(
321  kmp_ms_warning,
322  KMP_MSG( SetAffSysCallNotSupported, env_var ),
323  KMP_ERR( error ),
324  __kmp_msg_null
325  );
326  }
327  KMP_AFFINITY_DISABLE();
328  KMP_INTERNAL_FREE(buf);
329  }
330  if (errno == EFAULT) {
331  KMP_AFFINITY_ENABLE(gCode);
332  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
333  "affinity supported (mask size %d)\n",
334  (int)__kmp_affin_mask_size));
335  KMP_INTERNAL_FREE(buf);
336  return;
337  }
338  }
339  }
340 
341  //
342  // Call the getaffinity system call repeatedly with increasing set sizes
343  // until we succeed, or reach an upper bound on the search.
344  //
345  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
346  "searching for proper set size\n"));
347  int size;
348  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
349  gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
350  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
351  "getaffinity for mask size %d returned %d errno = %d\n", size,
352  gCode, errno));
353 
354  if (gCode < 0) {
355  if ( errno == ENOSYS )
356  {
357  //
358  // We shouldn't get here
359  //
360  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
361  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
362  size));
363  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
364  && (__kmp_affinity_type != affinity_none)
365  && (__kmp_affinity_type != affinity_default)
366  && (__kmp_affinity_type != affinity_disabled))) {
367  int error = errno;
368  __kmp_msg(
369  kmp_ms_warning,
370  KMP_MSG( GetAffSysCallNotSupported, env_var ),
371  KMP_ERR( error ),
372  __kmp_msg_null
373  );
374  }
375  KMP_AFFINITY_DISABLE();
376  KMP_INTERNAL_FREE(buf);
377  return;
378  }
379  continue;
380  }
381 
382  sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
383  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
384  "setaffinity for mask size %d returned %d errno = %d\n",
385  gCode, sCode, errno));
386  if (sCode < 0) {
387  if (errno == ENOSYS) { // Linux* OS only
388  //
389  // We shouldn't get here
390  //
391  KA_TRACE(30, ( "__kmp_affinity_determine_capable: "
392  "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
393  size));
394  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
395  && (__kmp_affinity_type != affinity_none)
396  && (__kmp_affinity_type != affinity_default)
397  && (__kmp_affinity_type != affinity_disabled))) {
398  int error = errno;
399  __kmp_msg(
400  kmp_ms_warning,
401  KMP_MSG( SetAffSysCallNotSupported, env_var ),
402  KMP_ERR( error ),
403  __kmp_msg_null
404  );
405  }
406  KMP_AFFINITY_DISABLE();
407  KMP_INTERNAL_FREE(buf);
408  return;
409  }
410  if (errno == EFAULT) {
411  KMP_AFFINITY_ENABLE(gCode);
412  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
413  "affinity supported (mask size %d)\n",
414  (int)__kmp_affin_mask_size));
415  KMP_INTERNAL_FREE(buf);
416  return;
417  }
418  }
419  }
420  //int error = errno; // save uncaught error code
421  KMP_INTERNAL_FREE(buf);
422  // errno = error; // restore uncaught error code, will be printed at the next KMP_WARNING below
423 
424  //
425  // Affinity is not supported
426  //
427  KMP_AFFINITY_DISABLE();
428  KA_TRACE(10, ( "__kmp_affinity_determine_capable: "
429  "cannot determine mask size - affinity not supported\n"));
430  if (__kmp_affinity_verbose || (__kmp_affinity_warnings
431  && (__kmp_affinity_type != affinity_none)
432  && (__kmp_affinity_type != affinity_default)
433  && (__kmp_affinity_type != affinity_disabled))) {
434  KMP_WARNING( AffCantGetMaskSize, env_var );
435  }
436 }
437 
438 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
439 
440 /* ------------------------------------------------------------------------ */
441 /* ------------------------------------------------------------------------ */
442 
443 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && !KMP_OS_CNK
444 
445 int
446 __kmp_futex_determine_capable()
447 {
448  int loc = 0;
449  int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
450  int retval = ( rc == 0 ) || ( errno != ENOSYS );
451 
452  KA_TRACE(10, ( "__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
453  errno ) );
454  KA_TRACE(10, ( "__kmp_futex_determine_capable: futex syscall%s supported\n",
455  retval ? "" : " not" ) );
456 
457  return retval;
458 }
459 
460 #endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM) && !KMP_OS_CNK
461 
462 /* ------------------------------------------------------------------------ */
463 /* ------------------------------------------------------------------------ */
464 
465 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS)
466 /*
467  * Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
468  * use compare_and_store for these routines
469  */
470 
471 kmp_int8
472 __kmp_test_then_or8( volatile kmp_int8 *p, kmp_int8 d )
473 {
474  kmp_int8 old_value, new_value;
475 
476  old_value = TCR_1( *p );
477  new_value = old_value | d;
478 
479  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
480  {
481  KMP_CPU_PAUSE();
482  old_value = TCR_1( *p );
483  new_value = old_value | d;
484  }
485  return old_value;
486 }
487 
488 kmp_int8
489 __kmp_test_then_and8( volatile kmp_int8 *p, kmp_int8 d )
490 {
491  kmp_int8 old_value, new_value;
492 
493  old_value = TCR_1( *p );
494  new_value = old_value & d;
495 
496  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
497  {
498  KMP_CPU_PAUSE();
499  old_value = TCR_1( *p );
500  new_value = old_value & d;
501  }
502  return old_value;
503 }
504 
505 kmp_int32
506 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 d )
507 {
508  kmp_int32 old_value, new_value;
509 
510  old_value = TCR_4( *p );
511  new_value = old_value | d;
512 
513  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
514  {
515  KMP_CPU_PAUSE();
516  old_value = TCR_4( *p );
517  new_value = old_value | d;
518  }
519  return old_value;
520 }
521 
522 kmp_int32
523 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 d )
524 {
525  kmp_int32 old_value, new_value;
526 
527  old_value = TCR_4( *p );
528  new_value = old_value & d;
529 
530  while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
531  {
532  KMP_CPU_PAUSE();
533  old_value = TCR_4( *p );
534  new_value = old_value & d;
535  }
536  return old_value;
537 }
538 
539 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
540 kmp_int8
541 __kmp_test_then_add8( volatile kmp_int8 *p, kmp_int8 d )
542 {
543  kmp_int8 old_value, new_value;
544 
545  old_value = TCR_1( *p );
546  new_value = old_value + d;
547 
548  while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
549  {
550  KMP_CPU_PAUSE();
551  old_value = TCR_1( *p );
552  new_value = old_value + d;
553  }
554  return old_value;
555 }
556 
557 kmp_int64
558 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d )
559 {
560  kmp_int64 old_value, new_value;
561 
562  old_value = TCR_8( *p );
563  new_value = old_value + d;
564 
565  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
566  {
567  KMP_CPU_PAUSE();
568  old_value = TCR_8( *p );
569  new_value = old_value + d;
570  }
571  return old_value;
572 }
573 # endif /* KMP_ARCH_X86 */
574 
575 kmp_int64
576 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 d )
577 {
578  kmp_int64 old_value, new_value;
579 
580  old_value = TCR_8( *p );
581  new_value = old_value | d;
582  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
583  {
584  KMP_CPU_PAUSE();
585  old_value = TCR_8( *p );
586  new_value = old_value | d;
587  }
588  return old_value;
589 }
590 
591 kmp_int64
592 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 d )
593 {
594  kmp_int64 old_value, new_value;
595 
596  old_value = TCR_8( *p );
597  new_value = old_value & d;
598  while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
599  {
600  KMP_CPU_PAUSE();
601  old_value = TCR_8( *p );
602  new_value = old_value & d;
603  }
604  return old_value;
605 }
606 
607 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
608 
609 void
610 __kmp_terminate_thread( int gtid )
611 {
612  int status;
613  kmp_info_t *th = __kmp_threads[ gtid ];
614 
615  if ( !th ) return;
616 
617  #ifdef KMP_CANCEL_THREADS
618  KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
619  status = pthread_cancel( th->th.th_info.ds.ds_thread );
620  if ( status != 0 && status != ESRCH ) {
621  __kmp_msg(
622  kmp_ms_fatal,
623  KMP_MSG( CantTerminateWorkerThread ),
624  KMP_ERR( status ),
625  __kmp_msg_null
626  );
627  }; // if
628  #endif
629  __kmp_yield( TRUE );
630 } //
631 
632 /* ------------------------------------------------------------------------ */
633 /* ------------------------------------------------------------------------ */
634 
635 /* ------------------------------------------------------------------------ */
636 /* ------------------------------------------------------------------------ */
637 
638 /*
639  * Set thread stack info according to values returned by
640  * pthread_getattr_np().
641  * If values are unreasonable, assume call failed and use
642  * incremental stack refinement method instead.
643  * Returns TRUE if the stack parameters could be determined exactly,
644  * FALSE if incremental refinement is necessary.
645  */
646 static kmp_int32
647 __kmp_set_stack_info( int gtid, kmp_info_t *th )
648 {
649  int stack_data;
650 #if KMP_OS_LINUX || KMP_OS_FREEBSD
651  /* Linux* OS only -- no pthread_getattr_np support on OS X* */
652  pthread_attr_t attr;
653  int status;
654  size_t size = 0;
655  void * addr = 0;
656 
657  /* Always do incremental stack refinement for ubermaster threads since the initial
658  thread stack range can be reduced by sibling thread creation so pthread_attr_getstack
659  may cause thread gtid aliasing */
660  if ( ! KMP_UBER_GTID(gtid) ) {
661 
662  /* Fetch the real thread attributes */
663  status = pthread_attr_init( &attr );
664  KMP_CHECK_SYSFAIL( "pthread_attr_init", status );
665 #if KMP_OS_FREEBSD
666  status = pthread_attr_get_np( pthread_self(), &attr );
667  KMP_CHECK_SYSFAIL( "pthread_attr_get_np", status );
668 #else
669  status = pthread_getattr_np( pthread_self(), &attr );
670  KMP_CHECK_SYSFAIL( "pthread_getattr_np", status );
671 #endif
672  status = pthread_attr_getstack( &attr, &addr, &size );
673  KMP_CHECK_SYSFAIL( "pthread_attr_getstack", status );
674  KA_TRACE( 60, ( "__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, "
675  "low addr: %p\n",
676  gtid, size, addr ));
677 
678  status = pthread_attr_destroy( &attr );
679  KMP_CHECK_SYSFAIL( "pthread_attr_destroy", status );
680  }
681 
682  if ( size != 0 && addr != 0 ) { /* was stack parameter determination successful? */
683  /* Store the correct base and size */
684  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
685  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
686  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
687  return TRUE;
688  }
689 #endif /* KMP_OS_LINUX || KMP_OS_FREEBSD */
690  /* Use incremental refinement starting from initial conservative estimate */
691  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
692  TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
693  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
694  return FALSE;
695 }
696 
697 static void*
698 __kmp_launch_worker( void *thr )
699 {
700  int status, old_type, old_state;
701 #ifdef KMP_BLOCK_SIGNALS
702  sigset_t new_set, old_set;
703 #endif /* KMP_BLOCK_SIGNALS */
704  void *exit_val;
705  void *padding = 0;
706  int gtid;
707  int error;
708 
709  gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
710  __kmp_gtid_set_specific( gtid );
711 #ifdef KMP_TDATA_GTID
712  __kmp_gtid = gtid;
713 #endif
714 #if KMP_STATS_ENABLED
715  // set __thread local index to point to thread-specific stats
716  __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
717 #endif
718 
719 #if USE_ITT_BUILD
720  __kmp_itt_thread_name( gtid );
721 #endif /* USE_ITT_BUILD */
722 
723 #if KMP_AFFINITY_SUPPORTED
724  __kmp_affinity_set_init_mask( gtid, FALSE );
725 #endif
726 
727 #ifdef KMP_CANCEL_THREADS
728  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
729  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
730  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
731  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
732  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
733 #endif
734 
735 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
736  //
737  // Set the FP control regs to be a copy of
738  // the parallel initialization thread's.
739  //
740  __kmp_clear_x87_fpu_status_word();
741  __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
742  __kmp_load_mxcsr( &__kmp_init_mxcsr );
743 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
744 
745 #ifdef KMP_BLOCK_SIGNALS
746  status = sigfillset( & new_set );
747  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
748  status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
749  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
750 #endif /* KMP_BLOCK_SIGNALS */
751 
752 #if KMP_OS_LINUX || KMP_OS_FREEBSD
753  if ( __kmp_stkoffset > 0 && gtid > 0 ) {
754  padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
755  }
756 #endif
757 
758  KMP_MB();
759  __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
760 
761  __kmp_check_stack_overlap( (kmp_info_t*)thr );
762 
763  exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
764 
765 #ifdef KMP_BLOCK_SIGNALS
766  status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
767  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
768 #endif /* KMP_BLOCK_SIGNALS */
769 
770  return exit_val;
771 }
772 
773 
774 /* The monitor thread controls all of the threads in the complex */
775 
776 static void*
777 __kmp_launch_monitor( void *thr )
778 {
779  int status, old_type, old_state;
780 #ifdef KMP_BLOCK_SIGNALS
781  sigset_t new_set;
782 #endif /* KMP_BLOCK_SIGNALS */
783  struct timespec interval;
784  int yield_count;
785  int yield_cycles = 0;
786  int error;
787 
788  KMP_MB(); /* Flush all pending memory write invalidates. */
789 
790  KA_TRACE( 10, ("__kmp_launch_monitor: #1 launched\n" ) );
791 
792  /* register us as the monitor thread */
793  __kmp_gtid_set_specific( KMP_GTID_MONITOR );
794 #ifdef KMP_TDATA_GTID
795  __kmp_gtid = KMP_GTID_MONITOR;
796 #endif
797 
798  KMP_MB();
799 
800 #if USE_ITT_BUILD
801  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
802 #endif /* USE_ITT_BUILD */
803 
804  __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
805 
806  __kmp_check_stack_overlap( (kmp_info_t*)thr );
807 
808 #ifdef KMP_CANCEL_THREADS
809  status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
810  KMP_CHECK_SYSFAIL( "pthread_setcanceltype", status );
811  /* josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads? */
812  status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
813  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
814 #endif
815 
816  #if KMP_REAL_TIME_FIX
817  // This is a potential fix which allows application with real-time scheduling policy work.
818  // However, decision about the fix is not made yet, so it is disabled by default.
819  { // Are program started with real-time scheduling policy?
820  int sched = sched_getscheduler( 0 );
821  if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
822  // Yes, we are a part of real-time application. Try to increase the priority of the
823  // monitor.
824  struct sched_param param;
825  int max_priority = sched_get_priority_max( sched );
826  int rc;
827  KMP_WARNING( RealTimeSchedNotSupported );
828  sched_getparam( 0, & param );
829  if ( param.sched_priority < max_priority ) {
830  param.sched_priority += 1;
831  rc = sched_setscheduler( 0, sched, & param );
832  if ( rc != 0 ) {
833  int error = errno;
834  __kmp_msg(
835  kmp_ms_warning,
836  KMP_MSG( CantChangeMonitorPriority ),
837  KMP_ERR( error ),
838  KMP_MSG( MonitorWillStarve ),
839  __kmp_msg_null
840  );
841  }; // if
842  } else {
843  // We cannot abort here, because number of CPUs may be enough for all the threads,
844  // including the monitor thread, so application could potentially work...
845  __kmp_msg(
846  kmp_ms_warning,
847  KMP_MSG( RunningAtMaxPriority ),
848  KMP_MSG( MonitorWillStarve ),
849  KMP_HNT( RunningAtMaxPriority ),
850  __kmp_msg_null
851  );
852  }; // if
853  }; // if
854  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 ); // AC: free thread that waits for monitor started
855  }
856  #endif // KMP_REAL_TIME_FIX
857 
858  KMP_MB(); /* Flush all pending memory write invalidates. */
859 
860  if ( __kmp_monitor_wakeups == 1 ) {
861  interval.tv_sec = 1;
862  interval.tv_nsec = 0;
863  } else {
864  interval.tv_sec = 0;
865  interval.tv_nsec = (NSEC_PER_SEC / __kmp_monitor_wakeups);
866  }
867 
868  KA_TRACE( 10, ("__kmp_launch_monitor: #2 monitor\n" ) );
869 
870  if (__kmp_yield_cycle) {
871  __kmp_yielding_on = 0; /* Start out with yielding shut off */
872  yield_count = __kmp_yield_off_count;
873  } else {
874  __kmp_yielding_on = 1; /* Yielding is on permanently */
875  }
876 
877  while( ! TCR_4( __kmp_global.g.g_done ) ) {
878  struct timespec now;
879  struct timeval tval;
880 
881  /* This thread monitors the state of the system */
882 
883  KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
884 
885  status = gettimeofday( &tval, NULL );
886  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
887  TIMEVAL_TO_TIMESPEC( &tval, &now );
888 
889  now.tv_sec += interval.tv_sec;
890  now.tv_nsec += interval.tv_nsec;
891 
892  if (now.tv_nsec >= NSEC_PER_SEC) {
893  now.tv_sec += 1;
894  now.tv_nsec -= NSEC_PER_SEC;
895  }
896 
897  status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
898  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
899  // AC: the monitor should not fall asleep if g_done has been set
900  if ( !TCR_4(__kmp_global.g.g_done) ) { // check once more under mutex
901  status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
902  if ( status != 0 ) {
903  if ( status != ETIMEDOUT && status != EINTR ) {
904  KMP_SYSFAIL( "pthread_cond_timedwait", status );
905  };
906  };
907  };
908  status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
909  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
910 
911  if (__kmp_yield_cycle) {
912  yield_cycles++;
913  if ( (yield_cycles % yield_count) == 0 ) {
914  if (__kmp_yielding_on) {
915  __kmp_yielding_on = 0; /* Turn it off now */
916  yield_count = __kmp_yield_off_count;
917  } else {
918  __kmp_yielding_on = 1; /* Turn it on now */
919  yield_count = __kmp_yield_on_count;
920  }
921  yield_cycles = 0;
922  }
923  } else {
924  __kmp_yielding_on = 1;
925  }
926 
927  TCW_4( __kmp_global.g.g_time.dt.t_value,
928  TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
929 
930  KMP_MB(); /* Flush all pending memory write invalidates. */
931  }
932 
933  KA_TRACE( 10, ("__kmp_launch_monitor: #3 cleanup\n" ) );
934 
935 #ifdef KMP_BLOCK_SIGNALS
936  status = sigfillset( & new_set );
937  KMP_CHECK_SYSFAIL_ERRNO( "sigfillset", status );
938  status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
939  KMP_CHECK_SYSFAIL( "pthread_sigmask", status );
940 #endif /* KMP_BLOCK_SIGNALS */
941 
942  KA_TRACE( 10, ("__kmp_launch_monitor: #4 finished\n" ) );
943 
944  if( __kmp_global.g.g_abort != 0 ) {
945  /* now we need to terminate the worker threads */
946  /* the value of t_abort is the signal we caught */
947 
948  int gtid;
949 
950  KA_TRACE( 10, ("__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
951 
952  /* terminate the OpenMP worker threads */
953  /* TODO this is not valid for sibling threads!!
954  * the uber master might not be 0 anymore.. */
955  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
956  __kmp_terminate_thread( gtid );
957 
958  __kmp_cleanup();
959 
960  KA_TRACE( 10, ("__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
961 
962  if (__kmp_global.g.g_abort > 0)
963  raise( __kmp_global.g.g_abort );
964 
965  }
966 
967  KA_TRACE( 10, ("__kmp_launch_monitor: #7 exit\n" ) );
968 
969  return thr;
970 }
971 
972 void
973 __kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
974 {
975  pthread_t handle;
976  pthread_attr_t thread_attr;
977  int status;
978 
979 
980  th->th.th_info.ds.ds_gtid = gtid;
981 
982 #if KMP_STATS_ENABLED
983  // sets up worker thread stats
984  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
985 
986  // th->th.th_stats is used to transfer thread specific stats-pointer to __kmp_launch_worker
987  // So when thread is created (goes into __kmp_launch_worker) it will
988  // set it's __thread local pointer to th->th.th_stats
989  th->th.th_stats = __kmp_stats_list.push_back(gtid);
990  if(KMP_UBER_GTID(gtid)) {
991  __kmp_stats_start_time = tsc_tick_count::now();
992  __kmp_stats_thread_ptr = th->th.th_stats;
993  __kmp_stats_init();
994  KMP_START_EXPLICIT_TIMER(OMP_serial);
995  KMP_START_EXPLICIT_TIMER(OMP_start_end);
996  }
997  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
998 
999 #endif // KMP_STATS_ENABLED
1000 
1001  if ( KMP_UBER_GTID(gtid) ) {
1002  KA_TRACE( 10, ("__kmp_create_worker: uber thread (%d)\n", gtid ) );
1003  th -> th.th_info.ds.ds_thread = pthread_self();
1004  __kmp_set_stack_info( gtid, th );
1005  __kmp_check_stack_overlap( th );
1006  return;
1007  }; // if
1008 
1009  KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1010 
1011  KMP_MB(); /* Flush all pending memory write invalidates. */
1012 
1013 #ifdef KMP_THREAD_ATTR
1014  {
1015  status = pthread_attr_init( &thread_attr );
1016  if ( status != 0 ) {
1017  __kmp_msg(
1018  kmp_ms_fatal,
1019  KMP_MSG( CantInitThreadAttrs ),
1020  KMP_ERR( status ),
1021  __kmp_msg_null
1022  );
1023  }; // if
1024  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1025  if ( status != 0 ) {
1026  __kmp_msg(
1027  kmp_ms_fatal,
1028  KMP_MSG( CantSetWorkerState ),
1029  KMP_ERR( status ),
1030  __kmp_msg_null
1031  );
1032  }; // if
1033 
1034  /* Set stack size for this thread now. */
1035  stack_size += gtid * __kmp_stkoffset;
1036 
1037  KA_TRACE( 10, ( "__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1038  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1039  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1040 
1041 # ifdef _POSIX_THREAD_ATTR_STACKSIZE
1042  status = pthread_attr_setstacksize( & thread_attr, stack_size );
1043 # ifdef KMP_BACKUP_STKSIZE
1044  if ( status != 0 ) {
1045  if ( ! __kmp_env_stksize ) {
1046  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1047  __kmp_stksize = KMP_BACKUP_STKSIZE;
1048  KA_TRACE( 10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
1049  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
1050  "bytes\n",
1051  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1052  );
1053  status = pthread_attr_setstacksize( &thread_attr, stack_size );
1054  }; // if
1055  }; // if
1056 # endif /* KMP_BACKUP_STKSIZE */
1057  if ( status != 0 ) {
1058  __kmp_msg(
1059  kmp_ms_fatal,
1060  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1061  KMP_ERR( status ),
1062  KMP_HNT( ChangeWorkerStackSize ),
1063  __kmp_msg_null
1064  );
1065  }; // if
1066 # endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1067  }
1068 #endif /* KMP_THREAD_ATTR */
1069 
1070  {
1071  status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (void *) th );
1072  if ( status != 0 || ! handle ) { // ??? Why do we check handle??
1073 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1074  if ( status == EINVAL ) {
1075  __kmp_msg(
1076  kmp_ms_fatal,
1077  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1078  KMP_ERR( status ),
1079  KMP_HNT( IncreaseWorkerStackSize ),
1080  __kmp_msg_null
1081  );
1082  };
1083  if ( status == ENOMEM ) {
1084  __kmp_msg(
1085  kmp_ms_fatal,
1086  KMP_MSG( CantSetWorkerStackSize, stack_size ),
1087  KMP_ERR( status ),
1088  KMP_HNT( DecreaseWorkerStackSize ),
1089  __kmp_msg_null
1090  );
1091  };
1092 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1093  if ( status == EAGAIN ) {
1094  __kmp_msg(
1095  kmp_ms_fatal,
1096  KMP_MSG( NoResourcesForWorkerThread ),
1097  KMP_ERR( status ),
1098  KMP_HNT( Decrease_NUM_THREADS ),
1099  __kmp_msg_null
1100  );
1101  }; // if
1102  KMP_SYSFAIL( "pthread_create", status );
1103  }; // if
1104 
1105  th->th.th_info.ds.ds_thread = handle;
1106  }
1107 
1108 #ifdef KMP_THREAD_ATTR
1109  {
1110  status = pthread_attr_destroy( & thread_attr );
1111  if ( status ) {
1112  __kmp_msg(
1113  kmp_ms_warning,
1114  KMP_MSG( CantDestroyThreadAttrs ),
1115  KMP_ERR( status ),
1116  __kmp_msg_null
1117  );
1118  }; // if
1119  }
1120 #endif /* KMP_THREAD_ATTR */
1121 
1122  KMP_MB(); /* Flush all pending memory write invalidates. */
1123 
1124  KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1125 
1126 } // __kmp_create_worker
1127 
1128 
1129 void
1130 __kmp_create_monitor( kmp_info_t *th )
1131 {
1132  pthread_t handle;
1133  pthread_attr_t thread_attr;
1134  size_t size;
1135  int status;
1136  int caller_gtid = __kmp_get_gtid();
1137  int auto_adj_size = FALSE;
1138 
1139  KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
1140 
1141  KMP_MB(); /* Flush all pending memory write invalidates. */
1142 
1143  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1144  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1145  #if KMP_REAL_TIME_FIX
1146  TCW_4( __kmp_global.g.g_time.dt.t_value, -1 ); // Will use it for synchronization a bit later.
1147  #else
1148  TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1149  #endif // KMP_REAL_TIME_FIX
1150 
1151  #ifdef KMP_THREAD_ATTR
1152  if ( __kmp_monitor_stksize == 0 ) {
1153  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1154  auto_adj_size = TRUE;
1155  }
1156  status = pthread_attr_init( &thread_attr );
1157  if ( status != 0 ) {
1158  __kmp_msg(
1159  kmp_ms_fatal,
1160  KMP_MSG( CantInitThreadAttrs ),
1161  KMP_ERR( status ),
1162  __kmp_msg_null
1163  );
1164  }; // if
1165  status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1166  if ( status != 0 ) {
1167  __kmp_msg(
1168  kmp_ms_fatal,
1169  KMP_MSG( CantSetMonitorState ),
1170  KMP_ERR( status ),
1171  __kmp_msg_null
1172  );
1173  }; // if
1174 
1175  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1176  status = pthread_attr_getstacksize( & thread_attr, & size );
1177  KMP_CHECK_SYSFAIL( "pthread_attr_getstacksize", status );
1178  #else
1179  size = __kmp_sys_min_stksize;
1180  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1181  #endif /* KMP_THREAD_ATTR */
1182 
1183  if ( __kmp_monitor_stksize == 0 ) {
1184  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1185  }
1186  if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1187  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1188  }
1189 
1190  KA_TRACE( 10, ( "__kmp_create_monitor: default stacksize = %lu bytes,"
1191  "requested stacksize = %lu bytes\n",
1192  size, __kmp_monitor_stksize ) );
1193 
1194  retry:
1195 
1196  /* Set stack size for this thread now. */
1197 
1198  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1199  KA_TRACE( 10, ( "__kmp_create_monitor: setting stacksize = %lu bytes,",
1200  __kmp_monitor_stksize ) );
1201  status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1202  if ( status != 0 ) {
1203  if ( auto_adj_size ) {
1204  __kmp_monitor_stksize *= 2;
1205  goto retry;
1206  }
1207  __kmp_msg(
1208  kmp_ms_warning, // should this be fatal? BB
1209  KMP_MSG( CantSetMonitorStackSize, (long int) __kmp_monitor_stksize ),
1210  KMP_ERR( status ),
1211  KMP_HNT( ChangeMonitorStackSize ),
1212  __kmp_msg_null
1213  );
1214  }; // if
1215  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1216 
1217  status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (void *) th );
1218 
1219  if ( status != 0 ) {
1220  #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1221  if ( status == EINVAL ) {
1222  if ( auto_adj_size && ( __kmp_monitor_stksize < (size_t)0x40000000 ) ) {
1223  __kmp_monitor_stksize *= 2;
1224  goto retry;
1225  }
1226  __kmp_msg(
1227  kmp_ms_fatal,
1228  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1229  KMP_ERR( status ),
1230  KMP_HNT( IncreaseMonitorStackSize ),
1231  __kmp_msg_null
1232  );
1233  }; // if
1234  if ( status == ENOMEM ) {
1235  __kmp_msg(
1236  kmp_ms_fatal,
1237  KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1238  KMP_ERR( status ),
1239  KMP_HNT( DecreaseMonitorStackSize ),
1240  __kmp_msg_null
1241  );
1242  }; // if
1243  #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1244  if ( status == EAGAIN ) {
1245  __kmp_msg(
1246  kmp_ms_fatal,
1247  KMP_MSG( NoResourcesForMonitorThread ),
1248  KMP_ERR( status ),
1249  KMP_HNT( DecreaseNumberOfThreadsInUse ),
1250  __kmp_msg_null
1251  );
1252  }; // if
1253  KMP_SYSFAIL( "pthread_create", status );
1254  }; // if
1255 
1256  th->th.th_info.ds.ds_thread = handle;
1257 
1258  #if KMP_REAL_TIME_FIX
1259  // Wait for the monitor thread is really started and set its *priority*.
1260  KMP_DEBUG_ASSERT( sizeof( kmp_uint32 ) == sizeof( __kmp_global.g.g_time.dt.t_value ) );
1261  __kmp_wait_yield_4(
1262  (kmp_uint32 volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1263  );
1264  #endif // KMP_REAL_TIME_FIX
1265 
1266  #ifdef KMP_THREAD_ATTR
1267  status = pthread_attr_destroy( & thread_attr );
1268  if ( status != 0 ) {
1269  __kmp_msg( //
1270  kmp_ms_warning,
1271  KMP_MSG( CantDestroyThreadAttrs ),
1272  KMP_ERR( status ),
1273  __kmp_msg_null
1274  );
1275  }; // if
1276  #endif
1277 
1278  KMP_MB(); /* Flush all pending memory write invalidates. */
1279 
1280  KA_TRACE( 10, ( "__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1281 
1282 } // __kmp_create_monitor
1283 
1284 void
1285 __kmp_exit_thread(
1286  int exit_status
1287 ) {
1288  pthread_exit( (void *)(intptr_t) exit_status );
1289 } // __kmp_exit_thread
1290 
1291 void __kmp_resume_monitor();
1292 
1293 void
1294 __kmp_reap_monitor( kmp_info_t *th )
1295 {
1296  int status, i;
1297  void *exit_val;
1298 
1299  KA_TRACE( 10, ("__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1300  th->th.th_info.ds.ds_thread ) );
1301 
1302  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1303  // If both tid and gtid are 0, it means the monitor did not ever start.
1304  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1305  KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1306  if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1307  return;
1308  }; // if
1309 
1310  KMP_MB(); /* Flush all pending memory write invalidates. */
1311 
1312 
1313  /* First, check to see whether the monitor thread exists. This could prevent a hang,
1314  but if the monitor dies after the pthread_kill call and before the pthread_join
1315  call, it will still hang. */
1316 
1317  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1318  if (status == ESRCH) {
1319 
1320  KA_TRACE( 10, ("__kmp_reap_monitor: monitor does not exist, returning\n") );
1321 
1322  } else
1323  {
1324  __kmp_resume_monitor(); // Wake up the monitor thread
1325  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1326  if (exit_val != th) {
1327  __kmp_msg(
1328  kmp_ms_fatal,
1329  KMP_MSG( ReapMonitorError ),
1330  KMP_ERR( status ),
1331  __kmp_msg_null
1332  );
1333  }
1334  }
1335 
1336  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1337  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1338 
1339  KA_TRACE( 10, ("__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1340  th->th.th_info.ds.ds_thread ) );
1341 
1342  KMP_MB(); /* Flush all pending memory write invalidates. */
1343 
1344 }
1345 
1346 void
1347 __kmp_reap_worker( kmp_info_t *th )
1348 {
1349  int status;
1350  void *exit_val;
1351 
1352  KMP_MB(); /* Flush all pending memory write invalidates. */
1353 
1354  KA_TRACE( 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1355 
1356  /* First, check to see whether the worker thread exists. This could prevent a hang,
1357  but if the worker dies after the pthread_kill call and before the pthread_join
1358  call, it will still hang. */
1359 
1360  {
1361  status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1362  if (status == ESRCH) {
1363  KA_TRACE( 10, ("__kmp_reap_worker: worker T#%d does not exist, returning\n",
1364  th->th.th_info.ds.ds_gtid ) );
1365  }
1366  else {
1367  KA_TRACE( 10, ("__kmp_reap_worker: try to join with worker T#%d\n",
1368  th->th.th_info.ds.ds_gtid ) );
1369 
1370  status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1371 #ifdef KMP_DEBUG
1372  /* Don't expose these to the user until we understand when they trigger */
1373  if ( status != 0 ) {
1374  __kmp_msg(
1375  kmp_ms_fatal,
1376  KMP_MSG( ReapWorkerError ),
1377  KMP_ERR( status ),
1378  __kmp_msg_null
1379  );
1380  }
1381  if ( exit_val != th ) {
1382  KA_TRACE( 10, ( "__kmp_reap_worker: worker T#%d did not reap properly, "
1383  "exit_val = %p\n",
1384  th->th.th_info.ds.ds_gtid, exit_val ) );
1385  }
1386 #endif /* KMP_DEBUG */
1387  }
1388  }
1389 
1390  KA_TRACE( 10, ("__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1391 
1392  KMP_MB(); /* Flush all pending memory write invalidates. */
1393 }
1394 
1395 
1396 /* ------------------------------------------------------------------------ */
1397 /* ------------------------------------------------------------------------ */
1398 
1399 #if KMP_HANDLE_SIGNALS
1400 
1401 
1402 static void
1403 __kmp_null_handler( int signo )
1404 {
1405  // Do nothing, for doing SIG_IGN-type actions.
1406 } // __kmp_null_handler
1407 
1408 
1409 static void
1410 __kmp_team_handler( int signo )
1411 {
1412  if ( __kmp_global.g.g_abort == 0 ) {
1413  /* Stage 1 signal handler, let's shut down all of the threads */
1414  #ifdef KMP_DEBUG
1415  __kmp_debug_printf( "__kmp_team_handler: caught signal = %d\n", signo );
1416  #endif
1417  switch ( signo ) {
1418  case SIGHUP :
1419  case SIGINT :
1420  case SIGQUIT :
1421  case SIGILL :
1422  case SIGABRT :
1423  case SIGFPE :
1424  case SIGBUS :
1425  case SIGSEGV :
1426  #ifdef SIGSYS
1427  case SIGSYS :
1428  #endif
1429  case SIGTERM :
1430  if ( __kmp_debug_buf ) {
1431  __kmp_dump_debug_buffer( );
1432  }; // if
1433  KMP_MB(); // Flush all pending memory write invalidates.
1434  TCW_4( __kmp_global.g.g_abort, signo );
1435  KMP_MB(); // Flush all pending memory write invalidates.
1436  TCW_4( __kmp_global.g.g_done, TRUE );
1437  KMP_MB(); // Flush all pending memory write invalidates.
1438  break;
1439  default:
1440  #ifdef KMP_DEBUG
1441  __kmp_debug_printf( "__kmp_team_handler: unknown signal type" );
1442  #endif
1443  break;
1444  }; // switch
1445  }; // if
1446 } // __kmp_team_handler
1447 
1448 
1449 static
1450 void __kmp_sigaction( int signum, const struct sigaction * act, struct sigaction * oldact ) {
1451  int rc = sigaction( signum, act, oldact );
1452  KMP_CHECK_SYSFAIL_ERRNO( "sigaction", rc );
1453 }
1454 
1455 
1456 static void
1457 __kmp_install_one_handler( int sig, sig_func_t handler_func, int parallel_init )
1458 {
1459  KMP_MB(); // Flush all pending memory write invalidates.
1460  KB_TRACE( 60, ( "__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1461  if ( parallel_init ) {
1462  struct sigaction new_action;
1463  struct sigaction old_action;
1464  new_action.sa_handler = handler_func;
1465  new_action.sa_flags = 0;
1466  sigfillset( & new_action.sa_mask );
1467  __kmp_sigaction( sig, & new_action, & old_action );
1468  if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1469  sigaddset( & __kmp_sigset, sig );
1470  } else {
1471  // Restore/keep user's handler if one previously installed.
1472  __kmp_sigaction( sig, & old_action, NULL );
1473  }; // if
1474  } else {
1475  // Save initial/system signal handlers to see if user handlers installed.
1476  __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1477  }; // if
1478  KMP_MB(); // Flush all pending memory write invalidates.
1479 } // __kmp_install_one_handler
1480 
1481 
1482 static void
1483 __kmp_remove_one_handler( int sig )
1484 {
1485  KB_TRACE( 60, ( "__kmp_remove_one_handler( %d )\n", sig ) );
1486  if ( sigismember( & __kmp_sigset, sig ) ) {
1487  struct sigaction old;
1488  KMP_MB(); // Flush all pending memory write invalidates.
1489  __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1490  if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1491  // Restore the users signal handler.
1492  KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1493  __kmp_sigaction( sig, & old, NULL );
1494  }; // if
1495  sigdelset( & __kmp_sigset, sig );
1496  KMP_MB(); // Flush all pending memory write invalidates.
1497  }; // if
1498 } // __kmp_remove_one_handler
1499 
1500 
1501 void
1502 __kmp_install_signals( int parallel_init )
1503 {
1504  KB_TRACE( 10, ( "__kmp_install_signals( %d )\n", parallel_init ) );
1505  if ( __kmp_handle_signals || ! parallel_init ) {
1506  // If ! parallel_init, we do not install handlers, just save original handlers.
1507  // Let us do it even __handle_signals is 0.
1508  sigemptyset( & __kmp_sigset );
1509  __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1510  __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1511  __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1512  __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1513  __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1514  __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1515  __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1516  __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1517  #ifdef SIGSYS
1518  __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1519  #endif // SIGSYS
1520  __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1521  #ifdef SIGPIPE
1522  __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1523  #endif // SIGPIPE
1524  }; // if
1525 } // __kmp_install_signals
1526 
1527 
1528 void
1529 __kmp_remove_signals( void )
1530 {
1531  int sig;
1532  KB_TRACE( 10, ( "__kmp_remove_signals()\n" ) );
1533  for ( sig = 1; sig < NSIG; ++ sig ) {
1534  __kmp_remove_one_handler( sig );
1535  }; // for sig
1536 } // __kmp_remove_signals
1537 
1538 
1539 #endif // KMP_HANDLE_SIGNALS
1540 
1541 /* ------------------------------------------------------------------------ */
1542 /* ------------------------------------------------------------------------ */
1543 
1544 void
1545 __kmp_enable( int new_state )
1546 {
1547  #ifdef KMP_CANCEL_THREADS
1548  int status, old_state;
1549  status = pthread_setcancelstate( new_state, & old_state );
1550  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1551  KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1552  #endif
1553 }
1554 
1555 void
1556 __kmp_disable( int * old_state )
1557 {
1558  #ifdef KMP_CANCEL_THREADS
1559  int status;
1560  status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1561  KMP_CHECK_SYSFAIL( "pthread_setcancelstate", status );
1562  #endif
1563 }
1564 
1565 /* ------------------------------------------------------------------------ */
1566 /* ------------------------------------------------------------------------ */
1567 
1568 static void
1569 __kmp_atfork_prepare (void)
1570 {
1571  /* nothing to do */
1572 }
1573 
1574 static void
1575 __kmp_atfork_parent (void)
1576 {
1577  /* nothing to do */
1578 }
1579 
1580 /*
1581  Reset the library so execution in the child starts "all over again" with
1582  clean data structures in initial states. Don't worry about freeing memory
1583  allocated by parent, just abandon it to be safe.
1584 */
1585 static void
1586 __kmp_atfork_child (void)
1587 {
1588  /* TODO make sure this is done right for nested/sibling */
1589  // ATT: Memory leaks are here? TODO: Check it and fix.
1590  /* KMP_ASSERT( 0 ); */
1591 
1592  ++__kmp_fork_count;
1593 
1594  __kmp_init_runtime = FALSE;
1595  __kmp_init_monitor = 0;
1596  __kmp_init_parallel = FALSE;
1597  __kmp_init_middle = FALSE;
1598  __kmp_init_serial = FALSE;
1599  TCW_4(__kmp_init_gtid, FALSE);
1600  __kmp_init_common = FALSE;
1601 
1602  TCW_4(__kmp_init_user_locks, FALSE);
1603 #if ! KMP_USE_DYNAMIC_LOCK
1604  __kmp_user_lock_table.used = 1;
1605  __kmp_user_lock_table.allocated = 0;
1606  __kmp_user_lock_table.table = NULL;
1607  __kmp_lock_blocks = NULL;
1608 #endif
1609 
1610  __kmp_all_nth = 0;
1611  TCW_4(__kmp_nth, 0);
1612 
1613  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate here
1614  so threadprivate doesn't use stale data */
1615  KA_TRACE( 10, ( "__kmp_atfork_child: checking cache address list %p\n",
1616  __kmp_threadpriv_cache_list ) );
1617 
1618  while ( __kmp_threadpriv_cache_list != NULL ) {
1619 
1620  if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1621  KC_TRACE( 50, ( "__kmp_atfork_child: zeroing cache at address %p\n",
1622  &(*__kmp_threadpriv_cache_list -> addr) ) );
1623 
1624  *__kmp_threadpriv_cache_list -> addr = NULL;
1625  }
1626  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1627  }
1628 
1629  __kmp_init_runtime = FALSE;
1630 
1631  /* reset statically initialized locks */
1632  __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1633  __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1634  __kmp_init_bootstrap_lock( &__kmp_console_lock );
1635 
1636  /* This is necessary to make sure no stale data is left around */
1637  /* AC: customers complain that we use unsafe routines in the atfork
1638  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1639  in dynamic_link when check the presence of shared tbbmalloc library.
1640  Suggestion is to make the library initialization lazier, similar
1641  to what done for __kmpc_begin(). */
1642  // TODO: synchronize all static initializations with regular library
1643  // startup; look at kmp_global.c and etc.
1644  //__kmp_internal_begin ();
1645 
1646 }
1647 
1648 void
1649 __kmp_register_atfork(void) {
1650  if ( __kmp_need_register_atfork ) {
1651  int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1652  KMP_CHECK_SYSFAIL( "pthread_atfork", status );
1653  __kmp_need_register_atfork = FALSE;
1654  }
1655 }
1656 
1657 void
1658 __kmp_suspend_initialize( void )
1659 {
1660  int status;
1661  status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1662  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
1663  status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1664  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
1665 }
1666 
1667 static void
1668 __kmp_suspend_initialize_thread( kmp_info_t *th )
1669 {
1670  if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1671  /* this means we haven't initialized the suspension pthread objects for this thread
1672  in this instance of the process */
1673  int status;
1674  status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1675  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
1676  status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1677  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
1678  *(volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1679  };
1680 }
1681 
1682 void
1683 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1684 {
1685  if(th->th.th_suspend_init_count > __kmp_fork_count) {
1686  /* this means we have initialize the suspension pthread objects for this thread
1687  in this instance of the process */
1688  int status;
1689 
1690  status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1691  if ( status != 0 && status != EBUSY ) {
1692  KMP_SYSFAIL( "pthread_cond_destroy", status );
1693  };
1694  status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1695  if ( status != 0 && status != EBUSY ) {
1696  KMP_SYSFAIL( "pthread_mutex_destroy", status );
1697  };
1698  --th->th.th_suspend_init_count;
1699  KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1700  }
1701 }
1702 
1703 /* This routine puts the calling thread to sleep after setting the
1704  * sleep bit for the indicated flag variable to true.
1705  */
1706 template <class C>
1707 static inline void __kmp_suspend_template( int th_gtid, C *flag )
1708 {
1709  KMP_TIME_BLOCK(USER_suspend);
1710  kmp_info_t *th = __kmp_threads[th_gtid];
1711  int status;
1712  typename C::flag_t old_spin;
1713 
1714  KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1715 
1716  __kmp_suspend_initialize_thread( th );
1717 
1718  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1719  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1720 
1721  KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1722  th_gtid, flag->get() ) );
1723 
1724  /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
1725  gets called first?
1726  */
1727  old_spin = flag->set_sleeping();
1728 
1729  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%d\n",
1730  th_gtid, flag->get(), *(flag->get()) ) );
1731 
1732  if ( flag->done_check_val(old_spin) ) {
1733  old_spin = flag->unset_sleeping();
1734  KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1735  th_gtid, flag->get()) );
1736  } else {
1737  /* Encapsulate in a loop as the documentation states that this may
1738  * "with low probability" return when the condition variable has
1739  * not been signaled or broadcast
1740  */
1741  int deactivated = FALSE;
1742  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1743  while ( flag->is_sleeping() ) {
1744 #ifdef DEBUG_SUSPEND
1745  char buffer[128];
1746  __kmp_suspend_count++;
1747  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1748  __kmp_printf( "__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1749 #endif
1750  // Mark the thread as no longer active (only in the first iteration of the loop).
1751  if ( ! deactivated ) {
1752  th->th.th_active = FALSE;
1753  if ( th->th.th_active_in_pool ) {
1754  th->th.th_active_in_pool = FALSE;
1755  KMP_TEST_THEN_DEC32(
1756  (kmp_int32 *) &__kmp_thread_pool_active_nth );
1757  KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1758  }
1759  deactivated = TRUE;
1760 
1761 
1762  }
1763 
1764 #if USE_SUSPEND_TIMEOUT
1765  struct timespec now;
1766  struct timeval tval;
1767  int msecs;
1768 
1769  status = gettimeofday( &tval, NULL );
1770  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
1771  TIMEVAL_TO_TIMESPEC( &tval, &now );
1772 
1773  msecs = (4*__kmp_dflt_blocktime) + 200;
1774  now.tv_sec += msecs / 1000;
1775  now.tv_nsec += (msecs % 1000)*1000;
1776 
1777  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1778  th_gtid ) );
1779  status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1780 #else
1781  KF_TRACE( 15, ( "__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1782  th_gtid ) );
1783 
1784  status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1785 #endif
1786 
1787  if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1788  KMP_SYSFAIL( "pthread_cond_wait", status );
1789  }
1790 #ifdef KMP_DEBUG
1791  if (status == ETIMEDOUT) {
1792  if ( flag->is_sleeping() ) {
1793  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1794  } else {
1795  KF_TRACE( 2, ( "__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1796  th_gtid ) );
1797  }
1798  } else if ( flag->is_sleeping() ) {
1799  KF_TRACE( 100, ( "__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1800  }
1801 #endif
1802  } // while
1803 
1804  // Mark the thread as active again (if it was previous marked as inactive)
1805  if ( deactivated ) {
1806  th->th.th_active = TRUE;
1807  if ( TCR_4(th->th.th_in_pool) ) {
1808  KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1809  th->th.th_active_in_pool = TRUE;
1810  }
1811  }
1812  }
1813 
1814 #ifdef DEBUG_SUSPEND
1815  {
1816  char buffer[128];
1817  __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1818  __kmp_printf( "__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1819  }
1820 #endif
1821 
1822 
1823  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1824  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1825 
1826  KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1827 }
1828 
1829 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
1830  __kmp_suspend_template(th_gtid, flag);
1831 }
1832 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
1833  __kmp_suspend_template(th_gtid, flag);
1834 }
1835 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1836  __kmp_suspend_template(th_gtid, flag);
1837 }
1838 
1839 
1840 /* This routine signals the thread specified by target_gtid to wake up
1841  * after setting the sleep bit indicated by the flag argument to FALSE.
1842  * The target thread must already have called __kmp_suspend_template()
1843  */
1844 template <class C>
1845 static inline void __kmp_resume_template( int target_gtid, C *flag )
1846 {
1847  kmp_info_t *th = __kmp_threads[target_gtid];
1848  int status;
1849 
1850 #ifdef KMP_DEBUG
1851  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1852 #endif
1853 
1854  KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1855  KMP_DEBUG_ASSERT( gtid != target_gtid );
1856 
1857  __kmp_suspend_initialize_thread( th );
1858 
1859  status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1860  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1861 
1862  if (!flag) {
1863  flag = (C *)th->th.th_sleep_loc;
1864  }
1865 
1866  if (!flag) {
1867  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1868  gtid, target_gtid, NULL ) );
1869  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1870  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1871  return;
1872  }
1873  else {
1874  typename C::flag_t old_spin = flag->unset_sleeping();
1875  if ( ! flag->is_sleeping_val(old_spin) ) {
1876  KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): "
1877  "%u => %u\n",
1878  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1879 
1880  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1881  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1882  return;
1883  }
1884  KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): "
1885  "%u => %u\n",
1886  gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1887  }
1888  TCW_PTR(th->th.th_sleep_loc, NULL);
1889 
1890 
1891 #ifdef DEBUG_SUSPEND
1892  {
1893  char buffer[128];
1894  __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1895  __kmp_printf( "__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1896  }
1897 #endif
1898 
1899 
1900  status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1901  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1902  status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1903  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1904  KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1905  gtid, target_gtid ) );
1906 }
1907 
1908 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
1909  __kmp_resume_template(target_gtid, flag);
1910 }
1911 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
1912  __kmp_resume_template(target_gtid, flag);
1913 }
1914 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1915  __kmp_resume_template(target_gtid, flag);
1916 }
1917 
1918 void
1919 __kmp_resume_monitor()
1920 {
1921  KMP_TIME_BLOCK(USER_resume);
1922  int status;
1923 #ifdef KMP_DEBUG
1924  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1925  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1926  gtid, KMP_GTID_MONITOR ) );
1927  KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1928 #endif
1929  status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1930  KMP_CHECK_SYSFAIL( "pthread_mutex_lock", status );
1931 #ifdef DEBUG_SUSPEND
1932  {
1933  char buffer[128];
1934  __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1935  __kmp_printf( "__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1936  }
1937 #endif
1938  status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1939  KMP_CHECK_SYSFAIL( "pthread_cond_signal", status );
1940  status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1941  KMP_CHECK_SYSFAIL( "pthread_mutex_unlock", status );
1942  KF_TRACE( 30, ( "__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1943  gtid, KMP_GTID_MONITOR ) );
1944 }
1945 
1946 /* ------------------------------------------------------------------------ */
1947 /* ------------------------------------------------------------------------ */
1948 
1949 void
1950 __kmp_yield( int cond )
1951 {
1952  if (cond && __kmp_yielding_on) {
1953  sched_yield();
1954  }
1955 }
1956 
1957 /* ------------------------------------------------------------------------ */
1958 /* ------------------------------------------------------------------------ */
1959 
1960 void
1961 __kmp_gtid_set_specific( int gtid )
1962 {
1963  int status;
1964  KMP_ASSERT( __kmp_init_runtime );
1965  status = pthread_setspecific( __kmp_gtid_threadprivate_key, (void*)(intptr_t)(gtid+1) );
1966  KMP_CHECK_SYSFAIL( "pthread_setspecific", status );
1967 }
1968 
1969 int
1970 __kmp_gtid_get_specific()
1971 {
1972  int gtid;
1973  if ( !__kmp_init_runtime ) {
1974  KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1975  return KMP_GTID_SHUTDOWN;
1976  }
1977  gtid = (int)(size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1978  if ( gtid == 0 ) {
1979  gtid = KMP_GTID_DNE;
1980  }
1981  else {
1982  gtid--;
1983  }
1984  KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1985  __kmp_gtid_threadprivate_key, gtid ));
1986  return gtid;
1987 }
1988 
1989 /* ------------------------------------------------------------------------ */
1990 /* ------------------------------------------------------------------------ */
1991 
1992 double
1993 __kmp_read_cpu_time( void )
1994 {
1995  /*clock_t t;*/
1996  struct tms buffer;
1997 
1998  /*t =*/ times( & buffer );
1999 
2000  return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
2001 }
2002 
2003 int
2004 __kmp_read_system_info( struct kmp_sys_info *info )
2005 {
2006  int status;
2007  struct rusage r_usage;
2008 
2009  memset( info, 0, sizeof( *info ) );
2010 
2011  status = getrusage( RUSAGE_SELF, &r_usage);
2012  KMP_CHECK_SYSFAIL_ERRNO( "getrusage", status );
2013 
2014  info->maxrss = r_usage.ru_maxrss; /* the maximum resident set size utilized (in kilobytes) */
2015  info->minflt = r_usage.ru_minflt; /* the number of page faults serviced without any I/O */
2016  info->majflt = r_usage.ru_majflt; /* the number of page faults serviced that required I/O */
2017  info->nswap = r_usage.ru_nswap; /* the number of times a process was "swapped" out of memory */
2018  info->inblock = r_usage.ru_inblock; /* the number of times the file system had to perform input */
2019  info->oublock = r_usage.ru_oublock; /* the number of times the file system had to perform output */
2020  info->nvcsw = r_usage.ru_nvcsw; /* the number of times a context switch was voluntarily */
2021  info->nivcsw = r_usage.ru_nivcsw; /* the number of times a context switch was forced */
2022 
2023  return (status != 0);
2024 }
2025 
2026 /* ------------------------------------------------------------------------ */
2027 /* ------------------------------------------------------------------------ */
2028 
2029 
2030 void
2031 __kmp_read_system_time( double *delta )
2032 {
2033  double t_ns;
2034  struct timeval tval;
2035  struct timespec stop;
2036  int status;
2037 
2038  status = gettimeofday( &tval, NULL );
2039  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2040  TIMEVAL_TO_TIMESPEC( &tval, &stop );
2041  t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
2042  *delta = (t_ns * 1e-9);
2043 }
2044 
2045 void
2046 __kmp_clear_system_time( void )
2047 {
2048  struct timeval tval;
2049  int status;
2050  status = gettimeofday( &tval, NULL );
2051  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2052  TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2053 }
2054 
2055 /* ------------------------------------------------------------------------ */
2056 /* ------------------------------------------------------------------------ */
2057 
2058 #ifdef BUILD_TV
2059 
2060 void
2061 __kmp_tv_threadprivate_store( kmp_info_t *th, void *global_addr, void *thread_addr )
2062 {
2063  struct tv_data *p;
2064 
2065  p = (struct tv_data *) __kmp_allocate( sizeof( *p ) );
2066 
2067  p->u.tp.global_addr = global_addr;
2068  p->u.tp.thread_addr = thread_addr;
2069 
2070  p->type = (void *) 1;
2071 
2072  p->next = th->th.th_local.tv_data;
2073  th->th.th_local.tv_data = p;
2074 
2075  if ( p->next == 0 ) {
2076  int rc = pthread_setspecific( __kmp_tv_key, p );
2077  KMP_CHECK_SYSFAIL( "pthread_setspecific", rc );
2078  }
2079 }
2080 
2081 #endif /* BUILD_TV */
2082 
2083 /* ------------------------------------------------------------------------ */
2084 /* ------------------------------------------------------------------------ */
2085 
2086 static int
2087 __kmp_get_xproc( void ) {
2088 
2089  int r = 0;
2090 
2091  #if KMP_OS_LINUX
2092 
2093  r = sysconf( _SC_NPROCESSORS_ONLN );
2094 
2095  #elif KMP_OS_DARWIN
2096 
2097  // Bug C77011 High "OpenMP Threads and number of active cores".
2098 
2099  // Find the number of available CPUs.
2100  kern_return_t rc;
2101  host_basic_info_data_t info;
2102  mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2103  rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2104  if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2105  // Cannot use KA_TRACE() here because this code works before trace support is
2106  // initialized.
2107  r = info.avail_cpus;
2108  } else {
2109  KMP_WARNING( CantGetNumAvailCPU );
2110  KMP_INFORM( AssumedNumCPU );
2111  }; // if
2112 
2113  #elif KMP_OS_FREEBSD
2114 
2115  int mib[] = { CTL_HW, HW_NCPU };
2116  size_t len = sizeof( r );
2117  if ( sysctl( mib, 2, &r, &len, NULL, 0 ) < 0 ) {
2118  r = 0;
2119  KMP_WARNING( CantGetNumAvailCPU );
2120  KMP_INFORM( AssumedNumCPU );
2121  }
2122 
2123  #else
2124 
2125  #error "Unknown or unsupported OS."
2126 
2127  #endif
2128 
2129  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
2130 
2131 } // __kmp_get_xproc
2132 
2133 int
2134 __kmp_read_from_file( char const *path, char const *format, ... )
2135 {
2136  int result;
2137  va_list args;
2138 
2139  va_start(args, format);
2140  FILE *f = fopen(path, "rb");
2141  if ( f == NULL )
2142  return 0;
2143  result = vfscanf(f, format, args);
2144  fclose(f);
2145 
2146  return result;
2147 }
2148 
2149 void
2150 __kmp_runtime_initialize( void )
2151 {
2152  int status;
2153  pthread_mutexattr_t mutex_attr;
2154  pthread_condattr_t cond_attr;
2155 
2156  if ( __kmp_init_runtime ) {
2157  return;
2158  }; // if
2159 
2160  #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
2161  if ( ! __kmp_cpuinfo.initialized ) {
2162  __kmp_query_cpuid( &__kmp_cpuinfo );
2163  }; // if
2164  #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
2165 
2166  __kmp_xproc = __kmp_get_xproc();
2167 
2168  if ( sysconf( _SC_THREADS ) ) {
2169 
2170  /* Query the maximum number of threads */
2171  __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2172  if ( __kmp_sys_max_nth == -1 ) {
2173  /* Unlimited threads for NPTL */
2174  __kmp_sys_max_nth = INT_MAX;
2175  }
2176  else if ( __kmp_sys_max_nth <= 1 ) {
2177  /* Can't tell, just use PTHREAD_THREADS_MAX */
2178  __kmp_sys_max_nth = KMP_MAX_NTH;
2179  }
2180 
2181  /* Query the minimum stack size */
2182  __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2183  if ( __kmp_sys_min_stksize <= 1 ) {
2184  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2185  }
2186  }
2187 
2188  /* Set up minimum number of threads to switch to TLS gtid */
2189  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2190 
2191 
2192  #ifdef BUILD_TV
2193  {
2194  int rc = pthread_key_create( & __kmp_tv_key, 0 );
2195  KMP_CHECK_SYSFAIL( "pthread_key_create", rc );
2196  }
2197  #endif
2198 
2199  status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2200  KMP_CHECK_SYSFAIL( "pthread_key_create", status );
2201  status = pthread_mutexattr_init( & mutex_attr );
2202  KMP_CHECK_SYSFAIL( "pthread_mutexattr_init", status );
2203  status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2204  KMP_CHECK_SYSFAIL( "pthread_mutex_init", status );
2205  status = pthread_condattr_init( & cond_attr );
2206  KMP_CHECK_SYSFAIL( "pthread_condattr_init", status );
2207  status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2208  KMP_CHECK_SYSFAIL( "pthread_cond_init", status );
2209 #if USE_ITT_BUILD
2210  __kmp_itt_initialize();
2211 #endif /* USE_ITT_BUILD */
2212 
2213  __kmp_init_runtime = TRUE;
2214 }
2215 
2216 void
2217 __kmp_runtime_destroy( void )
2218 {
2219  int status;
2220 
2221  if ( ! __kmp_init_runtime ) {
2222  return; // Nothing to do.
2223  };
2224 
2225 #if USE_ITT_BUILD
2226  __kmp_itt_destroy();
2227 #endif /* USE_ITT_BUILD */
2228 
2229  status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2230  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2231  #ifdef BUILD_TV
2232  status = pthread_key_delete( __kmp_tv_key );
2233  KMP_CHECK_SYSFAIL( "pthread_key_delete", status );
2234  #endif
2235 
2236  status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2237  if ( status != 0 && status != EBUSY ) {
2238  KMP_SYSFAIL( "pthread_mutex_destroy", status );
2239  }
2240  status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2241  if ( status != 0 && status != EBUSY ) {
2242  KMP_SYSFAIL( "pthread_cond_destroy", status );
2243  }
2244  #if KMP_AFFINITY_SUPPORTED
2245  __kmp_affinity_uninitialize();
2246  #endif
2247 
2248  __kmp_init_runtime = FALSE;
2249 }
2250 
2251 
2252 /* Put the thread to sleep for a time period */
2253 /* NOTE: not currently used anywhere */
2254 void
2255 __kmp_thread_sleep( int millis )
2256 {
2257  sleep( ( millis + 500 ) / 1000 );
2258 }
2259 
2260 /* Calculate the elapsed wall clock time for the user */
2261 void
2262 __kmp_elapsed( double *t )
2263 {
2264  int status;
2265 # ifdef FIX_SGI_CLOCK
2266  struct timespec ts;
2267 
2268  status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2269  KMP_CHECK_SYSFAIL_ERRNO( "clock_gettime", status );
2270  *t = (double) ts.tv_nsec * (1.0 / (double) NSEC_PER_SEC) +
2271  (double) ts.tv_sec;
2272 # else
2273  struct timeval tv;
2274 
2275  status = gettimeofday( & tv, NULL );
2276  KMP_CHECK_SYSFAIL_ERRNO( "gettimeofday", status );
2277  *t = (double) tv.tv_usec * (1.0 / (double) USEC_PER_SEC) +
2278  (double) tv.tv_sec;
2279 # endif
2280 }
2281 
2282 /* Calculate the elapsed wall clock tick for the user */
2283 void
2284 __kmp_elapsed_tick( double *t )
2285 {
2286  *t = 1 / (double) CLOCKS_PER_SEC;
2287 }
2288 
2289 /*
2290  Determine whether the given address is mapped into the current address space.
2291 */
2292 
2293 int
2294 __kmp_is_address_mapped( void * addr ) {
2295 
2296  int found = 0;
2297  int rc;
2298 
2299  #if KMP_OS_LINUX
2300 
2301  /*
2302  On Linux* OS, read the /proc/<pid>/maps pseudo-file to get all the address ranges mapped
2303  into the address space.
2304  */
2305 
2306  char * name = __kmp_str_format( "/proc/%d/maps", getpid() );
2307  FILE * file = NULL;
2308 
2309  file = fopen( name, "r" );
2310  KMP_ASSERT( file != NULL );
2311 
2312  for ( ; ; ) {
2313 
2314  void * beginning = NULL;
2315  void * ending = NULL;
2316  char perms[ 5 ];
2317 
2318  rc = fscanf( file, "%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2319  if ( rc == EOF ) {
2320  break;
2321  }; // if
2322  KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 ); // Make sure all fields are read.
2323 
2324  // Ending address is not included in the region, but beginning is.
2325  if ( ( addr >= beginning ) && ( addr < ending ) ) {
2326  perms[ 2 ] = 0; // 3th and 4th character does not matter.
2327  if ( strcmp( perms, "rw" ) == 0 ) {
2328  // Memory we are looking for should be readable and writable.
2329  found = 1;
2330  }; // if
2331  break;
2332  }; // if
2333 
2334  }; // forever
2335 
2336  // Free resources.
2337  fclose( file );
2338  KMP_INTERNAL_FREE( name );
2339 
2340  #elif KMP_OS_DARWIN
2341 
2342  /*
2343  On OS X*, /proc pseudo filesystem is not available. Try to read memory using vm
2344  interface.
2345  */
2346 
2347  int buffer;
2348  vm_size_t count;
2349  rc =
2350  vm_read_overwrite(
2351  mach_task_self(), // Task to read memory of.
2352  (vm_address_t)( addr ), // Address to read from.
2353  1, // Number of bytes to be read.
2354  (vm_address_t)( & buffer ), // Address of buffer to save read bytes in.
2355  & count // Address of var to save number of read bytes in.
2356  );
2357  if ( rc == 0 ) {
2358  // Memory successfully read.
2359  found = 1;
2360  }; // if
2361 
2362  #elif KMP_OS_FREEBSD
2363 
2364  // FIXME(FreeBSD*): Implement this
2365  found = 1;
2366 
2367  #else
2368 
2369  #error "Unknown or unsupported OS"
2370 
2371  #endif
2372 
2373  return found;
2374 
2375 } // __kmp_is_address_mapped
2376 
2377 #ifdef USE_LOAD_BALANCE
2378 
2379 
2380 # if KMP_OS_DARWIN
2381 
2382 // The function returns the rounded value of the system load average
2383 // during given time interval which depends on the value of
2384 // __kmp_load_balance_interval variable (default is 60 sec, other values
2385 // may be 300 sec or 900 sec).
2386 // It returns -1 in case of error.
2387 int
2388 __kmp_get_load_balance( int max )
2389 {
2390  double averages[3];
2391  int ret_avg = 0;
2392 
2393  int res = getloadavg( averages, 3 );
2394 
2395  //Check __kmp_load_balance_interval to determine which of averages to use.
2396  // getloadavg() may return the number of samples less than requested that is
2397  // less than 3.
2398  if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2399  ret_avg = averages[0];// 1 min
2400  } else if ( ( __kmp_load_balance_interval >= 180
2401  && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2402  ret_avg = averages[1];// 5 min
2403  } else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2404  ret_avg = averages[2];// 15 min
2405  } else {// Error occurred
2406  return -1;
2407  }
2408 
2409  return ret_avg;
2410 }
2411 
2412 # else // Linux* OS
2413 
2414 // The fuction returns number of running (not sleeping) threads, or -1 in case of error.
2415 // Error could be reported if Linux* OS kernel too old (without "/proc" support).
2416 // Counting running threads stops if max running threads encountered.
2417 int
2418 __kmp_get_load_balance( int max )
2419 {
2420  static int permanent_error = 0;
2421 
2422  static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
2423  static double glb_call_time = 0; /* Thread balance algorithm call time */
2424 
2425  int running_threads = 0; // Number of running threads in the system.
2426 
2427  DIR * proc_dir = NULL; // Handle of "/proc/" directory.
2428  struct dirent * proc_entry = NULL;
2429 
2430  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2431  DIR * task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2432  struct dirent * task_entry = NULL;
2433  int task_path_fixed_len;
2434 
2435  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2436  int stat_file = -1;
2437  int stat_path_fixed_len;
2438 
2439  int total_processes = 0; // Total number of processes in system.
2440  int total_threads = 0; // Total number of threads in system.
2441 
2442  double call_time = 0.0;
2443 
2444  __kmp_str_buf_init( & task_path );
2445  __kmp_str_buf_init( & stat_path );
2446 
2447  __kmp_elapsed( & call_time );
2448 
2449  if ( glb_call_time &&
2450  ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2451  running_threads = glb_running_threads;
2452  goto finish;
2453  }
2454 
2455  glb_call_time = call_time;
2456 
2457  // Do not spend time on scanning "/proc/" if we have a permanent error.
2458  if ( permanent_error ) {
2459  running_threads = -1;
2460  goto finish;
2461  }; // if
2462 
2463  if ( max <= 0 ) {
2464  max = INT_MAX;
2465  }; // if
2466 
2467  // Open "/proc/" directory.
2468  proc_dir = opendir( "/proc" );
2469  if ( proc_dir == NULL ) {
2470  // Cannot open "/prroc/". Probably the kernel does not support it. Return an error now and
2471  // in subsequent calls.
2472  running_threads = -1;
2473  permanent_error = 1;
2474  goto finish;
2475  }; // if
2476 
2477  // Initialize fixed part of task_path. This part will not change.
2478  __kmp_str_buf_cat( & task_path, "/proc/", 6 );
2479  task_path_fixed_len = task_path.used; // Remember number of used characters.
2480 
2481  proc_entry = readdir( proc_dir );
2482  while ( proc_entry != NULL ) {
2483  // Proc entry is a directory and name starts with a digit. Assume it is a process'
2484  // directory.
2485  if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2486 
2487  ++ total_processes;
2488  // Make sure init process is the very first in "/proc", so we can replace
2489  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes == 1.
2490  // We are going to check that total_processes == 1 => d_name == "1" is true (where
2491  // "=>" is implication). Since C++ does not have => operator, let us replace it with its
2492  // equivalent: a => b == ! a || b.
2493  KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name, "1" ) == 0 );
2494 
2495  // Construct task_path.
2496  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2497  __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2498  __kmp_str_buf_cat( & task_path, "/task", 5 );
2499 
2500  task_dir = opendir( task_path.str );
2501  if ( task_dir == NULL ) {
2502  // Process can finish between reading "/proc/" directory entry and opening process'
2503  // "task/" directory. So, in general case we should not complain, but have to skip
2504  // this process and read the next one.
2505  // But on systems with no "task/" support we will spend lot of time to scan "/proc/"
2506  // tree again and again without any benefit. "init" process (its pid is 1) should
2507  // exist always, so, if we cannot open "/proc/1/task/" directory, it means "task/"
2508  // is not supported by kernel. Report an error now and in the future.
2509  if ( strcmp( proc_entry->d_name, "1" ) == 0 ) {
2510  running_threads = -1;
2511  permanent_error = 1;
2512  goto finish;
2513  }; // if
2514  } else {
2515  // Construct fixed part of stat file path.
2516  __kmp_str_buf_clear( & stat_path );
2517  __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2518  __kmp_str_buf_cat( & stat_path, "/", 1 );
2519  stat_path_fixed_len = stat_path.used;
2520 
2521  task_entry = readdir( task_dir );
2522  while ( task_entry != NULL ) {
2523  // It is a directory and name starts with a digit.
2524  if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2525 
2526  ++ total_threads;
2527 
2528  // Consruct complete stat file path. Easiest way would be:
2529  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str, task_entry->d_name );
2530  // but seriae of __kmp_str_buf_cat works a bit faster.
2531  stat_path.used = stat_path_fixed_len; // Reset stat path to its fixed part.
2532  __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2533  __kmp_str_buf_cat( & stat_path, "/stat", 5 );
2534 
2535  // Note: Low-level API (open/read/close) is used. High-level API
2536  // (fopen/fclose) works ~ 30 % slower.
2537  stat_file = open( stat_path.str, O_RDONLY );
2538  if ( stat_file == -1 ) {
2539  // We cannot report an error because task (thread) can terminate just
2540  // before reading this file.
2541  } else {
2542  /*
2543  Content of "stat" file looks like:
2544 
2545  24285 (program) S ...
2546 
2547  It is a single line (if program name does not include fanny
2548  symbols). First number is a thread id, then name of executable file
2549  name in paretheses, then state of the thread. We need just thread
2550  state.
2551 
2552  Good news: Length of program name is 15 characters max. Longer
2553  names are truncated.
2554 
2555  Thus, we need rather short buffer: 15 chars for program name +
2556  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2557 
2558  Bad news: Program name may contain special symbols like space,
2559  closing parenthesis, or even new line. This makes parsing "stat"
2560  file not 100 % reliable. In case of fanny program names parsing
2561  may fail (report incorrect thread state).
2562 
2563  Parsing "status" file looks more promissing (due to different
2564  file structure and escaping special symbols) but reading and
2565  parsing of "status" file works slower.
2566 
2567  -- ln
2568  */
2569  char buffer[ 65 ];
2570  int len;
2571  len = read( stat_file, buffer, sizeof( buffer ) - 1 );
2572  if ( len >= 0 ) {
2573  buffer[ len ] = 0;
2574  // Using scanf:
2575  // sscanf( buffer, "%*d (%*s) %c ", & state );
2576  // looks very nice, but searching for a closing parenthesis works a
2577  // bit faster.
2578  char * close_parent = strstr( buffer, ") " );
2579  if ( close_parent != NULL ) {
2580  char state = * ( close_parent + 2 );
2581  if ( state == 'R' ) {
2582  ++ running_threads;
2583  if ( running_threads >= max ) {
2584  goto finish;
2585  }; // if
2586  }; // if
2587  }; // if
2588  }; // if
2589  close( stat_file );
2590  stat_file = -1;
2591  }; // if
2592  }; // if
2593  task_entry = readdir( task_dir );
2594  }; // while
2595  closedir( task_dir );
2596  task_dir = NULL;
2597  }; // if
2598  }; // if
2599  proc_entry = readdir( proc_dir );
2600  }; // while
2601 
2602  //
2603  // There _might_ be a timing hole where the thread executing this
2604  // code get skipped in the load balance, and running_threads is 0.
2605  // Assert in the debug builds only!!!
2606  //
2607  KMP_DEBUG_ASSERT( running_threads > 0 );
2608  if ( running_threads <= 0 ) {
2609  running_threads = 1;
2610  }
2611 
2612  finish: // Clean up and exit.
2613  if ( proc_dir != NULL ) {
2614  closedir( proc_dir );
2615  }; // if
2616  __kmp_str_buf_free( & task_path );
2617  if ( task_dir != NULL ) {
2618  closedir( task_dir );
2619  }; // if
2620  __kmp_str_buf_free( & stat_path );
2621  if ( stat_file != -1 ) {
2622  close( stat_file );
2623  }; // if
2624 
2625  glb_running_threads = running_threads;
2626 
2627  return running_threads;
2628 
2629 } // __kmp_get_load_balance
2630 
2631 # endif // KMP_OS_DARWIN
2632 
2633 #endif // USE_LOAD_BALANCE
2634 
2635 
2636 #if KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64)
2637 
2638 int __kmp_invoke_microtask( microtask_t pkfn, int gtid, int tid, int argc,
2639  void *p_argv[] )
2640 {
2641  int argc_full = argc + 2;
2642  int i;
2643  ffi_cif cif;
2644  ffi_type *types[argc_full];
2645  void *args[argc_full];
2646  void *idp[2];
2647 
2648  /* We're only passing pointers to the target. */
2649  for (i = 0; i < argc_full; i++)
2650  types[i] = &ffi_type_pointer;
2651 
2652  /* Ugly double-indirection, but that's how it goes... */
2653  idp[0] = &gtid;
2654  idp[1] = &tid;
2655  args[0] = &idp[0];
2656  args[1] = &idp[1];
2657 
2658  for (i = 0; i < argc; i++)
2659  args[2 + i] = &p_argv[i];
2660 
2661  if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, argc_full,
2662  &ffi_type_void, types) != FFI_OK)
2663  abort();
2664 
2665  ffi_call(&cif, (void (*)(void))pkfn, NULL, args);
2666 
2667  return 1;
2668 }
2669 
2670 #endif // KMP_COMPILER_GCC && !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_PPC64)
2671 
2672 #if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64
2673 
2674 // we really only need the case with 1 argument, because CLANG always build
2675 // a struct of pointers to shared variables referenced in the outlined function
2676 int
2677 __kmp_invoke_microtask( microtask_t pkfn,
2678  int gtid, int tid,
2679  int argc, void *p_argv[] ) {
2680  switch (argc) {
2681  default:
2682  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2683  fflush(stderr);
2684  exit(-1);
2685  case 0:
2686  (*pkfn)(&gtid, &tid);
2687  break;
2688  case 1:
2689  (*pkfn)(&gtid, &tid, p_argv[0]);
2690  break;
2691  case 2:
2692  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2693  break;
2694  case 3:
2695  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2696  break;
2697  case 4:
2698  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2699  break;
2700  case 5:
2701  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2702  break;
2703  case 6:
2704  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2705  p_argv[5]);
2706  break;
2707  case 7:
2708  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2709  p_argv[5], p_argv[6]);
2710  break;
2711  case 8:
2712  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2713  p_argv[5], p_argv[6], p_argv[7]);
2714  break;
2715  case 9:
2716  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2717  p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2718  break;
2719  case 10:
2720  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2721  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2722  break;
2723  case 11:
2724  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2725  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2726  break;
2727  case 12:
2728  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2729  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2730  p_argv[11]);
2731  break;
2732  case 13:
2733  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2734  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2735  p_argv[11], p_argv[12]);
2736  break;
2737  case 14:
2738  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2739  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2740  p_argv[11], p_argv[12], p_argv[13]);
2741  break;
2742  case 15:
2743  (*pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2744  p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2745  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2746  break;
2747  }
2748 
2749  return 1;
2750 }
2751 
2752 #endif
2753 
2754 // end of file //
2755 
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
Definition: kmp_stats.h:668
#define KMP_TIME_BLOCK(name)
Uses specified timer (name) to time code block.
Definition: kmp_stats.h:629