Intel® OpenMP* Runtime Library
 All Classes Functions Variables Typedefs Enumerations Enumerator Modules Pages
kmp_os.h
1 /*
2  * kmp_os.h -- KPTS runtime header file.
3  * $Revision: 43473 $
4  * $Date: 2014-09-26 15:02:57 -0500 (Fri, 26 Sep 2014) $
5  */
6 
7 /* <copyright>
8  Copyright (c) 1997-2014 Intel Corporation. All Rights Reserved.
9 
10  Redistribution and use in source and binary forms, with or without
11  modification, are permitted provided that the following conditions
12  are met:
13 
14  * Redistributions of source code must retain the above copyright
15  notice, this list of conditions and the following disclaimer.
16  * Redistributions in binary form must reproduce the above copyright
17  notice, this list of conditions and the following disclaimer in the
18  documentation and/or other materials provided with the distribution.
19  * Neither the name of Intel Corporation nor the names of its
20  contributors may be used to endorse or promote products derived
21  from this software without specific prior written permission.
22 
23  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 
35 </copyright> */
36 
37 #ifndef KMP_OS_H
38 #define KMP_OS_H
39 
40 #include <stdlib.h>
41 
42 #define KMP_FTN_PLAIN 1
43 #define KMP_FTN_APPEND 2
44 #define KMP_FTN_UPPER 3
45 /*
46 #define KMP_FTN_PREPEND 4
47 #define KMP_FTN_UAPPEND 5
48 */
49 
50 #define KMP_PTR_SKIP (sizeof(void*))
51 
52 /* -------------------------- Compiler variations ------------------------ */
53 
54 #define KMP_OFF 0
55 #define KMP_ON 1
56 
57 #define KMP_MEM_CONS_VOLATILE 0
58 #define KMP_MEM_CONS_FENCE 1
59 
60 #ifndef KMP_MEM_CONS_MODEL
61 # define KMP_MEM_CONS_MODEL KMP_MEM_CONS_VOLATILE
62 #endif
63 
64 /* ------------------------- Compiler recognition ---------------------- */
65 #define KMP_COMPILER_ICC 0
66 #define KMP_COMPILER_GCC 0
67 #define KMP_COMPILER_CLANG 0
68 #define KMP_COMPILER_MSVC 0
69 
70 #if defined( __INTEL_COMPILER )
71 # undef KMP_COMPILER_ICC
72 # define KMP_COMPILER_ICC 1
73 #elif defined( __clang__ )
74 # undef KMP_COMPILER_CLANG
75 # define KMP_COMPILER_CLANG 1
76 #elif defined( __GNUC__ )
77 # undef KMP_COMPILER_GCC
78 # define KMP_COMPILER_GCC 1
79 #elif defined( _MSC_VER )
80 # undef KMP_COMPILER_MSVC
81 # define KMP_COMPILER_MSVC 1
82 #else
83 # error Unknown compiler
84 #endif
85 
86 /* ---------------------- Operating system recognition ------------------- */
87 
88 #define KMP_OS_LINUX 0
89 #define KMP_OS_FREEBSD 0
90 #define KMP_OS_DARWIN 0
91 #define KMP_OS_WINDOWS 0
92 #define KMP_OS_CNK 0
93 #define KMP_OS_UNIX 0 /* disjunction of KMP_OS_LINUX, KMP_OS_DARWIN etc. */
94 
95 #define KMP_ARCH_X86 0
96 #define KMP_ARCH_X86_64 0
97 #define KMP_ARCH_PPC64 0
98 
99 #ifdef _WIN32
100 # undef KMP_OS_WINDOWS
101 # define KMP_OS_WINDOWS 1
102 #endif
103 
104 #if ( defined __APPLE__ && defined __MACH__ )
105 # undef KMP_OS_DARWIN
106 # define KMP_OS_DARWIN 1
107 #endif
108 
109 // in some ppc64 linux installations, only the second condition is met
110 #if ( defined __linux )
111 # undef KMP_OS_LINUX
112 # define KMP_OS_LINUX 1
113 #elif ( defined __linux__)
114 # undef KMP_OS_LINUX
115 # define KMP_OS_LINUX 1
116 #else
117 #endif
118 
119 #if ( defined __FreeBSD__ )
120 # undef KMP_OS_FREEBSD
121 # define KMP_OS_FREEBSD 1
122 #endif
123 
124 #if ( defined __bgq__ )
125 # undef KMP_OS_CNK
126 # define KMP_OS_CNK 1
127 #endif
128 
129 #if (1 != KMP_OS_LINUX + KMP_OS_FREEBSD + KMP_OS_DARWIN + KMP_OS_WINDOWS)
130 # error Unknown OS
131 #endif
132 
133 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_DARWIN
134 # undef KMP_OS_UNIX
135 # define KMP_OS_UNIX 1
136 #endif
137 
138 #if (KMP_OS_LINUX || KMP_OS_WINDOWS) && !KMP_OS_CNK && !KMP_ARCH_PPC64
139 # define KMP_AFFINITY_SUPPORTED 1
140 #else
141 # define KMP_AFFINITY_SUPPORTED 0
142 #endif
143 
144 #if KMP_OS_WINDOWS
145 # if defined _M_AMD64
146 # undef KMP_ARCH_X86_64
147 # define KMP_ARCH_X86_64 1
148 # else
149 # undef KMP_ARCH_X86
150 # define KMP_ARCH_X86 1
151 # endif
152 #endif
153 
154 #if KMP_OS_UNIX
155 # if defined __x86_64
156 # undef KMP_ARCH_X86_64
157 # define KMP_ARCH_X86_64 1
158 # elif defined __i386
159 # undef KMP_ARCH_X86
160 # define KMP_ARCH_X86 1
161 # elif defined __powerpc64__
162 # undef KMP_ARCH_PPC64
163 # define KMP_ARCH_PPC64 1
164 # endif
165 #endif
166 
167 #if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7R__) || \
168  defined(__ARM_ARCH_7A__)
169 # define KMP_ARCH_ARMV7 1
170 #endif
171 
172 #if defined(KMP_ARCH_ARMV7) || defined(__ARM_ARCH_6__) || \
173  defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || \
174  defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6T2__) || \
175  defined(__ARM_ARCH_6ZK__)
176 # define KMP_ARCH_ARMV6 1
177 #endif
178 
179 #if defined(KMP_ARCH_ARMV6) || defined(__ARM_ARCH_5T__) || \
180  defined(__ARM_ARCH_5E__) || defined(__ARM_ARCH_5TE__) || \
181  defined(__ARM_ARCH_5TEJ__)
182 # define KMP_ARCH_ARMV5 1
183 #endif
184 
185 #if defined(KMP_ARCH_ARMV5) || defined(__ARM_ARCH_4__) || \
186  defined(__ARM_ARCH_4T__)
187 # define KMP_ARCH_ARMV4 1
188 #endif
189 
190 #if defined(KMP_ARCH_ARMV4) || defined(__ARM_ARCH_3__) || \
191  defined(__ARM_ARCH_3M__)
192 # define KMP_ARCH_ARMV3 1
193 #endif
194 
195 #if defined(KMP_ARCH_ARMV3) || defined(__ARM_ARCH_2__)
196 # define KMP_ARCH_ARMV2 1
197 #endif
198 
199 #if defined(KMP_ARCH_ARMV2)
200 # define KMP_ARCH_ARM 1
201 #endif
202 
203 #if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64)
204 # error Unknown or unsupported architecture
205 #endif
206 
207 /* Check for quad-precision extension. */
208 #define KMP_HAVE_QUAD 0
209 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
210 # if KMP_COMPILER_ICC
211  /* _Quad is already defined for icc */
212 # undef KMP_HAVE_QUAD
213 # define KMP_HAVE_QUAD 1
214 # elif KMP_COMPILER_CLANG
215  /* Clang doesn't support a software-implemented
216  128-bit extended precision type yet */
217  typedef long double _Quad;
218 # elif KMP_COMPILER_GCC
219  typedef __float128 _Quad;
220 # undef KMP_HAVE_QUAD
221 # define KMP_HAVE_QUAD 1
222 # elif KMP_COMPILER_MSVC
223  typedef long double _Quad;
224 # endif
225 #else
226 # if __LDBL_MAX_EXP__ >= 16384 && KMP_COMPILER_GCC
227  typedef long double _Quad;
228 # undef KMP_HAVE_QUAD
229 # define KMP_HAVE_QUAD 1
230 # endif
231 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
232 
233 #if KMP_OS_WINDOWS
234  typedef char kmp_int8;
235  typedef unsigned char kmp_uint8;
236  typedef short kmp_int16;
237  typedef unsigned short kmp_uint16;
238  typedef int kmp_int32;
239  typedef unsigned int kmp_uint32;
240 # define KMP_INT32_SPEC "d"
241 # define KMP_UINT32_SPEC "u"
242 # ifndef KMP_STRUCT64
243  typedef __int64 kmp_int64;
244  typedef unsigned __int64 kmp_uint64;
245  #define KMP_INT64_SPEC "I64d"
246  #define KMP_UINT64_SPEC "I64u"
247 # else
248  struct kmp_struct64 {
249  kmp_int32 a,b;
250  };
251  typedef struct kmp_struct64 kmp_int64;
252  typedef struct kmp_struct64 kmp_uint64;
253  /* Not sure what to use for KMP_[U]INT64_SPEC here */
254 # endif
255 # if KMP_ARCH_X86_64
256 # define KMP_INTPTR 1
257  typedef __int64 kmp_intptr_t;
258  typedef unsigned __int64 kmp_uintptr_t;
259 # define KMP_INTPTR_SPEC "I64d"
260 # define KMP_UINTPTR_SPEC "I64u"
261 # endif
262 #endif /* KMP_OS_WINDOWS */
263 
264 #if KMP_OS_UNIX
265  typedef char kmp_int8;
266  typedef unsigned char kmp_uint8;
267  typedef short kmp_int16;
268  typedef unsigned short kmp_uint16;
269  typedef int kmp_int32;
270  typedef unsigned int kmp_uint32;
271  typedef long long kmp_int64;
272  typedef unsigned long long kmp_uint64;
273 # define KMP_INT32_SPEC "d"
274 # define KMP_UINT32_SPEC "u"
275 # define KMP_INT64_SPEC "lld"
276 # define KMP_UINT64_SPEC "llu"
277 #endif /* KMP_OS_UNIX */
278 
279 #if KMP_ARCH_X86 || KMP_ARCH_ARM
280 # define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
281 #elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64
282 # define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
283 #else
284 # error "Can't determine size_t printf format specifier."
285 #endif
286 
287 #if KMP_ARCH_X86
288 # define KMP_SIZE_T_MAX (0xFFFFFFFF)
289 #else
290 # define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)
291 #endif
292 
293 typedef size_t kmp_size_t;
294 typedef float kmp_real32;
295 typedef double kmp_real64;
296 
297 #ifndef KMP_INTPTR
298 # define KMP_INTPTR 1
299  typedef long kmp_intptr_t;
300  typedef unsigned long kmp_uintptr_t;
301 # define KMP_INTPTR_SPEC "ld"
302 # define KMP_UINTPTR_SPEC "lu"
303 #endif
304 
305 #ifdef KMP_I8
306  typedef kmp_int64 kmp_int;
307  typedef kmp_uint64 kmp_uint;
308 # define KMP_INT_SPEC KMP_INT64_SPEC
309 # define KMP_UINT_SPEC KMP_UINT64_SPEC
310 # define KMP_INT_MAX ((kmp_int64)0x7FFFFFFFFFFFFFFFLL)
311 # define KMP_INT_MIN ((kmp_int64)0x8000000000000000LL)
312 #else
313  typedef kmp_int32 kmp_int;
314  typedef kmp_uint32 kmp_uint;
315 # define KMP_INT_SPEC KMP_INT32_SPEC
316 # define KMP_UINT_SPEC KMP_UINT32_SPEC
317 # define KMP_INT_MAX ((kmp_int32)0x7FFFFFFF)
318 # define KMP_INT_MIN ((kmp_int32)0x80000000)
319 #endif /* KMP_I8 */
320 
321 #ifdef __cplusplus
322  //-------------------------------------------------------------------------
323  // template for debug prints specification ( d, u, lld, llu ), and to obtain
324  // signed/unsigned flavors of a type
325  template< typename T >
326  struct traits_t {
327  typedef T signed_t;
328  typedef T unsigned_t;
329  typedef T floating_t;
330  static char const * spec;
331  };
332  // int
333  template<>
334  struct traits_t< signed int > {
335  typedef signed int signed_t;
336  typedef unsigned int unsigned_t;
337  typedef double floating_t;
338  static char const * spec;
339  };
340  // unsigned int
341  template<>
342  struct traits_t< unsigned int > {
343  typedef signed int signed_t;
344  typedef unsigned int unsigned_t;
345  typedef double floating_t;
346  static char const * spec;
347  };
348  // long long
349  template<>
350  struct traits_t< signed long long > {
351  typedef signed long long signed_t;
352  typedef unsigned long long unsigned_t;
353  typedef long double floating_t;
354  static char const * spec;
355  };
356  // unsigned long long
357  template<>
358  struct traits_t< unsigned long long > {
359  typedef signed long long signed_t;
360  typedef unsigned long long unsigned_t;
361  typedef long double floating_t;
362  static char const * spec;
363  };
364  //-------------------------------------------------------------------------
365 #endif // __cplusplus
366 
367 #define KMP_EXPORT extern /* export declaration in guide libraries */
368 
369 #if __GNUC__ == 4
370  #define __forceinline __inline
371 #endif
372 
373 #define PAGE_SIZE (0x4000)
374 #define PAGE_ALIGNED(_addr) ( ! ((size_t) _addr & \
375  (size_t)(PAGE_SIZE - 1)))
376 #define ALIGN_TO_PAGE(x) (void *)(((size_t)(x)) & ~((size_t)(PAGE_SIZE - 1)))
377 
378 /* ---------------------- Support for cache alignment, padding, etc. -----------------*/
379 
380 #ifdef __cplusplus
381 extern "C" {
382 #endif // __cplusplus
383 
384 #define INTERNODE_CACHE_LINE 4096 /* for multi-node systems */
385 
386 /* Define the default size of the cache line */
387 #ifndef CACHE_LINE
388  #define CACHE_LINE 128 /* cache line size in bytes */
389 #else
390  #if ( CACHE_LINE < 64 ) && ! defined( KMP_OS_DARWIN )
391  // 2006-02-13: This produces too many warnings on OS X*. Disable it for a while...
392  #warning CACHE_LINE is too small.
393  #endif
394 #endif /* CACHE_LINE */
395 
396 #define KMP_CACHE_PREFETCH(ADDR) /* nothing */
397 
398 /* Temporary note: if performance testing of this passes, we can remove
399  all references to KMP_DO_ALIGN and replace with KMP_ALIGN. */
400 #if KMP_OS_UNIX && defined(__GNUC__)
401 # define KMP_DO_ALIGN(bytes) __attribute__((aligned(bytes)))
402 # define KMP_ALIGN_CACHE __attribute__((aligned(CACHE_LINE)))
403 # define KMP_ALIGN_CACHE_INTERNODE __attribute__((aligned(INTERNODE_CACHE_LINE)))
404 # define KMP_ALIGN(bytes) __attribute__((aligned(bytes)))
405 #else
406 # define KMP_DO_ALIGN(bytes) __declspec( align(bytes) )
407 # define KMP_ALIGN_CACHE __declspec( align(CACHE_LINE) )
408 # define KMP_ALIGN_CACHE_INTERNODE __declspec( align(INTERNODE_CACHE_LINE) )
409 # define KMP_ALIGN(bytes) __declspec( align(bytes) )
410 #endif
411 
412 #if defined(__MIC__) || defined(__MIC2__)
413  #define KMP_MIC 1
414 // Intel(R) Composer XE (13.0) defines both __MIC__ and __MIC2__ !
415 # if __MIC2__ || __KNC__
416  #define KMP_MIC1 0
417  #define KMP_MIC2 1
418 # else
419  #define KMP_MIC1 1
420  #define KMP_MIC2 0
421 # endif
422 #else
423  #define KMP_MIC 0
424  #define KMP_MIC1 0
425  #define KMP_MIC2 0
426 #endif
427 
428 /* General purpose fence types for memory operations */
429 enum kmp_mem_fence_type {
430  kmp_no_fence, /* No memory fence */
431  kmp_acquire_fence, /* Acquire (read) memory fence */
432  kmp_release_fence, /* Release (write) memory fence */
433  kmp_full_fence /* Full (read+write) memory fence */
434 };
435 
436 
437 //
438 // Synchronization primitives
439 //
440 
441 #if KMP_ASM_INTRINS && KMP_OS_WINDOWS
442 
443 #include <Windows.h>
444 
445 #pragma intrinsic(InterlockedExchangeAdd)
446 #pragma intrinsic(InterlockedCompareExchange)
447 #pragma intrinsic(InterlockedExchange)
448 #pragma intrinsic(InterlockedExchange64)
449 
450 //
451 // Using InterlockedIncrement / InterlockedDecrement causes a library loading
452 // ordering problem, so we use InterlockedExchangeAdd instead.
453 //
454 # define KMP_TEST_THEN_INC32(p) InterlockedExchangeAdd( (volatile long *)(p), 1 )
455 # define KMP_TEST_THEN_INC_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), 1 )
456 # define KMP_TEST_THEN_ADD4_32(p) InterlockedExchangeAdd( (volatile long *)(p), 4 )
457 # define KMP_TEST_THEN_ADD4_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), 4 )
458 # define KMP_TEST_THEN_DEC32(p) InterlockedExchangeAdd( (volatile long *)(p), -1 )
459 # define KMP_TEST_THEN_DEC_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), -1 )
460 # define KMP_TEST_THEN_ADD32(p, v) InterlockedExchangeAdd( (volatile long *)(p), (v) )
461 
462 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) InterlockedCompareExchange( (volatile long *)(p),(long)(sv),(long)(cv) )
463 
464 # define KMP_XCHG_FIXED32(p, v) InterlockedExchange( (volatile long *)(p), (long)(v) )
465 # define KMP_XCHG_FIXED64(p, v) InterlockedExchange64( (volatile kmp_int64 *)(p), (kmp_int64)(v) )
466 
467 inline kmp_real32 KMP_XCHG_REAL32( volatile kmp_real32 *p, kmp_real32 v)
468 {
469  kmp_int32 tmp = InterlockedExchange( (volatile long *)p, *(long *)&v);
470  return *(kmp_real32*)&tmp;
471 }
472 
473 //
474 // Routines that we still need to implement in assembly.
475 //
476 extern kmp_int32 __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 v );
477 extern kmp_int32 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 v );
478 extern kmp_int32 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 v );
479 extern kmp_int64 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 v );
480 extern kmp_int64 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 v );
481 extern kmp_int64 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 v );
482 
483 extern kmp_int8 __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
484 extern kmp_int16 __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
485 extern kmp_int32 __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
486 extern kmp_int32 __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
487 extern kmp_int8 __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
488 extern kmp_int16 __kmp_compare_and_store_ret16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
489 extern kmp_int32 __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
490 extern kmp_int64 __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
491 
492 extern kmp_int8 __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 v );
493 extern kmp_int16 __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 v );
494 extern kmp_int32 __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 v );
495 extern kmp_int64 __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 v );
496 extern kmp_real32 __kmp_xchg_real32( volatile kmp_real32 *p, kmp_real32 v );
497 extern kmp_real64 __kmp_xchg_real64( volatile kmp_real64 *p, kmp_real64 v );
498 
499 //# define KMP_TEST_THEN_INC32(p) __kmp_test_then_add32( (p), 1 )
500 //# define KMP_TEST_THEN_INC_ACQ32(p) __kmp_test_then_add32( (p), 1 )
501 # define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64( (p), 1LL )
502 # define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64( (p), 1LL )
503 //# define KMP_TEST_THEN_ADD4_32(p) __kmp_test_then_add32( (p), 4 )
504 //# define KMP_TEST_THEN_ADD4_ACQ32(p) __kmp_test_then_add32( (p), 4 )
505 # define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64( (p), 4LL )
506 # define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64( (p), 4LL )
507 //# define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32( (p), -1 )
508 //# define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32( (p), -1 )
509 # define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64( (p), -1LL )
510 # define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64( (p), -1LL )
511 //# define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32( (p), (v) )
512 # define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64( (p), (v) )
513 
514 # define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32( (p), (v) )
515 # define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32( (p), (v) )
516 # define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64( (p), (v) )
517 # define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64( (p), (v) )
518 
519 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
520 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
521 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
522 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
523 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
524 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
525 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
526 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
527 
528 # if KMP_ARCH_X86
529 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )
530 # else /* 64 bit pointers */
531 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )
532 # endif /* KMP_ARCH_X86 */
533 
534 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __kmp_compare_and_store_ret8( (p), (cv), (sv) )
535 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )
536 //# define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __kmp_compare_and_store_ret32( (p), (cv), (sv) )
537 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __kmp_compare_and_store_ret64( (p), (cv), (sv) )
538 
539 # define KMP_XCHG_FIXED8(p, v) __kmp_xchg_fixed8( (p), (v) );
540 # define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16( (p), (v) );
541 //# define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32( (p), (v) );
542 //# define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64( (p), (v) );
543 //# define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32( (p), (v) );
544 # define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64( (p), (v) );
545 
546 
547 #elif (KMP_ASM_INTRINS && KMP_OS_UNIX) || !(KMP_ARCH_X86 || KMP_ARCH_X86_64)
548 
549 /* cast p to correct type so that proper intrinsic will be used */
550 # define KMP_TEST_THEN_INC32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 1 )
551 # define KMP_TEST_THEN_INC_ACQ32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 1 )
552 # define KMP_TEST_THEN_INC64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 1LL )
553 # define KMP_TEST_THEN_INC_ACQ64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 1LL )
554 # define KMP_TEST_THEN_ADD4_32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 4 )
555 # define KMP_TEST_THEN_ADD4_ACQ32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 4 )
556 # define KMP_TEST_THEN_ADD4_64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 4LL )
557 # define KMP_TEST_THEN_ADD4_ACQ64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 4LL )
558 # define KMP_TEST_THEN_DEC32(p) __sync_fetch_and_sub( (kmp_int32 *)(p), 1 )
559 # define KMP_TEST_THEN_DEC_ACQ32(p) __sync_fetch_and_sub( (kmp_int32 *)(p), 1 )
560 # define KMP_TEST_THEN_DEC64(p) __sync_fetch_and_sub( (kmp_int64 *)(p), 1LL )
561 # define KMP_TEST_THEN_DEC_ACQ64(p) __sync_fetch_and_sub( (kmp_int64 *)(p), 1LL )
562 # define KMP_TEST_THEN_ADD32(p, v) __sync_fetch_and_add( (kmp_int32 *)(p), (v) )
563 # define KMP_TEST_THEN_ADD64(p, v) __sync_fetch_and_add( (kmp_int64 *)(p), (v) )
564 
565 # define KMP_TEST_THEN_OR32(p, v) __sync_fetch_and_or( (kmp_int32 *)(p), (v) )
566 # define KMP_TEST_THEN_AND32(p, v) __sync_fetch_and_and( (kmp_int32 *)(p), (v) )
567 # define KMP_TEST_THEN_OR64(p, v) __sync_fetch_and_or( (kmp_int64 *)(p), (v) )
568 # define KMP_TEST_THEN_AND64(p, v) __sync_fetch_and_and( (kmp_int64 *)(p), (v) )
569 
570 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
571 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
572 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
573 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
574 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
575 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
576 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
577 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
578 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __sync_bool_compare_and_swap( (volatile void **)(p),(void *)(cv),(void *)(sv) )
579 
580 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
581 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
582 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
583 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
584 
585 #define KMP_XCHG_FIXED8(p, v) __sync_lock_test_and_set( (volatile kmp_uint8 *)(p), (kmp_uint8)(v) )
586 #define KMP_XCHG_FIXED16(p, v) __sync_lock_test_and_set( (volatile kmp_uint16 *)(p), (kmp_uint16)(v) )
587 #define KMP_XCHG_FIXED32(p, v) __sync_lock_test_and_set( (volatile kmp_uint32 *)(p), (kmp_uint32)(v) )
588 #define KMP_XCHG_FIXED64(p, v) __sync_lock_test_and_set( (volatile kmp_uint64 *)(p), (kmp_uint64)(v) )
589 
590 inline kmp_real32 KMP_XCHG_REAL32( volatile kmp_real32 *p, kmp_real32 v)
591 {
592  kmp_int32 tmp = __sync_lock_test_and_set( (kmp_int32*)p, *(kmp_int32*)&v);
593  return *(kmp_real32*)&tmp;
594 }
595 
596 inline kmp_real64 KMP_XCHG_REAL64( volatile kmp_real64 *p, kmp_real64 v)
597 {
598  kmp_int64 tmp = __sync_lock_test_and_set( (kmp_int64*)p, *(kmp_int64*)&v);
599  return *(kmp_real64*)&tmp;
600 }
601 
602 #else
603 
604 extern kmp_int32 __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 v );
605 extern kmp_int32 __kmp_test_then_or32( volatile kmp_int32 *p, kmp_int32 v );
606 extern kmp_int32 __kmp_test_then_and32( volatile kmp_int32 *p, kmp_int32 v );
607 extern kmp_int64 __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 v );
608 extern kmp_int64 __kmp_test_then_or64( volatile kmp_int64 *p, kmp_int64 v );
609 extern kmp_int64 __kmp_test_then_and64( volatile kmp_int64 *p, kmp_int64 v );
610 
611 extern kmp_int8 __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
612 extern kmp_int16 __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
613 extern kmp_int32 __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
614 extern kmp_int32 __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
615 extern kmp_int8 __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
616 extern kmp_int16 __kmp_compare_and_store_ret16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
617 extern kmp_int32 __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
618 extern kmp_int64 __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
619 
620 extern kmp_int8 __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 v );
621 extern kmp_int16 __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 v );
622 extern kmp_int32 __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 v );
623 extern kmp_int64 __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 v );
624 extern kmp_real32 __kmp_xchg_real32( volatile kmp_real32 *p, kmp_real32 v );
625 extern kmp_real64 __kmp_xchg_real64( volatile kmp_real64 *p, kmp_real64 v );
626 
627 # define KMP_TEST_THEN_INC32(p) __kmp_test_then_add32( (p), 1 )
628 # define KMP_TEST_THEN_INC_ACQ32(p) __kmp_test_then_add32( (p), 1 )
629 # define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64( (p), 1LL )
630 # define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64( (p), 1LL )
631 # define KMP_TEST_THEN_ADD4_32(p) __kmp_test_then_add32( (p), 4 )
632 # define KMP_TEST_THEN_ADD4_ACQ32(p) __kmp_test_then_add32( (p), 4 )
633 # define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64( (p), 4LL )
634 # define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64( (p), 4LL )
635 # define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32( (p), -1 )
636 # define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32( (p), -1 )
637 # define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64( (p), -1LL )
638 # define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64( (p), -1LL )
639 # define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32( (p), (v) )
640 # define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64( (p), (v) )
641 
642 # define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32( (p), (v) )
643 # define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32( (p), (v) )
644 # define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64( (p), (v) )
645 # define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64( (p), (v) )
646 
647 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
648 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
649 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
650 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
651 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
652 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
653 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
654 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
655 
656 # if KMP_ARCH_X86
657 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )
658 # else /* 64 bit pointers */
659 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )
660 # endif /* KMP_ARCH_X86 */
661 
662 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __kmp_compare_and_store_ret8( (p), (cv), (sv) )
663 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )
664 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __kmp_compare_and_store_ret32( (p), (cv), (sv) )
665 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __kmp_compare_and_store_ret64( (p), (cv), (sv) )
666 
667 # define KMP_XCHG_FIXED8(p, v) __kmp_xchg_fixed8( (p), (v) );
668 # define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16( (p), (v) );
669 # define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32( (p), (v) );
670 # define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64( (p), (v) );
671 # define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32( (p), (v) );
672 # define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64( (p), (v) );
673 
674 #endif /* KMP_ASM_INTRINS */
675 
676 
677 /* ------------- relaxed consistency memory model stuff ------------------ */
678 
679 #if KMP_OS_WINDOWS
680 # ifdef __ABSOFT_WIN
681 # define KMP_MB() asm ("nop")
682 # define KMP_IMB() asm ("nop")
683 # else
684 # define KMP_MB() /* _asm{ nop } */
685 # define KMP_IMB() /* _asm{ nop } */
686 # endif
687 #endif /* KMP_OS_WINDOWS */
688 
689 #if KMP_ARCH_PPC64
690 # define KMP_MB() __sync_synchronize()
691 #endif
692 
693 #ifndef KMP_MB
694 # define KMP_MB() /* nothing to do */
695 #endif
696 
697 #ifndef KMP_IMB
698 # define KMP_IMB() /* nothing to do */
699 #endif
700 
701 #ifndef KMP_ST_REL32
702 # define KMP_ST_REL32(A,D) ( *(A) = (D) )
703 #endif
704 
705 #ifndef KMP_ST_REL64
706 # define KMP_ST_REL64(A,D) ( *(A) = (D) )
707 #endif
708 
709 #ifndef KMP_LD_ACQ32
710 # define KMP_LD_ACQ32(A) ( *(A) )
711 #endif
712 
713 #ifndef KMP_LD_ACQ64
714 # define KMP_LD_ACQ64(A) ( *(A) )
715 #endif
716 
717 /* ------------------------------------------------------------------------ */
718 //
719 // FIXME - maybe this should this be
720 //
721 // #define TCR_4(a) (*(volatile kmp_int32 *)(&a))
722 // #define TCW_4(a,b) (a) = (*(volatile kmp_int32 *)&(b))
723 //
724 // #define TCR_8(a) (*(volatile kmp_int64 *)(a))
725 // #define TCW_8(a,b) (a) = (*(volatile kmp_int64 *)(&b))
726 //
727 // I'm fairly certain this is the correct thing to do, but I'm afraid
728 // of performance regressions.
729 //
730 
731 #define TCR_4(a) (a)
732 #define TCW_4(a,b) (a) = (b)
733 #define TCR_8(a) (a)
734 #define TCW_8(a,b) (a) = (b)
735 #define TCR_SYNC_4(a) (a)
736 #define TCW_SYNC_4(a,b) (a) = (b)
737 #define TCX_SYNC_4(a,b,c) KMP_COMPARE_AND_STORE_REL32((volatile kmp_int32 *)(volatile void *)&(a), (kmp_int32)(b), (kmp_int32)(c))
738 #define TCR_SYNC_8(a) (a)
739 #define TCW_SYNC_8(a,b) (a) = (b)
740 #define TCX_SYNC_8(a,b,c) KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), (kmp_int64)(b), (kmp_int64)(c))
741 
742 #if KMP_ARCH_X86
743 
744  #define TCR_PTR(a) ((void *)TCR_4(a))
745  #define TCW_PTR(a,b) TCW_4((a),(b))
746  #define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_4(a))
747  #define TCW_SYNC_PTR(a,b) TCW_SYNC_4((a),(b))
748  #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_4((a),(b),(c)))
749 
750 #else /* 64 bit pointers */
751 
752  #define TCR_PTR(a) ((void *)TCR_8(a))
753  #define TCW_PTR(a,b) TCW_8((a),(b))
754  #define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_8(a))
755  #define TCW_SYNC_PTR(a,b) TCW_SYNC_8((a),(b))
756  #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_8((a),(b),(c)))
757 
758 #endif /* KMP_ARCH_X86 */
759 
760 /*
761  * If these FTN_{TRUE,FALSE} values change, may need to
762  * change several places where they are used to check that
763  * language is Fortran, not C.
764  */
765 
766 #ifndef FTN_TRUE
767 # define FTN_TRUE TRUE
768 #endif
769 
770 #ifndef FTN_FALSE
771 # define FTN_FALSE FALSE
772 #endif
773 
774 typedef void (*microtask_t)( int *gtid, int *npr, ... );
775 
776 #ifdef USE_VOLATILE_CAST
777 # define VOLATILE_CAST(x) (volatile x)
778 #else
779 # define VOLATILE_CAST(x) (x)
780 #endif
781 
782 #ifdef KMP_I8
783 # define KMP_WAIT_YIELD __kmp_wait_yield_8
784 # define KMP_EQ __kmp_eq_8
785 # define KMP_NEQ __kmp_neq_8
786 # define KMP_LT __kmp_lt_8
787 # define KMP_GE __kmp_ge_8
788 # define KMP_LE __kmp_le_8
789 #else
790 # define KMP_WAIT_YIELD __kmp_wait_yield_4
791 # define KMP_EQ __kmp_eq_4
792 # define KMP_NEQ __kmp_neq_4
793 # define KMP_LT __kmp_lt_4
794 # define KMP_GE __kmp_ge_4
795 # define KMP_LE __kmp_le_4
796 #endif /* KMP_I8 */
797 
798 /* Workaround for Intel(R) 64 code gen bug when taking address of static array (Intel(R) 64 Tracker #138) */
799 #if (KMP_ARCH_X86_64 || KMP_ARCH_PPC64) && KMP_OS_LINUX
800 # define STATIC_EFI2_WORKAROUND
801 #else
802 # define STATIC_EFI2_WORKAROUND static
803 #endif
804 
805 // Support of BGET usage
806 #ifndef KMP_USE_BGET
807 #define KMP_USE_BGET 1
808 #endif
809 
810 
811 // Switches for OSS builds
812 #ifndef USE_SYSFS_INFO
813 # define USE_SYSFS_INFO 0
814 #endif
815 #ifndef USE_CMPXCHG_FIX
816 # define USE_CMPXCHG_FIX 1
817 #endif
818 
819 // Warning levels
820 enum kmp_warnings_level {
821  kmp_warnings_off = 0, /* No warnings */
822  kmp_warnings_low, /* Minimal warnings (default) */
823  kmp_warnings_explicit = 6, /* Explicitly set to ON - more warnings */
824  kmp_warnings_verbose /* reserved */
825 };
826 
827 #ifdef __cplusplus
828 } // extern "C"
829 #endif // __cplusplus
830 
831 #endif /* KMP_OS_H */