42 #define KMP_FTN_PLAIN 1
43 #define KMP_FTN_APPEND 2
44 #define KMP_FTN_UPPER 3
50 #define KMP_PTR_SKIP (sizeof(void*))
57 #define KMP_MEM_CONS_VOLATILE 0
58 #define KMP_MEM_CONS_FENCE 1
60 #ifndef KMP_MEM_CONS_MODEL
61 # define KMP_MEM_CONS_MODEL KMP_MEM_CONS_VOLATILE
65 #define KMP_COMPILER_ICC 0
66 #define KMP_COMPILER_GCC 0
67 #define KMP_COMPILER_CLANG 0
69 #if defined( __INTEL_COMPILER )
70 # undef KMP_COMPILER_ICC
71 # define KMP_COMPILER_ICC 1
72 #elif defined( __clang__ )
73 # undef KMP_COMPILER_CLANG
74 # define KMP_COMPILER_CLANG 1
75 #elif defined( __GNUC__ )
76 # undef KMP_COMPILER_GCC
77 # define KMP_COMPILER_GCC 1
79 # error Unknown compiler
84 #define KMP_OS_LINUX 0
85 #define KMP_OS_DARWIN 0
86 #define KMP_OS_WINDOWS 0
89 #define KMP_ARCH_X86 0
90 #define KMP_ARCH_X86_64 0
93 # undef KMP_OS_WINDOWS
94 # define KMP_OS_WINDOWS 1
97 #if ( defined __APPLE__ && defined __MACH__ )
99 # define KMP_OS_DARWIN 1
102 #if ( defined __linux )
104 # define KMP_OS_LINUX 1
107 #if (1 != KMP_OS_LINUX + KMP_OS_DARWIN + KMP_OS_WINDOWS)
111 #if KMP_OS_LINUX || KMP_OS_DARWIN
113 # define KMP_OS_UNIX 1
117 # if defined _M_AMD64
118 # undef KMP_ARCH_X86_64
119 # define KMP_ARCH_X86_64 1
122 # define KMP_ARCH_X86 1
127 # if defined __x86_64
128 # undef KMP_ARCH_X86_64
129 # define KMP_ARCH_X86_64 1
130 # elif defined __i386
132 # define KMP_ARCH_X86 1
136 #if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7R__) || \
137 defined(__ARM_ARCH_7A__)
138 # define KMP_ARCH_ARMV7 1
141 #if defined(KMP_ARCH_ARMV7) || defined(__ARM_ARCH_6__) || \
142 defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || \
143 defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6T2__) || \
144 defined(__ARM_ARCH_6ZK__)
145 # define KMP_ARCH_ARMV6 1
148 #if defined(KMP_ARCH_ARMV6) || defined(__ARM_ARCH_5T__) || \
149 defined(__ARM_ARCH_5E__) || defined(__ARM_ARCH_5TE__) || \
150 defined(__ARM_ARCH_5TEJ__)
151 # define KMP_ARCH_ARMV5 1
154 #if defined(KMP_ARCH_ARMV5) || defined(__ARM_ARCH_4__) || \
155 defined(__ARM_ARCH_4T__)
156 # define KMP_ARCH_ARMV4 1
159 #if defined(KMP_ARCH_ARMV4) || defined(__ARM_ARCH_3__) || \
160 defined(__ARM_ARCH_3M__)
161 # define KMP_ARCH_ARMV3 1
164 #if defined(KMP_ARCH_ARMV3) || defined(__ARM_ARCH_2__)
165 # define KMP_ARCH_ARMV2 1
168 #if defined(KMP_ARCH_ARMV2)
169 # define KMP_ARCH_ARM 1
172 #if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM)
173 # error Unknown or unsupported architecture
177 #define KMP_HAVE_QUAD 0
178 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
179 # if KMP_COMPILER_ICC
181 # undef KMP_HAVE_QUAD
182 # define KMP_HAVE_QUAD 1
183 # elif KMP_COMPILER_CLANG
186 typedef long double _Quad;
187 # elif KMP_COMPILER_GCC
188 typedef __float128 _Quad;
189 # undef KMP_HAVE_QUAD
190 # define KMP_HAVE_QUAD 1
193 # if __LDBL_MAX_EXP__ >= 16384 && KMP_COMPILER_GCC
194 typedef long double _Quad;
195 # undef KMP_HAVE_QUAD
196 # define KMP_HAVE_QUAD 1
201 typedef char kmp_int8;
202 typedef unsigned char kmp_uint8;
203 typedef short kmp_int16;
204 typedef unsigned short kmp_uint16;
205 typedef int kmp_int32;
206 typedef unsigned int kmp_uint32;
207 # define KMP_INT32_SPEC "d"
208 # define KMP_UINT32_SPEC "u"
209 # ifndef KMP_STRUCT64
210 typedef __int64 kmp_int64;
211 typedef unsigned __int64 kmp_uint64;
212 #define KMP_INT64_SPEC "I64d"
213 #define KMP_UINT64_SPEC "I64u"
215 struct kmp_struct64 {
218 typedef struct kmp_struct64 kmp_int64;
219 typedef struct kmp_struct64 kmp_uint64;
223 # define KMP_INTPTR 1
224 typedef __int64 kmp_intptr_t;
225 typedef unsigned __int64 kmp_uintptr_t;
226 # define KMP_INTPTR_SPEC "I64d"
227 # define KMP_UINTPTR_SPEC "I64u"
232 typedef char kmp_int8;
233 typedef unsigned char kmp_uint8;
234 typedef short kmp_int16;
235 typedef unsigned short kmp_uint16;
236 typedef int kmp_int32;
237 typedef unsigned int kmp_uint32;
238 typedef long long kmp_int64;
239 typedef unsigned long long kmp_uint64;
240 # define KMP_INT32_SPEC "d"
241 # define KMP_UINT32_SPEC "u"
242 # define KMP_INT64_SPEC "lld"
243 # define KMP_UINT64_SPEC "llu"
246 #if KMP_ARCH_X86 || KMP_ARCH_ARM
247 # define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
248 #elif KMP_ARCH_X86_64
249 # define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
251 # error "Can't determine size_t printf format specifier."
255 # define KMP_SIZE_T_MAX (0xFFFFFFFF)
257 # define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)
260 typedef size_t kmp_size_t;
261 typedef float kmp_real32;
262 typedef double kmp_real64;
265 # define KMP_INTPTR 1
266 typedef long kmp_intptr_t;
267 typedef unsigned long kmp_uintptr_t;
268 # define KMP_INTPTR_SPEC "ld"
269 # define KMP_UINTPTR_SPEC "lu"
273 typedef kmp_int64 kmp_int;
274 typedef kmp_uint64 kmp_uint;
275 # define KMP_INT_SPEC KMP_INT64_SPEC
276 # define KMP_UINT_SPEC KMP_UINT64_SPEC
277 # define KMP_INT_MAX ((kmp_int64)0x7FFFFFFFFFFFFFFFLL)
278 # define KMP_INT_MIN ((kmp_int64)0x8000000000000000LL)
280 typedef kmp_int32 kmp_int;
281 typedef kmp_uint32 kmp_uint;
282 # define KMP_INT_SPEC KMP_INT32_SPEC
283 # define KMP_UINT_SPEC KMP_UINT32_SPEC
284 # define KMP_INT_MAX ((kmp_int32)0x7FFFFFFF)
285 # define KMP_INT_MIN ((kmp_int32)0x80000000)
292 template<
typename T >
295 typedef T unsigned_t;
296 typedef T floating_t;
297 static char const * spec;
301 struct traits_t< signed int > {
302 typedef signed int signed_t;
303 typedef unsigned int unsigned_t;
304 typedef double floating_t;
305 static char const * spec;
309 struct traits_t< unsigned int > {
310 typedef signed int signed_t;
311 typedef unsigned int unsigned_t;
312 typedef double floating_t;
313 static char const * spec;
317 struct traits_t< signed long long > {
318 typedef signed long long signed_t;
319 typedef unsigned long long unsigned_t;
320 typedef long double floating_t;
321 static char const * spec;
325 struct traits_t< unsigned long long > {
326 typedef signed long long signed_t;
327 typedef unsigned long long unsigned_t;
328 typedef long double floating_t;
329 static char const * spec;
332 #endif // __cplusplus
334 #define KMP_EXPORT extern
337 #define __forceinline __inline
340 #define PAGE_SIZE (0x4000)
341 #define PAGE_ALIGNED(_addr) ( ! ((size_t) _addr & \
342 (size_t)(PAGE_SIZE - 1)))
343 #define ALIGN_TO_PAGE(x) (void *)(((size_t)(x)) & ~((size_t)(PAGE_SIZE - 1)))
349 #endif // __cplusplus
353 #define CACHE_LINE 128
355 #if ( CACHE_LINE < 64 ) && ! defined( KMP_OS_DARWIN )
357 #warning CACHE_LINE is too small.
362 #if !defined KMP_PERF_V19
363 # define KMP_PERF_V19 KMP_ON
367 #if !defined KMP_PERF_V106
368 # define KMP_PERF_V106 KMP_ON
371 #define KMP_CACHE_PREFETCH(ADDR)
375 #if KMP_OS_UNIX && defined(__GNUC__)
376 # define KMP_DO_ALIGN(bytes) __attribute__((aligned(bytes)))
377 # define KMP_ALIGN_CACHE __attribute__((aligned(CACHE_LINE)))
378 # define KMP_ALIGN(bytes) __attribute__((aligned(bytes)))
380 # define KMP_DO_ALIGN(bytes) __declspec( align(bytes) )
381 # define KMP_ALIGN_CACHE __declspec( align(CACHE_LINE) )
382 # define KMP_ALIGN(bytes) __declspec( align(bytes) )
385 #if defined(__MIC__) || defined(__MIC2__)
388 # if __MIC2__ || __KNC__
402 enum kmp_mem_fence_type {
414 #if KMP_ASM_INTRINS && KMP_OS_WINDOWS
418 #pragma intrinsic(InterlockedExchangeAdd)
419 #pragma intrinsic(InterlockedCompareExchange)
420 #pragma intrinsic(InterlockedExchange)
421 #pragma intrinsic(InterlockedExchange64)
427 # define KMP_TEST_THEN_INC32(p) InterlockedExchangeAdd( (volatile long *)(p), 1 )
428 # define KMP_TEST_THEN_INC_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), 1 )
429 # define KMP_TEST_THEN_ADD4_32(p) InterlockedExchangeAdd( (volatile long *)(p), 4 )
430 # define KMP_TEST_THEN_ADD4_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), 4 )
431 # define KMP_TEST_THEN_DEC32(p) InterlockedExchangeAdd( (volatile long *)(p), -1 )
432 # define KMP_TEST_THEN_DEC_ACQ32(p) InterlockedExchangeAdd( (volatile long *)(p), -1 )
433 # define KMP_TEST_THEN_ADD32(p, v) InterlockedExchangeAdd( (volatile long *)(p), (v) )
435 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) InterlockedCompareExchange( (volatile long *)(p),(long)(sv),(long)(cv) )
437 # define KMP_XCHG_FIXED32(p, v) InterlockedExchange( (volatile long *)(p), (long)(v) )
438 # define KMP_XCHG_FIXED64(p, v) InterlockedExchange64( (volatile kmp_int64 *)(p), (kmp_int64)(v) )
440 inline kmp_real32 KMP_XCHG_REAL32(
volatile kmp_real32 *p, kmp_real32 v)
442 kmp_int32 tmp = InterlockedExchange( (
volatile long *)p, *(
long *)&v);
443 return *(kmp_real32*)&tmp;
449 extern kmp_int32 __kmp_test_then_add32(
volatile kmp_int32 *p, kmp_int32 v );
450 extern kmp_int32 __kmp_test_then_or32(
volatile kmp_int32 *p, kmp_int32 v );
451 extern kmp_int32 __kmp_test_then_and32(
volatile kmp_int32 *p, kmp_int32 v );
452 extern kmp_int64 __kmp_test_then_add64(
volatile kmp_int64 *p, kmp_int64 v );
453 extern kmp_int64 __kmp_test_then_or64(
volatile kmp_int64 *p, kmp_int64 v );
454 extern kmp_int64 __kmp_test_then_and64(
volatile kmp_int64 *p, kmp_int64 v );
456 extern kmp_int8 __kmp_compare_and_store8(
volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
457 extern kmp_int16 __kmp_compare_and_store16(
volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
458 extern kmp_int32 __kmp_compare_and_store32(
volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
459 extern kmp_int32 __kmp_compare_and_store64(
volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
460 extern kmp_int8 __kmp_compare_and_store_ret8(
volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
461 extern kmp_int16 __kmp_compare_and_store_ret16(
volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
462 extern kmp_int32 __kmp_compare_and_store_ret32(
volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
463 extern kmp_int64 __kmp_compare_and_store_ret64(
volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
465 extern kmp_int8 __kmp_xchg_fixed8(
volatile kmp_int8 *p, kmp_int8 v );
466 extern kmp_int16 __kmp_xchg_fixed16(
volatile kmp_int16 *p, kmp_int16 v );
467 extern kmp_int32 __kmp_xchg_fixed32(
volatile kmp_int32 *p, kmp_int32 v );
468 extern kmp_int64 __kmp_xchg_fixed64(
volatile kmp_int64 *p, kmp_int64 v );
469 extern kmp_real32 __kmp_xchg_real32(
volatile kmp_real32 *p, kmp_real32 v );
470 extern kmp_real64 __kmp_xchg_real64(
volatile kmp_real64 *p, kmp_real64 v );
474 # define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64( (p), 1LL )
475 # define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64( (p), 1LL )
478 # define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64( (p), 4LL )
479 # define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64( (p), 4LL )
482 # define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64( (p), -1LL )
483 # define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64( (p), -1LL )
485 # define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64( (p), (v) )
487 # define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32( (p), (v) )
488 # define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32( (p), (v) )
489 # define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64( (p), (v) )
490 # define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64( (p), (v) )
492 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
493 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
494 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
495 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
496 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
497 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
498 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
499 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
502 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )
504 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )
507 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __kmp_compare_and_store_ret8( (p), (cv), (sv) )
508 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )
510 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __kmp_compare_and_store_ret64( (p), (cv), (sv) )
512 # define KMP_XCHG_FIXED8(p, v) __kmp_xchg_fixed8( (p), (v) );
513 # define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16( (p), (v) );
517 # define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64( (p), (v) );
520 #elif (KMP_ASM_INTRINS && (KMP_OS_LINUX || KMP_OS_DARWIN)) || !(KMP_ARCH_X86 || KMP_ARCH_X86_64)
523 # define KMP_TEST_THEN_INC32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 1 )
524 # define KMP_TEST_THEN_INC_ACQ32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 1 )
525 # define KMP_TEST_THEN_INC64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 1LL )
526 # define KMP_TEST_THEN_INC_ACQ64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 1LL )
527 # define KMP_TEST_THEN_ADD4_32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 4 )
528 # define KMP_TEST_THEN_ADD4_ACQ32(p) __sync_fetch_and_add( (kmp_int32 *)(p), 4 )
529 # define KMP_TEST_THEN_ADD4_64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 4LL )
530 # define KMP_TEST_THEN_ADD4_ACQ64(p) __sync_fetch_and_add( (kmp_int64 *)(p), 4LL )
531 # define KMP_TEST_THEN_DEC32(p) __sync_fetch_and_sub( (kmp_int32 *)(p), 1 )
532 # define KMP_TEST_THEN_DEC_ACQ32(p) __sync_fetch_and_sub( (kmp_int32 *)(p), 1 )
533 # define KMP_TEST_THEN_DEC64(p) __sync_fetch_and_sub( (kmp_int64 *)(p), 1LL )
534 # define KMP_TEST_THEN_DEC_ACQ64(p) __sync_fetch_and_sub( (kmp_int64 *)(p), 1LL )
535 # define KMP_TEST_THEN_ADD32(p, v) __sync_fetch_and_add( (kmp_int32 *)(p), (v) )
536 # define KMP_TEST_THEN_ADD64(p, v) __sync_fetch_and_add( (kmp_int64 *)(p), (v) )
538 # define KMP_TEST_THEN_OR32(p, v) __sync_fetch_and_or( (kmp_int32 *)(p), (v) )
539 # define KMP_TEST_THEN_AND32(p, v) __sync_fetch_and_and( (kmp_int32 *)(p), (v) )
540 # define KMP_TEST_THEN_OR64(p, v) __sync_fetch_and_or( (kmp_int64 *)(p), (v) )
541 # define KMP_TEST_THEN_AND64(p, v) __sync_fetch_and_and( (kmp_int64 *)(p), (v) )
543 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
544 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
545 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
546 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
547 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
548 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
549 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
550 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __sync_bool_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
551 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __sync_bool_compare_and_swap( (volatile void **)(p),(void *)(cv),(void *)(sv) )
553 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint8 *)(p),(kmp_uint8)(cv),(kmp_uint8)(sv) )
554 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint16 *)(p),(kmp_uint16)(cv),(kmp_uint16)(sv) )
555 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint32 *)(p),(kmp_uint32)(cv),(kmp_uint32)(sv) )
556 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __sync_val_compare_and_swap( (volatile kmp_uint64 *)(p),(kmp_uint64)(cv),(kmp_uint64)(sv) )
558 #define KMP_XCHG_FIXED8(p, v) __sync_lock_test_and_set( (volatile kmp_uint8 *)(p), (kmp_uint8)(v) )
559 #define KMP_XCHG_FIXED16(p, v) __sync_lock_test_and_set( (volatile kmp_uint16 *)(p), (kmp_uint16)(v) )
560 #define KMP_XCHG_FIXED32(p, v) __sync_lock_test_and_set( (volatile kmp_uint32 *)(p), (kmp_uint32)(v) )
561 #define KMP_XCHG_FIXED64(p, v) __sync_lock_test_and_set( (volatile kmp_uint64 *)(p), (kmp_uint64)(v) )
563 inline kmp_real32 KMP_XCHG_REAL32(
volatile kmp_real32 *p, kmp_real32 v)
565 kmp_int32 tmp = __sync_lock_test_and_set( (kmp_int32*)p, *(kmp_int32*)&v);
566 return *(kmp_real32*)&tmp;
569 inline kmp_real64 KMP_XCHG_REAL64(
volatile kmp_real64 *p, kmp_real64 v)
571 kmp_int64 tmp = __sync_lock_test_and_set( (kmp_int64*)p, *(kmp_int64*)&v);
572 return *(kmp_real64*)&tmp;
577 extern kmp_int32 __kmp_test_then_add32(
volatile kmp_int32 *p, kmp_int32 v );
578 extern kmp_int32 __kmp_test_then_or32(
volatile kmp_int32 *p, kmp_int32 v );
579 extern kmp_int32 __kmp_test_then_and32(
volatile kmp_int32 *p, kmp_int32 v );
580 extern kmp_int64 __kmp_test_then_add64(
volatile kmp_int64 *p, kmp_int64 v );
581 extern kmp_int64 __kmp_test_then_or64(
volatile kmp_int64 *p, kmp_int64 v );
582 extern kmp_int64 __kmp_test_then_and64(
volatile kmp_int64 *p, kmp_int64 v );
584 extern kmp_int8 __kmp_compare_and_store8(
volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
585 extern kmp_int16 __kmp_compare_and_store16(
volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
586 extern kmp_int32 __kmp_compare_and_store32(
volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
587 extern kmp_int32 __kmp_compare_and_store64(
volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
588 extern kmp_int8 __kmp_compare_and_store_ret8(
volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
589 extern kmp_int16 __kmp_compare_and_store_ret16(
volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
590 extern kmp_int32 __kmp_compare_and_store_ret32(
volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
591 extern kmp_int64 __kmp_compare_and_store_ret64(
volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
593 extern kmp_int8 __kmp_xchg_fixed8(
volatile kmp_int8 *p, kmp_int8 v );
594 extern kmp_int16 __kmp_xchg_fixed16(
volatile kmp_int16 *p, kmp_int16 v );
595 extern kmp_int32 __kmp_xchg_fixed32(
volatile kmp_int32 *p, kmp_int32 v );
596 extern kmp_int64 __kmp_xchg_fixed64(
volatile kmp_int64 *p, kmp_int64 v );
597 extern kmp_real32 __kmp_xchg_real32(
volatile kmp_real32 *p, kmp_real32 v );
598 extern kmp_real64 __kmp_xchg_real64(
volatile kmp_real64 *p, kmp_real64 v );
600 # define KMP_TEST_THEN_INC32(p) __kmp_test_then_add32( (p), 1 )
601 # define KMP_TEST_THEN_INC_ACQ32(p) __kmp_test_then_add32( (p), 1 )
602 # define KMP_TEST_THEN_INC64(p) __kmp_test_then_add64( (p), 1LL )
603 # define KMP_TEST_THEN_INC_ACQ64(p) __kmp_test_then_add64( (p), 1LL )
604 # define KMP_TEST_THEN_ADD4_32(p) __kmp_test_then_add32( (p), 4 )
605 # define KMP_TEST_THEN_ADD4_ACQ32(p) __kmp_test_then_add32( (p), 4 )
606 # define KMP_TEST_THEN_ADD4_64(p) __kmp_test_then_add64( (p), 4LL )
607 # define KMP_TEST_THEN_ADD4_ACQ64(p) __kmp_test_then_add64( (p), 4LL )
608 # define KMP_TEST_THEN_DEC32(p) __kmp_test_then_add32( (p), -1 )
609 # define KMP_TEST_THEN_DEC_ACQ32(p) __kmp_test_then_add32( (p), -1 )
610 # define KMP_TEST_THEN_DEC64(p) __kmp_test_then_add64( (p), -1LL )
611 # define KMP_TEST_THEN_DEC_ACQ64(p) __kmp_test_then_add64( (p), -1LL )
612 # define KMP_TEST_THEN_ADD32(p, v) __kmp_test_then_add32( (p), (v) )
613 # define KMP_TEST_THEN_ADD64(p, v) __kmp_test_then_add64( (p), (v) )
615 # define KMP_TEST_THEN_OR32(p, v) __kmp_test_then_or32( (p), (v) )
616 # define KMP_TEST_THEN_AND32(p, v) __kmp_test_then_and32( (p), (v) )
617 # define KMP_TEST_THEN_OR64(p, v) __kmp_test_then_or64( (p), (v) )
618 # define KMP_TEST_THEN_AND64(p, v) __kmp_test_then_and64( (p), (v) )
620 # define KMP_COMPARE_AND_STORE_ACQ8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
621 # define KMP_COMPARE_AND_STORE_REL8(p, cv, sv) __kmp_compare_and_store8( (p), (cv), (sv) )
622 # define KMP_COMPARE_AND_STORE_ACQ16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
623 # define KMP_COMPARE_AND_STORE_REL16(p, cv, sv) __kmp_compare_and_store16( (p), (cv), (sv) )
624 # define KMP_COMPARE_AND_STORE_ACQ32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
625 # define KMP_COMPARE_AND_STORE_REL32(p, cv, sv) __kmp_compare_and_store32( (p), (cv), (sv) )
626 # define KMP_COMPARE_AND_STORE_ACQ64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
627 # define KMP_COMPARE_AND_STORE_REL64(p, cv, sv) __kmp_compare_and_store64( (p), (cv), (sv) )
630 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store32( (volatile kmp_int32*)(p), (kmp_int32)(cv), (kmp_int32)(sv) )
632 # define KMP_COMPARE_AND_STORE_PTR(p, cv, sv) __kmp_compare_and_store64( (volatile kmp_int64*)(p), (kmp_int64)(cv), (kmp_int64)(sv) )
635 # define KMP_COMPARE_AND_STORE_RET8(p, cv, sv) __kmp_compare_and_store_ret8( (p), (cv), (sv) )
636 # define KMP_COMPARE_AND_STORE_RET16(p, cv, sv) __kmp_compare_and_store_ret16( (p), (cv), (sv) )
637 # define KMP_COMPARE_AND_STORE_RET32(p, cv, sv) __kmp_compare_and_store_ret32( (p), (cv), (sv) )
638 # define KMP_COMPARE_AND_STORE_RET64(p, cv, sv) __kmp_compare_and_store_ret64( (p), (cv), (sv) )
640 # define KMP_XCHG_FIXED8(p, v) __kmp_xchg_fixed8( (p), (v) );
641 # define KMP_XCHG_FIXED16(p, v) __kmp_xchg_fixed16( (p), (v) );
642 # define KMP_XCHG_FIXED32(p, v) __kmp_xchg_fixed32( (p), (v) );
643 # define KMP_XCHG_FIXED64(p, v) __kmp_xchg_fixed64( (p), (v) );
644 # define KMP_XCHG_REAL32(p, v) __kmp_xchg_real32( (p), (v) );
645 # define KMP_XCHG_REAL64(p, v) __kmp_xchg_real64( (p), (v) );
654 extern kmp_real32 __kmp_test_then_add_real32 (
volatile kmp_real32 *p, kmp_real32 v );
655 extern kmp_real64 __kmp_test_then_add_real64 (
volatile kmp_real64 *p, kmp_real64 v );
656 # define KMP_TEST_THEN_ADD_REAL32(p, v) __kmp_test_then_add_real32( (p), (v) )
657 # define KMP_TEST_THEN_ADD_REAL64(p, v) __kmp_test_then_add_real64( (p), (v) )
665 # define KMP_MB() asm ("nop")
666 # define KMP_IMB() asm ("nop")
682 # define KMP_ST_REL32(A,D) ( *(A) = (D) )
686 # define KMP_ST_REL64(A,D) ( *(A) = (D) )
690 # define KMP_LD_ACQ32(A) ( *(A) )
694 # define KMP_LD_ACQ64(A) ( *(A) )
712 #define TCW_4(a,b) (a) = (b)
714 #define TCW_8(a,b) (a) = (b)
715 #define TCR_SYNC_4(a) (a)
716 #define TCW_SYNC_4(a,b) (a) = (b)
717 #define TCX_SYNC_4(a,b,c) KMP_COMPARE_AND_STORE_REL32((volatile kmp_int32 *)(volatile void *)&(a), (kmp_int32)(b), (kmp_int32)(c))
718 #define TCR_SYNC_8(a) (a)
719 #define TCW_SYNC_8(a,b) (a) = (b)
720 #define TCX_SYNC_8(a,b,c) KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a), (kmp_int64)(b), (kmp_int64)(c))
724 #define TCR_PTR(a) ((void *)TCR_4(a))
725 #define TCW_PTR(a,b) TCW_4((a),(b))
726 #define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_4(a))
727 #define TCW_SYNC_PTR(a,b) TCW_SYNC_4((a),(b))
728 #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_4((a),(b),(c)))
732 #define TCR_PTR(a) ((void *)TCR_8(a))
733 #define TCW_PTR(a,b) TCW_8((a),(b))
734 #define TCR_SYNC_PTR(a) ((void *)TCR_SYNC_8(a))
735 #define TCW_SYNC_PTR(a,b) TCW_SYNC_8((a),(b))
736 #define TCX_SYNC_PTR(a,b,c) ((void *)TCX_SYNC_8((a),(b),(c)))
747 # define FTN_TRUE TRUE
751 # define FTN_FALSE FALSE
754 typedef void (*microtask_t)(
int *gtid,
int *npr, ... );
756 #ifdef USE_VOLATILE_CAST
757 # define VOLATILE_CAST(x) (volatile x)
759 # define VOLATILE_CAST(x) (x)
763 # define KMP_WAIT_YIELD __kmp_wait_yield_8
764 # define KMP_EQ __kmp_eq_8
765 # define KMP_NEQ __kmp_neq_8
766 # define KMP_LT __kmp_lt_8
767 # define KMP_GE __kmp_ge_8
768 # define KMP_LE __kmp_le_8
770 # define KMP_WAIT_YIELD __kmp_wait_yield_4
771 # define KMP_EQ __kmp_eq_4
772 # define KMP_NEQ __kmp_neq_4
773 # define KMP_LT __kmp_lt_4
774 # define KMP_GE __kmp_ge_4
775 # define KMP_LE __kmp_le_4
779 #if KMP_ARCH_X86_64 && KMP_OS_LINUX
780 # define STATIC_EFI2_WORKAROUND
782 # define STATIC_EFI2_WORKAROUND static
787 #define KMP_USE_BGET 1
792 #ifndef USE_SYSFS_INFO
793 # define USE_SYSFS_INFO 0
795 #ifndef USE_CMPXCHG_FIX
796 # define USE_CMPXCHG_FIX 1
800 enum kmp_warnings_level {
801 kmp_warnings_off = 0,
803 kmp_warnings_explicit = 6,
809 #endif // __cplusplus