37 #include "kmp_atomic.h"
40 typedef unsigned char uchar;
41 typedef unsigned short ushort;
564 #ifndef KMP_GOMP_COMPAT
565 int __kmp_atomic_mode = 1;
567 int __kmp_atomic_mode = 2;
572 kmp_atomic_lock_t __kmp_atomic_lock;
573 kmp_atomic_lock_t __kmp_atomic_lock_1i;
574 kmp_atomic_lock_t __kmp_atomic_lock_2i;
575 kmp_atomic_lock_t __kmp_atomic_lock_4i;
576 kmp_atomic_lock_t __kmp_atomic_lock_4r;
577 kmp_atomic_lock_t __kmp_atomic_lock_8i;
578 kmp_atomic_lock_t __kmp_atomic_lock_8r;
579 kmp_atomic_lock_t __kmp_atomic_lock_8c;
580 kmp_atomic_lock_t __kmp_atomic_lock_10r;
581 kmp_atomic_lock_t __kmp_atomic_lock_16r;
582 kmp_atomic_lock_t __kmp_atomic_lock_16c;
583 kmp_atomic_lock_t __kmp_atomic_lock_20c;
584 kmp_atomic_lock_t __kmp_atomic_lock_32c;
594 #define KMP_ATOMIC_VOLATILE volatile
596 #if ( KMP_ARCH_X86 ) && KMP_HAVE_QUAD
598 static inline void operator +=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q += rhs.q; };
599 static inline void operator -=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q -= rhs.q; };
600 static inline void operator *=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q *= rhs.q; };
601 static inline void operator /=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q /= rhs.q; };
602 static inline bool operator < ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q < rhs.q; }
603 static inline bool operator > ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q > rhs.q; }
605 static inline void operator +=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q += rhs.q; };
606 static inline void operator -=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q -= rhs.q; };
607 static inline void operator *=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q *= rhs.q; };
608 static inline void operator /=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q /= rhs.q; };
609 static inline bool operator < ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q < rhs.q; }
610 static inline bool operator > ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q > rhs.q; }
612 static inline void operator +=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q += rhs.q; };
613 static inline void operator -=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q -= rhs.q; };
614 static inline void operator *=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q *= rhs.q; };
615 static inline void operator /=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q /= rhs.q; };
617 static inline void operator +=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q += rhs.q; };
618 static inline void operator -=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q -= rhs.q; };
619 static inline void operator *=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q *= rhs.q; };
620 static inline void operator /=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q /= rhs.q; };
633 #define KMP_CHECK_GTID \
634 if ( gtid == KMP_GTID_UNKNOWN ) { \
635 gtid = __kmp_entry_gtid(); \
636 } // check and get gtid when needed
642 #define ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
643 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
645 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
646 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
650 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
651 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
652 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
653 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
654 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
655 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
656 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
657 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
658 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
659 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
660 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
661 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
662 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
670 #define OP_CRITICAL(OP,LCK_ID) \
671 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
675 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
700 #ifdef KMP_GOMP_COMPAT
701 # define OP_GOMP_CRITICAL(OP,FLAG) \
702 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
704 OP_CRITICAL( OP, 0 ); \
708 # define OP_GOMP_CRITICAL(OP,FLAG)
712 # define KMP_DO_PAUSE _mm_delay_32( 1 )
714 # define KMP_DO_PAUSE KMP_CPU_PAUSE()
722 #define OP_CMPXCHG(TYPE,BITS,OP) \
724 TYPE old_value, new_value; \
725 old_value = *(TYPE volatile *)lhs; \
726 new_value = old_value OP rhs; \
727 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
728 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
729 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
733 old_value = *(TYPE volatile *)lhs; \
734 new_value = old_value OP rhs; \
746 #define OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
748 char anonym[ ( sizeof( TYPE ) == sizeof( kmp_int##BITS ) ) ? ( 1 ) : ( 0 ) ] = { 1 }; \
751 kmp_int##BITS *vvv; \
753 struct _sss old_value, new_value; \
754 old_value.vvv = ( kmp_int##BITS * )&old_value.cmp; \
755 new_value.vvv = ( kmp_int##BITS * )&new_value.cmp; \
756 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
757 new_value.cmp = old_value.cmp OP rhs; \
758 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
759 *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
760 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv ) ) \
764 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
765 new_value.cmp = old_value.cmp OP rhs; \
769 #endif // USE_CMPXCHG_FIX
771 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
775 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
776 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
777 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
779 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
782 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
783 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
784 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
785 OP_CMPXCHG(TYPE,BITS,OP) \
790 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
791 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
792 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
793 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
801 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
802 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
803 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
804 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
806 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
809 OP_CRITICAL(OP##=,LCK_ID) \
813 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
814 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
815 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
816 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
817 OP_CMPXCHG(TYPE,BITS,OP) \
820 OP_CRITICAL(OP##=,LCK_ID) \
826 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
827 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
828 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
829 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
830 OP_CMPXCHG(TYPE,BITS,OP) \
833 OP_CRITICAL(OP##=,LCK_ID) \
837 #endif // USE_CMPXCHG_FIX
841 ATOMIC_FIXED_ADD( fixed4, add, kmp_int32, 32, +, 4i, 3, 0 )
842 ATOMIC_FIXED_ADD( fixed4, sub, kmp_int32, 32, -, 4i, 3, 0 )
844 ATOMIC_CMPXCHG( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
845 ATOMIC_CMPXCHG( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
848 ATOMIC_FIXED_ADD( fixed8, add, kmp_int64, 64, +, 8i, 7, KMP_ARCH_X86 )
849 ATOMIC_FIXED_ADD( fixed8, sub, kmp_int64, 64, -, 8i, 7, KMP_ARCH_X86 )
851 ATOMIC_CMPXCHG( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
852 ATOMIC_CMPXCHG( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
869 ATOMIC_CMPXCHG( fixed1, add, kmp_int8, 8, +, 1i, 0, KMP_ARCH_X86 )
870 ATOMIC_CMPXCHG( fixed1, andb, kmp_int8, 8, &, 1i, 0, 0 )
871 ATOMIC_CMPXCHG( fixed1, div, kmp_int8, 8, /, 1i, 0, KMP_ARCH_X86 )
872 ATOMIC_CMPXCHG( fixed1u, div, kmp_uint8, 8, /, 1i, 0, KMP_ARCH_X86 )
873 ATOMIC_CMPXCHG( fixed1, mul, kmp_int8, 8, *, 1i, 0, KMP_ARCH_X86 )
874 ATOMIC_CMPXCHG( fixed1, orb, kmp_int8, 8, |, 1i, 0, 0 )
875 ATOMIC_CMPXCHG( fixed1, shl, kmp_int8, 8, <<, 1i, 0, KMP_ARCH_X86 )
876 ATOMIC_CMPXCHG( fixed1, shr, kmp_int8, 8, >>, 1i, 0, KMP_ARCH_X86 )
877 ATOMIC_CMPXCHG( fixed1u, shr, kmp_uint8, 8, >>, 1i, 0, KMP_ARCH_X86 )
878 ATOMIC_CMPXCHG( fixed1, sub, kmp_int8, 8, -, 1i, 0, KMP_ARCH_X86 )
879 ATOMIC_CMPXCHG( fixed1, xor, kmp_int8, 8, ^, 1i, 0, 0 )
880 ATOMIC_CMPXCHG( fixed2, add, kmp_int16, 16, +, 2i, 1, KMP_ARCH_X86 )
881 ATOMIC_CMPXCHG( fixed2, andb, kmp_int16, 16, &, 2i, 1, 0 )
882 ATOMIC_CMPXCHG( fixed2, div, kmp_int16, 16, /, 2i, 1, KMP_ARCH_X86 )
883 ATOMIC_CMPXCHG( fixed2u, div, kmp_uint16, 16, /, 2i, 1, KMP_ARCH_X86 )
884 ATOMIC_CMPXCHG( fixed2, mul, kmp_int16, 16, *, 2i, 1, KMP_ARCH_X86 )
885 ATOMIC_CMPXCHG( fixed2, orb, kmp_int16, 16, |, 2i, 1, 0 )
886 ATOMIC_CMPXCHG( fixed2, shl, kmp_int16, 16, <<, 2i, 1, KMP_ARCH_X86 )
887 ATOMIC_CMPXCHG( fixed2, shr, kmp_int16, 16, >>, 2i, 1, KMP_ARCH_X86 )
888 ATOMIC_CMPXCHG( fixed2u, shr, kmp_uint16, 16, >>, 2i, 1, KMP_ARCH_X86 )
889 ATOMIC_CMPXCHG( fixed2, sub, kmp_int16, 16, -, 2i, 1, KMP_ARCH_X86 )
890 ATOMIC_CMPXCHG( fixed2, xor, kmp_int16, 16, ^, 2i, 1, 0 )
891 ATOMIC_CMPXCHG( fixed4, andb, kmp_int32, 32, &, 4i, 3, 0 )
892 ATOMIC_CMPXCHG( fixed4, div, kmp_int32, 32, /, 4i, 3, KMP_ARCH_X86 )
893 ATOMIC_CMPXCHG( fixed4u, div, kmp_uint32, 32, /, 4i, 3, KMP_ARCH_X86 )
894 ATOMIC_CMPXCHG( fixed4, mul, kmp_int32, 32, *, 4i, 3, KMP_ARCH_X86 )
895 ATOMIC_CMPXCHG( fixed4, orb, kmp_int32, 32, |, 4i, 3, 0 )
896 ATOMIC_CMPXCHG( fixed4, shl, kmp_int32, 32, <<, 4i, 3, KMP_ARCH_X86 )
897 ATOMIC_CMPXCHG( fixed4, shr, kmp_int32, 32, >>, 4i, 3, KMP_ARCH_X86 )
898 ATOMIC_CMPXCHG( fixed4u, shr, kmp_uint32, 32, >>, 4i, 3, KMP_ARCH_X86 )
899 ATOMIC_CMPXCHG( fixed4, xor, kmp_int32, 32, ^, 4i, 3, 0 )
900 ATOMIC_CMPXCHG( fixed8, andb, kmp_int64, 64, &, 8i, 7, KMP_ARCH_X86 )
901 ATOMIC_CMPXCHG( fixed8, div, kmp_int64, 64, /, 8i, 7, KMP_ARCH_X86 )
902 ATOMIC_CMPXCHG( fixed8u, div, kmp_uint64, 64, /, 8i, 7, KMP_ARCH_X86 )
903 ATOMIC_CMPXCHG( fixed8, mul, kmp_int64, 64, *, 8i, 7, KMP_ARCH_X86 )
904 ATOMIC_CMPXCHG( fixed8, orb, kmp_int64, 64, |, 8i, 7, KMP_ARCH_X86 )
905 ATOMIC_CMPXCHG( fixed8, shl, kmp_int64, 64, <<, 8i, 7, KMP_ARCH_X86 )
906 ATOMIC_CMPXCHG( fixed8, shr, kmp_int64, 64, >>, 8i, 7, KMP_ARCH_X86 )
907 ATOMIC_CMPXCHG( fixed8u, shr, kmp_uint64, 64, >>, 8i, 7, KMP_ARCH_X86 )
908 ATOMIC_CMPXCHG( fixed8, xor, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
909 ATOMIC_CMPXCHG( float4, div, kmp_real32, 32, /, 4r, 3, KMP_ARCH_X86 )
910 ATOMIC_CMPXCHG( float4, mul, kmp_real32, 32, *, 4r, 3, KMP_ARCH_X86 )
911 ATOMIC_CMPXCHG( float8, div, kmp_real64, 64, /, 8r, 7, KMP_ARCH_X86 )
912 ATOMIC_CMPXCHG( float8, mul, kmp_real64, 64, *, 8r, 7, KMP_ARCH_X86 )
923 #define ATOMIC_CRIT_L(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
924 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
925 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
926 OP_CRITICAL( = *lhs OP, LCK_ID ) \
929 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
933 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
934 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
935 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
936 OP_CMPXCHG(TYPE,BITS,OP) \
942 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
943 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
944 OP_GOMP_CRITICAL(= *lhs OP,GOMP_FLAG) \
945 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
946 OP_CMPXCHG(TYPE,BITS,OP) \
949 OP_CRITICAL(= *lhs OP,LCK_ID) \
954 ATOMIC_CMPX_L( fixed1, andl,
char, 8, &&, 1i, 0, KMP_ARCH_X86 )
955 ATOMIC_CMPX_L( fixed1, orl,
char, 8, ||, 1i, 0, KMP_ARCH_X86 )
956 ATOMIC_CMPX_L( fixed2, andl,
short, 16, &&, 2i, 1, KMP_ARCH_X86 )
957 ATOMIC_CMPX_L( fixed2, orl,
short, 16, ||, 2i, 1, KMP_ARCH_X86 )
958 ATOMIC_CMPX_L( fixed4, andl, kmp_int32, 32, &&, 4i, 3, 0 )
959 ATOMIC_CMPX_L( fixed4, orl, kmp_int32, 32, ||, 4i, 3, 0 )
960 ATOMIC_CMPX_L( fixed8, andl, kmp_int64, 64, &&, 8i, 7, KMP_ARCH_X86 )
961 ATOMIC_CMPX_L( fixed8, orl, kmp_int64, 64, ||, 8i, 7, KMP_ARCH_X86 )
974 #define MIN_MAX_CRITSECT(OP,LCK_ID) \
975 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
977 if ( *lhs OP rhs ) { \
980 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
983 #ifdef KMP_GOMP_COMPAT
984 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) \
985 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
987 MIN_MAX_CRITSECT( OP, 0 ); \
991 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG)
995 #define MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
997 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1000 old_value = temp_val; \
1001 while ( old_value OP rhs && \
1002 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1003 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1004 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
1008 old_value = temp_val; \
1014 #define MIN_MAX_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1015 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1016 if ( *lhs OP rhs ) { \
1017 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1018 MIN_MAX_CRITSECT(OP,LCK_ID) \
1022 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1026 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1027 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1028 if ( *lhs OP rhs ) { \
1029 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1030 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1037 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1038 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1039 if ( *lhs OP rhs ) { \
1040 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1041 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1042 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1045 MIN_MAX_CRITSECT(OP,LCK_ID) \
1051 MIN_MAX_COMPXCHG( fixed1, max,
char, 8, <, 1i, 0, KMP_ARCH_X86 )
1052 MIN_MAX_COMPXCHG( fixed1, min,
char, 8, >, 1i, 0, KMP_ARCH_X86 )
1053 MIN_MAX_COMPXCHG( fixed2, max,
short, 16, <, 2i, 1, KMP_ARCH_X86 )
1054 MIN_MAX_COMPXCHG( fixed2, min,
short, 16, >, 2i, 1, KMP_ARCH_X86 )
1055 MIN_MAX_COMPXCHG( fixed4, max, kmp_int32, 32, <, 4i, 3, 0 )
1056 MIN_MAX_COMPXCHG( fixed4, min, kmp_int32, 32, >, 4i, 3, 0 )
1057 MIN_MAX_COMPXCHG( fixed8, max, kmp_int64, 64, <, 8i, 7, KMP_ARCH_X86 )
1058 MIN_MAX_COMPXCHG( fixed8, min, kmp_int64, 64, >, 8i, 7, KMP_ARCH_X86 )
1059 MIN_MAX_COMPXCHG( float4, max, kmp_real32, 32, <, 4r, 3, KMP_ARCH_X86 )
1060 MIN_MAX_COMPXCHG( float4, min, kmp_real32, 32, >, 4r, 3, KMP_ARCH_X86 )
1061 MIN_MAX_COMPXCHG( float8, max, kmp_real64, 64, <, 8r, 7, KMP_ARCH_X86 )
1062 MIN_MAX_COMPXCHG( float8, min, kmp_real64, 64, >, 8r, 7, KMP_ARCH_X86 )
1064 MIN_MAX_CRITICAL( float16, max, QUAD_LEGACY, <, 16r, 1 )
1065 MIN_MAX_CRITICAL( float16, min, QUAD_LEGACY, >, 16r, 1 )
1066 #if ( KMP_ARCH_X86 )
1067 MIN_MAX_CRITICAL( float16, max_a16, Quad_a16_t, <, 16r, 1 )
1068 MIN_MAX_CRITICAL( float16, min_a16, Quad_a16_t, >, 16r, 1 )
1074 #define ATOMIC_CRIT_EQV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1075 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1076 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1077 OP_CRITICAL(^=~,LCK_ID) \
1081 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1084 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1085 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1086 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1087 OP_CMPXCHG(TYPE,BITS,OP) \
1093 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1094 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1095 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1096 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1097 OP_CMPXCHG(TYPE,BITS,OP) \
1100 OP_CRITICAL(^=~,LCK_ID) \
1105 ATOMIC_CMPXCHG( fixed1, neqv, kmp_int8, 8, ^, 1i, 0, KMP_ARCH_X86 )
1106 ATOMIC_CMPXCHG( fixed2, neqv, kmp_int16, 16, ^, 2i, 1, KMP_ARCH_X86 )
1107 ATOMIC_CMPXCHG( fixed4, neqv, kmp_int32, 32, ^, 4i, 3, KMP_ARCH_X86 )
1108 ATOMIC_CMPXCHG( fixed8, neqv, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
1109 ATOMIC_CMPX_EQV( fixed1, eqv, kmp_int8, 8, ^~, 1i, 0, KMP_ARCH_X86 )
1110 ATOMIC_CMPX_EQV( fixed2, eqv, kmp_int16, 16, ^~, 2i, 1, KMP_ARCH_X86 )
1111 ATOMIC_CMPX_EQV( fixed4, eqv, kmp_int32, 32, ^~, 4i, 3, KMP_ARCH_X86 )
1112 ATOMIC_CMPX_EQV( fixed8, eqv, kmp_int64, 64, ^~, 8i, 7, KMP_ARCH_X86 )
1120 #define ATOMIC_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1121 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1122 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1123 OP_CRITICAL(OP##=,LCK_ID) \
1128 ATOMIC_CRITICAL( float10, add,
long double, +, 10r, 1 )
1129 ATOMIC_CRITICAL( float10, sub,
long double, -, 10r, 1 )
1130 ATOMIC_CRITICAL( float10, mul,
long double, *, 10r, 1 )
1131 ATOMIC_CRITICAL( float10, div,
long double, /, 10r, 1 )
1134 ATOMIC_CRITICAL( float16, add, QUAD_LEGACY, +, 16r, 1 )
1135 ATOMIC_CRITICAL( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1136 ATOMIC_CRITICAL( float16, mul, QUAD_LEGACY, *, 16r, 1 )
1137 ATOMIC_CRITICAL( float16, div, QUAD_LEGACY, /, 16r, 1 )
1138 #if ( KMP_ARCH_X86 )
1139 ATOMIC_CRITICAL( float16, add_a16, Quad_a16_t, +, 16r, 1 )
1140 ATOMIC_CRITICAL( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1141 ATOMIC_CRITICAL( float16, mul_a16, Quad_a16_t, *, 16r, 1 )
1142 ATOMIC_CRITICAL( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1149 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, add, kmp_cmplx32, 64, +, 8c, 7, 1 )
1150 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7, 1 )
1151 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7, 1 )
1152 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, div, kmp_cmplx32, 64, /, 8c, 7, 1 )
1155 ATOMIC_CRITICAL( cmplx4, add, kmp_cmplx32, +, 8c, 1 )
1156 ATOMIC_CRITICAL( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1157 ATOMIC_CRITICAL( cmplx4, mul, kmp_cmplx32, *, 8c, 1 )
1158 ATOMIC_CRITICAL( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1159 #endif // USE_CMPXCHG_FIX
1161 ATOMIC_CRITICAL( cmplx8, add, kmp_cmplx64, +, 16c, 1 )
1162 ATOMIC_CRITICAL( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1163 ATOMIC_CRITICAL( cmplx8, mul, kmp_cmplx64, *, 16c, 1 )
1164 ATOMIC_CRITICAL( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1165 ATOMIC_CRITICAL( cmplx10, add, kmp_cmplx80, +, 20c, 1 )
1166 ATOMIC_CRITICAL( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1167 ATOMIC_CRITICAL( cmplx10, mul, kmp_cmplx80, *, 20c, 1 )
1168 ATOMIC_CRITICAL( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1170 ATOMIC_CRITICAL( cmplx16, add, CPLX128_LEG, +, 32c, 1 )
1171 ATOMIC_CRITICAL( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1172 ATOMIC_CRITICAL( cmplx16, mul, CPLX128_LEG, *, 32c, 1 )
1173 ATOMIC_CRITICAL( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1174 #if ( KMP_ARCH_X86 )
1175 ATOMIC_CRITICAL( cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c, 1 )
1176 ATOMIC_CRITICAL( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1177 ATOMIC_CRITICAL( cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c, 1 )
1178 ATOMIC_CRITICAL( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1186 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1194 #define OP_CRITICAL_REV(OP,LCK_ID) \
1195 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1197 (*lhs) = (rhs) OP (*lhs); \
1199 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1201 #ifdef KMP_GOMP_COMPAT
1202 #define OP_GOMP_CRITICAL_REV(OP,FLAG) \
1203 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1205 OP_CRITICAL_REV( OP, 0 ); \
1209 #define OP_GOMP_CRITICAL_REV(OP,FLAG)
1217 #define ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1218 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
1220 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1221 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid ));
1230 #define OP_CMPXCHG_REV(TYPE,BITS,OP) \
1232 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1233 TYPE old_value, new_value; \
1235 old_value = temp_val; \
1236 new_value = rhs OP old_value; \
1237 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1238 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1239 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1244 old_value = temp_val; \
1245 new_value = rhs OP old_value; \
1250 #define ATOMIC_CMPXCHG_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,GOMP_FLAG) \
1251 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1252 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1253 OP_CMPXCHG_REV(TYPE,BITS,OP) \
1270 ATOMIC_CMPXCHG_REV( fixed1, div, kmp_int8, 8, /, 1i, KMP_ARCH_X86 )
1271 ATOMIC_CMPXCHG_REV( fixed1u, div, kmp_uint8, 8, /, 1i, KMP_ARCH_X86 )
1272 ATOMIC_CMPXCHG_REV( fixed1, shl, kmp_int8, 8, <<, 1i, KMP_ARCH_X86 )
1273 ATOMIC_CMPXCHG_REV( fixed1, shr, kmp_int8, 8, >>, 1i, KMP_ARCH_X86 )
1274 ATOMIC_CMPXCHG_REV( fixed1u, shr, kmp_uint8, 8, >>, 1i, KMP_ARCH_X86 )
1275 ATOMIC_CMPXCHG_REV( fixed1, sub, kmp_int8, 8, -, 1i, KMP_ARCH_X86 )
1277 ATOMIC_CMPXCHG_REV( fixed2, div, kmp_int16, 16, /, 2i, KMP_ARCH_X86 )
1278 ATOMIC_CMPXCHG_REV( fixed2u, div, kmp_uint16, 16, /, 2i, KMP_ARCH_X86 )
1279 ATOMIC_CMPXCHG_REV( fixed2, shl, kmp_int16, 16, <<, 2i, KMP_ARCH_X86 )
1280 ATOMIC_CMPXCHG_REV( fixed2, shr, kmp_int16, 16, >>, 2i, KMP_ARCH_X86 )
1281 ATOMIC_CMPXCHG_REV( fixed2u, shr, kmp_uint16, 16, >>, 2i, KMP_ARCH_X86 )
1282 ATOMIC_CMPXCHG_REV( fixed2, sub, kmp_int16, 16, -, 2i, KMP_ARCH_X86 )
1284 ATOMIC_CMPXCHG_REV( fixed4, div, kmp_int32, 32, /, 4i, KMP_ARCH_X86 )
1285 ATOMIC_CMPXCHG_REV( fixed4u, div, kmp_uint32, 32, /, 4i, KMP_ARCH_X86 )
1286 ATOMIC_CMPXCHG_REV( fixed4, shl, kmp_int32, 32, <<, 4i, KMP_ARCH_X86 )
1287 ATOMIC_CMPXCHG_REV( fixed4, shr, kmp_int32, 32, >>, 4i, KMP_ARCH_X86 )
1288 ATOMIC_CMPXCHG_REV( fixed4u, shr, kmp_uint32, 32, >>, 4i, KMP_ARCH_X86 )
1289 ATOMIC_CMPXCHG_REV( fixed4, sub, kmp_int32, 32, -, 4i, KMP_ARCH_X86 )
1291 ATOMIC_CMPXCHG_REV( fixed8, div, kmp_int64, 64, /, 8i, KMP_ARCH_X86 )
1292 ATOMIC_CMPXCHG_REV( fixed8u, div, kmp_uint64, 64, /, 8i, KMP_ARCH_X86 )
1293 ATOMIC_CMPXCHG_REV( fixed8, shl, kmp_int64, 64, <<, 8i, KMP_ARCH_X86 )
1294 ATOMIC_CMPXCHG_REV( fixed8, shr, kmp_int64, 64, >>, 8i, KMP_ARCH_X86 )
1295 ATOMIC_CMPXCHG_REV( fixed8u, shr, kmp_uint64, 64, >>, 8i, KMP_ARCH_X86 )
1296 ATOMIC_CMPXCHG_REV( fixed8, sub, kmp_int64, 64, -, 8i, KMP_ARCH_X86 )
1298 ATOMIC_CMPXCHG_REV( float4, div, kmp_real32, 32, /, 4r, KMP_ARCH_X86 )
1299 ATOMIC_CMPXCHG_REV( float4, sub, kmp_real32, 32, -, 4r, KMP_ARCH_X86 )
1301 ATOMIC_CMPXCHG_REV( float8, div, kmp_real64, 64, /, 8r, KMP_ARCH_X86 )
1302 ATOMIC_CMPXCHG_REV( float8, sub, kmp_real64, 64, -, 8r, KMP_ARCH_X86 )
1310 #define ATOMIC_CRITICAL_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1311 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1312 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1313 OP_CRITICAL_REV(OP,LCK_ID) \
1318 ATOMIC_CRITICAL_REV( float10, sub,
long double, -, 10r, 1 )
1319 ATOMIC_CRITICAL_REV( float10, div,
long double, /, 10r, 1 )
1322 ATOMIC_CRITICAL_REV( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1323 ATOMIC_CRITICAL_REV( float16, div, QUAD_LEGACY, /, 16r, 1 )
1324 #if ( KMP_ARCH_X86 )
1325 ATOMIC_CRITICAL_REV( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1326 ATOMIC_CRITICAL_REV( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1331 ATOMIC_CRITICAL_REV( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1332 ATOMIC_CRITICAL_REV( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1333 ATOMIC_CRITICAL_REV( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1334 ATOMIC_CRITICAL_REV( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1335 ATOMIC_CRITICAL_REV( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1336 ATOMIC_CRITICAL_REV( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1338 ATOMIC_CRITICAL_REV( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1339 ATOMIC_CRITICAL_REV( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1340 #if ( KMP_ARCH_X86 )
1341 ATOMIC_CRITICAL_REV( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1342 ATOMIC_CRITICAL_REV( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1347 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
1350 #endif //OMP_40_ENABLED
1364 #define ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1365 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs ) \
1367 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1368 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid ));
1371 #define ATOMIC_CRITICAL_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
1372 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1373 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1374 OP_CRITICAL(OP##=,LCK_ID) \
1378 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1381 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1382 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1383 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1384 OP_CMPXCHG(TYPE,BITS,OP) \
1390 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1391 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1392 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1393 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1394 OP_CMPXCHG(TYPE,BITS,OP) \
1397 OP_CRITICAL(OP##=,LCK_ID) \
1403 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1404 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1405 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1406 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1407 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3, 0 )
1408 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3, 0 )
1409 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1410 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1411 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1412 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1413 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1414 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1418 ATOMIC_CMPXCHG_MIX( fixed1,
char, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1419 ATOMIC_CMPXCHG_MIX( fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1420 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1421 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1422 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1424 ATOMIC_CMPXCHG_MIX( fixed2,
short, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1425 ATOMIC_CMPXCHG_MIX( fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1426 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1427 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1428 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1430 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3, 0 )
1431 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3, 0 )
1432 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3, 0 )
1433 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1434 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1436 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1437 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1438 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1439 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1440 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1442 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1443 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1444 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1445 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1447 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1448 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1449 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1450 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1452 ATOMIC_CRITICAL_FP( float10,
long double, add, +, fp, _Quad, 10r, 1 )
1453 ATOMIC_CRITICAL_FP( float10,
long double, sub, -, fp, _Quad, 10r, 1 )
1454 ATOMIC_CRITICAL_FP( float10,
long double, mul, *, fp, _Quad, 10r, 1 )
1455 ATOMIC_CRITICAL_FP( float10,
long double, div, /, fp, _Quad, 10r, 1 )
1458 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1463 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1464 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1465 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1466 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
1470 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1471 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1472 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1473 OP_CMPXCHG(TYPE,BITS,OP) \
1475 #endif // USE_CMPXCHG_FIX
1479 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1480 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1481 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1482 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1483 OP_CMPXCHG(TYPE,BITS,OP) \
1486 OP_CRITICAL(OP##=,LCK_ID) \
1491 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1492 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1493 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1494 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1497 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1509 #define ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1510 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * loc ) \
1512 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1513 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1525 #define OP_CMPXCHG_READ(TYPE,BITS,OP) \
1527 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1530 kmp_int##BITS i_val; \
1532 union f_i_union old_value; \
1534 old_value.f_val = temp_val; \
1535 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( (kmp_int##BITS *) loc, \
1536 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val, \
1537 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val ); \
1538 new_value = old_value.f_val; \
1548 #define OP_CRITICAL_READ(OP,LCK_ID) \
1549 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1551 new_value = (*loc); \
1553 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1556 #ifdef KMP_GOMP_COMPAT
1557 #define OP_GOMP_CRITICAL_READ(OP,FLAG) \
1558 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1560 OP_CRITICAL_READ( OP, 0 ); \
1564 #define OP_GOMP_CRITICAL_READ(OP,FLAG)
1568 #define ATOMIC_FIXED_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1569 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1571 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1572 new_value = KMP_TEST_THEN_ADD##BITS( loc, OP 0 ); \
1576 #define ATOMIC_CMPXCHG_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1577 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1579 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1580 OP_CMPXCHG_READ(TYPE,BITS,OP) \
1587 #define ATOMIC_CRITICAL_READ(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1588 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1590 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1591 OP_CRITICAL_READ(OP,LCK_ID) \
1599 #if ( KMP_OS_WINDOWS )
1601 #define OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1602 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1606 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1608 #ifdef KMP_GOMP_COMPAT
1609 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) \
1610 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1612 OP_CRITICAL_READ_WRK( OP, 0 ); \
1615 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG)
1618 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1619 void __kmpc_atomic_##TYPE_ID##_##OP_ID( TYPE * out, ident_t *id_ref, int gtid, TYPE * loc ) \
1621 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1622 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1625 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1626 ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1627 OP_GOMP_CRITICAL_READ_WRK(OP##=,GOMP_FLAG) \
1628 OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1631 #endif // KMP_OS_WINDOWS
1635 ATOMIC_FIXED_READ( fixed4, rd, kmp_int32, 32, +, 0 )
1636 ATOMIC_FIXED_READ( fixed8, rd, kmp_int64, 64, +, KMP_ARCH_X86 )
1637 ATOMIC_CMPXCHG_READ( float4, rd, kmp_real32, 32, +, KMP_ARCH_X86 )
1638 ATOMIC_CMPXCHG_READ( float8, rd, kmp_real64, 64, +, KMP_ARCH_X86 )
1641 ATOMIC_CMPXCHG_READ( fixed1, rd, kmp_int8, 8, +, KMP_ARCH_X86 )
1642 ATOMIC_CMPXCHG_READ( fixed2, rd, kmp_int16, 16, +, KMP_ARCH_X86 )
1644 ATOMIC_CRITICAL_READ( float10, rd,
long double, +, 10r, 1 )
1646 ATOMIC_CRITICAL_READ( float16, rd, QUAD_LEGACY, +, 16r, 1 )
1647 #endif // KMP_HAVE_QUAD
1650 #if ( KMP_OS_WINDOWS )
1651 ATOMIC_CRITICAL_READ_WRK( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1653 ATOMIC_CRITICAL_READ( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1655 ATOMIC_CRITICAL_READ( cmplx8, rd, kmp_cmplx64, +, 16c, 1 )
1656 ATOMIC_CRITICAL_READ( cmplx10, rd, kmp_cmplx80, +, 20c, 1 )
1658 ATOMIC_CRITICAL_READ( cmplx16, rd, CPLX128_LEG, +, 32c, 1 )
1659 #if ( KMP_ARCH_X86 )
1660 ATOMIC_CRITICAL_READ( float16, a16_rd, Quad_a16_t, +, 16r, 1 )
1661 ATOMIC_CRITICAL_READ( cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c, 1 )
1670 #define ATOMIC_XCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1671 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1672 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1673 KMP_XCHG_FIXED##BITS( lhs, rhs ); \
1676 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1677 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1678 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1679 KMP_XCHG_REAL##BITS( lhs, rhs ); \
1690 #define OP_CMPXCHG_WR(TYPE,BITS,OP) \
1692 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1693 TYPE old_value, new_value; \
1695 old_value = temp_val; \
1697 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1698 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1699 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1704 old_value = temp_val; \
1710 #define ATOMIC_CMPXCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1711 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1712 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1713 OP_CMPXCHG_WR(TYPE,BITS,OP) \
1721 #define ATOMIC_CRITICAL_WR(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1722 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1723 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1724 OP_CRITICAL(OP,LCK_ID) \
1728 ATOMIC_XCHG_WR( fixed1, wr, kmp_int8, 8, =, KMP_ARCH_X86 )
1729 ATOMIC_XCHG_WR( fixed2, wr, kmp_int16, 16, =, KMP_ARCH_X86 )
1730 ATOMIC_XCHG_WR( fixed4, wr, kmp_int32, 32, =, KMP_ARCH_X86 )
1731 #if ( KMP_ARCH_X86 )
1732 ATOMIC_CMPXCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1734 ATOMIC_XCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1737 ATOMIC_XCHG_FLOAT_WR( float4, wr, kmp_real32, 32, =, KMP_ARCH_X86 )
1738 #if ( KMP_ARCH_X86 )
1739 ATOMIC_CMPXCHG_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1741 ATOMIC_XCHG_FLOAT_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1744 ATOMIC_CRITICAL_WR( float10, wr,
long double, =, 10r, 1 )
1746 ATOMIC_CRITICAL_WR( float16, wr, QUAD_LEGACY, =, 16r, 1 )
1748 ATOMIC_CRITICAL_WR( cmplx4, wr, kmp_cmplx32, =, 8c, 1 )
1749 ATOMIC_CRITICAL_WR( cmplx8, wr, kmp_cmplx64, =, 16c, 1 )
1750 ATOMIC_CRITICAL_WR( cmplx10, wr, kmp_cmplx80, =, 20c, 1 )
1752 ATOMIC_CRITICAL_WR( cmplx16, wr, CPLX128_LEG, =, 32c, 1 )
1753 #if ( KMP_ARCH_X86 )
1754 ATOMIC_CRITICAL_WR( float16, a16_wr, Quad_a16_t, =, 16r, 1 )
1755 ATOMIC_CRITICAL_WR( cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c, 1 )
1768 #define ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,RET_TYPE) \
1769 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag ) \
1771 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1772 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1780 #define OP_CRITICAL_CPT(OP,LCK_ID) \
1781 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1785 new_value = (*lhs); \
1787 new_value = (*lhs); \
1791 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1795 #ifdef KMP_GOMP_COMPAT
1796 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) \
1797 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1799 OP_CRITICAL_CPT( OP##=, 0 ); \
1802 #define OP_GOMP_CRITICAL_CPT(OP,FLAG)
1812 #define OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1814 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1815 TYPE old_value, new_value; \
1817 old_value = temp_val; \
1818 new_value = old_value OP rhs; \
1819 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1820 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1821 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1826 old_value = temp_val; \
1827 new_value = old_value OP rhs; \
1836 #define ATOMIC_CMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1837 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1839 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1840 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1844 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1845 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1846 TYPE old_value, new_value; \
1847 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1849 old_value = KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
1851 return old_value OP rhs; \
1857 ATOMIC_FIXED_ADD_CPT( fixed4, add_cpt, kmp_int32, 32, +, 0 )
1858 ATOMIC_FIXED_ADD_CPT( fixed4, sub_cpt, kmp_int32, 32, -, 0 )
1859 ATOMIC_FIXED_ADD_CPT( fixed8, add_cpt, kmp_int64, 64, +, KMP_ARCH_X86 )
1860 ATOMIC_FIXED_ADD_CPT( fixed8, sub_cpt, kmp_int64, 64, -, KMP_ARCH_X86 )
1862 ATOMIC_CMPXCHG_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1863 ATOMIC_CMPXCHG_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1864 ATOMIC_CMPXCHG_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1865 ATOMIC_CMPXCHG_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1879 ATOMIC_CMPXCHG_CPT( fixed1, add_cpt, kmp_int8, 8, +, KMP_ARCH_X86 )
1880 ATOMIC_CMPXCHG_CPT( fixed1, andb_cpt, kmp_int8, 8, &, 0 )
1881 ATOMIC_CMPXCHG_CPT( fixed1, div_cpt, kmp_int8, 8, /, KMP_ARCH_X86 )
1882 ATOMIC_CMPXCHG_CPT( fixed1u, div_cpt, kmp_uint8, 8, /, KMP_ARCH_X86 )
1883 ATOMIC_CMPXCHG_CPT( fixed1, mul_cpt, kmp_int8, 8, *, KMP_ARCH_X86 )
1884 ATOMIC_CMPXCHG_CPT( fixed1, orb_cpt, kmp_int8, 8, |, 0 )
1885 ATOMIC_CMPXCHG_CPT( fixed1, shl_cpt, kmp_int8, 8, <<, KMP_ARCH_X86 )
1886 ATOMIC_CMPXCHG_CPT( fixed1, shr_cpt, kmp_int8, 8, >>, KMP_ARCH_X86 )
1887 ATOMIC_CMPXCHG_CPT( fixed1u, shr_cpt, kmp_uint8, 8, >>, KMP_ARCH_X86 )
1888 ATOMIC_CMPXCHG_CPT( fixed1, sub_cpt, kmp_int8, 8, -, KMP_ARCH_X86 )
1889 ATOMIC_CMPXCHG_CPT( fixed1, xor_cpt, kmp_int8, 8, ^, 0 )
1890 ATOMIC_CMPXCHG_CPT( fixed2, add_cpt, kmp_int16, 16, +, KMP_ARCH_X86 )
1891 ATOMIC_CMPXCHG_CPT( fixed2, andb_cpt, kmp_int16, 16, &, 0 )
1892 ATOMIC_CMPXCHG_CPT( fixed2, div_cpt, kmp_int16, 16, /, KMP_ARCH_X86 )
1893 ATOMIC_CMPXCHG_CPT( fixed2u, div_cpt, kmp_uint16, 16, /, KMP_ARCH_X86 )
1894 ATOMIC_CMPXCHG_CPT( fixed2, mul_cpt, kmp_int16, 16, *, KMP_ARCH_X86 )
1895 ATOMIC_CMPXCHG_CPT( fixed2, orb_cpt, kmp_int16, 16, |, 0 )
1896 ATOMIC_CMPXCHG_CPT( fixed2, shl_cpt, kmp_int16, 16, <<, KMP_ARCH_X86 )
1897 ATOMIC_CMPXCHG_CPT( fixed2, shr_cpt, kmp_int16, 16, >>, KMP_ARCH_X86 )
1898 ATOMIC_CMPXCHG_CPT( fixed2u, shr_cpt, kmp_uint16, 16, >>, KMP_ARCH_X86 )
1899 ATOMIC_CMPXCHG_CPT( fixed2, sub_cpt, kmp_int16, 16, -, KMP_ARCH_X86 )
1900 ATOMIC_CMPXCHG_CPT( fixed2, xor_cpt, kmp_int16, 16, ^, 0 )
1901 ATOMIC_CMPXCHG_CPT( fixed4, andb_cpt, kmp_int32, 32, &, 0 )
1902 ATOMIC_CMPXCHG_CPT( fixed4, div_cpt, kmp_int32, 32, /, KMP_ARCH_X86 )
1903 ATOMIC_CMPXCHG_CPT( fixed4u, div_cpt, kmp_uint32, 32, /, KMP_ARCH_X86 )
1904 ATOMIC_CMPXCHG_CPT( fixed4, mul_cpt, kmp_int32, 32, *, KMP_ARCH_X86 )
1905 ATOMIC_CMPXCHG_CPT( fixed4, orb_cpt, kmp_int32, 32, |, 0 )
1906 ATOMIC_CMPXCHG_CPT( fixed4, shl_cpt, kmp_int32, 32, <<, KMP_ARCH_X86 )
1907 ATOMIC_CMPXCHG_CPT( fixed4, shr_cpt, kmp_int32, 32, >>, KMP_ARCH_X86 )
1908 ATOMIC_CMPXCHG_CPT( fixed4u, shr_cpt, kmp_uint32, 32, >>, KMP_ARCH_X86 )
1909 ATOMIC_CMPXCHG_CPT( fixed4, xor_cpt, kmp_int32, 32, ^, 0 )
1910 ATOMIC_CMPXCHG_CPT( fixed8, andb_cpt, kmp_int64, 64, &, KMP_ARCH_X86 )
1911 ATOMIC_CMPXCHG_CPT( fixed8, div_cpt, kmp_int64, 64, /, KMP_ARCH_X86 )
1912 ATOMIC_CMPXCHG_CPT( fixed8u, div_cpt, kmp_uint64, 64, /, KMP_ARCH_X86 )
1913 ATOMIC_CMPXCHG_CPT( fixed8, mul_cpt, kmp_int64, 64, *, KMP_ARCH_X86 )
1914 ATOMIC_CMPXCHG_CPT( fixed8, orb_cpt, kmp_int64, 64, |, KMP_ARCH_X86 )
1915 ATOMIC_CMPXCHG_CPT( fixed8, shl_cpt, kmp_int64, 64, <<, KMP_ARCH_X86 )
1916 ATOMIC_CMPXCHG_CPT( fixed8, shr_cpt, kmp_int64, 64, >>, KMP_ARCH_X86 )
1917 ATOMIC_CMPXCHG_CPT( fixed8u, shr_cpt, kmp_uint64, 64, >>, KMP_ARCH_X86 )
1918 ATOMIC_CMPXCHG_CPT( fixed8, xor_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
1919 ATOMIC_CMPXCHG_CPT( float4, div_cpt, kmp_real32, 32, /, KMP_ARCH_X86 )
1920 ATOMIC_CMPXCHG_CPT( float4, mul_cpt, kmp_real32, 32, *, KMP_ARCH_X86 )
1921 ATOMIC_CMPXCHG_CPT( float8, div_cpt, kmp_real64, 64, /, KMP_ARCH_X86 )
1922 ATOMIC_CMPXCHG_CPT( float8, mul_cpt, kmp_real64, 64, *, KMP_ARCH_X86 )
1935 #define OP_CRITICAL_L_CPT(OP,LCK_ID) \
1936 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1941 new_value = (*lhs); \
1943 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1946 #ifdef KMP_GOMP_COMPAT
1947 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) \
1948 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1950 OP_CRITICAL_L_CPT( OP, 0 ); \
1954 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG)
1959 #define ATOMIC_CMPX_L_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1960 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1962 OP_GOMP_CRITICAL_L_CPT( = *lhs OP, GOMP_FLAG ) \
1963 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1966 ATOMIC_CMPX_L_CPT( fixed1, andl_cpt,
char, 8, &&, KMP_ARCH_X86 )
1967 ATOMIC_CMPX_L_CPT( fixed1, orl_cpt,
char, 8, ||, KMP_ARCH_X86 )
1968 ATOMIC_CMPX_L_CPT( fixed2, andl_cpt,
short, 16, &&, KMP_ARCH_X86 )
1969 ATOMIC_CMPX_L_CPT( fixed2, orl_cpt,
short, 16, ||, KMP_ARCH_X86 )
1970 ATOMIC_CMPX_L_CPT( fixed4, andl_cpt, kmp_int32, 32, &&, 0 )
1971 ATOMIC_CMPX_L_CPT( fixed4, orl_cpt, kmp_int32, 32, ||, 0 )
1972 ATOMIC_CMPX_L_CPT( fixed8, andl_cpt, kmp_int64, 64, &&, KMP_ARCH_X86 )
1973 ATOMIC_CMPX_L_CPT( fixed8, orl_cpt, kmp_int64, 64, ||, KMP_ARCH_X86 )
1986 #define MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
1987 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1989 if ( *lhs OP rhs ) { \
1995 new_value = old_value; \
1997 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2001 #ifdef KMP_GOMP_COMPAT
2002 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) \
2003 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
2005 MIN_MAX_CRITSECT_CPT( OP, 0 ); \
2008 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG)
2012 #define MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2014 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2017 old_value = temp_val; \
2018 while ( old_value OP rhs && \
2019 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2020 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2021 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
2025 old_value = temp_val; \
2035 #define MIN_MAX_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2036 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2037 TYPE new_value, old_value; \
2038 if ( *lhs OP rhs ) { \
2039 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2040 MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
2045 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2046 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2047 TYPE new_value, old_value; \
2048 if ( *lhs OP rhs ) { \
2049 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2050 MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2056 MIN_MAX_COMPXCHG_CPT( fixed1, max_cpt,
char, 8, <, KMP_ARCH_X86 )
2057 MIN_MAX_COMPXCHG_CPT( fixed1, min_cpt,
char, 8, >, KMP_ARCH_X86 )
2058 MIN_MAX_COMPXCHG_CPT( fixed2, max_cpt,
short, 16, <, KMP_ARCH_X86 )
2059 MIN_MAX_COMPXCHG_CPT( fixed2, min_cpt,
short, 16, >, KMP_ARCH_X86 )
2060 MIN_MAX_COMPXCHG_CPT( fixed4, max_cpt, kmp_int32, 32, <, 0 )
2061 MIN_MAX_COMPXCHG_CPT( fixed4, min_cpt, kmp_int32, 32, >, 0 )
2062 MIN_MAX_COMPXCHG_CPT( fixed8, max_cpt, kmp_int64, 64, <, KMP_ARCH_X86 )
2063 MIN_MAX_COMPXCHG_CPT( fixed8, min_cpt, kmp_int64, 64, >, KMP_ARCH_X86 )
2064 MIN_MAX_COMPXCHG_CPT( float4, max_cpt, kmp_real32, 32, <, KMP_ARCH_X86 )
2065 MIN_MAX_COMPXCHG_CPT( float4, min_cpt, kmp_real32, 32, >, KMP_ARCH_X86 )
2066 MIN_MAX_COMPXCHG_CPT( float8, max_cpt, kmp_real64, 64, <, KMP_ARCH_X86 )
2067 MIN_MAX_COMPXCHG_CPT( float8, min_cpt, kmp_real64, 64, >, KMP_ARCH_X86 )
2069 MIN_MAX_CRITICAL_CPT( float16, max_cpt, QUAD_LEGACY, <, 16r, 1 )
2070 MIN_MAX_CRITICAL_CPT( float16, min_cpt, QUAD_LEGACY, >, 16r, 1 )
2071 #if ( KMP_ARCH_X86 )
2072 MIN_MAX_CRITICAL_CPT( float16, max_a16_cpt, Quad_a16_t, <, 16r, 1 )
2073 MIN_MAX_CRITICAL_CPT( float16, min_a16_cpt, Quad_a16_t, >, 16r, 1 )
2078 #ifdef KMP_GOMP_COMPAT
2079 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) \
2080 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2082 OP_CRITICAL_CPT( OP, 0 ); \
2085 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG)
2088 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2089 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2091 OP_GOMP_CRITICAL_EQV_CPT(^=~,GOMP_FLAG) \
2092 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
2097 ATOMIC_CMPXCHG_CPT( fixed1, neqv_cpt, kmp_int8, 8, ^, KMP_ARCH_X86 )
2098 ATOMIC_CMPXCHG_CPT( fixed2, neqv_cpt, kmp_int16, 16, ^, KMP_ARCH_X86 )
2099 ATOMIC_CMPXCHG_CPT( fixed4, neqv_cpt, kmp_int32, 32, ^, KMP_ARCH_X86 )
2100 ATOMIC_CMPXCHG_CPT( fixed8, neqv_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
2101 ATOMIC_CMPX_EQV_CPT( fixed1, eqv_cpt, kmp_int8, 8, ^~, KMP_ARCH_X86 )
2102 ATOMIC_CMPX_EQV_CPT( fixed2, eqv_cpt, kmp_int16, 16, ^~, KMP_ARCH_X86 )
2103 ATOMIC_CMPX_EQV_CPT( fixed4, eqv_cpt, kmp_int32, 32, ^~, KMP_ARCH_X86 )
2104 ATOMIC_CMPX_EQV_CPT( fixed8, eqv_cpt, kmp_int64, 64, ^~, KMP_ARCH_X86 )
2111 #define ATOMIC_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2112 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2114 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
2115 OP_CRITICAL_CPT(OP##=,LCK_ID) \
2122 #define OP_CRITICAL_CPT_WRK(OP,LCK_ID) \
2123 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2133 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2137 #ifdef KMP_GOMP_COMPAT
2138 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) \
2139 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2141 OP_CRITICAL_CPT_WRK( OP##=, 0 ); \
2144 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG)
2148 #define ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2149 void __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out, int flag ) \
2151 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2152 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
2155 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2156 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2157 OP_GOMP_CRITICAL_CPT_WRK(OP,GOMP_FLAG) \
2158 OP_CRITICAL_CPT_WRK(OP##=,LCK_ID) \
2164 ATOMIC_CRITICAL_CPT( float10, add_cpt,
long double, +, 10r, 1 )
2165 ATOMIC_CRITICAL_CPT( float10, sub_cpt,
long double, -, 10r, 1 )
2166 ATOMIC_CRITICAL_CPT( float10, mul_cpt,
long double, *, 10r, 1 )
2167 ATOMIC_CRITICAL_CPT( float10, div_cpt,
long double, /, 10r, 1 )
2170 ATOMIC_CRITICAL_CPT( float16, add_cpt, QUAD_LEGACY, +, 16r, 1 )
2171 ATOMIC_CRITICAL_CPT( float16, sub_cpt, QUAD_LEGACY, -, 16r, 1 )
2172 ATOMIC_CRITICAL_CPT( float16, mul_cpt, QUAD_LEGACY, *, 16r, 1 )
2173 ATOMIC_CRITICAL_CPT( float16, div_cpt, QUAD_LEGACY, /, 16r, 1 )
2174 #if ( KMP_ARCH_X86 )
2175 ATOMIC_CRITICAL_CPT( float16, add_a16_cpt, Quad_a16_t, +, 16r, 1 )
2176 ATOMIC_CRITICAL_CPT( float16, sub_a16_cpt, Quad_a16_t, -, 16r, 1 )
2177 ATOMIC_CRITICAL_CPT( float16, mul_a16_cpt, Quad_a16_t, *, 16r, 1 )
2178 ATOMIC_CRITICAL_CPT( float16, div_a16_cpt, Quad_a16_t, /, 16r, 1 )
2185 ATOMIC_CRITICAL_CPT_WRK( cmplx4, add_cpt, kmp_cmplx32, +, 8c, 1 )
2186 ATOMIC_CRITICAL_CPT_WRK( cmplx4, sub_cpt, kmp_cmplx32, -, 8c, 1 )
2187 ATOMIC_CRITICAL_CPT_WRK( cmplx4, mul_cpt, kmp_cmplx32, *, 8c, 1 )
2188 ATOMIC_CRITICAL_CPT_WRK( cmplx4, div_cpt, kmp_cmplx32, /, 8c, 1 )
2190 ATOMIC_CRITICAL_CPT( cmplx8, add_cpt, kmp_cmplx64, +, 16c, 1 )
2191 ATOMIC_CRITICAL_CPT( cmplx8, sub_cpt, kmp_cmplx64, -, 16c, 1 )
2192 ATOMIC_CRITICAL_CPT( cmplx8, mul_cpt, kmp_cmplx64, *, 16c, 1 )
2193 ATOMIC_CRITICAL_CPT( cmplx8, div_cpt, kmp_cmplx64, /, 16c, 1 )
2194 ATOMIC_CRITICAL_CPT( cmplx10, add_cpt, kmp_cmplx80, +, 20c, 1 )
2195 ATOMIC_CRITICAL_CPT( cmplx10, sub_cpt, kmp_cmplx80, -, 20c, 1 )
2196 ATOMIC_CRITICAL_CPT( cmplx10, mul_cpt, kmp_cmplx80, *, 20c, 1 )
2197 ATOMIC_CRITICAL_CPT( cmplx10, div_cpt, kmp_cmplx80, /, 20c, 1 )
2199 ATOMIC_CRITICAL_CPT( cmplx16, add_cpt, CPLX128_LEG, +, 32c, 1 )
2200 ATOMIC_CRITICAL_CPT( cmplx16, sub_cpt, CPLX128_LEG, -, 32c, 1 )
2201 ATOMIC_CRITICAL_CPT( cmplx16, mul_cpt, CPLX128_LEG, *, 32c, 1 )
2202 ATOMIC_CRITICAL_CPT( cmplx16, div_cpt, CPLX128_LEG, /, 32c, 1 )
2203 #if ( KMP_ARCH_X86 )
2204 ATOMIC_CRITICAL_CPT( cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c, 1 )
2205 ATOMIC_CRITICAL_CPT( cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c, 1 )
2206 ATOMIC_CRITICAL_CPT( cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c, 1 )
2207 ATOMIC_CRITICAL_CPT( cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c, 1 )
2222 #define OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2223 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2227 (*lhs) = (rhs) OP (*lhs); \
2228 new_value = (*lhs); \
2230 new_value = (*lhs);\
2231 (*lhs) = (rhs) OP (*lhs); \
2233 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2237 #ifdef KMP_GOMP_COMPAT
2238 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) \
2239 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2241 OP_CRITICAL_CPT_REV( OP, 0 ); \
2244 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG)
2254 #define OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2256 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2257 TYPE old_value, new_value; \
2259 old_value = temp_val; \
2260 new_value = rhs OP old_value; \
2261 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2262 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2263 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2268 old_value = temp_val; \
2269 new_value = rhs OP old_value; \
2278 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2279 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2281 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2282 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2283 OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2287 ATOMIC_CMPXCHG_CPT_REV( fixed1, div_cpt_rev, kmp_int8, 8, /, KMP_ARCH_X86 )
2288 ATOMIC_CMPXCHG_CPT_REV( fixed1u, div_cpt_rev, kmp_uint8, 8, /, KMP_ARCH_X86 )
2289 ATOMIC_CMPXCHG_CPT_REV( fixed1, shl_cpt_rev, kmp_int8, 8, <<, KMP_ARCH_X86 )
2290 ATOMIC_CMPXCHG_CPT_REV( fixed1, shr_cpt_rev, kmp_int8, 8, >>, KMP_ARCH_X86 )
2291 ATOMIC_CMPXCHG_CPT_REV( fixed1u, shr_cpt_rev, kmp_uint8, 8, >>, KMP_ARCH_X86 )
2292 ATOMIC_CMPXCHG_CPT_REV( fixed1, sub_cpt_rev, kmp_int8, 8, -, KMP_ARCH_X86 )
2293 ATOMIC_CMPXCHG_CPT_REV( fixed2, div_cpt_rev, kmp_int16, 16, /, KMP_ARCH_X86 )
2294 ATOMIC_CMPXCHG_CPT_REV( fixed2u, div_cpt_rev, kmp_uint16, 16, /, KMP_ARCH_X86 )
2295 ATOMIC_CMPXCHG_CPT_REV( fixed2, shl_cpt_rev, kmp_int16, 16, <<, KMP_ARCH_X86 )
2296 ATOMIC_CMPXCHG_CPT_REV( fixed2, shr_cpt_rev, kmp_int16, 16, >>, KMP_ARCH_X86 )
2297 ATOMIC_CMPXCHG_CPT_REV( fixed2u, shr_cpt_rev, kmp_uint16, 16, >>, KMP_ARCH_X86 )
2298 ATOMIC_CMPXCHG_CPT_REV( fixed2, sub_cpt_rev, kmp_int16, 16, -, KMP_ARCH_X86 )
2299 ATOMIC_CMPXCHG_CPT_REV( fixed4, div_cpt_rev, kmp_int32, 32, /, KMP_ARCH_X86 )
2300 ATOMIC_CMPXCHG_CPT_REV( fixed4u, div_cpt_rev, kmp_uint32, 32, /, KMP_ARCH_X86 )
2301 ATOMIC_CMPXCHG_CPT_REV( fixed4, shl_cpt_rev, kmp_int32, 32, <<, KMP_ARCH_X86 )
2302 ATOMIC_CMPXCHG_CPT_REV( fixed4, shr_cpt_rev, kmp_int32, 32, >>, KMP_ARCH_X86 )
2303 ATOMIC_CMPXCHG_CPT_REV( fixed4u, shr_cpt_rev, kmp_uint32, 32, >>, KMP_ARCH_X86 )
2304 ATOMIC_CMPXCHG_CPT_REV( fixed4, sub_cpt_rev, kmp_int32, 32, -, KMP_ARCH_X86 )
2305 ATOMIC_CMPXCHG_CPT_REV( fixed8, div_cpt_rev, kmp_int64, 64, /, KMP_ARCH_X86 )
2306 ATOMIC_CMPXCHG_CPT_REV( fixed8u, div_cpt_rev, kmp_uint64, 64, /, KMP_ARCH_X86 )
2307 ATOMIC_CMPXCHG_CPT_REV( fixed8, shl_cpt_rev, kmp_int64, 64, <<, KMP_ARCH_X86 )
2308 ATOMIC_CMPXCHG_CPT_REV( fixed8, shr_cpt_rev, kmp_int64, 64, >>, KMP_ARCH_X86 )
2309 ATOMIC_CMPXCHG_CPT_REV( fixed8u, shr_cpt_rev, kmp_uint64, 64, >>, KMP_ARCH_X86 )
2310 ATOMIC_CMPXCHG_CPT_REV( fixed8, sub_cpt_rev, kmp_int64, 64, -, KMP_ARCH_X86 )
2311 ATOMIC_CMPXCHG_CPT_REV( float4, div_cpt_rev, kmp_real32, 32, /, KMP_ARCH_X86 )
2312 ATOMIC_CMPXCHG_CPT_REV( float4, sub_cpt_rev, kmp_real32, 32, -, KMP_ARCH_X86 )
2313 ATOMIC_CMPXCHG_CPT_REV( float8, div_cpt_rev, kmp_real64, 64, /, KMP_ARCH_X86 )
2314 ATOMIC_CMPXCHG_CPT_REV( float8, sub_cpt_rev, kmp_real64, 64, -, KMP_ARCH_X86 )
2323 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2324 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2326 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2328 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2329 OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2335 ATOMIC_CRITICAL_CPT_REV( float10, sub_cpt_rev,
long double, -, 10r, 1 )
2336 ATOMIC_CRITICAL_CPT_REV( float10, div_cpt_rev,
long double, /, 10r, 1 )
2339 ATOMIC_CRITICAL_CPT_REV( float16, sub_cpt_rev, QUAD_LEGACY, -, 16r, 1 )
2340 ATOMIC_CRITICAL_CPT_REV( float16, div_cpt_rev, QUAD_LEGACY, /, 16r, 1 )
2341 #if ( KMP_ARCH_X86 )
2342 ATOMIC_CRITICAL_CPT_REV( float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r, 1 )
2343 ATOMIC_CRITICAL_CPT_REV( float16, div_a16_cpt_rev, Quad_a16_t, /, 16r, 1 )
2353 #define OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2354 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2357 (*lhs) = (rhs) OP (*lhs); \
2361 (*lhs) = (rhs) OP (*lhs); \
2364 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2368 #ifdef KMP_GOMP_COMPAT
2369 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) \
2370 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2372 OP_CRITICAL_CPT_REV_WRK( OP, 0 ); \
2375 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG)
2379 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2380 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2381 OP_GOMP_CRITICAL_CPT_REV_WRK(OP,GOMP_FLAG) \
2382 OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2389 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c, 1 )
2390 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c, 1 )
2392 ATOMIC_CRITICAL_CPT_REV( cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c, 1 )
2393 ATOMIC_CRITICAL_CPT_REV( cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c, 1 )
2394 ATOMIC_CRITICAL_CPT_REV( cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c, 1 )
2395 ATOMIC_CRITICAL_CPT_REV( cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c, 1 )
2397 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c, 1 )
2398 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c, 1 )
2399 #if ( KMP_ARCH_X86 )
2400 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c, 1 )
2401 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c, 1 )
2407 #define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2408 TYPE __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
2410 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2411 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2413 #define CRITICAL_SWP(LCK_ID) \
2414 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2416 old_value = (*lhs); \
2419 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2423 #ifdef KMP_GOMP_COMPAT
2424 #define GOMP_CRITICAL_SWP(FLAG) \
2425 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2427 CRITICAL_SWP( 0 ); \
2430 #define GOMP_CRITICAL_SWP(FLAG)
2434 #define ATOMIC_XCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2435 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2437 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2438 old_value = KMP_XCHG_FIXED##BITS( lhs, rhs ); \
2442 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2443 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2445 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2446 old_value = KMP_XCHG_REAL##BITS( lhs, rhs ); \
2451 #define CMPXCHG_SWP(TYPE,BITS) \
2453 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2454 TYPE old_value, new_value; \
2456 old_value = temp_val; \
2458 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2459 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2460 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2465 old_value = temp_val; \
2472 #define ATOMIC_CMPXCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2473 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2475 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2476 CMPXCHG_SWP(TYPE,BITS) \
2479 ATOMIC_XCHG_SWP( fixed1, kmp_int8, 8, KMP_ARCH_X86 )
2480 ATOMIC_XCHG_SWP( fixed2, kmp_int16, 16, KMP_ARCH_X86 )
2481 ATOMIC_XCHG_SWP( fixed4, kmp_int32, 32, KMP_ARCH_X86 )
2483 ATOMIC_XCHG_FLOAT_SWP( float4, kmp_real32, 32, KMP_ARCH_X86 )
2485 #if ( KMP_ARCH_X86 )
2486 ATOMIC_CMPXCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2487 ATOMIC_CMPXCHG_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2489 ATOMIC_XCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2490 ATOMIC_XCHG_FLOAT_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2495 #define ATOMIC_CRITICAL_SWP(TYPE_ID,TYPE,LCK_ID,GOMP_FLAG) \
2496 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2498 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2499 CRITICAL_SWP(LCK_ID) \
2508 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2509 void __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out ) \
2511 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2512 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2515 #define CRITICAL_SWP_WRK(LCK_ID) \
2516 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2521 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2526 #ifdef KMP_GOMP_COMPAT
2527 #define GOMP_CRITICAL_SWP_WRK(FLAG) \
2528 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2530 CRITICAL_SWP_WRK( 0 ); \
2533 #define GOMP_CRITICAL_SWP_WRK(FLAG)
2537 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE,LCK_ID,GOMP_FLAG) \
2538 ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2540 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
2541 CRITICAL_SWP_WRK(LCK_ID) \
2546 ATOMIC_CRITICAL_SWP( float10,
long double, 10r, 1 )
2548 ATOMIC_CRITICAL_SWP( float16, QUAD_LEGACY, 16r, 1 )
2551 ATOMIC_CRITICAL_SWP_WRK( cmplx4, kmp_cmplx32, 8c, 1 )
2556 ATOMIC_CRITICAL_SWP( cmplx8, kmp_cmplx64, 16c, 1 )
2557 ATOMIC_CRITICAL_SWP( cmplx10, kmp_cmplx80, 20c, 1 )
2559 ATOMIC_CRITICAL_SWP( cmplx16, CPLX128_LEG, 32c, 1 )
2560 #if ( KMP_ARCH_X86 )
2561 ATOMIC_CRITICAL_SWP( float16_a16, Quad_a16_t, 16r, 1 )
2562 ATOMIC_CRITICAL_SWP( cmplx16_a16, kmp_cmplx128_a16_t, 32c, 1 )
2569 #endif //OMP_40_ENABLED
2571 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
2581 __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2583 KMP_DEBUG_ASSERT( __kmp_init_serial );
2586 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2593 kmp_int8 old_value, new_value;
2595 old_value = *(kmp_int8 *) lhs;
2596 (*f)( &new_value, &old_value, rhs );
2599 while ( ! KMP_COMPARE_AND_STORE_ACQ8 ( (kmp_int8 *) lhs,
2600 *(kmp_int8 *) &old_value, *(kmp_int8 *) &new_value ) )
2604 old_value = *(kmp_int8 *) lhs;
2605 (*f)( &new_value, &old_value, rhs );
2615 #ifdef KMP_GOMP_COMPAT
2616 if ( __kmp_atomic_mode == 2 ) {
2617 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2621 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2623 (*f)( lhs, lhs, rhs );
2625 #ifdef KMP_GOMP_COMPAT
2626 if ( __kmp_atomic_mode == 2 ) {
2627 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2631 __kmp_release_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2636 __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2639 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2641 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2644 ! ( (kmp_uintptr_t) lhs & 0x1)
2648 kmp_int16 old_value, new_value;
2650 old_value = *(kmp_int16 *) lhs;
2651 (*f)( &new_value, &old_value, rhs );
2654 while ( ! KMP_COMPARE_AND_STORE_ACQ16 ( (kmp_int16 *) lhs,
2655 *(kmp_int16 *) &old_value, *(kmp_int16 *) &new_value ) )
2659 old_value = *(kmp_int16 *) lhs;
2660 (*f)( &new_value, &old_value, rhs );
2670 #ifdef KMP_GOMP_COMPAT
2671 if ( __kmp_atomic_mode == 2 ) {
2672 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2676 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2678 (*f)( lhs, lhs, rhs );
2680 #ifdef KMP_GOMP_COMPAT
2681 if ( __kmp_atomic_mode == 2 ) {
2682 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2686 __kmp_release_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2691 __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2693 KMP_DEBUG_ASSERT( __kmp_init_serial );
2700 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
2703 ! ( (kmp_uintptr_t) lhs & 0x3)
2707 kmp_int32 old_value, new_value;
2709 old_value = *(kmp_int32 *) lhs;
2710 (*f)( &new_value, &old_value, rhs );
2713 while ( ! KMP_COMPARE_AND_STORE_ACQ32 ( (kmp_int32 *) lhs,
2714 *(kmp_int32 *) &old_value, *(kmp_int32 *) &new_value ) )
2718 old_value = *(kmp_int32 *) lhs;
2719 (*f)( &new_value, &old_value, rhs );
2730 #ifdef KMP_GOMP_COMPAT
2731 if ( __kmp_atomic_mode == 2 ) {
2732 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2736 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2738 (*f)( lhs, lhs, rhs );
2740 #ifdef KMP_GOMP_COMPAT
2741 if ( __kmp_atomic_mode == 2 ) {
2742 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2746 __kmp_release_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2751 __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2753 KMP_DEBUG_ASSERT( __kmp_init_serial );
2756 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2758 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2761 ! ( (kmp_uintptr_t) lhs & 0x7)
2765 kmp_int64 old_value, new_value;
2767 old_value = *(kmp_int64 *) lhs;
2768 (*f)( &new_value, &old_value, rhs );
2770 while ( ! KMP_COMPARE_AND_STORE_ACQ64 ( (kmp_int64 *) lhs,
2771 *(kmp_int64 *) &old_value,
2772 *(kmp_int64 *) &new_value ) )
2776 old_value = *(kmp_int64 *) lhs;
2777 (*f)( &new_value, &old_value, rhs );
2787 #ifdef KMP_GOMP_COMPAT
2788 if ( __kmp_atomic_mode == 2 ) {
2789 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2793 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2795 (*f)( lhs, lhs, rhs );
2797 #ifdef KMP_GOMP_COMPAT
2798 if ( __kmp_atomic_mode == 2 ) {
2799 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2803 __kmp_release_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2808 __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2810 KMP_DEBUG_ASSERT( __kmp_init_serial );
2812 #ifdef KMP_GOMP_COMPAT
2813 if ( __kmp_atomic_mode == 2 ) {
2814 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2818 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2820 (*f)( lhs, lhs, rhs );
2822 #ifdef KMP_GOMP_COMPAT
2823 if ( __kmp_atomic_mode == 2 ) {
2824 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2828 __kmp_release_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2832 __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2834 KMP_DEBUG_ASSERT( __kmp_init_serial );
2836 #ifdef KMP_GOMP_COMPAT
2837 if ( __kmp_atomic_mode == 2 ) {
2838 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2842 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2844 (*f)( lhs, lhs, rhs );
2846 #ifdef KMP_GOMP_COMPAT
2847 if ( __kmp_atomic_mode == 2 ) {
2848 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2852 __kmp_release_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2856 __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2858 KMP_DEBUG_ASSERT( __kmp_init_serial );
2860 #ifdef KMP_GOMP_COMPAT
2861 if ( __kmp_atomic_mode == 2 ) {
2862 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2866 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2868 (*f)( lhs, lhs, rhs );
2870 #ifdef KMP_GOMP_COMPAT
2871 if ( __kmp_atomic_mode == 2 ) {
2872 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2876 __kmp_release_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2880 __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2882 KMP_DEBUG_ASSERT( __kmp_init_serial );
2884 #ifdef KMP_GOMP_COMPAT
2885 if ( __kmp_atomic_mode == 2 ) {
2886 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2890 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2892 (*f)( lhs, lhs, rhs );
2894 #ifdef KMP_GOMP_COMPAT
2895 if ( __kmp_atomic_mode == 2 ) {
2896 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2900 __kmp_release_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2907 __kmpc_atomic_start(
void)
2909 int gtid = __kmp_entry_gtid();
2910 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
2911 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
2916 __kmpc_atomic_end(
void)
2918 int gtid = __kmp_get_gtid();
2919 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
2920 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);