37 #include "kmp_atomic.h"
40 typedef unsigned char uchar;
41 typedef unsigned short ushort;
564 #ifndef KMP_GOMP_COMPAT
565 int __kmp_atomic_mode = 1;
567 int __kmp_atomic_mode = 2;
572 kmp_atomic_lock_t __kmp_atomic_lock;
573 kmp_atomic_lock_t __kmp_atomic_lock_1i;
574 kmp_atomic_lock_t __kmp_atomic_lock_2i;
575 kmp_atomic_lock_t __kmp_atomic_lock_4i;
576 kmp_atomic_lock_t __kmp_atomic_lock_4r;
577 kmp_atomic_lock_t __kmp_atomic_lock_8i;
578 kmp_atomic_lock_t __kmp_atomic_lock_8r;
579 kmp_atomic_lock_t __kmp_atomic_lock_8c;
580 kmp_atomic_lock_t __kmp_atomic_lock_10r;
581 kmp_atomic_lock_t __kmp_atomic_lock_16r;
582 kmp_atomic_lock_t __kmp_atomic_lock_16c;
583 kmp_atomic_lock_t __kmp_atomic_lock_20c;
584 kmp_atomic_lock_t __kmp_atomic_lock_32c;
594 #define KMP_ATOMIC_VOLATILE volatile
596 #if ( KMP_ARCH_X86 ) && KMP_HAVE_QUAD
598 static inline void operator +=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q += rhs.q; };
599 static inline void operator -=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q -= rhs.q; };
600 static inline void operator *=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q *= rhs.q; };
601 static inline void operator /=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q /= rhs.q; };
602 static inline bool operator < ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q < rhs.q; }
603 static inline bool operator > ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q > rhs.q; }
605 static inline void operator +=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q += rhs.q; };
606 static inline void operator -=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q -= rhs.q; };
607 static inline void operator *=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q *= rhs.q; };
608 static inline void operator /=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q /= rhs.q; };
609 static inline bool operator < ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q < rhs.q; }
610 static inline bool operator > ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q > rhs.q; }
612 static inline void operator +=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q += rhs.q; };
613 static inline void operator -=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q -= rhs.q; };
614 static inline void operator *=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q *= rhs.q; };
615 static inline void operator /=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q /= rhs.q; };
617 static inline void operator +=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q += rhs.q; };
618 static inline void operator -=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q -= rhs.q; };
619 static inline void operator *=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q *= rhs.q; };
620 static inline void operator /=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q /= rhs.q; };
633 #define KMP_CHECK_GTID \
634 if ( gtid == KMP_GTID_UNKNOWN ) { \
635 gtid = __kmp_entry_gtid(); \
636 } // check and get gtid when needed
642 #define ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
643 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
645 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
646 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
650 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
651 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
652 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
653 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
654 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
655 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
656 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
657 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
658 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
659 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
660 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
661 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
662 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
670 #define OP_CRITICAL(OP,LCK_ID) \
671 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
675 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
700 #ifdef KMP_GOMP_COMPAT
701 # define OP_GOMP_CRITICAL(OP,FLAG) \
702 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
704 OP_CRITICAL( OP, 0 ); \
708 # define OP_GOMP_CRITICAL(OP,FLAG)
712 # define KMP_DO_PAUSE _mm_delay_32( 30 )
714 # define KMP_DO_PAUSE KMP_CPU_PAUSE()
724 #define OP_CMPXCHG(TYPE,BITS,OP) \
726 TYPE KMP_ATOMIC_VOLATILE temp_val; \
727 TYPE old_value, new_value; \
729 old_value = temp_val; \
730 new_value = old_value OP rhs; \
731 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
732 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
733 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
738 old_value = temp_val; \
739 new_value = old_value OP rhs; \
751 #define OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
753 char anonym[ ( sizeof( TYPE ) == sizeof( kmp_int##BITS ) ) ? ( 1 ) : ( 0 ) ] = { 1 }; \
756 kmp_int##BITS *vvv; \
758 struct _sss old_value, new_value; \
759 old_value.vvv = ( kmp_int##BITS * )&old_value.cmp; \
760 new_value.vvv = ( kmp_int##BITS * )&new_value.cmp; \
761 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
762 new_value.cmp = old_value.cmp OP rhs; \
763 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
764 *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
765 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv ) ) \
769 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
770 new_value.cmp = old_value.cmp OP rhs; \
774 #endif // USE_CMPXCHG_FIX
776 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
780 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
781 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
782 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
784 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
787 #define ATOMIC_FLOAT_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
788 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
789 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
791 KMP_TEST_THEN_ADD_REAL##BITS( lhs, OP rhs ); \
794 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
795 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
796 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
797 OP_CMPXCHG(TYPE,BITS,OP) \
802 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
803 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
804 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
805 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
813 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
814 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
815 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
816 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
818 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
821 OP_CRITICAL(OP##=,LCK_ID) \
825 #define ATOMIC_FLOAT_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
826 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
827 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
828 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
829 OP_CMPXCHG(TYPE,BITS,OP) \
832 OP_CRITICAL(OP##=,LCK_ID) \
836 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
837 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
838 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
839 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
840 OP_CMPXCHG(TYPE,BITS,OP) \
843 OP_CRITICAL(OP##=,LCK_ID) \
849 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
850 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
851 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
852 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
853 OP_CMPXCHG(TYPE,BITS,OP) \
856 OP_CRITICAL(OP##=,LCK_ID) \
860 #endif // USE_CMPXCHG_FIX
864 ATOMIC_FIXED_ADD( fixed4, add, kmp_int32, 32, +, 4i, 3, 0 )
865 ATOMIC_FIXED_ADD( fixed4, sub, kmp_int32, 32, -, 4i, 3, 0 )
868 ATOMIC_CMPXCHG( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
869 ATOMIC_CMPXCHG( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
871 ATOMIC_FLOAT_ADD( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
872 ATOMIC_FLOAT_ADD( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
876 ATOMIC_FIXED_ADD( fixed8, add, kmp_int64, 64, +, 8i, 7, KMP_ARCH_X86 )
877 ATOMIC_FIXED_ADD( fixed8, sub, kmp_int64, 64, -, 8i, 7, KMP_ARCH_X86 )
880 ATOMIC_CMPXCHG( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
881 ATOMIC_CMPXCHG( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
883 ATOMIC_FLOAT_ADD( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
884 ATOMIC_FLOAT_ADD( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
902 ATOMIC_CMPXCHG( fixed1, add, kmp_int8, 8, +, 1i, 0, KMP_ARCH_X86 )
903 ATOMIC_CMPXCHG( fixed1, andb, kmp_int8, 8, &, 1i, 0, 0 )
904 ATOMIC_CMPXCHG( fixed1, div, kmp_int8, 8, /, 1i, 0, KMP_ARCH_X86 )
905 ATOMIC_CMPXCHG( fixed1u, div, kmp_uint8, 8, /, 1i, 0, KMP_ARCH_X86 )
906 ATOMIC_CMPXCHG( fixed1, mul, kmp_int8, 8, *, 1i, 0, KMP_ARCH_X86 )
907 ATOMIC_CMPXCHG( fixed1, orb, kmp_int8, 8, |, 1i, 0, 0 )
908 ATOMIC_CMPXCHG( fixed1, shl, kmp_int8, 8, <<, 1i, 0, KMP_ARCH_X86 )
909 ATOMIC_CMPXCHG( fixed1, shr, kmp_int8, 8, >>, 1i, 0, KMP_ARCH_X86 )
910 ATOMIC_CMPXCHG( fixed1u, shr, kmp_uint8, 8, >>, 1i, 0, KMP_ARCH_X86 )
911 ATOMIC_CMPXCHG( fixed1, sub, kmp_int8, 8, -, 1i, 0, KMP_ARCH_X86 )
912 ATOMIC_CMPXCHG( fixed1, xor, kmp_int8, 8, ^, 1i, 0, 0 )
913 ATOMIC_CMPXCHG( fixed2, add, kmp_int16, 16, +, 2i, 1, KMP_ARCH_X86 )
914 ATOMIC_CMPXCHG( fixed2, andb, kmp_int16, 16, &, 2i, 1, 0 )
915 ATOMIC_CMPXCHG( fixed2, div, kmp_int16, 16, /, 2i, 1, KMP_ARCH_X86 )
916 ATOMIC_CMPXCHG( fixed2u, div, kmp_uint16, 16, /, 2i, 1, KMP_ARCH_X86 )
917 ATOMIC_CMPXCHG( fixed2, mul, kmp_int16, 16, *, 2i, 1, KMP_ARCH_X86 )
918 ATOMIC_CMPXCHG( fixed2, orb, kmp_int16, 16, |, 2i, 1, 0 )
919 ATOMIC_CMPXCHG( fixed2, shl, kmp_int16, 16, <<, 2i, 1, KMP_ARCH_X86 )
920 ATOMIC_CMPXCHG( fixed2, shr, kmp_int16, 16, >>, 2i, 1, KMP_ARCH_X86 )
921 ATOMIC_CMPXCHG( fixed2u, shr, kmp_uint16, 16, >>, 2i, 1, KMP_ARCH_X86 )
922 ATOMIC_CMPXCHG( fixed2, sub, kmp_int16, 16, -, 2i, 1, KMP_ARCH_X86 )
923 ATOMIC_CMPXCHG( fixed2, xor, kmp_int16, 16, ^, 2i, 1, 0 )
924 ATOMIC_CMPXCHG( fixed4, andb, kmp_int32, 32, &, 4i, 3, 0 )
925 ATOMIC_CMPXCHG( fixed4, div, kmp_int32, 32, /, 4i, 3, KMP_ARCH_X86 )
926 ATOMIC_CMPXCHG( fixed4u, div, kmp_uint32, 32, /, 4i, 3, KMP_ARCH_X86 )
927 ATOMIC_CMPXCHG( fixed4, mul, kmp_int32, 32, *, 4i, 3, KMP_ARCH_X86 )
928 ATOMIC_CMPXCHG( fixed4, orb, kmp_int32, 32, |, 4i, 3, 0 )
929 ATOMIC_CMPXCHG( fixed4, shl, kmp_int32, 32, <<, 4i, 3, KMP_ARCH_X86 )
930 ATOMIC_CMPXCHG( fixed4, shr, kmp_int32, 32, >>, 4i, 3, KMP_ARCH_X86 )
931 ATOMIC_CMPXCHG( fixed4u, shr, kmp_uint32, 32, >>, 4i, 3, KMP_ARCH_X86 )
932 ATOMIC_CMPXCHG( fixed4, xor, kmp_int32, 32, ^, 4i, 3, 0 )
933 ATOMIC_CMPXCHG( fixed8, andb, kmp_int64, 64, &, 8i, 7, KMP_ARCH_X86 )
934 ATOMIC_CMPXCHG( fixed8, div, kmp_int64, 64, /, 8i, 7, KMP_ARCH_X86 )
935 ATOMIC_CMPXCHG( fixed8u, div, kmp_uint64, 64, /, 8i, 7, KMP_ARCH_X86 )
936 ATOMIC_CMPXCHG( fixed8, mul, kmp_int64, 64, *, 8i, 7, KMP_ARCH_X86 )
937 ATOMIC_CMPXCHG( fixed8, orb, kmp_int64, 64, |, 8i, 7, KMP_ARCH_X86 )
938 ATOMIC_CMPXCHG( fixed8, shl, kmp_int64, 64, <<, 8i, 7, KMP_ARCH_X86 )
939 ATOMIC_CMPXCHG( fixed8, shr, kmp_int64, 64, >>, 8i, 7, KMP_ARCH_X86 )
940 ATOMIC_CMPXCHG( fixed8u, shr, kmp_uint64, 64, >>, 8i, 7, KMP_ARCH_X86 )
941 ATOMIC_CMPXCHG( fixed8, xor, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
942 ATOMIC_CMPXCHG( float4, div, kmp_real32, 32, /, 4r, 3, KMP_ARCH_X86 )
943 ATOMIC_CMPXCHG( float4, mul, kmp_real32, 32, *, 4r, 3, KMP_ARCH_X86 )
944 ATOMIC_CMPXCHG( float8, div, kmp_real64, 64, /, 8r, 7, KMP_ARCH_X86 )
945 ATOMIC_CMPXCHG( float8, mul, kmp_real64, 64, *, 8r, 7, KMP_ARCH_X86 )
956 #define ATOMIC_CRIT_L(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
957 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
958 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
959 OP_CRITICAL( = *lhs OP, LCK_ID ) \
962 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
966 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
967 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
968 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
969 OP_CMPXCHG(TYPE,BITS,OP) \
975 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
976 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
977 OP_GOMP_CRITICAL(= *lhs OP,GOMP_FLAG) \
978 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
979 OP_CMPXCHG(TYPE,BITS,OP) \
982 OP_CRITICAL(= *lhs OP,LCK_ID) \
987 ATOMIC_CMPX_L( fixed1, andl,
char, 8, &&, 1i, 0, KMP_ARCH_X86 )
988 ATOMIC_CMPX_L( fixed1, orl,
char, 8, ||, 1i, 0, KMP_ARCH_X86 )
989 ATOMIC_CMPX_L( fixed2, andl,
short, 16, &&, 2i, 1, KMP_ARCH_X86 )
990 ATOMIC_CMPX_L( fixed2, orl,
short, 16, ||, 2i, 1, KMP_ARCH_X86 )
991 ATOMIC_CMPX_L( fixed4, andl, kmp_int32, 32, &&, 4i, 3, 0 )
992 ATOMIC_CMPX_L( fixed4, orl, kmp_int32, 32, ||, 4i, 3, 0 )
993 ATOMIC_CMPX_L( fixed8, andl, kmp_int64, 64, &&, 8i, 7, KMP_ARCH_X86 )
994 ATOMIC_CMPX_L( fixed8, orl, kmp_int64, 64, ||, 8i, 7, KMP_ARCH_X86 )
1007 #define MIN_MAX_CRITSECT(OP,LCK_ID) \
1008 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1010 if ( *lhs OP rhs ) { \
1013 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1016 #ifdef KMP_GOMP_COMPAT
1017 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) \
1018 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
1020 MIN_MAX_CRITSECT( OP, 0 ); \
1024 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG)
1028 #define MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1030 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1033 old_value = temp_val; \
1034 while ( old_value OP rhs && \
1035 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1036 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1037 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
1041 old_value = temp_val; \
1047 #define MIN_MAX_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1048 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1049 if ( *lhs OP rhs ) { \
1050 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1051 MIN_MAX_CRITSECT(OP,LCK_ID) \
1055 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1059 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1060 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1061 if ( *lhs OP rhs ) { \
1062 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1063 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1070 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1071 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1072 if ( *lhs OP rhs ) { \
1073 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1074 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1075 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1078 MIN_MAX_CRITSECT(OP,LCK_ID) \
1084 MIN_MAX_COMPXCHG( fixed1, max,
char, 8, <, 1i, 0, KMP_ARCH_X86 )
1085 MIN_MAX_COMPXCHG( fixed1, min,
char, 8, >, 1i, 0, KMP_ARCH_X86 )
1086 MIN_MAX_COMPXCHG( fixed2, max,
short, 16, <, 2i, 1, KMP_ARCH_X86 )
1087 MIN_MAX_COMPXCHG( fixed2, min,
short, 16, >, 2i, 1, KMP_ARCH_X86 )
1088 MIN_MAX_COMPXCHG( fixed4, max, kmp_int32, 32, <, 4i, 3, 0 )
1089 MIN_MAX_COMPXCHG( fixed4, min, kmp_int32, 32, >, 4i, 3, 0 )
1090 MIN_MAX_COMPXCHG( fixed8, max, kmp_int64, 64, <, 8i, 7, KMP_ARCH_X86 )
1091 MIN_MAX_COMPXCHG( fixed8, min, kmp_int64, 64, >, 8i, 7, KMP_ARCH_X86 )
1092 MIN_MAX_COMPXCHG( float4, max, kmp_real32, 32, <, 4r, 3, KMP_ARCH_X86 )
1093 MIN_MAX_COMPXCHG( float4, min, kmp_real32, 32, >, 4r, 3, KMP_ARCH_X86 )
1094 MIN_MAX_COMPXCHG( float8, max, kmp_real64, 64, <, 8r, 7, KMP_ARCH_X86 )
1095 MIN_MAX_COMPXCHG( float8, min, kmp_real64, 64, >, 8r, 7, KMP_ARCH_X86 )
1097 MIN_MAX_CRITICAL( float16, max, QUAD_LEGACY, <, 16r, 1 )
1098 MIN_MAX_CRITICAL( float16, min, QUAD_LEGACY, >, 16r, 1 )
1099 #if ( KMP_ARCH_X86 )
1100 MIN_MAX_CRITICAL( float16, max_a16, Quad_a16_t, <, 16r, 1 )
1101 MIN_MAX_CRITICAL( float16, min_a16, Quad_a16_t, >, 16r, 1 )
1107 #define ATOMIC_CRIT_EQV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1108 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1109 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1110 OP_CRITICAL(^=~,LCK_ID) \
1114 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1117 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1118 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1119 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1120 OP_CMPXCHG(TYPE,BITS,OP) \
1126 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1127 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1128 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1129 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1130 OP_CMPXCHG(TYPE,BITS,OP) \
1133 OP_CRITICAL(^=~,LCK_ID) \
1138 ATOMIC_CMPXCHG( fixed1, neqv, kmp_int8, 8, ^, 1i, 0, KMP_ARCH_X86 )
1139 ATOMIC_CMPXCHG( fixed2, neqv, kmp_int16, 16, ^, 2i, 1, KMP_ARCH_X86 )
1140 ATOMIC_CMPXCHG( fixed4, neqv, kmp_int32, 32, ^, 4i, 3, KMP_ARCH_X86 )
1141 ATOMIC_CMPXCHG( fixed8, neqv, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
1142 ATOMIC_CMPX_EQV( fixed1, eqv, kmp_int8, 8, ^~, 1i, 0, KMP_ARCH_X86 )
1143 ATOMIC_CMPX_EQV( fixed2, eqv, kmp_int16, 16, ^~, 2i, 1, KMP_ARCH_X86 )
1144 ATOMIC_CMPX_EQV( fixed4, eqv, kmp_int32, 32, ^~, 4i, 3, KMP_ARCH_X86 )
1145 ATOMIC_CMPX_EQV( fixed8, eqv, kmp_int64, 64, ^~, 8i, 7, KMP_ARCH_X86 )
1153 #define ATOMIC_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1154 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1155 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1156 OP_CRITICAL(OP##=,LCK_ID) \
1161 ATOMIC_CRITICAL( float10, add,
long double, +, 10r, 1 )
1162 ATOMIC_CRITICAL( float10, sub,
long double, -, 10r, 1 )
1163 ATOMIC_CRITICAL( float10, mul,
long double, *, 10r, 1 )
1164 ATOMIC_CRITICAL( float10, div,
long double, /, 10r, 1 )
1167 ATOMIC_CRITICAL( float16, add, QUAD_LEGACY, +, 16r, 1 )
1168 ATOMIC_CRITICAL( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1169 ATOMIC_CRITICAL( float16, mul, QUAD_LEGACY, *, 16r, 1 )
1170 ATOMIC_CRITICAL( float16, div, QUAD_LEGACY, /, 16r, 1 )
1171 #if ( KMP_ARCH_X86 )
1172 ATOMIC_CRITICAL( float16, add_a16, Quad_a16_t, +, 16r, 1 )
1173 ATOMIC_CRITICAL( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1174 ATOMIC_CRITICAL( float16, mul_a16, Quad_a16_t, *, 16r, 1 )
1175 ATOMIC_CRITICAL( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1182 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, add, kmp_cmplx32, 64, +, 8c, 7, 1 )
1183 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7, 1 )
1184 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7, 1 )
1185 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, div, kmp_cmplx32, 64, /, 8c, 7, 1 )
1188 ATOMIC_CRITICAL( cmplx4, add, kmp_cmplx32, +, 8c, 1 )
1189 ATOMIC_CRITICAL( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1190 ATOMIC_CRITICAL( cmplx4, mul, kmp_cmplx32, *, 8c, 1 )
1191 ATOMIC_CRITICAL( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1192 #endif // USE_CMPXCHG_FIX
1194 ATOMIC_CRITICAL( cmplx8, add, kmp_cmplx64, +, 16c, 1 )
1195 ATOMIC_CRITICAL( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1196 ATOMIC_CRITICAL( cmplx8, mul, kmp_cmplx64, *, 16c, 1 )
1197 ATOMIC_CRITICAL( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1198 ATOMIC_CRITICAL( cmplx10, add, kmp_cmplx80, +, 20c, 1 )
1199 ATOMIC_CRITICAL( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1200 ATOMIC_CRITICAL( cmplx10, mul, kmp_cmplx80, *, 20c, 1 )
1201 ATOMIC_CRITICAL( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1203 ATOMIC_CRITICAL( cmplx16, add, CPLX128_LEG, +, 32c, 1 )
1204 ATOMIC_CRITICAL( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1205 ATOMIC_CRITICAL( cmplx16, mul, CPLX128_LEG, *, 32c, 1 )
1206 ATOMIC_CRITICAL( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1207 #if ( KMP_ARCH_X86 )
1208 ATOMIC_CRITICAL( cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c, 1 )
1209 ATOMIC_CRITICAL( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1210 ATOMIC_CRITICAL( cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c, 1 )
1211 ATOMIC_CRITICAL( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1219 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1227 #define OP_CRITICAL_REV(OP,LCK_ID) \
1228 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1230 (*lhs) = (rhs) OP (*lhs); \
1232 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1234 #ifdef KMP_GOMP_COMPAT
1235 #define OP_GOMP_CRITICAL_REV(OP,FLAG) \
1236 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1238 OP_CRITICAL_REV( OP, 0 ); \
1242 #define OP_GOMP_CRITICAL_REV(OP,FLAG)
1250 #define ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1251 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
1253 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1254 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid ));
1263 #define OP_CMPXCHG_REV(TYPE,BITS,OP) \
1265 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1266 TYPE old_value, new_value; \
1268 old_value = temp_val; \
1269 new_value = rhs OP old_value; \
1270 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1271 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1272 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1277 old_value = temp_val; \
1278 new_value = rhs OP old_value; \
1283 #define ATOMIC_CMPXCHG_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,GOMP_FLAG) \
1284 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1285 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1286 OP_CMPXCHG_REV(TYPE,BITS,OP) \
1303 ATOMIC_CMPXCHG_REV( fixed1, div, kmp_int8, 8, /, 1i, KMP_ARCH_X86 )
1304 ATOMIC_CMPXCHG_REV( fixed1u, div, kmp_uint8, 8, /, 1i, KMP_ARCH_X86 )
1305 ATOMIC_CMPXCHG_REV( fixed1, shl, kmp_int8, 8, <<, 1i, KMP_ARCH_X86 )
1306 ATOMIC_CMPXCHG_REV( fixed1, shr, kmp_int8, 8, >>, 1i, KMP_ARCH_X86 )
1307 ATOMIC_CMPXCHG_REV( fixed1u, shr, kmp_uint8, 8, >>, 1i, KMP_ARCH_X86 )
1308 ATOMIC_CMPXCHG_REV( fixed1, sub, kmp_int8, 8, -, 1i, KMP_ARCH_X86 )
1310 ATOMIC_CMPXCHG_REV( fixed2, div, kmp_int16, 16, /, 2i, KMP_ARCH_X86 )
1311 ATOMIC_CMPXCHG_REV( fixed2u, div, kmp_uint16, 16, /, 2i, KMP_ARCH_X86 )
1312 ATOMIC_CMPXCHG_REV( fixed2, shl, kmp_int16, 16, <<, 2i, KMP_ARCH_X86 )
1313 ATOMIC_CMPXCHG_REV( fixed2, shr, kmp_int16, 16, >>, 2i, KMP_ARCH_X86 )
1314 ATOMIC_CMPXCHG_REV( fixed2u, shr, kmp_uint16, 16, >>, 2i, KMP_ARCH_X86 )
1315 ATOMIC_CMPXCHG_REV( fixed2, sub, kmp_int16, 16, -, 2i, KMP_ARCH_X86 )
1317 ATOMIC_CMPXCHG_REV( fixed4, div, kmp_int32, 32, /, 4i, KMP_ARCH_X86 )
1318 ATOMIC_CMPXCHG_REV( fixed4u, div, kmp_uint32, 32, /, 4i, KMP_ARCH_X86 )
1319 ATOMIC_CMPXCHG_REV( fixed4, shl, kmp_int32, 32, <<, 4i, KMP_ARCH_X86 )
1320 ATOMIC_CMPXCHG_REV( fixed4, shr, kmp_int32, 32, >>, 4i, KMP_ARCH_X86 )
1321 ATOMIC_CMPXCHG_REV( fixed4u, shr, kmp_uint32, 32, >>, 4i, KMP_ARCH_X86 )
1322 ATOMIC_CMPXCHG_REV( fixed4, sub, kmp_int32, 32, -, 4i, KMP_ARCH_X86 )
1324 ATOMIC_CMPXCHG_REV( fixed8, div, kmp_int64, 64, /, 8i, KMP_ARCH_X86 )
1325 ATOMIC_CMPXCHG_REV( fixed8u, div, kmp_uint64, 64, /, 8i, KMP_ARCH_X86 )
1326 ATOMIC_CMPXCHG_REV( fixed8, shl, kmp_int64, 64, <<, 8i, KMP_ARCH_X86 )
1327 ATOMIC_CMPXCHG_REV( fixed8, shr, kmp_int64, 64, >>, 8i, KMP_ARCH_X86 )
1328 ATOMIC_CMPXCHG_REV( fixed8u, shr, kmp_uint64, 64, >>, 8i, KMP_ARCH_X86 )
1329 ATOMIC_CMPXCHG_REV( fixed8, sub, kmp_int64, 64, -, 8i, KMP_ARCH_X86 )
1331 ATOMIC_CMPXCHG_REV( float4, div, kmp_real32, 32, /, 4r, KMP_ARCH_X86 )
1332 ATOMIC_CMPXCHG_REV( float4, sub, kmp_real32, 32, -, 4r, KMP_ARCH_X86 )
1334 ATOMIC_CMPXCHG_REV( float8, div, kmp_real64, 64, /, 8r, KMP_ARCH_X86 )
1335 ATOMIC_CMPXCHG_REV( float8, sub, kmp_real64, 64, -, 8r, KMP_ARCH_X86 )
1343 #define ATOMIC_CRITICAL_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1344 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1345 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1346 OP_CRITICAL_REV(OP,LCK_ID) \
1351 ATOMIC_CRITICAL_REV( float10, sub,
long double, -, 10r, 1 )
1352 ATOMIC_CRITICAL_REV( float10, div,
long double, /, 10r, 1 )
1355 ATOMIC_CRITICAL_REV( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1356 ATOMIC_CRITICAL_REV( float16, div, QUAD_LEGACY, /, 16r, 1 )
1357 #if ( KMP_ARCH_X86 )
1358 ATOMIC_CRITICAL_REV( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1359 ATOMIC_CRITICAL_REV( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1364 ATOMIC_CRITICAL_REV( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1365 ATOMIC_CRITICAL_REV( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1366 ATOMIC_CRITICAL_REV( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1367 ATOMIC_CRITICAL_REV( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1368 ATOMIC_CRITICAL_REV( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1369 ATOMIC_CRITICAL_REV( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1371 ATOMIC_CRITICAL_REV( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1372 ATOMIC_CRITICAL_REV( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1373 #if ( KMP_ARCH_X86 )
1374 ATOMIC_CRITICAL_REV( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1375 ATOMIC_CRITICAL_REV( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1380 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
1383 #endif //OMP_40_ENABLED
1397 #define ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1398 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs ) \
1400 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1401 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid ));
1404 #define ATOMIC_CRITICAL_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
1405 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1406 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1407 OP_CRITICAL(OP##=,LCK_ID) \
1411 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1414 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1415 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1416 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1417 OP_CMPXCHG(TYPE,BITS,OP) \
1423 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1424 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1425 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1426 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1427 OP_CMPXCHG(TYPE,BITS,OP) \
1430 OP_CRITICAL(OP##=,LCK_ID) \
1436 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1437 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1438 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1439 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1440 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3, 0 )
1441 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3, 0 )
1442 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1443 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1444 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1445 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1446 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1447 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1451 ATOMIC_CMPXCHG_MIX( fixed1,
char, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1452 ATOMIC_CMPXCHG_MIX( fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1453 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1454 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1455 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1457 ATOMIC_CMPXCHG_MIX( fixed2,
short, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1458 ATOMIC_CMPXCHG_MIX( fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1459 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1460 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1461 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1463 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3, 0 )
1464 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3, 0 )
1465 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3, 0 )
1466 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1467 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1469 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1470 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1471 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1472 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1473 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1475 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1476 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1477 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1478 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1480 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1481 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1482 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1483 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1485 ATOMIC_CRITICAL_FP( float10,
long double, add, +, fp, _Quad, 10r, 1 )
1486 ATOMIC_CRITICAL_FP( float10,
long double, sub, -, fp, _Quad, 10r, 1 )
1487 ATOMIC_CRITICAL_FP( float10,
long double, mul, *, fp, _Quad, 10r, 1 )
1488 ATOMIC_CRITICAL_FP( float10,
long double, div, /, fp, _Quad, 10r, 1 )
1491 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1496 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1497 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1498 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1499 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
1503 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1504 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1505 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1506 OP_CMPXCHG(TYPE,BITS,OP) \
1508 #endif // USE_CMPXCHG_FIX
1512 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1513 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1514 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1515 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1516 OP_CMPXCHG(TYPE,BITS,OP) \
1519 OP_CRITICAL(OP##=,LCK_ID) \
1524 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1525 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1526 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1527 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1530 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1542 #define ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1543 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * loc ) \
1545 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1546 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1558 #define OP_CMPXCHG_READ(TYPE,BITS,OP) \
1560 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1563 kmp_int##BITS i_val; \
1565 union f_i_union old_value; \
1567 old_value.f_val = temp_val; \
1568 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( (kmp_int##BITS *) loc, \
1569 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val, \
1570 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val ); \
1571 new_value = old_value.f_val; \
1581 #define OP_CRITICAL_READ(OP,LCK_ID) \
1582 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1584 new_value = (*loc); \
1586 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1589 #ifdef KMP_GOMP_COMPAT
1590 #define OP_GOMP_CRITICAL_READ(OP,FLAG) \
1591 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1593 OP_CRITICAL_READ( OP, 0 ); \
1597 #define OP_GOMP_CRITICAL_READ(OP,FLAG)
1601 #define ATOMIC_FIXED_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1602 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1604 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1605 new_value = KMP_TEST_THEN_ADD##BITS( loc, OP 0 ); \
1609 #define ATOMIC_CMPXCHG_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1610 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1612 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1613 OP_CMPXCHG_READ(TYPE,BITS,OP) \
1620 #define ATOMIC_CRITICAL_READ(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1621 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1623 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1624 OP_CRITICAL_READ(OP,LCK_ID) \
1632 #if ( KMP_OS_WINDOWS )
1634 #define OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1635 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1639 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1641 #ifdef KMP_GOMP_COMPAT
1642 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) \
1643 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1645 OP_CRITICAL_READ_WRK( OP, 0 ); \
1648 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG)
1651 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1652 void __kmpc_atomic_##TYPE_ID##_##OP_ID( TYPE * out, ident_t *id_ref, int gtid, TYPE * loc ) \
1654 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1655 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1658 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1659 ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1660 OP_GOMP_CRITICAL_READ_WRK(OP##=,GOMP_FLAG) \
1661 OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1664 #endif // KMP_OS_WINDOWS
1668 ATOMIC_FIXED_READ( fixed4, rd, kmp_int32, 32, +, 0 )
1669 ATOMIC_FIXED_READ( fixed8, rd, kmp_int64, 64, +, KMP_ARCH_X86 )
1670 ATOMIC_CMPXCHG_READ( float4, rd, kmp_real32, 32, +, KMP_ARCH_X86 )
1671 ATOMIC_CMPXCHG_READ( float8, rd, kmp_real64, 64, +, KMP_ARCH_X86 )
1674 ATOMIC_CMPXCHG_READ( fixed1, rd, kmp_int8, 8, +, KMP_ARCH_X86 )
1675 ATOMIC_CMPXCHG_READ( fixed2, rd, kmp_int16, 16, +, KMP_ARCH_X86 )
1677 ATOMIC_CRITICAL_READ( float10, rd,
long double, +, 10r, 1 )
1679 ATOMIC_CRITICAL_READ( float16, rd, QUAD_LEGACY, +, 16r, 1 )
1680 #endif // KMP_HAVE_QUAD
1683 #if ( KMP_OS_WINDOWS )
1684 ATOMIC_CRITICAL_READ_WRK( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1686 ATOMIC_CRITICAL_READ( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1688 ATOMIC_CRITICAL_READ( cmplx8, rd, kmp_cmplx64, +, 16c, 1 )
1689 ATOMIC_CRITICAL_READ( cmplx10, rd, kmp_cmplx80, +, 20c, 1 )
1691 ATOMIC_CRITICAL_READ( cmplx16, rd, CPLX128_LEG, +, 32c, 1 )
1692 #if ( KMP_ARCH_X86 )
1693 ATOMIC_CRITICAL_READ( float16, a16_rd, Quad_a16_t, +, 16r, 1 )
1694 ATOMIC_CRITICAL_READ( cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c, 1 )
1703 #define ATOMIC_XCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1704 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1705 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1706 KMP_XCHG_FIXED##BITS( lhs, rhs ); \
1709 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1710 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1711 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1712 KMP_XCHG_REAL##BITS( lhs, rhs ); \
1723 #define OP_CMPXCHG_WR(TYPE,BITS,OP) \
1725 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1726 TYPE old_value, new_value; \
1728 old_value = temp_val; \
1730 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1731 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1732 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1737 old_value = temp_val; \
1743 #define ATOMIC_CMPXCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1744 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1745 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1746 OP_CMPXCHG_WR(TYPE,BITS,OP) \
1754 #define ATOMIC_CRITICAL_WR(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1755 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1756 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1757 OP_CRITICAL(OP,LCK_ID) \
1761 ATOMIC_XCHG_WR( fixed1, wr, kmp_int8, 8, =, KMP_ARCH_X86 )
1762 ATOMIC_XCHG_WR( fixed2, wr, kmp_int16, 16, =, KMP_ARCH_X86 )
1763 ATOMIC_XCHG_WR( fixed4, wr, kmp_int32, 32, =, KMP_ARCH_X86 )
1764 #if ( KMP_ARCH_X86 )
1765 ATOMIC_CMPXCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1767 ATOMIC_XCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1770 ATOMIC_XCHG_FLOAT_WR( float4, wr, kmp_real32, 32, =, KMP_ARCH_X86 )
1771 #if ( KMP_ARCH_X86 )
1772 ATOMIC_CMPXCHG_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1774 ATOMIC_XCHG_FLOAT_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1777 ATOMIC_CRITICAL_WR( float10, wr,
long double, =, 10r, 1 )
1779 ATOMIC_CRITICAL_WR( float16, wr, QUAD_LEGACY, =, 16r, 1 )
1781 ATOMIC_CRITICAL_WR( cmplx4, wr, kmp_cmplx32, =, 8c, 1 )
1782 ATOMIC_CRITICAL_WR( cmplx8, wr, kmp_cmplx64, =, 16c, 1 )
1783 ATOMIC_CRITICAL_WR( cmplx10, wr, kmp_cmplx80, =, 20c, 1 )
1785 ATOMIC_CRITICAL_WR( cmplx16, wr, CPLX128_LEG, =, 32c, 1 )
1786 #if ( KMP_ARCH_X86 )
1787 ATOMIC_CRITICAL_WR( float16, a16_wr, Quad_a16_t, =, 16r, 1 )
1788 ATOMIC_CRITICAL_WR( cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c, 1 )
1801 #define ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,RET_TYPE) \
1802 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag ) \
1804 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1805 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1813 #define OP_CRITICAL_CPT(OP,LCK_ID) \
1814 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1818 new_value = (*lhs); \
1820 new_value = (*lhs); \
1824 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1828 #ifdef KMP_GOMP_COMPAT
1829 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) \
1830 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1832 OP_CRITICAL_CPT( OP##=, 0 ); \
1835 #define OP_GOMP_CRITICAL_CPT(OP,FLAG)
1845 #define OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1847 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1848 TYPE old_value, new_value; \
1850 old_value = temp_val; \
1851 new_value = old_value OP rhs; \
1852 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1853 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1854 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1859 old_value = temp_val; \
1860 new_value = old_value OP rhs; \
1869 #define ATOMIC_CMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1870 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1872 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1873 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1877 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1878 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1879 TYPE old_value, new_value; \
1880 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1882 old_value = KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
1884 return old_value OP rhs; \
1889 #define ATOMIC_FLOAT_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1890 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1891 TYPE old_value, new_value; \
1892 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1894 old_value = KMP_TEST_THEN_ADD_REAL##BITS( lhs, OP rhs ); \
1896 return old_value OP rhs; \
1902 ATOMIC_FIXED_ADD_CPT( fixed4, add_cpt, kmp_int32, 32, +, 0 )
1903 ATOMIC_FIXED_ADD_CPT( fixed4, sub_cpt, kmp_int32, 32, -, 0 )
1904 ATOMIC_FIXED_ADD_CPT( fixed8, add_cpt, kmp_int64, 64, +, KMP_ARCH_X86 )
1905 ATOMIC_FIXED_ADD_CPT( fixed8, sub_cpt, kmp_int64, 64, -, KMP_ARCH_X86 )
1908 ATOMIC_CMPXCHG_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1909 ATOMIC_CMPXCHG_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1910 ATOMIC_CMPXCHG_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1911 ATOMIC_CMPXCHG_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1913 ATOMIC_FLOAT_ADD_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1914 ATOMIC_FLOAT_ADD_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1915 ATOMIC_FLOAT_ADD_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1916 ATOMIC_FLOAT_ADD_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1931 ATOMIC_CMPXCHG_CPT( fixed1, add_cpt, kmp_int8, 8, +, KMP_ARCH_X86 )
1932 ATOMIC_CMPXCHG_CPT( fixed1, andb_cpt, kmp_int8, 8, &, 0 )
1933 ATOMIC_CMPXCHG_CPT( fixed1, div_cpt, kmp_int8, 8, /, KMP_ARCH_X86 )
1934 ATOMIC_CMPXCHG_CPT( fixed1u, div_cpt, kmp_uint8, 8, /, KMP_ARCH_X86 )
1935 ATOMIC_CMPXCHG_CPT( fixed1, mul_cpt, kmp_int8, 8, *, KMP_ARCH_X86 )
1936 ATOMIC_CMPXCHG_CPT( fixed1, orb_cpt, kmp_int8, 8, |, 0 )
1937 ATOMIC_CMPXCHG_CPT( fixed1, shl_cpt, kmp_int8, 8, <<, KMP_ARCH_X86 )
1938 ATOMIC_CMPXCHG_CPT( fixed1, shr_cpt, kmp_int8, 8, >>, KMP_ARCH_X86 )
1939 ATOMIC_CMPXCHG_CPT( fixed1u, shr_cpt, kmp_uint8, 8, >>, KMP_ARCH_X86 )
1940 ATOMIC_CMPXCHG_CPT( fixed1, sub_cpt, kmp_int8, 8, -, KMP_ARCH_X86 )
1941 ATOMIC_CMPXCHG_CPT( fixed1, xor_cpt, kmp_int8, 8, ^, 0 )
1942 ATOMIC_CMPXCHG_CPT( fixed2, add_cpt, kmp_int16, 16, +, KMP_ARCH_X86 )
1943 ATOMIC_CMPXCHG_CPT( fixed2, andb_cpt, kmp_int16, 16, &, 0 )
1944 ATOMIC_CMPXCHG_CPT( fixed2, div_cpt, kmp_int16, 16, /, KMP_ARCH_X86 )
1945 ATOMIC_CMPXCHG_CPT( fixed2u, div_cpt, kmp_uint16, 16, /, KMP_ARCH_X86 )
1946 ATOMIC_CMPXCHG_CPT( fixed2, mul_cpt, kmp_int16, 16, *, KMP_ARCH_X86 )
1947 ATOMIC_CMPXCHG_CPT( fixed2, orb_cpt, kmp_int16, 16, |, 0 )
1948 ATOMIC_CMPXCHG_CPT( fixed2, shl_cpt, kmp_int16, 16, <<, KMP_ARCH_X86 )
1949 ATOMIC_CMPXCHG_CPT( fixed2, shr_cpt, kmp_int16, 16, >>, KMP_ARCH_X86 )
1950 ATOMIC_CMPXCHG_CPT( fixed2u, shr_cpt, kmp_uint16, 16, >>, KMP_ARCH_X86 )
1951 ATOMIC_CMPXCHG_CPT( fixed2, sub_cpt, kmp_int16, 16, -, KMP_ARCH_X86 )
1952 ATOMIC_CMPXCHG_CPT( fixed2, xor_cpt, kmp_int16, 16, ^, 0 )
1953 ATOMIC_CMPXCHG_CPT( fixed4, andb_cpt, kmp_int32, 32, &, 0 )
1954 ATOMIC_CMPXCHG_CPT( fixed4, div_cpt, kmp_int32, 32, /, KMP_ARCH_X86 )
1955 ATOMIC_CMPXCHG_CPT( fixed4u, div_cpt, kmp_uint32, 32, /, KMP_ARCH_X86 )
1956 ATOMIC_CMPXCHG_CPT( fixed4, mul_cpt, kmp_int32, 32, *, KMP_ARCH_X86 )
1957 ATOMIC_CMPXCHG_CPT( fixed4, orb_cpt, kmp_int32, 32, |, 0 )
1958 ATOMIC_CMPXCHG_CPT( fixed4, shl_cpt, kmp_int32, 32, <<, KMP_ARCH_X86 )
1959 ATOMIC_CMPXCHG_CPT( fixed4, shr_cpt, kmp_int32, 32, >>, KMP_ARCH_X86 )
1960 ATOMIC_CMPXCHG_CPT( fixed4u, shr_cpt, kmp_uint32, 32, >>, KMP_ARCH_X86 )
1961 ATOMIC_CMPXCHG_CPT( fixed4, xor_cpt, kmp_int32, 32, ^, 0 )
1962 ATOMIC_CMPXCHG_CPT( fixed8, andb_cpt, kmp_int64, 64, &, KMP_ARCH_X86 )
1963 ATOMIC_CMPXCHG_CPT( fixed8, div_cpt, kmp_int64, 64, /, KMP_ARCH_X86 )
1964 ATOMIC_CMPXCHG_CPT( fixed8u, div_cpt, kmp_uint64, 64, /, KMP_ARCH_X86 )
1965 ATOMIC_CMPXCHG_CPT( fixed8, mul_cpt, kmp_int64, 64, *, KMP_ARCH_X86 )
1966 ATOMIC_CMPXCHG_CPT( fixed8, orb_cpt, kmp_int64, 64, |, KMP_ARCH_X86 )
1967 ATOMIC_CMPXCHG_CPT( fixed8, shl_cpt, kmp_int64, 64, <<, KMP_ARCH_X86 )
1968 ATOMIC_CMPXCHG_CPT( fixed8, shr_cpt, kmp_int64, 64, >>, KMP_ARCH_X86 )
1969 ATOMIC_CMPXCHG_CPT( fixed8u, shr_cpt, kmp_uint64, 64, >>, KMP_ARCH_X86 )
1970 ATOMIC_CMPXCHG_CPT( fixed8, xor_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
1971 ATOMIC_CMPXCHG_CPT( float4, div_cpt, kmp_real32, 32, /, KMP_ARCH_X86 )
1972 ATOMIC_CMPXCHG_CPT( float4, mul_cpt, kmp_real32, 32, *, KMP_ARCH_X86 )
1973 ATOMIC_CMPXCHG_CPT( float8, div_cpt, kmp_real64, 64, /, KMP_ARCH_X86 )
1974 ATOMIC_CMPXCHG_CPT( float8, mul_cpt, kmp_real64, 64, *, KMP_ARCH_X86 )
1987 #define OP_CRITICAL_L_CPT(OP,LCK_ID) \
1988 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1993 new_value = (*lhs); \
1995 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1998 #ifdef KMP_GOMP_COMPAT
1999 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) \
2000 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2002 OP_CRITICAL_L_CPT( OP, 0 ); \
2006 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG)
2011 #define ATOMIC_CMPX_L_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2012 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2014 OP_GOMP_CRITICAL_L_CPT( = *lhs OP, GOMP_FLAG ) \
2015 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
2018 ATOMIC_CMPX_L_CPT( fixed1, andl_cpt,
char, 8, &&, KMP_ARCH_X86 )
2019 ATOMIC_CMPX_L_CPT( fixed1, orl_cpt,
char, 8, ||, KMP_ARCH_X86 )
2020 ATOMIC_CMPX_L_CPT( fixed2, andl_cpt,
short, 16, &&, KMP_ARCH_X86 )
2021 ATOMIC_CMPX_L_CPT( fixed2, orl_cpt,
short, 16, ||, KMP_ARCH_X86 )
2022 ATOMIC_CMPX_L_CPT( fixed4, andl_cpt, kmp_int32, 32, &&, 0 )
2023 ATOMIC_CMPX_L_CPT( fixed4, orl_cpt, kmp_int32, 32, ||, 0 )
2024 ATOMIC_CMPX_L_CPT( fixed8, andl_cpt, kmp_int64, 64, &&, KMP_ARCH_X86 )
2025 ATOMIC_CMPX_L_CPT( fixed8, orl_cpt, kmp_int64, 64, ||, KMP_ARCH_X86 )
2038 #define MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
2039 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2041 if ( *lhs OP rhs ) { \
2047 new_value = old_value; \
2049 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2053 #ifdef KMP_GOMP_COMPAT
2054 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) \
2055 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
2057 MIN_MAX_CRITSECT_CPT( OP, 0 ); \
2060 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG)
2064 #define MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2066 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2069 old_value = temp_val; \
2070 while ( old_value OP rhs && \
2071 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2072 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2073 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
2077 old_value = temp_val; \
2087 #define MIN_MAX_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2088 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2089 TYPE new_value, old_value; \
2090 if ( *lhs OP rhs ) { \
2091 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2092 MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
2097 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2098 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2099 TYPE new_value, old_value; \
2100 if ( *lhs OP rhs ) { \
2101 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2102 MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2108 MIN_MAX_COMPXCHG_CPT( fixed1, max_cpt,
char, 8, <, KMP_ARCH_X86 )
2109 MIN_MAX_COMPXCHG_CPT( fixed1, min_cpt,
char, 8, >, KMP_ARCH_X86 )
2110 MIN_MAX_COMPXCHG_CPT( fixed2, max_cpt,
short, 16, <, KMP_ARCH_X86 )
2111 MIN_MAX_COMPXCHG_CPT( fixed2, min_cpt,
short, 16, >, KMP_ARCH_X86 )
2112 MIN_MAX_COMPXCHG_CPT( fixed4, max_cpt, kmp_int32, 32, <, 0 )
2113 MIN_MAX_COMPXCHG_CPT( fixed4, min_cpt, kmp_int32, 32, >, 0 )
2114 MIN_MAX_COMPXCHG_CPT( fixed8, max_cpt, kmp_int64, 64, <, KMP_ARCH_X86 )
2115 MIN_MAX_COMPXCHG_CPT( fixed8, min_cpt, kmp_int64, 64, >, KMP_ARCH_X86 )
2116 MIN_MAX_COMPXCHG_CPT( float4, max_cpt, kmp_real32, 32, <, KMP_ARCH_X86 )
2117 MIN_MAX_COMPXCHG_CPT( float4, min_cpt, kmp_real32, 32, >, KMP_ARCH_X86 )
2118 MIN_MAX_COMPXCHG_CPT( float8, max_cpt, kmp_real64, 64, <, KMP_ARCH_X86 )
2119 MIN_MAX_COMPXCHG_CPT( float8, min_cpt, kmp_real64, 64, >, KMP_ARCH_X86 )
2121 MIN_MAX_CRITICAL_CPT( float16, max_cpt, QUAD_LEGACY, <, 16r, 1 )
2122 MIN_MAX_CRITICAL_CPT( float16, min_cpt, QUAD_LEGACY, >, 16r, 1 )
2123 #if ( KMP_ARCH_X86 )
2124 MIN_MAX_CRITICAL_CPT( float16, max_a16_cpt, Quad_a16_t, <, 16r, 1 )
2125 MIN_MAX_CRITICAL_CPT( float16, min_a16_cpt, Quad_a16_t, >, 16r, 1 )
2130 #ifdef KMP_GOMP_COMPAT
2131 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) \
2132 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2134 OP_CRITICAL_CPT( OP, 0 ); \
2137 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG)
2140 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2141 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2143 OP_GOMP_CRITICAL_EQV_CPT(^=~,GOMP_FLAG) \
2144 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
2149 ATOMIC_CMPXCHG_CPT( fixed1, neqv_cpt, kmp_int8, 8, ^, KMP_ARCH_X86 )
2150 ATOMIC_CMPXCHG_CPT( fixed2, neqv_cpt, kmp_int16, 16, ^, KMP_ARCH_X86 )
2151 ATOMIC_CMPXCHG_CPT( fixed4, neqv_cpt, kmp_int32, 32, ^, KMP_ARCH_X86 )
2152 ATOMIC_CMPXCHG_CPT( fixed8, neqv_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
2153 ATOMIC_CMPX_EQV_CPT( fixed1, eqv_cpt, kmp_int8, 8, ^~, KMP_ARCH_X86 )
2154 ATOMIC_CMPX_EQV_CPT( fixed2, eqv_cpt, kmp_int16, 16, ^~, KMP_ARCH_X86 )
2155 ATOMIC_CMPX_EQV_CPT( fixed4, eqv_cpt, kmp_int32, 32, ^~, KMP_ARCH_X86 )
2156 ATOMIC_CMPX_EQV_CPT( fixed8, eqv_cpt, kmp_int64, 64, ^~, KMP_ARCH_X86 )
2163 #define ATOMIC_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2164 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2166 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
2167 OP_CRITICAL_CPT(OP##=,LCK_ID) \
2174 #define OP_CRITICAL_CPT_WRK(OP,LCK_ID) \
2175 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2185 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2189 #ifdef KMP_GOMP_COMPAT
2190 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) \
2191 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2193 OP_CRITICAL_CPT_WRK( OP##=, 0 ); \
2196 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG)
2200 #define ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2201 void __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out, int flag ) \
2203 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2204 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
2207 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2208 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2209 OP_GOMP_CRITICAL_CPT_WRK(OP,GOMP_FLAG) \
2210 OP_CRITICAL_CPT_WRK(OP##=,LCK_ID) \
2216 ATOMIC_CRITICAL_CPT( float10, add_cpt,
long double, +, 10r, 1 )
2217 ATOMIC_CRITICAL_CPT( float10, sub_cpt,
long double, -, 10r, 1 )
2218 ATOMIC_CRITICAL_CPT( float10, mul_cpt,
long double, *, 10r, 1 )
2219 ATOMIC_CRITICAL_CPT( float10, div_cpt,
long double, /, 10r, 1 )
2222 ATOMIC_CRITICAL_CPT( float16, add_cpt, QUAD_LEGACY, +, 16r, 1 )
2223 ATOMIC_CRITICAL_CPT( float16, sub_cpt, QUAD_LEGACY, -, 16r, 1 )
2224 ATOMIC_CRITICAL_CPT( float16, mul_cpt, QUAD_LEGACY, *, 16r, 1 )
2225 ATOMIC_CRITICAL_CPT( float16, div_cpt, QUAD_LEGACY, /, 16r, 1 )
2226 #if ( KMP_ARCH_X86 )
2227 ATOMIC_CRITICAL_CPT( float16, add_a16_cpt, Quad_a16_t, +, 16r, 1 )
2228 ATOMIC_CRITICAL_CPT( float16, sub_a16_cpt, Quad_a16_t, -, 16r, 1 )
2229 ATOMIC_CRITICAL_CPT( float16, mul_a16_cpt, Quad_a16_t, *, 16r, 1 )
2230 ATOMIC_CRITICAL_CPT( float16, div_a16_cpt, Quad_a16_t, /, 16r, 1 )
2237 ATOMIC_CRITICAL_CPT_WRK( cmplx4, add_cpt, kmp_cmplx32, +, 8c, 1 )
2238 ATOMIC_CRITICAL_CPT_WRK( cmplx4, sub_cpt, kmp_cmplx32, -, 8c, 1 )
2239 ATOMIC_CRITICAL_CPT_WRK( cmplx4, mul_cpt, kmp_cmplx32, *, 8c, 1 )
2240 ATOMIC_CRITICAL_CPT_WRK( cmplx4, div_cpt, kmp_cmplx32, /, 8c, 1 )
2242 ATOMIC_CRITICAL_CPT( cmplx8, add_cpt, kmp_cmplx64, +, 16c, 1 )
2243 ATOMIC_CRITICAL_CPT( cmplx8, sub_cpt, kmp_cmplx64, -, 16c, 1 )
2244 ATOMIC_CRITICAL_CPT( cmplx8, mul_cpt, kmp_cmplx64, *, 16c, 1 )
2245 ATOMIC_CRITICAL_CPT( cmplx8, div_cpt, kmp_cmplx64, /, 16c, 1 )
2246 ATOMIC_CRITICAL_CPT( cmplx10, add_cpt, kmp_cmplx80, +, 20c, 1 )
2247 ATOMIC_CRITICAL_CPT( cmplx10, sub_cpt, kmp_cmplx80, -, 20c, 1 )
2248 ATOMIC_CRITICAL_CPT( cmplx10, mul_cpt, kmp_cmplx80, *, 20c, 1 )
2249 ATOMIC_CRITICAL_CPT( cmplx10, div_cpt, kmp_cmplx80, /, 20c, 1 )
2251 ATOMIC_CRITICAL_CPT( cmplx16, add_cpt, CPLX128_LEG, +, 32c, 1 )
2252 ATOMIC_CRITICAL_CPT( cmplx16, sub_cpt, CPLX128_LEG, -, 32c, 1 )
2253 ATOMIC_CRITICAL_CPT( cmplx16, mul_cpt, CPLX128_LEG, *, 32c, 1 )
2254 ATOMIC_CRITICAL_CPT( cmplx16, div_cpt, CPLX128_LEG, /, 32c, 1 )
2255 #if ( KMP_ARCH_X86 )
2256 ATOMIC_CRITICAL_CPT( cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c, 1 )
2257 ATOMIC_CRITICAL_CPT( cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c, 1 )
2258 ATOMIC_CRITICAL_CPT( cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c, 1 )
2259 ATOMIC_CRITICAL_CPT( cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c, 1 )
2274 #define OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2275 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2279 (*lhs) = (rhs) OP (*lhs); \
2280 new_value = (*lhs); \
2282 new_value = (*lhs);\
2283 (*lhs) = (rhs) OP (*lhs); \
2285 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2289 #ifdef KMP_GOMP_COMPAT
2290 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) \
2291 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2293 OP_CRITICAL_CPT_REV( OP, 0 ); \
2296 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG)
2306 #define OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2308 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2309 TYPE old_value, new_value; \
2311 old_value = temp_val; \
2312 new_value = rhs OP old_value; \
2313 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2314 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2315 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2320 old_value = temp_val; \
2321 new_value = rhs OP old_value; \
2330 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2331 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2333 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2334 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2335 OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2339 ATOMIC_CMPXCHG_CPT_REV( fixed1, div_cpt_rev, kmp_int8, 8, /, KMP_ARCH_X86 )
2340 ATOMIC_CMPXCHG_CPT_REV( fixed1u, div_cpt_rev, kmp_uint8, 8, /, KMP_ARCH_X86 )
2341 ATOMIC_CMPXCHG_CPT_REV( fixed1, shl_cpt_rev, kmp_int8, 8, <<, KMP_ARCH_X86 )
2342 ATOMIC_CMPXCHG_CPT_REV( fixed1, shr_cpt_rev, kmp_int8, 8, >>, KMP_ARCH_X86 )
2343 ATOMIC_CMPXCHG_CPT_REV( fixed1u, shr_cpt_rev, kmp_uint8, 8, >>, KMP_ARCH_X86 )
2344 ATOMIC_CMPXCHG_CPT_REV( fixed1, sub_cpt_rev, kmp_int8, 8, -, KMP_ARCH_X86 )
2345 ATOMIC_CMPXCHG_CPT_REV( fixed2, div_cpt_rev, kmp_int16, 16, /, KMP_ARCH_X86 )
2346 ATOMIC_CMPXCHG_CPT_REV( fixed2u, div_cpt_rev, kmp_uint16, 16, /, KMP_ARCH_X86 )
2347 ATOMIC_CMPXCHG_CPT_REV( fixed2, shl_cpt_rev, kmp_int16, 16, <<, KMP_ARCH_X86 )
2348 ATOMIC_CMPXCHG_CPT_REV( fixed2, shr_cpt_rev, kmp_int16, 16, >>, KMP_ARCH_X86 )
2349 ATOMIC_CMPXCHG_CPT_REV( fixed2u, shr_cpt_rev, kmp_uint16, 16, >>, KMP_ARCH_X86 )
2350 ATOMIC_CMPXCHG_CPT_REV( fixed2, sub_cpt_rev, kmp_int16, 16, -, KMP_ARCH_X86 )
2351 ATOMIC_CMPXCHG_CPT_REV( fixed4, div_cpt_rev, kmp_int32, 32, /, KMP_ARCH_X86 )
2352 ATOMIC_CMPXCHG_CPT_REV( fixed4u, div_cpt_rev, kmp_uint32, 32, /, KMP_ARCH_X86 )
2353 ATOMIC_CMPXCHG_CPT_REV( fixed4, shl_cpt_rev, kmp_int32, 32, <<, KMP_ARCH_X86 )
2354 ATOMIC_CMPXCHG_CPT_REV( fixed4, shr_cpt_rev, kmp_int32, 32, >>, KMP_ARCH_X86 )
2355 ATOMIC_CMPXCHG_CPT_REV( fixed4u, shr_cpt_rev, kmp_uint32, 32, >>, KMP_ARCH_X86 )
2356 ATOMIC_CMPXCHG_CPT_REV( fixed4, sub_cpt_rev, kmp_int32, 32, -, KMP_ARCH_X86 )
2357 ATOMIC_CMPXCHG_CPT_REV( fixed8, div_cpt_rev, kmp_int64, 64, /, KMP_ARCH_X86 )
2358 ATOMIC_CMPXCHG_CPT_REV( fixed8u, div_cpt_rev, kmp_uint64, 64, /, KMP_ARCH_X86 )
2359 ATOMIC_CMPXCHG_CPT_REV( fixed8, shl_cpt_rev, kmp_int64, 64, <<, KMP_ARCH_X86 )
2360 ATOMIC_CMPXCHG_CPT_REV( fixed8, shr_cpt_rev, kmp_int64, 64, >>, KMP_ARCH_X86 )
2361 ATOMIC_CMPXCHG_CPT_REV( fixed8u, shr_cpt_rev, kmp_uint64, 64, >>, KMP_ARCH_X86 )
2362 ATOMIC_CMPXCHG_CPT_REV( fixed8, sub_cpt_rev, kmp_int64, 64, -, KMP_ARCH_X86 )
2363 ATOMIC_CMPXCHG_CPT_REV( float4, div_cpt_rev, kmp_real32, 32, /, KMP_ARCH_X86 )
2364 ATOMIC_CMPXCHG_CPT_REV( float4, sub_cpt_rev, kmp_real32, 32, -, KMP_ARCH_X86 )
2365 ATOMIC_CMPXCHG_CPT_REV( float8, div_cpt_rev, kmp_real64, 64, /, KMP_ARCH_X86 )
2366 ATOMIC_CMPXCHG_CPT_REV( float8, sub_cpt_rev, kmp_real64, 64, -, KMP_ARCH_X86 )
2375 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2376 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2378 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2380 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2381 OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2387 ATOMIC_CRITICAL_CPT_REV( float10, sub_cpt_rev,
long double, -, 10r, 1 )
2388 ATOMIC_CRITICAL_CPT_REV( float10, div_cpt_rev,
long double, /, 10r, 1 )
2391 ATOMIC_CRITICAL_CPT_REV( float16, sub_cpt_rev, QUAD_LEGACY, -, 16r, 1 )
2392 ATOMIC_CRITICAL_CPT_REV( float16, div_cpt_rev, QUAD_LEGACY, /, 16r, 1 )
2393 #if ( KMP_ARCH_X86 )
2394 ATOMIC_CRITICAL_CPT_REV( float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r, 1 )
2395 ATOMIC_CRITICAL_CPT_REV( float16, div_a16_cpt_rev, Quad_a16_t, /, 16r, 1 )
2405 #define OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2406 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2409 (*lhs) = (rhs) OP (*lhs); \
2413 (*lhs) = (rhs) OP (*lhs); \
2416 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2420 #ifdef KMP_GOMP_COMPAT
2421 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) \
2422 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2424 OP_CRITICAL_CPT_REV_WRK( OP, 0 ); \
2427 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG)
2431 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2432 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2433 OP_GOMP_CRITICAL_CPT_REV_WRK(OP,GOMP_FLAG) \
2434 OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2441 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c, 1 )
2442 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c, 1 )
2444 ATOMIC_CRITICAL_CPT_REV( cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c, 1 )
2445 ATOMIC_CRITICAL_CPT_REV( cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c, 1 )
2446 ATOMIC_CRITICAL_CPT_REV( cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c, 1 )
2447 ATOMIC_CRITICAL_CPT_REV( cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c, 1 )
2449 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c, 1 )
2450 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c, 1 )
2451 #if ( KMP_ARCH_X86 )
2452 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c, 1 )
2453 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c, 1 )
2459 #define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2460 TYPE __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
2462 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2463 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2465 #define CRITICAL_SWP(LCK_ID) \
2466 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2468 old_value = (*lhs); \
2471 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2475 #ifdef KMP_GOMP_COMPAT
2476 #define GOMP_CRITICAL_SWP(FLAG) \
2477 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2479 CRITICAL_SWP( 0 ); \
2482 #define GOMP_CRITICAL_SWP(FLAG)
2486 #define ATOMIC_XCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2487 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2489 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2490 old_value = KMP_XCHG_FIXED##BITS( lhs, rhs ); \
2494 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2495 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2497 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2498 old_value = KMP_XCHG_REAL##BITS( lhs, rhs ); \
2503 #define CMPXCHG_SWP(TYPE,BITS) \
2505 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2506 TYPE old_value, new_value; \
2508 old_value = temp_val; \
2510 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2511 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2512 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2517 old_value = temp_val; \
2524 #define ATOMIC_CMPXCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2525 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2527 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2528 CMPXCHG_SWP(TYPE,BITS) \
2531 ATOMIC_XCHG_SWP( fixed1, kmp_int8, 8, KMP_ARCH_X86 )
2532 ATOMIC_XCHG_SWP( fixed2, kmp_int16, 16, KMP_ARCH_X86 )
2533 ATOMIC_XCHG_SWP( fixed4, kmp_int32, 32, KMP_ARCH_X86 )
2535 ATOMIC_XCHG_FLOAT_SWP( float4, kmp_real32, 32, KMP_ARCH_X86 )
2537 #if ( KMP_ARCH_X86 )
2538 ATOMIC_CMPXCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2539 ATOMIC_CMPXCHG_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2541 ATOMIC_XCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2542 ATOMIC_XCHG_FLOAT_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2547 #define ATOMIC_CRITICAL_SWP(TYPE_ID,TYPE,LCK_ID,GOMP_FLAG) \
2548 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2550 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2551 CRITICAL_SWP(LCK_ID) \
2560 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2561 void __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out ) \
2563 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2564 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2567 #define CRITICAL_SWP_WRK(LCK_ID) \
2568 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2573 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2578 #ifdef KMP_GOMP_COMPAT
2579 #define GOMP_CRITICAL_SWP_WRK(FLAG) \
2580 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2582 CRITICAL_SWP_WRK( 0 ); \
2585 #define GOMP_CRITICAL_SWP_WRK(FLAG)
2589 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE,LCK_ID,GOMP_FLAG) \
2590 ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2592 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
2593 CRITICAL_SWP_WRK(LCK_ID) \
2598 ATOMIC_CRITICAL_SWP( float10,
long double, 10r, 1 )
2600 ATOMIC_CRITICAL_SWP( float16, QUAD_LEGACY, 16r, 1 )
2603 ATOMIC_CRITICAL_SWP_WRK( cmplx4, kmp_cmplx32, 8c, 1 )
2608 ATOMIC_CRITICAL_SWP( cmplx8, kmp_cmplx64, 16c, 1 )
2609 ATOMIC_CRITICAL_SWP( cmplx10, kmp_cmplx80, 20c, 1 )
2611 ATOMIC_CRITICAL_SWP( cmplx16, CPLX128_LEG, 32c, 1 )
2612 #if ( KMP_ARCH_X86 )
2613 ATOMIC_CRITICAL_SWP( float16_a16, Quad_a16_t, 16r, 1 )
2614 ATOMIC_CRITICAL_SWP( cmplx16_a16, kmp_cmplx128_a16_t, 32c, 1 )
2621 #endif //OMP_40_ENABLED
2623 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
2633 __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2635 KMP_DEBUG_ASSERT( __kmp_init_serial );
2638 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2645 kmp_int8 old_value, new_value;
2647 old_value = *(kmp_int8 *) lhs;
2648 (*f)( &new_value, &old_value, rhs );
2651 while ( ! KMP_COMPARE_AND_STORE_ACQ8 ( (kmp_int8 *) lhs,
2652 *(kmp_int8 *) &old_value, *(kmp_int8 *) &new_value ) )
2656 old_value = *(kmp_int8 *) lhs;
2657 (*f)( &new_value, &old_value, rhs );
2667 #ifdef KMP_GOMP_COMPAT
2668 if ( __kmp_atomic_mode == 2 ) {
2669 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2673 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2675 (*f)( lhs, lhs, rhs );
2677 #ifdef KMP_GOMP_COMPAT
2678 if ( __kmp_atomic_mode == 2 ) {
2679 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2683 __kmp_release_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2688 __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2691 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2693 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2696 ! ( (kmp_uintptr_t) lhs & 0x1)
2700 kmp_int16 old_value, new_value;
2702 old_value = *(kmp_int16 *) lhs;
2703 (*f)( &new_value, &old_value, rhs );
2706 while ( ! KMP_COMPARE_AND_STORE_ACQ16 ( (kmp_int16 *) lhs,
2707 *(kmp_int16 *) &old_value, *(kmp_int16 *) &new_value ) )
2711 old_value = *(kmp_int16 *) lhs;
2712 (*f)( &new_value, &old_value, rhs );
2722 #ifdef KMP_GOMP_COMPAT
2723 if ( __kmp_atomic_mode == 2 ) {
2724 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2728 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2730 (*f)( lhs, lhs, rhs );
2732 #ifdef KMP_GOMP_COMPAT
2733 if ( __kmp_atomic_mode == 2 ) {
2734 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2738 __kmp_release_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2743 __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2745 KMP_DEBUG_ASSERT( __kmp_init_serial );
2752 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
2755 ! ( (kmp_uintptr_t) lhs & 0x3)
2759 kmp_int32 old_value, new_value;
2761 old_value = *(kmp_int32 *) lhs;
2762 (*f)( &new_value, &old_value, rhs );
2765 while ( ! KMP_COMPARE_AND_STORE_ACQ32 ( (kmp_int32 *) lhs,
2766 *(kmp_int32 *) &old_value, *(kmp_int32 *) &new_value ) )
2770 old_value = *(kmp_int32 *) lhs;
2771 (*f)( &new_value, &old_value, rhs );
2782 #ifdef KMP_GOMP_COMPAT
2783 if ( __kmp_atomic_mode == 2 ) {
2784 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2788 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2790 (*f)( lhs, lhs, rhs );
2792 #ifdef KMP_GOMP_COMPAT
2793 if ( __kmp_atomic_mode == 2 ) {
2794 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2798 __kmp_release_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2803 __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2805 KMP_DEBUG_ASSERT( __kmp_init_serial );
2808 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2810 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2813 ! ( (kmp_uintptr_t) lhs & 0x7)
2817 kmp_int64 old_value, new_value;
2819 old_value = *(kmp_int64 *) lhs;
2820 (*f)( &new_value, &old_value, rhs );
2822 while ( ! KMP_COMPARE_AND_STORE_ACQ64 ( (kmp_int64 *) lhs,
2823 *(kmp_int64 *) &old_value,
2824 *(kmp_int64 *) &new_value ) )
2828 old_value = *(kmp_int64 *) lhs;
2829 (*f)( &new_value, &old_value, rhs );
2839 #ifdef KMP_GOMP_COMPAT
2840 if ( __kmp_atomic_mode == 2 ) {
2841 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2845 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2847 (*f)( lhs, lhs, rhs );
2849 #ifdef KMP_GOMP_COMPAT
2850 if ( __kmp_atomic_mode == 2 ) {
2851 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2855 __kmp_release_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2860 __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2862 KMP_DEBUG_ASSERT( __kmp_init_serial );
2864 #ifdef KMP_GOMP_COMPAT
2865 if ( __kmp_atomic_mode == 2 ) {
2866 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2870 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2872 (*f)( lhs, lhs, rhs );
2874 #ifdef KMP_GOMP_COMPAT
2875 if ( __kmp_atomic_mode == 2 ) {
2876 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2880 __kmp_release_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2884 __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2886 KMP_DEBUG_ASSERT( __kmp_init_serial );
2888 #ifdef KMP_GOMP_COMPAT
2889 if ( __kmp_atomic_mode == 2 ) {
2890 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2894 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2896 (*f)( lhs, lhs, rhs );
2898 #ifdef KMP_GOMP_COMPAT
2899 if ( __kmp_atomic_mode == 2 ) {
2900 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2904 __kmp_release_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2908 __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2910 KMP_DEBUG_ASSERT( __kmp_init_serial );
2912 #ifdef KMP_GOMP_COMPAT
2913 if ( __kmp_atomic_mode == 2 ) {
2914 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2918 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2920 (*f)( lhs, lhs, rhs );
2922 #ifdef KMP_GOMP_COMPAT
2923 if ( __kmp_atomic_mode == 2 ) {
2924 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2928 __kmp_release_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2932 __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2934 KMP_DEBUG_ASSERT( __kmp_init_serial );
2936 #ifdef KMP_GOMP_COMPAT
2937 if ( __kmp_atomic_mode == 2 ) {
2938 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2942 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2944 (*f)( lhs, lhs, rhs );
2946 #ifdef KMP_GOMP_COMPAT
2947 if ( __kmp_atomic_mode == 2 ) {
2948 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2952 __kmp_release_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2959 __kmpc_atomic_start(
void)
2961 int gtid = __kmp_entry_gtid();
2962 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
2963 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
2968 __kmpc_atomic_end(
void)
2970 int gtid = __kmp_get_gtid();
2971 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
2972 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);