37 #include "kmp_atomic.h"
40 typedef unsigned char uchar;
41 typedef unsigned short ushort;
564 #ifndef KMP_GOMP_COMPAT
565 int __kmp_atomic_mode = 1;
567 int __kmp_atomic_mode = 2;
572 kmp_atomic_lock_t __kmp_atomic_lock;
573 kmp_atomic_lock_t __kmp_atomic_lock_1i;
574 kmp_atomic_lock_t __kmp_atomic_lock_2i;
575 kmp_atomic_lock_t __kmp_atomic_lock_4i;
576 kmp_atomic_lock_t __kmp_atomic_lock_4r;
577 kmp_atomic_lock_t __kmp_atomic_lock_8i;
578 kmp_atomic_lock_t __kmp_atomic_lock_8r;
579 kmp_atomic_lock_t __kmp_atomic_lock_8c;
580 kmp_atomic_lock_t __kmp_atomic_lock_10r;
581 kmp_atomic_lock_t __kmp_atomic_lock_16r;
582 kmp_atomic_lock_t __kmp_atomic_lock_16c;
583 kmp_atomic_lock_t __kmp_atomic_lock_20c;
584 kmp_atomic_lock_t __kmp_atomic_lock_32c;
594 #define KMP_ATOMIC_VOLATILE volatile
598 static inline void operator +=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q += rhs.q; };
599 static inline void operator -=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q -= rhs.q; };
600 static inline void operator *=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q *= rhs.q; };
601 static inline void operator /=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q /= rhs.q; };
602 static inline bool operator < ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q < rhs.q; }
603 static inline bool operator > ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q > rhs.q; }
605 static inline void operator +=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q += rhs.q; };
606 static inline void operator -=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q -= rhs.q; };
607 static inline void operator *=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q *= rhs.q; };
608 static inline void operator /=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q /= rhs.q; };
609 static inline bool operator < ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q < rhs.q; }
610 static inline bool operator > ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q > rhs.q; }
612 static inline void operator +=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q += rhs.q; };
613 static inline void operator -=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q -= rhs.q; };
614 static inline void operator *=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q *= rhs.q; };
615 static inline void operator /=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q /= rhs.q; };
617 static inline void operator +=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q += rhs.q; };
618 static inline void operator -=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q -= rhs.q; };
619 static inline void operator *=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q *= rhs.q; };
620 static inline void operator /=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q /= rhs.q; };
633 #define KMP_CHECK_GTID \
634 if ( gtid == KMP_GTID_UNKNOWN ) { \
635 gtid = __kmp_entry_gtid(); \
636 } // check and get gtid when needed
642 #define ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
643 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
645 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
646 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
650 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
651 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
652 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
653 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
654 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
655 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
656 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
657 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
658 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
659 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
660 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
661 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
662 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
670 #define OP_CRITICAL(OP,LCK_ID) \
671 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
675 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
700 #ifdef KMP_GOMP_COMPAT
701 # define OP_GOMP_CRITICAL(OP,FLAG) \
702 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
704 OP_CRITICAL( OP, 0 ); \
708 # define OP_GOMP_CRITICAL(OP,FLAG)
712 # define KMP_DO_PAUSE _mm_delay_32( 30 )
714 # define KMP_DO_PAUSE KMP_CPU_PAUSE()
724 #define OP_CMPXCHG(TYPE,BITS,OP) \
726 TYPE KMP_ATOMIC_VOLATILE temp_val; \
727 TYPE old_value, new_value; \
729 old_value = temp_val; \
730 new_value = old_value OP rhs; \
731 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
732 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
733 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
738 old_value = temp_val; \
739 new_value = old_value OP rhs; \
750 #define OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
752 char anonym[ ( sizeof( TYPE ) == sizeof( kmp_int##BITS ) ) ? ( 1 ) : ( 0 ) ] = { 1 }; \
755 kmp_int##BITS *vvv; \
757 struct _sss old_value, new_value; \
758 old_value.vvv = ( kmp_int##BITS * )&old_value.cmp; \
759 new_value.vvv = ( kmp_int##BITS * )&new_value.cmp; \
760 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
761 new_value.cmp = old_value.cmp OP rhs; \
762 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
763 *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
764 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv ) ) \
768 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
769 new_value.cmp = old_value.cmp OP rhs; \
774 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
778 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
779 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
780 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
782 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
785 #define ATOMIC_FLOAT_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
786 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
787 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
789 KMP_TEST_THEN_ADD_REAL##BITS( lhs, OP rhs ); \
792 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
793 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
794 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
795 OP_CMPXCHG(TYPE,BITS,OP) \
799 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
800 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
801 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
802 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
809 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
810 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
811 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
812 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
814 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
817 OP_CRITICAL(OP##=,LCK_ID) \
821 #define ATOMIC_FLOAT_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
822 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
823 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
824 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
825 OP_CMPXCHG(TYPE,BITS,OP) \
828 OP_CRITICAL(OP##=,LCK_ID) \
832 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
833 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
834 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
835 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
836 OP_CMPXCHG(TYPE,BITS,OP) \
839 OP_CRITICAL(OP##=,LCK_ID) \
844 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
845 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
846 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
847 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
848 OP_CMPXCHG(TYPE,BITS,OP) \
851 OP_CRITICAL(OP##=,LCK_ID) \
858 ATOMIC_FIXED_ADD( fixed4, add, kmp_int32, 32, +, 4i, 3, 0 )
859 ATOMIC_FIXED_ADD( fixed4, sub, kmp_int32, 32, -, 4i, 3, 0 )
862 ATOMIC_CMPXCHG( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
863 ATOMIC_CMPXCHG( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
865 ATOMIC_FLOAT_ADD( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
866 ATOMIC_FLOAT_ADD( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
870 ATOMIC_FIXED_ADD( fixed8, add, kmp_int64, 64, +, 8i, 7, KMP_ARCH_X86 )
871 ATOMIC_FIXED_ADD( fixed8, sub, kmp_int64, 64, -, 8i, 7, KMP_ARCH_X86 )
874 ATOMIC_CMPXCHG( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
875 ATOMIC_CMPXCHG( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
877 ATOMIC_FLOAT_ADD( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
878 ATOMIC_FLOAT_ADD( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
896 ATOMIC_CMPXCHG( fixed1, add, kmp_int8, 8, +, 1i, 0, KMP_ARCH_X86 )
897 ATOMIC_CMPXCHG( fixed1, andb, kmp_int8, 8, &, 1i, 0, 0 )
898 ATOMIC_CMPXCHG( fixed1, div, kmp_int8, 8, /, 1i, 0, KMP_ARCH_X86 )
899 ATOMIC_CMPXCHG( fixed1u, div, kmp_uint8, 8, /, 1i, 0, KMP_ARCH_X86 )
900 ATOMIC_CMPXCHG( fixed1, mul, kmp_int8, 8, *, 1i, 0, KMP_ARCH_X86 )
901 ATOMIC_CMPXCHG( fixed1, orb, kmp_int8, 8, |, 1i, 0, 0 )
902 ATOMIC_CMPXCHG( fixed1, shl, kmp_int8, 8, <<, 1i, 0, KMP_ARCH_X86 )
903 ATOMIC_CMPXCHG( fixed1, shr, kmp_int8, 8, >>, 1i, 0, KMP_ARCH_X86 )
904 ATOMIC_CMPXCHG( fixed1u, shr, kmp_uint8, 8, >>, 1i, 0, KMP_ARCH_X86 )
905 ATOMIC_CMPXCHG( fixed1, sub, kmp_int8, 8, -, 1i, 0, KMP_ARCH_X86 )
906 ATOMIC_CMPXCHG( fixed1, xor, kmp_int8, 8, ^, 1i, 0, 0 )
907 ATOMIC_CMPXCHG( fixed2, add, kmp_int16, 16, +, 2i, 1, KMP_ARCH_X86 )
908 ATOMIC_CMPXCHG( fixed2, andb, kmp_int16, 16, &, 2i, 1, 0 )
909 ATOMIC_CMPXCHG( fixed2, div, kmp_int16, 16, /, 2i, 1, KMP_ARCH_X86 )
910 ATOMIC_CMPXCHG( fixed2u, div, kmp_uint16, 16, /, 2i, 1, KMP_ARCH_X86 )
911 ATOMIC_CMPXCHG( fixed2, mul, kmp_int16, 16, *, 2i, 1, KMP_ARCH_X86 )
912 ATOMIC_CMPXCHG( fixed2, orb, kmp_int16, 16, |, 2i, 1, 0 )
913 ATOMIC_CMPXCHG( fixed2, shl, kmp_int16, 16, <<, 2i, 1, KMP_ARCH_X86 )
914 ATOMIC_CMPXCHG( fixed2, shr, kmp_int16, 16, >>, 2i, 1, KMP_ARCH_X86 )
915 ATOMIC_CMPXCHG( fixed2u, shr, kmp_uint16, 16, >>, 2i, 1, KMP_ARCH_X86 )
916 ATOMIC_CMPXCHG( fixed2, sub, kmp_int16, 16, -, 2i, 1, KMP_ARCH_X86 )
917 ATOMIC_CMPXCHG( fixed2, xor, kmp_int16, 16, ^, 2i, 1, 0 )
918 ATOMIC_CMPXCHG( fixed4, andb, kmp_int32, 32, &, 4i, 3, 0 )
919 ATOMIC_CMPXCHG( fixed4, div, kmp_int32, 32, /, 4i, 3, KMP_ARCH_X86 )
920 ATOMIC_CMPXCHG( fixed4u, div, kmp_uint32, 32, /, 4i, 3, KMP_ARCH_X86 )
921 ATOMIC_CMPXCHG( fixed4, mul, kmp_int32, 32, *, 4i, 3, KMP_ARCH_X86 )
922 ATOMIC_CMPXCHG( fixed4, orb, kmp_int32, 32, |, 4i, 3, 0 )
923 ATOMIC_CMPXCHG( fixed4, shl, kmp_int32, 32, <<, 4i, 3, KMP_ARCH_X86 )
924 ATOMIC_CMPXCHG( fixed4, shr, kmp_int32, 32, >>, 4i, 3, KMP_ARCH_X86 )
925 ATOMIC_CMPXCHG( fixed4u, shr, kmp_uint32, 32, >>, 4i, 3, KMP_ARCH_X86 )
926 ATOMIC_CMPXCHG( fixed4, xor, kmp_int32, 32, ^, 4i, 3, 0 )
927 ATOMIC_CMPXCHG( fixed8, andb, kmp_int64, 64, &, 8i, 7, KMP_ARCH_X86 )
928 ATOMIC_CMPXCHG( fixed8, div, kmp_int64, 64, /, 8i, 7, KMP_ARCH_X86 )
929 ATOMIC_CMPXCHG( fixed8u, div, kmp_uint64, 64, /, 8i, 7, KMP_ARCH_X86 )
930 ATOMIC_CMPXCHG( fixed8, mul, kmp_int64, 64, *, 8i, 7, KMP_ARCH_X86 )
931 ATOMIC_CMPXCHG( fixed8, orb, kmp_int64, 64, |, 8i, 7, KMP_ARCH_X86 )
932 ATOMIC_CMPXCHG( fixed8, shl, kmp_int64, 64, <<, 8i, 7, KMP_ARCH_X86 )
933 ATOMIC_CMPXCHG( fixed8, shr, kmp_int64, 64, >>, 8i, 7, KMP_ARCH_X86 )
934 ATOMIC_CMPXCHG( fixed8u, shr, kmp_uint64, 64, >>, 8i, 7, KMP_ARCH_X86 )
935 ATOMIC_CMPXCHG( fixed8, xor, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
936 ATOMIC_CMPXCHG( float4, div, kmp_real32, 32, /, 4r, 3, KMP_ARCH_X86 )
937 ATOMIC_CMPXCHG( float4, mul, kmp_real32, 32, *, 4r, 3, KMP_ARCH_X86 )
938 ATOMIC_CMPXCHG( float8, div, kmp_real64, 64, /, 8r, 7, KMP_ARCH_X86 )
939 ATOMIC_CMPXCHG( float8, mul, kmp_real64, 64, *, 8r, 7, KMP_ARCH_X86 )
950 #define ATOMIC_CRIT_L(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
951 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
952 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
953 OP_CRITICAL( = *lhs OP, LCK_ID ) \
956 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
960 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
961 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
962 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
963 OP_CMPXCHG(TYPE,BITS,OP) \
969 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
970 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
971 OP_GOMP_CRITICAL(= *lhs OP,GOMP_FLAG) \
972 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
973 OP_CMPXCHG(TYPE,BITS,OP) \
976 OP_CRITICAL(= *lhs OP,LCK_ID) \
981 ATOMIC_CMPX_L( fixed1, andl,
char, 8, &&, 1i, 0, KMP_ARCH_X86 )
982 ATOMIC_CMPX_L( fixed1, orl,
char, 8, ||, 1i, 0, KMP_ARCH_X86 )
983 ATOMIC_CMPX_L( fixed2, andl,
short, 16, &&, 2i, 1, KMP_ARCH_X86 )
984 ATOMIC_CMPX_L( fixed2, orl,
short, 16, ||, 2i, 1, KMP_ARCH_X86 )
985 ATOMIC_CMPX_L( fixed4, andl, kmp_int32, 32, &&, 4i, 3, 0 )
986 ATOMIC_CMPX_L( fixed4, orl, kmp_int32, 32, ||, 4i, 3, 0 )
987 ATOMIC_CMPX_L( fixed8, andl, kmp_int64, 64, &&, 8i, 7, KMP_ARCH_X86 )
988 ATOMIC_CMPX_L( fixed8, orl, kmp_int64, 64, ||, 8i, 7, KMP_ARCH_X86 )
1001 #define MIN_MAX_CRITSECT(OP,LCK_ID) \
1002 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1004 if ( *lhs OP rhs ) { \
1007 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1010 #ifdef KMP_GOMP_COMPAT
1011 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) \
1012 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
1014 MIN_MAX_CRITSECT( OP, 0 ); \
1018 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG)
1022 #define MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1024 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1027 old_value = temp_val; \
1028 while ( old_value OP rhs && \
1029 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1030 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1031 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
1035 old_value = temp_val; \
1041 #define MIN_MAX_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1042 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1043 if ( *lhs OP rhs ) { \
1044 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1045 MIN_MAX_CRITSECT(OP,LCK_ID) \
1049 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1053 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1054 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1055 if ( *lhs OP rhs ) { \
1056 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1057 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1064 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1065 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1066 if ( *lhs OP rhs ) { \
1067 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
1068 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1069 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
1072 MIN_MAX_CRITSECT(OP,LCK_ID) \
1078 MIN_MAX_COMPXCHG( fixed1, max,
char, 8, <, 1i, 0, KMP_ARCH_X86 )
1079 MIN_MAX_COMPXCHG( fixed1, min,
char, 8, >, 1i, 0, KMP_ARCH_X86 )
1080 MIN_MAX_COMPXCHG( fixed2, max,
short, 16, <, 2i, 1, KMP_ARCH_X86 )
1081 MIN_MAX_COMPXCHG( fixed2, min,
short, 16, >, 2i, 1, KMP_ARCH_X86 )
1082 MIN_MAX_COMPXCHG( fixed4, max, kmp_int32, 32, <, 4i, 3, 0 )
1083 MIN_MAX_COMPXCHG( fixed4, min, kmp_int32, 32, >, 4i, 3, 0 )
1084 MIN_MAX_COMPXCHG( fixed8, max, kmp_int64, 64, <, 8i, 7, KMP_ARCH_X86 )
1085 MIN_MAX_COMPXCHG( fixed8, min, kmp_int64, 64, >, 8i, 7, KMP_ARCH_X86 )
1086 MIN_MAX_COMPXCHG( float4, max, kmp_real32, 32, <, 4r, 3, KMP_ARCH_X86 )
1087 MIN_MAX_COMPXCHG( float4, min, kmp_real32, 32, >, 4r, 3, KMP_ARCH_X86 )
1088 MIN_MAX_COMPXCHG( float8, max, kmp_real64, 64, <, 8r, 7, KMP_ARCH_X86 )
1089 MIN_MAX_COMPXCHG( float8, min, kmp_real64, 64, >, 8r, 7, KMP_ARCH_X86 )
1090 MIN_MAX_CRITICAL( float16, max, QUAD_LEGACY, <, 16r, 1 )
1091 MIN_MAX_CRITICAL( float16, min, QUAD_LEGACY, >, 16r, 1 )
1092 #if ( KMP_ARCH_X86 )
1093 MIN_MAX_CRITICAL( float16, max_a16, Quad_a16_t, <, 16r, 1 )
1094 MIN_MAX_CRITICAL( float16, min_a16, Quad_a16_t, >, 16r, 1 )
1099 #define ATOMIC_CRIT_EQV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1100 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1101 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1102 OP_CRITICAL(^=~,LCK_ID) \
1106 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1109 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1110 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1111 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1112 OP_CMPXCHG(TYPE,BITS,OP) \
1118 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
1119 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1120 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
1121 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1122 OP_CMPXCHG(TYPE,BITS,OP) \
1125 OP_CRITICAL(^=~,LCK_ID) \
1130 ATOMIC_CMPXCHG( fixed1, neqv, kmp_int8, 8, ^, 1i, 0, KMP_ARCH_X86 )
1131 ATOMIC_CMPXCHG( fixed2, neqv, kmp_int16, 16, ^, 2i, 1, KMP_ARCH_X86 )
1132 ATOMIC_CMPXCHG( fixed4, neqv, kmp_int32, 32, ^, 4i, 3, KMP_ARCH_X86 )
1133 ATOMIC_CMPXCHG( fixed8, neqv, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
1134 ATOMIC_CMPX_EQV( fixed1, eqv, kmp_int8, 8, ^~, 1i, 0, KMP_ARCH_X86 )
1135 ATOMIC_CMPX_EQV( fixed2, eqv, kmp_int16, 16, ^~, 2i, 1, KMP_ARCH_X86 )
1136 ATOMIC_CMPX_EQV( fixed4, eqv, kmp_int32, 32, ^~, 4i, 3, KMP_ARCH_X86 )
1137 ATOMIC_CMPX_EQV( fixed8, eqv, kmp_int64, 64, ^~, 8i, 7, KMP_ARCH_X86 )
1145 #define ATOMIC_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1146 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1147 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1148 OP_CRITICAL(OP##=,LCK_ID) \
1153 ATOMIC_CRITICAL( float10, add,
long double, +, 10r, 1 )
1154 ATOMIC_CRITICAL( float10, sub,
long double, -, 10r, 1 )
1155 ATOMIC_CRITICAL( float10, mul,
long double, *, 10r, 1 )
1156 ATOMIC_CRITICAL( float10, div,
long double, /, 10r, 1 )
1158 ATOMIC_CRITICAL( float16, add, QUAD_LEGACY, +, 16r, 1 )
1159 ATOMIC_CRITICAL( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1160 ATOMIC_CRITICAL( float16, mul, QUAD_LEGACY, *, 16r, 1 )
1161 ATOMIC_CRITICAL( float16, div, QUAD_LEGACY, /, 16r, 1 )
1162 #if ( KMP_ARCH_X86 )
1163 ATOMIC_CRITICAL( float16, add_a16, Quad_a16_t, +, 16r, 1 )
1164 ATOMIC_CRITICAL( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1165 ATOMIC_CRITICAL( float16, mul_a16, Quad_a16_t, *, 16r, 1 )
1166 ATOMIC_CRITICAL( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1171 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, add, kmp_cmplx32, 64, +, 8c, 7, 1 )
1172 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7, 1 )
1173 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7, 1 )
1174 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, div, kmp_cmplx32, 64, /, 8c, 7, 1 )
1177 ATOMIC_CRITICAL( cmplx8, add, kmp_cmplx64, +, 16c, 1 )
1178 ATOMIC_CRITICAL( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1179 ATOMIC_CRITICAL( cmplx8, mul, kmp_cmplx64, *, 16c, 1 )
1180 ATOMIC_CRITICAL( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1181 ATOMIC_CRITICAL( cmplx10, add, kmp_cmplx80, +, 20c, 1 )
1182 ATOMIC_CRITICAL( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1183 ATOMIC_CRITICAL( cmplx10, mul, kmp_cmplx80, *, 20c, 1 )
1184 ATOMIC_CRITICAL( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1185 ATOMIC_CRITICAL( cmplx16, add, CPLX128_LEG, +, 32c, 1 )
1186 ATOMIC_CRITICAL( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1187 ATOMIC_CRITICAL( cmplx16, mul, CPLX128_LEG, *, 32c, 1 )
1188 ATOMIC_CRITICAL( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1189 #if ( KMP_ARCH_X86 )
1190 ATOMIC_CRITICAL( cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c, 1 )
1191 ATOMIC_CRITICAL( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1192 ATOMIC_CRITICAL( cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c, 1 )
1193 ATOMIC_CRITICAL( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1200 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1208 #define OP_CRITICAL_REV(OP,LCK_ID) \
1209 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1211 (*lhs) = (rhs) OP (*lhs); \
1213 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1215 #ifdef KMP_GOMP_COMPAT
1216 #define OP_GOMP_CRITICAL_REV(OP,FLAG) \
1217 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1219 OP_CRITICAL_REV( OP, 0 ); \
1223 #define OP_GOMP_CRITICAL_REV(OP,FLAG)
1231 #define ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1232 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
1234 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1235 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid ));
1244 #define OP_CMPXCHG_REV(TYPE,BITS,OP) \
1246 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1247 TYPE old_value, new_value; \
1249 old_value = temp_val; \
1250 new_value = rhs OP old_value; \
1251 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1252 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1253 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1258 old_value = temp_val; \
1259 new_value = rhs OP old_value; \
1264 #define ATOMIC_CMPXCHG_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,GOMP_FLAG) \
1265 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1266 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1267 OP_CMPXCHG_REV(TYPE,BITS,OP) \
1284 ATOMIC_CMPXCHG_REV( fixed1, div, kmp_int8, 8, /, 1i, KMP_ARCH_X86 )
1285 ATOMIC_CMPXCHG_REV( fixed1u, div, kmp_uint8, 8, /, 1i, KMP_ARCH_X86 )
1286 ATOMIC_CMPXCHG_REV( fixed1, shl, kmp_int8, 8, <<, 1i, KMP_ARCH_X86 )
1287 ATOMIC_CMPXCHG_REV( fixed1, shr, kmp_int8, 8, >>, 1i, KMP_ARCH_X86 )
1288 ATOMIC_CMPXCHG_REV( fixed1u, shr, kmp_uint8, 8, >>, 1i, KMP_ARCH_X86 )
1289 ATOMIC_CMPXCHG_REV( fixed1, sub, kmp_int8, 8, -, 1i, KMP_ARCH_X86 )
1291 ATOMIC_CMPXCHG_REV( fixed2, div, kmp_int16, 16, /, 2i, KMP_ARCH_X86 )
1292 ATOMIC_CMPXCHG_REV( fixed2u, div, kmp_uint16, 16, /, 2i, KMP_ARCH_X86 )
1293 ATOMIC_CMPXCHG_REV( fixed2, shl, kmp_int16, 16, <<, 2i, KMP_ARCH_X86 )
1294 ATOMIC_CMPXCHG_REV( fixed2, shr, kmp_int16, 16, >>, 2i, KMP_ARCH_X86 )
1295 ATOMIC_CMPXCHG_REV( fixed2u, shr, kmp_uint16, 16, >>, 2i, KMP_ARCH_X86 )
1296 ATOMIC_CMPXCHG_REV( fixed2, sub, kmp_int16, 16, -, 2i, KMP_ARCH_X86 )
1298 ATOMIC_CMPXCHG_REV( fixed4, div, kmp_int32, 32, /, 4i, KMP_ARCH_X86 )
1299 ATOMIC_CMPXCHG_REV( fixed4u, div, kmp_uint32, 32, /, 4i, KMP_ARCH_X86 )
1300 ATOMIC_CMPXCHG_REV( fixed4, shl, kmp_int32, 32, <<, 4i, KMP_ARCH_X86 )
1301 ATOMIC_CMPXCHG_REV( fixed4, shr, kmp_int32, 32, >>, 4i, KMP_ARCH_X86 )
1302 ATOMIC_CMPXCHG_REV( fixed4u, shr, kmp_uint32, 32, >>, 4i, KMP_ARCH_X86 )
1303 ATOMIC_CMPXCHG_REV( fixed4, sub, kmp_int32, 32, -, 4i, KMP_ARCH_X86 )
1305 ATOMIC_CMPXCHG_REV( fixed8, div, kmp_int64, 64, /, 8i, KMP_ARCH_X86 )
1306 ATOMIC_CMPXCHG_REV( fixed8u, div, kmp_uint64, 64, /, 8i, KMP_ARCH_X86 )
1307 ATOMIC_CMPXCHG_REV( fixed8, shl, kmp_int64, 64, <<, 8i, KMP_ARCH_X86 )
1308 ATOMIC_CMPXCHG_REV( fixed8, shr, kmp_int64, 64, >>, 8i, KMP_ARCH_X86 )
1309 ATOMIC_CMPXCHG_REV( fixed8u, shr, kmp_uint64, 64, >>, 8i, KMP_ARCH_X86 )
1310 ATOMIC_CMPXCHG_REV( fixed8, sub, kmp_int64, 64, -, 8i, KMP_ARCH_X86 )
1312 ATOMIC_CMPXCHG_REV( float4, div, kmp_real32, 32, /, 4r, KMP_ARCH_X86 )
1313 ATOMIC_CMPXCHG_REV( float4, sub, kmp_real32, 32, -, 4r, KMP_ARCH_X86 )
1315 ATOMIC_CMPXCHG_REV( float8, div, kmp_real64, 64, /, 8r, KMP_ARCH_X86 )
1316 ATOMIC_CMPXCHG_REV( float8, sub, kmp_real64, 64, -, 8r, KMP_ARCH_X86 )
1324 #define ATOMIC_CRITICAL_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1325 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
1326 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
1327 OP_CRITICAL_REV(OP,LCK_ID) \
1332 ATOMIC_CRITICAL_REV( float10, sub,
long double, -, 10r, 1 )
1333 ATOMIC_CRITICAL_REV( float10, div,
long double, /, 10r, 1 )
1335 ATOMIC_CRITICAL_REV( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1336 ATOMIC_CRITICAL_REV( float16, div, QUAD_LEGACY, /, 16r, 1 )
1337 #if ( KMP_ARCH_X86 )
1338 ATOMIC_CRITICAL_REV( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1339 ATOMIC_CRITICAL_REV( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1343 ATOMIC_CRITICAL_REV( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1344 ATOMIC_CRITICAL_REV( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1345 ATOMIC_CRITICAL_REV( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1346 ATOMIC_CRITICAL_REV( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1347 ATOMIC_CRITICAL_REV( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1348 ATOMIC_CRITICAL_REV( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1349 ATOMIC_CRITICAL_REV( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1350 ATOMIC_CRITICAL_REV( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1351 #if ( KMP_ARCH_X86 )
1352 ATOMIC_CRITICAL_REV( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1353 ATOMIC_CRITICAL_REV( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1357 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
1360 #endif //OMP_40_ENABLED
1374 #define ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1375 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs ) \
1377 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1378 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid ));
1381 #define ATOMIC_CRITICAL_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
1382 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1383 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1384 OP_CRITICAL(OP##=,LCK_ID) \
1388 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1391 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1392 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1393 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1394 OP_CMPXCHG(TYPE,BITS,OP) \
1400 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1401 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1402 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1403 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1404 OP_CMPXCHG(TYPE,BITS,OP) \
1407 OP_CRITICAL(OP##=,LCK_ID) \
1413 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1414 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1415 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1416 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1417 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3, 0 )
1418 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3, 0 )
1419 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1420 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1421 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1422 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1423 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1424 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1428 ATOMIC_CMPXCHG_MIX( fixed1,
char, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1429 ATOMIC_CMPXCHG_MIX( fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1430 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1431 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1432 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1434 ATOMIC_CMPXCHG_MIX( fixed2,
short, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1435 ATOMIC_CMPXCHG_MIX( fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1436 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1437 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1438 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1440 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3, 0 )
1441 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3, 0 )
1442 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3, 0 )
1443 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1444 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1446 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1447 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1448 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1449 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1450 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1452 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1453 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1454 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1455 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1457 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1458 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1459 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1460 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1462 ATOMIC_CRITICAL_FP( float10,
long double, add, +, fp, _Quad, 10r, 1 )
1463 ATOMIC_CRITICAL_FP( float10,
long double, sub, -, fp, _Quad, 10r, 1 )
1464 ATOMIC_CRITICAL_FP( float10,
long double, mul, *, fp, _Quad, 10r, 1 )
1465 ATOMIC_CRITICAL_FP( float10,
long double, div, /, fp, _Quad, 10r, 1 )
1467 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1471 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1472 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1473 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1474 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
1480 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
1481 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
1482 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
1483 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
1484 OP_CMPXCHG(TYPE,BITS,OP) \
1487 OP_CRITICAL(OP##=,LCK_ID) \
1492 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1493 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1494 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1495 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1498 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1510 #define ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
1511 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * loc ) \
1513 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1514 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1526 #define OP_CMPXCHG_READ(TYPE,BITS,OP) \
1528 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1531 kmp_int##BITS i_val; \
1533 union f_i_union old_value; \
1535 old_value.f_val = temp_val; \
1536 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( (kmp_int##BITS *) loc, \
1537 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val, \
1538 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val ); \
1539 new_value = old_value.f_val; \
1549 #define OP_CRITICAL_READ(OP,LCK_ID) \
1550 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1552 new_value = (*loc); \
1554 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1557 #ifdef KMP_GOMP_COMPAT
1558 #define OP_GOMP_CRITICAL_READ(OP,FLAG) \
1559 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1561 OP_CRITICAL_READ( OP, 0 ); \
1565 #define OP_GOMP_CRITICAL_READ(OP,FLAG)
1569 #define ATOMIC_FIXED_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1570 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1572 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1573 new_value = KMP_TEST_THEN_ADD##BITS( loc, OP 0 ); \
1577 #define ATOMIC_CMPXCHG_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1578 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1580 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1581 OP_CMPXCHG_READ(TYPE,BITS,OP) \
1588 #define ATOMIC_CRITICAL_READ(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1589 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
1591 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
1592 OP_CRITICAL_READ(OP,LCK_ID) \
1600 #if ( KMP_OS_WINDOWS )
1602 #define OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1603 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1607 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1609 #ifdef KMP_GOMP_COMPAT
1610 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) \
1611 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1613 OP_CRITICAL_READ_WRK( OP, 0 ); \
1616 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG)
1619 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1620 void __kmpc_atomic_##TYPE_ID##_##OP_ID( TYPE * out, ident_t *id_ref, int gtid, TYPE * loc ) \
1622 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1623 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1626 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1627 ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
1628 OP_GOMP_CRITICAL_READ_WRK(OP##=,GOMP_FLAG) \
1629 OP_CRITICAL_READ_WRK(OP,LCK_ID) \
1632 #endif // KMP_OS_WINDOWS
1636 ATOMIC_FIXED_READ( fixed4, rd, kmp_int32, 32, +, 0 )
1637 ATOMIC_FIXED_READ( fixed8, rd, kmp_int64, 64, +, KMP_ARCH_X86 )
1638 ATOMIC_CMPXCHG_READ( float4, rd, kmp_real32, 32, +, KMP_ARCH_X86 )
1639 ATOMIC_CMPXCHG_READ( float8, rd, kmp_real64, 64, +, KMP_ARCH_X86 )
1642 ATOMIC_CMPXCHG_READ( fixed1, rd, kmp_int8, 8, +, KMP_ARCH_X86 )
1643 ATOMIC_CMPXCHG_READ( fixed2, rd, kmp_int16, 16, +, KMP_ARCH_X86 )
1645 ATOMIC_CRITICAL_READ( float10, rd,
long double, +, 10r, 1 )
1646 ATOMIC_CRITICAL_READ( float16, rd, QUAD_LEGACY, +, 16r, 1 )
1649 #if ( KMP_OS_WINDOWS )
1650 ATOMIC_CRITICAL_READ_WRK( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1652 ATOMIC_CRITICAL_READ( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1654 ATOMIC_CRITICAL_READ( cmplx8, rd, kmp_cmplx64, +, 16c, 1 )
1655 ATOMIC_CRITICAL_READ( cmplx10, rd, kmp_cmplx80, +, 20c, 1 )
1656 ATOMIC_CRITICAL_READ( cmplx16, rd, CPLX128_LEG, +, 32c, 1 )
1657 #if ( KMP_ARCH_X86 )
1658 ATOMIC_CRITICAL_READ( float16, a16_rd, Quad_a16_t, +, 16r, 1 )
1659 ATOMIC_CRITICAL_READ( cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c, 1 )
1667 #define ATOMIC_XCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1668 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1669 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1670 KMP_XCHG_FIXED##BITS( lhs, rhs ); \
1673 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1674 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1675 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1676 KMP_XCHG_REAL##BITS( lhs, rhs ); \
1687 #define OP_CMPXCHG_WR(TYPE,BITS,OP) \
1689 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1690 TYPE old_value, new_value; \
1692 old_value = temp_val; \
1694 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1695 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1696 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1701 old_value = temp_val; \
1707 #define ATOMIC_CMPXCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1708 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1709 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1710 OP_CMPXCHG_WR(TYPE,BITS,OP) \
1718 #define ATOMIC_CRITICAL_WR(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
1719 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
1720 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
1721 OP_CRITICAL(OP,LCK_ID) \
1725 ATOMIC_XCHG_WR( fixed1, wr, kmp_int8, 8, =, KMP_ARCH_X86 )
1726 ATOMIC_XCHG_WR( fixed2, wr, kmp_int16, 16, =, KMP_ARCH_X86 )
1727 ATOMIC_XCHG_WR( fixed4, wr, kmp_int32, 32, =, KMP_ARCH_X86 )
1728 #if ( KMP_ARCH_X86 )
1729 ATOMIC_CMPXCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1731 ATOMIC_XCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1734 ATOMIC_XCHG_FLOAT_WR( float4, wr, kmp_real32, 32, =, KMP_ARCH_X86 )
1735 #if ( KMP_ARCH_X86 )
1736 ATOMIC_CMPXCHG_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1738 ATOMIC_XCHG_FLOAT_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1741 ATOMIC_CRITICAL_WR( float10, wr,
long double, =, 10r, 1 )
1742 ATOMIC_CRITICAL_WR( float16, wr, QUAD_LEGACY, =, 16r, 1 )
1743 ATOMIC_CRITICAL_WR( cmplx4, wr, kmp_cmplx32, =, 8c, 1 )
1744 ATOMIC_CRITICAL_WR( cmplx8, wr, kmp_cmplx64, =, 16c, 1 )
1745 ATOMIC_CRITICAL_WR( cmplx10, wr, kmp_cmplx80, =, 20c, 1 )
1746 ATOMIC_CRITICAL_WR( cmplx16, wr, CPLX128_LEG, =, 32c, 1 )
1747 #if ( KMP_ARCH_X86 )
1748 ATOMIC_CRITICAL_WR( float16, a16_wr, Quad_a16_t, =, 16r, 1 )
1749 ATOMIC_CRITICAL_WR( cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c, 1 )
1761 #define ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,RET_TYPE) \
1762 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag ) \
1764 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
1765 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
1773 #define OP_CRITICAL_CPT(OP,LCK_ID) \
1774 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1778 new_value = (*lhs); \
1780 new_value = (*lhs); \
1784 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1788 #ifdef KMP_GOMP_COMPAT
1789 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) \
1790 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1792 OP_CRITICAL_CPT( OP##=, 0 ); \
1795 #define OP_GOMP_CRITICAL_CPT(OP,FLAG)
1805 #define OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1807 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1808 TYPE old_value, new_value; \
1810 old_value = temp_val; \
1811 new_value = old_value OP rhs; \
1812 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
1813 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
1814 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
1819 old_value = temp_val; \
1820 new_value = old_value OP rhs; \
1829 #define ATOMIC_CMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1830 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1832 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1833 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1837 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1838 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1839 TYPE old_value, new_value; \
1840 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1842 old_value = KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
1844 return old_value OP rhs; \
1849 #define ATOMIC_FLOAT_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1850 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1851 TYPE old_value, new_value; \
1852 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
1854 old_value = KMP_TEST_THEN_ADD_REAL##BITS( lhs, OP rhs ); \
1856 return old_value OP rhs; \
1862 ATOMIC_FIXED_ADD_CPT( fixed4, add_cpt, kmp_int32, 32, +, 0 )
1863 ATOMIC_FIXED_ADD_CPT( fixed4, sub_cpt, kmp_int32, 32, -, 0 )
1864 ATOMIC_FIXED_ADD_CPT( fixed8, add_cpt, kmp_int64, 64, +, KMP_ARCH_X86 )
1865 ATOMIC_FIXED_ADD_CPT( fixed8, sub_cpt, kmp_int64, 64, -, KMP_ARCH_X86 )
1868 ATOMIC_CMPXCHG_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1869 ATOMIC_CMPXCHG_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1870 ATOMIC_CMPXCHG_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1871 ATOMIC_CMPXCHG_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1873 ATOMIC_FLOAT_ADD_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1874 ATOMIC_FLOAT_ADD_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1875 ATOMIC_FLOAT_ADD_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1876 ATOMIC_FLOAT_ADD_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1891 ATOMIC_CMPXCHG_CPT( fixed1, add_cpt, kmp_int8, 8, +, KMP_ARCH_X86 )
1892 ATOMIC_CMPXCHG_CPT( fixed1, andb_cpt, kmp_int8, 8, &, 0 )
1893 ATOMIC_CMPXCHG_CPT( fixed1, div_cpt, kmp_int8, 8, /, KMP_ARCH_X86 )
1894 ATOMIC_CMPXCHG_CPT( fixed1u, div_cpt, kmp_uint8, 8, /, KMP_ARCH_X86 )
1895 ATOMIC_CMPXCHG_CPT( fixed1, mul_cpt, kmp_int8, 8, *, KMP_ARCH_X86 )
1896 ATOMIC_CMPXCHG_CPT( fixed1, orb_cpt, kmp_int8, 8, |, 0 )
1897 ATOMIC_CMPXCHG_CPT( fixed1, shl_cpt, kmp_int8, 8, <<, KMP_ARCH_X86 )
1898 ATOMIC_CMPXCHG_CPT( fixed1, shr_cpt, kmp_int8, 8, >>, KMP_ARCH_X86 )
1899 ATOMIC_CMPXCHG_CPT( fixed1u, shr_cpt, kmp_uint8, 8, >>, KMP_ARCH_X86 )
1900 ATOMIC_CMPXCHG_CPT( fixed1, sub_cpt, kmp_int8, 8, -, KMP_ARCH_X86 )
1901 ATOMIC_CMPXCHG_CPT( fixed1, xor_cpt, kmp_int8, 8, ^, 0 )
1902 ATOMIC_CMPXCHG_CPT( fixed2, add_cpt, kmp_int16, 16, +, KMP_ARCH_X86 )
1903 ATOMIC_CMPXCHG_CPT( fixed2, andb_cpt, kmp_int16, 16, &, 0 )
1904 ATOMIC_CMPXCHG_CPT( fixed2, div_cpt, kmp_int16, 16, /, KMP_ARCH_X86 )
1905 ATOMIC_CMPXCHG_CPT( fixed2u, div_cpt, kmp_uint16, 16, /, KMP_ARCH_X86 )
1906 ATOMIC_CMPXCHG_CPT( fixed2, mul_cpt, kmp_int16, 16, *, KMP_ARCH_X86 )
1907 ATOMIC_CMPXCHG_CPT( fixed2, orb_cpt, kmp_int16, 16, |, 0 )
1908 ATOMIC_CMPXCHG_CPT( fixed2, shl_cpt, kmp_int16, 16, <<, KMP_ARCH_X86 )
1909 ATOMIC_CMPXCHG_CPT( fixed2, shr_cpt, kmp_int16, 16, >>, KMP_ARCH_X86 )
1910 ATOMIC_CMPXCHG_CPT( fixed2u, shr_cpt, kmp_uint16, 16, >>, KMP_ARCH_X86 )
1911 ATOMIC_CMPXCHG_CPT( fixed2, sub_cpt, kmp_int16, 16, -, KMP_ARCH_X86 )
1912 ATOMIC_CMPXCHG_CPT( fixed2, xor_cpt, kmp_int16, 16, ^, 0 )
1913 ATOMIC_CMPXCHG_CPT( fixed4, andb_cpt, kmp_int32, 32, &, 0 )
1914 ATOMIC_CMPXCHG_CPT( fixed4, div_cpt, kmp_int32, 32, /, KMP_ARCH_X86 )
1915 ATOMIC_CMPXCHG_CPT( fixed4u, div_cpt, kmp_uint32, 32, /, KMP_ARCH_X86 )
1916 ATOMIC_CMPXCHG_CPT( fixed4, mul_cpt, kmp_int32, 32, *, KMP_ARCH_X86 )
1917 ATOMIC_CMPXCHG_CPT( fixed4, orb_cpt, kmp_int32, 32, |, 0 )
1918 ATOMIC_CMPXCHG_CPT( fixed4, shl_cpt, kmp_int32, 32, <<, KMP_ARCH_X86 )
1919 ATOMIC_CMPXCHG_CPT( fixed4, shr_cpt, kmp_int32, 32, >>, KMP_ARCH_X86 )
1920 ATOMIC_CMPXCHG_CPT( fixed4u, shr_cpt, kmp_uint32, 32, >>, KMP_ARCH_X86 )
1921 ATOMIC_CMPXCHG_CPT( fixed4, xor_cpt, kmp_int32, 32, ^, 0 )
1922 ATOMIC_CMPXCHG_CPT( fixed8, andb_cpt, kmp_int64, 64, &, KMP_ARCH_X86 )
1923 ATOMIC_CMPXCHG_CPT( fixed8, div_cpt, kmp_int64, 64, /, KMP_ARCH_X86 )
1924 ATOMIC_CMPXCHG_CPT( fixed8u, div_cpt, kmp_uint64, 64, /, KMP_ARCH_X86 )
1925 ATOMIC_CMPXCHG_CPT( fixed8, mul_cpt, kmp_int64, 64, *, KMP_ARCH_X86 )
1926 ATOMIC_CMPXCHG_CPT( fixed8, orb_cpt, kmp_int64, 64, |, KMP_ARCH_X86 )
1927 ATOMIC_CMPXCHG_CPT( fixed8, shl_cpt, kmp_int64, 64, <<, KMP_ARCH_X86 )
1928 ATOMIC_CMPXCHG_CPT( fixed8, shr_cpt, kmp_int64, 64, >>, KMP_ARCH_X86 )
1929 ATOMIC_CMPXCHG_CPT( fixed8u, shr_cpt, kmp_uint64, 64, >>, KMP_ARCH_X86 )
1930 ATOMIC_CMPXCHG_CPT( fixed8, xor_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
1931 ATOMIC_CMPXCHG_CPT( float4, div_cpt, kmp_real32, 32, /, KMP_ARCH_X86 )
1932 ATOMIC_CMPXCHG_CPT( float4, mul_cpt, kmp_real32, 32, *, KMP_ARCH_X86 )
1933 ATOMIC_CMPXCHG_CPT( float8, div_cpt, kmp_real64, 64, /, KMP_ARCH_X86 )
1934 ATOMIC_CMPXCHG_CPT( float8, mul_cpt, kmp_real64, 64, *, KMP_ARCH_X86 )
1947 #define OP_CRITICAL_L_CPT(OP,LCK_ID) \
1948 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
1953 new_value = (*lhs); \
1955 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
1958 #ifdef KMP_GOMP_COMPAT
1959 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) \
1960 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
1962 OP_CRITICAL_L_CPT( OP, 0 ); \
1966 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG)
1971 #define ATOMIC_CMPX_L_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
1972 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
1974 OP_GOMP_CRITICAL_L_CPT( = *lhs OP, GOMP_FLAG ) \
1975 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
1978 ATOMIC_CMPX_L_CPT( fixed1, andl_cpt,
char, 8, &&, KMP_ARCH_X86 )
1979 ATOMIC_CMPX_L_CPT( fixed1, orl_cpt,
char, 8, ||, KMP_ARCH_X86 )
1980 ATOMIC_CMPX_L_CPT( fixed2, andl_cpt,
short, 16, &&, KMP_ARCH_X86 )
1981 ATOMIC_CMPX_L_CPT( fixed2, orl_cpt,
short, 16, ||, KMP_ARCH_X86 )
1982 ATOMIC_CMPX_L_CPT( fixed4, andl_cpt, kmp_int32, 32, &&, 0 )
1983 ATOMIC_CMPX_L_CPT( fixed4, orl_cpt, kmp_int32, 32, ||, 0 )
1984 ATOMIC_CMPX_L_CPT( fixed8, andl_cpt, kmp_int64, 64, &&, KMP_ARCH_X86 )
1985 ATOMIC_CMPX_L_CPT( fixed8, orl_cpt, kmp_int64, 64, ||, KMP_ARCH_X86 )
1998 #define MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
1999 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2001 if ( *lhs OP rhs ) { \
2007 new_value = old_value; \
2009 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2013 #ifdef KMP_GOMP_COMPAT
2014 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) \
2015 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
2017 MIN_MAX_CRITSECT_CPT( OP, 0 ); \
2020 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG)
2024 #define MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2026 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2029 old_value = temp_val; \
2030 while ( old_value OP rhs && \
2031 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2032 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2033 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
2037 old_value = temp_val; \
2047 #define MIN_MAX_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2048 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2049 TYPE new_value, old_value; \
2050 if ( *lhs OP rhs ) { \
2051 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2052 MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
2057 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2058 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2059 TYPE new_value, old_value; \
2060 if ( *lhs OP rhs ) { \
2061 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
2062 MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
2068 MIN_MAX_COMPXCHG_CPT( fixed1, max_cpt,
char, 8, <, KMP_ARCH_X86 )
2069 MIN_MAX_COMPXCHG_CPT( fixed1, min_cpt,
char, 8, >, KMP_ARCH_X86 )
2070 MIN_MAX_COMPXCHG_CPT( fixed2, max_cpt,
short, 16, <, KMP_ARCH_X86 )
2071 MIN_MAX_COMPXCHG_CPT( fixed2, min_cpt,
short, 16, >, KMP_ARCH_X86 )
2072 MIN_MAX_COMPXCHG_CPT( fixed4, max_cpt, kmp_int32, 32, <, 0 )
2073 MIN_MAX_COMPXCHG_CPT( fixed4, min_cpt, kmp_int32, 32, >, 0 )
2074 MIN_MAX_COMPXCHG_CPT( fixed8, max_cpt, kmp_int64, 64, <, KMP_ARCH_X86 )
2075 MIN_MAX_COMPXCHG_CPT( fixed8, min_cpt, kmp_int64, 64, >, KMP_ARCH_X86 )
2076 MIN_MAX_COMPXCHG_CPT( float4, max_cpt, kmp_real32, 32, <, KMP_ARCH_X86 )
2077 MIN_MAX_COMPXCHG_CPT( float4, min_cpt, kmp_real32, 32, >, KMP_ARCH_X86 )
2078 MIN_MAX_COMPXCHG_CPT( float8, max_cpt, kmp_real64, 64, <, KMP_ARCH_X86 )
2079 MIN_MAX_COMPXCHG_CPT( float8, min_cpt, kmp_real64, 64, >, KMP_ARCH_X86 )
2080 MIN_MAX_CRITICAL_CPT( float16, max_cpt, QUAD_LEGACY, <, 16r, 1 )
2081 MIN_MAX_CRITICAL_CPT( float16, min_cpt, QUAD_LEGACY, >, 16r, 1 )
2082 #if ( KMP_ARCH_X86 )
2083 MIN_MAX_CRITICAL_CPT( float16, max_a16_cpt, Quad_a16_t, <, 16r, 1 )
2084 MIN_MAX_CRITICAL_CPT( float16, min_a16_cpt, Quad_a16_t, >, 16r, 1 )
2088 #ifdef KMP_GOMP_COMPAT
2089 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) \
2090 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2092 OP_CRITICAL_CPT( OP, 0 ); \
2095 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG)
2098 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2099 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2101 OP_GOMP_CRITICAL_EQV_CPT(^=~,GOMP_FLAG) \
2102 OP_CMPXCHG_CPT(TYPE,BITS,OP) \
2107 ATOMIC_CMPXCHG_CPT( fixed1, neqv_cpt, kmp_int8, 8, ^, KMP_ARCH_X86 )
2108 ATOMIC_CMPXCHG_CPT( fixed2, neqv_cpt, kmp_int16, 16, ^, KMP_ARCH_X86 )
2109 ATOMIC_CMPXCHG_CPT( fixed4, neqv_cpt, kmp_int32, 32, ^, KMP_ARCH_X86 )
2110 ATOMIC_CMPXCHG_CPT( fixed8, neqv_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
2111 ATOMIC_CMPX_EQV_CPT( fixed1, eqv_cpt, kmp_int8, 8, ^~, KMP_ARCH_X86 )
2112 ATOMIC_CMPX_EQV_CPT( fixed2, eqv_cpt, kmp_int16, 16, ^~, KMP_ARCH_X86 )
2113 ATOMIC_CMPX_EQV_CPT( fixed4, eqv_cpt, kmp_int32, 32, ^~, KMP_ARCH_X86 )
2114 ATOMIC_CMPX_EQV_CPT( fixed8, eqv_cpt, kmp_int64, 64, ^~, KMP_ARCH_X86 )
2121 #define ATOMIC_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2122 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2124 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
2125 OP_CRITICAL_CPT(OP##=,LCK_ID) \
2132 #define OP_CRITICAL_CPT_WRK(OP,LCK_ID) \
2133 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2143 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2147 #ifdef KMP_GOMP_COMPAT
2148 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) \
2149 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2151 OP_CRITICAL_CPT_WRK( OP##=, 0 ); \
2154 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG)
2158 #define ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2159 void __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out, int flag ) \
2161 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2162 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
2165 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2166 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2167 OP_GOMP_CRITICAL_CPT_WRK(OP,GOMP_FLAG) \
2168 OP_CRITICAL_CPT_WRK(OP##=,LCK_ID) \
2174 ATOMIC_CRITICAL_CPT( float10, add_cpt,
long double, +, 10r, 1 )
2175 ATOMIC_CRITICAL_CPT( float10, sub_cpt,
long double, -, 10r, 1 )
2176 ATOMIC_CRITICAL_CPT( float10, mul_cpt,
long double, *, 10r, 1 )
2177 ATOMIC_CRITICAL_CPT( float10, div_cpt,
long double, /, 10r, 1 )
2179 ATOMIC_CRITICAL_CPT( float16, add_cpt, QUAD_LEGACY, +, 16r, 1 )
2180 ATOMIC_CRITICAL_CPT( float16, sub_cpt, QUAD_LEGACY, -, 16r, 1 )
2181 ATOMIC_CRITICAL_CPT( float16, mul_cpt, QUAD_LEGACY, *, 16r, 1 )
2182 ATOMIC_CRITICAL_CPT( float16, div_cpt, QUAD_LEGACY, /, 16r, 1 )
2183 #if ( KMP_ARCH_X86 )
2184 ATOMIC_CRITICAL_CPT( float16, add_a16_cpt, Quad_a16_t, +, 16r, 1 )
2185 ATOMIC_CRITICAL_CPT( float16, sub_a16_cpt, Quad_a16_t, -, 16r, 1 )
2186 ATOMIC_CRITICAL_CPT( float16, mul_a16_cpt, Quad_a16_t, *, 16r, 1 )
2187 ATOMIC_CRITICAL_CPT( float16, div_a16_cpt, Quad_a16_t, /, 16r, 1 )
2193 ATOMIC_CRITICAL_CPT_WRK( cmplx4, add_cpt, kmp_cmplx32, +, 8c, 1 )
2194 ATOMIC_CRITICAL_CPT_WRK( cmplx4, sub_cpt, kmp_cmplx32, -, 8c, 1 )
2195 ATOMIC_CRITICAL_CPT_WRK( cmplx4, mul_cpt, kmp_cmplx32, *, 8c, 1 )
2196 ATOMIC_CRITICAL_CPT_WRK( cmplx4, div_cpt, kmp_cmplx32, /, 8c, 1 )
2198 ATOMIC_CRITICAL_CPT( cmplx8, add_cpt, kmp_cmplx64, +, 16c, 1 )
2199 ATOMIC_CRITICAL_CPT( cmplx8, sub_cpt, kmp_cmplx64, -, 16c, 1 )
2200 ATOMIC_CRITICAL_CPT( cmplx8, mul_cpt, kmp_cmplx64, *, 16c, 1 )
2201 ATOMIC_CRITICAL_CPT( cmplx8, div_cpt, kmp_cmplx64, /, 16c, 1 )
2202 ATOMIC_CRITICAL_CPT( cmplx10, add_cpt, kmp_cmplx80, +, 20c, 1 )
2203 ATOMIC_CRITICAL_CPT( cmplx10, sub_cpt, kmp_cmplx80, -, 20c, 1 )
2204 ATOMIC_CRITICAL_CPT( cmplx10, mul_cpt, kmp_cmplx80, *, 20c, 1 )
2205 ATOMIC_CRITICAL_CPT( cmplx10, div_cpt, kmp_cmplx80, /, 20c, 1 )
2206 ATOMIC_CRITICAL_CPT( cmplx16, add_cpt, CPLX128_LEG, +, 32c, 1 )
2207 ATOMIC_CRITICAL_CPT( cmplx16, sub_cpt, CPLX128_LEG, -, 32c, 1 )
2208 ATOMIC_CRITICAL_CPT( cmplx16, mul_cpt, CPLX128_LEG, *, 32c, 1 )
2209 ATOMIC_CRITICAL_CPT( cmplx16, div_cpt, CPLX128_LEG, /, 32c, 1 )
2210 #if ( KMP_ARCH_X86 )
2211 ATOMIC_CRITICAL_CPT( cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c, 1 )
2212 ATOMIC_CRITICAL_CPT( cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c, 1 )
2213 ATOMIC_CRITICAL_CPT( cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c, 1 )
2214 ATOMIC_CRITICAL_CPT( cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c, 1 )
2228 #define OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2229 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2233 (*lhs) = (rhs) OP (*lhs); \
2234 new_value = (*lhs); \
2236 new_value = (*lhs);\
2237 (*lhs) = (rhs) OP (*lhs); \
2239 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2243 #ifdef KMP_GOMP_COMPAT
2244 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) \
2245 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2247 OP_CRITICAL_CPT_REV( OP, 0 ); \
2250 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG)
2260 #define OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2262 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2263 TYPE old_value, new_value; \
2265 old_value = temp_val; \
2266 new_value = rhs OP old_value; \
2267 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2268 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2269 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2274 old_value = temp_val; \
2275 new_value = rhs OP old_value; \
2284 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
2285 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2287 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2288 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2289 OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
2293 ATOMIC_CMPXCHG_CPT_REV( fixed1, div_cpt_rev, kmp_int8, 8, /, KMP_ARCH_X86 )
2294 ATOMIC_CMPXCHG_CPT_REV( fixed1u, div_cpt_rev, kmp_uint8, 8, /, KMP_ARCH_X86 )
2295 ATOMIC_CMPXCHG_CPT_REV( fixed1, shl_cpt_rev, kmp_int8, 8, <<, KMP_ARCH_X86 )
2296 ATOMIC_CMPXCHG_CPT_REV( fixed1, shr_cpt_rev, kmp_int8, 8, >>, KMP_ARCH_X86 )
2297 ATOMIC_CMPXCHG_CPT_REV( fixed1u, shr_cpt_rev, kmp_uint8, 8, >>, KMP_ARCH_X86 )
2298 ATOMIC_CMPXCHG_CPT_REV( fixed1, sub_cpt_rev, kmp_int8, 8, -, KMP_ARCH_X86 )
2299 ATOMIC_CMPXCHG_CPT_REV( fixed2, div_cpt_rev, kmp_int16, 16, /, KMP_ARCH_X86 )
2300 ATOMIC_CMPXCHG_CPT_REV( fixed2u, div_cpt_rev, kmp_uint16, 16, /, KMP_ARCH_X86 )
2301 ATOMIC_CMPXCHG_CPT_REV( fixed2, shl_cpt_rev, kmp_int16, 16, <<, KMP_ARCH_X86 )
2302 ATOMIC_CMPXCHG_CPT_REV( fixed2, shr_cpt_rev, kmp_int16, 16, >>, KMP_ARCH_X86 )
2303 ATOMIC_CMPXCHG_CPT_REV( fixed2u, shr_cpt_rev, kmp_uint16, 16, >>, KMP_ARCH_X86 )
2304 ATOMIC_CMPXCHG_CPT_REV( fixed2, sub_cpt_rev, kmp_int16, 16, -, KMP_ARCH_X86 )
2305 ATOMIC_CMPXCHG_CPT_REV( fixed4, div_cpt_rev, kmp_int32, 32, /, KMP_ARCH_X86 )
2306 ATOMIC_CMPXCHG_CPT_REV( fixed4u, div_cpt_rev, kmp_uint32, 32, /, KMP_ARCH_X86 )
2307 ATOMIC_CMPXCHG_CPT_REV( fixed4, shl_cpt_rev, kmp_int32, 32, <<, KMP_ARCH_X86 )
2308 ATOMIC_CMPXCHG_CPT_REV( fixed4, shr_cpt_rev, kmp_int32, 32, >>, KMP_ARCH_X86 )
2309 ATOMIC_CMPXCHG_CPT_REV( fixed4u, shr_cpt_rev, kmp_uint32, 32, >>, KMP_ARCH_X86 )
2310 ATOMIC_CMPXCHG_CPT_REV( fixed4, sub_cpt_rev, kmp_int32, 32, -, KMP_ARCH_X86 )
2311 ATOMIC_CMPXCHG_CPT_REV( fixed8, div_cpt_rev, kmp_int64, 64, /, KMP_ARCH_X86 )
2312 ATOMIC_CMPXCHG_CPT_REV( fixed8u, div_cpt_rev, kmp_uint64, 64, /, KMP_ARCH_X86 )
2313 ATOMIC_CMPXCHG_CPT_REV( fixed8, shl_cpt_rev, kmp_int64, 64, <<, KMP_ARCH_X86 )
2314 ATOMIC_CMPXCHG_CPT_REV( fixed8, shr_cpt_rev, kmp_int64, 64, >>, KMP_ARCH_X86 )
2315 ATOMIC_CMPXCHG_CPT_REV( fixed8u, shr_cpt_rev, kmp_uint64, 64, >>, KMP_ARCH_X86 )
2316 ATOMIC_CMPXCHG_CPT_REV( fixed8, sub_cpt_rev, kmp_int64, 64, -, KMP_ARCH_X86 )
2317 ATOMIC_CMPXCHG_CPT_REV( float4, div_cpt_rev, kmp_real32, 32, /, KMP_ARCH_X86 )
2318 ATOMIC_CMPXCHG_CPT_REV( float4, sub_cpt_rev, kmp_real32, 32, -, KMP_ARCH_X86 )
2319 ATOMIC_CMPXCHG_CPT_REV( float8, div_cpt_rev, kmp_real64, 64, /, KMP_ARCH_X86 )
2320 ATOMIC_CMPXCHG_CPT_REV( float8, sub_cpt_rev, kmp_real64, 64, -, KMP_ARCH_X86 )
2329 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2330 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
2332 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2334 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
2335 OP_CRITICAL_CPT_REV(OP,LCK_ID) \
2341 ATOMIC_CRITICAL_CPT_REV( float10, sub_cpt_rev,
long double, -, 10r, 1 )
2342 ATOMIC_CRITICAL_CPT_REV( float10, div_cpt_rev,
long double, /, 10r, 1 )
2344 ATOMIC_CRITICAL_CPT_REV( float16, sub_cpt_rev, QUAD_LEGACY, -, 16r, 1 )
2345 ATOMIC_CRITICAL_CPT_REV( float16, div_cpt_rev, QUAD_LEGACY, /, 16r, 1 )
2346 #if ( KMP_ARCH_X86 )
2347 ATOMIC_CRITICAL_CPT_REV( float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r, 1 )
2348 ATOMIC_CRITICAL_CPT_REV( float16, div_a16_cpt_rev, Quad_a16_t, /, 16r, 1 )
2357 #define OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2358 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2361 (*lhs) = (rhs) OP (*lhs); \
2365 (*lhs) = (rhs) OP (*lhs); \
2368 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2372 #ifdef KMP_GOMP_COMPAT
2373 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) \
2374 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2376 OP_CRITICAL_CPT_REV_WRK( OP, 0 ); \
2379 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG)
2383 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
2384 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
2385 OP_GOMP_CRITICAL_CPT_REV_WRK(OP,GOMP_FLAG) \
2386 OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
2393 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c, 1 )
2394 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c, 1 )
2396 ATOMIC_CRITICAL_CPT_REV( cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c, 1 )
2397 ATOMIC_CRITICAL_CPT_REV( cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c, 1 )
2398 ATOMIC_CRITICAL_CPT_REV( cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c, 1 )
2399 ATOMIC_CRITICAL_CPT_REV( cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c, 1 )
2400 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c, 1 )
2401 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c, 1 )
2402 #if ( KMP_ARCH_X86 )
2403 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c, 1 )
2404 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c, 1 )
2409 #define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2410 TYPE __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
2412 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2413 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2415 #define CRITICAL_SWP(LCK_ID) \
2416 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2418 old_value = (*lhs); \
2421 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2425 #ifdef KMP_GOMP_COMPAT
2426 #define GOMP_CRITICAL_SWP(FLAG) \
2427 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2429 CRITICAL_SWP( 0 ); \
2432 #define GOMP_CRITICAL_SWP(FLAG)
2436 #define ATOMIC_XCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2437 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2439 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2440 old_value = KMP_XCHG_FIXED##BITS( lhs, rhs ); \
2444 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2445 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2447 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2448 old_value = KMP_XCHG_REAL##BITS( lhs, rhs ); \
2453 #define CMPXCHG_SWP(TYPE,BITS) \
2455 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2456 TYPE old_value, new_value; \
2458 old_value = temp_val; \
2460 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
2461 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
2462 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
2467 old_value = temp_val; \
2474 #define ATOMIC_CMPXCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
2475 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2477 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2478 CMPXCHG_SWP(TYPE,BITS) \
2481 ATOMIC_XCHG_SWP( fixed1, kmp_int8, 8, KMP_ARCH_X86 )
2482 ATOMIC_XCHG_SWP( fixed2, kmp_int16, 16, KMP_ARCH_X86 )
2483 ATOMIC_XCHG_SWP( fixed4, kmp_int32, 32, KMP_ARCH_X86 )
2485 ATOMIC_XCHG_FLOAT_SWP( float4, kmp_real32, 32, KMP_ARCH_X86 )
2487 #if ( KMP_ARCH_X86 )
2488 ATOMIC_CMPXCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2489 ATOMIC_CMPXCHG_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2491 ATOMIC_XCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2492 ATOMIC_XCHG_FLOAT_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2497 #define ATOMIC_CRITICAL_SWP(TYPE_ID,TYPE,LCK_ID,GOMP_FLAG) \
2498 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
2500 GOMP_CRITICAL_SWP(GOMP_FLAG) \
2501 CRITICAL_SWP(LCK_ID) \
2510 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2511 void __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out ) \
2513 KMP_DEBUG_ASSERT( __kmp_init_serial ); \
2514 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
2517 #define CRITICAL_SWP_WRK(LCK_ID) \
2518 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2523 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
2528 #ifdef KMP_GOMP_COMPAT
2529 #define GOMP_CRITICAL_SWP_WRK(FLAG) \
2530 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
2532 CRITICAL_SWP_WRK( 0 ); \
2535 #define GOMP_CRITICAL_SWP_WRK(FLAG)
2539 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE,LCK_ID,GOMP_FLAG) \
2540 ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
2542 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
2543 CRITICAL_SWP_WRK(LCK_ID) \
2548 ATOMIC_CRITICAL_SWP( float10,
long double, 10r, 1 )
2549 ATOMIC_CRITICAL_SWP( float16, QUAD_LEGACY, 16r, 1 )
2551 ATOMIC_CRITICAL_SWP_WRK( cmplx4, kmp_cmplx32, 8c, 1 )
2556 ATOMIC_CRITICAL_SWP( cmplx8, kmp_cmplx64, 16c, 1 )
2557 ATOMIC_CRITICAL_SWP( cmplx10, kmp_cmplx80, 20c, 1 )
2558 ATOMIC_CRITICAL_SWP( cmplx16, CPLX128_LEG, 32c, 1 )
2559 #if ( KMP_ARCH_X86 )
2560 ATOMIC_CRITICAL_SWP( float16_a16, Quad_a16_t, 16r, 1 )
2561 ATOMIC_CRITICAL_SWP( cmplx16_a16, kmp_cmplx128_a16_t, 32c, 1 )
2567 #endif //OMP_40_ENABLED
2569 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
2579 __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2581 KMP_DEBUG_ASSERT( __kmp_init_serial );
2584 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2591 kmp_int8 old_value, new_value;
2593 old_value = *(kmp_int8 *) lhs;
2594 (*f)( &new_value, &old_value, rhs );
2597 while ( ! KMP_COMPARE_AND_STORE_ACQ8 ( (kmp_int8 *) lhs,
2598 *(kmp_int8 *) &old_value, *(kmp_int8 *) &new_value ) )
2602 old_value = *(kmp_int8 *) lhs;
2603 (*f)( &new_value, &old_value, rhs );
2613 #ifdef KMP_GOMP_COMPAT
2614 if ( __kmp_atomic_mode == 2 ) {
2615 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2619 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2621 (*f)( lhs, lhs, rhs );
2623 #ifdef KMP_GOMP_COMPAT
2624 if ( __kmp_atomic_mode == 2 ) {
2625 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2629 __kmp_release_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2634 __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2637 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2639 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2642 ! ( (kmp_uintptr_t) lhs & 0x1)
2646 kmp_int16 old_value, new_value;
2648 old_value = *(kmp_int16 *) lhs;
2649 (*f)( &new_value, &old_value, rhs );
2652 while ( ! KMP_COMPARE_AND_STORE_ACQ16 ( (kmp_int16 *) lhs,
2653 *(kmp_int16 *) &old_value, *(kmp_int16 *) &new_value ) )
2657 old_value = *(kmp_int16 *) lhs;
2658 (*f)( &new_value, &old_value, rhs );
2668 #ifdef KMP_GOMP_COMPAT
2669 if ( __kmp_atomic_mode == 2 ) {
2670 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2674 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2676 (*f)( lhs, lhs, rhs );
2678 #ifdef KMP_GOMP_COMPAT
2679 if ( __kmp_atomic_mode == 2 ) {
2680 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2684 __kmp_release_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2689 __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2691 KMP_DEBUG_ASSERT( __kmp_init_serial );
2698 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
2701 ! ( (kmp_uintptr_t) lhs & 0x3)
2705 kmp_int32 old_value, new_value;
2707 old_value = *(kmp_int32 *) lhs;
2708 (*f)( &new_value, &old_value, rhs );
2711 while ( ! KMP_COMPARE_AND_STORE_ACQ32 ( (kmp_int32 *) lhs,
2712 *(kmp_int32 *) &old_value, *(kmp_int32 *) &new_value ) )
2716 old_value = *(kmp_int32 *) lhs;
2717 (*f)( &new_value, &old_value, rhs );
2728 #ifdef KMP_GOMP_COMPAT
2729 if ( __kmp_atomic_mode == 2 ) {
2730 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2734 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2736 (*f)( lhs, lhs, rhs );
2738 #ifdef KMP_GOMP_COMPAT
2739 if ( __kmp_atomic_mode == 2 ) {
2740 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2744 __kmp_release_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2749 __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2751 KMP_DEBUG_ASSERT( __kmp_init_serial );
2754 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2756 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2759 ! ( (kmp_uintptr_t) lhs & 0x7)
2763 kmp_int64 old_value, new_value;
2765 old_value = *(kmp_int64 *) lhs;
2766 (*f)( &new_value, &old_value, rhs );
2768 while ( ! KMP_COMPARE_AND_STORE_ACQ64 ( (kmp_int64 *) lhs,
2769 *(kmp_int64 *) &old_value,
2770 *(kmp_int64 *) &new_value ) )
2774 old_value = *(kmp_int64 *) lhs;
2775 (*f)( &new_value, &old_value, rhs );
2785 #ifdef KMP_GOMP_COMPAT
2786 if ( __kmp_atomic_mode == 2 ) {
2787 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2791 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2793 (*f)( lhs, lhs, rhs );
2795 #ifdef KMP_GOMP_COMPAT
2796 if ( __kmp_atomic_mode == 2 ) {
2797 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2801 __kmp_release_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2806 __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2808 KMP_DEBUG_ASSERT( __kmp_init_serial );
2810 #ifdef KMP_GOMP_COMPAT
2811 if ( __kmp_atomic_mode == 2 ) {
2812 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2816 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2818 (*f)( lhs, lhs, rhs );
2820 #ifdef KMP_GOMP_COMPAT
2821 if ( __kmp_atomic_mode == 2 ) {
2822 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2826 __kmp_release_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2830 __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2832 KMP_DEBUG_ASSERT( __kmp_init_serial );
2834 #ifdef KMP_GOMP_COMPAT
2835 if ( __kmp_atomic_mode == 2 ) {
2836 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2840 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2842 (*f)( lhs, lhs, rhs );
2844 #ifdef KMP_GOMP_COMPAT
2845 if ( __kmp_atomic_mode == 2 ) {
2846 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2850 __kmp_release_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2854 __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2856 KMP_DEBUG_ASSERT( __kmp_init_serial );
2858 #ifdef KMP_GOMP_COMPAT
2859 if ( __kmp_atomic_mode == 2 ) {
2860 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2864 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2866 (*f)( lhs, lhs, rhs );
2868 #ifdef KMP_GOMP_COMPAT
2869 if ( __kmp_atomic_mode == 2 ) {
2870 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2874 __kmp_release_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2878 __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2880 KMP_DEBUG_ASSERT( __kmp_init_serial );
2882 #ifdef KMP_GOMP_COMPAT
2883 if ( __kmp_atomic_mode == 2 ) {
2884 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2888 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2890 (*f)( lhs, lhs, rhs );
2892 #ifdef KMP_GOMP_COMPAT
2893 if ( __kmp_atomic_mode == 2 ) {
2894 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2898 __kmp_release_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2905 __kmpc_atomic_start(
void)
2907 int gtid = __kmp_entry_gtid();
2908 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
2909 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
2914 __kmpc_atomic_end(
void)
2916 int gtid = __kmp_get_gtid();
2917 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
2918 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);