13#include "kmp_atomic.h"
16typedef unsigned char uchar;
17typedef unsigned short ushort;
565#ifndef KMP_GOMP_COMPAT
566int __kmp_atomic_mode = 1;
568int __kmp_atomic_mode = 2;
574kmp_atomic_lock_t __kmp_atomic_lock;
576kmp_atomic_lock_t __kmp_atomic_lock_1i;
578kmp_atomic_lock_t __kmp_atomic_lock_2i;
580kmp_atomic_lock_t __kmp_atomic_lock_4i;
582kmp_atomic_lock_t __kmp_atomic_lock_4r;
584kmp_atomic_lock_t __kmp_atomic_lock_8i;
586kmp_atomic_lock_t __kmp_atomic_lock_8r;
588kmp_atomic_lock_t __kmp_atomic_lock_8c;
590kmp_atomic_lock_t __kmp_atomic_lock_10r;
592kmp_atomic_lock_t __kmp_atomic_lock_16r;
594kmp_atomic_lock_t __kmp_atomic_lock_16c;
596kmp_atomic_lock_t __kmp_atomic_lock_20c;
598kmp_atomic_lock_t __kmp_atomic_lock_32c;
605#define KMP_ATOMIC_VOLATILE volatile
607#if (KMP_ARCH_X86) && KMP_HAVE_QUAD
609static inline void operator+=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
612static inline void operator-=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
615static inline void operator*=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
618static inline void operator/=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
621static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 return lhs.q < rhs.q;
624static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q > rhs.q;
628static inline void operator+=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
631static inline void operator-=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
634static inline void operator*=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
637static inline void operator/=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
640static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 return lhs.q < rhs.q;
643static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q > rhs.q;
647static inline void operator+=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
650static inline void operator-=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
653static inline void operator*=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
656static inline void operator/=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
660static inline void operator+=(kmp_cmplx128_a16_t &lhs,
661 kmp_cmplx128_a16_t &rhs) {
664static inline void operator-=(kmp_cmplx128_a16_t &lhs,
665 kmp_cmplx128_a16_t &rhs) {
668static inline void operator*=(kmp_cmplx128_a16_t &lhs,
669 kmp_cmplx128_a16_t &rhs) {
672static inline void operator/=(kmp_cmplx128_a16_t &lhs,
673 kmp_cmplx128_a16_t &rhs) {
684#define KMP_CHECK_GTID \
685 if (gtid == KMP_GTID_UNKNOWN) { \
686 gtid = __kmp_entry_gtid(); \
694#define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
695 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
696 TYPE *lhs, TYPE rhs) { \
697 KMP_DEBUG_ASSERT(__kmp_init_serial); \
698 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
702#define ATOMIC_LOCK0 __kmp_atomic_lock
703#define ATOMIC_LOCK1i __kmp_atomic_lock_1i
704#define ATOMIC_LOCK2i __kmp_atomic_lock_2i
705#define ATOMIC_LOCK4i __kmp_atomic_lock_4i
706#define ATOMIC_LOCK4r __kmp_atomic_lock_4r
707#define ATOMIC_LOCK8i __kmp_atomic_lock_8i
708#define ATOMIC_LOCK8r __kmp_atomic_lock_8r
709#define ATOMIC_LOCK8c __kmp_atomic_lock_8c
710#define ATOMIC_LOCK10r __kmp_atomic_lock_10r
711#define ATOMIC_LOCK16r __kmp_atomic_lock_16r
712#define ATOMIC_LOCK16c __kmp_atomic_lock_16c
713#define ATOMIC_LOCK20c __kmp_atomic_lock_20c
714#define ATOMIC_LOCK32c __kmp_atomic_lock_32c
722#define OP_CRITICAL(OP, LCK_ID) \
723 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
727 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
751#ifdef KMP_GOMP_COMPAT
752#define OP_GOMP_CRITICAL(OP, FLAG) \
753 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
755 OP_CRITICAL(OP, 0); \
759#define OP_GOMP_CRITICAL(OP, FLAG)
763#define KMP_DO_PAUSE _mm_delay_32(1)
765#define KMP_DO_PAUSE KMP_CPU_PAUSE()
773#define OP_CMPXCHG(TYPE, BITS, OP) \
775 TYPE old_value, new_value; \
776 old_value = *(TYPE volatile *)lhs; \
777 new_value = old_value OP rhs; \
778 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
779 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
780 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
783 old_value = *(TYPE volatile *)lhs; \
784 new_value = old_value OP rhs; \
795#define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
799 kmp_int##BITS *vvv; \
801 struct _sss old_value, new_value; \
802 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
803 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
804 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
805 new_value.cmp = old_value.cmp OP rhs; \
806 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
807 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
808 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
811 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
812 new_value.cmp = old_value.cmp OP rhs; \
818#if KMP_ARCH_X86 || KMP_ARCH_X86_64
822#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
824 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
825 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
827 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
830#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
832 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
833 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
834 OP_CMPXCHG(TYPE, BITS, OP) \
839#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
841 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
842 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
843 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
851#define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
853 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
854 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
855 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
857 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
860 OP_CRITICAL(OP## =, LCK_ID) \
864#define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
866 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
867 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
868 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
869 OP_CMPXCHG(TYPE, BITS, OP) \
872 OP_CRITICAL(OP## =, LCK_ID) \
878#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
880 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
881 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
882 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
883 OP_CMPXCHG(TYPE, BITS, OP) \
886 OP_CRITICAL(OP## =, LCK_ID) \
894ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
896ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
899ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
901ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
905ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
907ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
910ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
912ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
930ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
932ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
934ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
936ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
938ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
940ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
942ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
944ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
946ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
948ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
950ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
952ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
954ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
956ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
958ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
960ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
962ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
964ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
966ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
968ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
970ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
972ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
974ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
976ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
978ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
980ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
982ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
984ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
986ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
988ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
990ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
992ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
994ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
996ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
998ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1000ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1002ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1004ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1006ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1008ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1010ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1012ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1014ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1016ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1026#define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1027 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1028 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1029 OP_CRITICAL(= *lhs OP, LCK_ID) \
1032#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1036#define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1037 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1038 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1039 OP_CMPXCHG(TYPE, BITS, OP) \
1045#define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1046 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1047 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1048 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1049 OP_CMPXCHG(TYPE, BITS, OP) \
1052 OP_CRITICAL(= *lhs OP, LCK_ID) \
1057ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1059ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1061ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1063ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1065ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1067ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1069ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1071ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1083#define MIN_MAX_CRITSECT(OP, LCK_ID) \
1084 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1086 if (*lhs OP rhs) { \
1089 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1092#ifdef KMP_GOMP_COMPAT
1093#define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \
1094 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1096 MIN_MAX_CRITSECT(OP, 0); \
1100#define GOMP_MIN_MAX_CRITSECT(OP, FLAG)
1104#define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1106 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1109 old_value = temp_val; \
1110 while (old_value OP rhs && \
1111 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
1112 (kmp_int##BITS *)lhs, \
1113 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1114 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
1117 old_value = temp_val; \
1123#define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1124 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1125 if (*lhs OP rhs) { \
1126 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1127 MIN_MAX_CRITSECT(OP, LCK_ID) \
1131#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1135#define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1137 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1138 if (*lhs OP rhs) { \
1139 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1140 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1147#define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1149 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1150 if (*lhs OP rhs) { \
1151 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1152 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1153 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1156 MIN_MAX_CRITSECT(OP, LCK_ID) \
1162MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1164MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1166MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1168MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1170MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1172MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1174MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1176MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1178MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1180MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1182MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1184MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1187MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1189MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1192MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1194MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1201#define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1202 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1203 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \
1204 OP_CRITICAL(^= ~, LCK_ID) \
1208#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1211#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1213 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1214 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \
1215 OP_CMPXCHG(TYPE, BITS, OP) \
1221#define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1223 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1224 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \
1225 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1226 OP_CMPXCHG(TYPE, BITS, OP) \
1229 OP_CRITICAL(^= ~, LCK_ID) \
1234ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1236ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1238ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1240ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1242ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1244ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1246ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1248ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1257#define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1258 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1259 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1260 OP_CRITICAL(OP## =, LCK_ID) \
1265ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1267ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1269ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1271ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1275ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1277ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1279ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1281ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1284ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1286ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1288ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1290ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1298ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1300ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1302ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1304ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1308ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1309ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1310ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1311ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1314ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1315ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1316ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1317ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1318ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1320ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1322ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1324ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1327ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1329ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1331ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1333ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1336ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1338ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1340ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1342ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1349#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1357#define OP_CRITICAL_REV(OP, LCK_ID) \
1358 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1360 (*lhs) = (rhs)OP(*lhs); \
1362 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1364#ifdef KMP_GOMP_COMPAT
1365#define OP_GOMP_CRITICAL_REV(OP, FLAG) \
1366 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1368 OP_CRITICAL_REV(OP, 0); \
1372#define OP_GOMP_CRITICAL_REV(OP, FLAG)
1380#define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1381 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \
1382 TYPE *lhs, TYPE rhs) { \
1383 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1384 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid));
1393#define OP_CMPXCHG_REV(TYPE, BITS, OP) \
1395 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1396 TYPE old_value, new_value; \
1398 old_value = temp_val; \
1399 new_value = rhs OP old_value; \
1400 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
1401 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1402 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
1406 old_value = temp_val; \
1407 new_value = rhs OP old_value; \
1412#define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \
1413 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1414 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1415 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1432ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1434ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1436ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1438ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1440ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1442ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1445ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1447ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1449ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1451ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1453ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1455ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1458ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1460ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1462ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1464ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1466ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1468ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1471ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1473ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1475ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1477ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1479ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1481ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1484ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1486ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1489ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1491ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1501#define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1502 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1503 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1504 OP_CRITICAL_REV(OP, LCK_ID) \
1509ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1511ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1515ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1517ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1520ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1522ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1528ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1530ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1532ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1534ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1536ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1538ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1541ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1543ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1546ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1548ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1567#define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1568 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
1569 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \
1570 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1572 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
1576#define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \
1578 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1579 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1580 OP_CRITICAL(OP## =, LCK_ID) \
1584#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1587#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1588 LCK_ID, MASK, GOMP_FLAG) \
1589 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1590 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1591 OP_CMPXCHG(TYPE, BITS, OP) \
1597#define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1598 LCK_ID, MASK, GOMP_FLAG) \
1599 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1600 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1601 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1602 OP_CMPXCHG(TYPE, BITS, OP) \
1605 OP_CRITICAL(OP## =, LCK_ID) \
1611#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1613#define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
1614 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
1615 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1616 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1617 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1619#define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
1620 LCK_ID, GOMP_FLAG) \
1621 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1622 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1623 OP_CRITICAL_REV(OP, LCK_ID) \
1628ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1630ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1632ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1634ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1636ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1638ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1640ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1642ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1644ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1646ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1648ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1650ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1656ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1658ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1660ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1662ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1664ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1666ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1668ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1670ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1673ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1675ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1677ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1679ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1681ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1683ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1685ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1687ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1690ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1692ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1694ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1696ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1698ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1700ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1702ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1704ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1707ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1709ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1711ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1713ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1715ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1717ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1719ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1721ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1724ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1726ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1728ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1730ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1733ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1735ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1737ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1739ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1742ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1744ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1746ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1748ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1751#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1753ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1755ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1757ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1759ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1762ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1764ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1766ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1768ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1771ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1773ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1775ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1777ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1780ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1782ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1784ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1786ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1789ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1791ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1794ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1796ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1799ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1801ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1807#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1812#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1813 LCK_ID, MASK, GOMP_FLAG) \
1814 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1815 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1816 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
1820#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1821 LCK_ID, MASK, GOMP_FLAG) \
1822 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1823 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1824 OP_CMPXCHG(TYPE, BITS, OP) \
1830#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1831 LCK_ID, MASK, GOMP_FLAG) \
1832 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1833 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1834 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1835 OP_CMPXCHG(TYPE, BITS, OP) \
1838 OP_CRITICAL(OP## =, LCK_ID) \
1843ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1845ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1847ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1849ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1853#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1864#define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1865 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
1867 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1868 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
1879#define OP_CMPXCHG_READ(TYPE, BITS, OP) \
1881 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1884 kmp_int##BITS i_val; \
1886 union f_i_union old_value; \
1888 old_value.f_val = temp_val; \
1889 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \
1890 (kmp_int##BITS *)loc, \
1891 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \
1892 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \
1893 new_value = old_value.f_val; \
1903#define OP_CRITICAL_READ(OP, LCK_ID) \
1904 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1906 new_value = (*loc); \
1908 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1911#ifdef KMP_GOMP_COMPAT
1912#define OP_GOMP_CRITICAL_READ(OP, FLAG) \
1913 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1915 OP_CRITICAL_READ(OP, 0); \
1919#define OP_GOMP_CRITICAL_READ(OP, FLAG)
1923#define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1924 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1926 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1927 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \
1931#define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1932 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1934 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1935 OP_CMPXCHG_READ(TYPE, BITS, OP) \
1943#define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1944 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1946 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1947 OP_CRITICAL_READ(OP, LCK_ID) \
1957#define OP_CRITICAL_READ_WRK(OP, LCK_ID) \
1958 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1962 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1964#ifdef KMP_GOMP_COMPAT
1965#define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \
1966 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1968 OP_CRITICAL_READ_WRK(OP, 0); \
1971#define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG)
1974#define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
1975 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \
1977 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1978 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
1981#define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1982 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
1983 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \
1984 OP_CRITICAL_READ_WRK(OP, LCK_ID) \
1991ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
1992ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
1994ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
1996ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2000ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2002ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2005ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2008ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2014ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2017ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2020ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2022ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2025ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2028ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2030ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2038#define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2039 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2040 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2041 KMP_XCHG_FIXED##BITS(lhs, rhs); \
2044#define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2045 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2046 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2047 KMP_XCHG_REAL##BITS(lhs, rhs); \
2057#define OP_CMPXCHG_WR(TYPE, BITS, OP) \
2059 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2060 TYPE old_value, new_value; \
2062 old_value = temp_val; \
2064 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2065 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2066 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2070 old_value = temp_val; \
2076#define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2077 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2078 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2079 OP_CMPXCHG_WR(TYPE, BITS, OP) \
2088#define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2089 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2090 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2091 OP_CRITICAL(OP, LCK_ID) \
2095ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2097ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2099ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2102ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2105ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2109ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2112ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2115ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2119ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2122ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2125ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2126ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2128ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2131ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2134ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2136ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2149#define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
2150 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
2151 TYPE *lhs, TYPE rhs, int flag) { \
2152 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2153 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2161#define OP_CRITICAL_CPT(OP, LCK_ID) \
2162 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2166 new_value = (*lhs); \
2168 new_value = (*lhs); \
2172 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2176#ifdef KMP_GOMP_COMPAT
2177#define OP_GOMP_CRITICAL_CPT(OP, FLAG) \
2178 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2180 OP_CRITICAL_CPT(OP## =, 0); \
2183#define OP_GOMP_CRITICAL_CPT(OP, FLAG)
2193#define OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2195 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2196 TYPE old_value, new_value; \
2198 old_value = temp_val; \
2199 new_value = old_value OP rhs; \
2200 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2201 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2202 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2206 old_value = temp_val; \
2207 new_value = old_value OP rhs; \
2216#define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2217 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2219 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2220 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2224#define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2225 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2226 TYPE old_value, new_value; \
2227 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2229 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
2231 return old_value OP rhs; \
2237ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2239ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2241ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2243ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2246ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2248ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2250ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2252ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2267ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2269ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2271ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2273ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2275ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2277ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2279ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2281ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2283ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2285ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2287ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2289ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2291ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2293ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2295ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2297ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2299ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2301ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2303ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2305ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2307ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2309ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2311ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2313ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2315ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2317ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2319ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2321ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2323ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2325ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2327ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2329ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2331ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2333ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2335ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2337ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2339ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2341ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2343ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2345ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2347ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2349ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2351ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2353ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2365#define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2366 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
2367 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \
2368 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2370 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
2374#define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
2375 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
2376 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2378 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2379 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2383#define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
2384 LCK_ID, GOMP_FLAG) \
2385 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2387 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2388 OP_CRITICAL_CPT(OP## =, LCK_ID) \
2391ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2393ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2395ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2397ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2399ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2401ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2403ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2405ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2408ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2410ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2412ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2414ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2416ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2418ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2420ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2422ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2425ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2427ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2429ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2431ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2433ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2435ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2437ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2439ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2442ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2444ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2446ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2448ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2450ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2452ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2454ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2456ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2459ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2461ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2463ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2465ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2468ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2470ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2472ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2474ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2477ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2479ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2481ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2483ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2497#define OP_CRITICAL_L_CPT(OP, LCK_ID) \
2498 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2503 new_value = (*lhs); \
2505 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
2508#ifdef KMP_GOMP_COMPAT
2509#define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \
2510 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2512 OP_CRITICAL_L_CPT(OP, 0); \
2516#define OP_GOMP_CRITICAL_L_CPT(OP, FLAG)
2521#define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2522 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2524 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \
2525 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2528ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2530ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2532ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2534ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2536ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2538ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2540ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2542ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2554#define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2555 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2557 if (*lhs OP rhs) { \
2563 new_value = old_value; \
2567 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2571#ifdef KMP_GOMP_COMPAT
2572#define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \
2573 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2575 MIN_MAX_CRITSECT_CPT(OP, 0); \
2578#define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG)
2582#define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2584 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2587 old_value = temp_val; \
2588 while (old_value OP rhs && \
2589 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
2590 (kmp_int##BITS *)lhs, \
2591 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2592 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
2595 old_value = temp_val; \
2605#define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2606 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2607 TYPE new_value, old_value; \
2608 if (*lhs OP rhs) { \
2609 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2610 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2615#define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2616 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2617 TYPE new_value, old_value; \
2618 if (*lhs OP rhs) { \
2619 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2620 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2625MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2627MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2629MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2631MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2633MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2635MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2637MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2639MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2641MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2643MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2645MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2647MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2650MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2652MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2655MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2657MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2663#ifdef KMP_GOMP_COMPAT
2664#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \
2665 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2667 OP_CRITICAL_CPT(OP, 0); \
2670#define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG)
2673#define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2674 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2676 OP_GOMP_CRITICAL_EQV_CPT(^= ~, GOMP_FLAG) \
2677 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2682ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2684ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2686ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2688ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2690ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2692ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2694ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2696ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2705#define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2706 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2708 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2709 OP_CRITICAL_CPT(OP## =, LCK_ID) \
2715#define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \
2716 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2726 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2730#ifdef KMP_GOMP_COMPAT
2731#define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \
2732 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2734 OP_CRITICAL_CPT_WRK(OP## =, 0); \
2737#define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG)
2741#define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2742 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \
2743 TYPE rhs, TYPE *out, int flag) { \
2744 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2745 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2748#define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2749 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2750 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \
2751 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \
2757ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2759ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2761ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2763ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2767ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2769ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2771ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2773ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2776ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2778ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2780ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2782ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2790ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2792ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2794ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2796ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2799ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2801ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2803ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2805ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2807ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2809ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2811ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2813ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2816ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2818ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2820ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2822ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2825ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2827ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2829ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2831ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2846#define OP_CRITICAL_CPT_REV(OP, LCK_ID) \
2847 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2851 (*lhs) = (rhs)OP(*lhs); \
2852 new_value = (*lhs); \
2854 new_value = (*lhs); \
2855 (*lhs) = (rhs)OP(*lhs); \
2857 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2861#ifdef KMP_GOMP_COMPAT
2862#define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) \
2863 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2865 OP_CRITICAL_CPT_REV(OP, 0); \
2868#define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG)
2878#define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2880 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2881 TYPE old_value, new_value; \
2883 old_value = temp_val; \
2884 new_value = rhs OP old_value; \
2885 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2886 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2887 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2891 old_value = temp_val; \
2892 new_value = rhs OP old_value; \
2901#define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2902 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2904 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
2905 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2908ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2910ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2912ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
2914ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
2916ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
2918ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
2920ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
2922ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
2924ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
2926ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
2928ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
2930ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
2932ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
2934ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
2936ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
2938ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
2940ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
2942ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
2944ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
2946ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
2948ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
2950ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
2952ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
2954ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
2956ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
2958ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
2960ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
2962ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
2972#define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2973 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2976 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
2977 OP_CRITICAL_CPT_REV(OP, LCK_ID) \
2982ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
2984ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
2988ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
2990ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
2993ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
2995ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
3005#define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3006 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3009 (*lhs) = (rhs)OP(*lhs); \
3013 (*lhs) = (rhs)OP(*lhs); \
3016 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3020#ifdef KMP_GOMP_COMPAT
3021#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \
3022 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3024 OP_CRITICAL_CPT_REV_WRK(OP, 0); \
3027#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG)
3031#define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \
3033 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
3034 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \
3035 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3041ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3043ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3046ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3048ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3050ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3052ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3055ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3057ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3060ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3062ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3076#define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
3077 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
3078 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3080 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
3081 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
3085#define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
3086 LCK_ID, GOMP_FLAG) \
3087 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3089 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
3090 OP_CRITICAL_CPT_REV(OP, LCK_ID) \
3093ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3095ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3097ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3099ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3102ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3104ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3107ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3109ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3113ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3115ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3117ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3119ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3122ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3125ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3128ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3131ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3135ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3138ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3142ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3145ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3149ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3151ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3158#define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3159 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3161 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3162 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3164#define CRITICAL_SWP(LCK_ID) \
3165 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3167 old_value = (*lhs); \
3170 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3174#ifdef KMP_GOMP_COMPAT
3175#define GOMP_CRITICAL_SWP(FLAG) \
3176 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3181#define GOMP_CRITICAL_SWP(FLAG)
3184#define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3185 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3187 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3188 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \
3192#define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3193 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3195 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3196 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \
3201#define CMPXCHG_SWP(TYPE, BITS) \
3203 TYPE KMP_ATOMIC_VOLATILE temp_val; \
3204 TYPE old_value, new_value; \
3206 old_value = temp_val; \
3208 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
3209 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
3210 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
3214 old_value = temp_val; \
3221#define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3222 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3224 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3225 CMPXCHG_SWP(TYPE, BITS) \
3228ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3229ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3230ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3232ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3236ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3238ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3241ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3242ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3249#define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3250 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3252 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3253 CRITICAL_SWP(LCK_ID) \
3261#define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3262 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3263 TYPE rhs, TYPE *out) { \
3264 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3265 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3267#define CRITICAL_SWP_WRK(LCK_ID) \
3268 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3273 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3277#ifdef KMP_GOMP_COMPAT
3278#define GOMP_CRITICAL_SWP_WRK(FLAG) \
3279 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3281 CRITICAL_SWP_WRK(0); \
3284#define GOMP_CRITICAL_SWP_WRK(FLAG)
3288#define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3289 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3291 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
3292 CRITICAL_SWP_WRK(LCK_ID) \
3296ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3298ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3301ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3306ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3307ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3309ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3311ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3313ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3327void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3328 void (*f)(
void *,
void *,
void *)) {
3329 KMP_DEBUG_ASSERT(__kmp_init_serial);
3332#
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3338 kmp_int8 old_value, new_value;
3340 old_value = *(kmp_int8 *)lhs;
3341 (*f)(&new_value, &old_value, rhs);
3344 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3345 *(kmp_int8 *)&new_value)) {
3348 old_value = *(kmp_int8 *)lhs;
3349 (*f)(&new_value, &old_value, rhs);
3356#ifdef KMP_GOMP_COMPAT
3357 if (__kmp_atomic_mode == 2) {
3358 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3361 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3363 (*f)(lhs, lhs, rhs);
3365#ifdef KMP_GOMP_COMPAT
3366 if (__kmp_atomic_mode == 2) {
3367 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3370 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3374void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3375 void (*f)(
void *,
void *,
void *)) {
3377#
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3379#elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3382 !((kmp_uintptr_t)lhs & 0x1)
3385 kmp_int16 old_value, new_value;
3387 old_value = *(kmp_int16 *)lhs;
3388 (*f)(&new_value, &old_value, rhs);
3391 while (!KMP_COMPARE_AND_STORE_ACQ16(
3392 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3395 old_value = *(kmp_int16 *)lhs;
3396 (*f)(&new_value, &old_value, rhs);
3403#ifdef KMP_GOMP_COMPAT
3404 if (__kmp_atomic_mode == 2) {
3405 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3408 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3410 (*f)(lhs, lhs, rhs);
3412#ifdef KMP_GOMP_COMPAT
3413 if (__kmp_atomic_mode == 2) {
3414 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3417 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3421void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3422 void (*f)(
void *,
void *,
void *)) {
3423 KMP_DEBUG_ASSERT(__kmp_init_serial);
3428#
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3431 !((kmp_uintptr_t)lhs & 0x3)
3434 kmp_int32 old_value, new_value;
3436 old_value = *(kmp_int32 *)lhs;
3437 (*f)(&new_value, &old_value, rhs);
3440 while (!KMP_COMPARE_AND_STORE_ACQ32(
3441 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3444 old_value = *(kmp_int32 *)lhs;
3445 (*f)(&new_value, &old_value, rhs);
3453#ifdef KMP_GOMP_COMPAT
3454 if (__kmp_atomic_mode == 2) {
3455 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3458 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3460 (*f)(lhs, lhs, rhs);
3462#ifdef KMP_GOMP_COMPAT
3463 if (__kmp_atomic_mode == 2) {
3464 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3467 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3471void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3472 void (*f)(
void *,
void *,
void *)) {
3473 KMP_DEBUG_ASSERT(__kmp_init_serial);
3476#
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3478#elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3481 !((kmp_uintptr_t)lhs & 0x7)
3484 kmp_int64 old_value, new_value;
3486 old_value = *(kmp_int64 *)lhs;
3487 (*f)(&new_value, &old_value, rhs);
3489 while (!KMP_COMPARE_AND_STORE_ACQ64(
3490 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3493 old_value = *(kmp_int64 *)lhs;
3494 (*f)(&new_value, &old_value, rhs);
3502#ifdef KMP_GOMP_COMPAT
3503 if (__kmp_atomic_mode == 2) {
3504 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3507 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3509 (*f)(lhs, lhs, rhs);
3511#ifdef KMP_GOMP_COMPAT
3512 if (__kmp_atomic_mode == 2) {
3513 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3516 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3520void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3521 void (*f)(
void *,
void *,
void *)) {
3522 KMP_DEBUG_ASSERT(__kmp_init_serial);
3524#ifdef KMP_GOMP_COMPAT
3525 if (__kmp_atomic_mode == 2) {
3526 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3529 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3531 (*f)(lhs, lhs, rhs);
3533#ifdef KMP_GOMP_COMPAT
3534 if (__kmp_atomic_mode == 2) {
3535 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3538 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3541void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3542 void (*f)(
void *,
void *,
void *)) {
3543 KMP_DEBUG_ASSERT(__kmp_init_serial);
3545#ifdef KMP_GOMP_COMPAT
3546 if (__kmp_atomic_mode == 2) {
3547 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3550 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3552 (*f)(lhs, lhs, rhs);
3554#ifdef KMP_GOMP_COMPAT
3555 if (__kmp_atomic_mode == 2) {
3556 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3559 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3562void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3563 void (*f)(
void *,
void *,
void *)) {
3564 KMP_DEBUG_ASSERT(__kmp_init_serial);
3566#ifdef KMP_GOMP_COMPAT
3567 if (__kmp_atomic_mode == 2) {
3568 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3571 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3573 (*f)(lhs, lhs, rhs);
3575#ifdef KMP_GOMP_COMPAT
3576 if (__kmp_atomic_mode == 2) {
3577 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3580 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3583void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3584 void (*f)(
void *,
void *,
void *)) {
3585 KMP_DEBUG_ASSERT(__kmp_init_serial);
3587#ifdef KMP_GOMP_COMPAT
3588 if (__kmp_atomic_mode == 2) {
3589 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3592 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3594 (*f)(lhs, lhs, rhs);
3596#ifdef KMP_GOMP_COMPAT
3597 if (__kmp_atomic_mode == 2) {
3598 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3601 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3607void __kmpc_atomic_start(
void) {
3608 int gtid = __kmp_entry_gtid();
3609 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3610 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3613void __kmpc_atomic_end(
void) {
3614 int gtid = __kmp_get_gtid();
3615 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3616 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);