mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
261 lines
7.8 KiB
261 lines
7.8 KiB
/* SPDX-License-Identifier: GPL-2.0 */ |
|
#ifndef ASM_X86_CMPXCHG_H |
|
#define ASM_X86_CMPXCHG_H |
|
|
|
#include <linux/compiler.h> |
|
#include <asm/cpufeatures.h> |
|
#include <asm/alternative.h> /* Provides LOCK_PREFIX */ |
|
|
|
/* |
|
* Non-existent functions to indicate usage errors at link time |
|
* (or compile-time if the compiler implements __compiletime_error(). |
|
*/ |
|
extern void __xchg_wrong_size(void) |
|
__compiletime_error("Bad argument size for xchg"); |
|
extern void __cmpxchg_wrong_size(void) |
|
__compiletime_error("Bad argument size for cmpxchg"); |
|
extern void __xadd_wrong_size(void) |
|
__compiletime_error("Bad argument size for xadd"); |
|
extern void __add_wrong_size(void) |
|
__compiletime_error("Bad argument size for add"); |
|
|
|
/* |
|
* Constants for operation sizes. On 32-bit, the 64-bit size it set to |
|
* -1 because sizeof will never return -1, thereby making those switch |
|
* case statements guaranteeed dead code which the compiler will |
|
* eliminate, and allowing the "missing symbol in the default case" to |
|
* indicate a usage error. |
|
*/ |
|
#define __X86_CASE_B 1 |
|
#define __X86_CASE_W 2 |
|
#define __X86_CASE_L 4 |
|
#ifdef CONFIG_64BIT |
|
#define __X86_CASE_Q 8 |
|
#else |
|
#define __X86_CASE_Q -1 /* sizeof will never return -1 */ |
|
#endif |
|
|
|
/* |
|
* An exchange-type operation, which takes a value and a pointer, and |
|
* returns the old value. |
|
*/ |
|
#define __xchg_op(ptr, arg, op, lock) \ |
|
({ \ |
|
__typeof__ (*(ptr)) __ret = (arg); \ |
|
switch (sizeof(*(ptr))) { \ |
|
case __X86_CASE_B: \ |
|
asm volatile (lock #op "b %b0, %1\n" \ |
|
: "+q" (__ret), "+m" (*(ptr)) \ |
|
: : "memory", "cc"); \ |
|
break; \ |
|
case __X86_CASE_W: \ |
|
asm volatile (lock #op "w %w0, %1\n" \ |
|
: "+r" (__ret), "+m" (*(ptr)) \ |
|
: : "memory", "cc"); \ |
|
break; \ |
|
case __X86_CASE_L: \ |
|
asm volatile (lock #op "l %0, %1\n" \ |
|
: "+r" (__ret), "+m" (*(ptr)) \ |
|
: : "memory", "cc"); \ |
|
break; \ |
|
case __X86_CASE_Q: \ |
|
asm volatile (lock #op "q %q0, %1\n" \ |
|
: "+r" (__ret), "+m" (*(ptr)) \ |
|
: : "memory", "cc"); \ |
|
break; \ |
|
default: \ |
|
__ ## op ## _wrong_size(); \ |
|
} \ |
|
__ret; \ |
|
}) |
|
|
|
/* |
|
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway. |
|
* Since this is generally used to protect other memory information, we |
|
* use "asm volatile" and "memory" clobbers to prevent gcc from moving |
|
* information around. |
|
*/ |
|
#define arch_xchg(ptr, v) __xchg_op((ptr), (v), xchg, "") |
|
|
|
/* |
|
* Atomic compare and exchange. Compare OLD with MEM, if identical, |
|
* store NEW in MEM. Return the initial value in MEM. Success is |
|
* indicated by comparing RETURN with OLD. |
|
*/ |
|
#define __raw_cmpxchg(ptr, old, new, size, lock) \ |
|
({ \ |
|
__typeof__(*(ptr)) __ret; \ |
|
__typeof__(*(ptr)) __old = (old); \ |
|
__typeof__(*(ptr)) __new = (new); \ |
|
switch (size) { \ |
|
case __X86_CASE_B: \ |
|
{ \ |
|
volatile u8 *__ptr = (volatile u8 *)(ptr); \ |
|
asm volatile(lock "cmpxchgb %2,%1" \ |
|
: "=a" (__ret), "+m" (*__ptr) \ |
|
: "q" (__new), "0" (__old) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
case __X86_CASE_W: \ |
|
{ \ |
|
volatile u16 *__ptr = (volatile u16 *)(ptr); \ |
|
asm volatile(lock "cmpxchgw %2,%1" \ |
|
: "=a" (__ret), "+m" (*__ptr) \ |
|
: "r" (__new), "0" (__old) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
case __X86_CASE_L: \ |
|
{ \ |
|
volatile u32 *__ptr = (volatile u32 *)(ptr); \ |
|
asm volatile(lock "cmpxchgl %2,%1" \ |
|
: "=a" (__ret), "+m" (*__ptr) \ |
|
: "r" (__new), "0" (__old) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
case __X86_CASE_Q: \ |
|
{ \ |
|
volatile u64 *__ptr = (volatile u64 *)(ptr); \ |
|
asm volatile(lock "cmpxchgq %2,%1" \ |
|
: "=a" (__ret), "+m" (*__ptr) \ |
|
: "r" (__new), "0" (__old) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
default: \ |
|
__cmpxchg_wrong_size(); \ |
|
} \ |
|
__ret; \ |
|
}) |
|
|
|
#define __cmpxchg(ptr, old, new, size) \ |
|
__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX) |
|
|
|
#define __sync_cmpxchg(ptr, old, new, size) \ |
|
__raw_cmpxchg((ptr), (old), (new), (size), "lock; ") |
|
|
|
#define __cmpxchg_local(ptr, old, new, size) \ |
|
__raw_cmpxchg((ptr), (old), (new), (size), "") |
|
|
|
#ifdef CONFIG_X86_32 |
|
# include <asm/cmpxchg_32.h> |
|
#else |
|
# include <asm/cmpxchg_64.h> |
|
#endif |
|
|
|
#define arch_cmpxchg(ptr, old, new) \ |
|
__cmpxchg(ptr, old, new, sizeof(*(ptr))) |
|
|
|
#define arch_sync_cmpxchg(ptr, old, new) \ |
|
__sync_cmpxchg(ptr, old, new, sizeof(*(ptr))) |
|
|
|
#define arch_cmpxchg_local(ptr, old, new) \ |
|
__cmpxchg_local(ptr, old, new, sizeof(*(ptr))) |
|
|
|
|
|
#define __raw_try_cmpxchg(_ptr, _pold, _new, size, lock) \ |
|
({ \ |
|
bool success; \ |
|
__typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ |
|
__typeof__(*(_ptr)) __old = *_old; \ |
|
__typeof__(*(_ptr)) __new = (_new); \ |
|
switch (size) { \ |
|
case __X86_CASE_B: \ |
|
{ \ |
|
volatile u8 *__ptr = (volatile u8 *)(_ptr); \ |
|
asm volatile(lock "cmpxchgb %[new], %[ptr]" \ |
|
CC_SET(z) \ |
|
: CC_OUT(z) (success), \ |
|
[ptr] "+m" (*__ptr), \ |
|
[old] "+a" (__old) \ |
|
: [new] "q" (__new) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
case __X86_CASE_W: \ |
|
{ \ |
|
volatile u16 *__ptr = (volatile u16 *)(_ptr); \ |
|
asm volatile(lock "cmpxchgw %[new], %[ptr]" \ |
|
CC_SET(z) \ |
|
: CC_OUT(z) (success), \ |
|
[ptr] "+m" (*__ptr), \ |
|
[old] "+a" (__old) \ |
|
: [new] "r" (__new) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
case __X86_CASE_L: \ |
|
{ \ |
|
volatile u32 *__ptr = (volatile u32 *)(_ptr); \ |
|
asm volatile(lock "cmpxchgl %[new], %[ptr]" \ |
|
CC_SET(z) \ |
|
: CC_OUT(z) (success), \ |
|
[ptr] "+m" (*__ptr), \ |
|
[old] "+a" (__old) \ |
|
: [new] "r" (__new) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
case __X86_CASE_Q: \ |
|
{ \ |
|
volatile u64 *__ptr = (volatile u64 *)(_ptr); \ |
|
asm volatile(lock "cmpxchgq %[new], %[ptr]" \ |
|
CC_SET(z) \ |
|
: CC_OUT(z) (success), \ |
|
[ptr] "+m" (*__ptr), \ |
|
[old] "+a" (__old) \ |
|
: [new] "r" (__new) \ |
|
: "memory"); \ |
|
break; \ |
|
} \ |
|
default: \ |
|
__cmpxchg_wrong_size(); \ |
|
} \ |
|
if (unlikely(!success)) \ |
|
*_old = __old; \ |
|
likely(success); \ |
|
}) |
|
|
|
#define __try_cmpxchg(ptr, pold, new, size) \ |
|
__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX) |
|
|
|
#define arch_try_cmpxchg(ptr, pold, new) \ |
|
__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr))) |
|
|
|
/* |
|
* xadd() adds "inc" to "*ptr" and atomically returns the previous |
|
* value of "*ptr". |
|
* |
|
* xadd() is locked when multiple CPUs are online |
|
*/ |
|
#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock) |
|
#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX) |
|
|
|
#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \ |
|
({ \ |
|
bool __ret; \ |
|
__typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \ |
|
__typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \ |
|
BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ |
|
BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ |
|
VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \ |
|
VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \ |
|
asm volatile(pfx "cmpxchg%c5b %1" \ |
|
CC_SET(e) \ |
|
: CC_OUT(e) (__ret), \ |
|
"+m" (*(p1)), "+m" (*(p2)), \ |
|
"+a" (__old1), "+d" (__old2) \ |
|
: "i" (2 * sizeof(long)), \ |
|
"b" (__new1), "c" (__new2)); \ |
|
__ret; \ |
|
}) |
|
|
|
#define arch_cmpxchg_double(p1, p2, o1, o2, n1, n2) \ |
|
__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2) |
|
|
|
#define arch_cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \ |
|
__cmpxchg_double(, p1, p2, o1, o2, n1, n2) |
|
|
|
#endif /* ASM_X86_CMPXCHG_H */
|
|
|