forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
387 lines
10 KiB
387 lines
10 KiB
// SPDX-License-Identifier: GPL-2.0+ |
|
/* |
|
* Restartable sequences system call |
|
* |
|
* Copyright (C) 2015, Google, Inc., |
|
* Paul Turner <[email protected]> and Andrew Hunter <[email protected]> |
|
* Copyright (C) 2015-2018, EfficiOS Inc., |
|
* Mathieu Desnoyers <[email protected]> |
|
*/ |
|
|
|
#include <linux/sched.h> |
|
#include <linux/uaccess.h> |
|
#include <linux/syscalls.h> |
|
#include <linux/rseq.h> |
|
#include <linux/types.h> |
|
#include <asm/ptrace.h> |
|
|
|
#define CREATE_TRACE_POINTS |
|
#include <trace/events/rseq.h> |
|
|
|
#define RSEQ_CS_PREEMPT_MIGRATE_FLAGS (RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE | \ |
|
RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT) |
|
|
|
/* |
|
* |
|
* Restartable sequences are a lightweight interface that allows |
|
* user-level code to be executed atomically relative to scheduler |
|
* preemption and signal delivery. Typically used for implementing |
|
* per-cpu operations. |
|
* |
|
* It allows user-space to perform update operations on per-cpu data |
|
* without requiring heavy-weight atomic operations. |
|
* |
|
* Detailed algorithm of rseq user-space assembly sequences: |
|
* |
|
* init(rseq_cs) |
|
* cpu = TLS->rseq::cpu_id_start |
|
* [1] TLS->rseq::rseq_cs = rseq_cs |
|
* [start_ip] ---------------------------- |
|
* [2] if (cpu != TLS->rseq::cpu_id) |
|
* goto abort_ip; |
|
* [3] <last_instruction_in_cs> |
|
* [post_commit_ip] ---------------------------- |
|
* |
|
* The address of jump target abort_ip must be outside the critical |
|
* region, i.e.: |
|
* |
|
* [abort_ip] < [start_ip] || [abort_ip] >= [post_commit_ip] |
|
* |
|
* Steps [2]-[3] (inclusive) need to be a sequence of instructions in |
|
* userspace that can handle being interrupted between any of those |
|
* instructions, and then resumed to the abort_ip. |
|
* |
|
* 1. Userspace stores the address of the struct rseq_cs assembly |
|
* block descriptor into the rseq_cs field of the registered |
|
* struct rseq TLS area. This update is performed through a single |
|
* store within the inline assembly instruction sequence. |
|
* [start_ip] |
|
* |
|
* 2. Userspace tests to check whether the current cpu_id field match |
|
* the cpu number loaded before start_ip, branching to abort_ip |
|
* in case of a mismatch. |
|
* |
|
* If the sequence is preempted or interrupted by a signal |
|
* at or after start_ip and before post_commit_ip, then the kernel |
|
* clears TLS->__rseq_abi::rseq_cs, and sets the user-space return |
|
* ip to abort_ip before returning to user-space, so the preempted |
|
* execution resumes at abort_ip. |
|
* |
|
* 3. Userspace critical section final instruction before |
|
* post_commit_ip is the commit. The critical section is |
|
* self-terminating. |
|
* [post_commit_ip] |
|
* |
|
* 4. <success> |
|
* |
|
* On failure at [2], or if interrupted by preempt or signal delivery |
|
* between [1] and [3]: |
|
* |
|
* [abort_ip] |
|
* F1. <failure> |
|
*/ |
|
|
|
static int rseq_update_cpu_id(struct task_struct *t) |
|
{ |
|
u32 cpu_id = raw_smp_processor_id(); |
|
struct rseq __user *rseq = t->rseq; |
|
|
|
if (!user_write_access_begin(rseq, sizeof(*rseq))) |
|
goto efault; |
|
unsafe_put_user(cpu_id, &rseq->cpu_id_start, efault_end); |
|
unsafe_put_user(cpu_id, &rseq->cpu_id, efault_end); |
|
user_write_access_end(); |
|
trace_rseq_update(t); |
|
return 0; |
|
|
|
efault_end: |
|
user_write_access_end(); |
|
efault: |
|
return -EFAULT; |
|
} |
|
|
|
static int rseq_reset_rseq_cpu_id(struct task_struct *t) |
|
{ |
|
u32 cpu_id_start = 0, cpu_id = RSEQ_CPU_ID_UNINITIALIZED; |
|
|
|
/* |
|
* Reset cpu_id_start to its initial state (0). |
|
*/ |
|
if (put_user(cpu_id_start, &t->rseq->cpu_id_start)) |
|
return -EFAULT; |
|
/* |
|
* Reset cpu_id to RSEQ_CPU_ID_UNINITIALIZED, so any user coming |
|
* in after unregistration can figure out that rseq needs to be |
|
* registered again. |
|
*/ |
|
if (put_user(cpu_id, &t->rseq->cpu_id)) |
|
return -EFAULT; |
|
return 0; |
|
} |
|
|
|
static int rseq_get_rseq_cs(struct task_struct *t, struct rseq_cs *rseq_cs) |
|
{ |
|
struct rseq_cs __user *urseq_cs; |
|
u64 ptr; |
|
u32 __user *usig; |
|
u32 sig; |
|
int ret; |
|
|
|
#ifdef CONFIG_64BIT |
|
if (get_user(ptr, &t->rseq->rseq_cs.ptr64)) |
|
return -EFAULT; |
|
#else |
|
if (copy_from_user(&ptr, &t->rseq->rseq_cs.ptr64, sizeof(ptr))) |
|
return -EFAULT; |
|
#endif |
|
if (!ptr) { |
|
memset(rseq_cs, 0, sizeof(*rseq_cs)); |
|
return 0; |
|
} |
|
if (ptr >= TASK_SIZE) |
|
return -EINVAL; |
|
urseq_cs = (struct rseq_cs __user *)(unsigned long)ptr; |
|
if (copy_from_user(rseq_cs, urseq_cs, sizeof(*rseq_cs))) |
|
return -EFAULT; |
|
|
|
if (rseq_cs->start_ip >= TASK_SIZE || |
|
rseq_cs->start_ip + rseq_cs->post_commit_offset >= TASK_SIZE || |
|
rseq_cs->abort_ip >= TASK_SIZE || |
|
rseq_cs->version > 0) |
|
return -EINVAL; |
|
/* Check for overflow. */ |
|
if (rseq_cs->start_ip + rseq_cs->post_commit_offset < rseq_cs->start_ip) |
|
return -EINVAL; |
|
/* Ensure that abort_ip is not in the critical section. */ |
|
if (rseq_cs->abort_ip - rseq_cs->start_ip < rseq_cs->post_commit_offset) |
|
return -EINVAL; |
|
|
|
usig = (u32 __user *)(unsigned long)(rseq_cs->abort_ip - sizeof(u32)); |
|
ret = get_user(sig, usig); |
|
if (ret) |
|
return ret; |
|
|
|
if (current->rseq_sig != sig) { |
|
printk_ratelimited(KERN_WARNING |
|
"Possible attack attempt. Unexpected rseq signature 0x%x, expecting 0x%x (pid=%d, addr=%p).\n", |
|
sig, current->rseq_sig, current->pid, usig); |
|
return -EINVAL; |
|
} |
|
return 0; |
|
} |
|
|
|
static int rseq_need_restart(struct task_struct *t, u32 cs_flags) |
|
{ |
|
u32 flags, event_mask; |
|
int ret; |
|
|
|
/* Get thread flags. */ |
|
ret = get_user(flags, &t->rseq->flags); |
|
if (ret) |
|
return ret; |
|
|
|
/* Take critical section flags into account. */ |
|
flags |= cs_flags; |
|
|
|
/* |
|
* Restart on signal can only be inhibited when restart on |
|
* preempt and restart on migrate are inhibited too. Otherwise, |
|
* a preempted signal handler could fail to restart the prior |
|
* execution context on sigreturn. |
|
*/ |
|
if (unlikely((flags & RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL) && |
|
(flags & RSEQ_CS_PREEMPT_MIGRATE_FLAGS) != |
|
RSEQ_CS_PREEMPT_MIGRATE_FLAGS)) |
|
return -EINVAL; |
|
|
|
/* |
|
* Load and clear event mask atomically with respect to |
|
* scheduler preemption. |
|
*/ |
|
preempt_disable(); |
|
event_mask = t->rseq_event_mask; |
|
t->rseq_event_mask = 0; |
|
preempt_enable(); |
|
|
|
return !!(event_mask & ~flags); |
|
} |
|
|
|
static int clear_rseq_cs(struct task_struct *t) |
|
{ |
|
/* |
|
* The rseq_cs field is set to NULL on preemption or signal |
|
* delivery on top of rseq assembly block, as well as on top |
|
* of code outside of the rseq assembly block. This performs |
|
* a lazy clear of the rseq_cs field. |
|
* |
|
* Set rseq_cs to NULL. |
|
*/ |
|
#ifdef CONFIG_64BIT |
|
return put_user(0UL, &t->rseq->rseq_cs.ptr64); |
|
#else |
|
if (clear_user(&t->rseq->rseq_cs.ptr64, sizeof(t->rseq->rseq_cs.ptr64))) |
|
return -EFAULT; |
|
return 0; |
|
#endif |
|
} |
|
|
|
/* |
|
* Unsigned comparison will be true when ip >= start_ip, and when |
|
* ip < start_ip + post_commit_offset. |
|
*/ |
|
static bool in_rseq_cs(unsigned long ip, struct rseq_cs *rseq_cs) |
|
{ |
|
return ip - rseq_cs->start_ip < rseq_cs->post_commit_offset; |
|
} |
|
|
|
static int rseq_ip_fixup(struct pt_regs *regs) |
|
{ |
|
unsigned long ip = instruction_pointer(regs); |
|
struct task_struct *t = current; |
|
struct rseq_cs rseq_cs; |
|
int ret; |
|
|
|
ret = rseq_get_rseq_cs(t, &rseq_cs); |
|
if (ret) |
|
return ret; |
|
|
|
/* |
|
* Handle potentially not being within a critical section. |
|
* If not nested over a rseq critical section, restart is useless. |
|
* Clear the rseq_cs pointer and return. |
|
*/ |
|
if (!in_rseq_cs(ip, &rseq_cs)) |
|
return clear_rseq_cs(t); |
|
ret = rseq_need_restart(t, rseq_cs.flags); |
|
if (ret <= 0) |
|
return ret; |
|
ret = clear_rseq_cs(t); |
|
if (ret) |
|
return ret; |
|
trace_rseq_ip_fixup(ip, rseq_cs.start_ip, rseq_cs.post_commit_offset, |
|
rseq_cs.abort_ip); |
|
instruction_pointer_set(regs, (unsigned long)rseq_cs.abort_ip); |
|
return 0; |
|
} |
|
|
|
/* |
|
* This resume handler must always be executed between any of: |
|
* - preemption, |
|
* - signal delivery, |
|
* and return to user-space. |
|
* |
|
* This is how we can ensure that the entire rseq critical section |
|
* will issue the commit instruction only if executed atomically with |
|
* respect to other threads scheduled on the same CPU, and with respect |
|
* to signal handlers. |
|
*/ |
|
void __rseq_handle_notify_resume(struct ksignal *ksig, struct pt_regs *regs) |
|
{ |
|
struct task_struct *t = current; |
|
int ret, sig; |
|
|
|
if (unlikely(t->flags & PF_EXITING)) |
|
return; |
|
|
|
/* |
|
* regs is NULL if and only if the caller is in a syscall path. Skip |
|
* fixup and leave rseq_cs as is so that rseq_sycall() will detect and |
|
* kill a misbehaving userspace on debug kernels. |
|
*/ |
|
if (regs) { |
|
ret = rseq_ip_fixup(regs); |
|
if (unlikely(ret < 0)) |
|
goto error; |
|
} |
|
if (unlikely(rseq_update_cpu_id(t))) |
|
goto error; |
|
return; |
|
|
|
error: |
|
sig = ksig ? ksig->sig : 0; |
|
force_sigsegv(sig); |
|
} |
|
|
|
#ifdef CONFIG_DEBUG_RSEQ |
|
|
|
/* |
|
* Terminate the process if a syscall is issued within a restartable |
|
* sequence. |
|
*/ |
|
void rseq_syscall(struct pt_regs *regs) |
|
{ |
|
unsigned long ip = instruction_pointer(regs); |
|
struct task_struct *t = current; |
|
struct rseq_cs rseq_cs; |
|
|
|
if (!t->rseq) |
|
return; |
|
if (rseq_get_rseq_cs(t, &rseq_cs) || in_rseq_cs(ip, &rseq_cs)) |
|
force_sig(SIGSEGV); |
|
} |
|
|
|
#endif |
|
|
|
/* |
|
* sys_rseq - setup restartable sequences for caller thread. |
|
*/ |
|
SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, |
|
int, flags, u32, sig) |
|
{ |
|
int ret; |
|
|
|
if (flags & RSEQ_FLAG_UNREGISTER) { |
|
if (flags & ~RSEQ_FLAG_UNREGISTER) |
|
return -EINVAL; |
|
/* Unregister rseq for current thread. */ |
|
if (current->rseq != rseq || !current->rseq) |
|
return -EINVAL; |
|
if (rseq_len != sizeof(*rseq)) |
|
return -EINVAL; |
|
if (current->rseq_sig != sig) |
|
return -EPERM; |
|
ret = rseq_reset_rseq_cpu_id(current); |
|
if (ret) |
|
return ret; |
|
current->rseq = NULL; |
|
current->rseq_sig = 0; |
|
return 0; |
|
} |
|
|
|
if (unlikely(flags)) |
|
return -EINVAL; |
|
|
|
if (current->rseq) { |
|
/* |
|
* If rseq is already registered, check whether |
|
* the provided address differs from the prior |
|
* one. |
|
*/ |
|
if (current->rseq != rseq || rseq_len != sizeof(*rseq)) |
|
return -EINVAL; |
|
if (current->rseq_sig != sig) |
|
return -EPERM; |
|
/* Already registered. */ |
|
return -EBUSY; |
|
} |
|
|
|
/* |
|
* If there was no rseq previously registered, |
|
* ensure the provided rseq is properly aligned and valid. |
|
*/ |
|
if (!IS_ALIGNED((unsigned long)rseq, __alignof__(*rseq)) || |
|
rseq_len != sizeof(*rseq)) |
|
return -EINVAL; |
|
if (!access_ok(rseq, rseq_len)) |
|
return -EFAULT; |
|
current->rseq = rseq; |
|
current->rseq_sig = sig; |
|
/* |
|
* If rseq was previously inactive, and has just been |
|
* registered, ensure the cpu_id_start and cpu_id fields |
|
* are updated before returning to user-space. |
|
*/ |
|
rseq_set_notify_resume(current); |
|
|
|
return 0; |
|
}
|
|
|