mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
986 lines
28 KiB
986 lines
28 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* |
|
* Kernel Probes (KProbes) |
|
* |
|
* Copyright (C) IBM Corporation, 2002, 2004 |
|
* |
|
* 2002-Oct Created by Vamsi Krishna S <[email protected]> Kernel |
|
* Probes initial implementation ( includes contributions from |
|
* Rusty Russell). |
|
* 2004-July Suparna Bhattacharya <[email protected]> added jumper probes |
|
* interface to access function arguments. |
|
* 2004-Oct Jim Keniston <[email protected]> and Prasanna S Panchamukhi |
|
* <[email protected]> adapted for x86_64 from i386. |
|
* 2005-Mar Roland McGrath <[email protected]> |
|
* Fixed to handle %rip-relative addressing mode correctly. |
|
* 2005-May Hien Nguyen <[email protected]>, Jim Keniston |
|
* <[email protected]> and Prasanna S Panchamukhi |
|
* <[email protected]> added function-return probes. |
|
* 2005-May Rusty Lynch <[email protected]> |
|
* Added function return probes functionality |
|
* 2006-Feb Masami Hiramatsu <[email protected]> added |
|
* kprobe-booster and kretprobe-booster for i386. |
|
* 2007-Dec Masami Hiramatsu <[email protected]> added kprobe-booster |
|
* and kretprobe-booster for x86-64 |
|
* 2007-Dec Masami Hiramatsu <[email protected]>, Arjan van de Ven |
|
* <[email protected]> and Jim Keniston <[email protected]> |
|
* unified x86 kprobes code. |
|
*/ |
|
#include <linux/kprobes.h> |
|
#include <linux/ptrace.h> |
|
#include <linux/string.h> |
|
#include <linux/slab.h> |
|
#include <linux/hardirq.h> |
|
#include <linux/preempt.h> |
|
#include <linux/sched/debug.h> |
|
#include <linux/perf_event.h> |
|
#include <linux/extable.h> |
|
#include <linux/kdebug.h> |
|
#include <linux/kallsyms.h> |
|
#include <linux/ftrace.h> |
|
#include <linux/kasan.h> |
|
#include <linux/moduleloader.h> |
|
#include <linux/objtool.h> |
|
#include <linux/vmalloc.h> |
|
#include <linux/pgtable.h> |
|
|
|
#include <asm/text-patching.h> |
|
#include <asm/cacheflush.h> |
|
#include <asm/desc.h> |
|
#include <linux/uaccess.h> |
|
#include <asm/alternative.h> |
|
#include <asm/insn.h> |
|
#include <asm/debugreg.h> |
|
#include <asm/set_memory.h> |
|
|
|
#include "common.h" |
|
|
|
DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
|
DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
|
|
|
#define stack_addr(regs) ((unsigned long *)regs->sp) |
|
|
|
#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ |
|
(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \ |
|
(b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \ |
|
(b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \ |
|
(bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \ |
|
<< (row % 32)) |
|
/* |
|
* Undefined/reserved opcodes, conditional jump, Opcode Extension |
|
* Groups, and some special opcodes can not boost. |
|
* This is non-const and volatile to keep gcc from statically |
|
* optimizing it out, as variable_test_bit makes gcc think only |
|
* *(unsigned long*) is used. |
|
*/ |
|
static volatile u32 twobyte_is_boostable[256 / 32] = { |
|
/* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ |
|
/* ---------------------------------------------- */ |
|
W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */ |
|
W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1) , /* 10 */ |
|
W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */ |
|
W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */ |
|
W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */ |
|
W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */ |
|
W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */ |
|
W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */ |
|
W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */ |
|
W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */ |
|
W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */ |
|
W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */ |
|
W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */ |
|
W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */ |
|
W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */ |
|
W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */ |
|
/* ----------------------------------------------- */ |
|
/* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ |
|
}; |
|
#undef W |
|
|
|
struct kretprobe_blackpoint kretprobe_blacklist[] = { |
|
{"__switch_to", }, /* This function switches only current task, but |
|
doesn't switch kernel stack.*/ |
|
{NULL, NULL} /* Terminator */ |
|
}; |
|
|
|
const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); |
|
|
|
static nokprobe_inline void |
|
__synthesize_relative_insn(void *dest, void *from, void *to, u8 op) |
|
{ |
|
struct __arch_relative_insn { |
|
u8 op; |
|
s32 raddr; |
|
} __packed *insn; |
|
|
|
insn = (struct __arch_relative_insn *)dest; |
|
insn->raddr = (s32)((long)(to) - ((long)(from) + 5)); |
|
insn->op = op; |
|
} |
|
|
|
/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ |
|
void synthesize_reljump(void *dest, void *from, void *to) |
|
{ |
|
__synthesize_relative_insn(dest, from, to, JMP32_INSN_OPCODE); |
|
} |
|
NOKPROBE_SYMBOL(synthesize_reljump); |
|
|
|
/* Insert a call instruction at address 'from', which calls address 'to'.*/ |
|
void synthesize_relcall(void *dest, void *from, void *to) |
|
{ |
|
__synthesize_relative_insn(dest, from, to, CALL_INSN_OPCODE); |
|
} |
|
NOKPROBE_SYMBOL(synthesize_relcall); |
|
|
|
/* |
|
* Returns non-zero if INSN is boostable. |
|
* RIP relative instructions are adjusted at copying time in 64 bits mode |
|
*/ |
|
int can_boost(struct insn *insn, void *addr) |
|
{ |
|
kprobe_opcode_t opcode; |
|
insn_byte_t prefix; |
|
int i; |
|
|
|
if (search_exception_tables((unsigned long)addr)) |
|
return 0; /* Page fault may occur on this address. */ |
|
|
|
/* 2nd-byte opcode */ |
|
if (insn->opcode.nbytes == 2) |
|
return test_bit(insn->opcode.bytes[1], |
|
(unsigned long *)twobyte_is_boostable); |
|
|
|
if (insn->opcode.nbytes != 1) |
|
return 0; |
|
|
|
for_each_insn_prefix(insn, i, prefix) { |
|
insn_attr_t attr; |
|
|
|
attr = inat_get_opcode_attribute(prefix); |
|
/* Can't boost Address-size override prefix and CS override prefix */ |
|
if (prefix == 0x2e || inat_is_address_size_prefix(attr)) |
|
return 0; |
|
} |
|
|
|
opcode = insn->opcode.bytes[0]; |
|
|
|
switch (opcode & 0xf0) { |
|
case 0x60: |
|
/* can't boost "bound" */ |
|
return (opcode != 0x62); |
|
case 0x70: |
|
return 0; /* can't boost conditional jump */ |
|
case 0x90: |
|
return opcode != 0x9a; /* can't boost call far */ |
|
case 0xc0: |
|
/* can't boost software-interruptions */ |
|
return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf; |
|
case 0xd0: |
|
/* can boost AA* and XLAT */ |
|
return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7); |
|
case 0xe0: |
|
/* can boost in/out and absolute jmps */ |
|
return ((opcode & 0x04) || opcode == 0xea); |
|
case 0xf0: |
|
/* clear and set flags are boostable */ |
|
return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe)); |
|
default: |
|
/* call is not boostable */ |
|
return opcode != 0x9a; |
|
} |
|
} |
|
|
|
static unsigned long |
|
__recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr) |
|
{ |
|
struct kprobe *kp; |
|
unsigned long faddr; |
|
|
|
kp = get_kprobe((void *)addr); |
|
faddr = ftrace_location(addr); |
|
/* |
|
* Addresses inside the ftrace location are refused by |
|
* arch_check_ftrace_location(). Something went terribly wrong |
|
* if such an address is checked here. |
|
*/ |
|
if (WARN_ON(faddr && faddr != addr)) |
|
return 0UL; |
|
/* |
|
* Use the current code if it is not modified by Kprobe |
|
* and it cannot be modified by ftrace. |
|
*/ |
|
if (!kp && !faddr) |
|
return addr; |
|
|
|
/* |
|
* Basically, kp->ainsn.insn has an original instruction. |
|
* However, RIP-relative instruction can not do single-stepping |
|
* at different place, __copy_instruction() tweaks the displacement of |
|
* that instruction. In that case, we can't recover the instruction |
|
* from the kp->ainsn.insn. |
|
* |
|
* On the other hand, in case on normal Kprobe, kp->opcode has a copy |
|
* of the first byte of the probed instruction, which is overwritten |
|
* by int3. And the instruction at kp->addr is not modified by kprobes |
|
* except for the first byte, we can recover the original instruction |
|
* from it and kp->opcode. |
|
* |
|
* In case of Kprobes using ftrace, we do not have a copy of |
|
* the original instruction. In fact, the ftrace location might |
|
* be modified at anytime and even could be in an inconsistent state. |
|
* Fortunately, we know that the original code is the ideal 5-byte |
|
* long NOP. |
|
*/ |
|
if (copy_from_kernel_nofault(buf, (void *)addr, |
|
MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) |
|
return 0UL; |
|
|
|
if (faddr) |
|
memcpy(buf, ideal_nops[NOP_ATOMIC5], 5); |
|
else |
|
buf[0] = kp->opcode; |
|
return (unsigned long)buf; |
|
} |
|
|
|
/* |
|
* Recover the probed instruction at addr for further analysis. |
|
* Caller must lock kprobes by kprobe_mutex, or disable preemption |
|
* for preventing to release referencing kprobes. |
|
* Returns zero if the instruction can not get recovered (or access failed). |
|
*/ |
|
unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr) |
|
{ |
|
unsigned long __addr; |
|
|
|
__addr = __recover_optprobed_insn(buf, addr); |
|
if (__addr != addr) |
|
return __addr; |
|
|
|
return __recover_probed_insn(buf, addr); |
|
} |
|
|
|
/* Check if paddr is at an instruction boundary */ |
|
static int can_probe(unsigned long paddr) |
|
{ |
|
unsigned long addr, __addr, offset = 0; |
|
struct insn insn; |
|
kprobe_opcode_t buf[MAX_INSN_SIZE]; |
|
|
|
if (!kallsyms_lookup_size_offset(paddr, NULL, &offset)) |
|
return 0; |
|
|
|
/* Decode instructions */ |
|
addr = paddr - offset; |
|
while (addr < paddr) { |
|
/* |
|
* Check if the instruction has been modified by another |
|
* kprobe, in which case we replace the breakpoint by the |
|
* original instruction in our buffer. |
|
* Also, jump optimization will change the breakpoint to |
|
* relative-jump. Since the relative-jump itself is |
|
* normally used, we just go through if there is no kprobe. |
|
*/ |
|
__addr = recover_probed_instruction(buf, addr); |
|
if (!__addr) |
|
return 0; |
|
kernel_insn_init(&insn, (void *)__addr, MAX_INSN_SIZE); |
|
insn_get_length(&insn); |
|
|
|
/* |
|
* Another debugging subsystem might insert this breakpoint. |
|
* In that case, we can't recover it. |
|
*/ |
|
if (insn.opcode.bytes[0] == INT3_INSN_OPCODE) |
|
return 0; |
|
addr += insn.length; |
|
} |
|
|
|
return (addr == paddr); |
|
} |
|
|
|
/* |
|
* Copy an instruction with recovering modified instruction by kprobes |
|
* and adjust the displacement if the instruction uses the %rip-relative |
|
* addressing mode. Note that since @real will be the final place of copied |
|
* instruction, displacement must be adjust by @real, not @dest. |
|
* This returns the length of copied instruction, or 0 if it has an error. |
|
*/ |
|
int __copy_instruction(u8 *dest, u8 *src, u8 *real, struct insn *insn) |
|
{ |
|
kprobe_opcode_t buf[MAX_INSN_SIZE]; |
|
unsigned long recovered_insn = |
|
recover_probed_instruction(buf, (unsigned long)src); |
|
|
|
if (!recovered_insn || !insn) |
|
return 0; |
|
|
|
/* This can access kernel text if given address is not recovered */ |
|
if (copy_from_kernel_nofault(dest, (void *)recovered_insn, |
|
MAX_INSN_SIZE)) |
|
return 0; |
|
|
|
kernel_insn_init(insn, dest, MAX_INSN_SIZE); |
|
insn_get_length(insn); |
|
|
|
/* We can not probe force emulate prefixed instruction */ |
|
if (insn_has_emulate_prefix(insn)) |
|
return 0; |
|
|
|
/* Another subsystem puts a breakpoint, failed to recover */ |
|
if (insn->opcode.bytes[0] == INT3_INSN_OPCODE) |
|
return 0; |
|
|
|
/* We should not singlestep on the exception masking instructions */ |
|
if (insn_masking_exception(insn)) |
|
return 0; |
|
|
|
#ifdef CONFIG_X86_64 |
|
/* Only x86_64 has RIP relative instructions */ |
|
if (insn_rip_relative(insn)) { |
|
s64 newdisp; |
|
u8 *disp; |
|
/* |
|
* The copied instruction uses the %rip-relative addressing |
|
* mode. Adjust the displacement for the difference between |
|
* the original location of this instruction and the location |
|
* of the copy that will actually be run. The tricky bit here |
|
* is making sure that the sign extension happens correctly in |
|
* this calculation, since we need a signed 32-bit result to |
|
* be sign-extended to 64 bits when it's added to the %rip |
|
* value and yield the same 64-bit result that the sign- |
|
* extension of the original signed 32-bit displacement would |
|
* have given. |
|
*/ |
|
newdisp = (u8 *) src + (s64) insn->displacement.value |
|
- (u8 *) real; |
|
if ((s64) (s32) newdisp != newdisp) { |
|
pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp); |
|
return 0; |
|
} |
|
disp = (u8 *) dest + insn_offset_displacement(insn); |
|
*(s32 *) disp = (s32) newdisp; |
|
} |
|
#endif |
|
return insn->length; |
|
} |
|
|
|
/* Prepare reljump right after instruction to boost */ |
|
static int prepare_boost(kprobe_opcode_t *buf, struct kprobe *p, |
|
struct insn *insn) |
|
{ |
|
int len = insn->length; |
|
|
|
if (can_boost(insn, p->addr) && |
|
MAX_INSN_SIZE - len >= JMP32_INSN_SIZE) { |
|
/* |
|
* These instructions can be executed directly if it |
|
* jumps back to correct address. |
|
*/ |
|
synthesize_reljump(buf + len, p->ainsn.insn + len, |
|
p->addr + insn->length); |
|
len += JMP32_INSN_SIZE; |
|
p->ainsn.boostable = 1; |
|
} else { |
|
p->ainsn.boostable = 0; |
|
} |
|
|
|
return len; |
|
} |
|
|
|
/* Make page to RO mode when allocate it */ |
|
void *alloc_insn_page(void) |
|
{ |
|
void *page; |
|
|
|
page = module_alloc(PAGE_SIZE); |
|
if (!page) |
|
return NULL; |
|
|
|
set_vm_flush_reset_perms(page); |
|
/* |
|
* First make the page read-only, and only then make it executable to |
|
* prevent it from being W+X in between. |
|
*/ |
|
set_memory_ro((unsigned long)page, 1); |
|
|
|
/* |
|
* TODO: Once additional kernel code protection mechanisms are set, ensure |
|
* that the page was not maliciously altered and it is still zeroed. |
|
*/ |
|
set_memory_x((unsigned long)page, 1); |
|
|
|
return page; |
|
} |
|
|
|
/* Recover page to RW mode before releasing it */ |
|
void free_insn_page(void *page) |
|
{ |
|
module_memfree(page); |
|
} |
|
|
|
static void set_resume_flags(struct kprobe *p, struct insn *insn) |
|
{ |
|
insn_byte_t opcode = insn->opcode.bytes[0]; |
|
|
|
switch (opcode) { |
|
case 0xfa: /* cli */ |
|
case 0xfb: /* sti */ |
|
case 0x9d: /* popf/popfd */ |
|
/* Check whether the instruction modifies Interrupt Flag or not */ |
|
p->ainsn.if_modifier = 1; |
|
break; |
|
case 0x9c: /* pushfl */ |
|
p->ainsn.is_pushf = 1; |
|
break; |
|
case 0xcf: /* iret */ |
|
p->ainsn.if_modifier = 1; |
|
fallthrough; |
|
case 0xc2: /* ret/lret */ |
|
case 0xc3: |
|
case 0xca: |
|
case 0xcb: |
|
case 0xea: /* jmp absolute -- ip is correct */ |
|
/* ip is already adjusted, no more changes required */ |
|
p->ainsn.is_abs_ip = 1; |
|
/* Without resume jump, this is boostable */ |
|
p->ainsn.boostable = 1; |
|
break; |
|
case 0xe8: /* call relative - Fix return addr */ |
|
p->ainsn.is_call = 1; |
|
break; |
|
#ifdef CONFIG_X86_32 |
|
case 0x9a: /* call absolute -- same as call absolute, indirect */ |
|
p->ainsn.is_call = 1; |
|
p->ainsn.is_abs_ip = 1; |
|
break; |
|
#endif |
|
case 0xff: |
|
/* |
|
* Since the 0xff is an extended group opcode, the instruction |
|
* is determined by the MOD/RM byte. |
|
*/ |
|
opcode = insn->modrm.bytes[0]; |
|
if ((opcode & 0x30) == 0x10) { |
|
/* |
|
* call absolute, indirect |
|
* Fix return addr; ip is correct. |
|
* But this is not boostable |
|
*/ |
|
p->ainsn.is_call = 1; |
|
p->ainsn.is_abs_ip = 1; |
|
break; |
|
} else if (((opcode & 0x31) == 0x20) || |
|
((opcode & 0x31) == 0x21)) { |
|
/* |
|
* jmp near and far, absolute indirect |
|
* ip is correct. |
|
*/ |
|
p->ainsn.is_abs_ip = 1; |
|
/* Without resume jump, this is boostable */ |
|
p->ainsn.boostable = 1; |
|
} |
|
break; |
|
} |
|
} |
|
|
|
static int arch_copy_kprobe(struct kprobe *p) |
|
{ |
|
struct insn insn; |
|
kprobe_opcode_t buf[MAX_INSN_SIZE]; |
|
int len; |
|
|
|
/* Copy an instruction with recovering if other optprobe modifies it.*/ |
|
len = __copy_instruction(buf, p->addr, p->ainsn.insn, &insn); |
|
if (!len) |
|
return -EINVAL; |
|
|
|
/* |
|
* __copy_instruction can modify the displacement of the instruction, |
|
* but it doesn't affect boostable check. |
|
*/ |
|
len = prepare_boost(buf, p, &insn); |
|
|
|
/* Analyze the opcode and set resume flags */ |
|
set_resume_flags(p, &insn); |
|
|
|
/* Also, displacement change doesn't affect the first byte */ |
|
p->opcode = buf[0]; |
|
|
|
p->ainsn.tp_len = len; |
|
perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len); |
|
|
|
/* OK, write back the instruction(s) into ROX insn buffer */ |
|
text_poke(p->ainsn.insn, buf, len); |
|
|
|
return 0; |
|
} |
|
|
|
int arch_prepare_kprobe(struct kprobe *p) |
|
{ |
|
int ret; |
|
|
|
if (alternatives_text_reserved(p->addr, p->addr)) |
|
return -EINVAL; |
|
|
|
if (!can_probe((unsigned long)p->addr)) |
|
return -EILSEQ; |
|
|
|
memset(&p->ainsn, 0, sizeof(p->ainsn)); |
|
|
|
/* insn: must be on special executable page on x86. */ |
|
p->ainsn.insn = get_insn_slot(); |
|
if (!p->ainsn.insn) |
|
return -ENOMEM; |
|
|
|
ret = arch_copy_kprobe(p); |
|
if (ret) { |
|
free_insn_slot(p->ainsn.insn, 0); |
|
p->ainsn.insn = NULL; |
|
} |
|
|
|
return ret; |
|
} |
|
|
|
void arch_arm_kprobe(struct kprobe *p) |
|
{ |
|
u8 int3 = INT3_INSN_OPCODE; |
|
|
|
text_poke(p->addr, &int3, 1); |
|
text_poke_sync(); |
|
perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1); |
|
} |
|
|
|
void arch_disarm_kprobe(struct kprobe *p) |
|
{ |
|
u8 int3 = INT3_INSN_OPCODE; |
|
|
|
perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1); |
|
text_poke(p->addr, &p->opcode, 1); |
|
text_poke_sync(); |
|
} |
|
|
|
void arch_remove_kprobe(struct kprobe *p) |
|
{ |
|
if (p->ainsn.insn) { |
|
/* Record the perf event before freeing the slot */ |
|
perf_event_text_poke(p->ainsn.insn, p->ainsn.insn, |
|
p->ainsn.tp_len, NULL, 0); |
|
free_insn_slot(p->ainsn.insn, p->ainsn.boostable); |
|
p->ainsn.insn = NULL; |
|
} |
|
} |
|
|
|
static nokprobe_inline void |
|
save_previous_kprobe(struct kprobe_ctlblk *kcb) |
|
{ |
|
kcb->prev_kprobe.kp = kprobe_running(); |
|
kcb->prev_kprobe.status = kcb->kprobe_status; |
|
kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags; |
|
kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags; |
|
} |
|
|
|
static nokprobe_inline void |
|
restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
|
{ |
|
__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
|
kcb->kprobe_status = kcb->prev_kprobe.status; |
|
kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags; |
|
kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags; |
|
} |
|
|
|
static nokprobe_inline void |
|
set_current_kprobe(struct kprobe *p, struct pt_regs *regs, |
|
struct kprobe_ctlblk *kcb) |
|
{ |
|
__this_cpu_write(current_kprobe, p); |
|
kcb->kprobe_saved_flags = kcb->kprobe_old_flags |
|
= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF)); |
|
if (p->ainsn.if_modifier) |
|
kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF; |
|
} |
|
|
|
static nokprobe_inline void clear_btf(void) |
|
{ |
|
if (test_thread_flag(TIF_BLOCKSTEP)) { |
|
unsigned long debugctl = get_debugctlmsr(); |
|
|
|
debugctl &= ~DEBUGCTLMSR_BTF; |
|
update_debugctlmsr(debugctl); |
|
} |
|
} |
|
|
|
static nokprobe_inline void restore_btf(void) |
|
{ |
|
if (test_thread_flag(TIF_BLOCKSTEP)) { |
|
unsigned long debugctl = get_debugctlmsr(); |
|
|
|
debugctl |= DEBUGCTLMSR_BTF; |
|
update_debugctlmsr(debugctl); |
|
} |
|
} |
|
|
|
void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) |
|
{ |
|
unsigned long *sara = stack_addr(regs); |
|
|
|
ri->ret_addr = (kprobe_opcode_t *) *sara; |
|
ri->fp = sara; |
|
|
|
/* Replace the return addr with trampoline addr */ |
|
*sara = (unsigned long) &kretprobe_trampoline; |
|
} |
|
NOKPROBE_SYMBOL(arch_prepare_kretprobe); |
|
|
|
static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, |
|
struct kprobe_ctlblk *kcb, int reenter) |
|
{ |
|
if (setup_detour_execution(p, regs, reenter)) |
|
return; |
|
|
|
#if !defined(CONFIG_PREEMPTION) |
|
if (p->ainsn.boostable && !p->post_handler) { |
|
/* Boost up -- we can execute copied instructions directly */ |
|
if (!reenter) |
|
reset_current_kprobe(); |
|
/* |
|
* Reentering boosted probe doesn't reset current_kprobe, |
|
* nor set current_kprobe, because it doesn't use single |
|
* stepping. |
|
*/ |
|
regs->ip = (unsigned long)p->ainsn.insn; |
|
return; |
|
} |
|
#endif |
|
if (reenter) { |
|
save_previous_kprobe(kcb); |
|
set_current_kprobe(p, regs, kcb); |
|
kcb->kprobe_status = KPROBE_REENTER; |
|
} else |
|
kcb->kprobe_status = KPROBE_HIT_SS; |
|
/* Prepare real single stepping */ |
|
clear_btf(); |
|
regs->flags |= X86_EFLAGS_TF; |
|
regs->flags &= ~X86_EFLAGS_IF; |
|
/* single step inline if the instruction is an int3 */ |
|
if (p->opcode == INT3_INSN_OPCODE) |
|
regs->ip = (unsigned long)p->addr; |
|
else |
|
regs->ip = (unsigned long)p->ainsn.insn; |
|
} |
|
NOKPROBE_SYMBOL(setup_singlestep); |
|
|
|
/* |
|
* We have reentered the kprobe_handler(), since another probe was hit while |
|
* within the handler. We save the original kprobes variables and just single |
|
* step on the instruction of the new probe without calling any user handlers. |
|
*/ |
|
static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs, |
|
struct kprobe_ctlblk *kcb) |
|
{ |
|
switch (kcb->kprobe_status) { |
|
case KPROBE_HIT_SSDONE: |
|
case KPROBE_HIT_ACTIVE: |
|
case KPROBE_HIT_SS: |
|
kprobes_inc_nmissed_count(p); |
|
setup_singlestep(p, regs, kcb, 1); |
|
break; |
|
case KPROBE_REENTER: |
|
/* A probe has been hit in the codepath leading up to, or just |
|
* after, single-stepping of a probed instruction. This entire |
|
* codepath should strictly reside in .kprobes.text section. |
|
* Raise a BUG or we'll continue in an endless reentering loop |
|
* and eventually a stack overflow. |
|
*/ |
|
pr_err("Unrecoverable kprobe detected.\n"); |
|
dump_kprobe(p); |
|
BUG(); |
|
default: |
|
/* impossible cases */ |
|
WARN_ON(1); |
|
return 0; |
|
} |
|
|
|
return 1; |
|
} |
|
NOKPROBE_SYMBOL(reenter_kprobe); |
|
|
|
/* |
|
* Interrupts are disabled on entry as trap3 is an interrupt gate and they |
|
* remain disabled throughout this function. |
|
*/ |
|
int kprobe_int3_handler(struct pt_regs *regs) |
|
{ |
|
kprobe_opcode_t *addr; |
|
struct kprobe *p; |
|
struct kprobe_ctlblk *kcb; |
|
|
|
if (user_mode(regs)) |
|
return 0; |
|
|
|
addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t)); |
|
/* |
|
* We don't want to be preempted for the entire duration of kprobe |
|
* processing. Since int3 and debug trap disables irqs and we clear |
|
* IF while singlestepping, it must be no preemptible. |
|
*/ |
|
|
|
kcb = get_kprobe_ctlblk(); |
|
p = get_kprobe(addr); |
|
|
|
if (p) { |
|
if (kprobe_running()) { |
|
if (reenter_kprobe(p, regs, kcb)) |
|
return 1; |
|
} else { |
|
set_current_kprobe(p, regs, kcb); |
|
kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
|
|
|
/* |
|
* If we have no pre-handler or it returned 0, we |
|
* continue with normal processing. If we have a |
|
* pre-handler and it returned non-zero, that means |
|
* user handler setup registers to exit to another |
|
* instruction, we must skip the single stepping. |
|
*/ |
|
if (!p->pre_handler || !p->pre_handler(p, regs)) |
|
setup_singlestep(p, regs, kcb, 0); |
|
else |
|
reset_current_kprobe(); |
|
return 1; |
|
} |
|
} else if (*addr != INT3_INSN_OPCODE) { |
|
/* |
|
* The breakpoint instruction was removed right |
|
* after we hit it. Another cpu has removed |
|
* either a probepoint or a debugger breakpoint |
|
* at this address. In either case, no further |
|
* handling of this interrupt is appropriate. |
|
* Back up over the (now missing) int3 and run |
|
* the original instruction. |
|
*/ |
|
regs->ip = (unsigned long)addr; |
|
return 1; |
|
} /* else: not a kprobe fault; let the kernel handle it */ |
|
|
|
return 0; |
|
} |
|
NOKPROBE_SYMBOL(kprobe_int3_handler); |
|
|
|
/* |
|
* When a retprobed function returns, this code saves registers and |
|
* calls trampoline_handler() runs, which calls the kretprobe's handler. |
|
*/ |
|
asm( |
|
".text\n" |
|
".global kretprobe_trampoline\n" |
|
".type kretprobe_trampoline, @function\n" |
|
"kretprobe_trampoline:\n" |
|
/* We don't bother saving the ss register */ |
|
#ifdef CONFIG_X86_64 |
|
" pushq %rsp\n" |
|
" pushfq\n" |
|
SAVE_REGS_STRING |
|
" movq %rsp, %rdi\n" |
|
" call trampoline_handler\n" |
|
/* Replace saved sp with true return address. */ |
|
" movq %rax, 19*8(%rsp)\n" |
|
RESTORE_REGS_STRING |
|
" popfq\n" |
|
#else |
|
" pushl %esp\n" |
|
" pushfl\n" |
|
SAVE_REGS_STRING |
|
" movl %esp, %eax\n" |
|
" call trampoline_handler\n" |
|
/* Replace saved sp with true return address. */ |
|
" movl %eax, 15*4(%esp)\n" |
|
RESTORE_REGS_STRING |
|
" popfl\n" |
|
#endif |
|
" ret\n" |
|
".size kretprobe_trampoline, .-kretprobe_trampoline\n" |
|
); |
|
NOKPROBE_SYMBOL(kretprobe_trampoline); |
|
STACK_FRAME_NON_STANDARD(kretprobe_trampoline); |
|
|
|
|
|
/* |
|
* Called from kretprobe_trampoline |
|
*/ |
|
__used __visible void *trampoline_handler(struct pt_regs *regs) |
|
{ |
|
/* fixup registers */ |
|
regs->cs = __KERNEL_CS; |
|
#ifdef CONFIG_X86_32 |
|
regs->gs = 0; |
|
#endif |
|
regs->ip = (unsigned long)&kretprobe_trampoline; |
|
regs->orig_ax = ~0UL; |
|
|
|
return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, ®s->sp); |
|
} |
|
NOKPROBE_SYMBOL(trampoline_handler); |
|
|
|
/* |
|
* Called after single-stepping. p->addr is the address of the |
|
* instruction whose first byte has been replaced by the "int 3" |
|
* instruction. To avoid the SMP problems that can occur when we |
|
* temporarily put back the original opcode to single-step, we |
|
* single-stepped a copy of the instruction. The address of this |
|
* copy is p->ainsn.insn. |
|
* |
|
* This function prepares to return from the post-single-step |
|
* interrupt. We have to fix up the stack as follows: |
|
* |
|
* 0) Except in the case of absolute or indirect jump or call instructions, |
|
* the new ip is relative to the copied instruction. We need to make |
|
* it relative to the original instruction. |
|
* |
|
* 1) If the single-stepped instruction was pushfl, then the TF and IF |
|
* flags are set in the just-pushed flags, and may need to be cleared. |
|
* |
|
* 2) If the single-stepped instruction was a call, the return address |
|
* that is atop the stack is the address following the copied instruction. |
|
* We need to make it the address following the original instruction. |
|
*/ |
|
static void resume_execution(struct kprobe *p, struct pt_regs *regs, |
|
struct kprobe_ctlblk *kcb) |
|
{ |
|
unsigned long *tos = stack_addr(regs); |
|
unsigned long copy_ip = (unsigned long)p->ainsn.insn; |
|
unsigned long orig_ip = (unsigned long)p->addr; |
|
|
|
regs->flags &= ~X86_EFLAGS_TF; |
|
|
|
/* Fixup the contents of top of stack */ |
|
if (p->ainsn.is_pushf) { |
|
*tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF); |
|
*tos |= kcb->kprobe_old_flags; |
|
} else if (p->ainsn.is_call) { |
|
*tos = orig_ip + (*tos - copy_ip); |
|
} |
|
|
|
if (!p->ainsn.is_abs_ip) |
|
regs->ip += orig_ip - copy_ip; |
|
|
|
restore_btf(); |
|
} |
|
NOKPROBE_SYMBOL(resume_execution); |
|
|
|
/* |
|
* Interrupts are disabled on entry as trap1 is an interrupt gate and they |
|
* remain disabled throughout this function. |
|
*/ |
|
int kprobe_debug_handler(struct pt_regs *regs) |
|
{ |
|
struct kprobe *cur = kprobe_running(); |
|
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
|
|
|
if (!cur) |
|
return 0; |
|
|
|
resume_execution(cur, regs, kcb); |
|
regs->flags |= kcb->kprobe_saved_flags; |
|
|
|
if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
|
kcb->kprobe_status = KPROBE_HIT_SSDONE; |
|
cur->post_handler(cur, regs, 0); |
|
} |
|
|
|
/* Restore back the original saved kprobes variables and continue. */ |
|
if (kcb->kprobe_status == KPROBE_REENTER) { |
|
restore_previous_kprobe(kcb); |
|
goto out; |
|
} |
|
reset_current_kprobe(); |
|
out: |
|
/* |
|
* if somebody else is singlestepping across a probe point, flags |
|
* will have TF set, in which case, continue the remaining processing |
|
* of do_debug, as if this is not a probe hit. |
|
*/ |
|
if (regs->flags & X86_EFLAGS_TF) |
|
return 0; |
|
|
|
return 1; |
|
} |
|
NOKPROBE_SYMBOL(kprobe_debug_handler); |
|
|
|
int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
|
{ |
|
struct kprobe *cur = kprobe_running(); |
|
struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
|
|
|
if (unlikely(regs->ip == (unsigned long)cur->ainsn.insn)) { |
|
/* This must happen on single-stepping */ |
|
WARN_ON(kcb->kprobe_status != KPROBE_HIT_SS && |
|
kcb->kprobe_status != KPROBE_REENTER); |
|
/* |
|
* We are here because the instruction being single |
|
* stepped caused a page fault. We reset the current |
|
* kprobe and the ip points back to the probe address |
|
* and allow the page fault handler to continue as a |
|
* normal page fault. |
|
*/ |
|
regs->ip = (unsigned long)cur->addr; |
|
/* |
|
* Trap flag (TF) has been set here because this fault |
|
* happened where the single stepping will be done. |
|
* So clear it by resetting the current kprobe: |
|
*/ |
|
regs->flags &= ~X86_EFLAGS_TF; |
|
/* |
|
* Since the single step (trap) has been cancelled, |
|
* we need to restore BTF here. |
|
*/ |
|
restore_btf(); |
|
|
|
/* |
|
* If the TF flag was set before the kprobe hit, |
|
* don't touch it: |
|
*/ |
|
regs->flags |= kcb->kprobe_old_flags; |
|
|
|
if (kcb->kprobe_status == KPROBE_REENTER) |
|
restore_previous_kprobe(kcb); |
|
else |
|
reset_current_kprobe(); |
|
} else if (kcb->kprobe_status == KPROBE_HIT_ACTIVE || |
|
kcb->kprobe_status == KPROBE_HIT_SSDONE) { |
|
/* |
|
* We increment the nmissed count for accounting, |
|
* we can also use npre/npostfault count for accounting |
|
* these specific fault cases. |
|
*/ |
|
kprobes_inc_nmissed_count(cur); |
|
|
|
/* |
|
* We come here because instructions in the pre/post |
|
* handler caused the page_fault, this could happen |
|
* if handler tries to access user space by |
|
* copy_from_user(), get_user() etc. Let the |
|
* user-specified handler try to fix it first. |
|
*/ |
|
if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) |
|
return 1; |
|
} |
|
|
|
return 0; |
|
} |
|
NOKPROBE_SYMBOL(kprobe_fault_handler); |
|
|
|
int __init arch_populate_kprobe_blacklist(void) |
|
{ |
|
return kprobe_add_area_blacklist((unsigned long)__entry_text_start, |
|
(unsigned long)__entry_text_end); |
|
} |
|
|
|
int __init arch_init_kprobes(void) |
|
{ |
|
return 0; |
|
} |
|
|
|
int arch_trampoline_kprobe(struct kprobe *p) |
|
{ |
|
return 0; |
|
}
|
|
|