forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
133 lines
3.6 KiB
133 lines
3.6 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* preemptoff and irqoff tracepoints |
|
* |
|
* Copyright (C) Joel Fernandes (Google) <[email protected]> |
|
*/ |
|
|
|
#include <linux/kallsyms.h> |
|
#include <linux/uaccess.h> |
|
#include <linux/module.h> |
|
#include <linux/ftrace.h> |
|
#include <linux/kprobes.h> |
|
#include "trace.h" |
|
|
|
#define CREATE_TRACE_POINTS |
|
#include <trace/events/preemptirq.h> |
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS |
|
/* Per-cpu variable to prevent redundant calls when IRQs already off */ |
|
static DEFINE_PER_CPU(int, tracing_irq_cpu); |
|
|
|
/* |
|
* Like trace_hardirqs_on() but without the lockdep invocation. This is |
|
* used in the low level entry code where the ordering vs. RCU is important |
|
* and lockdep uses a staged approach which splits the lockdep hardirq |
|
* tracking into a RCU on and a RCU off section. |
|
*/ |
|
void trace_hardirqs_on_prepare(void) |
|
{ |
|
if (this_cpu_read(tracing_irq_cpu)) { |
|
if (!in_nmi()) |
|
trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); |
|
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
|
this_cpu_write(tracing_irq_cpu, 0); |
|
} |
|
} |
|
EXPORT_SYMBOL(trace_hardirqs_on_prepare); |
|
NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); |
|
|
|
void trace_hardirqs_on(void) |
|
{ |
|
if (this_cpu_read(tracing_irq_cpu)) { |
|
if (!in_nmi()) |
|
trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); |
|
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); |
|
this_cpu_write(tracing_irq_cpu, 0); |
|
} |
|
|
|
lockdep_hardirqs_on_prepare(CALLER_ADDR0); |
|
lockdep_hardirqs_on(CALLER_ADDR0); |
|
} |
|
EXPORT_SYMBOL(trace_hardirqs_on); |
|
NOKPROBE_SYMBOL(trace_hardirqs_on); |
|
|
|
/* |
|
* Like trace_hardirqs_off() but without the lockdep invocation. This is |
|
* used in the low level entry code where the ordering vs. RCU is important |
|
* and lockdep uses a staged approach which splits the lockdep hardirq |
|
* tracking into a RCU on and a RCU off section. |
|
*/ |
|
void trace_hardirqs_off_finish(void) |
|
{ |
|
if (!this_cpu_read(tracing_irq_cpu)) { |
|
this_cpu_write(tracing_irq_cpu, 1); |
|
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
|
if (!in_nmi()) |
|
trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); |
|
} |
|
|
|
} |
|
EXPORT_SYMBOL(trace_hardirqs_off_finish); |
|
NOKPROBE_SYMBOL(trace_hardirqs_off_finish); |
|
|
|
void trace_hardirqs_off(void) |
|
{ |
|
lockdep_hardirqs_off(CALLER_ADDR0); |
|
|
|
if (!this_cpu_read(tracing_irq_cpu)) { |
|
this_cpu_write(tracing_irq_cpu, 1); |
|
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); |
|
if (!in_nmi()) |
|
trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); |
|
} |
|
} |
|
EXPORT_SYMBOL(trace_hardirqs_off); |
|
NOKPROBE_SYMBOL(trace_hardirqs_off); |
|
|
|
__visible void trace_hardirqs_on_caller(unsigned long caller_addr) |
|
{ |
|
if (this_cpu_read(tracing_irq_cpu)) { |
|
if (!in_nmi()) |
|
trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr); |
|
tracer_hardirqs_on(CALLER_ADDR0, caller_addr); |
|
this_cpu_write(tracing_irq_cpu, 0); |
|
} |
|
|
|
lockdep_hardirqs_on_prepare(CALLER_ADDR0); |
|
lockdep_hardirqs_on(CALLER_ADDR0); |
|
} |
|
EXPORT_SYMBOL(trace_hardirqs_on_caller); |
|
NOKPROBE_SYMBOL(trace_hardirqs_on_caller); |
|
|
|
__visible void trace_hardirqs_off_caller(unsigned long caller_addr) |
|
{ |
|
lockdep_hardirqs_off(CALLER_ADDR0); |
|
|
|
if (!this_cpu_read(tracing_irq_cpu)) { |
|
this_cpu_write(tracing_irq_cpu, 1); |
|
tracer_hardirqs_off(CALLER_ADDR0, caller_addr); |
|
if (!in_nmi()) |
|
trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr); |
|
} |
|
} |
|
EXPORT_SYMBOL(trace_hardirqs_off_caller); |
|
NOKPROBE_SYMBOL(trace_hardirqs_off_caller); |
|
#endif /* CONFIG_TRACE_IRQFLAGS */ |
|
|
|
#ifdef CONFIG_TRACE_PREEMPT_TOGGLE |
|
|
|
void trace_preempt_on(unsigned long a0, unsigned long a1) |
|
{ |
|
if (!in_nmi()) |
|
trace_preempt_enable_rcuidle(a0, a1); |
|
tracer_preempt_on(a0, a1); |
|
} |
|
|
|
void trace_preempt_off(unsigned long a0, unsigned long a1) |
|
{ |
|
if (!in_nmi()) |
|
trace_preempt_disable_rcuidle(a0, a1); |
|
tracer_preempt_off(a0, a1); |
|
} |
|
#endif
|
|
|