mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1521 lines
37 KiB
1521 lines
37 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* Copyright (C) 2008-2017 Andes Technology Corporation |
|
* |
|
* Reference ARMv7: Jean Pihet <[email protected]> |
|
* 2010 (c) MontaVista Software, LLC. |
|
*/ |
|
|
|
#include <linux/perf_event.h> |
|
#include <linux/bitmap.h> |
|
#include <linux/export.h> |
|
#include <linux/kernel.h> |
|
#include <linux/of.h> |
|
#include <linux/platform_device.h> |
|
#include <linux/slab.h> |
|
#include <linux/spinlock.h> |
|
#include <linux/pm_runtime.h> |
|
#include <linux/ftrace.h> |
|
#include <linux/uaccess.h> |
|
#include <linux/sched/clock.h> |
|
#include <linux/percpu-defs.h> |
|
|
|
#include <asm/pmu.h> |
|
#include <asm/irq_regs.h> |
|
#include <asm/nds32.h> |
|
#include <asm/stacktrace.h> |
|
#include <asm/perf_event.h> |
|
#include <nds32_intrinsic.h> |
|
|
|
/* Set at runtime when we know what CPU type we are. */ |
|
static struct nds32_pmu *cpu_pmu; |
|
|
|
static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events); |
|
static void nds32_pmu_start(struct nds32_pmu *cpu_pmu); |
|
static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu); |
|
static struct platform_device_id cpu_pmu_plat_device_ids[] = { |
|
{.name = "nds32-pfm"}, |
|
{}, |
|
}; |
|
|
|
static int nds32_pmu_map_cache_event(const unsigned int (*cache_map) |
|
[PERF_COUNT_HW_CACHE_MAX] |
|
[PERF_COUNT_HW_CACHE_OP_MAX] |
|
[PERF_COUNT_HW_CACHE_RESULT_MAX], u64 config) |
|
{ |
|
unsigned int cache_type, cache_op, cache_result, ret; |
|
|
|
cache_type = (config >> 0) & 0xff; |
|
if (cache_type >= PERF_COUNT_HW_CACHE_MAX) |
|
return -EINVAL; |
|
|
|
cache_op = (config >> 8) & 0xff; |
|
if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) |
|
return -EINVAL; |
|
|
|
cache_result = (config >> 16) & 0xff; |
|
if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
|
return -EINVAL; |
|
|
|
ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; |
|
|
|
if (ret == CACHE_OP_UNSUPPORTED) |
|
return -ENOENT; |
|
|
|
return ret; |
|
} |
|
|
|
static int |
|
nds32_pmu_map_hw_event(const unsigned int (*event_map)[PERF_COUNT_HW_MAX], |
|
u64 config) |
|
{ |
|
int mapping; |
|
|
|
if (config >= PERF_COUNT_HW_MAX) |
|
return -ENOENT; |
|
|
|
mapping = (*event_map)[config]; |
|
return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; |
|
} |
|
|
|
static int nds32_pmu_map_raw_event(u32 raw_event_mask, u64 config) |
|
{ |
|
int ev_type = (int)(config & raw_event_mask); |
|
int idx = config >> 8; |
|
|
|
switch (idx) { |
|
case 0: |
|
ev_type = PFM_OFFSET_MAGIC_0 + ev_type; |
|
if (ev_type >= SPAV3_0_SEL_LAST || ev_type <= SPAV3_0_SEL_BASE) |
|
return -ENOENT; |
|
break; |
|
case 1: |
|
ev_type = PFM_OFFSET_MAGIC_1 + ev_type; |
|
if (ev_type >= SPAV3_1_SEL_LAST || ev_type <= SPAV3_1_SEL_BASE) |
|
return -ENOENT; |
|
break; |
|
case 2: |
|
ev_type = PFM_OFFSET_MAGIC_2 + ev_type; |
|
if (ev_type >= SPAV3_2_SEL_LAST || ev_type <= SPAV3_2_SEL_BASE) |
|
return -ENOENT; |
|
break; |
|
default: |
|
return -ENOENT; |
|
} |
|
|
|
return ev_type; |
|
} |
|
|
|
int |
|
nds32_pmu_map_event(struct perf_event *event, |
|
const unsigned int (*event_map)[PERF_COUNT_HW_MAX], |
|
const unsigned int (*cache_map) |
|
[PERF_COUNT_HW_CACHE_MAX] |
|
[PERF_COUNT_HW_CACHE_OP_MAX] |
|
[PERF_COUNT_HW_CACHE_RESULT_MAX], u32 raw_event_mask) |
|
{ |
|
u64 config = event->attr.config; |
|
|
|
switch (event->attr.type) { |
|
case PERF_TYPE_HARDWARE: |
|
return nds32_pmu_map_hw_event(event_map, config); |
|
case PERF_TYPE_HW_CACHE: |
|
return nds32_pmu_map_cache_event(cache_map, config); |
|
case PERF_TYPE_RAW: |
|
return nds32_pmu_map_raw_event(raw_event_mask, config); |
|
} |
|
|
|
return -ENOENT; |
|
} |
|
|
|
static int nds32_spav3_map_event(struct perf_event *event) |
|
{ |
|
return nds32_pmu_map_event(event, &nds32_pfm_perf_map, |
|
&nds32_pfm_perf_cache_map, SOFTWARE_EVENT_MASK); |
|
} |
|
|
|
static inline u32 nds32_pfm_getreset_flags(void) |
|
{ |
|
/* Read overflow status */ |
|
u32 val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
u32 old_val = val; |
|
|
|
/* Write overflow bit to clear status, and others keep it 0 */ |
|
u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]; |
|
|
|
__nds32__mtsr(val | ov_flag, NDS32_SR_PFM_CTL); |
|
|
|
return old_val; |
|
} |
|
|
|
static inline int nds32_pfm_has_overflowed(u32 pfm) |
|
{ |
|
u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]; |
|
|
|
return pfm & ov_flag; |
|
} |
|
|
|
static inline int nds32_pfm_counter_has_overflowed(u32 pfm, int idx) |
|
{ |
|
u32 mask = 0; |
|
|
|
switch (idx) { |
|
case 0: |
|
mask = PFM_CTL_OVF[0]; |
|
break; |
|
case 1: |
|
mask = PFM_CTL_OVF[1]; |
|
break; |
|
case 2: |
|
mask = PFM_CTL_OVF[2]; |
|
break; |
|
default: |
|
pr_err("%s index wrong\n", __func__); |
|
break; |
|
} |
|
return pfm & mask; |
|
} |
|
|
|
/* |
|
* Set the next IRQ period, based on the hwc->period_left value. |
|
* To be called with the event disabled in hw: |
|
*/ |
|
int nds32_pmu_event_set_period(struct perf_event *event) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
s64 left = local64_read(&hwc->period_left); |
|
s64 period = hwc->sample_period; |
|
int ret = 0; |
|
|
|
/* The period may have been changed by PERF_EVENT_IOC_PERIOD */ |
|
if (unlikely(period != hwc->last_period)) |
|
left = period - (hwc->last_period - left); |
|
|
|
if (unlikely(left <= -period)) { |
|
left = period; |
|
local64_set(&hwc->period_left, left); |
|
hwc->last_period = period; |
|
ret = 1; |
|
} |
|
|
|
if (unlikely(left <= 0)) { |
|
left += period; |
|
local64_set(&hwc->period_left, left); |
|
hwc->last_period = period; |
|
ret = 1; |
|
} |
|
|
|
if (left > (s64)nds32_pmu->max_period) |
|
left = nds32_pmu->max_period; |
|
|
|
/* |
|
* The hw event starts counting from this event offset, |
|
* mark it to be able to extract future "deltas": |
|
*/ |
|
local64_set(&hwc->prev_count, (u64)(-left)); |
|
|
|
nds32_pmu->write_counter(event, (u64)(-left) & nds32_pmu->max_period); |
|
|
|
perf_event_update_userpage(event); |
|
|
|
return ret; |
|
} |
|
|
|
static irqreturn_t nds32_pmu_handle_irq(int irq_num, void *dev) |
|
{ |
|
u32 pfm; |
|
struct perf_sample_data data; |
|
struct nds32_pmu *cpu_pmu = (struct nds32_pmu *)dev; |
|
struct pmu_hw_events *cpuc = cpu_pmu->get_hw_events(); |
|
struct pt_regs *regs; |
|
int idx; |
|
/* |
|
* Get and reset the IRQ flags |
|
*/ |
|
pfm = nds32_pfm_getreset_flags(); |
|
|
|
/* |
|
* Did an overflow occur? |
|
*/ |
|
if (!nds32_pfm_has_overflowed(pfm)) |
|
return IRQ_NONE; |
|
|
|
/* |
|
* Handle the counter(s) overflow(s) |
|
*/ |
|
regs = get_irq_regs(); |
|
|
|
nds32_pmu_stop(cpu_pmu); |
|
for (idx = 0; idx < cpu_pmu->num_events; ++idx) { |
|
struct perf_event *event = cpuc->events[idx]; |
|
struct hw_perf_event *hwc; |
|
|
|
/* Ignore if we don't have an event. */ |
|
if (!event) |
|
continue; |
|
|
|
/* |
|
* We have a single interrupt for all counters. Check that |
|
* each counter has overflowed before we process it. |
|
*/ |
|
if (!nds32_pfm_counter_has_overflowed(pfm, idx)) |
|
continue; |
|
|
|
hwc = &event->hw; |
|
nds32_pmu_event_update(event); |
|
perf_sample_data_init(&data, 0, hwc->last_period); |
|
if (!nds32_pmu_event_set_period(event)) |
|
continue; |
|
|
|
if (perf_event_overflow(event, &data, regs)) |
|
cpu_pmu->disable(event); |
|
} |
|
nds32_pmu_start(cpu_pmu); |
|
/* |
|
* Handle the pending perf events. |
|
* |
|
* Note: this call *must* be run with interrupts disabled. For |
|
* platforms that can have the PMU interrupts raised as an NMI, this |
|
* will not work. |
|
*/ |
|
irq_work_run(); |
|
|
|
return IRQ_HANDLED; |
|
} |
|
|
|
static inline int nds32_pfm_counter_valid(struct nds32_pmu *cpu_pmu, int idx) |
|
{ |
|
return ((idx >= 0) && (idx < cpu_pmu->num_events)); |
|
} |
|
|
|
static inline int nds32_pfm_disable_counter(int idx) |
|
{ |
|
unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
u32 mask = 0; |
|
|
|
mask = PFM_CTL_EN[idx]; |
|
val &= ~mask; |
|
val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); |
|
return idx; |
|
} |
|
|
|
/* |
|
* Add an event filter to a given event. |
|
*/ |
|
static int nds32_pmu_set_event_filter(struct hw_perf_event *event, |
|
struct perf_event_attr *attr) |
|
{ |
|
unsigned long config_base = 0; |
|
int idx = event->idx; |
|
unsigned long no_kernel_tracing = 0; |
|
unsigned long no_user_tracing = 0; |
|
/* If index is -1, do not do anything */ |
|
if (idx == -1) |
|
return 0; |
|
|
|
no_kernel_tracing = PFM_CTL_KS[idx]; |
|
no_user_tracing = PFM_CTL_KU[idx]; |
|
/* |
|
* Default: enable both kernel and user mode tracing. |
|
*/ |
|
if (attr->exclude_user) |
|
config_base |= no_user_tracing; |
|
|
|
if (attr->exclude_kernel) |
|
config_base |= no_kernel_tracing; |
|
|
|
/* |
|
* Install the filter into config_base as this is used to |
|
* construct the event type. |
|
*/ |
|
event->config_base |= config_base; |
|
return 0; |
|
} |
|
|
|
static inline void nds32_pfm_write_evtsel(int idx, u32 evnum) |
|
{ |
|
u32 offset = 0; |
|
u32 ori_val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
u32 ev_mask = 0; |
|
u32 no_kernel_mask = 0; |
|
u32 no_user_mask = 0; |
|
u32 val; |
|
|
|
offset = PFM_CTL_OFFSEL[idx]; |
|
/* Clear previous mode selection, and write new one */ |
|
no_kernel_mask = PFM_CTL_KS[idx]; |
|
no_user_mask = PFM_CTL_KU[idx]; |
|
ori_val &= ~no_kernel_mask; |
|
ori_val &= ~no_user_mask; |
|
if (evnum & no_kernel_mask) |
|
ori_val |= no_kernel_mask; |
|
|
|
if (evnum & no_user_mask) |
|
ori_val |= no_user_mask; |
|
|
|
/* Clear previous event selection */ |
|
ev_mask = PFM_CTL_SEL[idx]; |
|
ori_val &= ~ev_mask; |
|
evnum &= SOFTWARE_EVENT_MASK; |
|
|
|
/* undo the linear mapping */ |
|
evnum = get_converted_evet_hw_num(evnum); |
|
val = ori_val | (evnum << offset); |
|
val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); |
|
} |
|
|
|
static inline int nds32_pfm_enable_counter(int idx) |
|
{ |
|
unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
u32 mask = 0; |
|
|
|
mask = PFM_CTL_EN[idx]; |
|
val |= mask; |
|
val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); |
|
return idx; |
|
} |
|
|
|
static inline int nds32_pfm_enable_intens(int idx) |
|
{ |
|
unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
u32 mask = 0; |
|
|
|
mask = PFM_CTL_IE[idx]; |
|
val |= mask; |
|
val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); |
|
return idx; |
|
} |
|
|
|
static inline int nds32_pfm_disable_intens(int idx) |
|
{ |
|
unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
u32 mask = 0; |
|
|
|
mask = PFM_CTL_IE[idx]; |
|
val &= ~mask; |
|
val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); |
|
return idx; |
|
} |
|
|
|
static int event_requires_mode_exclusion(struct perf_event_attr *attr) |
|
{ |
|
/* Other modes NDS32 does not support */ |
|
return attr->exclude_user || attr->exclude_kernel; |
|
} |
|
|
|
static void nds32_pmu_enable_event(struct perf_event *event) |
|
{ |
|
unsigned long flags; |
|
unsigned int evnum = 0; |
|
struct hw_perf_event *hwc = &event->hw; |
|
struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); |
|
struct pmu_hw_events *events = cpu_pmu->get_hw_events(); |
|
int idx = hwc->idx; |
|
|
|
if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { |
|
pr_err("CPU enabling wrong pfm counter IRQ enable\n"); |
|
return; |
|
} |
|
|
|
/* |
|
* Enable counter and interrupt, and set the counter to count |
|
* the event that we're interested in. |
|
*/ |
|
raw_spin_lock_irqsave(&events->pmu_lock, flags); |
|
|
|
/* |
|
* Disable counter |
|
*/ |
|
nds32_pfm_disable_counter(idx); |
|
|
|
/* |
|
* Check whether we need to exclude the counter from certain modes. |
|
*/ |
|
if ((!cpu_pmu->set_event_filter || |
|
cpu_pmu->set_event_filter(hwc, &event->attr)) && |
|
event_requires_mode_exclusion(&event->attr)) { |
|
pr_notice |
|
("NDS32 performance counters do not support mode exclusion\n"); |
|
hwc->config_base = 0; |
|
} |
|
/* Write event */ |
|
evnum = hwc->config_base; |
|
nds32_pfm_write_evtsel(idx, evnum); |
|
|
|
/* |
|
* Enable interrupt for this counter |
|
*/ |
|
nds32_pfm_enable_intens(idx); |
|
|
|
/* |
|
* Enable counter |
|
*/ |
|
nds32_pfm_enable_counter(idx); |
|
|
|
raw_spin_unlock_irqrestore(&events->pmu_lock, flags); |
|
} |
|
|
|
static void nds32_pmu_disable_event(struct perf_event *event) |
|
{ |
|
unsigned long flags; |
|
struct hw_perf_event *hwc = &event->hw; |
|
struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); |
|
struct pmu_hw_events *events = cpu_pmu->get_hw_events(); |
|
int idx = hwc->idx; |
|
|
|
if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { |
|
pr_err("CPU disabling wrong pfm counter IRQ enable %d\n", idx); |
|
return; |
|
} |
|
|
|
/* |
|
* Disable counter and interrupt |
|
*/ |
|
raw_spin_lock_irqsave(&events->pmu_lock, flags); |
|
|
|
/* |
|
* Disable counter |
|
*/ |
|
nds32_pfm_disable_counter(idx); |
|
|
|
/* |
|
* Disable interrupt for this counter |
|
*/ |
|
nds32_pfm_disable_intens(idx); |
|
|
|
raw_spin_unlock_irqrestore(&events->pmu_lock, flags); |
|
} |
|
|
|
static inline u32 nds32_pmu_read_counter(struct perf_event *event) |
|
{ |
|
struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int idx = hwc->idx; |
|
u32 count = 0; |
|
|
|
if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { |
|
pr_err("CPU reading wrong counter %d\n", idx); |
|
} else { |
|
switch (idx) { |
|
case PFMC0: |
|
count = __nds32__mfsr(NDS32_SR_PFMC0); |
|
break; |
|
case PFMC1: |
|
count = __nds32__mfsr(NDS32_SR_PFMC1); |
|
break; |
|
case PFMC2: |
|
count = __nds32__mfsr(NDS32_SR_PFMC2); |
|
break; |
|
default: |
|
pr_err |
|
("%s: CPU has no performance counters %d\n", |
|
__func__, idx); |
|
} |
|
} |
|
return count; |
|
} |
|
|
|
static inline void nds32_pmu_write_counter(struct perf_event *event, u32 value) |
|
{ |
|
struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int idx = hwc->idx; |
|
|
|
if (!nds32_pfm_counter_valid(cpu_pmu, idx)) { |
|
pr_err("CPU writing wrong counter %d\n", idx); |
|
} else { |
|
switch (idx) { |
|
case PFMC0: |
|
__nds32__mtsr_isb(value, NDS32_SR_PFMC0); |
|
break; |
|
case PFMC1: |
|
__nds32__mtsr_isb(value, NDS32_SR_PFMC1); |
|
break; |
|
case PFMC2: |
|
__nds32__mtsr_isb(value, NDS32_SR_PFMC2); |
|
break; |
|
default: |
|
pr_err |
|
("%s: CPU has no performance counters %d\n", |
|
__func__, idx); |
|
} |
|
} |
|
} |
|
|
|
static int nds32_pmu_get_event_idx(struct pmu_hw_events *cpuc, |
|
struct perf_event *event) |
|
{ |
|
int idx; |
|
struct hw_perf_event *hwc = &event->hw; |
|
/* |
|
* Current implementation maps cycles, instruction count and cache-miss |
|
* to specific counter. |
|
* However, multiple of the 3 counters are able to count these events. |
|
* |
|
* |
|
* SOFTWARE_EVENT_MASK mask for getting event num , |
|
* This is defined by Jia-Rung, you can change the polocies. |
|
* However, do not exceed 8 bits. This is hardware specific. |
|
* The last number is SPAv3_2_SEL_LAST. |
|
*/ |
|
unsigned long evtype = hwc->config_base & SOFTWARE_EVENT_MASK; |
|
|
|
idx = get_converted_event_idx(evtype); |
|
/* |
|
* Try to get the counter for correpsonding event |
|
*/ |
|
if (evtype == SPAV3_0_SEL_TOTAL_CYCLES) { |
|
if (!test_and_set_bit(idx, cpuc->used_mask)) |
|
return idx; |
|
if (!test_and_set_bit(NDS32_IDX_COUNTER0, cpuc->used_mask)) |
|
return NDS32_IDX_COUNTER0; |
|
if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask)) |
|
return NDS32_IDX_COUNTER1; |
|
} else if (evtype == SPAV3_1_SEL_COMPLETED_INSTRUCTION) { |
|
if (!test_and_set_bit(idx, cpuc->used_mask)) |
|
return idx; |
|
else if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask)) |
|
return NDS32_IDX_COUNTER1; |
|
else if (!test_and_set_bit |
|
(NDS32_IDX_CYCLE_COUNTER, cpuc->used_mask)) |
|
return NDS32_IDX_CYCLE_COUNTER; |
|
} else { |
|
if (!test_and_set_bit(idx, cpuc->used_mask)) |
|
return idx; |
|
} |
|
return -EAGAIN; |
|
} |
|
|
|
static void nds32_pmu_start(struct nds32_pmu *cpu_pmu) |
|
{ |
|
unsigned long flags; |
|
unsigned int val; |
|
struct pmu_hw_events *events = cpu_pmu->get_hw_events(); |
|
|
|
raw_spin_lock_irqsave(&events->pmu_lock, flags); |
|
|
|
/* Enable all counters , NDS PFM has 3 counters */ |
|
val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
val |= (PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]); |
|
val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); |
|
|
|
raw_spin_unlock_irqrestore(&events->pmu_lock, flags); |
|
} |
|
|
|
static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu) |
|
{ |
|
unsigned long flags; |
|
unsigned int val; |
|
struct pmu_hw_events *events = cpu_pmu->get_hw_events(); |
|
|
|
raw_spin_lock_irqsave(&events->pmu_lock, flags); |
|
|
|
/* Disable all counters , NDS PFM has 3 counters */ |
|
val = __nds32__mfsr(NDS32_SR_PFM_CTL); |
|
val &= ~(PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]); |
|
val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr_isb(val, NDS32_SR_PFM_CTL); |
|
|
|
raw_spin_unlock_irqrestore(&events->pmu_lock, flags); |
|
} |
|
|
|
static void nds32_pmu_reset(void *info) |
|
{ |
|
u32 val = 0; |
|
|
|
val |= (PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]); |
|
__nds32__mtsr(val, NDS32_SR_PFM_CTL); |
|
__nds32__mtsr(0, NDS32_SR_PFM_CTL); |
|
__nds32__mtsr(0, NDS32_SR_PFMC0); |
|
__nds32__mtsr(0, NDS32_SR_PFMC1); |
|
__nds32__mtsr(0, NDS32_SR_PFMC2); |
|
} |
|
|
|
static void nds32_pmu_init(struct nds32_pmu *cpu_pmu) |
|
{ |
|
cpu_pmu->handle_irq = nds32_pmu_handle_irq; |
|
cpu_pmu->enable = nds32_pmu_enable_event; |
|
cpu_pmu->disable = nds32_pmu_disable_event; |
|
cpu_pmu->read_counter = nds32_pmu_read_counter; |
|
cpu_pmu->write_counter = nds32_pmu_write_counter; |
|
cpu_pmu->get_event_idx = nds32_pmu_get_event_idx; |
|
cpu_pmu->start = nds32_pmu_start; |
|
cpu_pmu->stop = nds32_pmu_stop; |
|
cpu_pmu->reset = nds32_pmu_reset; |
|
cpu_pmu->max_period = 0xFFFFFFFF; /* Maximum counts */ |
|
}; |
|
|
|
static u32 nds32_read_num_pfm_events(void) |
|
{ |
|
/* NDS32 SPAv3 PMU support 3 counter */ |
|
return 3; |
|
} |
|
|
|
static int device_pmu_init(struct nds32_pmu *cpu_pmu) |
|
{ |
|
nds32_pmu_init(cpu_pmu); |
|
/* |
|
* This name should be devive-specific name, whatever you like :) |
|
* I think "PMU" will be a good generic name. |
|
*/ |
|
cpu_pmu->name = "nds32v3-pmu"; |
|
cpu_pmu->map_event = nds32_spav3_map_event; |
|
cpu_pmu->num_events = nds32_read_num_pfm_events(); |
|
cpu_pmu->set_event_filter = nds32_pmu_set_event_filter; |
|
return 0; |
|
} |
|
|
|
/* |
|
* CPU PMU identification and probing. |
|
*/ |
|
static int probe_current_pmu(struct nds32_pmu *pmu) |
|
{ |
|
int ret; |
|
|
|
get_cpu(); |
|
ret = -ENODEV; |
|
/* |
|
* If ther are various CPU types with its own PMU, initialize with |
|
* |
|
* the corresponding one |
|
*/ |
|
device_pmu_init(pmu); |
|
put_cpu(); |
|
return ret; |
|
} |
|
|
|
static void nds32_pmu_enable(struct pmu *pmu) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu); |
|
struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); |
|
int enabled = bitmap_weight(hw_events->used_mask, |
|
nds32_pmu->num_events); |
|
|
|
if (enabled) |
|
nds32_pmu->start(nds32_pmu); |
|
} |
|
|
|
static void nds32_pmu_disable(struct pmu *pmu) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu); |
|
|
|
nds32_pmu->stop(nds32_pmu); |
|
} |
|
|
|
static void nds32_pmu_release_hardware(struct nds32_pmu *nds32_pmu) |
|
{ |
|
nds32_pmu->free_irq(nds32_pmu); |
|
pm_runtime_put_sync(&nds32_pmu->plat_device->dev); |
|
} |
|
|
|
static irqreturn_t nds32_pmu_dispatch_irq(int irq, void *dev) |
|
{ |
|
struct nds32_pmu *nds32_pmu = (struct nds32_pmu *)dev; |
|
int ret; |
|
u64 start_clock, finish_clock; |
|
|
|
start_clock = local_clock(); |
|
ret = nds32_pmu->handle_irq(irq, dev); |
|
finish_clock = local_clock(); |
|
|
|
perf_sample_event_took(finish_clock - start_clock); |
|
return ret; |
|
} |
|
|
|
static int nds32_pmu_reserve_hardware(struct nds32_pmu *nds32_pmu) |
|
{ |
|
int err; |
|
struct platform_device *pmu_device = nds32_pmu->plat_device; |
|
|
|
if (!pmu_device) |
|
return -ENODEV; |
|
|
|
pm_runtime_get_sync(&pmu_device->dev); |
|
err = nds32_pmu->request_irq(nds32_pmu, nds32_pmu_dispatch_irq); |
|
if (err) { |
|
nds32_pmu_release_hardware(nds32_pmu); |
|
return err; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static int |
|
validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, |
|
struct perf_event *event) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
|
|
if (is_software_event(event)) |
|
return 1; |
|
|
|
if (event->pmu != pmu) |
|
return 0; |
|
|
|
if (event->state < PERF_EVENT_STATE_OFF) |
|
return 1; |
|
|
|
if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) |
|
return 1; |
|
|
|
return nds32_pmu->get_event_idx(hw_events, event) >= 0; |
|
} |
|
|
|
static int validate_group(struct perf_event *event) |
|
{ |
|
struct perf_event *sibling, *leader = event->group_leader; |
|
struct pmu_hw_events fake_pmu; |
|
DECLARE_BITMAP(fake_used_mask, MAX_COUNTERS); |
|
/* |
|
* Initialize the fake PMU. We only need to populate the |
|
* used_mask for the purposes of validation. |
|
*/ |
|
memset(fake_used_mask, 0, sizeof(fake_used_mask)); |
|
|
|
if (!validate_event(event->pmu, &fake_pmu, leader)) |
|
return -EINVAL; |
|
|
|
for_each_sibling_event(sibling, leader) { |
|
if (!validate_event(event->pmu, &fake_pmu, sibling)) |
|
return -EINVAL; |
|
} |
|
|
|
if (!validate_event(event->pmu, &fake_pmu, event)) |
|
return -EINVAL; |
|
|
|
return 0; |
|
} |
|
|
|
static int __hw_perf_event_init(struct perf_event *event) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int mapping; |
|
|
|
mapping = nds32_pmu->map_event(event); |
|
|
|
if (mapping < 0) { |
|
pr_debug("event %x:%llx not supported\n", event->attr.type, |
|
event->attr.config); |
|
return mapping; |
|
} |
|
|
|
/* |
|
* We don't assign an index until we actually place the event onto |
|
* hardware. Use -1 to signify that we haven't decided where to put it |
|
* yet. For SMP systems, each core has it's own PMU so we can't do any |
|
* clever allocation or constraints checking at this point. |
|
*/ |
|
hwc->idx = -1; |
|
hwc->config_base = 0; |
|
hwc->config = 0; |
|
hwc->event_base = 0; |
|
|
|
/* |
|
* Check whether we need to exclude the counter from certain modes. |
|
*/ |
|
if ((!nds32_pmu->set_event_filter || |
|
nds32_pmu->set_event_filter(hwc, &event->attr)) && |
|
event_requires_mode_exclusion(&event->attr)) { |
|
pr_debug |
|
("NDS performance counters do not support mode exclusion\n"); |
|
return -EOPNOTSUPP; |
|
} |
|
|
|
/* |
|
* Store the event encoding into the config_base field. |
|
*/ |
|
hwc->config_base |= (unsigned long)mapping; |
|
|
|
if (!hwc->sample_period) { |
|
/* |
|
* For non-sampling runs, limit the sample_period to half |
|
* of the counter width. That way, the new counter value |
|
* is far less likely to overtake the previous one unless |
|
* you have some serious IRQ latency issues. |
|
*/ |
|
hwc->sample_period = nds32_pmu->max_period >> 1; |
|
hwc->last_period = hwc->sample_period; |
|
local64_set(&hwc->period_left, hwc->sample_period); |
|
} |
|
|
|
if (event->group_leader != event) { |
|
if (validate_group(event) != 0) |
|
return -EINVAL; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static int nds32_pmu_event_init(struct perf_event *event) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
int err = 0; |
|
atomic_t *active_events = &nds32_pmu->active_events; |
|
|
|
/* does not support taken branch sampling */ |
|
if (has_branch_stack(event)) |
|
return -EOPNOTSUPP; |
|
|
|
if (nds32_pmu->map_event(event) == -ENOENT) |
|
return -ENOENT; |
|
|
|
if (!atomic_inc_not_zero(active_events)) { |
|
if (atomic_read(active_events) == 0) { |
|
/* Register irq handler */ |
|
err = nds32_pmu_reserve_hardware(nds32_pmu); |
|
} |
|
|
|
if (!err) |
|
atomic_inc(active_events); |
|
} |
|
|
|
if (err) |
|
return err; |
|
|
|
err = __hw_perf_event_init(event); |
|
|
|
return err; |
|
} |
|
|
|
static void nds32_start(struct perf_event *event, int flags) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
/* |
|
* NDS pmu always has to reprogram the period, so ignore |
|
* PERF_EF_RELOAD, see the comment below. |
|
*/ |
|
if (flags & PERF_EF_RELOAD) |
|
WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
|
|
|
hwc->state = 0; |
|
/* Set the period for the event. */ |
|
nds32_pmu_event_set_period(event); |
|
|
|
nds32_pmu->enable(event); |
|
} |
|
|
|
static int nds32_pmu_add(struct perf_event *event, int flags) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int idx; |
|
int err = 0; |
|
|
|
perf_pmu_disable(event->pmu); |
|
|
|
/* If we don't have a space for the counter then finish early. */ |
|
idx = nds32_pmu->get_event_idx(hw_events, event); |
|
if (idx < 0) { |
|
err = idx; |
|
goto out; |
|
} |
|
|
|
/* |
|
* If there is an event in the counter we are going to use then make |
|
* sure it is disabled. |
|
*/ |
|
event->hw.idx = idx; |
|
nds32_pmu->disable(event); |
|
hw_events->events[idx] = event; |
|
|
|
hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
|
if (flags & PERF_EF_START) |
|
nds32_start(event, PERF_EF_RELOAD); |
|
|
|
/* Propagate our changes to the userspace mapping. */ |
|
perf_event_update_userpage(event); |
|
|
|
out: |
|
perf_pmu_enable(event->pmu); |
|
return err; |
|
} |
|
|
|
u64 nds32_pmu_event_update(struct perf_event *event) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
u64 delta, prev_raw_count, new_raw_count; |
|
|
|
again: |
|
prev_raw_count = local64_read(&hwc->prev_count); |
|
new_raw_count = nds32_pmu->read_counter(event); |
|
|
|
if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, |
|
new_raw_count) != prev_raw_count) { |
|
goto again; |
|
} |
|
/* |
|
* Whether overflow or not, "unsigned substraction" |
|
* will always get their delta |
|
*/ |
|
delta = (new_raw_count - prev_raw_count) & nds32_pmu->max_period; |
|
|
|
local64_add(delta, &event->count); |
|
local64_sub(delta, &hwc->period_left); |
|
|
|
return new_raw_count; |
|
} |
|
|
|
static void nds32_stop(struct perf_event *event, int flags) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
/* |
|
* NDS pmu always has to update the counter, so ignore |
|
* PERF_EF_UPDATE, see comments in nds32_start(). |
|
*/ |
|
if (!(hwc->state & PERF_HES_STOPPED)) { |
|
nds32_pmu->disable(event); |
|
nds32_pmu_event_update(event); |
|
hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
|
} |
|
} |
|
|
|
static void nds32_pmu_del(struct perf_event *event, int flags) |
|
{ |
|
struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu); |
|
struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events(); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int idx = hwc->idx; |
|
|
|
nds32_stop(event, PERF_EF_UPDATE); |
|
hw_events->events[idx] = NULL; |
|
clear_bit(idx, hw_events->used_mask); |
|
|
|
perf_event_update_userpage(event); |
|
} |
|
|
|
static void nds32_pmu_read(struct perf_event *event) |
|
{ |
|
nds32_pmu_event_update(event); |
|
} |
|
|
|
/* Please refer to SPAv3 for more hardware specific details */ |
|
PMU_FORMAT_ATTR(event, "config:0-63"); |
|
|
|
static struct attribute *nds32_arch_formats_attr[] = { |
|
&format_attr_event.attr, |
|
NULL, |
|
}; |
|
|
|
static struct attribute_group nds32_pmu_format_group = { |
|
.name = "format", |
|
.attrs = nds32_arch_formats_attr, |
|
}; |
|
|
|
static ssize_t nds32_pmu_cpumask_show(struct device *dev, |
|
struct device_attribute *attr, |
|
char *buf) |
|
{ |
|
return 0; |
|
} |
|
|
|
static DEVICE_ATTR(cpus, 0444, nds32_pmu_cpumask_show, NULL); |
|
|
|
static struct attribute *nds32_pmu_common_attrs[] = { |
|
&dev_attr_cpus.attr, |
|
NULL, |
|
}; |
|
|
|
static struct attribute_group nds32_pmu_common_group = { |
|
.attrs = nds32_pmu_common_attrs, |
|
}; |
|
|
|
static const struct attribute_group *nds32_pmu_attr_groups[] = { |
|
&nds32_pmu_format_group, |
|
&nds32_pmu_common_group, |
|
NULL, |
|
}; |
|
|
|
static void nds32_init(struct nds32_pmu *nds32_pmu) |
|
{ |
|
atomic_set(&nds32_pmu->active_events, 0); |
|
|
|
nds32_pmu->pmu = (struct pmu) { |
|
.pmu_enable = nds32_pmu_enable, |
|
.pmu_disable = nds32_pmu_disable, |
|
.attr_groups = nds32_pmu_attr_groups, |
|
.event_init = nds32_pmu_event_init, |
|
.add = nds32_pmu_add, |
|
.del = nds32_pmu_del, |
|
.start = nds32_start, |
|
.stop = nds32_stop, |
|
.read = nds32_pmu_read, |
|
}; |
|
} |
|
|
|
int nds32_pmu_register(struct nds32_pmu *nds32_pmu, int type) |
|
{ |
|
nds32_init(nds32_pmu); |
|
pm_runtime_enable(&nds32_pmu->plat_device->dev); |
|
pr_info("enabled with %s PMU driver, %d counters available\n", |
|
nds32_pmu->name, nds32_pmu->num_events); |
|
return perf_pmu_register(&nds32_pmu->pmu, nds32_pmu->name, type); |
|
} |
|
|
|
static struct pmu_hw_events *cpu_pmu_get_cpu_events(void) |
|
{ |
|
return this_cpu_ptr(&cpu_hw_events); |
|
} |
|
|
|
static int cpu_pmu_request_irq(struct nds32_pmu *cpu_pmu, irq_handler_t handler) |
|
{ |
|
int err, irq, irqs; |
|
struct platform_device *pmu_device = cpu_pmu->plat_device; |
|
|
|
if (!pmu_device) |
|
return -ENODEV; |
|
|
|
irqs = min(pmu_device->num_resources, num_possible_cpus()); |
|
if (irqs < 1) { |
|
pr_err("no irqs for PMUs defined\n"); |
|
return -ENODEV; |
|
} |
|
|
|
irq = platform_get_irq(pmu_device, 0); |
|
err = request_irq(irq, handler, IRQF_NOBALANCING, "nds32-pfm", |
|
cpu_pmu); |
|
if (err) { |
|
pr_err("unable to request IRQ%d for NDS PMU counters\n", |
|
irq); |
|
return err; |
|
} |
|
return 0; |
|
} |
|
|
|
static void cpu_pmu_free_irq(struct nds32_pmu *cpu_pmu) |
|
{ |
|
int irq; |
|
struct platform_device *pmu_device = cpu_pmu->plat_device; |
|
|
|
irq = platform_get_irq(pmu_device, 0); |
|
if (irq >= 0) |
|
free_irq(irq, cpu_pmu); |
|
} |
|
|
|
static void cpu_pmu_init(struct nds32_pmu *cpu_pmu) |
|
{ |
|
int cpu; |
|
struct pmu_hw_events *events = &per_cpu(cpu_hw_events, cpu); |
|
|
|
raw_spin_lock_init(&events->pmu_lock); |
|
|
|
cpu_pmu->get_hw_events = cpu_pmu_get_cpu_events; |
|
cpu_pmu->request_irq = cpu_pmu_request_irq; |
|
cpu_pmu->free_irq = cpu_pmu_free_irq; |
|
|
|
/* Ensure the PMU has sane values out of reset. */ |
|
if (cpu_pmu->reset) |
|
on_each_cpu(cpu_pmu->reset, cpu_pmu, 1); |
|
} |
|
|
|
static const struct of_device_id cpu_pmu_of_device_ids[] = { |
|
{.compatible = "andestech,nds32v3-pmu", |
|
.data = device_pmu_init}, |
|
{}, |
|
}; |
|
|
|
static int cpu_pmu_device_probe(struct platform_device *pdev) |
|
{ |
|
const struct of_device_id *of_id; |
|
int (*init_fn)(struct nds32_pmu *nds32_pmu); |
|
struct device_node *node = pdev->dev.of_node; |
|
struct nds32_pmu *pmu; |
|
int ret = -ENODEV; |
|
|
|
if (cpu_pmu) { |
|
pr_notice("[perf] attempt to register multiple PMU devices!\n"); |
|
return -ENOSPC; |
|
} |
|
|
|
pmu = kzalloc(sizeof(*pmu), GFP_KERNEL); |
|
if (!pmu) |
|
return -ENOMEM; |
|
|
|
of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node); |
|
if (node && of_id) { |
|
init_fn = of_id->data; |
|
ret = init_fn(pmu); |
|
} else { |
|
ret = probe_current_pmu(pmu); |
|
} |
|
|
|
if (ret) { |
|
pr_notice("[perf] failed to probe PMU!\n"); |
|
goto out_free; |
|
} |
|
|
|
cpu_pmu = pmu; |
|
cpu_pmu->plat_device = pdev; |
|
cpu_pmu_init(cpu_pmu); |
|
ret = nds32_pmu_register(cpu_pmu, PERF_TYPE_RAW); |
|
|
|
if (!ret) |
|
return 0; |
|
|
|
out_free: |
|
pr_notice("[perf] failed to register PMU devices!\n"); |
|
kfree(pmu); |
|
return ret; |
|
} |
|
|
|
static struct platform_driver cpu_pmu_driver = { |
|
.driver = { |
|
.name = "nds32-pfm", |
|
.of_match_table = cpu_pmu_of_device_ids, |
|
}, |
|
.probe = cpu_pmu_device_probe, |
|
.id_table = cpu_pmu_plat_device_ids, |
|
}; |
|
|
|
static int __init register_pmu_driver(void) |
|
{ |
|
int err = 0; |
|
|
|
err = platform_driver_register(&cpu_pmu_driver); |
|
if (err) |
|
pr_notice("[perf] PMU initialization failed\n"); |
|
else |
|
pr_notice("[perf] PMU initialization done\n"); |
|
|
|
return err; |
|
} |
|
|
|
device_initcall(register_pmu_driver); |
|
|
|
/* |
|
* References: arch/nds32/kernel/traps.c:__dump() |
|
* You will need to know the NDS ABI first. |
|
*/ |
|
static int unwind_frame_kernel(struct stackframe *frame) |
|
{ |
|
int graph = 0; |
|
#ifdef CONFIG_FRAME_POINTER |
|
/* 0x3 means misalignment */ |
|
if (!kstack_end((void *)frame->fp) && |
|
!((unsigned long)frame->fp & 0x3) && |
|
((unsigned long)frame->fp >= TASK_SIZE)) { |
|
/* |
|
* The array index is based on the ABI, the below graph |
|
* illustrate the reasons. |
|
* Function call procedure: "smw" and "lmw" will always |
|
* update SP and FP for you automatically. |
|
* |
|
* Stack Relative Address |
|
* | | 0 |
|
* ---- |
|
* |LP| <-- SP(before smw) <-- FP(after smw) -1 |
|
* ---- |
|
* |FP| -2 |
|
* ---- |
|
* | | <-- SP(after smw) -3 |
|
*/ |
|
frame->lp = ((unsigned long *)frame->fp)[-1]; |
|
frame->fp = ((unsigned long *)frame->fp)[FP_OFFSET]; |
|
/* make sure CONFIG_FUNCTION_GRAPH_TRACER is turned on */ |
|
if (__kernel_text_address(frame->lp)) |
|
frame->lp = ftrace_graph_ret_addr |
|
(NULL, &graph, frame->lp, NULL); |
|
|
|
return 0; |
|
} else { |
|
return -EPERM; |
|
} |
|
#else |
|
/* |
|
* You can refer to arch/nds32/kernel/traps.c:__dump() |
|
* Treat "sp" as "fp", but the "sp" is one frame ahead of "fp". |
|
* And, the "sp" is not always correct. |
|
* |
|
* Stack Relative Address |
|
* | | 0 |
|
* ---- |
|
* |LP| <-- SP(before smw) -1 |
|
* ---- |
|
* | | <-- SP(after smw) -2 |
|
* ---- |
|
*/ |
|
if (!kstack_end((void *)frame->sp)) { |
|
frame->lp = ((unsigned long *)frame->sp)[1]; |
|
/* TODO: How to deal with the value in first |
|
* "sp" is not correct? |
|
*/ |
|
if (__kernel_text_address(frame->lp)) |
|
frame->lp = ftrace_graph_ret_addr |
|
(tsk, &graph, frame->lp, NULL); |
|
|
|
frame->sp = ((unsigned long *)frame->sp) + 1; |
|
|
|
return 0; |
|
} else { |
|
return -EPERM; |
|
} |
|
#endif |
|
} |
|
|
|
static void notrace |
|
walk_stackframe(struct stackframe *frame, |
|
int (*fn_record)(struct stackframe *, void *), |
|
void *data) |
|
{ |
|
while (1) { |
|
int ret; |
|
|
|
if (fn_record(frame, data)) |
|
break; |
|
|
|
ret = unwind_frame_kernel(frame); |
|
if (ret < 0) |
|
break; |
|
} |
|
} |
|
|
|
/* |
|
* Gets called by walk_stackframe() for every stackframe. This will be called |
|
* whist unwinding the stackframe and is like a subroutine return so we use |
|
* the PC. |
|
*/ |
|
static int callchain_trace(struct stackframe *fr, void *data) |
|
{ |
|
struct perf_callchain_entry_ctx *entry = data; |
|
|
|
perf_callchain_store(entry, fr->lp); |
|
return 0; |
|
} |
|
|
|
/* |
|
* Get the return address for a single stackframe and return a pointer to the |
|
* next frame tail. |
|
*/ |
|
static unsigned long |
|
user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp) |
|
{ |
|
struct frame_tail buftail; |
|
unsigned long lp = 0; |
|
unsigned long *user_frame_tail = |
|
(unsigned long *)(fp - (unsigned long)sizeof(buftail)); |
|
|
|
/* Check accessibility of one struct frame_tail beyond */ |
|
if (!access_ok(user_frame_tail, sizeof(buftail))) |
|
return 0; |
|
if (__copy_from_user_inatomic |
|
(&buftail, user_frame_tail, sizeof(buftail))) |
|
return 0; |
|
|
|
/* |
|
* Refer to unwind_frame_kernel() for more illurstration |
|
*/ |
|
lp = buftail.stack_lp; /* ((unsigned long *)fp)[-1] */ |
|
fp = buftail.stack_fp; /* ((unsigned long *)fp)[FP_OFFSET] */ |
|
perf_callchain_store(entry, lp); |
|
return fp; |
|
} |
|
|
|
static unsigned long |
|
user_backtrace_opt_size(struct perf_callchain_entry_ctx *entry, |
|
unsigned long fp) |
|
{ |
|
struct frame_tail_opt_size buftail; |
|
unsigned long lp = 0; |
|
|
|
unsigned long *user_frame_tail = |
|
(unsigned long *)(fp - (unsigned long)sizeof(buftail)); |
|
|
|
/* Check accessibility of one struct frame_tail beyond */ |
|
if (!access_ok(user_frame_tail, sizeof(buftail))) |
|
return 0; |
|
if (__copy_from_user_inatomic |
|
(&buftail, user_frame_tail, sizeof(buftail))) |
|
return 0; |
|
|
|
/* |
|
* Refer to unwind_frame_kernel() for more illurstration |
|
*/ |
|
lp = buftail.stack_lp; /* ((unsigned long *)fp)[-1] */ |
|
fp = buftail.stack_fp; /* ((unsigned long *)fp)[FP_OFFSET] */ |
|
|
|
perf_callchain_store(entry, lp); |
|
return fp; |
|
} |
|
|
|
/* |
|
* This will be called when the target is in user mode |
|
* This function will only be called when we use |
|
* "PERF_SAMPLE_CALLCHAIN" in |
|
* kernel/events/core.c:perf_prepare_sample() |
|
* |
|
* How to trigger perf_callchain_[user/kernel] : |
|
* $ perf record -e cpu-clock --call-graph fp ./program |
|
* $ perf report --call-graph |
|
*/ |
|
unsigned long leaf_fp; |
|
void |
|
perf_callchain_user(struct perf_callchain_entry_ctx *entry, |
|
struct pt_regs *regs) |
|
{ |
|
unsigned long fp = 0; |
|
unsigned long gp = 0; |
|
unsigned long lp = 0; |
|
unsigned long sp = 0; |
|
unsigned long *user_frame_tail; |
|
|
|
leaf_fp = 0; |
|
|
|
if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { |
|
/* We don't support guest os callchain now */ |
|
return; |
|
} |
|
|
|
perf_callchain_store(entry, regs->ipc); |
|
fp = regs->fp; |
|
gp = regs->gp; |
|
lp = regs->lp; |
|
sp = regs->sp; |
|
if (entry->nr < PERF_MAX_STACK_DEPTH && |
|
(unsigned long)fp && !((unsigned long)fp & 0x7) && fp > sp) { |
|
user_frame_tail = |
|
(unsigned long *)(fp - (unsigned long)sizeof(fp)); |
|
|
|
if (!access_ok(user_frame_tail, sizeof(fp))) |
|
return; |
|
|
|
if (__copy_from_user_inatomic |
|
(&leaf_fp, user_frame_tail, sizeof(fp))) |
|
return; |
|
|
|
if (leaf_fp == lp) { |
|
/* |
|
* Maybe this is non leaf function |
|
* with optimize for size, |
|
* or maybe this is the function |
|
* with optimize for size |
|
*/ |
|
struct frame_tail buftail; |
|
|
|
user_frame_tail = |
|
(unsigned long *)(fp - |
|
(unsigned long)sizeof(buftail)); |
|
|
|
if (!access_ok(user_frame_tail, sizeof(buftail))) |
|
return; |
|
|
|
if (__copy_from_user_inatomic |
|
(&buftail, user_frame_tail, sizeof(buftail))) |
|
return; |
|
|
|
if (buftail.stack_fp == gp) { |
|
/* non leaf function with optimize |
|
* for size condition |
|
*/ |
|
struct frame_tail_opt_size buftail_opt_size; |
|
|
|
user_frame_tail = |
|
(unsigned long *)(fp - (unsigned long) |
|
sizeof(buftail_opt_size)); |
|
|
|
if (!access_ok(user_frame_tail, |
|
sizeof(buftail_opt_size))) |
|
return; |
|
|
|
if (__copy_from_user_inatomic |
|
(&buftail_opt_size, user_frame_tail, |
|
sizeof(buftail_opt_size))) |
|
return; |
|
|
|
perf_callchain_store(entry, lp); |
|
fp = buftail_opt_size.stack_fp; |
|
|
|
while ((entry->nr < PERF_MAX_STACK_DEPTH) && |
|
(unsigned long)fp && |
|
!((unsigned long)fp & 0x7) && |
|
fp > sp) { |
|
sp = fp; |
|
fp = user_backtrace_opt_size(entry, fp); |
|
} |
|
|
|
} else { |
|
/* this is the function |
|
* without optimize for size |
|
*/ |
|
fp = buftail.stack_fp; |
|
perf_callchain_store(entry, lp); |
|
while ((entry->nr < PERF_MAX_STACK_DEPTH) && |
|
(unsigned long)fp && |
|
!((unsigned long)fp & 0x7) && |
|
fp > sp) { |
|
sp = fp; |
|
fp = user_backtrace(entry, fp); |
|
} |
|
} |
|
} else { |
|
/* this is leaf function */ |
|
fp = leaf_fp; |
|
perf_callchain_store(entry, lp); |
|
|
|
/* previous function callcahin */ |
|
while ((entry->nr < PERF_MAX_STACK_DEPTH) && |
|
(unsigned long)fp && |
|
!((unsigned long)fp & 0x7) && fp > sp) { |
|
sp = fp; |
|
fp = user_backtrace(entry, fp); |
|
} |
|
} |
|
return; |
|
} |
|
} |
|
|
|
/* This will be called when the target is in kernel mode */ |
|
void |
|
perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, |
|
struct pt_regs *regs) |
|
{ |
|
struct stackframe fr; |
|
|
|
if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { |
|
/* We don't support guest os callchain now */ |
|
return; |
|
} |
|
fr.fp = regs->fp; |
|
fr.lp = regs->lp; |
|
fr.sp = regs->sp; |
|
walk_stackframe(&fr, callchain_trace, entry); |
|
} |
|
|
|
unsigned long perf_instruction_pointer(struct pt_regs *regs) |
|
{ |
|
/* However, NDS32 does not support virtualization */ |
|
if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) |
|
return perf_guest_cbs->get_guest_ip(); |
|
|
|
return instruction_pointer(regs); |
|
} |
|
|
|
unsigned long perf_misc_flags(struct pt_regs *regs) |
|
{ |
|
int misc = 0; |
|
|
|
/* However, NDS32 does not support virtualization */ |
|
if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { |
|
if (perf_guest_cbs->is_user_mode()) |
|
misc |= PERF_RECORD_MISC_GUEST_USER; |
|
else |
|
misc |= PERF_RECORD_MISC_GUEST_KERNEL; |
|
} else { |
|
if (user_mode(regs)) |
|
misc |= PERF_RECORD_MISC_USER; |
|
else |
|
misc |= PERF_RECORD_MISC_KERNEL; |
|
} |
|
|
|
return misc; |
|
}
|
|
|