mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
565 lines
13 KiB
565 lines
13 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* L220/L310 cache controller support |
|
* |
|
* Copyright (C) 2016 ARM Limited |
|
*/ |
|
#include <linux/errno.h> |
|
#include <linux/hrtimer.h> |
|
#include <linux/io.h> |
|
#include <linux/list.h> |
|
#include <linux/perf_event.h> |
|
#include <linux/printk.h> |
|
#include <linux/slab.h> |
|
#include <linux/types.h> |
|
|
|
#include <asm/hardware/cache-l2x0.h> |
|
|
|
#define PMU_NR_COUNTERS 2 |
|
|
|
static void __iomem *l2x0_base; |
|
static struct pmu *l2x0_pmu; |
|
static cpumask_t pmu_cpu; |
|
|
|
static const char *l2x0_name; |
|
|
|
static ktime_t l2x0_pmu_poll_period; |
|
static struct hrtimer l2x0_pmu_hrtimer; |
|
|
|
/* |
|
* The L220/PL310 PMU has two equivalent counters, Counter1 and Counter0. |
|
* Registers controlling these are laid out in pairs, in descending order, i.e. |
|
* the register for Counter1 comes first, followed by the register for |
|
* Counter0. |
|
* We ensure that idx 0 -> Counter0, and idx1 -> Counter1. |
|
*/ |
|
static struct perf_event *events[PMU_NR_COUNTERS]; |
|
|
|
/* Find an unused counter */ |
|
static int l2x0_pmu_find_idx(void) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < PMU_NR_COUNTERS; i++) { |
|
if (!events[i]) |
|
return i; |
|
} |
|
|
|
return -1; |
|
} |
|
|
|
/* How many counters are allocated? */ |
|
static int l2x0_pmu_num_active_counters(void) |
|
{ |
|
int i, cnt = 0; |
|
|
|
for (i = 0; i < PMU_NR_COUNTERS; i++) { |
|
if (events[i]) |
|
cnt++; |
|
} |
|
|
|
return cnt; |
|
} |
|
|
|
static void l2x0_pmu_counter_config_write(int idx, u32 val) |
|
{ |
|
writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_CFG - 4 * idx); |
|
} |
|
|
|
static u32 l2x0_pmu_counter_read(int idx) |
|
{ |
|
return readl_relaxed(l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx); |
|
} |
|
|
|
static void l2x0_pmu_counter_write(int idx, u32 val) |
|
{ |
|
writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx); |
|
} |
|
|
|
static void __l2x0_pmu_enable(void) |
|
{ |
|
u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL); |
|
val |= L2X0_EVENT_CNT_CTRL_ENABLE; |
|
writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL); |
|
} |
|
|
|
static void __l2x0_pmu_disable(void) |
|
{ |
|
u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL); |
|
val &= ~L2X0_EVENT_CNT_CTRL_ENABLE; |
|
writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL); |
|
} |
|
|
|
static void l2x0_pmu_enable(struct pmu *pmu) |
|
{ |
|
if (l2x0_pmu_num_active_counters() == 0) |
|
return; |
|
|
|
__l2x0_pmu_enable(); |
|
} |
|
|
|
static void l2x0_pmu_disable(struct pmu *pmu) |
|
{ |
|
if (l2x0_pmu_num_active_counters() == 0) |
|
return; |
|
|
|
__l2x0_pmu_disable(); |
|
} |
|
|
|
static void warn_if_saturated(u32 count) |
|
{ |
|
if (count != 0xffffffff) |
|
return; |
|
|
|
pr_warn_ratelimited("L2X0 counter saturated. Poll period too long\n"); |
|
} |
|
|
|
static void l2x0_pmu_event_read(struct perf_event *event) |
|
{ |
|
struct hw_perf_event *hw = &event->hw; |
|
u64 prev_count, new_count, mask; |
|
|
|
do { |
|
prev_count = local64_read(&hw->prev_count); |
|
new_count = l2x0_pmu_counter_read(hw->idx); |
|
} while (local64_xchg(&hw->prev_count, new_count) != prev_count); |
|
|
|
mask = GENMASK_ULL(31, 0); |
|
local64_add((new_count - prev_count) & mask, &event->count); |
|
|
|
warn_if_saturated(new_count); |
|
} |
|
|
|
static void l2x0_pmu_event_configure(struct perf_event *event) |
|
{ |
|
struct hw_perf_event *hw = &event->hw; |
|
|
|
/* |
|
* The L2X0 counters saturate at 0xffffffff rather than wrapping, so we |
|
* will *always* lose some number of events when a counter saturates, |
|
* and have no way of detecting how many were lost. |
|
* |
|
* To minimize the impact of this, we try to maximize the period by |
|
* always starting counters at zero. To ensure that group ratios are |
|
* representative, we poll periodically to avoid counters saturating. |
|
* See l2x0_pmu_poll(). |
|
*/ |
|
local64_set(&hw->prev_count, 0); |
|
l2x0_pmu_counter_write(hw->idx, 0); |
|
} |
|
|
|
static enum hrtimer_restart l2x0_pmu_poll(struct hrtimer *hrtimer) |
|
{ |
|
unsigned long flags; |
|
int i; |
|
|
|
local_irq_save(flags); |
|
__l2x0_pmu_disable(); |
|
|
|
for (i = 0; i < PMU_NR_COUNTERS; i++) { |
|
struct perf_event *event = events[i]; |
|
|
|
if (!event) |
|
continue; |
|
|
|
l2x0_pmu_event_read(event); |
|
l2x0_pmu_event_configure(event); |
|
} |
|
|
|
__l2x0_pmu_enable(); |
|
local_irq_restore(flags); |
|
|
|
hrtimer_forward_now(hrtimer, l2x0_pmu_poll_period); |
|
return HRTIMER_RESTART; |
|
} |
|
|
|
|
|
static void __l2x0_pmu_event_enable(int idx, u32 event) |
|
{ |
|
u32 val; |
|
|
|
val = event << L2X0_EVENT_CNT_CFG_SRC_SHIFT; |
|
val |= L2X0_EVENT_CNT_CFG_INT_DISABLED; |
|
l2x0_pmu_counter_config_write(idx, val); |
|
} |
|
|
|
static void l2x0_pmu_event_start(struct perf_event *event, int flags) |
|
{ |
|
struct hw_perf_event *hw = &event->hw; |
|
|
|
if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) |
|
return; |
|
|
|
if (flags & PERF_EF_RELOAD) { |
|
WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE)); |
|
l2x0_pmu_event_configure(event); |
|
} |
|
|
|
hw->state = 0; |
|
|
|
__l2x0_pmu_event_enable(hw->idx, hw->config_base); |
|
} |
|
|
|
static void __l2x0_pmu_event_disable(int idx) |
|
{ |
|
u32 val; |
|
|
|
val = L2X0_EVENT_CNT_CFG_SRC_DISABLED << L2X0_EVENT_CNT_CFG_SRC_SHIFT; |
|
val |= L2X0_EVENT_CNT_CFG_INT_DISABLED; |
|
l2x0_pmu_counter_config_write(idx, val); |
|
} |
|
|
|
static void l2x0_pmu_event_stop(struct perf_event *event, int flags) |
|
{ |
|
struct hw_perf_event *hw = &event->hw; |
|
|
|
if (WARN_ON_ONCE(event->hw.state & PERF_HES_STOPPED)) |
|
return; |
|
|
|
__l2x0_pmu_event_disable(hw->idx); |
|
|
|
hw->state |= PERF_HES_STOPPED; |
|
|
|
if (flags & PERF_EF_UPDATE) { |
|
l2x0_pmu_event_read(event); |
|
hw->state |= PERF_HES_UPTODATE; |
|
} |
|
} |
|
|
|
static int l2x0_pmu_event_add(struct perf_event *event, int flags) |
|
{ |
|
struct hw_perf_event *hw = &event->hw; |
|
int idx = l2x0_pmu_find_idx(); |
|
|
|
if (idx == -1) |
|
return -EAGAIN; |
|
|
|
/* |
|
* Pin the timer, so that the overflows are handled by the chosen |
|
* event->cpu (this is the same one as presented in "cpumask" |
|
* attribute). |
|
*/ |
|
if (l2x0_pmu_num_active_counters() == 0) |
|
hrtimer_start(&l2x0_pmu_hrtimer, l2x0_pmu_poll_period, |
|
HRTIMER_MODE_REL_PINNED); |
|
|
|
events[idx] = event; |
|
hw->idx = idx; |
|
|
|
l2x0_pmu_event_configure(event); |
|
|
|
hw->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
|
|
|
if (flags & PERF_EF_START) |
|
l2x0_pmu_event_start(event, 0); |
|
|
|
return 0; |
|
} |
|
|
|
static void l2x0_pmu_event_del(struct perf_event *event, int flags) |
|
{ |
|
struct hw_perf_event *hw = &event->hw; |
|
|
|
l2x0_pmu_event_stop(event, PERF_EF_UPDATE); |
|
|
|
events[hw->idx] = NULL; |
|
hw->idx = -1; |
|
|
|
if (l2x0_pmu_num_active_counters() == 0) |
|
hrtimer_cancel(&l2x0_pmu_hrtimer); |
|
} |
|
|
|
static bool l2x0_pmu_group_is_valid(struct perf_event *event) |
|
{ |
|
struct pmu *pmu = event->pmu; |
|
struct perf_event *leader = event->group_leader; |
|
struct perf_event *sibling; |
|
int num_hw = 0; |
|
|
|
if (leader->pmu == pmu) |
|
num_hw++; |
|
else if (!is_software_event(leader)) |
|
return false; |
|
|
|
for_each_sibling_event(sibling, leader) { |
|
if (sibling->pmu == pmu) |
|
num_hw++; |
|
else if (!is_software_event(sibling)) |
|
return false; |
|
} |
|
|
|
return num_hw <= PMU_NR_COUNTERS; |
|
} |
|
|
|
static int l2x0_pmu_event_init(struct perf_event *event) |
|
{ |
|
struct hw_perf_event *hw = &event->hw; |
|
|
|
if (event->attr.type != l2x0_pmu->type) |
|
return -ENOENT; |
|
|
|
if (is_sampling_event(event) || |
|
event->attach_state & PERF_ATTACH_TASK) |
|
return -EINVAL; |
|
|
|
if (event->cpu < 0) |
|
return -EINVAL; |
|
|
|
if (event->attr.config & ~L2X0_EVENT_CNT_CFG_SRC_MASK) |
|
return -EINVAL; |
|
|
|
hw->config_base = event->attr.config; |
|
|
|
if (!l2x0_pmu_group_is_valid(event)) |
|
return -EINVAL; |
|
|
|
event->cpu = cpumask_first(&pmu_cpu); |
|
|
|
return 0; |
|
} |
|
|
|
struct l2x0_event_attribute { |
|
struct device_attribute attr; |
|
unsigned int config; |
|
bool pl310_only; |
|
}; |
|
|
|
#define L2X0_EVENT_ATTR(_name, _config, _pl310_only) \ |
|
(&((struct l2x0_event_attribute[]) {{ \ |
|
.attr = __ATTR(_name, S_IRUGO, l2x0_pmu_event_show, NULL), \ |
|
.config = _config, \ |
|
.pl310_only = _pl310_only, \ |
|
}})[0].attr.attr) |
|
|
|
#define L220_PLUS_EVENT_ATTR(_name, _config) \ |
|
L2X0_EVENT_ATTR(_name, _config, false) |
|
|
|
#define PL310_EVENT_ATTR(_name, _config) \ |
|
L2X0_EVENT_ATTR(_name, _config, true) |
|
|
|
static ssize_t l2x0_pmu_event_show(struct device *dev, |
|
struct device_attribute *attr, char *buf) |
|
{ |
|
struct l2x0_event_attribute *lattr; |
|
|
|
lattr = container_of(attr, typeof(*lattr), attr); |
|
return snprintf(buf, PAGE_SIZE, "config=0x%x\n", lattr->config); |
|
} |
|
|
|
static umode_t l2x0_pmu_event_attr_is_visible(struct kobject *kobj, |
|
struct attribute *attr, |
|
int unused) |
|
{ |
|
struct device *dev = kobj_to_dev(kobj); |
|
struct pmu *pmu = dev_get_drvdata(dev); |
|
struct l2x0_event_attribute *lattr; |
|
|
|
lattr = container_of(attr, typeof(*lattr), attr.attr); |
|
|
|
if (!lattr->pl310_only || strcmp("l2c_310", pmu->name) == 0) |
|
return attr->mode; |
|
|
|
return 0; |
|
} |
|
|
|
static struct attribute *l2x0_pmu_event_attrs[] = { |
|
L220_PLUS_EVENT_ATTR(co, 0x1), |
|
L220_PLUS_EVENT_ATTR(drhit, 0x2), |
|
L220_PLUS_EVENT_ATTR(drreq, 0x3), |
|
L220_PLUS_EVENT_ATTR(dwhit, 0x4), |
|
L220_PLUS_EVENT_ATTR(dwreq, 0x5), |
|
L220_PLUS_EVENT_ATTR(dwtreq, 0x6), |
|
L220_PLUS_EVENT_ATTR(irhit, 0x7), |
|
L220_PLUS_EVENT_ATTR(irreq, 0x8), |
|
L220_PLUS_EVENT_ATTR(wa, 0x9), |
|
PL310_EVENT_ATTR(ipfalloc, 0xa), |
|
PL310_EVENT_ATTR(epfhit, 0xb), |
|
PL310_EVENT_ATTR(epfalloc, 0xc), |
|
PL310_EVENT_ATTR(srrcvd, 0xd), |
|
PL310_EVENT_ATTR(srconf, 0xe), |
|
PL310_EVENT_ATTR(epfrcvd, 0xf), |
|
NULL |
|
}; |
|
|
|
static struct attribute_group l2x0_pmu_event_attrs_group = { |
|
.name = "events", |
|
.attrs = l2x0_pmu_event_attrs, |
|
.is_visible = l2x0_pmu_event_attr_is_visible, |
|
}; |
|
|
|
static ssize_t l2x0_pmu_cpumask_show(struct device *dev, |
|
struct device_attribute *attr, char *buf) |
|
{ |
|
return cpumap_print_to_pagebuf(true, buf, &pmu_cpu); |
|
} |
|
|
|
static struct device_attribute l2x0_pmu_cpumask_attr = |
|
__ATTR(cpumask, S_IRUGO, l2x0_pmu_cpumask_show, NULL); |
|
|
|
static struct attribute *l2x0_pmu_cpumask_attrs[] = { |
|
&l2x0_pmu_cpumask_attr.attr, |
|
NULL, |
|
}; |
|
|
|
static struct attribute_group l2x0_pmu_cpumask_attr_group = { |
|
.attrs = l2x0_pmu_cpumask_attrs, |
|
}; |
|
|
|
static const struct attribute_group *l2x0_pmu_attr_groups[] = { |
|
&l2x0_pmu_event_attrs_group, |
|
&l2x0_pmu_cpumask_attr_group, |
|
NULL, |
|
}; |
|
|
|
static void l2x0_pmu_reset(void) |
|
{ |
|
int i; |
|
|
|
__l2x0_pmu_disable(); |
|
|
|
for (i = 0; i < PMU_NR_COUNTERS; i++) |
|
__l2x0_pmu_event_disable(i); |
|
} |
|
|
|
static int l2x0_pmu_offline_cpu(unsigned int cpu) |
|
{ |
|
unsigned int target; |
|
|
|
if (!cpumask_test_and_clear_cpu(cpu, &pmu_cpu)) |
|
return 0; |
|
|
|
target = cpumask_any_but(cpu_online_mask, cpu); |
|
if (target >= nr_cpu_ids) |
|
return 0; |
|
|
|
perf_pmu_migrate_context(l2x0_pmu, cpu, target); |
|
cpumask_set_cpu(target, &pmu_cpu); |
|
|
|
return 0; |
|
} |
|
|
|
void l2x0_pmu_suspend(void) |
|
{ |
|
int i; |
|
|
|
if (!l2x0_pmu) |
|
return; |
|
|
|
l2x0_pmu_disable(l2x0_pmu); |
|
|
|
for (i = 0; i < PMU_NR_COUNTERS; i++) { |
|
if (events[i]) |
|
l2x0_pmu_event_stop(events[i], PERF_EF_UPDATE); |
|
} |
|
|
|
} |
|
|
|
void l2x0_pmu_resume(void) |
|
{ |
|
int i; |
|
|
|
if (!l2x0_pmu) |
|
return; |
|
|
|
l2x0_pmu_reset(); |
|
|
|
for (i = 0; i < PMU_NR_COUNTERS; i++) { |
|
if (events[i]) |
|
l2x0_pmu_event_start(events[i], PERF_EF_RELOAD); |
|
} |
|
|
|
l2x0_pmu_enable(l2x0_pmu); |
|
} |
|
|
|
void __init l2x0_pmu_register(void __iomem *base, u32 part) |
|
{ |
|
/* |
|
* Determine whether we support the PMU, and choose the name for sysfs. |
|
* This is also used by l2x0_pmu_event_attr_is_visible to determine |
|
* which events to display, as the PL310 PMU supports a superset of |
|
* L220 events. |
|
* |
|
* The L210 PMU has a different programmer's interface, and is not |
|
* supported by this driver. |
|
* |
|
* We must defer registering the PMU until the perf subsystem is up and |
|
* running, so just stash the name and base, and leave that to another |
|
* initcall. |
|
*/ |
|
switch (part & L2X0_CACHE_ID_PART_MASK) { |
|
case L2X0_CACHE_ID_PART_L220: |
|
l2x0_name = "l2c_220"; |
|
break; |
|
case L2X0_CACHE_ID_PART_L310: |
|
l2x0_name = "l2c_310"; |
|
break; |
|
default: |
|
return; |
|
} |
|
|
|
l2x0_base = base; |
|
} |
|
|
|
static __init int l2x0_pmu_init(void) |
|
{ |
|
int ret; |
|
|
|
if (!l2x0_base) |
|
return 0; |
|
|
|
l2x0_pmu = kzalloc(sizeof(*l2x0_pmu), GFP_KERNEL); |
|
if (!l2x0_pmu) { |
|
pr_warn("Unable to allocate L2x0 PMU\n"); |
|
return -ENOMEM; |
|
} |
|
|
|
*l2x0_pmu = (struct pmu) { |
|
.task_ctx_nr = perf_invalid_context, |
|
.pmu_enable = l2x0_pmu_enable, |
|
.pmu_disable = l2x0_pmu_disable, |
|
.read = l2x0_pmu_event_read, |
|
.start = l2x0_pmu_event_start, |
|
.stop = l2x0_pmu_event_stop, |
|
.add = l2x0_pmu_event_add, |
|
.del = l2x0_pmu_event_del, |
|
.event_init = l2x0_pmu_event_init, |
|
.attr_groups = l2x0_pmu_attr_groups, |
|
.capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
|
}; |
|
|
|
l2x0_pmu_reset(); |
|
|
|
/* |
|
* We always use a hrtimer rather than an interrupt. |
|
* See comments in l2x0_pmu_event_configure and l2x0_pmu_poll. |
|
* |
|
* Polling once a second allows the counters to fill up to 1/128th on a |
|
* quad-core test chip with cores clocked at 400MHz. Hopefully this |
|
* leaves sufficient headroom to avoid overflow on production silicon |
|
* at higher frequencies. |
|
*/ |
|
l2x0_pmu_poll_period = ms_to_ktime(1000); |
|
hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
|
l2x0_pmu_hrtimer.function = l2x0_pmu_poll; |
|
|
|
cpumask_set_cpu(0, &pmu_cpu); |
|
ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE, |
|
"perf/arm/l2x0:online", NULL, |
|
l2x0_pmu_offline_cpu); |
|
if (ret) |
|
goto out_pmu; |
|
|
|
ret = perf_pmu_register(l2x0_pmu, l2x0_name, -1); |
|
if (ret) |
|
goto out_cpuhp; |
|
|
|
return 0; |
|
|
|
out_cpuhp: |
|
cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE); |
|
out_pmu: |
|
kfree(l2x0_pmu); |
|
l2x0_pmu = NULL; |
|
return ret; |
|
} |
|
device_initcall(l2x0_pmu_init);
|
|
|