mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
758 lines
22 KiB
758 lines
22 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver |
|
* |
|
* Copyright (C) 2021 Marvell. |
|
*/ |
|
|
|
#include <linux/init.h> |
|
#include <linux/io.h> |
|
#include <linux/module.h> |
|
#include <linux/of.h> |
|
#include <linux/of_address.h> |
|
#include <linux/of_device.h> |
|
#include <linux/perf_event.h> |
|
#include <linux/hrtimer.h> |
|
|
|
/* Performance Counters Operating Mode Control Registers */ |
|
#define DDRC_PERF_CNT_OP_MODE_CTRL 0x8020 |
|
#define OP_MODE_CTRL_VAL_MANNUAL 0x1 |
|
|
|
/* Performance Counters Start Operation Control Registers */ |
|
#define DDRC_PERF_CNT_START_OP_CTRL 0x8028 |
|
#define START_OP_CTRL_VAL_START 0x1ULL |
|
#define START_OP_CTRL_VAL_ACTIVE 0x2 |
|
|
|
/* Performance Counters End Operation Control Registers */ |
|
#define DDRC_PERF_CNT_END_OP_CTRL 0x8030 |
|
#define END_OP_CTRL_VAL_END 0x1ULL |
|
|
|
/* Performance Counters End Status Registers */ |
|
#define DDRC_PERF_CNT_END_STATUS 0x8038 |
|
#define END_STATUS_VAL_END_TIMER_MODE_END 0x1 |
|
|
|
/* Performance Counters Configuration Registers */ |
|
#define DDRC_PERF_CFG_BASE 0x8040 |
|
|
|
/* 8 Generic event counter + 2 fixed event counters */ |
|
#define DDRC_PERF_NUM_GEN_COUNTERS 8 |
|
#define DDRC_PERF_NUM_FIX_COUNTERS 2 |
|
#define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS |
|
#define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1) |
|
#define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \ |
|
DDRC_PERF_NUM_FIX_COUNTERS) |
|
|
|
/* Generic event counter registers */ |
|
#define DDRC_PERF_CFG(n) (DDRC_PERF_CFG_BASE + 8 * (n)) |
|
#define EVENT_ENABLE BIT_ULL(63) |
|
|
|
/* Two dedicated event counters for DDR reads and writes */ |
|
#define EVENT_DDR_READS 101 |
|
#define EVENT_DDR_WRITES 100 |
|
|
|
/* |
|
* programmable events IDs in programmable event counters. |
|
* DO NOT change these event-id numbers, they are used to |
|
* program event bitmap in h/w. |
|
*/ |
|
#define EVENT_OP_IS_ZQLATCH 55 |
|
#define EVENT_OP_IS_ZQSTART 54 |
|
#define EVENT_OP_IS_TCR_MRR 53 |
|
#define EVENT_OP_IS_DQSOSC_MRR 52 |
|
#define EVENT_OP_IS_DQSOSC_MPC 51 |
|
#define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 50 |
|
#define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 49 |
|
#define EVENT_BSM_STARVATION 48 |
|
#define EVENT_BSM_ALLOC 47 |
|
#define EVENT_LPR_REQ_WITH_NOCREDIT 46 |
|
#define EVENT_HPR_REQ_WITH_NOCREDIT 45 |
|
#define EVENT_OP_IS_ZQCS 44 |
|
#define EVENT_OP_IS_ZQCL 43 |
|
#define EVENT_OP_IS_LOAD_MODE 42 |
|
#define EVENT_OP_IS_SPEC_REF 41 |
|
#define EVENT_OP_IS_CRIT_REF 40 |
|
#define EVENT_OP_IS_REFRESH 39 |
|
#define EVENT_OP_IS_ENTER_MPSM 35 |
|
#define EVENT_OP_IS_ENTER_POWERDOWN 31 |
|
#define EVENT_OP_IS_ENTER_SELFREF 27 |
|
#define EVENT_WAW_HAZARD 26 |
|
#define EVENT_RAW_HAZARD 25 |
|
#define EVENT_WAR_HAZARD 24 |
|
#define EVENT_WRITE_COMBINE 23 |
|
#define EVENT_RDWR_TRANSITIONS 22 |
|
#define EVENT_PRECHARGE_FOR_OTHER 21 |
|
#define EVENT_PRECHARGE_FOR_RDWR 20 |
|
#define EVENT_OP_IS_PRECHARGE 19 |
|
#define EVENT_OP_IS_MWR 18 |
|
#define EVENT_OP_IS_WR 17 |
|
#define EVENT_OP_IS_RD 16 |
|
#define EVENT_OP_IS_RD_ACTIVATE 15 |
|
#define EVENT_OP_IS_RD_OR_WR 14 |
|
#define EVENT_OP_IS_ACTIVATE 13 |
|
#define EVENT_WR_XACT_WHEN_CRITICAL 12 |
|
#define EVENT_LPR_XACT_WHEN_CRITICAL 11 |
|
#define EVENT_HPR_XACT_WHEN_CRITICAL 10 |
|
#define EVENT_DFI_RD_DATA_CYCLES 9 |
|
#define EVENT_DFI_WR_DATA_CYCLES 8 |
|
#define EVENT_ACT_BYPASS 7 |
|
#define EVENT_READ_BYPASS 6 |
|
#define EVENT_HIF_HI_PRI_RD 5 |
|
#define EVENT_HIF_RMW 4 |
|
#define EVENT_HIF_RD 3 |
|
#define EVENT_HIF_WR 2 |
|
#define EVENT_HIF_RD_OR_WR 1 |
|
|
|
/* Event counter value registers */ |
|
#define DDRC_PERF_CNT_VALUE_BASE 0x8080 |
|
#define DDRC_PERF_CNT_VALUE(n) (DDRC_PERF_CNT_VALUE_BASE + 8 * (n)) |
|
|
|
/* Fixed event counter enable/disable register */ |
|
#define DDRC_PERF_CNT_FREERUN_EN 0x80C0 |
|
#define DDRC_PERF_FREERUN_WRITE_EN 0x1 |
|
#define DDRC_PERF_FREERUN_READ_EN 0x2 |
|
|
|
/* Fixed event counter control register */ |
|
#define DDRC_PERF_CNT_FREERUN_CTRL 0x80C8 |
|
#define DDRC_FREERUN_WRITE_CNT_CLR 0x1 |
|
#define DDRC_FREERUN_READ_CNT_CLR 0x2 |
|
|
|
/* Fixed event counter value register */ |
|
#define DDRC_PERF_CNT_VALUE_WR_OP 0x80D0 |
|
#define DDRC_PERF_CNT_VALUE_RD_OP 0x80D8 |
|
#define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48) |
|
#define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0) |
|
|
|
struct cn10k_ddr_pmu { |
|
struct pmu pmu; |
|
void __iomem *base; |
|
unsigned int cpu; |
|
struct device *dev; |
|
int active_events; |
|
struct perf_event *events[DDRC_PERF_NUM_COUNTERS]; |
|
struct hrtimer hrtimer; |
|
struct hlist_node node; |
|
}; |
|
|
|
#define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu) |
|
|
|
static ssize_t cn10k_ddr_pmu_event_show(struct device *dev, |
|
struct device_attribute *attr, |
|
char *page) |
|
{ |
|
struct perf_pmu_events_attr *pmu_attr; |
|
|
|
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); |
|
return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); |
|
|
|
} |
|
|
|
#define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \ |
|
PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id) |
|
|
|
static struct attribute *cn10k_ddr_perf_events_attrs[] = { |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access, |
|
EVENT_HPR_XACT_WHEN_CRITICAL), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access, |
|
EVENT_LPR_XACT_WHEN_CRITICAL), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access, |
|
EVENT_WR_XACT_WHEN_CRITICAL), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other, |
|
EVENT_PRECHARGE_FOR_OTHER), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit, |
|
EVENT_HPR_REQ_WITH_NOCREDIT), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit, |
|
EVENT_LPR_REQ_WITH_NOCREDIT), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd, |
|
EVENT_VISIBLE_WIN_LIMIT_REACHED_RD), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr, |
|
EVENT_VISIBLE_WIN_LIMIT_REACHED_WR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH), |
|
/* Free run event counters */ |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS), |
|
CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES), |
|
NULL |
|
}; |
|
|
|
static struct attribute_group cn10k_ddr_perf_events_attr_group = { |
|
.name = "events", |
|
.attrs = cn10k_ddr_perf_events_attrs, |
|
}; |
|
|
|
PMU_FORMAT_ATTR(event, "config:0-8"); |
|
|
|
static struct attribute *cn10k_ddr_perf_format_attrs[] = { |
|
&format_attr_event.attr, |
|
NULL, |
|
}; |
|
|
|
static struct attribute_group cn10k_ddr_perf_format_attr_group = { |
|
.name = "format", |
|
.attrs = cn10k_ddr_perf_format_attrs, |
|
}; |
|
|
|
static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev, |
|
struct device_attribute *attr, |
|
char *buf) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev); |
|
|
|
return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu)); |
|
} |
|
|
|
static struct device_attribute cn10k_ddr_perf_cpumask_attr = |
|
__ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL); |
|
|
|
static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = { |
|
&cn10k_ddr_perf_cpumask_attr.attr, |
|
NULL, |
|
}; |
|
|
|
static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = { |
|
.attrs = cn10k_ddr_perf_cpumask_attrs, |
|
}; |
|
|
|
static const struct attribute_group *cn10k_attr_groups[] = { |
|
&cn10k_ddr_perf_events_attr_group, |
|
&cn10k_ddr_perf_format_attr_group, |
|
&cn10k_ddr_perf_cpumask_attr_group, |
|
NULL, |
|
}; |
|
|
|
/* Default poll timeout is 100 sec, which is very sufficient for |
|
* 48 bit counter incremented max at 5.6 GT/s, which may take many |
|
* hours to overflow. |
|
*/ |
|
static unsigned long cn10k_ddr_pmu_poll_period_sec = 100; |
|
module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644); |
|
|
|
static ktime_t cn10k_ddr_pmu_timer_period(void) |
|
{ |
|
return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC); |
|
} |
|
|
|
static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap) |
|
{ |
|
switch (eventid) { |
|
case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD: |
|
case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH: |
|
*event_bitmap = (1ULL << (eventid - 1)); |
|
break; |
|
case EVENT_OP_IS_ENTER_SELFREF: |
|
case EVENT_OP_IS_ENTER_POWERDOWN: |
|
case EVENT_OP_IS_ENTER_MPSM: |
|
*event_bitmap = (0xFULL << (eventid - 1)); |
|
break; |
|
default: |
|
pr_err("%s Invalid eventid %d\n", __func__, eventid); |
|
return -EINVAL; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu, |
|
struct perf_event *event) |
|
{ |
|
u8 config = event->attr.config; |
|
int i; |
|
|
|
/* DDR read free-run counter index */ |
|
if (config == EVENT_DDR_READS) { |
|
pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event; |
|
return DDRC_PERF_READ_COUNTER_IDX; |
|
} |
|
|
|
/* DDR write free-run counter index */ |
|
if (config == EVENT_DDR_WRITES) { |
|
pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event; |
|
return DDRC_PERF_WRITE_COUNTER_IDX; |
|
} |
|
|
|
/* Allocate DDR generic counters */ |
|
for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { |
|
if (pmu->events[i] == NULL) { |
|
pmu->events[i] = event; |
|
return i; |
|
} |
|
} |
|
|
|
return -ENOENT; |
|
} |
|
|
|
static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter) |
|
{ |
|
pmu->events[counter] = NULL; |
|
} |
|
|
|
static int cn10k_ddr_perf_event_init(struct perf_event *event) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
|
|
if (event->attr.type != event->pmu->type) |
|
return -ENOENT; |
|
|
|
if (is_sampling_event(event)) { |
|
dev_info(pmu->dev, "Sampling not supported!\n"); |
|
return -EOPNOTSUPP; |
|
} |
|
|
|
if (event->cpu < 0) { |
|
dev_warn(pmu->dev, "Can't provide per-task data!\n"); |
|
return -EOPNOTSUPP; |
|
} |
|
|
|
/* We must NOT create groups containing mixed PMUs */ |
|
if (event->group_leader->pmu != event->pmu && |
|
!is_software_event(event->group_leader)) |
|
return -EINVAL; |
|
|
|
/* Set ownership of event to one CPU, same event can not be observed |
|
* on multiple cpus at same time. |
|
*/ |
|
event->cpu = pmu->cpu; |
|
hwc->idx = -1; |
|
return 0; |
|
} |
|
|
|
static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu, |
|
int counter, bool enable) |
|
{ |
|
u32 reg; |
|
u64 val; |
|
|
|
if (counter > DDRC_PERF_NUM_COUNTERS) { |
|
pr_err("Error: unsupported counter %d\n", counter); |
|
return; |
|
} |
|
|
|
if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { |
|
reg = DDRC_PERF_CFG(counter); |
|
val = readq_relaxed(pmu->base + reg); |
|
|
|
if (enable) |
|
val |= EVENT_ENABLE; |
|
else |
|
val &= ~EVENT_ENABLE; |
|
|
|
writeq_relaxed(val, pmu->base + reg); |
|
} else { |
|
val = readq_relaxed(pmu->base + DDRC_PERF_CNT_FREERUN_EN); |
|
if (enable) { |
|
if (counter == DDRC_PERF_READ_COUNTER_IDX) |
|
val |= DDRC_PERF_FREERUN_READ_EN; |
|
else |
|
val |= DDRC_PERF_FREERUN_WRITE_EN; |
|
} else { |
|
if (counter == DDRC_PERF_READ_COUNTER_IDX) |
|
val &= ~DDRC_PERF_FREERUN_READ_EN; |
|
else |
|
val &= ~DDRC_PERF_FREERUN_WRITE_EN; |
|
} |
|
writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_EN); |
|
} |
|
} |
|
|
|
static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter) |
|
{ |
|
u64 val; |
|
|
|
if (counter == DDRC_PERF_READ_COUNTER_IDX) |
|
return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_RD_OP); |
|
|
|
if (counter == DDRC_PERF_WRITE_COUNTER_IDX) |
|
return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_WR_OP); |
|
|
|
val = readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE(counter)); |
|
return val; |
|
} |
|
|
|
static void cn10k_ddr_perf_event_update(struct perf_event *event) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
u64 prev_count, new_count, mask; |
|
|
|
do { |
|
prev_count = local64_read(&hwc->prev_count); |
|
new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); |
|
} while (local64_xchg(&hwc->prev_count, new_count) != prev_count); |
|
|
|
mask = DDRC_PERF_CNT_MAX_VALUE; |
|
|
|
local64_add((new_count - prev_count) & mask, &event->count); |
|
} |
|
|
|
static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int counter = hwc->idx; |
|
|
|
local64_set(&hwc->prev_count, 0); |
|
|
|
cn10k_ddr_perf_counter_enable(pmu, counter, true); |
|
|
|
hwc->state = 0; |
|
} |
|
|
|
static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
u8 config = event->attr.config; |
|
int counter, ret; |
|
u32 reg_offset; |
|
u64 val; |
|
|
|
counter = cn10k_ddr_perf_alloc_counter(pmu, event); |
|
if (counter < 0) |
|
return -EAGAIN; |
|
|
|
pmu->active_events++; |
|
hwc->idx = counter; |
|
|
|
if (pmu->active_events == 1) |
|
hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(), |
|
HRTIMER_MODE_REL_PINNED); |
|
|
|
if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { |
|
/* Generic counters, configure event id */ |
|
reg_offset = DDRC_PERF_CFG(counter); |
|
ret = ddr_perf_get_event_bitmap(config, &val); |
|
if (ret) |
|
return ret; |
|
|
|
writeq_relaxed(val, pmu->base + reg_offset); |
|
} else { |
|
/* fixed event counter, clear counter value */ |
|
if (counter == DDRC_PERF_READ_COUNTER_IDX) |
|
val = DDRC_FREERUN_READ_CNT_CLR; |
|
else |
|
val = DDRC_FREERUN_WRITE_CNT_CLR; |
|
|
|
writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_CTRL); |
|
} |
|
|
|
hwc->state |= PERF_HES_STOPPED; |
|
|
|
if (flags & PERF_EF_START) |
|
cn10k_ddr_perf_event_start(event, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int counter = hwc->idx; |
|
|
|
cn10k_ddr_perf_counter_enable(pmu, counter, false); |
|
|
|
if (flags & PERF_EF_UPDATE) |
|
cn10k_ddr_perf_event_update(event); |
|
|
|
hwc->state |= PERF_HES_STOPPED; |
|
} |
|
|
|
static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); |
|
struct hw_perf_event *hwc = &event->hw; |
|
int counter = hwc->idx; |
|
|
|
cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE); |
|
|
|
cn10k_ddr_perf_free_counter(pmu, counter); |
|
pmu->active_events--; |
|
hwc->idx = -1; |
|
|
|
/* Cancel timer when no events to capture */ |
|
if (pmu->active_events == 0) |
|
hrtimer_cancel(&pmu->hrtimer); |
|
} |
|
|
|
static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu) |
|
{ |
|
struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu); |
|
|
|
writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base + |
|
DDRC_PERF_CNT_START_OP_CTRL); |
|
} |
|
|
|
static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu) |
|
{ |
|
struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu); |
|
|
|
writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base + |
|
DDRC_PERF_CNT_END_OP_CTRL); |
|
} |
|
|
|
static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu) |
|
{ |
|
struct hw_perf_event *hwc; |
|
int i; |
|
|
|
for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { |
|
if (pmu->events[i] == NULL) |
|
continue; |
|
|
|
cn10k_ddr_perf_event_update(pmu->events[i]); |
|
} |
|
|
|
/* Reset previous count as h/w counter are reset */ |
|
for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { |
|
if (pmu->events[i] == NULL) |
|
continue; |
|
|
|
hwc = &pmu->events[i]->hw; |
|
local64_set(&hwc->prev_count, 0); |
|
} |
|
} |
|
|
|
static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu) |
|
{ |
|
struct perf_event *event; |
|
struct hw_perf_event *hwc; |
|
u64 prev_count, new_count; |
|
u64 value; |
|
int i; |
|
|
|
event = pmu->events[DDRC_PERF_READ_COUNTER_IDX]; |
|
if (event) { |
|
hwc = &event->hw; |
|
prev_count = local64_read(&hwc->prev_count); |
|
new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); |
|
|
|
/* Overflow condition is when new count less than |
|
* previous count |
|
*/ |
|
if (new_count < prev_count) |
|
cn10k_ddr_perf_event_update(event); |
|
} |
|
|
|
event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX]; |
|
if (event) { |
|
hwc = &event->hw; |
|
prev_count = local64_read(&hwc->prev_count); |
|
new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); |
|
|
|
/* Overflow condition is when new count less than |
|
* previous count |
|
*/ |
|
if (new_count < prev_count) |
|
cn10k_ddr_perf_event_update(event); |
|
} |
|
|
|
for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { |
|
if (pmu->events[i] == NULL) |
|
continue; |
|
|
|
value = cn10k_ddr_perf_read_counter(pmu, i); |
|
if (value == DDRC_PERF_CNT_MAX_VALUE) { |
|
pr_info("Counter-(%d) reached max value\n", i); |
|
cn10k_ddr_perf_event_update_all(pmu); |
|
cn10k_ddr_perf_pmu_disable(&pmu->pmu); |
|
cn10k_ddr_perf_pmu_enable(&pmu->pmu); |
|
} |
|
} |
|
|
|
return IRQ_HANDLED; |
|
} |
|
|
|
static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu, |
|
hrtimer); |
|
unsigned long flags; |
|
|
|
local_irq_save(flags); |
|
cn10k_ddr_pmu_overflow_handler(pmu); |
|
local_irq_restore(flags); |
|
|
|
hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period()); |
|
return HRTIMER_RESTART; |
|
} |
|
|
|
static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) |
|
{ |
|
struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu, |
|
node); |
|
unsigned int target; |
|
|
|
if (cpu != pmu->cpu) |
|
return 0; |
|
|
|
target = cpumask_any_but(cpu_online_mask, cpu); |
|
if (target >= nr_cpu_ids) |
|
return 0; |
|
|
|
perf_pmu_migrate_context(&pmu->pmu, cpu, target); |
|
pmu->cpu = target; |
|
return 0; |
|
} |
|
|
|
static int cn10k_ddr_perf_probe(struct platform_device *pdev) |
|
{ |
|
struct cn10k_ddr_pmu *ddr_pmu; |
|
struct resource *res; |
|
void __iomem *base; |
|
char *name; |
|
int ret; |
|
|
|
ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL); |
|
if (!ddr_pmu) |
|
return -ENOMEM; |
|
|
|
ddr_pmu->dev = &pdev->dev; |
|
platform_set_drvdata(pdev, ddr_pmu); |
|
|
|
base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); |
|
if (IS_ERR(base)) |
|
return PTR_ERR(base); |
|
|
|
ddr_pmu->base = base; |
|
|
|
/* Setup the PMU counter to work in manual mode */ |
|
writeq_relaxed(OP_MODE_CTRL_VAL_MANNUAL, ddr_pmu->base + |
|
DDRC_PERF_CNT_OP_MODE_CTRL); |
|
|
|
ddr_pmu->pmu = (struct pmu) { |
|
.module = THIS_MODULE, |
|
.capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
|
.task_ctx_nr = perf_invalid_context, |
|
.attr_groups = cn10k_attr_groups, |
|
.event_init = cn10k_ddr_perf_event_init, |
|
.add = cn10k_ddr_perf_event_add, |
|
.del = cn10k_ddr_perf_event_del, |
|
.start = cn10k_ddr_perf_event_start, |
|
.stop = cn10k_ddr_perf_event_stop, |
|
.read = cn10k_ddr_perf_event_update, |
|
.pmu_enable = cn10k_ddr_perf_pmu_enable, |
|
.pmu_disable = cn10k_ddr_perf_pmu_disable, |
|
}; |
|
|
|
/* Choose this cpu to collect perf data */ |
|
ddr_pmu->cpu = raw_smp_processor_id(); |
|
|
|
name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx", |
|
res->start); |
|
if (!name) |
|
return -ENOMEM; |
|
|
|
hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
|
ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler; |
|
|
|
cpuhp_state_add_instance_nocalls( |
|
CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, |
|
&ddr_pmu->node); |
|
|
|
ret = perf_pmu_register(&ddr_pmu->pmu, name, -1); |
|
if (ret) |
|
goto error; |
|
|
|
pr_info("CN10K DDR PMU Driver for ddrc@%llx\n", res->start); |
|
return 0; |
|
error: |
|
cpuhp_state_remove_instance_nocalls( |
|
CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, |
|
&ddr_pmu->node); |
|
return ret; |
|
} |
|
|
|
static int cn10k_ddr_perf_remove(struct platform_device *pdev) |
|
{ |
|
struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev); |
|
|
|
cpuhp_state_remove_instance_nocalls( |
|
CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, |
|
&ddr_pmu->node); |
|
|
|
perf_pmu_unregister(&ddr_pmu->pmu); |
|
return 0; |
|
} |
|
|
|
#ifdef CONFIG_OF |
|
static const struct of_device_id cn10k_ddr_pmu_of_match[] = { |
|
{ .compatible = "marvell,cn10k-ddr-pmu", }, |
|
{ }, |
|
}; |
|
MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match); |
|
#endif |
|
|
|
static struct platform_driver cn10k_ddr_pmu_driver = { |
|
.driver = { |
|
.name = "cn10k-ddr-pmu", |
|
.of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match), |
|
.suppress_bind_attrs = true, |
|
}, |
|
.probe = cn10k_ddr_perf_probe, |
|
.remove = cn10k_ddr_perf_remove, |
|
}; |
|
|
|
static int __init cn10k_ddr_pmu_init(void) |
|
{ |
|
int ret; |
|
|
|
ret = cpuhp_setup_state_multi( |
|
CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, |
|
"perf/marvell/cn10k/ddr:online", NULL, |
|
cn10k_ddr_pmu_offline_cpu); |
|
if (ret) |
|
return ret; |
|
|
|
ret = platform_driver_register(&cn10k_ddr_pmu_driver); |
|
if (ret) |
|
cpuhp_remove_multi_state( |
|
CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE); |
|
return ret; |
|
} |
|
|
|
static void __exit cn10k_ddr_pmu_exit(void) |
|
{ |
|
platform_driver_unregister(&cn10k_ddr_pmu_driver); |
|
cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE); |
|
} |
|
|
|
module_init(cn10k_ddr_pmu_init); |
|
module_exit(cn10k_ddr_pmu_exit); |
|
|
|
MODULE_AUTHOR("Bharat Bhushan <[email protected]>"); |
|
MODULE_LICENSE("GPL v2");
|
|
|