mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
542 lines
13 KiB
542 lines
13 KiB
/* |
|
* @file op_model_amd.c |
|
* athlon / K7 / K8 / Family 10h model-specific MSR operations |
|
* |
|
* @remark Copyright 2002-2009 OProfile authors |
|
* @remark Read the file COPYING |
|
* |
|
* @author John Levon |
|
* @author Philippe Elie |
|
* @author Graydon Hoare |
|
* @author Robert Richter <[email protected]> |
|
* @author Barry Kasindorf <[email protected]> |
|
* @author Jason Yeh <[email protected]> |
|
* @author Suravee Suthikulpanit <[email protected]> |
|
*/ |
|
|
|
#include <linux/oprofile.h> |
|
#include <linux/device.h> |
|
#include <linux/pci.h> |
|
#include <linux/percpu.h> |
|
|
|
#include <asm/ptrace.h> |
|
#include <asm/msr.h> |
|
#include <asm/nmi.h> |
|
#include <asm/apic.h> |
|
#include <asm/processor.h> |
|
|
|
#include "op_x86_model.h" |
|
#include "op_counter.h" |
|
|
|
#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX |
|
#define NUM_VIRT_COUNTERS 32 |
|
#else |
|
#define NUM_VIRT_COUNTERS 0 |
|
#endif |
|
|
|
#define OP_EVENT_MASK 0x0FFF |
|
#define OP_CTR_OVERFLOW (1ULL<<31) |
|
|
|
#define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21)) |
|
|
|
static int num_counters; |
|
static unsigned long reset_value[OP_MAX_COUNTER]; |
|
|
|
#define IBS_FETCH_SIZE 6 |
|
#define IBS_OP_SIZE 12 |
|
|
|
static u32 ibs_caps; |
|
|
|
struct ibs_config { |
|
unsigned long op_enabled; |
|
unsigned long fetch_enabled; |
|
unsigned long max_cnt_fetch; |
|
unsigned long max_cnt_op; |
|
unsigned long rand_en; |
|
unsigned long dispatched_ops; |
|
unsigned long branch_target; |
|
}; |
|
|
|
struct ibs_state { |
|
u64 ibs_op_ctl; |
|
int branch_target; |
|
unsigned long sample_size; |
|
}; |
|
|
|
static struct ibs_config ibs_config; |
|
static struct ibs_state ibs_state; |
|
|
|
/* |
|
* IBS randomization macros |
|
*/ |
|
#define IBS_RANDOM_BITS 12 |
|
#define IBS_RANDOM_MASK ((1ULL << IBS_RANDOM_BITS) - 1) |
|
#define IBS_RANDOM_MAXCNT_OFFSET (1ULL << (IBS_RANDOM_BITS - 5)) |
|
|
|
/* |
|
* 16-bit Linear Feedback Shift Register (LFSR) |
|
* |
|
* 16 14 13 11 |
|
* Feedback polynomial = X + X + X + X + 1 |
|
*/ |
|
static unsigned int lfsr_random(void) |
|
{ |
|
static unsigned int lfsr_value = 0xF00D; |
|
unsigned int bit; |
|
|
|
/* Compute next bit to shift in */ |
|
bit = ((lfsr_value >> 0) ^ |
|
(lfsr_value >> 2) ^ |
|
(lfsr_value >> 3) ^ |
|
(lfsr_value >> 5)) & 0x0001; |
|
|
|
/* Advance to next register value */ |
|
lfsr_value = (lfsr_value >> 1) | (bit << 15); |
|
|
|
return lfsr_value; |
|
} |
|
|
|
/* |
|
* IBS software randomization |
|
* |
|
* The IBS periodic op counter is randomized in software. The lower 12 |
|
* bits of the 20 bit counter are randomized. IbsOpCurCnt is |
|
* initialized with a 12 bit random value. |
|
*/ |
|
static inline u64 op_amd_randomize_ibs_op(u64 val) |
|
{ |
|
unsigned int random = lfsr_random(); |
|
|
|
if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) |
|
/* |
|
* Work around if the hw can not write to IbsOpCurCnt |
|
* |
|
* Randomize the lower 8 bits of the 16 bit |
|
* IbsOpMaxCnt [15:0] value in the range of -128 to |
|
* +127 by adding/subtracting an offset to the |
|
* maximum count (IbsOpMaxCnt). |
|
* |
|
* To avoid over or underflows and protect upper bits |
|
* starting at bit 16, the initial value for |
|
* IbsOpMaxCnt must fit in the range from 0x0081 to |
|
* 0xff80. |
|
*/ |
|
val += (s8)(random >> 4); |
|
else |
|
val |= (u64)(random & IBS_RANDOM_MASK) << 32; |
|
|
|
return val; |
|
} |
|
|
|
static inline void |
|
op_amd_handle_ibs(struct pt_regs * const regs, |
|
struct op_msrs const * const msrs) |
|
{ |
|
u64 val, ctl; |
|
struct op_entry entry; |
|
|
|
if (!ibs_caps) |
|
return; |
|
|
|
if (ibs_config.fetch_enabled) { |
|
rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl); |
|
if (ctl & IBS_FETCH_VAL) { |
|
rdmsrl(MSR_AMD64_IBSFETCHLINAD, val); |
|
oprofile_write_reserve(&entry, regs, val, |
|
IBS_FETCH_CODE, IBS_FETCH_SIZE); |
|
oprofile_add_data64(&entry, val); |
|
oprofile_add_data64(&entry, ctl); |
|
rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val); |
|
oprofile_add_data64(&entry, val); |
|
oprofile_write_commit(&entry); |
|
|
|
/* reenable the IRQ */ |
|
ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT); |
|
ctl |= IBS_FETCH_ENABLE; |
|
wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl); |
|
} |
|
} |
|
|
|
if (ibs_config.op_enabled) { |
|
rdmsrl(MSR_AMD64_IBSOPCTL, ctl); |
|
if (ctl & IBS_OP_VAL) { |
|
rdmsrl(MSR_AMD64_IBSOPRIP, val); |
|
oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE, |
|
ibs_state.sample_size); |
|
oprofile_add_data64(&entry, val); |
|
rdmsrl(MSR_AMD64_IBSOPDATA, val); |
|
oprofile_add_data64(&entry, val); |
|
rdmsrl(MSR_AMD64_IBSOPDATA2, val); |
|
oprofile_add_data64(&entry, val); |
|
rdmsrl(MSR_AMD64_IBSOPDATA3, val); |
|
oprofile_add_data64(&entry, val); |
|
rdmsrl(MSR_AMD64_IBSDCLINAD, val); |
|
oprofile_add_data64(&entry, val); |
|
rdmsrl(MSR_AMD64_IBSDCPHYSAD, val); |
|
oprofile_add_data64(&entry, val); |
|
if (ibs_state.branch_target) { |
|
rdmsrl(MSR_AMD64_IBSBRTARGET, val); |
|
oprofile_add_data(&entry, (unsigned long)val); |
|
} |
|
oprofile_write_commit(&entry); |
|
|
|
/* reenable the IRQ */ |
|
ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl); |
|
wrmsrl(MSR_AMD64_IBSOPCTL, ctl); |
|
} |
|
} |
|
} |
|
|
|
static inline void op_amd_start_ibs(void) |
|
{ |
|
u64 val; |
|
|
|
if (!ibs_caps) |
|
return; |
|
|
|
memset(&ibs_state, 0, sizeof(ibs_state)); |
|
|
|
/* |
|
* Note: Since the max count settings may out of range we |
|
* write back the actual used values so that userland can read |
|
* it. |
|
*/ |
|
|
|
if (ibs_config.fetch_enabled) { |
|
val = ibs_config.max_cnt_fetch >> 4; |
|
val = min(val, IBS_FETCH_MAX_CNT); |
|
ibs_config.max_cnt_fetch = val << 4; |
|
val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0; |
|
val |= IBS_FETCH_ENABLE; |
|
wrmsrl(MSR_AMD64_IBSFETCHCTL, val); |
|
} |
|
|
|
if (ibs_config.op_enabled) { |
|
val = ibs_config.max_cnt_op >> 4; |
|
if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) { |
|
/* |
|
* IbsOpCurCnt not supported. See |
|
* op_amd_randomize_ibs_op() for details. |
|
*/ |
|
val = clamp(val, 0x0081ULL, 0xFF80ULL); |
|
ibs_config.max_cnt_op = val << 4; |
|
} else { |
|
/* |
|
* The start value is randomized with a |
|
* positive offset, we need to compensate it |
|
* with the half of the randomized range. Also |
|
* avoid underflows. |
|
*/ |
|
val += IBS_RANDOM_MAXCNT_OFFSET; |
|
if (ibs_caps & IBS_CAPS_OPCNTEXT) |
|
val = min(val, IBS_OP_MAX_CNT_EXT); |
|
else |
|
val = min(val, IBS_OP_MAX_CNT); |
|
ibs_config.max_cnt_op = |
|
(val - IBS_RANDOM_MAXCNT_OFFSET) << 4; |
|
} |
|
val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT); |
|
val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0; |
|
val |= IBS_OP_ENABLE; |
|
ibs_state.ibs_op_ctl = val; |
|
ibs_state.sample_size = IBS_OP_SIZE; |
|
if (ibs_config.branch_target) { |
|
ibs_state.branch_target = 1; |
|
ibs_state.sample_size++; |
|
} |
|
val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl); |
|
wrmsrl(MSR_AMD64_IBSOPCTL, val); |
|
} |
|
} |
|
|
|
static void op_amd_stop_ibs(void) |
|
{ |
|
if (!ibs_caps) |
|
return; |
|
|
|
if (ibs_config.fetch_enabled) |
|
/* clear max count and enable */ |
|
wrmsrl(MSR_AMD64_IBSFETCHCTL, 0); |
|
|
|
if (ibs_config.op_enabled) |
|
/* clear max count and enable */ |
|
wrmsrl(MSR_AMD64_IBSOPCTL, 0); |
|
} |
|
|
|
#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX |
|
|
|
static void op_mux_switch_ctrl(struct op_x86_model_spec const *model, |
|
struct op_msrs const * const msrs) |
|
{ |
|
u64 val; |
|
int i; |
|
|
|
/* enable active counters */ |
|
for (i = 0; i < num_counters; ++i) { |
|
int virt = op_x86_phys_to_virt(i); |
|
if (!reset_value[virt]) |
|
continue; |
|
rdmsrl(msrs->controls[i].addr, val); |
|
val &= model->reserved; |
|
val |= op_x86_get_ctrl(model, &counter_config[virt]); |
|
wrmsrl(msrs->controls[i].addr, val); |
|
} |
|
} |
|
|
|
#endif |
|
|
|
/* functions for op_amd_spec */ |
|
|
|
static void op_amd_shutdown(struct op_msrs const * const msrs) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < num_counters; ++i) { |
|
if (!msrs->counters[i].addr) |
|
continue; |
|
release_perfctr_nmi(MSR_K7_PERFCTR0 + i); |
|
release_evntsel_nmi(MSR_K7_EVNTSEL0 + i); |
|
} |
|
} |
|
|
|
static int op_amd_fill_in_addresses(struct op_msrs * const msrs) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < num_counters; i++) { |
|
if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i)) |
|
goto fail; |
|
if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) { |
|
release_perfctr_nmi(MSR_K7_PERFCTR0 + i); |
|
goto fail; |
|
} |
|
/* both registers must be reserved */ |
|
if (num_counters == AMD64_NUM_COUNTERS_CORE) { |
|
msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1); |
|
msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1); |
|
} else { |
|
msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i; |
|
msrs->counters[i].addr = MSR_K7_PERFCTR0 + i; |
|
} |
|
continue; |
|
fail: |
|
if (!counter_config[i].enabled) |
|
continue; |
|
op_x86_warn_reserved(i); |
|
op_amd_shutdown(msrs); |
|
return -EBUSY; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, |
|
struct op_msrs const * const msrs) |
|
{ |
|
u64 val; |
|
int i; |
|
|
|
/* setup reset_value */ |
|
for (i = 0; i < OP_MAX_COUNTER; ++i) { |
|
if (counter_config[i].enabled |
|
&& msrs->counters[op_x86_virt_to_phys(i)].addr) |
|
reset_value[i] = counter_config[i].count; |
|
else |
|
reset_value[i] = 0; |
|
} |
|
|
|
/* clear all counters */ |
|
for (i = 0; i < num_counters; ++i) { |
|
if (!msrs->controls[i].addr) |
|
continue; |
|
rdmsrl(msrs->controls[i].addr, val); |
|
if (val & ARCH_PERFMON_EVENTSEL_ENABLE) |
|
op_x86_warn_in_use(i); |
|
val &= model->reserved; |
|
wrmsrl(msrs->controls[i].addr, val); |
|
/* |
|
* avoid a false detection of ctr overflows in NMI |
|
* handler |
|
*/ |
|
wrmsrl(msrs->counters[i].addr, -1LL); |
|
} |
|
|
|
/* enable active counters */ |
|
for (i = 0; i < num_counters; ++i) { |
|
int virt = op_x86_phys_to_virt(i); |
|
if (!reset_value[virt]) |
|
continue; |
|
|
|
/* setup counter registers */ |
|
wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]); |
|
|
|
/* setup control registers */ |
|
rdmsrl(msrs->controls[i].addr, val); |
|
val &= model->reserved; |
|
val |= op_x86_get_ctrl(model, &counter_config[virt]); |
|
wrmsrl(msrs->controls[i].addr, val); |
|
} |
|
} |
|
|
|
static int op_amd_check_ctrs(struct pt_regs * const regs, |
|
struct op_msrs const * const msrs) |
|
{ |
|
u64 val; |
|
int i; |
|
|
|
for (i = 0; i < num_counters; ++i) { |
|
int virt = op_x86_phys_to_virt(i); |
|
if (!reset_value[virt]) |
|
continue; |
|
rdmsrl(msrs->counters[i].addr, val); |
|
/* bit is clear if overflowed: */ |
|
if (val & OP_CTR_OVERFLOW) |
|
continue; |
|
oprofile_add_sample(regs, virt); |
|
wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]); |
|
} |
|
|
|
op_amd_handle_ibs(regs, msrs); |
|
|
|
/* See op_model_ppro.c */ |
|
return 1; |
|
} |
|
|
|
static void op_amd_start(struct op_msrs const * const msrs) |
|
{ |
|
u64 val; |
|
int i; |
|
|
|
for (i = 0; i < num_counters; ++i) { |
|
if (!reset_value[op_x86_phys_to_virt(i)]) |
|
continue; |
|
rdmsrl(msrs->controls[i].addr, val); |
|
val |= ARCH_PERFMON_EVENTSEL_ENABLE; |
|
wrmsrl(msrs->controls[i].addr, val); |
|
} |
|
|
|
op_amd_start_ibs(); |
|
} |
|
|
|
static void op_amd_stop(struct op_msrs const * const msrs) |
|
{ |
|
u64 val; |
|
int i; |
|
|
|
/* |
|
* Subtle: stop on all counters to avoid race with setting our |
|
* pm callback |
|
*/ |
|
for (i = 0; i < num_counters; ++i) { |
|
if (!reset_value[op_x86_phys_to_virt(i)]) |
|
continue; |
|
rdmsrl(msrs->controls[i].addr, val); |
|
val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; |
|
wrmsrl(msrs->controls[i].addr, val); |
|
} |
|
|
|
op_amd_stop_ibs(); |
|
} |
|
|
|
/* |
|
* check and reserve APIC extended interrupt LVT offset for IBS if |
|
* available |
|
*/ |
|
|
|
static void init_ibs(void) |
|
{ |
|
ibs_caps = get_ibs_caps(); |
|
|
|
if (!ibs_caps) |
|
return; |
|
|
|
printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps); |
|
} |
|
|
|
static int (*create_arch_files)(struct dentry *root); |
|
|
|
static int setup_ibs_files(struct dentry *root) |
|
{ |
|
struct dentry *dir; |
|
int ret = 0; |
|
|
|
/* architecture specific files */ |
|
if (create_arch_files) |
|
ret = create_arch_files(root); |
|
|
|
if (ret) |
|
return ret; |
|
|
|
if (!ibs_caps) |
|
return ret; |
|
|
|
/* model specific files */ |
|
|
|
/* setup some reasonable defaults */ |
|
memset(&ibs_config, 0, sizeof(ibs_config)); |
|
ibs_config.max_cnt_fetch = 250000; |
|
ibs_config.max_cnt_op = 250000; |
|
|
|
if (ibs_caps & IBS_CAPS_FETCHSAM) { |
|
dir = oprofilefs_mkdir(root, "ibs_fetch"); |
|
oprofilefs_create_ulong(dir, "enable", |
|
&ibs_config.fetch_enabled); |
|
oprofilefs_create_ulong(dir, "max_count", |
|
&ibs_config.max_cnt_fetch); |
|
oprofilefs_create_ulong(dir, "rand_enable", |
|
&ibs_config.rand_en); |
|
} |
|
|
|
if (ibs_caps & IBS_CAPS_OPSAM) { |
|
dir = oprofilefs_mkdir(root, "ibs_op"); |
|
oprofilefs_create_ulong(dir, "enable", |
|
&ibs_config.op_enabled); |
|
oprofilefs_create_ulong(dir, "max_count", |
|
&ibs_config.max_cnt_op); |
|
if (ibs_caps & IBS_CAPS_OPCNT) |
|
oprofilefs_create_ulong(dir, "dispatched_ops", |
|
&ibs_config.dispatched_ops); |
|
if (ibs_caps & IBS_CAPS_BRNTRGT) |
|
oprofilefs_create_ulong(dir, "branch_target", |
|
&ibs_config.branch_target); |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
struct op_x86_model_spec op_amd_spec; |
|
|
|
static int op_amd_init(struct oprofile_operations *ops) |
|
{ |
|
init_ibs(); |
|
create_arch_files = ops->create_files; |
|
ops->create_files = setup_ibs_files; |
|
|
|
if (boot_cpu_data.x86 == 0x15) { |
|
num_counters = AMD64_NUM_COUNTERS_CORE; |
|
} else { |
|
num_counters = AMD64_NUM_COUNTERS; |
|
} |
|
|
|
op_amd_spec.num_counters = num_counters; |
|
op_amd_spec.num_controls = num_counters; |
|
op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS); |
|
|
|
return 0; |
|
} |
|
|
|
struct op_x86_model_spec op_amd_spec = { |
|
/* num_counters/num_controls filled in at runtime */ |
|
.reserved = MSR_AMD_EVENTSEL_RESERVED, |
|
.event_mask = OP_EVENT_MASK, |
|
.init = op_amd_init, |
|
.fill_in_addresses = &op_amd_fill_in_addresses, |
|
.setup_ctrs = &op_amd_setup_ctrs, |
|
.check_ctrs = &op_amd_check_ctrs, |
|
.start = &op_amd_start, |
|
.stop = &op_amd_stop, |
|
.shutdown = &op_amd_shutdown, |
|
#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX |
|
.switch_ctrl = &op_mux_switch_ctrl, |
|
#endif |
|
};
|
|
|