mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
581 lines
14 KiB
581 lines
14 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
#include <linux/types.h> |
|
#include <linux/interrupt.h> |
|
|
|
#include <asm/xen/hypercall.h> |
|
#include <xen/xen.h> |
|
#include <xen/page.h> |
|
#include <xen/interface/xen.h> |
|
#include <xen/interface/vcpu.h> |
|
#include <xen/interface/xenpmu.h> |
|
|
|
#include "xen-ops.h" |
|
#include "pmu.h" |
|
|
|
/* x86_pmu.handle_irq definition */ |
|
#include "../events/perf_event.h" |
|
|
|
#define XENPMU_IRQ_PROCESSING 1 |
|
struct xenpmu { |
|
/* Shared page between hypervisor and domain */ |
|
struct xen_pmu_data *xenpmu_data; |
|
|
|
uint8_t flags; |
|
}; |
|
static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared); |
|
#define get_xenpmu_data() (this_cpu_ptr(&xenpmu_shared)->xenpmu_data) |
|
#define get_xenpmu_flags() (this_cpu_ptr(&xenpmu_shared)->flags) |
|
|
|
/* Macro for computing address of a PMU MSR bank */ |
|
#define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \ |
|
(uintptr_t)ctxt->field)) |
|
|
|
/* AMD PMU */ |
|
#define F15H_NUM_COUNTERS 6 |
|
#define F10H_NUM_COUNTERS 4 |
|
|
|
static __read_mostly uint32_t amd_counters_base; |
|
static __read_mostly uint32_t amd_ctrls_base; |
|
static __read_mostly int amd_msr_step; |
|
static __read_mostly int k7_counters_mirrored; |
|
static __read_mostly int amd_num_counters; |
|
|
|
/* Intel PMU */ |
|
#define MSR_TYPE_COUNTER 0 |
|
#define MSR_TYPE_CTRL 1 |
|
#define MSR_TYPE_GLOBAL 2 |
|
#define MSR_TYPE_ARCH_COUNTER 3 |
|
#define MSR_TYPE_ARCH_CTRL 4 |
|
|
|
/* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */ |
|
#define PMU_GENERAL_NR_SHIFT 8 |
|
#define PMU_GENERAL_NR_BITS 8 |
|
#define PMU_GENERAL_NR_MASK (((1 << PMU_GENERAL_NR_BITS) - 1) \ |
|
<< PMU_GENERAL_NR_SHIFT) |
|
|
|
/* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */ |
|
#define PMU_FIXED_NR_SHIFT 0 |
|
#define PMU_FIXED_NR_BITS 5 |
|
#define PMU_FIXED_NR_MASK (((1 << PMU_FIXED_NR_BITS) - 1) \ |
|
<< PMU_FIXED_NR_SHIFT) |
|
|
|
/* Alias registers (0x4c1) for full-width writes to PMCs */ |
|
#define MSR_PMC_ALIAS_MASK (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0)) |
|
|
|
#define INTEL_PMC_TYPE_SHIFT 30 |
|
|
|
static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters; |
|
|
|
|
|
static void xen_pmu_arch_init(void) |
|
{ |
|
if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { |
|
|
|
switch (boot_cpu_data.x86) { |
|
case 0x15: |
|
amd_num_counters = F15H_NUM_COUNTERS; |
|
amd_counters_base = MSR_F15H_PERF_CTR; |
|
amd_ctrls_base = MSR_F15H_PERF_CTL; |
|
amd_msr_step = 2; |
|
k7_counters_mirrored = 1; |
|
break; |
|
case 0x10: |
|
case 0x12: |
|
case 0x14: |
|
case 0x16: |
|
default: |
|
amd_num_counters = F10H_NUM_COUNTERS; |
|
amd_counters_base = MSR_K7_PERFCTR0; |
|
amd_ctrls_base = MSR_K7_EVNTSEL0; |
|
amd_msr_step = 1; |
|
k7_counters_mirrored = 0; |
|
break; |
|
} |
|
} else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { |
|
amd_num_counters = F10H_NUM_COUNTERS; |
|
amd_counters_base = MSR_K7_PERFCTR0; |
|
amd_ctrls_base = MSR_K7_EVNTSEL0; |
|
amd_msr_step = 1; |
|
k7_counters_mirrored = 0; |
|
} else { |
|
uint32_t eax, ebx, ecx, edx; |
|
|
|
cpuid(0xa, &eax, &ebx, &ecx, &edx); |
|
|
|
intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >> |
|
PMU_GENERAL_NR_SHIFT; |
|
intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >> |
|
PMU_FIXED_NR_SHIFT; |
|
} |
|
} |
|
|
|
static inline uint32_t get_fam15h_addr(u32 addr) |
|
{ |
|
switch (addr) { |
|
case MSR_K7_PERFCTR0: |
|
case MSR_K7_PERFCTR1: |
|
case MSR_K7_PERFCTR2: |
|
case MSR_K7_PERFCTR3: |
|
return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0); |
|
case MSR_K7_EVNTSEL0: |
|
case MSR_K7_EVNTSEL1: |
|
case MSR_K7_EVNTSEL2: |
|
case MSR_K7_EVNTSEL3: |
|
return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0); |
|
default: |
|
break; |
|
} |
|
|
|
return addr; |
|
} |
|
|
|
static inline bool is_amd_pmu_msr(unsigned int msr) |
|
{ |
|
if ((msr >= MSR_F15H_PERF_CTL && |
|
msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) || |
|
(msr >= MSR_K7_EVNTSEL0 && |
|
msr < MSR_K7_PERFCTR0 + amd_num_counters)) |
|
return true; |
|
|
|
return false; |
|
} |
|
|
|
static int is_intel_pmu_msr(u32 msr_index, int *type, int *index) |
|
{ |
|
u32 msr_index_pmc; |
|
|
|
switch (msr_index) { |
|
case MSR_CORE_PERF_FIXED_CTR_CTRL: |
|
case MSR_IA32_DS_AREA: |
|
case MSR_IA32_PEBS_ENABLE: |
|
*type = MSR_TYPE_CTRL; |
|
return true; |
|
|
|
case MSR_CORE_PERF_GLOBAL_CTRL: |
|
case MSR_CORE_PERF_GLOBAL_STATUS: |
|
case MSR_CORE_PERF_GLOBAL_OVF_CTRL: |
|
*type = MSR_TYPE_GLOBAL; |
|
return true; |
|
|
|
default: |
|
|
|
if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) && |
|
(msr_index < MSR_CORE_PERF_FIXED_CTR0 + |
|
intel_num_fixed_counters)) { |
|
*index = msr_index - MSR_CORE_PERF_FIXED_CTR0; |
|
*type = MSR_TYPE_COUNTER; |
|
return true; |
|
} |
|
|
|
if ((msr_index >= MSR_P6_EVNTSEL0) && |
|
(msr_index < MSR_P6_EVNTSEL0 + intel_num_arch_counters)) { |
|
*index = msr_index - MSR_P6_EVNTSEL0; |
|
*type = MSR_TYPE_ARCH_CTRL; |
|
return true; |
|
} |
|
|
|
msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK; |
|
if ((msr_index_pmc >= MSR_IA32_PERFCTR0) && |
|
(msr_index_pmc < MSR_IA32_PERFCTR0 + |
|
intel_num_arch_counters)) { |
|
*type = MSR_TYPE_ARCH_COUNTER; |
|
*index = msr_index_pmc - MSR_IA32_PERFCTR0; |
|
return true; |
|
} |
|
return false; |
|
} |
|
} |
|
|
|
static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type, |
|
int index, bool is_read) |
|
{ |
|
uint64_t *reg = NULL; |
|
struct xen_pmu_intel_ctxt *ctxt; |
|
uint64_t *fix_counters; |
|
struct xen_pmu_cntr_pair *arch_cntr_pair; |
|
struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
uint8_t xenpmu_flags = get_xenpmu_flags(); |
|
|
|
|
|
if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) |
|
return false; |
|
|
|
ctxt = &xenpmu_data->pmu.c.intel; |
|
|
|
switch (msr) { |
|
case MSR_CORE_PERF_GLOBAL_OVF_CTRL: |
|
reg = &ctxt->global_ovf_ctrl; |
|
break; |
|
case MSR_CORE_PERF_GLOBAL_STATUS: |
|
reg = &ctxt->global_status; |
|
break; |
|
case MSR_CORE_PERF_GLOBAL_CTRL: |
|
reg = &ctxt->global_ctrl; |
|
break; |
|
case MSR_CORE_PERF_FIXED_CTR_CTRL: |
|
reg = &ctxt->fixed_ctrl; |
|
break; |
|
default: |
|
switch (type) { |
|
case MSR_TYPE_COUNTER: |
|
fix_counters = field_offset(ctxt, fixed_counters); |
|
reg = &fix_counters[index]; |
|
break; |
|
case MSR_TYPE_ARCH_COUNTER: |
|
arch_cntr_pair = field_offset(ctxt, arch_counters); |
|
reg = &arch_cntr_pair[index].counter; |
|
break; |
|
case MSR_TYPE_ARCH_CTRL: |
|
arch_cntr_pair = field_offset(ctxt, arch_counters); |
|
reg = &arch_cntr_pair[index].control; |
|
break; |
|
default: |
|
return false; |
|
} |
|
} |
|
|
|
if (reg) { |
|
if (is_read) |
|
*val = *reg; |
|
else { |
|
*reg = *val; |
|
|
|
if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL) |
|
ctxt->global_status &= (~(*val)); |
|
} |
|
return true; |
|
} |
|
|
|
return false; |
|
} |
|
|
|
static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) |
|
{ |
|
uint64_t *reg = NULL; |
|
int i, off = 0; |
|
struct xen_pmu_amd_ctxt *ctxt; |
|
uint64_t *counter_regs, *ctrl_regs; |
|
struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
uint8_t xenpmu_flags = get_xenpmu_flags(); |
|
|
|
if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) |
|
return false; |
|
|
|
if (k7_counters_mirrored && |
|
((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3))) |
|
msr = get_fam15h_addr(msr); |
|
|
|
ctxt = &xenpmu_data->pmu.c.amd; |
|
for (i = 0; i < amd_num_counters; i++) { |
|
if (msr == amd_ctrls_base + off) { |
|
ctrl_regs = field_offset(ctxt, ctrls); |
|
reg = &ctrl_regs[i]; |
|
break; |
|
} else if (msr == amd_counters_base + off) { |
|
counter_regs = field_offset(ctxt, counters); |
|
reg = &counter_regs[i]; |
|
break; |
|
} |
|
off += amd_msr_step; |
|
} |
|
|
|
if (reg) { |
|
if (is_read) |
|
*val = *reg; |
|
else |
|
*reg = *val; |
|
|
|
return true; |
|
} |
|
return false; |
|
} |
|
|
|
bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err) |
|
{ |
|
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { |
|
if (is_amd_pmu_msr(msr)) { |
|
if (!xen_amd_pmu_emulate(msr, val, 1)) |
|
*val = native_read_msr_safe(msr, err); |
|
return true; |
|
} |
|
} else { |
|
int type, index; |
|
|
|
if (is_intel_pmu_msr(msr, &type, &index)) { |
|
if (!xen_intel_pmu_emulate(msr, val, type, index, 1)) |
|
*val = native_read_msr_safe(msr, err); |
|
return true; |
|
} |
|
} |
|
|
|
return false; |
|
} |
|
|
|
bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err) |
|
{ |
|
uint64_t val = ((uint64_t)high << 32) | low; |
|
|
|
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { |
|
if (is_amd_pmu_msr(msr)) { |
|
if (!xen_amd_pmu_emulate(msr, &val, 0)) |
|
*err = native_write_msr_safe(msr, low, high); |
|
return true; |
|
} |
|
} else { |
|
int type, index; |
|
|
|
if (is_intel_pmu_msr(msr, &type, &index)) { |
|
if (!xen_intel_pmu_emulate(msr, &val, type, index, 0)) |
|
*err = native_write_msr_safe(msr, low, high); |
|
return true; |
|
} |
|
} |
|
|
|
return false; |
|
} |
|
|
|
static unsigned long long xen_amd_read_pmc(int counter) |
|
{ |
|
struct xen_pmu_amd_ctxt *ctxt; |
|
uint64_t *counter_regs; |
|
struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
uint8_t xenpmu_flags = get_xenpmu_flags(); |
|
|
|
if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { |
|
uint32_t msr; |
|
int err; |
|
|
|
msr = amd_counters_base + (counter * amd_msr_step); |
|
return native_read_msr_safe(msr, &err); |
|
} |
|
|
|
ctxt = &xenpmu_data->pmu.c.amd; |
|
counter_regs = field_offset(ctxt, counters); |
|
return counter_regs[counter]; |
|
} |
|
|
|
static unsigned long long xen_intel_read_pmc(int counter) |
|
{ |
|
struct xen_pmu_intel_ctxt *ctxt; |
|
uint64_t *fixed_counters; |
|
struct xen_pmu_cntr_pair *arch_cntr_pair; |
|
struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
uint8_t xenpmu_flags = get_xenpmu_flags(); |
|
|
|
if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { |
|
uint32_t msr; |
|
int err; |
|
|
|
if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) |
|
msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff); |
|
else |
|
msr = MSR_IA32_PERFCTR0 + counter; |
|
|
|
return native_read_msr_safe(msr, &err); |
|
} |
|
|
|
ctxt = &xenpmu_data->pmu.c.intel; |
|
if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) { |
|
fixed_counters = field_offset(ctxt, fixed_counters); |
|
return fixed_counters[counter & 0xffff]; |
|
} |
|
|
|
arch_cntr_pair = field_offset(ctxt, arch_counters); |
|
return arch_cntr_pair[counter].counter; |
|
} |
|
|
|
unsigned long long xen_read_pmc(int counter) |
|
{ |
|
if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) |
|
return xen_amd_read_pmc(counter); |
|
else |
|
return xen_intel_read_pmc(counter); |
|
} |
|
|
|
int pmu_apic_update(uint32_t val) |
|
{ |
|
int ret; |
|
struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
|
|
if (!xenpmu_data) { |
|
pr_warn_once("%s: pmudata not initialized\n", __func__); |
|
return -EINVAL; |
|
} |
|
|
|
xenpmu_data->pmu.l.lapic_lvtpc = val; |
|
|
|
if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING) |
|
return 0; |
|
|
|
ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL); |
|
|
|
return ret; |
|
} |
|
|
|
/* perf callbacks */ |
|
static int xen_is_in_guest(void) |
|
{ |
|
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
|
|
if (!xenpmu_data) { |
|
pr_warn_once("%s: pmudata not initialized\n", __func__); |
|
return 0; |
|
} |
|
|
|
if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF)) |
|
return 0; |
|
|
|
return 1; |
|
} |
|
|
|
static int xen_is_user_mode(void) |
|
{ |
|
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
|
|
if (!xenpmu_data) { |
|
pr_warn_once("%s: pmudata not initialized\n", __func__); |
|
return 0; |
|
} |
|
|
|
if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) |
|
return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER); |
|
else |
|
return !!(xenpmu_data->pmu.r.regs.cpl & 3); |
|
} |
|
|
|
static unsigned long xen_get_guest_ip(void) |
|
{ |
|
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
|
|
if (!xenpmu_data) { |
|
pr_warn_once("%s: pmudata not initialized\n", __func__); |
|
return 0; |
|
} |
|
|
|
return xenpmu_data->pmu.r.regs.ip; |
|
} |
|
|
|
static struct perf_guest_info_callbacks xen_guest_cbs = { |
|
.is_in_guest = xen_is_in_guest, |
|
.is_user_mode = xen_is_user_mode, |
|
.get_guest_ip = xen_get_guest_ip, |
|
}; |
|
|
|
/* Convert registers from Xen's format to Linux' */ |
|
static void xen_convert_regs(const struct xen_pmu_regs *xen_regs, |
|
struct pt_regs *regs, uint64_t pmu_flags) |
|
{ |
|
regs->ip = xen_regs->ip; |
|
regs->cs = xen_regs->cs; |
|
regs->sp = xen_regs->sp; |
|
|
|
if (pmu_flags & PMU_SAMPLE_PV) { |
|
if (pmu_flags & PMU_SAMPLE_USER) |
|
regs->cs |= 3; |
|
else |
|
regs->cs &= ~3; |
|
} else { |
|
if (xen_regs->cpl) |
|
regs->cs |= 3; |
|
else |
|
regs->cs &= ~3; |
|
} |
|
} |
|
|
|
irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) |
|
{ |
|
int err, ret = IRQ_NONE; |
|
struct pt_regs regs = {0}; |
|
const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); |
|
uint8_t xenpmu_flags = get_xenpmu_flags(); |
|
|
|
if (!xenpmu_data) { |
|
pr_warn_once("%s: pmudata not initialized\n", __func__); |
|
return ret; |
|
} |
|
|
|
this_cpu_ptr(&xenpmu_shared)->flags = |
|
xenpmu_flags | XENPMU_IRQ_PROCESSING; |
|
xen_convert_regs(&xenpmu_data->pmu.r.regs, ®s, |
|
xenpmu_data->pmu.pmu_flags); |
|
if (x86_pmu.handle_irq(®s)) |
|
ret = IRQ_HANDLED; |
|
|
|
/* Write out cached context to HW */ |
|
err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL); |
|
this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags; |
|
if (err) { |
|
pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err); |
|
return IRQ_NONE; |
|
} |
|
|
|
return ret; |
|
} |
|
|
|
bool is_xen_pmu(int cpu) |
|
{ |
|
return (get_xenpmu_data() != NULL); |
|
} |
|
|
|
void xen_pmu_init(int cpu) |
|
{ |
|
int err; |
|
struct xen_pmu_params xp; |
|
unsigned long pfn; |
|
struct xen_pmu_data *xenpmu_data; |
|
|
|
BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE); |
|
|
|
if (xen_hvm_domain()) |
|
return; |
|
|
|
xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL); |
|
if (!xenpmu_data) { |
|
pr_err("VPMU init: No memory\n"); |
|
return; |
|
} |
|
pfn = virt_to_pfn(xenpmu_data); |
|
|
|
xp.val = pfn_to_mfn(pfn); |
|
xp.vcpu = cpu; |
|
xp.version.maj = XENPMU_VER_MAJ; |
|
xp.version.min = XENPMU_VER_MIN; |
|
err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp); |
|
if (err) |
|
goto fail; |
|
|
|
per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data; |
|
per_cpu(xenpmu_shared, cpu).flags = 0; |
|
|
|
if (cpu == 0) { |
|
perf_register_guest_info_callbacks(&xen_guest_cbs); |
|
xen_pmu_arch_init(); |
|
} |
|
|
|
return; |
|
|
|
fail: |
|
if (err == -EOPNOTSUPP || err == -ENOSYS) |
|
pr_info_once("VPMU disabled by hypervisor.\n"); |
|
else |
|
pr_info_once("Could not initialize VPMU for cpu %d, error %d\n", |
|
cpu, err); |
|
free_pages((unsigned long)xenpmu_data, 0); |
|
} |
|
|
|
void xen_pmu_finish(int cpu) |
|
{ |
|
struct xen_pmu_params xp; |
|
|
|
if (xen_hvm_domain()) |
|
return; |
|
|
|
xp.vcpu = cpu; |
|
xp.version.maj = XENPMU_VER_MAJ; |
|
xp.version.min = XENPMU_VER_MIN; |
|
|
|
(void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp); |
|
|
|
free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0); |
|
per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL; |
|
}
|
|
|