mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
294 lines
7.2 KiB
294 lines
7.2 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
|
|
/* |
|
* Hyper-V specific APIC code. |
|
* |
|
* Copyright (C) 2018, Microsoft, Inc. |
|
* |
|
* Author : K. Y. Srinivasan <[email protected]> |
|
* |
|
* This program is free software; you can redistribute it and/or modify it |
|
* under the terms of the GNU General Public License version 2 as published |
|
* by the Free Software Foundation. |
|
* |
|
* This program is distributed in the hope that it will be useful, but |
|
* WITHOUT ANY WARRANTY; without even the implied warranty of |
|
* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or |
|
* NON INFRINGEMENT. See the GNU General Public License for more |
|
* details. |
|
* |
|
*/ |
|
|
|
#include <linux/types.h> |
|
#include <linux/vmalloc.h> |
|
#include <linux/mm.h> |
|
#include <linux/clockchips.h> |
|
#include <linux/hyperv.h> |
|
#include <linux/slab.h> |
|
#include <linux/cpuhotplug.h> |
|
#include <asm/hypervisor.h> |
|
#include <asm/mshyperv.h> |
|
#include <asm/apic.h> |
|
|
|
#include <asm/trace/hyperv.h> |
|
|
|
static struct apic orig_apic; |
|
|
|
static u64 hv_apic_icr_read(void) |
|
{ |
|
u64 reg_val; |
|
|
|
rdmsrl(HV_X64_MSR_ICR, reg_val); |
|
return reg_val; |
|
} |
|
|
|
static void hv_apic_icr_write(u32 low, u32 id) |
|
{ |
|
u64 reg_val; |
|
|
|
reg_val = SET_APIC_DEST_FIELD(id); |
|
reg_val = reg_val << 32; |
|
reg_val |= low; |
|
|
|
wrmsrl(HV_X64_MSR_ICR, reg_val); |
|
} |
|
|
|
static u32 hv_apic_read(u32 reg) |
|
{ |
|
u32 reg_val, hi; |
|
|
|
switch (reg) { |
|
case APIC_EOI: |
|
rdmsr(HV_X64_MSR_EOI, reg_val, hi); |
|
return reg_val; |
|
case APIC_TASKPRI: |
|
rdmsr(HV_X64_MSR_TPR, reg_val, hi); |
|
return reg_val; |
|
|
|
default: |
|
return native_apic_mem_read(reg); |
|
} |
|
} |
|
|
|
static void hv_apic_write(u32 reg, u32 val) |
|
{ |
|
switch (reg) { |
|
case APIC_EOI: |
|
wrmsr(HV_X64_MSR_EOI, val, 0); |
|
break; |
|
case APIC_TASKPRI: |
|
wrmsr(HV_X64_MSR_TPR, val, 0); |
|
break; |
|
default: |
|
native_apic_mem_write(reg, val); |
|
} |
|
} |
|
|
|
static void hv_apic_eoi_write(u32 reg, u32 val) |
|
{ |
|
struct hv_vp_assist_page *hvp = hv_vp_assist_page[smp_processor_id()]; |
|
|
|
if (hvp && (xchg(&hvp->apic_assist, 0) & 0x1)) |
|
return; |
|
|
|
wrmsr(HV_X64_MSR_EOI, val, 0); |
|
} |
|
|
|
/* |
|
* IPI implementation on Hyper-V. |
|
*/ |
|
static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector) |
|
{ |
|
struct hv_send_ipi_ex **arg; |
|
struct hv_send_ipi_ex *ipi_arg; |
|
unsigned long flags; |
|
int nr_bank = 0; |
|
int ret = 1; |
|
|
|
if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) |
|
return false; |
|
|
|
local_irq_save(flags); |
|
arg = (struct hv_send_ipi_ex **)this_cpu_ptr(hyperv_pcpu_input_arg); |
|
|
|
ipi_arg = *arg; |
|
if (unlikely(!ipi_arg)) |
|
goto ipi_mask_ex_done; |
|
|
|
ipi_arg->vector = vector; |
|
ipi_arg->reserved = 0; |
|
ipi_arg->vp_set.valid_bank_mask = 0; |
|
|
|
if (!cpumask_equal(mask, cpu_present_mask)) { |
|
ipi_arg->vp_set.format = HV_GENERIC_SET_SPARSE_4K; |
|
nr_bank = cpumask_to_vpset(&(ipi_arg->vp_set), mask); |
|
} |
|
if (nr_bank < 0) |
|
goto ipi_mask_ex_done; |
|
if (!nr_bank) |
|
ipi_arg->vp_set.format = HV_GENERIC_SET_ALL; |
|
|
|
ret = hv_do_rep_hypercall(HVCALL_SEND_IPI_EX, 0, nr_bank, |
|
ipi_arg, NULL); |
|
|
|
ipi_mask_ex_done: |
|
local_irq_restore(flags); |
|
return ((ret == 0) ? true : false); |
|
} |
|
|
|
static bool __send_ipi_mask(const struct cpumask *mask, int vector) |
|
{ |
|
int cur_cpu, vcpu; |
|
struct hv_send_ipi ipi_arg; |
|
int ret = 1; |
|
|
|
trace_hyperv_send_ipi_mask(mask, vector); |
|
|
|
if (cpumask_empty(mask)) |
|
return true; |
|
|
|
if (!hv_hypercall_pg) |
|
return false; |
|
|
|
if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) |
|
return false; |
|
|
|
/* |
|
* From the supplied CPU set we need to figure out if we can get away |
|
* with cheaper HVCALL_SEND_IPI hypercall. This is possible when the |
|
* highest VP number in the set is < 64. As VP numbers are usually in |
|
* ascending order and match Linux CPU ids, here is an optimization: |
|
* we check the VP number for the highest bit in the supplied set first |
|
* so we can quickly find out if using HVCALL_SEND_IPI_EX hypercall is |
|
* a must. We will also check all VP numbers when walking the supplied |
|
* CPU set to remain correct in all cases. |
|
*/ |
|
if (hv_cpu_number_to_vp_number(cpumask_last(mask)) >= 64) |
|
goto do_ex_hypercall; |
|
|
|
ipi_arg.vector = vector; |
|
ipi_arg.cpu_mask = 0; |
|
|
|
for_each_cpu(cur_cpu, mask) { |
|
vcpu = hv_cpu_number_to_vp_number(cur_cpu); |
|
if (vcpu == VP_INVAL) |
|
return false; |
|
|
|
/* |
|
* This particular version of the IPI hypercall can |
|
* only target upto 64 CPUs. |
|
*/ |
|
if (vcpu >= 64) |
|
goto do_ex_hypercall; |
|
|
|
__set_bit(vcpu, (unsigned long *)&ipi_arg.cpu_mask); |
|
} |
|
|
|
ret = hv_do_fast_hypercall16(HVCALL_SEND_IPI, ipi_arg.vector, |
|
ipi_arg.cpu_mask); |
|
return ((ret == 0) ? true : false); |
|
|
|
do_ex_hypercall: |
|
return __send_ipi_mask_ex(mask, vector); |
|
} |
|
|
|
static bool __send_ipi_one(int cpu, int vector) |
|
{ |
|
int vp = hv_cpu_number_to_vp_number(cpu); |
|
|
|
trace_hyperv_send_ipi_one(cpu, vector); |
|
|
|
if (!hv_hypercall_pg || (vp == VP_INVAL)) |
|
return false; |
|
|
|
if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR)) |
|
return false; |
|
|
|
if (vp >= 64) |
|
return __send_ipi_mask_ex(cpumask_of(cpu), vector); |
|
|
|
return !hv_do_fast_hypercall16(HVCALL_SEND_IPI, vector, BIT_ULL(vp)); |
|
} |
|
|
|
static void hv_send_ipi(int cpu, int vector) |
|
{ |
|
if (!__send_ipi_one(cpu, vector)) |
|
orig_apic.send_IPI(cpu, vector); |
|
} |
|
|
|
static void hv_send_ipi_mask(const struct cpumask *mask, int vector) |
|
{ |
|
if (!__send_ipi_mask(mask, vector)) |
|
orig_apic.send_IPI_mask(mask, vector); |
|
} |
|
|
|
static void hv_send_ipi_mask_allbutself(const struct cpumask *mask, int vector) |
|
{ |
|
unsigned int this_cpu = smp_processor_id(); |
|
struct cpumask new_mask; |
|
const struct cpumask *local_mask; |
|
|
|
cpumask_copy(&new_mask, mask); |
|
cpumask_clear_cpu(this_cpu, &new_mask); |
|
local_mask = &new_mask; |
|
if (!__send_ipi_mask(local_mask, vector)) |
|
orig_apic.send_IPI_mask_allbutself(mask, vector); |
|
} |
|
|
|
static void hv_send_ipi_allbutself(int vector) |
|
{ |
|
hv_send_ipi_mask_allbutself(cpu_online_mask, vector); |
|
} |
|
|
|
static void hv_send_ipi_all(int vector) |
|
{ |
|
if (!__send_ipi_mask(cpu_online_mask, vector)) |
|
orig_apic.send_IPI_all(vector); |
|
} |
|
|
|
static void hv_send_ipi_self(int vector) |
|
{ |
|
if (!__send_ipi_one(smp_processor_id(), vector)) |
|
orig_apic.send_IPI_self(vector); |
|
} |
|
|
|
void __init hv_apic_init(void) |
|
{ |
|
if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) { |
|
pr_info("Hyper-V: Using IPI hypercalls\n"); |
|
/* |
|
* Set the IPI entry points. |
|
*/ |
|
orig_apic = *apic; |
|
|
|
apic->send_IPI = hv_send_ipi; |
|
apic->send_IPI_mask = hv_send_ipi_mask; |
|
apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself; |
|
apic->send_IPI_allbutself = hv_send_ipi_allbutself; |
|
apic->send_IPI_all = hv_send_ipi_all; |
|
apic->send_IPI_self = hv_send_ipi_self; |
|
} |
|
|
|
if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) { |
|
pr_info("Hyper-V: Using enlightened APIC (%s mode)", |
|
x2apic_enabled() ? "x2apic" : "xapic"); |
|
/* |
|
* When in x2apic mode, don't use the Hyper-V specific APIC |
|
* accessors since the field layout in the ICR register is |
|
* different in x2apic mode. Furthermore, the architectural |
|
* x2apic MSRs function just as well as the Hyper-V |
|
* synthetic APIC MSRs, so there's no benefit in having |
|
* separate Hyper-V accessors for x2apic mode. The only |
|
* exception is hv_apic_eoi_write, because it benefits from |
|
* lazy EOI when available, but the same accessor works for |
|
* both xapic and x2apic because the field layout is the same. |
|
*/ |
|
apic_set_eoi_write(hv_apic_eoi_write); |
|
if (!x2apic_enabled()) { |
|
apic->read = hv_apic_read; |
|
apic->write = hv_apic_write; |
|
apic->icr_write = hv_apic_icr_write; |
|
apic->icr_read = hv_apic_icr_read; |
|
} |
|
} |
|
}
|
|
|