forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
346 lines
9.1 KiB
346 lines
9.1 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
#include <linux/kvm_host.h> |
|
|
|
#include <asm/irq_remapping.h> |
|
#include <asm/cpu.h> |
|
|
|
#include "lapic.h" |
|
#include "posted_intr.h" |
|
#include "trace.h" |
|
#include "vmx.h" |
|
|
|
/* |
|
* We maintain a per-CPU linked-list of vCPU, so in wakeup_handler() we |
|
* can find which vCPU should be waken up. |
|
*/ |
|
static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu); |
|
static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock); |
|
|
|
static inline struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu) |
|
{ |
|
return &(to_vmx(vcpu)->pi_desc); |
|
} |
|
|
|
void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) |
|
{ |
|
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); |
|
struct pi_desc old, new; |
|
unsigned int dest; |
|
|
|
/* |
|
* In case of hot-plug or hot-unplug, we may have to undo |
|
* vmx_vcpu_pi_put even if there is no assigned device. And we |
|
* always keep PI.NDST up to date for simplicity: it makes the |
|
* code easier, and CPU migration is not a fast path. |
|
*/ |
|
if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu) |
|
return; |
|
|
|
/* |
|
* If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change |
|
* PI.NDST: pi_post_block is the one expected to change PID.NDST and the |
|
* wakeup handler expects the vCPU to be on the blocked_vcpu_list that |
|
* matches PI.NDST. Otherwise, a vcpu may not be able to be woken up |
|
* correctly. |
|
*/ |
|
if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR || vcpu->cpu == cpu) { |
|
pi_clear_sn(pi_desc); |
|
goto after_clear_sn; |
|
} |
|
|
|
/* The full case. */ |
|
do { |
|
old.control = new.control = pi_desc->control; |
|
|
|
dest = cpu_physical_id(cpu); |
|
|
|
if (x2apic_mode) |
|
new.ndst = dest; |
|
else |
|
new.ndst = (dest << 8) & 0xFF00; |
|
|
|
new.sn = 0; |
|
} while (cmpxchg64(&pi_desc->control, old.control, |
|
new.control) != old.control); |
|
|
|
after_clear_sn: |
|
|
|
/* |
|
* Clear SN before reading the bitmap. The VT-d firmware |
|
* writes the bitmap and reads SN atomically (5.2.3 in the |
|
* spec), so it doesn't really have a memory barrier that |
|
* pairs with this, but we cannot do that and we need one. |
|
*/ |
|
smp_mb__after_atomic(); |
|
|
|
if (!pi_is_pir_empty(pi_desc)) |
|
pi_set_on(pi_desc); |
|
} |
|
|
|
void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) |
|
{ |
|
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); |
|
|
|
if (!kvm_arch_has_assigned_device(vcpu->kvm) || |
|
!irq_remapping_cap(IRQ_POSTING_CAP) || |
|
!kvm_vcpu_apicv_active(vcpu)) |
|
return; |
|
|
|
/* Set SN when the vCPU is preempted */ |
|
if (vcpu->preempted) |
|
pi_set_sn(pi_desc); |
|
} |
|
|
|
static void __pi_post_block(struct kvm_vcpu *vcpu) |
|
{ |
|
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); |
|
struct pi_desc old, new; |
|
unsigned int dest; |
|
|
|
do { |
|
old.control = new.control = pi_desc->control; |
|
WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR, |
|
"Wakeup handler not enabled while the VCPU is blocked\n"); |
|
|
|
dest = cpu_physical_id(vcpu->cpu); |
|
|
|
if (x2apic_mode) |
|
new.ndst = dest; |
|
else |
|
new.ndst = (dest << 8) & 0xFF00; |
|
|
|
/* set 'NV' to 'notification vector' */ |
|
new.nv = POSTED_INTR_VECTOR; |
|
} while (cmpxchg64(&pi_desc->control, old.control, |
|
new.control) != old.control); |
|
|
|
if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) { |
|
spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); |
|
list_del(&vcpu->blocked_vcpu_list); |
|
spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); |
|
vcpu->pre_pcpu = -1; |
|
} |
|
} |
|
|
|
/* |
|
* This routine does the following things for vCPU which is going |
|
* to be blocked if VT-d PI is enabled. |
|
* - Store the vCPU to the wakeup list, so when interrupts happen |
|
* we can find the right vCPU to wake up. |
|
* - Change the Posted-interrupt descriptor as below: |
|
* 'NDST' <-- vcpu->pre_pcpu |
|
* 'NV' <-- POSTED_INTR_WAKEUP_VECTOR |
|
* - If 'ON' is set during this process, which means at least one |
|
* interrupt is posted for this vCPU, we cannot block it, in |
|
* this case, return 1, otherwise, return 0. |
|
* |
|
*/ |
|
int pi_pre_block(struct kvm_vcpu *vcpu) |
|
{ |
|
unsigned int dest; |
|
struct pi_desc old, new; |
|
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); |
|
|
|
if (!kvm_arch_has_assigned_device(vcpu->kvm) || |
|
!irq_remapping_cap(IRQ_POSTING_CAP) || |
|
!kvm_vcpu_apicv_active(vcpu)) |
|
return 0; |
|
|
|
WARN_ON(irqs_disabled()); |
|
local_irq_disable(); |
|
if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) { |
|
vcpu->pre_pcpu = vcpu->cpu; |
|
spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); |
|
list_add_tail(&vcpu->blocked_vcpu_list, |
|
&per_cpu(blocked_vcpu_on_cpu, |
|
vcpu->pre_pcpu)); |
|
spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu)); |
|
} |
|
|
|
do { |
|
old.control = new.control = pi_desc->control; |
|
|
|
WARN((pi_desc->sn == 1), |
|
"Warning: SN field of posted-interrupts " |
|
"is set before blocking\n"); |
|
|
|
/* |
|
* Since vCPU can be preempted during this process, |
|
* vcpu->cpu could be different with pre_pcpu, we |
|
* need to set pre_pcpu as the destination of wakeup |
|
* notification event, then we can find the right vCPU |
|
* to wakeup in wakeup handler if interrupts happen |
|
* when the vCPU is in blocked state. |
|
*/ |
|
dest = cpu_physical_id(vcpu->pre_pcpu); |
|
|
|
if (x2apic_mode) |
|
new.ndst = dest; |
|
else |
|
new.ndst = (dest << 8) & 0xFF00; |
|
|
|
/* set 'NV' to 'wakeup vector' */ |
|
new.nv = POSTED_INTR_WAKEUP_VECTOR; |
|
} while (cmpxchg64(&pi_desc->control, old.control, |
|
new.control) != old.control); |
|
|
|
/* We should not block the vCPU if an interrupt is posted for it. */ |
|
if (pi_test_on(pi_desc) == 1) |
|
__pi_post_block(vcpu); |
|
|
|
local_irq_enable(); |
|
return (vcpu->pre_pcpu == -1); |
|
} |
|
|
|
void pi_post_block(struct kvm_vcpu *vcpu) |
|
{ |
|
if (vcpu->pre_pcpu == -1) |
|
return; |
|
|
|
WARN_ON(irqs_disabled()); |
|
local_irq_disable(); |
|
__pi_post_block(vcpu); |
|
local_irq_enable(); |
|
} |
|
|
|
/* |
|
* Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. |
|
*/ |
|
void pi_wakeup_handler(void) |
|
{ |
|
struct kvm_vcpu *vcpu; |
|
int cpu = smp_processor_id(); |
|
|
|
spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); |
|
list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu), |
|
blocked_vcpu_list) { |
|
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); |
|
|
|
if (pi_test_on(pi_desc) == 1) |
|
kvm_vcpu_kick(vcpu); |
|
} |
|
spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); |
|
} |
|
|
|
void __init pi_init_cpu(int cpu) |
|
{ |
|
INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu)); |
|
spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu)); |
|
} |
|
|
|
bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) |
|
{ |
|
struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); |
|
|
|
return pi_test_on(pi_desc) || |
|
(pi_test_sn(pi_desc) && !pi_is_pir_empty(pi_desc)); |
|
} |
|
|
|
|
|
/* |
|
* Bail out of the block loop if the VM has an assigned |
|
* device, but the blocking vCPU didn't reconfigure the |
|
* PI.NV to the wakeup vector, i.e. the assigned device |
|
* came along after the initial check in pi_pre_block(). |
|
*/ |
|
void vmx_pi_start_assignment(struct kvm *kvm) |
|
{ |
|
if (!irq_remapping_cap(IRQ_POSTING_CAP)) |
|
return; |
|
|
|
kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK); |
|
} |
|
|
|
/* |
|
* pi_update_irte - set IRTE for Posted-Interrupts |
|
* |
|
* @kvm: kvm |
|
* @host_irq: host irq of the interrupt |
|
* @guest_irq: gsi of the interrupt |
|
* @set: set or unset PI |
|
* returns 0 on success, < 0 on failure |
|
*/ |
|
int pi_update_irte(struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, |
|
bool set) |
|
{ |
|
struct kvm_kernel_irq_routing_entry *e; |
|
struct kvm_irq_routing_table *irq_rt; |
|
struct kvm_lapic_irq irq; |
|
struct kvm_vcpu *vcpu; |
|
struct vcpu_data vcpu_info; |
|
int idx, ret = 0; |
|
|
|
if (!kvm_arch_has_assigned_device(kvm) || |
|
!irq_remapping_cap(IRQ_POSTING_CAP) || |
|
!kvm_vcpu_apicv_active(kvm->vcpus[0])) |
|
return 0; |
|
|
|
idx = srcu_read_lock(&kvm->irq_srcu); |
|
irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); |
|
if (guest_irq >= irq_rt->nr_rt_entries || |
|
hlist_empty(&irq_rt->map[guest_irq])) { |
|
pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", |
|
guest_irq, irq_rt->nr_rt_entries); |
|
goto out; |
|
} |
|
|
|
hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { |
|
if (e->type != KVM_IRQ_ROUTING_MSI) |
|
continue; |
|
/* |
|
* VT-d PI cannot support posting multicast/broadcast |
|
* interrupts to a vCPU, we still use interrupt remapping |
|
* for these kind of interrupts. |
|
* |
|
* For lowest-priority interrupts, we only support |
|
* those with single CPU as the destination, e.g. user |
|
* configures the interrupts via /proc/irq or uses |
|
* irqbalance to make the interrupts single-CPU. |
|
* |
|
* We will support full lowest-priority interrupt later. |
|
* |
|
* In addition, we can only inject generic interrupts using |
|
* the PI mechanism, refuse to route others through it. |
|
*/ |
|
|
|
kvm_set_msi_irq(kvm, e, &irq); |
|
if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || |
|
!kvm_irq_is_postable(&irq)) { |
|
/* |
|
* Make sure the IRTE is in remapped mode if |
|
* we don't handle it in posted mode. |
|
*/ |
|
ret = irq_set_vcpu_affinity(host_irq, NULL); |
|
if (ret < 0) { |
|
printk(KERN_INFO |
|
"failed to back to remapped mode, irq: %u\n", |
|
host_irq); |
|
goto out; |
|
} |
|
|
|
continue; |
|
} |
|
|
|
vcpu_info.pi_desc_addr = __pa(&to_vmx(vcpu)->pi_desc); |
|
vcpu_info.vector = irq.vector; |
|
|
|
trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi, |
|
vcpu_info.vector, vcpu_info.pi_desc_addr, set); |
|
|
|
if (set) |
|
ret = irq_set_vcpu_affinity(host_irq, &vcpu_info); |
|
else |
|
ret = irq_set_vcpu_affinity(host_irq, NULL); |
|
|
|
if (ret < 0) { |
|
printk(KERN_INFO "%s: failed to update PI IRTE\n", |
|
__func__); |
|
goto out; |
|
} |
|
} |
|
|
|
ret = 0; |
|
out: |
|
srcu_read_unlock(&kvm->irq_srcu, idx); |
|
return ret; |
|
}
|
|
|