mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
416 lines
10 KiB
416 lines
10 KiB
/* SPDX-License-Identifier: GPL-2.0-only */ |
|
#include <asm/asm-offsets.h> |
|
#include <asm/cache.h> |
|
#include <asm/code-patching-asm.h> |
|
#include <asm/exception-64s.h> |
|
#include <asm/export.h> |
|
#include <asm/kvm_asm.h> |
|
#include <asm/kvm_book3s_asm.h> |
|
#include <asm/mmu.h> |
|
#include <asm/ppc_asm.h> |
|
#include <asm/ptrace.h> |
|
#include <asm/reg.h> |
|
#include <asm/ultravisor-api.h> |
|
|
|
/* |
|
* These are branched to from interrupt handlers in exception-64s.S which set |
|
* IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero. |
|
*/ |
|
|
|
/* |
|
* This is a hcall, so register convention is as |
|
* Documentation/powerpc/papr_hcalls.rst. |
|
* |
|
* This may also be a syscall from PR-KVM userspace that is to be |
|
* reflected to the PR guest kernel, so registers may be set up for |
|
* a system call rather than hcall. We don't currently clobber |
|
* anything here, but the 0xc00 handler has already clobbered CTR |
|
* and CR0, so PR-KVM can not support a guest kernel that preserves |
|
* those registers across its system calls. |
|
* |
|
* The state of registers is as kvmppc_interrupt, except CFAR is not |
|
* saved, R13 is not in SCRATCH0, and R10 does not contain the trap. |
|
*/ |
|
.global kvmppc_hcall |
|
.balign IFETCH_ALIGN_BYTES |
|
kvmppc_hcall: |
|
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
|
lbz r10,HSTATE_IN_GUEST(r13) |
|
cmpwi r10,KVM_GUEST_MODE_HV_P9 |
|
beq kvmppc_p9_exit_hcall |
|
#endif |
|
ld r10,PACA_EXGEN+EX_R13(r13) |
|
SET_SCRATCH0(r10) |
|
li r10,0xc00 |
|
/* Now we look like kvmppc_interrupt */ |
|
li r11,PACA_EXGEN |
|
b .Lgot_save_area |
|
|
|
/* |
|
* KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that |
|
* call convention: |
|
* |
|
* guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area |
|
* guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts |
|
* guest R13 also saved in SCRATCH0 |
|
* R13 = PACA |
|
* R11 = (H)SRR0 |
|
* R12 = (H)SRR1 |
|
* R9 = guest CR |
|
* PPR is set to medium |
|
* |
|
* With the addition for KVM: |
|
* R10 = trap vector |
|
*/ |
|
.global kvmppc_interrupt |
|
.balign IFETCH_ALIGN_BYTES |
|
kvmppc_interrupt: |
|
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
|
std r10,HSTATE_SCRATCH0(r13) |
|
lbz r10,HSTATE_IN_GUEST(r13) |
|
cmpwi r10,KVM_GUEST_MODE_HV_P9 |
|
beq kvmppc_p9_exit_interrupt |
|
ld r10,HSTATE_SCRATCH0(r13) |
|
#endif |
|
li r11,PACA_EXGEN |
|
cmpdi r10,0x200 |
|
bgt+ .Lgot_save_area |
|
li r11,PACA_EXMC |
|
beq .Lgot_save_area |
|
li r11,PACA_EXNMI |
|
.Lgot_save_area: |
|
add r11,r11,r13 |
|
BEGIN_FTR_SECTION |
|
ld r12,EX_CFAR(r11) |
|
std r12,HSTATE_CFAR(r13) |
|
END_FTR_SECTION_IFSET(CPU_FTR_CFAR) |
|
ld r12,EX_CTR(r11) |
|
mtctr r12 |
|
BEGIN_FTR_SECTION |
|
ld r12,EX_PPR(r11) |
|
std r12,HSTATE_PPR(r13) |
|
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
|
ld r12,EX_R12(r11) |
|
std r12,HSTATE_SCRATCH0(r13) |
|
sldi r12,r9,32 |
|
or r12,r12,r10 |
|
ld r9,EX_R9(r11) |
|
ld r10,EX_R10(r11) |
|
ld r11,EX_R11(r11) |
|
|
|
/* |
|
* Hcalls and other interrupts come here after normalising register |
|
* contents and save locations: |
|
* |
|
* R12 = (guest CR << 32) | interrupt vector |
|
* R13 = PACA |
|
* guest R12 saved in shadow HSTATE_SCRATCH0 |
|
* guest R13 saved in SPRN_SCRATCH0 |
|
*/ |
|
std r9,HSTATE_SCRATCH2(r13) |
|
lbz r9,HSTATE_IN_GUEST(r13) |
|
cmpwi r9,KVM_GUEST_MODE_SKIP |
|
beq- .Lmaybe_skip |
|
.Lno_skip: |
|
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
|
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE |
|
cmpwi r9,KVM_GUEST_MODE_GUEST |
|
beq kvmppc_interrupt_pr |
|
#endif |
|
b kvmppc_interrupt_hv |
|
#else |
|
b kvmppc_interrupt_pr |
|
#endif |
|
|
|
/* |
|
* "Skip" interrupts are part of a trick KVM uses a with hash guests to load |
|
* the faulting instruction in guest memory from the the hypervisor without |
|
* walking page tables. |
|
* |
|
* When the guest takes a fault that requires the hypervisor to load the |
|
* instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1 |
|
* and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets |
|
* MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions |
|
* but loads and stores will access the guest context. This is used to load |
|
* the faulting instruction using the faulting guest effective address. |
|
* |
|
* However the guest context may not be able to translate, or it may cause a |
|
* machine check or other issue, which results in a fault in the host |
|
* (even with KVM-HV). |
|
* |
|
* These faults come here because KVM_GUEST_MODE_SKIP was set, so if they |
|
* are (or are likely) caused by that load, the instruction is skipped by |
|
* just returning with the PC advanced +4, where it is noticed the load did |
|
* not execute and it goes to the slow path which walks the page tables to |
|
* read guest memory. |
|
*/ |
|
.Lmaybe_skip: |
|
cmpwi r12,BOOK3S_INTERRUPT_MACHINE_CHECK |
|
beq 1f |
|
cmpwi r12,BOOK3S_INTERRUPT_DATA_STORAGE |
|
beq 1f |
|
cmpwi r12,BOOK3S_INTERRUPT_DATA_SEGMENT |
|
beq 1f |
|
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
|
/* HSRR interrupts get 2 added to interrupt number */ |
|
cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2 |
|
beq 2f |
|
#endif |
|
b .Lno_skip |
|
1: mfspr r9,SPRN_SRR0 |
|
addi r9,r9,4 |
|
mtspr SPRN_SRR0,r9 |
|
ld r12,HSTATE_SCRATCH0(r13) |
|
ld r9,HSTATE_SCRATCH2(r13) |
|
GET_SCRATCH0(r13) |
|
RFI_TO_KERNEL |
|
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
|
2: mfspr r9,SPRN_HSRR0 |
|
addi r9,r9,4 |
|
mtspr SPRN_HSRR0,r9 |
|
ld r12,HSTATE_SCRATCH0(r13) |
|
ld r9,HSTATE_SCRATCH2(r13) |
|
GET_SCRATCH0(r13) |
|
HRFI_TO_KERNEL |
|
#endif |
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
|
|
|
/* Stack frame offsets for kvmppc_p9_enter_guest */ |
|
#define SFS (144 + STACK_FRAME_MIN_SIZE) |
|
#define STACK_SLOT_NVGPRS (SFS - 144) /* 18 gprs */ |
|
|
|
/* |
|
* void kvmppc_p9_enter_guest(struct vcpu *vcpu); |
|
* |
|
* Enter the guest on a ISAv3.0 or later system. |
|
*/ |
|
.balign IFETCH_ALIGN_BYTES |
|
_GLOBAL(kvmppc_p9_enter_guest) |
|
EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest) |
|
mflr r0 |
|
std r0,PPC_LR_STKOFF(r1) |
|
stdu r1,-SFS(r1) |
|
|
|
std r1,HSTATE_HOST_R1(r13) |
|
|
|
mfcr r4 |
|
stw r4,SFS+8(r1) |
|
|
|
reg = 14 |
|
.rept 18 |
|
std reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) |
|
reg = reg + 1 |
|
.endr |
|
|
|
ld r4,VCPU_LR(r3) |
|
mtlr r4 |
|
ld r4,VCPU_CTR(r3) |
|
mtctr r4 |
|
ld r4,VCPU_XER(r3) |
|
mtspr SPRN_XER,r4 |
|
|
|
ld r1,VCPU_CR(r3) |
|
|
|
BEGIN_FTR_SECTION |
|
ld r4,VCPU_CFAR(r3) |
|
mtspr SPRN_CFAR,r4 |
|
END_FTR_SECTION_IFSET(CPU_FTR_CFAR) |
|
BEGIN_FTR_SECTION |
|
ld r4,VCPU_PPR(r3) |
|
mtspr SPRN_PPR,r4 |
|
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) |
|
|
|
reg = 4 |
|
.rept 28 |
|
ld reg,__VCPU_GPR(reg)(r3) |
|
reg = reg + 1 |
|
.endr |
|
|
|
ld r4,VCPU_KVM(r3) |
|
lbz r4,KVM_SECURE_GUEST(r4) |
|
cmpdi r4,0 |
|
ld r4,VCPU_GPR(R4)(r3) |
|
bne .Lret_to_ultra |
|
|
|
mtcr r1 |
|
|
|
ld r0,VCPU_GPR(R0)(r3) |
|
ld r1,VCPU_GPR(R1)(r3) |
|
ld r2,VCPU_GPR(R2)(r3) |
|
ld r3,VCPU_GPR(R3)(r3) |
|
|
|
HRFI_TO_GUEST |
|
b . |
|
|
|
/* |
|
* Use UV_RETURN ultracall to return control back to the Ultravisor |
|
* after processing an hypercall or interrupt that was forwarded |
|
* (a.k.a. reflected) to the Hypervisor. |
|
* |
|
* All registers have already been reloaded except the ucall requires: |
|
* R0 = hcall result |
|
* R2 = SRR1, so UV can detect a synthesized interrupt (if any) |
|
* R3 = UV_RETURN |
|
*/ |
|
.Lret_to_ultra: |
|
mtcr r1 |
|
ld r1,VCPU_GPR(R1)(r3) |
|
|
|
ld r0,VCPU_GPR(R3)(r3) |
|
mfspr r2,SPRN_SRR1 |
|
LOAD_REG_IMMEDIATE(r3, UV_RETURN) |
|
sc 2 |
|
|
|
/* |
|
* kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from |
|
* above if the interrupt was taken for a guest that was entered via |
|
* kvmppc_p9_enter_guest(). |
|
* |
|
* The exit code recovers the host stack and vcpu pointer, saves all guest GPRs |
|
* and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re- |
|
* establishes the host stack and registers to return from the |
|
* kvmppc_p9_enter_guest() function, which saves CTR and other guest registers |
|
* (SPRs and FP, VEC, etc). |
|
*/ |
|
.balign IFETCH_ALIGN_BYTES |
|
kvmppc_p9_exit_hcall: |
|
mfspr r11,SPRN_SRR0 |
|
mfspr r12,SPRN_SRR1 |
|
li r10,0xc00 |
|
std r10,HSTATE_SCRATCH0(r13) |
|
|
|
.balign IFETCH_ALIGN_BYTES |
|
kvmppc_p9_exit_interrupt: |
|
/* |
|
* If set to KVM_GUEST_MODE_HV_P9 but we're still in the |
|
* hypervisor, that means we can't return from the entry stack. |
|
*/ |
|
rldicl. r10,r12,64-MSR_HV_LG,63 |
|
bne- kvmppc_p9_bad_interrupt |
|
|
|
std r1,HSTATE_SCRATCH1(r13) |
|
std r3,HSTATE_SCRATCH2(r13) |
|
ld r1,HSTATE_HOST_R1(r13) |
|
ld r3,HSTATE_KVM_VCPU(r13) |
|
|
|
std r9,VCPU_CR(r3) |
|
|
|
1: |
|
std r11,VCPU_PC(r3) |
|
std r12,VCPU_MSR(r3) |
|
|
|
reg = 14 |
|
.rept 18 |
|
std reg,__VCPU_GPR(reg)(r3) |
|
reg = reg + 1 |
|
.endr |
|
|
|
/* r1, r3, r9-r13 are saved to vcpu by C code */ |
|
std r0,VCPU_GPR(R0)(r3) |
|
std r2,VCPU_GPR(R2)(r3) |
|
reg = 4 |
|
.rept 5 |
|
std reg,__VCPU_GPR(reg)(r3) |
|
reg = reg + 1 |
|
.endr |
|
|
|
ld r2,PACATOC(r13) |
|
|
|
mflr r4 |
|
std r4,VCPU_LR(r3) |
|
mfspr r4,SPRN_XER |
|
std r4,VCPU_XER(r3) |
|
|
|
reg = 14 |
|
.rept 18 |
|
ld reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) |
|
reg = reg + 1 |
|
.endr |
|
|
|
lwz r4,SFS+8(r1) |
|
mtcr r4 |
|
|
|
/* |
|
* Flush the link stack here, before executing the first blr on the |
|
* way out of the guest. |
|
* |
|
* The link stack won't match coming out of the guest anyway so the |
|
* only cost is the flush itself. The call clobbers r0. |
|
*/ |
|
1: nop |
|
patch_site 1b patch__call_kvm_flush_link_stack_p9 |
|
|
|
addi r1,r1,SFS |
|
ld r0,PPC_LR_STKOFF(r1) |
|
mtlr r0 |
|
blr |
|
|
|
/* |
|
* Took an interrupt somewhere right before HRFID to guest, so registers are |
|
* in a bad way. Return things hopefully enough to run host virtual code and |
|
* run the Linux interrupt handler (SRESET or MCE) to print something useful. |
|
* |
|
* We could be really clever and save all host registers in known locations |
|
* before setting HSTATE_IN_GUEST, then restoring them all here, and setting |
|
* return address to a fixup that sets them up again. But that's a lot of |
|
* effort for a small bit of code. Lots of other things to do first. |
|
*/ |
|
kvmppc_p9_bad_interrupt: |
|
BEGIN_MMU_FTR_SECTION |
|
/* |
|
* Hash host doesn't try to recover MMU (requires host SLB reload) |
|
*/ |
|
b . |
|
END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) |
|
/* |
|
* Clean up guest registers to give host a chance to run. |
|
*/ |
|
li r10,0 |
|
mtspr SPRN_AMR,r10 |
|
mtspr SPRN_IAMR,r10 |
|
mtspr SPRN_CIABR,r10 |
|
mtspr SPRN_DAWRX0,r10 |
|
BEGIN_FTR_SECTION |
|
mtspr SPRN_DAWRX1,r10 |
|
END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) |
|
mtspr SPRN_PID,r10 |
|
|
|
/* |
|
* Switch to host MMU mode |
|
*/ |
|
ld r10, HSTATE_KVM_VCPU(r13) |
|
ld r10, VCPU_KVM(r10) |
|
lwz r10, KVM_HOST_LPID(r10) |
|
mtspr SPRN_LPID,r10 |
|
|
|
ld r10, HSTATE_KVM_VCPU(r13) |
|
ld r10, VCPU_KVM(r10) |
|
ld r10, KVM_HOST_LPCR(r10) |
|
mtspr SPRN_LPCR,r10 |
|
|
|
/* |
|
* Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear |
|
* MSR_RI in r12 ([H]SRR1) so the handler won't try to return. |
|
*/ |
|
li r10,KVM_GUEST_MODE_NONE |
|
stb r10,HSTATE_IN_GUEST(r13) |
|
li r10,MSR_RI |
|
andc r12,r12,r10 |
|
|
|
/* |
|
* Go back to interrupt handler. MCE and SRESET have their specific |
|
* PACA save area so they should be used directly. They set up their |
|
* own stack. The other handlers all use EXGEN. They will use the |
|
* guest r1 if it looks like a kernel stack, so just load the |
|
* emergency stack and go to program check for all other interrupts. |
|
*/ |
|
ld r10,HSTATE_SCRATCH0(r13) |
|
cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK |
|
beq machine_check_common |
|
|
|
cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET |
|
beq system_reset_common |
|
|
|
b . |
|
#endif
|
|
|