mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
417 lines
11 KiB
417 lines
11 KiB
/* SPDX-License-Identifier: GPL-2.0 */ |
|
#include <linux/init.h> |
|
#include <linux/linkage.h> |
|
|
|
#include <asm/assembler.h> |
|
#include <asm/asm-offsets.h> |
|
#include <asm/errno.h> |
|
#include <asm/thread_info.h> |
|
#include <asm/uaccess-asm.h> |
|
#include <asm/v7m.h> |
|
|
|
@ Bad Abort numbers |
|
@ ----------------- |
|
@ |
|
#define BAD_PREFETCH 0 |
|
#define BAD_DATA 1 |
|
#define BAD_ADDREXCPTN 2 |
|
#define BAD_IRQ 3 |
|
#define BAD_UNDEFINSTR 4 |
|
|
|
@ |
|
@ Most of the stack format comes from struct pt_regs, but with |
|
@ the addition of 8 bytes for storing syscall args 5 and 6. |
|
@ This _must_ remain a multiple of 8 for EABI. |
|
@ |
|
#define S_OFF 8 |
|
|
|
/* |
|
* The SWI code relies on the fact that R0 is at the bottom of the stack |
|
* (due to slow/fast restore user regs). |
|
*/ |
|
#if S_R0 != 0 |
|
#error "Please fix" |
|
#endif |
|
|
|
.macro zero_fp |
|
#ifdef CONFIG_FRAME_POINTER |
|
mov fp, #0 |
|
#endif |
|
.endm |
|
|
|
#ifdef CONFIG_ALIGNMENT_TRAP |
|
#define ATRAP(x...) x |
|
#else |
|
#define ATRAP(x...) |
|
#endif |
|
|
|
.macro alignment_trap, rtmp1, rtmp2, label |
|
#ifdef CONFIG_ALIGNMENT_TRAP |
|
mrc p15, 0, \rtmp2, c1, c0, 0 |
|
ldr \rtmp1, \label |
|
ldr \rtmp1, [\rtmp1] |
|
teq \rtmp1, \rtmp2 |
|
mcrne p15, 0, \rtmp1, c1, c0, 0 |
|
#endif |
|
.endm |
|
|
|
#ifdef CONFIG_CPU_V7M |
|
/* |
|
* ARMv7-M exception entry/exit macros. |
|
* |
|
* xPSR, ReturnAddress(), LR (R14), R12, R3, R2, R1, and R0 are |
|
* automatically saved on the current stack (32 words) before |
|
* switching to the exception stack (SP_main). |
|
* |
|
* If exception is taken while in user mode, SP_main is |
|
* empty. Otherwise, SP_main is aligned to 64 bit automatically |
|
* (CCR.STKALIGN set). |
|
* |
|
* Linux assumes that the interrupts are disabled when entering an |
|
* exception handler and it may BUG if this is not the case. Interrupts |
|
* are disabled during entry and reenabled in the exit macro. |
|
* |
|
* v7m_exception_slow_exit is used when returning from SVC or PendSV. |
|
* When returning to kernel mode, we don't return from exception. |
|
*/ |
|
.macro v7m_exception_entry |
|
@ determine the location of the registers saved by the core during |
|
@ exception entry. Depending on the mode the cpu was in when the |
|
@ exception happend that is either on the main or the process stack. |
|
@ Bit 2 of EXC_RETURN stored in the lr register specifies which stack |
|
@ was used. |
|
tst lr, #EXC_RET_STACK_MASK |
|
mrsne r12, psp |
|
moveq r12, sp |
|
|
|
@ we cannot rely on r0-r3 and r12 matching the value saved in the |
|
@ exception frame because of tail-chaining. So these have to be |
|
@ reloaded. |
|
ldmia r12!, {r0-r3} |
|
|
|
@ Linux expects to have irqs off. Do it here before taking stack space |
|
cpsid i |
|
|
|
sub sp, #PT_REGS_SIZE-S_IP |
|
stmdb sp!, {r0-r11} |
|
|
|
@ load saved r12, lr, return address and xPSR. |
|
@ r0-r7 are used for signals and never touched from now on. Clobbering |
|
@ r8-r12 is OK. |
|
mov r9, r12 |
|
ldmia r9!, {r8, r10-r12} |
|
|
|
@ calculate the original stack pointer value. |
|
@ r9 currently points to the memory location just above the auto saved |
|
@ xPSR. |
|
@ The cpu might automatically 8-byte align the stack. Bit 9 |
|
@ of the saved xPSR specifies if stack aligning took place. In this case |
|
@ another 32-bit value is included in the stack. |
|
|
|
tst r12, V7M_xPSR_FRAMEPTRALIGN |
|
addne r9, r9, #4 |
|
|
|
@ store saved r12 using str to have a register to hold the base for stm |
|
str r8, [sp, #S_IP] |
|
add r8, sp, #S_SP |
|
@ store r13-r15, xPSR |
|
stmia r8!, {r9-r12} |
|
@ store old_r0 |
|
str r0, [r8] |
|
.endm |
|
|
|
/* |
|
* PENDSV and SVCALL are configured to have the same exception |
|
* priorities. As a kernel thread runs at SVCALL execution priority it |
|
* can never be preempted and so we will never have to return to a |
|
* kernel thread here. |
|
*/ |
|
.macro v7m_exception_slow_exit ret_r0 |
|
cpsid i |
|
ldr lr, =exc_ret |
|
ldr lr, [lr] |
|
|
|
@ read original r12, sp, lr, pc and xPSR |
|
add r12, sp, #S_IP |
|
ldmia r12, {r1-r5} |
|
|
|
@ an exception frame is always 8-byte aligned. To tell the hardware if |
|
@ the sp to be restored is aligned or not set bit 9 of the saved xPSR |
|
@ accordingly. |
|
tst r2, #4 |
|
subne r2, r2, #4 |
|
orrne r5, V7M_xPSR_FRAMEPTRALIGN |
|
biceq r5, V7M_xPSR_FRAMEPTRALIGN |
|
|
|
@ ensure bit 0 is cleared in the PC, otherwise behaviour is |
|
@ unpredictable |
|
bic r4, #1 |
|
|
|
@ write basic exception frame |
|
stmdb r2!, {r1, r3-r5} |
|
ldmia sp, {r1, r3-r5} |
|
.if \ret_r0 |
|
stmdb r2!, {r0, r3-r5} |
|
.else |
|
stmdb r2!, {r1, r3-r5} |
|
.endif |
|
|
|
@ restore process sp |
|
msr psp, r2 |
|
|
|
@ restore original r4-r11 |
|
ldmia sp!, {r0-r11} |
|
|
|
@ restore main sp |
|
add sp, sp, #PT_REGS_SIZE-S_IP |
|
|
|
cpsie i |
|
bx lr |
|
.endm |
|
#endif /* CONFIG_CPU_V7M */ |
|
|
|
@ |
|
@ Store/load the USER SP and LR registers by switching to the SYS |
|
@ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not |
|
@ available. Should only be called from SVC mode |
|
@ |
|
.macro store_user_sp_lr, rd, rtemp, offset = 0 |
|
mrs \rtemp, cpsr |
|
eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
|
msr cpsr_c, \rtemp @ switch to the SYS mode |
|
|
|
str sp, [\rd, #\offset] @ save sp_usr |
|
str lr, [\rd, #\offset + 4] @ save lr_usr |
|
|
|
eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
|
msr cpsr_c, \rtemp @ switch back to the SVC mode |
|
.endm |
|
|
|
.macro load_user_sp_lr, rd, rtemp, offset = 0 |
|
mrs \rtemp, cpsr |
|
eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
|
msr cpsr_c, \rtemp @ switch to the SYS mode |
|
|
|
ldr sp, [\rd, #\offset] @ load sp_usr |
|
ldr lr, [\rd, #\offset + 4] @ load lr_usr |
|
|
|
eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) |
|
msr cpsr_c, \rtemp @ switch back to the SVC mode |
|
.endm |
|
|
|
|
|
.macro svc_exit, rpsr, irq = 0 |
|
.if \irq != 0 |
|
@ IRQs already off |
|
#ifdef CONFIG_TRACE_IRQFLAGS |
|
@ The parent context IRQs must have been enabled to get here in |
|
@ the first place, so there's no point checking the PSR I bit. |
|
bl trace_hardirqs_on |
|
#endif |
|
.else |
|
@ IRQs off again before pulling preserved data off the stack |
|
disable_irq_notrace |
|
#ifdef CONFIG_TRACE_IRQFLAGS |
|
tst \rpsr, #PSR_I_BIT |
|
bleq trace_hardirqs_on |
|
tst \rpsr, #PSR_I_BIT |
|
blne trace_hardirqs_off |
|
#endif |
|
.endif |
|
uaccess_exit tsk, r0, r1 |
|
|
|
#ifndef CONFIG_THUMB2_KERNEL |
|
@ ARM mode SVC restore |
|
msr spsr_cxsf, \rpsr |
|
#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) |
|
@ We must avoid clrex due to Cortex-A15 erratum #830321 |
|
sub r0, sp, #4 @ uninhabited address |
|
strex r1, r2, [r0] @ clear the exclusive monitor |
|
#endif |
|
ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr |
|
#else |
|
@ Thumb mode SVC restore |
|
ldr lr, [sp, #S_SP] @ top of the stack |
|
ldrd r0, r1, [sp, #S_LR] @ calling lr and pc |
|
|
|
@ We must avoid clrex due to Cortex-A15 erratum #830321 |
|
strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor |
|
|
|
stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context |
|
ldmia sp, {r0 - r12} |
|
mov sp, lr |
|
ldr lr, [sp], #4 |
|
rfeia sp! |
|
#endif |
|
.endm |
|
|
|
@ |
|
@ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit |
|
@ |
|
@ This macro acts in a similar manner to svc_exit but switches to FIQ |
|
@ mode to restore the final part of the register state. |
|
@ |
|
@ We cannot use the normal svc_exit procedure because that would |
|
@ clobber spsr_svc (FIQ could be delivered during the first few |
|
@ instructions of vector_swi meaning its contents have not been |
|
@ saved anywhere). |
|
@ |
|
@ Note that, unlike svc_exit, this macro also does not allow a caller |
|
@ supplied rpsr. This is because the FIQ exceptions are not re-entrant |
|
@ and the handlers cannot call into the scheduler (meaning the value |
|
@ on the stack remains correct). |
|
@ |
|
.macro svc_exit_via_fiq |
|
uaccess_exit tsk, r0, r1 |
|
#ifndef CONFIG_THUMB2_KERNEL |
|
@ ARM mode restore |
|
mov r0, sp |
|
ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will |
|
@ clobber state restored below) |
|
msr cpsr_c, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT |
|
add r8, r0, #S_PC |
|
ldr r9, [r0, #S_PSR] |
|
msr spsr_cxsf, r9 |
|
ldr r0, [r0, #S_R0] |
|
ldmia r8, {pc}^ |
|
#else |
|
@ Thumb mode restore |
|
add r0, sp, #S_R2 |
|
ldr lr, [sp, #S_LR] |
|
ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will |
|
@ clobber state restored below) |
|
ldmia r0, {r2 - r12} |
|
mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT |
|
msr cpsr_c, r1 |
|
sub r0, #S_R2 |
|
add r8, r0, #S_PC |
|
ldmia r0, {r0 - r1} |
|
rfeia r8 |
|
#endif |
|
.endm |
|
|
|
|
|
.macro restore_user_regs, fast = 0, offset = 0 |
|
uaccess_enable r1, isb=0 |
|
#ifndef CONFIG_THUMB2_KERNEL |
|
@ ARM mode restore |
|
mov r2, sp |
|
ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr |
|
ldr lr, [r2, #\offset + S_PC]! @ get pc |
|
tst r1, #PSR_I_BIT | 0x0f |
|
bne 1f |
|
msr spsr_cxsf, r1 @ save in spsr_svc |
|
#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) |
|
@ We must avoid clrex due to Cortex-A15 erratum #830321 |
|
strex r1, r2, [r2] @ clear the exclusive monitor |
|
#endif |
|
.if \fast |
|
ldmdb r2, {r1 - lr}^ @ get calling r1 - lr |
|
.else |
|
ldmdb r2, {r0 - lr}^ @ get calling r0 - lr |
|
.endif |
|
mov r0, r0 @ ARMv5T and earlier require a nop |
|
@ after ldm {}^ |
|
add sp, sp, #\offset + PT_REGS_SIZE |
|
movs pc, lr @ return & move spsr_svc into cpsr |
|
1: bug "Returning to usermode but unexpected PSR bits set?", \@ |
|
#elif defined(CONFIG_CPU_V7M) |
|
@ V7M restore. |
|
@ Note that we don't need to do clrex here as clearing the local |
|
@ monitor is part of the exception entry and exit sequence. |
|
.if \offset |
|
add sp, #\offset |
|
.endif |
|
v7m_exception_slow_exit ret_r0 = \fast |
|
#else |
|
@ Thumb mode restore |
|
mov r2, sp |
|
load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr |
|
ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr |
|
ldr lr, [sp, #\offset + S_PC] @ get pc |
|
add sp, sp, #\offset + S_SP |
|
tst r1, #PSR_I_BIT | 0x0f |
|
bne 1f |
|
msr spsr_cxsf, r1 @ save in spsr_svc |
|
|
|
@ We must avoid clrex due to Cortex-A15 erratum #830321 |
|
strex r1, r2, [sp] @ clear the exclusive monitor |
|
|
|
.if \fast |
|
ldmdb sp, {r1 - r12} @ get calling r1 - r12 |
|
.else |
|
ldmdb sp, {r0 - r12} @ get calling r0 - r12 |
|
.endif |
|
add sp, sp, #PT_REGS_SIZE - S_SP |
|
movs pc, lr @ return & move spsr_svc into cpsr |
|
1: bug "Returning to usermode but unexpected PSR bits set?", \@ |
|
#endif /* !CONFIG_THUMB2_KERNEL */ |
|
.endm |
|
|
|
/* |
|
* Context tracking subsystem. Used to instrument transitions |
|
* between user and kernel mode. |
|
*/ |
|
.macro ct_user_exit, save = 1 |
|
#ifdef CONFIG_CONTEXT_TRACKING |
|
.if \save |
|
stmdb sp!, {r0-r3, ip, lr} |
|
bl context_tracking_user_exit |
|
ldmia sp!, {r0-r3, ip, lr} |
|
.else |
|
bl context_tracking_user_exit |
|
.endif |
|
#endif |
|
.endm |
|
|
|
.macro ct_user_enter, save = 1 |
|
#ifdef CONFIG_CONTEXT_TRACKING |
|
.if \save |
|
stmdb sp!, {r0-r3, ip, lr} |
|
bl context_tracking_user_enter |
|
ldmia sp!, {r0-r3, ip, lr} |
|
.else |
|
bl context_tracking_user_enter |
|
.endif |
|
#endif |
|
.endm |
|
|
|
.macro invoke_syscall, table, nr, tmp, ret, reload=0 |
|
#ifdef CONFIG_CPU_SPECTRE |
|
mov \tmp, \nr |
|
cmp \tmp, #NR_syscalls @ check upper syscall limit |
|
movcs \tmp, #0 |
|
csdb |
|
badr lr, \ret @ return address |
|
.if \reload |
|
add r1, sp, #S_R0 + S_OFF @ pointer to regs |
|
ldmiacc r1, {r0 - r6} @ reload r0-r6 |
|
stmiacc sp, {r4, r5} @ update stack arguments |
|
.endif |
|
ldrcc pc, [\table, \tmp, lsl #2] @ call sys_* routine |
|
#else |
|
cmp \nr, #NR_syscalls @ check upper syscall limit |
|
badr lr, \ret @ return address |
|
.if \reload |
|
add r1, sp, #S_R0 + S_OFF @ pointer to regs |
|
ldmiacc r1, {r0 - r6} @ reload r0-r6 |
|
stmiacc sp, {r4, r5} @ update stack arguments |
|
.endif |
|
ldrcc pc, [\table, \nr, lsl #2] @ call sys_* routine |
|
#endif |
|
.endm |
|
|
|
/* |
|
* These are the registers used in the syscall handler, and allow us to |
|
* have in theory up to 7 arguments to a function - r0 to r6. |
|
* |
|
* r7 is reserved for the system call number for thumb mode. |
|
* |
|
* Note that tbl == why is intentional. |
|
* |
|
* We must set at least "tsk" and "why" when calling ret_with_reschedule. |
|
*/ |
|
scno .req r7 @ syscall number |
|
tbl .req r8 @ syscall table pointer |
|
why .req r8 @ Linux syscall (!= 0) |
|
tsk .req r9 @ current thread_info
|
|
|