mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
229 lines
5.1 KiB
229 lines
5.1 KiB
/* SPDX-License-Identifier: GPL-2.0-only */ |
|
/* |
|
* arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM |
|
* |
|
* Created by: Nicolas Pitre, March 2012 |
|
* Copyright: (C) 2012-2013 Linaro Limited |
|
* |
|
* Refer to Documentation/arm/cluster-pm-race-avoidance.rst |
|
* for details of the synchronisation algorithms used here. |
|
*/ |
|
|
|
#include <linux/linkage.h> |
|
#include <asm/mcpm.h> |
|
#include <asm/assembler.h> |
|
|
|
#include "vlock.h" |
|
|
|
.if MCPM_SYNC_CLUSTER_CPUS |
|
.error "cpus must be the first member of struct mcpm_sync_struct" |
|
.endif |
|
|
|
.macro pr_dbg string |
|
#if defined(CONFIG_DEBUG_LL) && defined(DEBUG) |
|
b 1901f |
|
1902: .asciz "CPU" |
|
1903: .asciz " cluster" |
|
1904: .asciz ": \string" |
|
.align |
|
1901: adr r0, 1902b |
|
bl printascii |
|
mov r0, r9 |
|
bl printhex2 |
|
adr r0, 1903b |
|
bl printascii |
|
mov r0, r10 |
|
bl printhex2 |
|
adr r0, 1904b |
|
bl printascii |
|
#endif |
|
.endm |
|
|
|
.arm |
|
.align |
|
|
|
ENTRY(mcpm_entry_point) |
|
|
|
ARM_BE8(setend be) |
|
THUMB( badr r12, 1f ) |
|
THUMB( bx r12 ) |
|
THUMB( .thumb ) |
|
1: |
|
mrc p15, 0, r0, c0, c0, 5 @ MPIDR |
|
ubfx r9, r0, #0, #8 @ r9 = cpu |
|
ubfx r10, r0, #8, #8 @ r10 = cluster |
|
mov r3, #MAX_CPUS_PER_CLUSTER |
|
mla r4, r3, r10, r9 @ r4 = canonical CPU index |
|
cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS) |
|
blo 2f |
|
|
|
/* We didn't expect this CPU. Try to cheaply make it quiet. */ |
|
1: wfi |
|
wfe |
|
b 1b |
|
|
|
2: pr_dbg "kernel mcpm_entry_point\n" |
|
|
|
/* |
|
* MMU is off so we need to get to various variables in a |
|
* position independent way. |
|
*/ |
|
adr r5, 3f |
|
ldmia r5, {r0, r6, r7, r8, r11} |
|
add r0, r5, r0 @ r0 = mcpm_entry_early_pokes |
|
add r6, r5, r6 @ r6 = mcpm_entry_vectors |
|
ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys |
|
add r8, r5, r8 @ r8 = mcpm_sync |
|
add r11, r5, r11 @ r11 = first_man_locks |
|
|
|
@ Perform an early poke, if any |
|
add r0, r0, r4, lsl #3 |
|
ldmia r0, {r0, r1} |
|
teq r0, #0 |
|
strne r1, [r0] |
|
|
|
mov r0, #MCPM_SYNC_CLUSTER_SIZE |
|
mla r8, r0, r10, r8 @ r8 = sync cluster base |
|
|
|
@ Signal that this CPU is coming UP: |
|
mov r0, #CPU_COMING_UP |
|
mov r5, #MCPM_SYNC_CPU_SIZE |
|
mla r5, r9, r5, r8 @ r5 = sync cpu address |
|
strb r0, [r5] |
|
|
|
@ At this point, the cluster cannot unexpectedly enter the GOING_DOWN |
|
@ state, because there is at least one active CPU (this CPU). |
|
|
|
mov r0, #VLOCK_SIZE |
|
mla r11, r0, r10, r11 @ r11 = cluster first man lock |
|
mov r0, r11 |
|
mov r1, r9 @ cpu |
|
bl vlock_trylock @ implies DMB |
|
|
|
cmp r0, #0 @ failed to get the lock? |
|
bne mcpm_setup_wait @ wait for cluster setup if so |
|
|
|
ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
|
cmp r0, #CLUSTER_UP @ cluster already up? |
|
bne mcpm_setup @ if not, set up the cluster |
|
|
|
@ Otherwise, release the first man lock and skip setup: |
|
mov r0, r11 |
|
bl vlock_unlock |
|
b mcpm_setup_complete |
|
|
|
mcpm_setup: |
|
@ Control dependency implies strb not observable before previous ldrb. |
|
|
|
@ Signal that the cluster is being brought up: |
|
mov r0, #INBOUND_COMING_UP |
|
strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] |
|
dmb |
|
|
|
@ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this |
|
@ point onwards will observe INBOUND_COMING_UP and abort. |
|
|
|
@ Wait for any previously-pending cluster teardown operations to abort |
|
@ or complete: |
|
mcpm_teardown_wait: |
|
ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
|
cmp r0, #CLUSTER_GOING_DOWN |
|
bne first_man_setup |
|
wfe |
|
b mcpm_teardown_wait |
|
|
|
first_man_setup: |
|
dmb |
|
|
|
@ If the outbound gave up before teardown started, skip cluster setup: |
|
|
|
cmp r0, #CLUSTER_UP |
|
beq mcpm_setup_leave |
|
|
|
@ power_up_setup is now responsible for setting up the cluster: |
|
|
|
cmp r7, #0 |
|
mov r0, #1 @ second (cluster) affinity level |
|
blxne r7 @ Call power_up_setup if defined |
|
dmb |
|
|
|
mov r0, #CLUSTER_UP |
|
strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
|
dmb |
|
|
|
mcpm_setup_leave: |
|
@ Leave the cluster setup critical section: |
|
|
|
mov r0, #INBOUND_NOT_COMING_UP |
|
strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] |
|
dsb st |
|
sev |
|
|
|
mov r0, r11 |
|
bl vlock_unlock @ implies DMB |
|
b mcpm_setup_complete |
|
|
|
@ In the contended case, non-first men wait here for cluster setup |
|
@ to complete: |
|
mcpm_setup_wait: |
|
ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
|
cmp r0, #CLUSTER_UP |
|
wfene |
|
bne mcpm_setup_wait |
|
dmb |
|
|
|
mcpm_setup_complete: |
|
@ If a platform-specific CPU setup hook is needed, it is |
|
@ called from here. |
|
|
|
cmp r7, #0 |
|
mov r0, #0 @ first (CPU) affinity level |
|
blxne r7 @ Call power_up_setup if defined |
|
dmb |
|
|
|
@ Mark the CPU as up: |
|
|
|
mov r0, #CPU_UP |
|
strb r0, [r5] |
|
|
|
@ Observability order of CPU_UP and opening of the gate does not matter. |
|
|
|
mcpm_entry_gated: |
|
ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector |
|
cmp r5, #0 |
|
wfeeq |
|
beq mcpm_entry_gated |
|
dmb |
|
|
|
pr_dbg "released\n" |
|
bx r5 |
|
|
|
.align 2 |
|
|
|
3: .word mcpm_entry_early_pokes - . |
|
.word mcpm_entry_vectors - 3b |
|
.word mcpm_power_up_setup_phys - 3b |
|
.word mcpm_sync - 3b |
|
.word first_man_locks - 3b |
|
|
|
ENDPROC(mcpm_entry_point) |
|
|
|
.bss |
|
|
|
.align CACHE_WRITEBACK_ORDER |
|
.type first_man_locks, #object |
|
first_man_locks: |
|
.space VLOCK_SIZE * MAX_NR_CLUSTERS |
|
.align CACHE_WRITEBACK_ORDER |
|
|
|
.type mcpm_entry_vectors, #object |
|
ENTRY(mcpm_entry_vectors) |
|
.space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER |
|
|
|
.type mcpm_entry_early_pokes, #object |
|
ENTRY(mcpm_entry_early_pokes) |
|
.space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER |
|
|
|
.type mcpm_power_up_setup_phys, #object |
|
ENTRY(mcpm_power_up_setup_phys) |
|
.space 4 @ set by mcpm_sync_init()
|
|
|