mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
456 lines
13 KiB
456 lines
13 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM |
|
* |
|
* Created by: Nicolas Pitre, March 2012 |
|
* Copyright: (C) 2012-2013 Linaro Limited |
|
*/ |
|
|
|
#include <linux/export.h> |
|
#include <linux/kernel.h> |
|
#include <linux/init.h> |
|
#include <linux/irqflags.h> |
|
#include <linux/cpu_pm.h> |
|
|
|
#include <asm/mcpm.h> |
|
#include <asm/cacheflush.h> |
|
#include <asm/idmap.h> |
|
#include <asm/cputype.h> |
|
#include <asm/suspend.h> |
|
|
|
/* |
|
* The public API for this code is documented in arch/arm/include/asm/mcpm.h. |
|
* For a comprehensive description of the main algorithm used here, please |
|
* see Documentation/arm/cluster-pm-race-avoidance.rst. |
|
*/ |
|
|
|
struct sync_struct mcpm_sync; |
|
|
|
/* |
|
* __mcpm_cpu_going_down: Indicates that the cpu is being torn down. |
|
* This must be called at the point of committing to teardown of a CPU. |
|
* The CPU cache (SCTRL.C bit) is expected to still be active. |
|
*/ |
|
static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster) |
|
{ |
|
mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN; |
|
sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu); |
|
} |
|
|
|
/* |
|
* __mcpm_cpu_down: Indicates that cpu teardown is complete and that the |
|
* cluster can be torn down without disrupting this CPU. |
|
* To avoid deadlocks, this must be called before a CPU is powered down. |
|
* The CPU cache (SCTRL.C bit) is expected to be off. |
|
* However L2 cache might or might not be active. |
|
*/ |
|
static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster) |
|
{ |
|
dmb(); |
|
mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN; |
|
sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu); |
|
sev(); |
|
} |
|
|
|
/* |
|
* __mcpm_outbound_leave_critical: Leave the cluster teardown critical section. |
|
* @state: the final state of the cluster: |
|
* CLUSTER_UP: no destructive teardown was done and the cluster has been |
|
* restored to the previous state (CPU cache still active); or |
|
* CLUSTER_DOWN: the cluster has been torn-down, ready for power-off |
|
* (CPU cache disabled, L2 cache either enabled or disabled). |
|
*/ |
|
static void __mcpm_outbound_leave_critical(unsigned int cluster, int state) |
|
{ |
|
dmb(); |
|
mcpm_sync.clusters[cluster].cluster = state; |
|
sync_cache_w(&mcpm_sync.clusters[cluster].cluster); |
|
sev(); |
|
} |
|
|
|
/* |
|
* __mcpm_outbound_enter_critical: Enter the cluster teardown critical section. |
|
* This function should be called by the last man, after local CPU teardown |
|
* is complete. CPU cache expected to be active. |
|
* |
|
* Returns: |
|
* false: the critical section was not entered because an inbound CPU was |
|
* observed, or the cluster is already being set up; |
|
* true: the critical section was entered: it is now safe to tear down the |
|
* cluster. |
|
*/ |
|
static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster) |
|
{ |
|
unsigned int i; |
|
struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster]; |
|
|
|
/* Warn inbound CPUs that the cluster is being torn down: */ |
|
c->cluster = CLUSTER_GOING_DOWN; |
|
sync_cache_w(&c->cluster); |
|
|
|
/* Back out if the inbound cluster is already in the critical region: */ |
|
sync_cache_r(&c->inbound); |
|
if (c->inbound == INBOUND_COMING_UP) |
|
goto abort; |
|
|
|
/* |
|
* Wait for all CPUs to get out of the GOING_DOWN state, so that local |
|
* teardown is complete on each CPU before tearing down the cluster. |
|
* |
|
* If any CPU has been woken up again from the DOWN state, then we |
|
* shouldn't be taking the cluster down at all: abort in that case. |
|
*/ |
|
sync_cache_r(&c->cpus); |
|
for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) { |
|
int cpustate; |
|
|
|
if (i == cpu) |
|
continue; |
|
|
|
while (1) { |
|
cpustate = c->cpus[i].cpu; |
|
if (cpustate != CPU_GOING_DOWN) |
|
break; |
|
|
|
wfe(); |
|
sync_cache_r(&c->cpus[i].cpu); |
|
} |
|
|
|
switch (cpustate) { |
|
case CPU_DOWN: |
|
continue; |
|
|
|
default: |
|
goto abort; |
|
} |
|
} |
|
|
|
return true; |
|
|
|
abort: |
|
__mcpm_outbound_leave_critical(cluster, CLUSTER_UP); |
|
return false; |
|
} |
|
|
|
static int __mcpm_cluster_state(unsigned int cluster) |
|
{ |
|
sync_cache_r(&mcpm_sync.clusters[cluster].cluster); |
|
return mcpm_sync.clusters[cluster].cluster; |
|
} |
|
|
|
extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER]; |
|
|
|
void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr) |
|
{ |
|
unsigned long val = ptr ? __pa_symbol(ptr) : 0; |
|
mcpm_entry_vectors[cluster][cpu] = val; |
|
sync_cache_w(&mcpm_entry_vectors[cluster][cpu]); |
|
} |
|
|
|
extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2]; |
|
|
|
void mcpm_set_early_poke(unsigned cpu, unsigned cluster, |
|
unsigned long poke_phys_addr, unsigned long poke_val) |
|
{ |
|
unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0]; |
|
poke[0] = poke_phys_addr; |
|
poke[1] = poke_val; |
|
__sync_cache_range_w(poke, 2 * sizeof(*poke)); |
|
} |
|
|
|
static const struct mcpm_platform_ops *platform_ops; |
|
|
|
int __init mcpm_platform_register(const struct mcpm_platform_ops *ops) |
|
{ |
|
if (platform_ops) |
|
return -EBUSY; |
|
platform_ops = ops; |
|
return 0; |
|
} |
|
|
|
bool mcpm_is_available(void) |
|
{ |
|
return (platform_ops) ? true : false; |
|
} |
|
EXPORT_SYMBOL_GPL(mcpm_is_available); |
|
|
|
/* |
|
* We can't use regular spinlocks. In the switcher case, it is possible |
|
* for an outbound CPU to call power_down() after its inbound counterpart |
|
* is already live using the same logical CPU number which trips lockdep |
|
* debugging. |
|
*/ |
|
static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED; |
|
|
|
static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER]; |
|
|
|
static inline bool mcpm_cluster_unused(unsigned int cluster) |
|
{ |
|
int i, cnt; |
|
for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++) |
|
cnt |= mcpm_cpu_use_count[cluster][i]; |
|
return !cnt; |
|
} |
|
|
|
int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster) |
|
{ |
|
bool cpu_is_down, cluster_is_down; |
|
int ret = 0; |
|
|
|
pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); |
|
if (!platform_ops) |
|
return -EUNATCH; /* try not to shadow power_up errors */ |
|
might_sleep(); |
|
|
|
/* |
|
* Since this is called with IRQs enabled, and no arch_spin_lock_irq |
|
* variant exists, we need to disable IRQs manually here. |
|
*/ |
|
local_irq_disable(); |
|
arch_spin_lock(&mcpm_lock); |
|
|
|
cpu_is_down = !mcpm_cpu_use_count[cluster][cpu]; |
|
cluster_is_down = mcpm_cluster_unused(cluster); |
|
|
|
mcpm_cpu_use_count[cluster][cpu]++; |
|
/* |
|
* The only possible values are: |
|
* 0 = CPU down |
|
* 1 = CPU (still) up |
|
* 2 = CPU requested to be up before it had a chance |
|
* to actually make itself down. |
|
* Any other value is a bug. |
|
*/ |
|
BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 && |
|
mcpm_cpu_use_count[cluster][cpu] != 2); |
|
|
|
if (cluster_is_down) |
|
ret = platform_ops->cluster_powerup(cluster); |
|
if (cpu_is_down && !ret) |
|
ret = platform_ops->cpu_powerup(cpu, cluster); |
|
|
|
arch_spin_unlock(&mcpm_lock); |
|
local_irq_enable(); |
|
return ret; |
|
} |
|
|
|
typedef typeof(cpu_reset) phys_reset_t; |
|
|
|
void mcpm_cpu_power_down(void) |
|
{ |
|
unsigned int mpidr, cpu, cluster; |
|
bool cpu_going_down, last_man; |
|
phys_reset_t phys_reset; |
|
|
|
mpidr = read_cpuid_mpidr(); |
|
cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
|
cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
|
pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); |
|
if (WARN_ON_ONCE(!platform_ops)) |
|
return; |
|
BUG_ON(!irqs_disabled()); |
|
|
|
setup_mm_for_reboot(); |
|
|
|
__mcpm_cpu_going_down(cpu, cluster); |
|
arch_spin_lock(&mcpm_lock); |
|
BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP); |
|
|
|
mcpm_cpu_use_count[cluster][cpu]--; |
|
BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 && |
|
mcpm_cpu_use_count[cluster][cpu] != 1); |
|
cpu_going_down = !mcpm_cpu_use_count[cluster][cpu]; |
|
last_man = mcpm_cluster_unused(cluster); |
|
|
|
if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) { |
|
platform_ops->cpu_powerdown_prepare(cpu, cluster); |
|
platform_ops->cluster_powerdown_prepare(cluster); |
|
arch_spin_unlock(&mcpm_lock); |
|
platform_ops->cluster_cache_disable(); |
|
__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN); |
|
} else { |
|
if (cpu_going_down) |
|
platform_ops->cpu_powerdown_prepare(cpu, cluster); |
|
arch_spin_unlock(&mcpm_lock); |
|
/* |
|
* If cpu_going_down is false here, that means a power_up |
|
* request raced ahead of us. Even if we do not want to |
|
* shut this CPU down, the caller still expects execution |
|
* to return through the system resume entry path, like |
|
* when the WFI is aborted due to a new IRQ or the like.. |
|
* So let's continue with cache cleaning in all cases. |
|
*/ |
|
platform_ops->cpu_cache_disable(); |
|
} |
|
|
|
__mcpm_cpu_down(cpu, cluster); |
|
|
|
/* Now we are prepared for power-down, do it: */ |
|
if (cpu_going_down) |
|
wfi(); |
|
|
|
/* |
|
* It is possible for a power_up request to happen concurrently |
|
* with a power_down request for the same CPU. In this case the |
|
* CPU might not be able to actually enter a powered down state |
|
* with the WFI instruction if the power_up request has removed |
|
* the required reset condition. We must perform a re-entry in |
|
* the kernel as if the power_up method just had deasserted reset |
|
* on the CPU. |
|
*/ |
|
phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset); |
|
phys_reset(__pa_symbol(mcpm_entry_point), false); |
|
|
|
/* should never get here */ |
|
BUG(); |
|
} |
|
|
|
int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster) |
|
{ |
|
int ret; |
|
|
|
if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown)) |
|
return -EUNATCH; |
|
|
|
ret = platform_ops->wait_for_powerdown(cpu, cluster); |
|
if (ret) |
|
pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n", |
|
__func__, cpu, cluster, ret); |
|
|
|
return ret; |
|
} |
|
|
|
void mcpm_cpu_suspend(void) |
|
{ |
|
if (WARN_ON_ONCE(!platform_ops)) |
|
return; |
|
|
|
/* Some platforms might have to enable special resume modes, etc. */ |
|
if (platform_ops->cpu_suspend_prepare) { |
|
unsigned int mpidr = read_cpuid_mpidr(); |
|
unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
|
unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
|
arch_spin_lock(&mcpm_lock); |
|
platform_ops->cpu_suspend_prepare(cpu, cluster); |
|
arch_spin_unlock(&mcpm_lock); |
|
} |
|
mcpm_cpu_power_down(); |
|
} |
|
|
|
int mcpm_cpu_powered_up(void) |
|
{ |
|
unsigned int mpidr, cpu, cluster; |
|
bool cpu_was_down, first_man; |
|
unsigned long flags; |
|
|
|
if (!platform_ops) |
|
return -EUNATCH; |
|
|
|
mpidr = read_cpuid_mpidr(); |
|
cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
|
cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
|
local_irq_save(flags); |
|
arch_spin_lock(&mcpm_lock); |
|
|
|
cpu_was_down = !mcpm_cpu_use_count[cluster][cpu]; |
|
first_man = mcpm_cluster_unused(cluster); |
|
|
|
if (first_man && platform_ops->cluster_is_up) |
|
platform_ops->cluster_is_up(cluster); |
|
if (cpu_was_down) |
|
mcpm_cpu_use_count[cluster][cpu] = 1; |
|
if (platform_ops->cpu_is_up) |
|
platform_ops->cpu_is_up(cpu, cluster); |
|
|
|
arch_spin_unlock(&mcpm_lock); |
|
local_irq_restore(flags); |
|
|
|
return 0; |
|
} |
|
|
|
#ifdef CONFIG_ARM_CPU_SUSPEND |
|
|
|
static int __init nocache_trampoline(unsigned long _arg) |
|
{ |
|
void (*cache_disable)(void) = (void *)_arg; |
|
unsigned int mpidr = read_cpuid_mpidr(); |
|
unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
|
unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
|
phys_reset_t phys_reset; |
|
|
|
mcpm_set_entry_vector(cpu, cluster, cpu_resume_no_hyp); |
|
setup_mm_for_reboot(); |
|
|
|
__mcpm_cpu_going_down(cpu, cluster); |
|
BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster)); |
|
cache_disable(); |
|
__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN); |
|
__mcpm_cpu_down(cpu, cluster); |
|
|
|
phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset); |
|
phys_reset(__pa_symbol(mcpm_entry_point), false); |
|
BUG(); |
|
} |
|
|
|
int __init mcpm_loopback(void (*cache_disable)(void)) |
|
{ |
|
int ret; |
|
|
|
/* |
|
* We're going to soft-restart the current CPU through the |
|
* low-level MCPM code by leveraging the suspend/resume |
|
* infrastructure. Let's play it safe by using cpu_pm_enter() |
|
* in case the CPU init code path resets the VFP or similar. |
|
*/ |
|
local_irq_disable(); |
|
local_fiq_disable(); |
|
ret = cpu_pm_enter(); |
|
if (!ret) { |
|
ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline); |
|
cpu_pm_exit(); |
|
} |
|
local_fiq_enable(); |
|
local_irq_enable(); |
|
if (ret) |
|
pr_err("%s returned %d\n", __func__, ret); |
|
return ret; |
|
} |
|
|
|
#endif |
|
|
|
extern unsigned long mcpm_power_up_setup_phys; |
|
|
|
int __init mcpm_sync_init( |
|
void (*power_up_setup)(unsigned int affinity_level)) |
|
{ |
|
unsigned int i, j, mpidr, this_cluster; |
|
|
|
BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync); |
|
BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1)); |
|
|
|
/* |
|
* Set initial CPU and cluster states. |
|
* Only one cluster is assumed to be active at this point. |
|
*/ |
|
for (i = 0; i < MAX_NR_CLUSTERS; i++) { |
|
mcpm_sync.clusters[i].cluster = CLUSTER_DOWN; |
|
mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP; |
|
for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++) |
|
mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN; |
|
} |
|
mpidr = read_cpuid_mpidr(); |
|
this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
|
for_each_online_cpu(i) { |
|
mcpm_cpu_use_count[this_cluster][i] = 1; |
|
mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP; |
|
} |
|
mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP; |
|
sync_cache_w(&mcpm_sync); |
|
|
|
if (power_up_setup) { |
|
mcpm_power_up_setup_phys = __pa_symbol(power_up_setup); |
|
sync_cache_w(&mcpm_power_up_setup_phys); |
|
} |
|
|
|
return 0; |
|
}
|
|
|