forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
543 lines
16 KiB
543 lines
16 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
#include <linux/kdebug.h> |
|
#include <linux/kprobes.h> |
|
#include <linux/export.h> |
|
#include <linux/notifier.h> |
|
#include <linux/rcupdate.h> |
|
#include <linux/vmalloc.h> |
|
#include <linux/reboot.h> |
|
|
|
/* |
|
* Notifier list for kernel code which wants to be called |
|
* at shutdown. This is used to stop any idling DMA operations |
|
* and the like. |
|
*/ |
|
BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); |
|
|
|
/* |
|
* Notifier chain core routines. The exported routines below |
|
* are layered on top of these, with appropriate locking added. |
|
*/ |
|
|
|
static int notifier_chain_register(struct notifier_block **nl, |
|
struct notifier_block *n) |
|
{ |
|
while ((*nl) != NULL) { |
|
if (unlikely((*nl) == n)) { |
|
WARN(1, "double register detected"); |
|
return 0; |
|
} |
|
if (n->priority > (*nl)->priority) |
|
break; |
|
nl = &((*nl)->next); |
|
} |
|
n->next = *nl; |
|
rcu_assign_pointer(*nl, n); |
|
return 0; |
|
} |
|
|
|
static int notifier_chain_unregister(struct notifier_block **nl, |
|
struct notifier_block *n) |
|
{ |
|
while ((*nl) != NULL) { |
|
if ((*nl) == n) { |
|
rcu_assign_pointer(*nl, n->next); |
|
return 0; |
|
} |
|
nl = &((*nl)->next); |
|
} |
|
return -ENOENT; |
|
} |
|
|
|
/** |
|
* notifier_call_chain - Informs the registered notifiers about an event. |
|
* @nl: Pointer to head of the blocking notifier chain |
|
* @val: Value passed unmodified to notifier function |
|
* @v: Pointer passed unmodified to notifier function |
|
* @nr_to_call: Number of notifier functions to be called. Don't care |
|
* value of this parameter is -1. |
|
* @nr_calls: Records the number of notifications sent. Don't care |
|
* value of this field is NULL. |
|
* @returns: notifier_call_chain returns the value returned by the |
|
* last notifier function called. |
|
*/ |
|
static int notifier_call_chain(struct notifier_block **nl, |
|
unsigned long val, void *v, |
|
int nr_to_call, int *nr_calls) |
|
{ |
|
int ret = NOTIFY_DONE; |
|
struct notifier_block *nb, *next_nb; |
|
|
|
nb = rcu_dereference_raw(*nl); |
|
|
|
while (nb && nr_to_call) { |
|
next_nb = rcu_dereference_raw(nb->next); |
|
|
|
#ifdef CONFIG_DEBUG_NOTIFIERS |
|
if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { |
|
WARN(1, "Invalid notifier called!"); |
|
nb = next_nb; |
|
continue; |
|
} |
|
#endif |
|
ret = nb->notifier_call(nb, val, v); |
|
|
|
if (nr_calls) |
|
(*nr_calls)++; |
|
|
|
if (ret & NOTIFY_STOP_MASK) |
|
break; |
|
nb = next_nb; |
|
nr_to_call--; |
|
} |
|
return ret; |
|
} |
|
NOKPROBE_SYMBOL(notifier_call_chain); |
|
|
|
/** |
|
* notifier_call_chain_robust - Inform the registered notifiers about an event |
|
* and rollback on error. |
|
* @nl: Pointer to head of the blocking notifier chain |
|
* @val_up: Value passed unmodified to the notifier function |
|
* @val_down: Value passed unmodified to the notifier function when recovering |
|
* from an error on @val_up |
|
* @v Pointer passed unmodified to the notifier function |
|
* |
|
* NOTE: It is important the @nl chain doesn't change between the two |
|
* invocations of notifier_call_chain() such that we visit the |
|
* exact same notifier callbacks; this rules out any RCU usage. |
|
* |
|
* Returns: the return value of the @val_up call. |
|
*/ |
|
static int notifier_call_chain_robust(struct notifier_block **nl, |
|
unsigned long val_up, unsigned long val_down, |
|
void *v) |
|
{ |
|
int ret, nr = 0; |
|
|
|
ret = notifier_call_chain(nl, val_up, v, -1, &nr); |
|
if (ret & NOTIFY_STOP_MASK) |
|
notifier_call_chain(nl, val_down, v, nr-1, NULL); |
|
|
|
return ret; |
|
} |
|
|
|
/* |
|
* Atomic notifier chain routines. Registration and unregistration |
|
* use a spinlock, and call_chain is synchronized by RCU (no locks). |
|
*/ |
|
|
|
/** |
|
* atomic_notifier_chain_register - Add notifier to an atomic notifier chain |
|
* @nh: Pointer to head of the atomic notifier chain |
|
* @n: New entry in notifier chain |
|
* |
|
* Adds a notifier to an atomic notifier chain. |
|
* |
|
* Currently always returns zero. |
|
*/ |
|
int atomic_notifier_chain_register(struct atomic_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
unsigned long flags; |
|
int ret; |
|
|
|
spin_lock_irqsave(&nh->lock, flags); |
|
ret = notifier_chain_register(&nh->head, n); |
|
spin_unlock_irqrestore(&nh->lock, flags); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); |
|
|
|
/** |
|
* atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain |
|
* @nh: Pointer to head of the atomic notifier chain |
|
* @n: Entry to remove from notifier chain |
|
* |
|
* Removes a notifier from an atomic notifier chain. |
|
* |
|
* Returns zero on success or %-ENOENT on failure. |
|
*/ |
|
int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
unsigned long flags; |
|
int ret; |
|
|
|
spin_lock_irqsave(&nh->lock, flags); |
|
ret = notifier_chain_unregister(&nh->head, n); |
|
spin_unlock_irqrestore(&nh->lock, flags); |
|
synchronize_rcu(); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); |
|
|
|
/** |
|
* atomic_notifier_call_chain - Call functions in an atomic notifier chain |
|
* @nh: Pointer to head of the atomic notifier chain |
|
* @val: Value passed unmodified to notifier function |
|
* @v: Pointer passed unmodified to notifier function |
|
* |
|
* Calls each function in a notifier chain in turn. The functions |
|
* run in an atomic context, so they must not block. |
|
* This routine uses RCU to synchronize with changes to the chain. |
|
* |
|
* If the return value of the notifier can be and'ed |
|
* with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() |
|
* will return immediately, with the return value of |
|
* the notifier function which halted execution. |
|
* Otherwise the return value is the return value |
|
* of the last notifier function called. |
|
*/ |
|
int atomic_notifier_call_chain(struct atomic_notifier_head *nh, |
|
unsigned long val, void *v) |
|
{ |
|
int ret; |
|
|
|
rcu_read_lock(); |
|
ret = notifier_call_chain(&nh->head, val, v, -1, NULL); |
|
rcu_read_unlock(); |
|
|
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); |
|
NOKPROBE_SYMBOL(atomic_notifier_call_chain); |
|
|
|
/* |
|
* Blocking notifier chain routines. All access to the chain is |
|
* synchronized by an rwsem. |
|
*/ |
|
|
|
/** |
|
* blocking_notifier_chain_register - Add notifier to a blocking notifier chain |
|
* @nh: Pointer to head of the blocking notifier chain |
|
* @n: New entry in notifier chain |
|
* |
|
* Adds a notifier to a blocking notifier chain. |
|
* Must be called in process context. |
|
* |
|
* Currently always returns zero. |
|
*/ |
|
int blocking_notifier_chain_register(struct blocking_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
int ret; |
|
|
|
/* |
|
* This code gets used during boot-up, when task switching is |
|
* not yet working and interrupts must remain disabled. At |
|
* such times we must not call down_write(). |
|
*/ |
|
if (unlikely(system_state == SYSTEM_BOOTING)) |
|
return notifier_chain_register(&nh->head, n); |
|
|
|
down_write(&nh->rwsem); |
|
ret = notifier_chain_register(&nh->head, n); |
|
up_write(&nh->rwsem); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); |
|
|
|
/** |
|
* blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain |
|
* @nh: Pointer to head of the blocking notifier chain |
|
* @n: Entry to remove from notifier chain |
|
* |
|
* Removes a notifier from a blocking notifier chain. |
|
* Must be called from process context. |
|
* |
|
* Returns zero on success or %-ENOENT on failure. |
|
*/ |
|
int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
int ret; |
|
|
|
/* |
|
* This code gets used during boot-up, when task switching is |
|
* not yet working and interrupts must remain disabled. At |
|
* such times we must not call down_write(). |
|
*/ |
|
if (unlikely(system_state == SYSTEM_BOOTING)) |
|
return notifier_chain_unregister(&nh->head, n); |
|
|
|
down_write(&nh->rwsem); |
|
ret = notifier_chain_unregister(&nh->head, n); |
|
up_write(&nh->rwsem); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); |
|
|
|
int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, |
|
unsigned long val_up, unsigned long val_down, void *v) |
|
{ |
|
int ret = NOTIFY_DONE; |
|
|
|
/* |
|
* We check the head outside the lock, but if this access is |
|
* racy then it does not matter what the result of the test |
|
* is, we re-check the list after having taken the lock anyway: |
|
*/ |
|
if (rcu_access_pointer(nh->head)) { |
|
down_read(&nh->rwsem); |
|
ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); |
|
up_read(&nh->rwsem); |
|
} |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); |
|
|
|
/** |
|
* blocking_notifier_call_chain - Call functions in a blocking notifier chain |
|
* @nh: Pointer to head of the blocking notifier chain |
|
* @val: Value passed unmodified to notifier function |
|
* @v: Pointer passed unmodified to notifier function |
|
* |
|
* Calls each function in a notifier chain in turn. The functions |
|
* run in a process context, so they are allowed to block. |
|
* |
|
* If the return value of the notifier can be and'ed |
|
* with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() |
|
* will return immediately, with the return value of |
|
* the notifier function which halted execution. |
|
* Otherwise the return value is the return value |
|
* of the last notifier function called. |
|
*/ |
|
int blocking_notifier_call_chain(struct blocking_notifier_head *nh, |
|
unsigned long val, void *v) |
|
{ |
|
int ret = NOTIFY_DONE; |
|
|
|
/* |
|
* We check the head outside the lock, but if this access is |
|
* racy then it does not matter what the result of the test |
|
* is, we re-check the list after having taken the lock anyway: |
|
*/ |
|
if (rcu_access_pointer(nh->head)) { |
|
down_read(&nh->rwsem); |
|
ret = notifier_call_chain(&nh->head, val, v, -1, NULL); |
|
up_read(&nh->rwsem); |
|
} |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); |
|
|
|
/* |
|
* Raw notifier chain routines. There is no protection; |
|
* the caller must provide it. Use at your own risk! |
|
*/ |
|
|
|
/** |
|
* raw_notifier_chain_register - Add notifier to a raw notifier chain |
|
* @nh: Pointer to head of the raw notifier chain |
|
* @n: New entry in notifier chain |
|
* |
|
* Adds a notifier to a raw notifier chain. |
|
* All locking must be provided by the caller. |
|
* |
|
* Currently always returns zero. |
|
*/ |
|
int raw_notifier_chain_register(struct raw_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
return notifier_chain_register(&nh->head, n); |
|
} |
|
EXPORT_SYMBOL_GPL(raw_notifier_chain_register); |
|
|
|
/** |
|
* raw_notifier_chain_unregister - Remove notifier from a raw notifier chain |
|
* @nh: Pointer to head of the raw notifier chain |
|
* @n: Entry to remove from notifier chain |
|
* |
|
* Removes a notifier from a raw notifier chain. |
|
* All locking must be provided by the caller. |
|
* |
|
* Returns zero on success or %-ENOENT on failure. |
|
*/ |
|
int raw_notifier_chain_unregister(struct raw_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
return notifier_chain_unregister(&nh->head, n); |
|
} |
|
EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); |
|
|
|
int raw_notifier_call_chain_robust(struct raw_notifier_head *nh, |
|
unsigned long val_up, unsigned long val_down, void *v) |
|
{ |
|
return notifier_call_chain_robust(&nh->head, val_up, val_down, v); |
|
} |
|
EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust); |
|
|
|
/** |
|
* raw_notifier_call_chain - Call functions in a raw notifier chain |
|
* @nh: Pointer to head of the raw notifier chain |
|
* @val: Value passed unmodified to notifier function |
|
* @v: Pointer passed unmodified to notifier function |
|
* |
|
* Calls each function in a notifier chain in turn. The functions |
|
* run in an undefined context. |
|
* All locking must be provided by the caller. |
|
* |
|
* If the return value of the notifier can be and'ed |
|
* with %NOTIFY_STOP_MASK then raw_notifier_call_chain() |
|
* will return immediately, with the return value of |
|
* the notifier function which halted execution. |
|
* Otherwise the return value is the return value |
|
* of the last notifier function called. |
|
*/ |
|
int raw_notifier_call_chain(struct raw_notifier_head *nh, |
|
unsigned long val, void *v) |
|
{ |
|
return notifier_call_chain(&nh->head, val, v, -1, NULL); |
|
} |
|
EXPORT_SYMBOL_GPL(raw_notifier_call_chain); |
|
|
|
#ifdef CONFIG_SRCU |
|
/* |
|
* SRCU notifier chain routines. Registration and unregistration |
|
* use a mutex, and call_chain is synchronized by SRCU (no locks). |
|
*/ |
|
|
|
/** |
|
* srcu_notifier_chain_register - Add notifier to an SRCU notifier chain |
|
* @nh: Pointer to head of the SRCU notifier chain |
|
* @n: New entry in notifier chain |
|
* |
|
* Adds a notifier to an SRCU notifier chain. |
|
* Must be called in process context. |
|
* |
|
* Currently always returns zero. |
|
*/ |
|
int srcu_notifier_chain_register(struct srcu_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
int ret; |
|
|
|
/* |
|
* This code gets used during boot-up, when task switching is |
|
* not yet working and interrupts must remain disabled. At |
|
* such times we must not call mutex_lock(). |
|
*/ |
|
if (unlikely(system_state == SYSTEM_BOOTING)) |
|
return notifier_chain_register(&nh->head, n); |
|
|
|
mutex_lock(&nh->mutex); |
|
ret = notifier_chain_register(&nh->head, n); |
|
mutex_unlock(&nh->mutex); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); |
|
|
|
/** |
|
* srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain |
|
* @nh: Pointer to head of the SRCU notifier chain |
|
* @n: Entry to remove from notifier chain |
|
* |
|
* Removes a notifier from an SRCU notifier chain. |
|
* Must be called from process context. |
|
* |
|
* Returns zero on success or %-ENOENT on failure. |
|
*/ |
|
int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, |
|
struct notifier_block *n) |
|
{ |
|
int ret; |
|
|
|
/* |
|
* This code gets used during boot-up, when task switching is |
|
* not yet working and interrupts must remain disabled. At |
|
* such times we must not call mutex_lock(). |
|
*/ |
|
if (unlikely(system_state == SYSTEM_BOOTING)) |
|
return notifier_chain_unregister(&nh->head, n); |
|
|
|
mutex_lock(&nh->mutex); |
|
ret = notifier_chain_unregister(&nh->head, n); |
|
mutex_unlock(&nh->mutex); |
|
synchronize_srcu(&nh->srcu); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); |
|
|
|
/** |
|
* srcu_notifier_call_chain - Call functions in an SRCU notifier chain |
|
* @nh: Pointer to head of the SRCU notifier chain |
|
* @val: Value passed unmodified to notifier function |
|
* @v: Pointer passed unmodified to notifier function |
|
* |
|
* Calls each function in a notifier chain in turn. The functions |
|
* run in a process context, so they are allowed to block. |
|
* |
|
* If the return value of the notifier can be and'ed |
|
* with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() |
|
* will return immediately, with the return value of |
|
* the notifier function which halted execution. |
|
* Otherwise the return value is the return value |
|
* of the last notifier function called. |
|
*/ |
|
int srcu_notifier_call_chain(struct srcu_notifier_head *nh, |
|
unsigned long val, void *v) |
|
{ |
|
int ret; |
|
int idx; |
|
|
|
idx = srcu_read_lock(&nh->srcu); |
|
ret = notifier_call_chain(&nh->head, val, v, -1, NULL); |
|
srcu_read_unlock(&nh->srcu, idx); |
|
return ret; |
|
} |
|
EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); |
|
|
|
/** |
|
* srcu_init_notifier_head - Initialize an SRCU notifier head |
|
* @nh: Pointer to head of the srcu notifier chain |
|
* |
|
* Unlike other sorts of notifier heads, SRCU notifier heads require |
|
* dynamic initialization. Be sure to call this routine before |
|
* calling any of the other SRCU notifier routines for this head. |
|
* |
|
* If an SRCU notifier head is deallocated, it must first be cleaned |
|
* up by calling srcu_cleanup_notifier_head(). Otherwise the head's |
|
* per-cpu data (used by the SRCU mechanism) will leak. |
|
*/ |
|
void srcu_init_notifier_head(struct srcu_notifier_head *nh) |
|
{ |
|
mutex_init(&nh->mutex); |
|
if (init_srcu_struct(&nh->srcu) < 0) |
|
BUG(); |
|
nh->head = NULL; |
|
} |
|
EXPORT_SYMBOL_GPL(srcu_init_notifier_head); |
|
|
|
#endif /* CONFIG_SRCU */ |
|
|
|
static ATOMIC_NOTIFIER_HEAD(die_chain); |
|
|
|
int notrace notify_die(enum die_val val, const char *str, |
|
struct pt_regs *regs, long err, int trap, int sig) |
|
{ |
|
struct die_args args = { |
|
.regs = regs, |
|
.str = str, |
|
.err = err, |
|
.trapnr = trap, |
|
.signr = sig, |
|
|
|
}; |
|
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
|
"notify_die called but RCU thinks we're quiescent"); |
|
return atomic_notifier_call_chain(&die_chain, val, &args); |
|
} |
|
NOKPROBE_SYMBOL(notify_die); |
|
|
|
int register_die_notifier(struct notifier_block *nb) |
|
{ |
|
return atomic_notifier_chain_register(&die_chain, nb); |
|
} |
|
EXPORT_SYMBOL_GPL(register_die_notifier); |
|
|
|
int unregister_die_notifier(struct notifier_block *nb) |
|
{ |
|
return atomic_notifier_chain_unregister(&die_chain, nb); |
|
} |
|
EXPORT_SYMBOL_GPL(unregister_die_notifier);
|
|
|