forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
778 lines
18 KiB
778 lines
18 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* Copyright (c) 2020 Facebook */ |
|
|
|
#include <linux/fs.h> |
|
#include <linux/anon_inodes.h> |
|
#include <linux/filter.h> |
|
#include <linux/bpf.h> |
|
#include <linux/rcupdate_trace.h> |
|
|
|
struct bpf_iter_target_info { |
|
struct list_head list; |
|
const struct bpf_iter_reg *reg_info; |
|
u32 btf_id; /* cached value */ |
|
}; |
|
|
|
struct bpf_iter_link { |
|
struct bpf_link link; |
|
struct bpf_iter_aux_info aux; |
|
struct bpf_iter_target_info *tinfo; |
|
}; |
|
|
|
struct bpf_iter_priv_data { |
|
struct bpf_iter_target_info *tinfo; |
|
const struct bpf_iter_seq_info *seq_info; |
|
struct bpf_prog *prog; |
|
u64 session_id; |
|
u64 seq_num; |
|
bool done_stop; |
|
u8 target_private[] __aligned(8); |
|
}; |
|
|
|
static struct list_head targets = LIST_HEAD_INIT(targets); |
|
static DEFINE_MUTEX(targets_mutex); |
|
|
|
/* protect bpf_iter_link changes */ |
|
static DEFINE_MUTEX(link_mutex); |
|
|
|
/* incremented on every opened seq_file */ |
|
static atomic64_t session_id; |
|
|
|
static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, |
|
const struct bpf_iter_seq_info *seq_info); |
|
|
|
static void bpf_iter_inc_seq_num(struct seq_file *seq) |
|
{ |
|
struct bpf_iter_priv_data *iter_priv; |
|
|
|
iter_priv = container_of(seq->private, struct bpf_iter_priv_data, |
|
target_private); |
|
iter_priv->seq_num++; |
|
} |
|
|
|
static void bpf_iter_dec_seq_num(struct seq_file *seq) |
|
{ |
|
struct bpf_iter_priv_data *iter_priv; |
|
|
|
iter_priv = container_of(seq->private, struct bpf_iter_priv_data, |
|
target_private); |
|
iter_priv->seq_num--; |
|
} |
|
|
|
static void bpf_iter_done_stop(struct seq_file *seq) |
|
{ |
|
struct bpf_iter_priv_data *iter_priv; |
|
|
|
iter_priv = container_of(seq->private, struct bpf_iter_priv_data, |
|
target_private); |
|
iter_priv->done_stop = true; |
|
} |
|
|
|
static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo) |
|
{ |
|
return tinfo->reg_info->feature & BPF_ITER_RESCHED; |
|
} |
|
|
|
static bool bpf_iter_support_resched(struct seq_file *seq) |
|
{ |
|
struct bpf_iter_priv_data *iter_priv; |
|
|
|
iter_priv = container_of(seq->private, struct bpf_iter_priv_data, |
|
target_private); |
|
return bpf_iter_target_support_resched(iter_priv->tinfo); |
|
} |
|
|
|
/* maximum visited objects before bailing out */ |
|
#define MAX_ITER_OBJECTS 1000000 |
|
|
|
/* bpf_seq_read, a customized and simpler version for bpf iterator. |
|
* The following are differences from seq_read(): |
|
* . fixed buffer size (PAGE_SIZE) |
|
* . assuming NULL ->llseek() |
|
* . stop() may call bpf program, handling potential overflow there |
|
*/ |
|
static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, |
|
loff_t *ppos) |
|
{ |
|
struct seq_file *seq = file->private_data; |
|
size_t n, offs, copied = 0; |
|
int err = 0, num_objs = 0; |
|
bool can_resched; |
|
void *p; |
|
|
|
mutex_lock(&seq->lock); |
|
|
|
if (!seq->buf) { |
|
seq->size = PAGE_SIZE << 3; |
|
seq->buf = kvmalloc(seq->size, GFP_KERNEL); |
|
if (!seq->buf) { |
|
err = -ENOMEM; |
|
goto done; |
|
} |
|
} |
|
|
|
if (seq->count) { |
|
n = min(seq->count, size); |
|
err = copy_to_user(buf, seq->buf + seq->from, n); |
|
if (err) { |
|
err = -EFAULT; |
|
goto done; |
|
} |
|
seq->count -= n; |
|
seq->from += n; |
|
copied = n; |
|
goto done; |
|
} |
|
|
|
seq->from = 0; |
|
p = seq->op->start(seq, &seq->index); |
|
if (!p) |
|
goto stop; |
|
if (IS_ERR(p)) { |
|
err = PTR_ERR(p); |
|
seq->op->stop(seq, p); |
|
seq->count = 0; |
|
goto done; |
|
} |
|
|
|
err = seq->op->show(seq, p); |
|
if (err > 0) { |
|
/* object is skipped, decrease seq_num, so next |
|
* valid object can reuse the same seq_num. |
|
*/ |
|
bpf_iter_dec_seq_num(seq); |
|
seq->count = 0; |
|
} else if (err < 0 || seq_has_overflowed(seq)) { |
|
if (!err) |
|
err = -E2BIG; |
|
seq->op->stop(seq, p); |
|
seq->count = 0; |
|
goto done; |
|
} |
|
|
|
can_resched = bpf_iter_support_resched(seq); |
|
while (1) { |
|
loff_t pos = seq->index; |
|
|
|
num_objs++; |
|
offs = seq->count; |
|
p = seq->op->next(seq, p, &seq->index); |
|
if (pos == seq->index) { |
|
pr_info_ratelimited("buggy seq_file .next function %ps " |
|
"did not updated position index\n", |
|
seq->op->next); |
|
seq->index++; |
|
} |
|
|
|
if (IS_ERR_OR_NULL(p)) |
|
break; |
|
|
|
/* got a valid next object, increase seq_num */ |
|
bpf_iter_inc_seq_num(seq); |
|
|
|
if (seq->count >= size) |
|
break; |
|
|
|
if (num_objs >= MAX_ITER_OBJECTS) { |
|
if (offs == 0) { |
|
err = -EAGAIN; |
|
seq->op->stop(seq, p); |
|
goto done; |
|
} |
|
break; |
|
} |
|
|
|
err = seq->op->show(seq, p); |
|
if (err > 0) { |
|
bpf_iter_dec_seq_num(seq); |
|
seq->count = offs; |
|
} else if (err < 0 || seq_has_overflowed(seq)) { |
|
seq->count = offs; |
|
if (offs == 0) { |
|
if (!err) |
|
err = -E2BIG; |
|
seq->op->stop(seq, p); |
|
goto done; |
|
} |
|
break; |
|
} |
|
|
|
if (can_resched) |
|
cond_resched(); |
|
} |
|
stop: |
|
offs = seq->count; |
|
if (IS_ERR(p)) { |
|
seq->op->stop(seq, NULL); |
|
err = PTR_ERR(p); |
|
goto done; |
|
} |
|
/* bpf program called if !p */ |
|
seq->op->stop(seq, p); |
|
if (!p) { |
|
if (!seq_has_overflowed(seq)) { |
|
bpf_iter_done_stop(seq); |
|
} else { |
|
seq->count = offs; |
|
if (offs == 0) { |
|
err = -E2BIG; |
|
goto done; |
|
} |
|
} |
|
} |
|
|
|
n = min(seq->count, size); |
|
err = copy_to_user(buf, seq->buf, n); |
|
if (err) { |
|
err = -EFAULT; |
|
goto done; |
|
} |
|
copied = n; |
|
seq->count -= n; |
|
seq->from = n; |
|
done: |
|
if (!copied) |
|
copied = err; |
|
else |
|
*ppos += copied; |
|
mutex_unlock(&seq->lock); |
|
return copied; |
|
} |
|
|
|
static const struct bpf_iter_seq_info * |
|
__get_seq_info(struct bpf_iter_link *link) |
|
{ |
|
const struct bpf_iter_seq_info *seq_info; |
|
|
|
if (link->aux.map) { |
|
seq_info = link->aux.map->ops->iter_seq_info; |
|
if (seq_info) |
|
return seq_info; |
|
} |
|
|
|
return link->tinfo->reg_info->seq_info; |
|
} |
|
|
|
static int iter_open(struct inode *inode, struct file *file) |
|
{ |
|
struct bpf_iter_link *link = inode->i_private; |
|
|
|
return prepare_seq_file(file, link, __get_seq_info(link)); |
|
} |
|
|
|
static int iter_release(struct inode *inode, struct file *file) |
|
{ |
|
struct bpf_iter_priv_data *iter_priv; |
|
struct seq_file *seq; |
|
|
|
seq = file->private_data; |
|
if (!seq) |
|
return 0; |
|
|
|
iter_priv = container_of(seq->private, struct bpf_iter_priv_data, |
|
target_private); |
|
|
|
if (iter_priv->seq_info->fini_seq_private) |
|
iter_priv->seq_info->fini_seq_private(seq->private); |
|
|
|
bpf_prog_put(iter_priv->prog); |
|
seq->private = iter_priv; |
|
|
|
return seq_release_private(inode, file); |
|
} |
|
|
|
const struct file_operations bpf_iter_fops = { |
|
.open = iter_open, |
|
.llseek = no_llseek, |
|
.read = bpf_seq_read, |
|
.release = iter_release, |
|
}; |
|
|
|
/* The argument reg_info will be cached in bpf_iter_target_info. |
|
* The common practice is to declare target reg_info as |
|
* a const static variable and passed as an argument to |
|
* bpf_iter_reg_target(). |
|
*/ |
|
int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) |
|
{ |
|
struct bpf_iter_target_info *tinfo; |
|
|
|
tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL); |
|
if (!tinfo) |
|
return -ENOMEM; |
|
|
|
tinfo->reg_info = reg_info; |
|
INIT_LIST_HEAD(&tinfo->list); |
|
|
|
mutex_lock(&targets_mutex); |
|
list_add(&tinfo->list, &targets); |
|
mutex_unlock(&targets_mutex); |
|
|
|
return 0; |
|
} |
|
|
|
void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info) |
|
{ |
|
struct bpf_iter_target_info *tinfo; |
|
bool found = false; |
|
|
|
mutex_lock(&targets_mutex); |
|
list_for_each_entry(tinfo, &targets, list) { |
|
if (reg_info == tinfo->reg_info) { |
|
list_del(&tinfo->list); |
|
kfree(tinfo); |
|
found = true; |
|
break; |
|
} |
|
} |
|
mutex_unlock(&targets_mutex); |
|
|
|
WARN_ON(found == false); |
|
} |
|
|
|
static void cache_btf_id(struct bpf_iter_target_info *tinfo, |
|
struct bpf_prog *prog) |
|
{ |
|
tinfo->btf_id = prog->aux->attach_btf_id; |
|
} |
|
|
|
bool bpf_iter_prog_supported(struct bpf_prog *prog) |
|
{ |
|
const char *attach_fname = prog->aux->attach_func_name; |
|
struct bpf_iter_target_info *tinfo = NULL, *iter; |
|
u32 prog_btf_id = prog->aux->attach_btf_id; |
|
const char *prefix = BPF_ITER_FUNC_PREFIX; |
|
int prefix_len = strlen(prefix); |
|
|
|
if (strncmp(attach_fname, prefix, prefix_len)) |
|
return false; |
|
|
|
mutex_lock(&targets_mutex); |
|
list_for_each_entry(iter, &targets, list) { |
|
if (iter->btf_id && iter->btf_id == prog_btf_id) { |
|
tinfo = iter; |
|
break; |
|
} |
|
if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) { |
|
cache_btf_id(iter, prog); |
|
tinfo = iter; |
|
break; |
|
} |
|
} |
|
mutex_unlock(&targets_mutex); |
|
|
|
if (tinfo) { |
|
prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size; |
|
prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info; |
|
} |
|
|
|
return tinfo != NULL; |
|
} |
|
|
|
const struct bpf_func_proto * |
|
bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) |
|
{ |
|
const struct bpf_iter_target_info *tinfo; |
|
const struct bpf_func_proto *fn = NULL; |
|
|
|
mutex_lock(&targets_mutex); |
|
list_for_each_entry(tinfo, &targets, list) { |
|
if (tinfo->btf_id == prog->aux->attach_btf_id) { |
|
const struct bpf_iter_reg *reg_info; |
|
|
|
reg_info = tinfo->reg_info; |
|
if (reg_info->get_func_proto) |
|
fn = reg_info->get_func_proto(func_id, prog); |
|
break; |
|
} |
|
} |
|
mutex_unlock(&targets_mutex); |
|
|
|
return fn; |
|
} |
|
|
|
static void bpf_iter_link_release(struct bpf_link *link) |
|
{ |
|
struct bpf_iter_link *iter_link = |
|
container_of(link, struct bpf_iter_link, link); |
|
|
|
if (iter_link->tinfo->reg_info->detach_target) |
|
iter_link->tinfo->reg_info->detach_target(&iter_link->aux); |
|
} |
|
|
|
static void bpf_iter_link_dealloc(struct bpf_link *link) |
|
{ |
|
struct bpf_iter_link *iter_link = |
|
container_of(link, struct bpf_iter_link, link); |
|
|
|
kfree(iter_link); |
|
} |
|
|
|
static int bpf_iter_link_replace(struct bpf_link *link, |
|
struct bpf_prog *new_prog, |
|
struct bpf_prog *old_prog) |
|
{ |
|
int ret = 0; |
|
|
|
mutex_lock(&link_mutex); |
|
if (old_prog && link->prog != old_prog) { |
|
ret = -EPERM; |
|
goto out_unlock; |
|
} |
|
|
|
if (link->prog->type != new_prog->type || |
|
link->prog->expected_attach_type != new_prog->expected_attach_type || |
|
link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { |
|
ret = -EINVAL; |
|
goto out_unlock; |
|
} |
|
|
|
old_prog = xchg(&link->prog, new_prog); |
|
bpf_prog_put(old_prog); |
|
|
|
out_unlock: |
|
mutex_unlock(&link_mutex); |
|
return ret; |
|
} |
|
|
|
static void bpf_iter_link_show_fdinfo(const struct bpf_link *link, |
|
struct seq_file *seq) |
|
{ |
|
struct bpf_iter_link *iter_link = |
|
container_of(link, struct bpf_iter_link, link); |
|
bpf_iter_show_fdinfo_t show_fdinfo; |
|
|
|
seq_printf(seq, |
|
"target_name:\t%s\n", |
|
iter_link->tinfo->reg_info->target); |
|
|
|
show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo; |
|
if (show_fdinfo) |
|
show_fdinfo(&iter_link->aux, seq); |
|
} |
|
|
|
static int bpf_iter_link_fill_link_info(const struct bpf_link *link, |
|
struct bpf_link_info *info) |
|
{ |
|
struct bpf_iter_link *iter_link = |
|
container_of(link, struct bpf_iter_link, link); |
|
char __user *ubuf = u64_to_user_ptr(info->iter.target_name); |
|
bpf_iter_fill_link_info_t fill_link_info; |
|
u32 ulen = info->iter.target_name_len; |
|
const char *target_name; |
|
u32 target_len; |
|
|
|
if (!ulen ^ !ubuf) |
|
return -EINVAL; |
|
|
|
target_name = iter_link->tinfo->reg_info->target; |
|
target_len = strlen(target_name); |
|
info->iter.target_name_len = target_len + 1; |
|
|
|
if (ubuf) { |
|
if (ulen >= target_len + 1) { |
|
if (copy_to_user(ubuf, target_name, target_len + 1)) |
|
return -EFAULT; |
|
} else { |
|
char zero = '\0'; |
|
|
|
if (copy_to_user(ubuf, target_name, ulen - 1)) |
|
return -EFAULT; |
|
if (put_user(zero, ubuf + ulen - 1)) |
|
return -EFAULT; |
|
return -ENOSPC; |
|
} |
|
} |
|
|
|
fill_link_info = iter_link->tinfo->reg_info->fill_link_info; |
|
if (fill_link_info) |
|
return fill_link_info(&iter_link->aux, info); |
|
|
|
return 0; |
|
} |
|
|
|
static const struct bpf_link_ops bpf_iter_link_lops = { |
|
.release = bpf_iter_link_release, |
|
.dealloc = bpf_iter_link_dealloc, |
|
.update_prog = bpf_iter_link_replace, |
|
.show_fdinfo = bpf_iter_link_show_fdinfo, |
|
.fill_link_info = bpf_iter_link_fill_link_info, |
|
}; |
|
|
|
bool bpf_link_is_iter(struct bpf_link *link) |
|
{ |
|
return link->ops == &bpf_iter_link_lops; |
|
} |
|
|
|
int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, |
|
struct bpf_prog *prog) |
|
{ |
|
struct bpf_iter_target_info *tinfo = NULL, *iter; |
|
struct bpf_link_primer link_primer; |
|
union bpf_iter_link_info linfo; |
|
struct bpf_iter_link *link; |
|
u32 prog_btf_id, linfo_len; |
|
bpfptr_t ulinfo; |
|
int err; |
|
|
|
if (attr->link_create.target_fd || attr->link_create.flags) |
|
return -EINVAL; |
|
|
|
memset(&linfo, 0, sizeof(union bpf_iter_link_info)); |
|
|
|
ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel); |
|
linfo_len = attr->link_create.iter_info_len; |
|
if (bpfptr_is_null(ulinfo) ^ !linfo_len) |
|
return -EINVAL; |
|
|
|
if (!bpfptr_is_null(ulinfo)) { |
|
err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo), |
|
linfo_len); |
|
if (err) |
|
return err; |
|
linfo_len = min_t(u32, linfo_len, sizeof(linfo)); |
|
if (copy_from_bpfptr(&linfo, ulinfo, linfo_len)) |
|
return -EFAULT; |
|
} |
|
|
|
prog_btf_id = prog->aux->attach_btf_id; |
|
mutex_lock(&targets_mutex); |
|
list_for_each_entry(iter, &targets, list) { |
|
if (iter->btf_id == prog_btf_id) { |
|
tinfo = iter; |
|
break; |
|
} |
|
} |
|
mutex_unlock(&targets_mutex); |
|
if (!tinfo) |
|
return -ENOENT; |
|
|
|
/* Only allow sleepable program for resched-able iterator */ |
|
if (prog->aux->sleepable && !bpf_iter_target_support_resched(tinfo)) |
|
return -EINVAL; |
|
|
|
link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN); |
|
if (!link) |
|
return -ENOMEM; |
|
|
|
bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog); |
|
link->tinfo = tinfo; |
|
|
|
err = bpf_link_prime(&link->link, &link_primer); |
|
if (err) { |
|
kfree(link); |
|
return err; |
|
} |
|
|
|
if (tinfo->reg_info->attach_target) { |
|
err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux); |
|
if (err) { |
|
bpf_link_cleanup(&link_primer); |
|
return err; |
|
} |
|
} |
|
|
|
return bpf_link_settle(&link_primer); |
|
} |
|
|
|
static void init_seq_meta(struct bpf_iter_priv_data *priv_data, |
|
struct bpf_iter_target_info *tinfo, |
|
const struct bpf_iter_seq_info *seq_info, |
|
struct bpf_prog *prog) |
|
{ |
|
priv_data->tinfo = tinfo; |
|
priv_data->seq_info = seq_info; |
|
priv_data->prog = prog; |
|
priv_data->session_id = atomic64_inc_return(&session_id); |
|
priv_data->seq_num = 0; |
|
priv_data->done_stop = false; |
|
} |
|
|
|
static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, |
|
const struct bpf_iter_seq_info *seq_info) |
|
{ |
|
struct bpf_iter_priv_data *priv_data; |
|
struct bpf_iter_target_info *tinfo; |
|
struct bpf_prog *prog; |
|
u32 total_priv_dsize; |
|
struct seq_file *seq; |
|
int err = 0; |
|
|
|
mutex_lock(&link_mutex); |
|
prog = link->link.prog; |
|
bpf_prog_inc(prog); |
|
mutex_unlock(&link_mutex); |
|
|
|
tinfo = link->tinfo; |
|
total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) + |
|
seq_info->seq_priv_size; |
|
priv_data = __seq_open_private(file, seq_info->seq_ops, |
|
total_priv_dsize); |
|
if (!priv_data) { |
|
err = -ENOMEM; |
|
goto release_prog; |
|
} |
|
|
|
if (seq_info->init_seq_private) { |
|
err = seq_info->init_seq_private(priv_data->target_private, &link->aux); |
|
if (err) |
|
goto release_seq_file; |
|
} |
|
|
|
init_seq_meta(priv_data, tinfo, seq_info, prog); |
|
seq = file->private_data; |
|
seq->private = priv_data->target_private; |
|
|
|
return 0; |
|
|
|
release_seq_file: |
|
seq_release_private(file->f_inode, file); |
|
file->private_data = NULL; |
|
release_prog: |
|
bpf_prog_put(prog); |
|
return err; |
|
} |
|
|
|
int bpf_iter_new_fd(struct bpf_link *link) |
|
{ |
|
struct bpf_iter_link *iter_link; |
|
struct file *file; |
|
unsigned int flags; |
|
int err, fd; |
|
|
|
if (link->ops != &bpf_iter_link_lops) |
|
return -EINVAL; |
|
|
|
flags = O_RDONLY | O_CLOEXEC; |
|
fd = get_unused_fd_flags(flags); |
|
if (fd < 0) |
|
return fd; |
|
|
|
file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags); |
|
if (IS_ERR(file)) { |
|
err = PTR_ERR(file); |
|
goto free_fd; |
|
} |
|
|
|
iter_link = container_of(link, struct bpf_iter_link, link); |
|
err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link)); |
|
if (err) |
|
goto free_file; |
|
|
|
fd_install(fd, file); |
|
return fd; |
|
|
|
free_file: |
|
fput(file); |
|
free_fd: |
|
put_unused_fd(fd); |
|
return err; |
|
} |
|
|
|
struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop) |
|
{ |
|
struct bpf_iter_priv_data *iter_priv; |
|
struct seq_file *seq; |
|
void *seq_priv; |
|
|
|
seq = meta->seq; |
|
if (seq->file->f_op != &bpf_iter_fops) |
|
return NULL; |
|
|
|
seq_priv = seq->private; |
|
iter_priv = container_of(seq_priv, struct bpf_iter_priv_data, |
|
target_private); |
|
|
|
if (in_stop && iter_priv->done_stop) |
|
return NULL; |
|
|
|
meta->session_id = iter_priv->session_id; |
|
meta->seq_num = iter_priv->seq_num; |
|
|
|
return iter_priv->prog; |
|
} |
|
|
|
int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) |
|
{ |
|
struct bpf_run_ctx run_ctx, *old_run_ctx; |
|
int ret; |
|
|
|
if (prog->aux->sleepable) { |
|
rcu_read_lock_trace(); |
|
migrate_disable(); |
|
might_fault(); |
|
old_run_ctx = bpf_set_run_ctx(&run_ctx); |
|
ret = bpf_prog_run(prog, ctx); |
|
bpf_reset_run_ctx(old_run_ctx); |
|
migrate_enable(); |
|
rcu_read_unlock_trace(); |
|
} else { |
|
rcu_read_lock(); |
|
migrate_disable(); |
|
old_run_ctx = bpf_set_run_ctx(&run_ctx); |
|
ret = bpf_prog_run(prog, ctx); |
|
bpf_reset_run_ctx(old_run_ctx); |
|
migrate_enable(); |
|
rcu_read_unlock(); |
|
} |
|
|
|
/* bpf program can only return 0 or 1: |
|
* 0 : okay |
|
* 1 : retry the same object |
|
* The bpf_iter_run_prog() return value |
|
* will be seq_ops->show() return value. |
|
*/ |
|
return ret == 0 ? 0 : -EAGAIN; |
|
} |
|
|
|
BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn, |
|
void *, callback_ctx, u64, flags) |
|
{ |
|
return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags); |
|
} |
|
|
|
const struct bpf_func_proto bpf_for_each_map_elem_proto = { |
|
.func = bpf_for_each_map_elem, |
|
.gpl_only = false, |
|
.ret_type = RET_INTEGER, |
|
.arg1_type = ARG_CONST_MAP_PTR, |
|
.arg2_type = ARG_PTR_TO_FUNC, |
|
.arg3_type = ARG_PTR_TO_STACK_OR_NULL, |
|
.arg4_type = ARG_ANYTHING, |
|
}; |
|
|
|
BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx, |
|
u64, flags) |
|
{ |
|
bpf_callback_t callback = (bpf_callback_t)callback_fn; |
|
u64 ret; |
|
u32 i; |
|
|
|
/* Note: these safety checks are also verified when bpf_loop |
|
* is inlined, be careful to modify this code in sync. See |
|
* function verifier.c:inline_bpf_loop. |
|
*/ |
|
if (flags) |
|
return -EINVAL; |
|
if (nr_loops > BPF_MAX_LOOPS) |
|
return -E2BIG; |
|
|
|
for (i = 0; i < nr_loops; i++) { |
|
ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0); |
|
/* return value: 0 - continue, 1 - stop and return */ |
|
if (ret) |
|
return i + 1; |
|
} |
|
|
|
return i; |
|
} |
|
|
|
const struct bpf_func_proto bpf_loop_proto = { |
|
.func = bpf_loop, |
|
.gpl_only = false, |
|
.ret_type = RET_INTEGER, |
|
.arg1_type = ARG_ANYTHING, |
|
.arg2_type = ARG_PTR_TO_FUNC, |
|
.arg3_type = ARG_PTR_TO_STACK_OR_NULL, |
|
.arg4_type = ARG_ANYTHING, |
|
};
|
|
|