forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
221 lines
4.8 KiB
221 lines
4.8 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* Block stat tracking code |
|
* |
|
* Copyright (C) 2016 Jens Axboe |
|
*/ |
|
#include <linux/kernel.h> |
|
#include <linux/rculist.h> |
|
#include <linux/blk-mq.h> |
|
|
|
#include "blk-stat.h" |
|
#include "blk-mq.h" |
|
#include "blk.h" |
|
|
|
struct blk_queue_stats { |
|
struct list_head callbacks; |
|
spinlock_t lock; |
|
bool enable_accounting; |
|
}; |
|
|
|
void blk_rq_stat_init(struct blk_rq_stat *stat) |
|
{ |
|
stat->min = -1ULL; |
|
stat->max = stat->nr_samples = stat->mean = 0; |
|
stat->batch = 0; |
|
} |
|
|
|
/* src is a per-cpu stat, mean isn't initialized */ |
|
void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) |
|
{ |
|
if (!src->nr_samples) |
|
return; |
|
|
|
dst->min = min(dst->min, src->min); |
|
dst->max = max(dst->max, src->max); |
|
|
|
dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples, |
|
dst->nr_samples + src->nr_samples); |
|
|
|
dst->nr_samples += src->nr_samples; |
|
} |
|
|
|
void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value) |
|
{ |
|
stat->min = min(stat->min, value); |
|
stat->max = max(stat->max, value); |
|
stat->batch += value; |
|
stat->nr_samples++; |
|
} |
|
|
|
void blk_stat_add(struct request *rq, u64 now) |
|
{ |
|
struct request_queue *q = rq->q; |
|
struct blk_stat_callback *cb; |
|
struct blk_rq_stat *stat; |
|
int bucket, cpu; |
|
u64 value; |
|
|
|
value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0; |
|
|
|
blk_throtl_stat_add(rq, value); |
|
|
|
rcu_read_lock(); |
|
cpu = get_cpu(); |
|
list_for_each_entry_rcu(cb, &q->stats->callbacks, list) { |
|
if (!blk_stat_is_active(cb)) |
|
continue; |
|
|
|
bucket = cb->bucket_fn(rq); |
|
if (bucket < 0) |
|
continue; |
|
|
|
stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket]; |
|
blk_rq_stat_add(stat, value); |
|
} |
|
put_cpu(); |
|
rcu_read_unlock(); |
|
} |
|
|
|
static void blk_stat_timer_fn(struct timer_list *t) |
|
{ |
|
struct blk_stat_callback *cb = from_timer(cb, t, timer); |
|
unsigned int bucket; |
|
int cpu; |
|
|
|
for (bucket = 0; bucket < cb->buckets; bucket++) |
|
blk_rq_stat_init(&cb->stat[bucket]); |
|
|
|
for_each_online_cpu(cpu) { |
|
struct blk_rq_stat *cpu_stat; |
|
|
|
cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
|
for (bucket = 0; bucket < cb->buckets; bucket++) { |
|
blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]); |
|
blk_rq_stat_init(&cpu_stat[bucket]); |
|
} |
|
} |
|
|
|
cb->timer_fn(cb); |
|
} |
|
|
|
struct blk_stat_callback * |
|
blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), |
|
int (*bucket_fn)(const struct request *), |
|
unsigned int buckets, void *data) |
|
{ |
|
struct blk_stat_callback *cb; |
|
|
|
cb = kmalloc(sizeof(*cb), GFP_KERNEL); |
|
if (!cb) |
|
return NULL; |
|
|
|
cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat), |
|
GFP_KERNEL); |
|
if (!cb->stat) { |
|
kfree(cb); |
|
return NULL; |
|
} |
|
cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat), |
|
__alignof__(struct blk_rq_stat)); |
|
if (!cb->cpu_stat) { |
|
kfree(cb->stat); |
|
kfree(cb); |
|
return NULL; |
|
} |
|
|
|
cb->timer_fn = timer_fn; |
|
cb->bucket_fn = bucket_fn; |
|
cb->data = data; |
|
cb->buckets = buckets; |
|
timer_setup(&cb->timer, blk_stat_timer_fn, 0); |
|
|
|
return cb; |
|
} |
|
|
|
void blk_stat_add_callback(struct request_queue *q, |
|
struct blk_stat_callback *cb) |
|
{ |
|
unsigned int bucket; |
|
unsigned long flags; |
|
int cpu; |
|
|
|
for_each_possible_cpu(cpu) { |
|
struct blk_rq_stat *cpu_stat; |
|
|
|
cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); |
|
for (bucket = 0; bucket < cb->buckets; bucket++) |
|
blk_rq_stat_init(&cpu_stat[bucket]); |
|
} |
|
|
|
spin_lock_irqsave(&q->stats->lock, flags); |
|
list_add_tail_rcu(&cb->list, &q->stats->callbacks); |
|
blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
|
spin_unlock_irqrestore(&q->stats->lock, flags); |
|
} |
|
|
|
void blk_stat_remove_callback(struct request_queue *q, |
|
struct blk_stat_callback *cb) |
|
{ |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&q->stats->lock, flags); |
|
list_del_rcu(&cb->list); |
|
if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting) |
|
blk_queue_flag_clear(QUEUE_FLAG_STATS, q); |
|
spin_unlock_irqrestore(&q->stats->lock, flags); |
|
|
|
del_timer_sync(&cb->timer); |
|
} |
|
|
|
static void blk_stat_free_callback_rcu(struct rcu_head *head) |
|
{ |
|
struct blk_stat_callback *cb; |
|
|
|
cb = container_of(head, struct blk_stat_callback, rcu); |
|
free_percpu(cb->cpu_stat); |
|
kfree(cb->stat); |
|
kfree(cb); |
|
} |
|
|
|
void blk_stat_free_callback(struct blk_stat_callback *cb) |
|
{ |
|
if (cb) |
|
call_rcu(&cb->rcu, blk_stat_free_callback_rcu); |
|
} |
|
|
|
void blk_stat_enable_accounting(struct request_queue *q) |
|
{ |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&q->stats->lock, flags); |
|
q->stats->enable_accounting = true; |
|
blk_queue_flag_set(QUEUE_FLAG_STATS, q); |
|
spin_unlock_irqrestore(&q->stats->lock, flags); |
|
} |
|
EXPORT_SYMBOL_GPL(blk_stat_enable_accounting); |
|
|
|
struct blk_queue_stats *blk_alloc_queue_stats(void) |
|
{ |
|
struct blk_queue_stats *stats; |
|
|
|
stats = kmalloc(sizeof(*stats), GFP_KERNEL); |
|
if (!stats) |
|
return NULL; |
|
|
|
INIT_LIST_HEAD(&stats->callbacks); |
|
spin_lock_init(&stats->lock); |
|
stats->enable_accounting = false; |
|
|
|
return stats; |
|
} |
|
|
|
void blk_free_queue_stats(struct blk_queue_stats *stats) |
|
{ |
|
if (!stats) |
|
return; |
|
|
|
WARN_ON(!list_empty(&stats->callbacks)); |
|
|
|
kfree(stats); |
|
}
|
|
|