mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
565 lines
14 KiB
565 lines
14 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* Historical Service Time |
|
* |
|
* Keeps a time-weighted exponential moving average of the historical |
|
* service time. Estimates future service time based on the historical |
|
* service time and the number of outstanding requests. |
|
* |
|
* Marks paths stale if they have not finished within hst * |
|
* num_paths. If a path is stale and unused, we will send a single |
|
* request to probe in case the path has improved. This situation |
|
* generally arises if the path is so much worse than others that it |
|
* will never have the best estimated service time, or if the entire |
|
* multipath device is unused. If a path is stale and in use, limit the |
|
* number of requests it can receive with the assumption that the path |
|
* has become degraded. |
|
* |
|
* To avoid repeatedly calculating exponents for time weighting, times |
|
* are split into HST_WEIGHT_COUNT buckets each (1 >> HST_BUCKET_SHIFT) |
|
* ns, and the weighting is pre-calculated. |
|
* |
|
*/ |
|
|
|
#include "dm.h" |
|
#include "dm-path-selector.h" |
|
|
|
#include <linux/blkdev.h> |
|
#include <linux/slab.h> |
|
#include <linux/module.h> |
|
#include <linux/sched/clock.h> |
|
|
|
|
|
#define DM_MSG_PREFIX "multipath historical-service-time" |
|
#define HST_MIN_IO 1 |
|
#define HST_VERSION "0.1.1" |
|
|
|
#define HST_FIXED_SHIFT 10 /* 10 bits of decimal precision */ |
|
#define HST_FIXED_MAX (ULLONG_MAX >> HST_FIXED_SHIFT) |
|
#define HST_FIXED_1 (1 << HST_FIXED_SHIFT) |
|
#define HST_FIXED_95 972 |
|
#define HST_MAX_INFLIGHT HST_FIXED_1 |
|
#define HST_BUCKET_SHIFT 24 /* Buckets are ~ 16ms */ |
|
#define HST_WEIGHT_COUNT 64ULL |
|
|
|
struct selector { |
|
struct list_head valid_paths; |
|
struct list_head failed_paths; |
|
int valid_count; |
|
spinlock_t lock; |
|
|
|
unsigned int weights[HST_WEIGHT_COUNT]; |
|
unsigned int threshold_multiplier; |
|
}; |
|
|
|
struct path_info { |
|
struct list_head list; |
|
struct dm_path *path; |
|
unsigned int repeat_count; |
|
|
|
spinlock_t lock; |
|
|
|
u64 historical_service_time; /* Fixed point */ |
|
|
|
u64 stale_after; |
|
u64 last_finish; |
|
|
|
u64 outstanding; |
|
}; |
|
|
|
/** |
|
* fixed_power - compute: x^n, in O(log n) time |
|
* |
|
* @x: base of the power |
|
* @frac_bits: fractional bits of @x |
|
* @n: power to raise @x to. |
|
* |
|
* By exploiting the relation between the definition of the natural power |
|
* function: x^n := x*x*...*x (x multiplied by itself for n times), and |
|
* the binary encoding of numbers used by computers: n := \Sum n_i * 2^i, |
|
* (where: n_i \elem {0, 1}, the binary vector representing n), |
|
* we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is |
|
* of course trivially computable in O(log_2 n), the length of our binary |
|
* vector. |
|
* |
|
* (see: kernel/sched/loadavg.c) |
|
*/ |
|
static u64 fixed_power(u64 x, unsigned int frac_bits, unsigned int n) |
|
{ |
|
unsigned long result = 1UL << frac_bits; |
|
|
|
if (n) { |
|
for (;;) { |
|
if (n & 1) { |
|
result *= x; |
|
result += 1UL << (frac_bits - 1); |
|
result >>= frac_bits; |
|
} |
|
n >>= 1; |
|
if (!n) |
|
break; |
|
x *= x; |
|
x += 1UL << (frac_bits - 1); |
|
x >>= frac_bits; |
|
} |
|
} |
|
|
|
return result; |
|
} |
|
|
|
/* |
|
* Calculate the next value of an exponential moving average |
|
* a_1 = a_0 * e + a * (1 - e) |
|
* |
|
* @last: [0, ULLONG_MAX >> HST_FIXED_SHIFT] |
|
* @next: [0, ULLONG_MAX >> HST_FIXED_SHIFT] |
|
* @weight: [0, HST_FIXED_1] |
|
* |
|
* Note: |
|
* To account for multiple periods in the same calculation, |
|
* a_n = a_0 * e^n + a * (1 - e^n), |
|
* so call fixed_ema(last, next, pow(weight, N)) |
|
*/ |
|
static u64 fixed_ema(u64 last, u64 next, u64 weight) |
|
{ |
|
last *= weight; |
|
last += next * (HST_FIXED_1 - weight); |
|
last += 1ULL << (HST_FIXED_SHIFT - 1); |
|
return last >> HST_FIXED_SHIFT; |
|
} |
|
|
|
static struct selector *alloc_selector(void) |
|
{ |
|
struct selector *s = kmalloc(sizeof(*s), GFP_KERNEL); |
|
|
|
if (s) { |
|
INIT_LIST_HEAD(&s->valid_paths); |
|
INIT_LIST_HEAD(&s->failed_paths); |
|
spin_lock_init(&s->lock); |
|
s->valid_count = 0; |
|
} |
|
|
|
return s; |
|
} |
|
|
|
/* |
|
* Get the weight for a given time span. |
|
*/ |
|
static u64 hst_weight(struct path_selector *ps, u64 delta) |
|
{ |
|
struct selector *s = ps->context; |
|
int bucket = clamp(delta >> HST_BUCKET_SHIFT, 0ULL, |
|
HST_WEIGHT_COUNT - 1); |
|
|
|
return s->weights[bucket]; |
|
} |
|
|
|
/* |
|
* Set up the weights array. |
|
* |
|
* weights[len-1] = 0 |
|
* weights[n] = base ^ (n + 1) |
|
*/ |
|
static void hst_set_weights(struct path_selector *ps, unsigned int base) |
|
{ |
|
struct selector *s = ps->context; |
|
int i; |
|
|
|
if (base >= HST_FIXED_1) |
|
return; |
|
|
|
for (i = 0; i < HST_WEIGHT_COUNT - 1; i++) |
|
s->weights[i] = fixed_power(base, HST_FIXED_SHIFT, i + 1); |
|
s->weights[HST_WEIGHT_COUNT - 1] = 0; |
|
} |
|
|
|
static int hst_create(struct path_selector *ps, unsigned int argc, char **argv) |
|
{ |
|
struct selector *s; |
|
unsigned int base_weight = HST_FIXED_95; |
|
unsigned int threshold_multiplier = 0; |
|
char dummy; |
|
|
|
/* |
|
* Arguments: [<base_weight> [<threshold_multiplier>]] |
|
* <base_weight>: Base weight for ema [0, 1024) 10-bit fixed point. A |
|
* value of 0 will completely ignore any history. |
|
* If not given, default (HST_FIXED_95) is used. |
|
* <threshold_multiplier>: Minimum threshold multiplier for paths to |
|
* be considered different. That is, a path is |
|
* considered different iff (p1 > N * p2) where p1 |
|
* is the path with higher service time. A threshold |
|
* of 1 or 0 has no effect. Defaults to 0. |
|
*/ |
|
if (argc > 2) |
|
return -EINVAL; |
|
|
|
if (argc && (sscanf(argv[0], "%u%c", &base_weight, &dummy) != 1 || |
|
base_weight >= HST_FIXED_1)) { |
|
return -EINVAL; |
|
} |
|
|
|
if (argc > 1 && (sscanf(argv[1], "%u%c", |
|
&threshold_multiplier, &dummy) != 1)) { |
|
return -EINVAL; |
|
} |
|
|
|
s = alloc_selector(); |
|
if (!s) |
|
return -ENOMEM; |
|
|
|
ps->context = s; |
|
|
|
hst_set_weights(ps, base_weight); |
|
s->threshold_multiplier = threshold_multiplier; |
|
return 0; |
|
} |
|
|
|
static void free_paths(struct list_head *paths) |
|
{ |
|
struct path_info *pi, *next; |
|
|
|
list_for_each_entry_safe(pi, next, paths, list) { |
|
list_del(&pi->list); |
|
kfree(pi); |
|
} |
|
} |
|
|
|
static void hst_destroy(struct path_selector *ps) |
|
{ |
|
struct selector *s = ps->context; |
|
|
|
free_paths(&s->valid_paths); |
|
free_paths(&s->failed_paths); |
|
kfree(s); |
|
ps->context = NULL; |
|
} |
|
|
|
static int hst_status(struct path_selector *ps, struct dm_path *path, |
|
status_type_t type, char *result, unsigned int maxlen) |
|
{ |
|
unsigned int sz = 0; |
|
struct path_info *pi; |
|
|
|
if (!path) { |
|
struct selector *s = ps->context; |
|
|
|
DMEMIT("2 %u %u ", s->weights[0], s->threshold_multiplier); |
|
} else { |
|
pi = path->pscontext; |
|
|
|
switch (type) { |
|
case STATUSTYPE_INFO: |
|
DMEMIT("%llu %llu %llu ", pi->historical_service_time, |
|
pi->outstanding, pi->stale_after); |
|
break; |
|
case STATUSTYPE_TABLE: |
|
DMEMIT("0 "); |
|
break; |
|
case STATUSTYPE_IMA: |
|
*result = '\0'; |
|
break; |
|
} |
|
} |
|
|
|
return sz; |
|
} |
|
|
|
static int hst_add_path(struct path_selector *ps, struct dm_path *path, |
|
int argc, char **argv, char **error) |
|
{ |
|
struct selector *s = ps->context; |
|
struct path_info *pi; |
|
unsigned int repeat_count = HST_MIN_IO; |
|
char dummy; |
|
unsigned long flags; |
|
|
|
/* |
|
* Arguments: [<repeat_count>] |
|
* <repeat_count>: The number of I/Os before switching path. |
|
* If not given, default (HST_MIN_IO) is used. |
|
*/ |
|
if (argc > 1) { |
|
*error = "historical-service-time ps: incorrect number of arguments"; |
|
return -EINVAL; |
|
} |
|
|
|
if (argc && (sscanf(argv[0], "%u%c", &repeat_count, &dummy) != 1)) { |
|
*error = "historical-service-time ps: invalid repeat count"; |
|
return -EINVAL; |
|
} |
|
|
|
/* allocate the path */ |
|
pi = kmalloc(sizeof(*pi), GFP_KERNEL); |
|
if (!pi) { |
|
*error = "historical-service-time ps: Error allocating path context"; |
|
return -ENOMEM; |
|
} |
|
|
|
pi->path = path; |
|
pi->repeat_count = repeat_count; |
|
|
|
pi->historical_service_time = HST_FIXED_1; |
|
|
|
spin_lock_init(&pi->lock); |
|
pi->outstanding = 0; |
|
|
|
pi->stale_after = 0; |
|
pi->last_finish = 0; |
|
|
|
path->pscontext = pi; |
|
|
|
spin_lock_irqsave(&s->lock, flags); |
|
list_add_tail(&pi->list, &s->valid_paths); |
|
s->valid_count++; |
|
spin_unlock_irqrestore(&s->lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static void hst_fail_path(struct path_selector *ps, struct dm_path *path) |
|
{ |
|
struct selector *s = ps->context; |
|
struct path_info *pi = path->pscontext; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&s->lock, flags); |
|
list_move(&pi->list, &s->failed_paths); |
|
s->valid_count--; |
|
spin_unlock_irqrestore(&s->lock, flags); |
|
} |
|
|
|
static int hst_reinstate_path(struct path_selector *ps, struct dm_path *path) |
|
{ |
|
struct selector *s = ps->context; |
|
struct path_info *pi = path->pscontext; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&s->lock, flags); |
|
list_move_tail(&pi->list, &s->valid_paths); |
|
s->valid_count++; |
|
spin_unlock_irqrestore(&s->lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static void hst_fill_compare(struct path_info *pi, u64 *hst, |
|
u64 *out, u64 *stale) |
|
{ |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&pi->lock, flags); |
|
*hst = pi->historical_service_time; |
|
*out = pi->outstanding; |
|
*stale = pi->stale_after; |
|
spin_unlock_irqrestore(&pi->lock, flags); |
|
} |
|
|
|
/* |
|
* Compare the estimated service time of 2 paths, pi1 and pi2, |
|
* for the incoming I/O. |
|
* |
|
* Returns: |
|
* < 0 : pi1 is better |
|
* 0 : no difference between pi1 and pi2 |
|
* > 0 : pi2 is better |
|
* |
|
*/ |
|
static long long hst_compare(struct path_info *pi1, struct path_info *pi2, |
|
u64 time_now, struct path_selector *ps) |
|
{ |
|
struct selector *s = ps->context; |
|
u64 hst1, hst2; |
|
long long out1, out2, stale1, stale2; |
|
int pi2_better, over_threshold; |
|
|
|
hst_fill_compare(pi1, &hst1, &out1, &stale1); |
|
hst_fill_compare(pi2, &hst2, &out2, &stale2); |
|
|
|
/* Check here if estimated latency for two paths are too similar. |
|
* If this is the case, we skip extra calculation and just compare |
|
* outstanding requests. In this case, any unloaded paths will |
|
* be preferred. |
|
*/ |
|
if (hst1 > hst2) |
|
over_threshold = hst1 > (s->threshold_multiplier * hst2); |
|
else |
|
over_threshold = hst2 > (s->threshold_multiplier * hst1); |
|
|
|
if (!over_threshold) |
|
return out1 - out2; |
|
|
|
/* |
|
* If an unloaded path is stale, choose it. If both paths are unloaded, |
|
* choose path that is the most stale. |
|
* (If one path is loaded, choose the other) |
|
*/ |
|
if ((!out1 && stale1 < time_now) || (!out2 && stale2 < time_now) || |
|
(!out1 && !out2)) |
|
return (!out2 * stale1) - (!out1 * stale2); |
|
|
|
/* Compare estimated service time. If outstanding is the same, we |
|
* don't need to multiply |
|
*/ |
|
if (out1 == out2) { |
|
pi2_better = hst1 > hst2; |
|
} else { |
|
/* Potential overflow with out >= 1024 */ |
|
if (unlikely(out1 >= HST_MAX_INFLIGHT || |
|
out2 >= HST_MAX_INFLIGHT)) { |
|
/* If over 1023 in-flights, we may overflow if hst |
|
* is at max. (With this shift we still overflow at |
|
* 1048576 in-flights, which is high enough). |
|
*/ |
|
hst1 >>= HST_FIXED_SHIFT; |
|
hst2 >>= HST_FIXED_SHIFT; |
|
} |
|
pi2_better = (1 + out1) * hst1 > (1 + out2) * hst2; |
|
} |
|
|
|
/* In the case that the 'winner' is stale, limit to equal usage. */ |
|
if (pi2_better) { |
|
if (stale2 < time_now) |
|
return out1 - out2; |
|
return 1; |
|
} |
|
if (stale1 < time_now) |
|
return out1 - out2; |
|
return -1; |
|
} |
|
|
|
static struct dm_path *hst_select_path(struct path_selector *ps, |
|
size_t nr_bytes) |
|
{ |
|
struct selector *s = ps->context; |
|
struct path_info *pi = NULL, *best = NULL; |
|
u64 time_now = sched_clock(); |
|
struct dm_path *ret = NULL; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&s->lock, flags); |
|
if (list_empty(&s->valid_paths)) |
|
goto out; |
|
|
|
list_for_each_entry(pi, &s->valid_paths, list) { |
|
if (!best || (hst_compare(pi, best, time_now, ps) < 0)) |
|
best = pi; |
|
} |
|
|
|
if (!best) |
|
goto out; |
|
|
|
/* Move last used path to end (least preferred in case of ties) */ |
|
list_move_tail(&best->list, &s->valid_paths); |
|
|
|
ret = best->path; |
|
|
|
out: |
|
spin_unlock_irqrestore(&s->lock, flags); |
|
return ret; |
|
} |
|
|
|
static int hst_start_io(struct path_selector *ps, struct dm_path *path, |
|
size_t nr_bytes) |
|
{ |
|
struct path_info *pi = path->pscontext; |
|
unsigned long flags; |
|
|
|
spin_lock_irqsave(&pi->lock, flags); |
|
pi->outstanding++; |
|
spin_unlock_irqrestore(&pi->lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static u64 path_service_time(struct path_info *pi, u64 start_time) |
|
{ |
|
u64 sched_now = ktime_get_ns(); |
|
|
|
/* if a previous disk request has finished after this IO was |
|
* sent to the hardware, pretend the submission happened |
|
* serially. |
|
*/ |
|
if (time_after64(pi->last_finish, start_time)) |
|
start_time = pi->last_finish; |
|
|
|
pi->last_finish = sched_now; |
|
if (time_before64(sched_now, start_time)) |
|
return 0; |
|
|
|
return sched_now - start_time; |
|
} |
|
|
|
static int hst_end_io(struct path_selector *ps, struct dm_path *path, |
|
size_t nr_bytes, u64 start_time) |
|
{ |
|
struct path_info *pi = path->pscontext; |
|
struct selector *s = ps->context; |
|
unsigned long flags; |
|
u64 st; |
|
|
|
spin_lock_irqsave(&pi->lock, flags); |
|
|
|
st = path_service_time(pi, start_time); |
|
pi->outstanding--; |
|
pi->historical_service_time = |
|
fixed_ema(pi->historical_service_time, |
|
min(st * HST_FIXED_1, HST_FIXED_MAX), |
|
hst_weight(ps, st)); |
|
|
|
/* |
|
* On request end, mark path as fresh. If a path hasn't |
|
* finished any requests within the fresh period, the estimated |
|
* service time is considered too optimistic and we limit the |
|
* maximum requests on that path. |
|
*/ |
|
pi->stale_after = pi->last_finish + |
|
(s->valid_count * (pi->historical_service_time >> HST_FIXED_SHIFT)); |
|
|
|
spin_unlock_irqrestore(&pi->lock, flags); |
|
|
|
return 0; |
|
} |
|
|
|
static struct path_selector_type hst_ps = { |
|
.name = "historical-service-time", |
|
.module = THIS_MODULE, |
|
.table_args = 1, |
|
.info_args = 3, |
|
.create = hst_create, |
|
.destroy = hst_destroy, |
|
.status = hst_status, |
|
.add_path = hst_add_path, |
|
.fail_path = hst_fail_path, |
|
.reinstate_path = hst_reinstate_path, |
|
.select_path = hst_select_path, |
|
.start_io = hst_start_io, |
|
.end_io = hst_end_io, |
|
}; |
|
|
|
static int __init dm_hst_init(void) |
|
{ |
|
int r = dm_register_path_selector(&hst_ps); |
|
|
|
if (r < 0) |
|
DMERR("register failed %d", r); |
|
|
|
DMINFO("version " HST_VERSION " loaded"); |
|
|
|
return r; |
|
} |
|
|
|
static void __exit dm_hst_exit(void) |
|
{ |
|
int r = dm_unregister_path_selector(&hst_ps); |
|
|
|
if (r < 0) |
|
DMERR("unregister failed %d", r); |
|
} |
|
|
|
module_init(dm_hst_init); |
|
module_exit(dm_hst_exit); |
|
|
|
MODULE_DESCRIPTION(DM_NAME " measured service time oriented path selector"); |
|
MODULE_AUTHOR("Khazhismel Kumykov <[email protected]>"); |
|
MODULE_LICENSE("GPL");
|
|
|