mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
520 lines
15 KiB
520 lines
15 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* Volume-level cache cookie handling. |
|
* |
|
* Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. |
|
* Written by David Howells ([email protected]) |
|
*/ |
|
|
|
#define FSCACHE_DEBUG_LEVEL COOKIE |
|
#include <linux/export.h> |
|
#include <linux/slab.h> |
|
#include "internal.h" |
|
|
|
#define fscache_volume_hash_shift 10 |
|
static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; |
|
static atomic_t fscache_volume_debug_id; |
|
static LIST_HEAD(fscache_volumes); |
|
|
|
static void fscache_create_volume_work(struct work_struct *work); |
|
|
|
struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, |
|
enum fscache_volume_trace where) |
|
{ |
|
int ref; |
|
|
|
__refcount_inc(&volume->ref, &ref); |
|
trace_fscache_volume(volume->debug_id, ref + 1, where); |
|
return volume; |
|
} |
|
|
|
static void fscache_see_volume(struct fscache_volume *volume, |
|
enum fscache_volume_trace where) |
|
{ |
|
int ref = refcount_read(&volume->ref); |
|
|
|
trace_fscache_volume(volume->debug_id, ref, where); |
|
} |
|
|
|
/* |
|
* Pin the cache behind a volume so that we can access it. |
|
*/ |
|
static void __fscache_begin_volume_access(struct fscache_volume *volume, |
|
struct fscache_cookie *cookie, |
|
enum fscache_access_trace why) |
|
{ |
|
int n_accesses; |
|
|
|
n_accesses = atomic_inc_return(&volume->n_accesses); |
|
smp_mb__after_atomic(); |
|
trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, |
|
refcount_read(&volume->ref), |
|
n_accesses, why); |
|
} |
|
|
|
/** |
|
* fscache_begin_volume_access - Pin a cache so a volume can be accessed |
|
* @volume: The volume cookie |
|
* @cookie: A datafile cookie for a tracing reference (or NULL) |
|
* @why: An indication of the circumstances of the access for tracing |
|
* |
|
* Attempt to pin the cache to prevent it from going away whilst we're |
|
* accessing a volume and returns true if successful. This works as follows: |
|
* |
|
* (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), |
|
* then we return false to indicate access was not permitted. |
|
* |
|
* (2) If the cache tests as live, then we increment the volume's n_accesses |
|
* count and then recheck the cache liveness, ending the access if it |
|
* ceased to be live. |
|
* |
|
* (3) When we end the access, we decrement the volume's n_accesses and wake |
|
* up the any waiters if it reaches 0. |
|
* |
|
* (4) Whilst the cache is caching, the volume's n_accesses is kept |
|
* artificially incremented to prevent wakeups from happening. |
|
* |
|
* (5) When the cache is taken offline, the state is changed to prevent new |
|
* accesses, the volume's n_accesses is decremented and we wait for it to |
|
* become 0. |
|
* |
|
* The datafile @cookie and the @why indicator are merely provided for tracing |
|
* purposes. |
|
*/ |
|
bool fscache_begin_volume_access(struct fscache_volume *volume, |
|
struct fscache_cookie *cookie, |
|
enum fscache_access_trace why) |
|
{ |
|
if (!fscache_cache_is_live(volume->cache)) |
|
return false; |
|
__fscache_begin_volume_access(volume, cookie, why); |
|
if (!fscache_cache_is_live(volume->cache)) { |
|
fscache_end_volume_access(volume, cookie, fscache_access_unlive); |
|
return false; |
|
} |
|
return true; |
|
} |
|
|
|
/** |
|
* fscache_end_volume_access - Unpin a cache at the end of an access. |
|
* @volume: The volume cookie |
|
* @cookie: A datafile cookie for a tracing reference (or NULL) |
|
* @why: An indication of the circumstances of the access for tracing |
|
* |
|
* Unpin a cache volume after we've accessed it. The datafile @cookie and the |
|
* @why indicator are merely provided for tracing purposes. |
|
*/ |
|
void fscache_end_volume_access(struct fscache_volume *volume, |
|
struct fscache_cookie *cookie, |
|
enum fscache_access_trace why) |
|
{ |
|
int n_accesses; |
|
|
|
smp_mb__before_atomic(); |
|
n_accesses = atomic_dec_return(&volume->n_accesses); |
|
trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, |
|
refcount_read(&volume->ref), |
|
n_accesses, why); |
|
if (n_accesses == 0) |
|
wake_up_var(&volume->n_accesses); |
|
} |
|
EXPORT_SYMBOL(fscache_end_volume_access); |
|
|
|
static bool fscache_volume_same(const struct fscache_volume *a, |
|
const struct fscache_volume *b) |
|
{ |
|
size_t klen; |
|
|
|
if (a->key_hash != b->key_hash || |
|
a->cache != b->cache || |
|
a->key[0] != b->key[0]) |
|
return false; |
|
|
|
klen = round_up(a->key[0] + 1, sizeof(__le32)); |
|
return memcmp(a->key, b->key, klen) == 0; |
|
} |
|
|
|
static bool fscache_is_acquire_pending(struct fscache_volume *volume) |
|
{ |
|
return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); |
|
} |
|
|
|
static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, |
|
unsigned int collidee_debug_id) |
|
{ |
|
wait_var_event_timeout(&candidate->flags, |
|
!fscache_is_acquire_pending(candidate), 20 * HZ); |
|
if (fscache_is_acquire_pending(candidate)) { |
|
pr_notice("Potential volume collision new=%08x old=%08x", |
|
candidate->debug_id, collidee_debug_id); |
|
fscache_stat(&fscache_n_volumes_collision); |
|
wait_var_event(&candidate->flags, !fscache_is_acquire_pending(candidate)); |
|
} |
|
} |
|
|
|
/* |
|
* Attempt to insert the new volume into the hash. If there's a collision, we |
|
* wait for the old volume to complete if it's being relinquished and an error |
|
* otherwise. |
|
*/ |
|
static bool fscache_hash_volume(struct fscache_volume *candidate) |
|
{ |
|
struct fscache_volume *cursor; |
|
struct hlist_bl_head *h; |
|
struct hlist_bl_node *p; |
|
unsigned int bucket, collidee_debug_id = 0; |
|
|
|
bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); |
|
h = &fscache_volume_hash[bucket]; |
|
|
|
hlist_bl_lock(h); |
|
hlist_bl_for_each_entry(cursor, p, h, hash_link) { |
|
if (fscache_volume_same(candidate, cursor)) { |
|
if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) |
|
goto collision; |
|
fscache_see_volume(cursor, fscache_volume_get_hash_collision); |
|
set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); |
|
set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); |
|
collidee_debug_id = cursor->debug_id; |
|
break; |
|
} |
|
} |
|
|
|
hlist_bl_add_head(&candidate->hash_link, h); |
|
hlist_bl_unlock(h); |
|
|
|
if (fscache_is_acquire_pending(candidate)) |
|
fscache_wait_on_volume_collision(candidate, collidee_debug_id); |
|
return true; |
|
|
|
collision: |
|
fscache_see_volume(cursor, fscache_volume_collision); |
|
hlist_bl_unlock(h); |
|
return false; |
|
} |
|
|
|
/* |
|
* Allocate and initialise a volume representation cookie. |
|
*/ |
|
static struct fscache_volume *fscache_alloc_volume(const char *volume_key, |
|
const char *cache_name, |
|
const void *coherency_data, |
|
size_t coherency_len) |
|
{ |
|
struct fscache_volume *volume; |
|
struct fscache_cache *cache; |
|
size_t klen, hlen; |
|
u8 *key; |
|
|
|
klen = strlen(volume_key); |
|
if (klen > NAME_MAX) |
|
return NULL; |
|
|
|
if (!coherency_data) |
|
coherency_len = 0; |
|
|
|
cache = fscache_lookup_cache(cache_name, false); |
|
if (IS_ERR(cache)) |
|
return NULL; |
|
|
|
volume = kzalloc(struct_size(volume, coherency, coherency_len), |
|
GFP_KERNEL); |
|
if (!volume) |
|
goto err_cache; |
|
|
|
volume->cache = cache; |
|
volume->coherency_len = coherency_len; |
|
if (coherency_data) |
|
memcpy(volume->coherency, coherency_data, coherency_len); |
|
INIT_LIST_HEAD(&volume->proc_link); |
|
INIT_WORK(&volume->work, fscache_create_volume_work); |
|
refcount_set(&volume->ref, 1); |
|
spin_lock_init(&volume->lock); |
|
|
|
/* Stick the length on the front of the key and pad it out to make |
|
* hashing easier. |
|
*/ |
|
hlen = round_up(1 + klen + 1, sizeof(__le32)); |
|
key = kzalloc(hlen, GFP_KERNEL); |
|
if (!key) |
|
goto err_vol; |
|
key[0] = klen; |
|
memcpy(key + 1, volume_key, klen); |
|
|
|
volume->key = key; |
|
volume->key_hash = fscache_hash(0, key, hlen); |
|
|
|
volume->debug_id = atomic_inc_return(&fscache_volume_debug_id); |
|
down_write(&fscache_addremove_sem); |
|
atomic_inc(&cache->n_volumes); |
|
list_add_tail(&volume->proc_link, &fscache_volumes); |
|
fscache_see_volume(volume, fscache_volume_new_acquire); |
|
fscache_stat(&fscache_n_volumes); |
|
up_write(&fscache_addremove_sem); |
|
_leave(" = v=%x", volume->debug_id); |
|
return volume; |
|
|
|
err_vol: |
|
kfree(volume); |
|
err_cache: |
|
fscache_put_cache(cache, fscache_cache_put_alloc_volume); |
|
fscache_stat(&fscache_n_volumes_nomem); |
|
return NULL; |
|
} |
|
|
|
/* |
|
* Create a volume's representation on disk. Have a volume ref and a cache |
|
* access we have to release. |
|
*/ |
|
static void fscache_create_volume_work(struct work_struct *work) |
|
{ |
|
const struct fscache_cache_ops *ops; |
|
struct fscache_volume *volume = |
|
container_of(work, struct fscache_volume, work); |
|
|
|
fscache_see_volume(volume, fscache_volume_see_create_work); |
|
|
|
ops = volume->cache->ops; |
|
if (ops->acquire_volume) |
|
ops->acquire_volume(volume); |
|
fscache_end_cache_access(volume->cache, |
|
fscache_access_acquire_volume_end); |
|
|
|
clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); |
|
wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); |
|
fscache_put_volume(volume, fscache_volume_put_create_work); |
|
} |
|
|
|
/* |
|
* Dispatch a worker thread to create a volume's representation on disk. |
|
*/ |
|
void fscache_create_volume(struct fscache_volume *volume, bool wait) |
|
{ |
|
if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags)) |
|
goto maybe_wait; |
|
if (volume->cache_priv) |
|
goto no_wait; /* We raced */ |
|
if (!fscache_begin_cache_access(volume->cache, |
|
fscache_access_acquire_volume)) |
|
goto no_wait; |
|
|
|
fscache_get_volume(volume, fscache_volume_get_create_work); |
|
if (!schedule_work(&volume->work)) |
|
fscache_put_volume(volume, fscache_volume_put_create_work); |
|
|
|
maybe_wait: |
|
if (wait) { |
|
fscache_see_volume(volume, fscache_volume_wait_create_work); |
|
wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING, |
|
TASK_UNINTERRUPTIBLE); |
|
} |
|
return; |
|
no_wait: |
|
clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); |
|
wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); |
|
} |
|
|
|
/* |
|
* Acquire a volume representation cookie and link it to a (proposed) cache. |
|
*/ |
|
struct fscache_volume *__fscache_acquire_volume(const char *volume_key, |
|
const char *cache_name, |
|
const void *coherency_data, |
|
size_t coherency_len) |
|
{ |
|
struct fscache_volume *volume; |
|
|
|
volume = fscache_alloc_volume(volume_key, cache_name, |
|
coherency_data, coherency_len); |
|
if (!volume) |
|
return ERR_PTR(-ENOMEM); |
|
|
|
if (!fscache_hash_volume(volume)) { |
|
fscache_put_volume(volume, fscache_volume_put_hash_collision); |
|
return ERR_PTR(-EBUSY); |
|
} |
|
|
|
fscache_create_volume(volume, false); |
|
return volume; |
|
} |
|
EXPORT_SYMBOL(__fscache_acquire_volume); |
|
|
|
static void fscache_wake_pending_volume(struct fscache_volume *volume, |
|
struct hlist_bl_head *h) |
|
{ |
|
struct fscache_volume *cursor; |
|
struct hlist_bl_node *p; |
|
|
|
hlist_bl_for_each_entry(cursor, p, h, hash_link) { |
|
if (fscache_volume_same(cursor, volume)) { |
|
fscache_see_volume(cursor, fscache_volume_see_hash_wake); |
|
clear_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &cursor->flags); |
|
wake_up_bit(&cursor->flags, FSCACHE_VOLUME_ACQUIRE_PENDING); |
|
return; |
|
} |
|
} |
|
} |
|
|
|
/* |
|
* Remove a volume cookie from the hash table. |
|
*/ |
|
static void fscache_unhash_volume(struct fscache_volume *volume) |
|
{ |
|
struct hlist_bl_head *h; |
|
unsigned int bucket; |
|
|
|
bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); |
|
h = &fscache_volume_hash[bucket]; |
|
|
|
hlist_bl_lock(h); |
|
hlist_bl_del(&volume->hash_link); |
|
if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) |
|
fscache_wake_pending_volume(volume, h); |
|
hlist_bl_unlock(h); |
|
} |
|
|
|
/* |
|
* Drop a cache's volume attachments. |
|
*/ |
|
static void fscache_free_volume(struct fscache_volume *volume) |
|
{ |
|
struct fscache_cache *cache = volume->cache; |
|
|
|
if (volume->cache_priv) { |
|
__fscache_begin_volume_access(volume, NULL, |
|
fscache_access_relinquish_volume); |
|
if (volume->cache_priv) |
|
cache->ops->free_volume(volume); |
|
fscache_end_volume_access(volume, NULL, |
|
fscache_access_relinquish_volume_end); |
|
} |
|
|
|
down_write(&fscache_addremove_sem); |
|
list_del_init(&volume->proc_link); |
|
atomic_dec(&volume->cache->n_volumes); |
|
up_write(&fscache_addremove_sem); |
|
|
|
if (!hlist_bl_unhashed(&volume->hash_link)) |
|
fscache_unhash_volume(volume); |
|
|
|
trace_fscache_volume(volume->debug_id, 0, fscache_volume_free); |
|
kfree(volume->key); |
|
kfree(volume); |
|
fscache_stat_d(&fscache_n_volumes); |
|
fscache_put_cache(cache, fscache_cache_put_volume); |
|
} |
|
|
|
/* |
|
* Drop a reference to a volume cookie. |
|
*/ |
|
void fscache_put_volume(struct fscache_volume *volume, |
|
enum fscache_volume_trace where) |
|
{ |
|
if (volume) { |
|
unsigned int debug_id = volume->debug_id; |
|
bool zero; |
|
int ref; |
|
|
|
zero = __refcount_dec_and_test(&volume->ref, &ref); |
|
trace_fscache_volume(debug_id, ref - 1, where); |
|
if (zero) |
|
fscache_free_volume(volume); |
|
} |
|
} |
|
|
|
/* |
|
* Relinquish a volume representation cookie. |
|
*/ |
|
void __fscache_relinquish_volume(struct fscache_volume *volume, |
|
const void *coherency_data, |
|
bool invalidate) |
|
{ |
|
if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) |
|
return; |
|
|
|
if (invalidate) { |
|
set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); |
|
} else if (coherency_data) { |
|
memcpy(volume->coherency, coherency_data, volume->coherency_len); |
|
} |
|
|
|
fscache_put_volume(volume, fscache_volume_put_relinquish); |
|
} |
|
EXPORT_SYMBOL(__fscache_relinquish_volume); |
|
|
|
/** |
|
* fscache_withdraw_volume - Withdraw a volume from being cached |
|
* @volume: Volume cookie |
|
* |
|
* Withdraw a cache volume from service, waiting for all accesses to complete |
|
* before returning. |
|
*/ |
|
void fscache_withdraw_volume(struct fscache_volume *volume) |
|
{ |
|
int n_accesses; |
|
|
|
_debug("withdraw V=%x", volume->debug_id); |
|
|
|
/* Allow wakeups on dec-to-0 */ |
|
n_accesses = atomic_dec_return(&volume->n_accesses); |
|
trace_fscache_access_volume(volume->debug_id, 0, |
|
refcount_read(&volume->ref), |
|
n_accesses, fscache_access_cache_unpin); |
|
|
|
wait_var_event(&volume->n_accesses, |
|
atomic_read(&volume->n_accesses) == 0); |
|
} |
|
EXPORT_SYMBOL(fscache_withdraw_volume); |
|
|
|
#ifdef CONFIG_PROC_FS |
|
/* |
|
* Generate a list of volumes in /proc/fs/fscache/volumes |
|
*/ |
|
static int fscache_volumes_seq_show(struct seq_file *m, void *v) |
|
{ |
|
struct fscache_volume *volume; |
|
|
|
if (v == &fscache_volumes) { |
|
seq_puts(m, |
|
"VOLUME REF nCOOK ACC FL CACHE KEY\n" |
|
"======== ===== ===== === == =============== ================\n"); |
|
return 0; |
|
} |
|
|
|
volume = list_entry(v, struct fscache_volume, proc_link); |
|
seq_printf(m, |
|
"%08x %5d %5d %3d %02lx %-15.15s %s\n", |
|
volume->debug_id, |
|
refcount_read(&volume->ref), |
|
atomic_read(&volume->n_cookies), |
|
atomic_read(&volume->n_accesses), |
|
volume->flags, |
|
volume->cache->name ?: "-", |
|
volume->key + 1); |
|
return 0; |
|
} |
|
|
|
static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos) |
|
__acquires(&fscache_addremove_sem) |
|
{ |
|
down_read(&fscache_addremove_sem); |
|
return seq_list_start_head(&fscache_volumes, *_pos); |
|
} |
|
|
|
static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos) |
|
{ |
|
return seq_list_next(v, &fscache_volumes, _pos); |
|
} |
|
|
|
static void fscache_volumes_seq_stop(struct seq_file *m, void *v) |
|
__releases(&fscache_addremove_sem) |
|
{ |
|
up_read(&fscache_addremove_sem); |
|
} |
|
|
|
const struct seq_operations fscache_volumes_seq_ops = { |
|
.start = fscache_volumes_seq_start, |
|
.next = fscache_volumes_seq_next, |
|
.stop = fscache_volumes_seq_stop, |
|
.show = fscache_volumes_seq_show, |
|
}; |
|
#endif /* CONFIG_PROC_FS */
|
|
|