forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
258 lines
6.6 KiB
258 lines
6.6 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* fence-chain: chain fences together in a timeline |
|
* |
|
* Copyright (C) 2018 Advanced Micro Devices, Inc. |
|
* Authors: |
|
* Christian König <[email protected]> |
|
*/ |
|
|
|
#include <linux/dma-fence-chain.h> |
|
|
|
static bool dma_fence_chain_enable_signaling(struct dma_fence *fence); |
|
|
|
/** |
|
* dma_fence_chain_get_prev - use RCU to get a reference to the previous fence |
|
* @chain: chain node to get the previous node from |
|
* |
|
* Use dma_fence_get_rcu_safe to get a reference to the previous fence of the |
|
* chain node. |
|
*/ |
|
static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain) |
|
{ |
|
struct dma_fence *prev; |
|
|
|
rcu_read_lock(); |
|
prev = dma_fence_get_rcu_safe(&chain->prev); |
|
rcu_read_unlock(); |
|
return prev; |
|
} |
|
|
|
/** |
|
* dma_fence_chain_walk - chain walking function |
|
* @fence: current chain node |
|
* |
|
* Walk the chain to the next node. Returns the next fence or NULL if we are at |
|
* the end of the chain. Garbage collects chain nodes which are already |
|
* signaled. |
|
*/ |
|
struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence) |
|
{ |
|
struct dma_fence_chain *chain, *prev_chain; |
|
struct dma_fence *prev, *replacement, *tmp; |
|
|
|
chain = to_dma_fence_chain(fence); |
|
if (!chain) { |
|
dma_fence_put(fence); |
|
return NULL; |
|
} |
|
|
|
while ((prev = dma_fence_chain_get_prev(chain))) { |
|
|
|
prev_chain = to_dma_fence_chain(prev); |
|
if (prev_chain) { |
|
if (!dma_fence_is_signaled(prev_chain->fence)) |
|
break; |
|
|
|
replacement = dma_fence_chain_get_prev(prev_chain); |
|
} else { |
|
if (!dma_fence_is_signaled(prev)) |
|
break; |
|
|
|
replacement = NULL; |
|
} |
|
|
|
tmp = cmpxchg((struct dma_fence __force **)&chain->prev, |
|
prev, replacement); |
|
if (tmp == prev) |
|
dma_fence_put(tmp); |
|
else |
|
dma_fence_put(replacement); |
|
dma_fence_put(prev); |
|
} |
|
|
|
dma_fence_put(fence); |
|
return prev; |
|
} |
|
EXPORT_SYMBOL(dma_fence_chain_walk); |
|
|
|
/** |
|
* dma_fence_chain_find_seqno - find fence chain node by seqno |
|
* @pfence: pointer to the chain node where to start |
|
* @seqno: the sequence number to search for |
|
* |
|
* Advance the fence pointer to the chain node which will signal this sequence |
|
* number. If no sequence number is provided then this is a no-op. |
|
* |
|
* Returns EINVAL if the fence is not a chain node or the sequence number has |
|
* not yet advanced far enough. |
|
*/ |
|
int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno) |
|
{ |
|
struct dma_fence_chain *chain; |
|
|
|
if (!seqno) |
|
return 0; |
|
|
|
chain = to_dma_fence_chain(*pfence); |
|
if (!chain || chain->base.seqno < seqno) |
|
return -EINVAL; |
|
|
|
dma_fence_chain_for_each(*pfence, &chain->base) { |
|
if ((*pfence)->context != chain->base.context || |
|
to_dma_fence_chain(*pfence)->prev_seqno < seqno) |
|
break; |
|
} |
|
dma_fence_put(&chain->base); |
|
|
|
return 0; |
|
} |
|
EXPORT_SYMBOL(dma_fence_chain_find_seqno); |
|
|
|
static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence) |
|
{ |
|
return "dma_fence_chain"; |
|
} |
|
|
|
static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence) |
|
{ |
|
return "unbound"; |
|
} |
|
|
|
static void dma_fence_chain_irq_work(struct irq_work *work) |
|
{ |
|
struct dma_fence_chain *chain; |
|
|
|
chain = container_of(work, typeof(*chain), work); |
|
|
|
/* Try to rearm the callback */ |
|
if (!dma_fence_chain_enable_signaling(&chain->base)) |
|
/* Ok, we are done. No more unsignaled fences left */ |
|
dma_fence_signal(&chain->base); |
|
dma_fence_put(&chain->base); |
|
} |
|
|
|
static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb) |
|
{ |
|
struct dma_fence_chain *chain; |
|
|
|
chain = container_of(cb, typeof(*chain), cb); |
|
irq_work_queue(&chain->work); |
|
dma_fence_put(f); |
|
} |
|
|
|
static bool dma_fence_chain_enable_signaling(struct dma_fence *fence) |
|
{ |
|
struct dma_fence_chain *head = to_dma_fence_chain(fence); |
|
|
|
dma_fence_get(&head->base); |
|
dma_fence_chain_for_each(fence, &head->base) { |
|
struct dma_fence_chain *chain = to_dma_fence_chain(fence); |
|
struct dma_fence *f = chain ? chain->fence : fence; |
|
|
|
dma_fence_get(f); |
|
if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) { |
|
dma_fence_put(fence); |
|
return true; |
|
} |
|
dma_fence_put(f); |
|
} |
|
dma_fence_put(&head->base); |
|
return false; |
|
} |
|
|
|
static bool dma_fence_chain_signaled(struct dma_fence *fence) |
|
{ |
|
dma_fence_chain_for_each(fence, fence) { |
|
struct dma_fence_chain *chain = to_dma_fence_chain(fence); |
|
struct dma_fence *f = chain ? chain->fence : fence; |
|
|
|
if (!dma_fence_is_signaled(f)) { |
|
dma_fence_put(fence); |
|
return false; |
|
} |
|
} |
|
|
|
return true; |
|
} |
|
|
|
static void dma_fence_chain_release(struct dma_fence *fence) |
|
{ |
|
struct dma_fence_chain *chain = to_dma_fence_chain(fence); |
|
struct dma_fence *prev; |
|
|
|
/* Manually unlink the chain as much as possible to avoid recursion |
|
* and potential stack overflow. |
|
*/ |
|
while ((prev = rcu_dereference_protected(chain->prev, true))) { |
|
struct dma_fence_chain *prev_chain; |
|
|
|
if (kref_read(&prev->refcount) > 1) |
|
break; |
|
|
|
prev_chain = to_dma_fence_chain(prev); |
|
if (!prev_chain) |
|
break; |
|
|
|
/* No need for atomic operations since we hold the last |
|
* reference to prev_chain. |
|
*/ |
|
chain->prev = prev_chain->prev; |
|
RCU_INIT_POINTER(prev_chain->prev, NULL); |
|
dma_fence_put(prev); |
|
} |
|
dma_fence_put(prev); |
|
|
|
dma_fence_put(chain->fence); |
|
dma_fence_free(fence); |
|
} |
|
|
|
const struct dma_fence_ops dma_fence_chain_ops = { |
|
.use_64bit_seqno = true, |
|
.get_driver_name = dma_fence_chain_get_driver_name, |
|
.get_timeline_name = dma_fence_chain_get_timeline_name, |
|
.enable_signaling = dma_fence_chain_enable_signaling, |
|
.signaled = dma_fence_chain_signaled, |
|
.release = dma_fence_chain_release, |
|
}; |
|
EXPORT_SYMBOL(dma_fence_chain_ops); |
|
|
|
/** |
|
* dma_fence_chain_init - initialize a fence chain |
|
* @chain: the chain node to initialize |
|
* @prev: the previous fence |
|
* @fence: the current fence |
|
* @seqno: the sequence number to use for the fence chain |
|
* |
|
* Initialize a new chain node and either start a new chain or add the node to |
|
* the existing chain of the previous fence. |
|
*/ |
|
void dma_fence_chain_init(struct dma_fence_chain *chain, |
|
struct dma_fence *prev, |
|
struct dma_fence *fence, |
|
uint64_t seqno) |
|
{ |
|
struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev); |
|
uint64_t context; |
|
|
|
spin_lock_init(&chain->lock); |
|
rcu_assign_pointer(chain->prev, prev); |
|
chain->fence = fence; |
|
chain->prev_seqno = 0; |
|
init_irq_work(&chain->work, dma_fence_chain_irq_work); |
|
|
|
/* Try to reuse the context of the previous chain node. */ |
|
if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) { |
|
context = prev->context; |
|
chain->prev_seqno = prev->seqno; |
|
} else { |
|
context = dma_fence_context_alloc(1); |
|
/* Make sure that we always have a valid sequence number. */ |
|
if (prev_chain) |
|
seqno = max(prev->seqno, seqno); |
|
} |
|
|
|
dma_fence_init(&chain->base, &dma_fence_chain_ops, |
|
&chain->lock, context, seqno); |
|
} |
|
EXPORT_SYMBOL(dma_fence_chain_init);
|
|
|