forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1143 lines
31 KiB
1143 lines
31 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* Copyright (c) 2016-2018 Oracle. All rights reserved. |
|
* |
|
* Use the core R/W API to move RPC-over-RDMA Read and Write chunks. |
|
*/ |
|
|
|
#include <rdma/rw.h> |
|
|
|
#include <linux/sunrpc/xdr.h> |
|
#include <linux/sunrpc/rpc_rdma.h> |
|
#include <linux/sunrpc/svc_rdma.h> |
|
|
|
#include "xprt_rdma.h" |
|
#include <trace/events/rpcrdma.h> |
|
|
|
static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc); |
|
static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); |
|
|
|
/* Each R/W context contains state for one chain of RDMA Read or |
|
* Write Work Requests. |
|
* |
|
* Each WR chain handles a single contiguous server-side buffer, |
|
* because scatterlist entries after the first have to start on |
|
* page alignment. xdr_buf iovecs cannot guarantee alignment. |
|
* |
|
* Each WR chain handles only one R_key. Each RPC-over-RDMA segment |
|
* from a client may contain a unique R_key, so each WR chain moves |
|
* up to one segment at a time. |
|
* |
|
* The scatterlist makes this data structure over 4KB in size. To |
|
* make it less likely to fail, and to handle the allocation for |
|
* smaller I/O requests without disabling bottom-halves, these |
|
* contexts are created on demand, but cached and reused until the |
|
* controlling svcxprt_rdma is destroyed. |
|
*/ |
|
struct svc_rdma_rw_ctxt { |
|
struct llist_node rw_node; |
|
struct list_head rw_list; |
|
struct rdma_rw_ctx rw_ctx; |
|
unsigned int rw_nents; |
|
struct sg_table rw_sg_table; |
|
struct scatterlist rw_first_sgl[]; |
|
}; |
|
|
|
static inline struct svc_rdma_rw_ctxt * |
|
svc_rdma_next_ctxt(struct list_head *list) |
|
{ |
|
return list_first_entry_or_null(list, struct svc_rdma_rw_ctxt, |
|
rw_list); |
|
} |
|
|
|
static struct svc_rdma_rw_ctxt * |
|
svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges) |
|
{ |
|
struct svc_rdma_rw_ctxt *ctxt; |
|
struct llist_node *node; |
|
|
|
spin_lock(&rdma->sc_rw_ctxt_lock); |
|
node = llist_del_first(&rdma->sc_rw_ctxts); |
|
spin_unlock(&rdma->sc_rw_ctxt_lock); |
|
if (node) { |
|
ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); |
|
} else { |
|
ctxt = kmalloc(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE), |
|
GFP_KERNEL); |
|
if (!ctxt) |
|
goto out_noctx; |
|
|
|
INIT_LIST_HEAD(&ctxt->rw_list); |
|
} |
|
|
|
ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl; |
|
if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges, |
|
ctxt->rw_sg_table.sgl, |
|
SG_CHUNK_SIZE)) |
|
goto out_free; |
|
return ctxt; |
|
|
|
out_free: |
|
kfree(ctxt); |
|
out_noctx: |
|
trace_svcrdma_no_rwctx_err(rdma, sges); |
|
return NULL; |
|
} |
|
|
|
static void __svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, |
|
struct svc_rdma_rw_ctxt *ctxt, |
|
struct llist_head *list) |
|
{ |
|
sg_free_table_chained(&ctxt->rw_sg_table, SG_CHUNK_SIZE); |
|
llist_add(&ctxt->rw_node, list); |
|
} |
|
|
|
static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, |
|
struct svc_rdma_rw_ctxt *ctxt) |
|
{ |
|
__svc_rdma_put_rw_ctxt(rdma, ctxt, &rdma->sc_rw_ctxts); |
|
} |
|
|
|
/** |
|
* svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts |
|
* @rdma: transport about to be destroyed |
|
* |
|
*/ |
|
void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma) |
|
{ |
|
struct svc_rdma_rw_ctxt *ctxt; |
|
struct llist_node *node; |
|
|
|
while ((node = llist_del_first(&rdma->sc_rw_ctxts)) != NULL) { |
|
ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node); |
|
kfree(ctxt); |
|
} |
|
} |
|
|
|
/** |
|
* svc_rdma_rw_ctx_init - Prepare a R/W context for I/O |
|
* @rdma: controlling transport instance |
|
* @ctxt: R/W context to prepare |
|
* @offset: RDMA offset |
|
* @handle: RDMA tag/handle |
|
* @direction: I/O direction |
|
* |
|
* Returns on success, the number of WQEs that will be needed |
|
* on the workqueue, or a negative errno. |
|
*/ |
|
static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma, |
|
struct svc_rdma_rw_ctxt *ctxt, |
|
u64 offset, u32 handle, |
|
enum dma_data_direction direction) |
|
{ |
|
int ret; |
|
|
|
ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, rdma->sc_port_num, |
|
ctxt->rw_sg_table.sgl, ctxt->rw_nents, |
|
0, offset, handle, direction); |
|
if (unlikely(ret < 0)) { |
|
svc_rdma_put_rw_ctxt(rdma, ctxt); |
|
trace_svcrdma_dma_map_rw_err(rdma, ctxt->rw_nents, ret); |
|
} |
|
return ret; |
|
} |
|
|
|
/* A chunk context tracks all I/O for moving one Read or Write |
|
* chunk. This is a set of rdma_rw's that handle data movement |
|
* for all segments of one chunk. |
|
* |
|
* These are small, acquired with a single allocator call, and |
|
* no more than one is needed per chunk. They are allocated on |
|
* demand, and not cached. |
|
*/ |
|
struct svc_rdma_chunk_ctxt { |
|
struct rpc_rdma_cid cc_cid; |
|
struct ib_cqe cc_cqe; |
|
struct svcxprt_rdma *cc_rdma; |
|
struct list_head cc_rwctxts; |
|
int cc_sqecount; |
|
enum ib_wc_status cc_status; |
|
struct completion cc_done; |
|
}; |
|
|
|
static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma, |
|
struct rpc_rdma_cid *cid) |
|
{ |
|
cid->ci_queue_id = rdma->sc_sq_cq->res.id; |
|
cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids); |
|
} |
|
|
|
static void svc_rdma_cc_init(struct svcxprt_rdma *rdma, |
|
struct svc_rdma_chunk_ctxt *cc) |
|
{ |
|
svc_rdma_cc_cid_init(rdma, &cc->cc_cid); |
|
cc->cc_rdma = rdma; |
|
|
|
INIT_LIST_HEAD(&cc->cc_rwctxts); |
|
cc->cc_sqecount = 0; |
|
} |
|
|
|
/* |
|
* The consumed rw_ctx's are cleaned and placed on a local llist so |
|
* that only one atomic llist operation is needed to put them all |
|
* back on the free list. |
|
*/ |
|
static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc, |
|
enum dma_data_direction dir) |
|
{ |
|
struct svcxprt_rdma *rdma = cc->cc_rdma; |
|
struct llist_node *first, *last; |
|
struct svc_rdma_rw_ctxt *ctxt; |
|
LLIST_HEAD(free); |
|
|
|
first = last = NULL; |
|
while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) { |
|
list_del(&ctxt->rw_list); |
|
|
|
rdma_rw_ctx_destroy(&ctxt->rw_ctx, rdma->sc_qp, |
|
rdma->sc_port_num, ctxt->rw_sg_table.sgl, |
|
ctxt->rw_nents, dir); |
|
__svc_rdma_put_rw_ctxt(rdma, ctxt, &free); |
|
|
|
ctxt->rw_node.next = first; |
|
first = &ctxt->rw_node; |
|
if (!last) |
|
last = first; |
|
} |
|
if (first) |
|
llist_add_batch(first, last, &rdma->sc_rw_ctxts); |
|
} |
|
|
|
/* State for sending a Write or Reply chunk. |
|
* - Tracks progress of writing one chunk over all its segments |
|
* - Stores arguments for the SGL constructor functions |
|
*/ |
|
struct svc_rdma_write_info { |
|
const struct svc_rdma_chunk *wi_chunk; |
|
|
|
/* write state of this chunk */ |
|
unsigned int wi_seg_off; |
|
unsigned int wi_seg_no; |
|
|
|
/* SGL constructor arguments */ |
|
const struct xdr_buf *wi_xdr; |
|
unsigned char *wi_base; |
|
unsigned int wi_next_off; |
|
|
|
struct svc_rdma_chunk_ctxt wi_cc; |
|
}; |
|
|
|
static struct svc_rdma_write_info * |
|
svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, |
|
const struct svc_rdma_chunk *chunk) |
|
{ |
|
struct svc_rdma_write_info *info; |
|
|
|
info = kmalloc(sizeof(*info), GFP_KERNEL); |
|
if (!info) |
|
return info; |
|
|
|
info->wi_chunk = chunk; |
|
info->wi_seg_off = 0; |
|
info->wi_seg_no = 0; |
|
svc_rdma_cc_init(rdma, &info->wi_cc); |
|
info->wi_cc.cc_cqe.done = svc_rdma_write_done; |
|
return info; |
|
} |
|
|
|
static void svc_rdma_write_info_free(struct svc_rdma_write_info *info) |
|
{ |
|
svc_rdma_cc_release(&info->wi_cc, DMA_TO_DEVICE); |
|
kfree(info); |
|
} |
|
|
|
/** |
|
* svc_rdma_write_done - Write chunk completion |
|
* @cq: controlling Completion Queue |
|
* @wc: Work Completion |
|
* |
|
* Pages under I/O are freed by a subsequent Send completion. |
|
*/ |
|
static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) |
|
{ |
|
struct ib_cqe *cqe = wc->wr_cqe; |
|
struct svc_rdma_chunk_ctxt *cc = |
|
container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); |
|
struct svcxprt_rdma *rdma = cc->cc_rdma; |
|
struct svc_rdma_write_info *info = |
|
container_of(cc, struct svc_rdma_write_info, wi_cc); |
|
|
|
trace_svcrdma_wc_write(wc, &cc->cc_cid); |
|
|
|
svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); |
|
|
|
if (unlikely(wc->status != IB_WC_SUCCESS)) |
|
svc_xprt_deferred_close(&rdma->sc_xprt); |
|
|
|
svc_rdma_write_info_free(info); |
|
} |
|
|
|
/* State for pulling a Read chunk. |
|
*/ |
|
struct svc_rdma_read_info { |
|
struct svc_rqst *ri_rqst; |
|
struct svc_rdma_recv_ctxt *ri_readctxt; |
|
unsigned int ri_pageno; |
|
unsigned int ri_pageoff; |
|
unsigned int ri_totalbytes; |
|
|
|
struct svc_rdma_chunk_ctxt ri_cc; |
|
}; |
|
|
|
static struct svc_rdma_read_info * |
|
svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma) |
|
{ |
|
struct svc_rdma_read_info *info; |
|
|
|
info = kmalloc(sizeof(*info), GFP_KERNEL); |
|
if (!info) |
|
return info; |
|
|
|
svc_rdma_cc_init(rdma, &info->ri_cc); |
|
info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done; |
|
return info; |
|
} |
|
|
|
static void svc_rdma_read_info_free(struct svc_rdma_read_info *info) |
|
{ |
|
svc_rdma_cc_release(&info->ri_cc, DMA_FROM_DEVICE); |
|
kfree(info); |
|
} |
|
|
|
/** |
|
* svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx |
|
* @cq: controlling Completion Queue |
|
* @wc: Work Completion |
|
* |
|
*/ |
|
static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc) |
|
{ |
|
struct ib_cqe *cqe = wc->wr_cqe; |
|
struct svc_rdma_chunk_ctxt *cc = |
|
container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe); |
|
struct svcxprt_rdma *rdma = cc->cc_rdma; |
|
|
|
trace_svcrdma_wc_read(wc, &cc->cc_cid); |
|
|
|
svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount); |
|
cc->cc_status = wc->status; |
|
complete(&cc->cc_done); |
|
return; |
|
} |
|
|
|
/* This function sleeps when the transport's Send Queue is congested. |
|
* |
|
* Assumptions: |
|
* - If ib_post_send() succeeds, only one completion is expected, |
|
* even if one or more WRs are flushed. This is true when posting |
|
* an rdma_rw_ctx or when posting a single signaled WR. |
|
*/ |
|
static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc) |
|
{ |
|
struct svcxprt_rdma *rdma = cc->cc_rdma; |
|
struct ib_send_wr *first_wr; |
|
const struct ib_send_wr *bad_wr; |
|
struct list_head *tmp; |
|
struct ib_cqe *cqe; |
|
int ret; |
|
|
|
if (cc->cc_sqecount > rdma->sc_sq_depth) |
|
return -EINVAL; |
|
|
|
first_wr = NULL; |
|
cqe = &cc->cc_cqe; |
|
list_for_each(tmp, &cc->cc_rwctxts) { |
|
struct svc_rdma_rw_ctxt *ctxt; |
|
|
|
ctxt = list_entry(tmp, struct svc_rdma_rw_ctxt, rw_list); |
|
first_wr = rdma_rw_ctx_wrs(&ctxt->rw_ctx, rdma->sc_qp, |
|
rdma->sc_port_num, cqe, first_wr); |
|
cqe = NULL; |
|
} |
|
|
|
do { |
|
if (atomic_sub_return(cc->cc_sqecount, |
|
&rdma->sc_sq_avail) > 0) { |
|
ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr); |
|
if (ret) |
|
break; |
|
return 0; |
|
} |
|
|
|
percpu_counter_inc(&svcrdma_stat_sq_starve); |
|
trace_svcrdma_sq_full(rdma); |
|
atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); |
|
wait_event(rdma->sc_send_wait, |
|
atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount); |
|
trace_svcrdma_sq_retry(rdma); |
|
} while (1); |
|
|
|
trace_svcrdma_sq_post_err(rdma, ret); |
|
svc_xprt_deferred_close(&rdma->sc_xprt); |
|
|
|
/* If even one was posted, there will be a completion. */ |
|
if (bad_wr != first_wr) |
|
return 0; |
|
|
|
atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail); |
|
wake_up(&rdma->sc_send_wait); |
|
return -ENOTCONN; |
|
} |
|
|
|
/* Build and DMA-map an SGL that covers one kvec in an xdr_buf |
|
*/ |
|
static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info, |
|
unsigned int len, |
|
struct svc_rdma_rw_ctxt *ctxt) |
|
{ |
|
struct scatterlist *sg = ctxt->rw_sg_table.sgl; |
|
|
|
sg_set_buf(&sg[0], info->wi_base, len); |
|
info->wi_base += len; |
|
|
|
ctxt->rw_nents = 1; |
|
} |
|
|
|
/* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist. |
|
*/ |
|
static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info, |
|
unsigned int remaining, |
|
struct svc_rdma_rw_ctxt *ctxt) |
|
{ |
|
unsigned int sge_no, sge_bytes, page_off, page_no; |
|
const struct xdr_buf *xdr = info->wi_xdr; |
|
struct scatterlist *sg; |
|
struct page **page; |
|
|
|
page_off = info->wi_next_off + xdr->page_base; |
|
page_no = page_off >> PAGE_SHIFT; |
|
page_off = offset_in_page(page_off); |
|
page = xdr->pages + page_no; |
|
info->wi_next_off += remaining; |
|
sg = ctxt->rw_sg_table.sgl; |
|
sge_no = 0; |
|
do { |
|
sge_bytes = min_t(unsigned int, remaining, |
|
PAGE_SIZE - page_off); |
|
sg_set_page(sg, *page, sge_bytes, page_off); |
|
|
|
remaining -= sge_bytes; |
|
sg = sg_next(sg); |
|
page_off = 0; |
|
sge_no++; |
|
page++; |
|
} while (remaining); |
|
|
|
ctxt->rw_nents = sge_no; |
|
} |
|
|
|
/* Construct RDMA Write WRs to send a portion of an xdr_buf containing |
|
* an RPC Reply. |
|
*/ |
|
static int |
|
svc_rdma_build_writes(struct svc_rdma_write_info *info, |
|
void (*constructor)(struct svc_rdma_write_info *info, |
|
unsigned int len, |
|
struct svc_rdma_rw_ctxt *ctxt), |
|
unsigned int remaining) |
|
{ |
|
struct svc_rdma_chunk_ctxt *cc = &info->wi_cc; |
|
struct svcxprt_rdma *rdma = cc->cc_rdma; |
|
const struct svc_rdma_segment *seg; |
|
struct svc_rdma_rw_ctxt *ctxt; |
|
int ret; |
|
|
|
do { |
|
unsigned int write_len; |
|
u64 offset; |
|
|
|
seg = &info->wi_chunk->ch_segments[info->wi_seg_no]; |
|
if (!seg) |
|
goto out_overflow; |
|
|
|
write_len = min(remaining, seg->rs_length - info->wi_seg_off); |
|
if (!write_len) |
|
goto out_overflow; |
|
ctxt = svc_rdma_get_rw_ctxt(rdma, |
|
(write_len >> PAGE_SHIFT) + 2); |
|
if (!ctxt) |
|
return -ENOMEM; |
|
|
|
constructor(info, write_len, ctxt); |
|
offset = seg->rs_offset + info->wi_seg_off; |
|
ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, seg->rs_handle, |
|
DMA_TO_DEVICE); |
|
if (ret < 0) |
|
return -EIO; |
|
percpu_counter_inc(&svcrdma_stat_write); |
|
|
|
list_add(&ctxt->rw_list, &cc->cc_rwctxts); |
|
cc->cc_sqecount += ret; |
|
if (write_len == seg->rs_length - info->wi_seg_off) { |
|
info->wi_seg_no++; |
|
info->wi_seg_off = 0; |
|
} else { |
|
info->wi_seg_off += write_len; |
|
} |
|
remaining -= write_len; |
|
} while (remaining); |
|
|
|
return 0; |
|
|
|
out_overflow: |
|
trace_svcrdma_small_wrch_err(rdma, remaining, info->wi_seg_no, |
|
info->wi_chunk->ch_segcount); |
|
return -E2BIG; |
|
} |
|
|
|
/** |
|
* svc_rdma_iov_write - Construct RDMA Writes from an iov |
|
* @info: pointer to write arguments |
|
* @iov: kvec to write |
|
* |
|
* Returns: |
|
* On success, returns zero |
|
* %-E2BIG if the client-provided Write chunk is too small |
|
* %-ENOMEM if a resource has been exhausted |
|
* %-EIO if an rdma-rw error occurred |
|
*/ |
|
static int svc_rdma_iov_write(struct svc_rdma_write_info *info, |
|
const struct kvec *iov) |
|
{ |
|
info->wi_base = iov->iov_base; |
|
return svc_rdma_build_writes(info, svc_rdma_vec_to_sg, |
|
iov->iov_len); |
|
} |
|
|
|
/** |
|
* svc_rdma_pages_write - Construct RDMA Writes from pages |
|
* @info: pointer to write arguments |
|
* @xdr: xdr_buf with pages to write |
|
* @offset: offset into the content of @xdr |
|
* @length: number of bytes to write |
|
* |
|
* Returns: |
|
* On success, returns zero |
|
* %-E2BIG if the client-provided Write chunk is too small |
|
* %-ENOMEM if a resource has been exhausted |
|
* %-EIO if an rdma-rw error occurred |
|
*/ |
|
static int svc_rdma_pages_write(struct svc_rdma_write_info *info, |
|
const struct xdr_buf *xdr, |
|
unsigned int offset, |
|
unsigned long length) |
|
{ |
|
info->wi_xdr = xdr; |
|
info->wi_next_off = offset - xdr->head[0].iov_len; |
|
return svc_rdma_build_writes(info, svc_rdma_pagelist_to_sg, |
|
length); |
|
} |
|
|
|
/** |
|
* svc_rdma_xb_write - Construct RDMA Writes to write an xdr_buf |
|
* @xdr: xdr_buf to write |
|
* @data: pointer to write arguments |
|
* |
|
* Returns: |
|
* On success, returns zero |
|
* %-E2BIG if the client-provided Write chunk is too small |
|
* %-ENOMEM if a resource has been exhausted |
|
* %-EIO if an rdma-rw error occurred |
|
*/ |
|
static int svc_rdma_xb_write(const struct xdr_buf *xdr, void *data) |
|
{ |
|
struct svc_rdma_write_info *info = data; |
|
int ret; |
|
|
|
if (xdr->head[0].iov_len) { |
|
ret = svc_rdma_iov_write(info, &xdr->head[0]); |
|
if (ret < 0) |
|
return ret; |
|
} |
|
|
|
if (xdr->page_len) { |
|
ret = svc_rdma_pages_write(info, xdr, xdr->head[0].iov_len, |
|
xdr->page_len); |
|
if (ret < 0) |
|
return ret; |
|
} |
|
|
|
if (xdr->tail[0].iov_len) { |
|
ret = svc_rdma_iov_write(info, &xdr->tail[0]); |
|
if (ret < 0) |
|
return ret; |
|
} |
|
|
|
return xdr->len; |
|
} |
|
|
|
/** |
|
* svc_rdma_send_write_chunk - Write all segments in a Write chunk |
|
* @rdma: controlling RDMA transport |
|
* @chunk: Write chunk provided by the client |
|
* @xdr: xdr_buf containing the data payload |
|
* |
|
* Returns a non-negative number of bytes the chunk consumed, or |
|
* %-E2BIG if the payload was larger than the Write chunk, |
|
* %-EINVAL if client provided too many segments, |
|
* %-ENOMEM if rdma_rw context pool was exhausted, |
|
* %-ENOTCONN if posting failed (connection is lost), |
|
* %-EIO if rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, |
|
const struct svc_rdma_chunk *chunk, |
|
const struct xdr_buf *xdr) |
|
{ |
|
struct svc_rdma_write_info *info; |
|
struct svc_rdma_chunk_ctxt *cc; |
|
int ret; |
|
|
|
info = svc_rdma_write_info_alloc(rdma, chunk); |
|
if (!info) |
|
return -ENOMEM; |
|
cc = &info->wi_cc; |
|
|
|
ret = svc_rdma_xb_write(xdr, info); |
|
if (ret != xdr->len) |
|
goto out_err; |
|
|
|
trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount); |
|
ret = svc_rdma_post_chunk_ctxt(cc); |
|
if (ret < 0) |
|
goto out_err; |
|
return xdr->len; |
|
|
|
out_err: |
|
svc_rdma_write_info_free(info); |
|
return ret; |
|
} |
|
|
|
/** |
|
* svc_rdma_send_reply_chunk - Write all segments in the Reply chunk |
|
* @rdma: controlling RDMA transport |
|
* @rctxt: Write and Reply chunks from client |
|
* @xdr: xdr_buf containing an RPC Reply |
|
* |
|
* Returns a non-negative number of bytes the chunk consumed, or |
|
* %-E2BIG if the payload was larger than the Reply chunk, |
|
* %-EINVAL if client provided too many segments, |
|
* %-ENOMEM if rdma_rw context pool was exhausted, |
|
* %-ENOTCONN if posting failed (connection is lost), |
|
* %-EIO if rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma, |
|
const struct svc_rdma_recv_ctxt *rctxt, |
|
const struct xdr_buf *xdr) |
|
{ |
|
struct svc_rdma_write_info *info; |
|
struct svc_rdma_chunk_ctxt *cc; |
|
struct svc_rdma_chunk *chunk; |
|
int ret; |
|
|
|
if (pcl_is_empty(&rctxt->rc_reply_pcl)) |
|
return 0; |
|
|
|
chunk = pcl_first_chunk(&rctxt->rc_reply_pcl); |
|
info = svc_rdma_write_info_alloc(rdma, chunk); |
|
if (!info) |
|
return -ENOMEM; |
|
cc = &info->wi_cc; |
|
|
|
ret = pcl_process_nonpayloads(&rctxt->rc_write_pcl, xdr, |
|
svc_rdma_xb_write, info); |
|
if (ret < 0) |
|
goto out_err; |
|
|
|
trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount); |
|
ret = svc_rdma_post_chunk_ctxt(cc); |
|
if (ret < 0) |
|
goto out_err; |
|
|
|
return xdr->len; |
|
|
|
out_err: |
|
svc_rdma_write_info_free(info); |
|
return ret; |
|
} |
|
|
|
/** |
|
* svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment |
|
* @info: context for ongoing I/O |
|
* @segment: co-ordinates of remote memory to be read |
|
* |
|
* Returns: |
|
* %0: the Read WR chain was constructed successfully |
|
* %-EINVAL: there were not enough rq_pages to finish |
|
* %-ENOMEM: allocating a local resources failed |
|
* %-EIO: a DMA mapping error occurred |
|
*/ |
|
static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, |
|
const struct svc_rdma_segment *segment) |
|
{ |
|
struct svc_rdma_recv_ctxt *head = info->ri_readctxt; |
|
struct svc_rdma_chunk_ctxt *cc = &info->ri_cc; |
|
struct svc_rqst *rqstp = info->ri_rqst; |
|
unsigned int sge_no, seg_len, len; |
|
struct svc_rdma_rw_ctxt *ctxt; |
|
struct scatterlist *sg; |
|
int ret; |
|
|
|
len = segment->rs_length; |
|
sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT; |
|
ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no); |
|
if (!ctxt) |
|
return -ENOMEM; |
|
ctxt->rw_nents = sge_no; |
|
|
|
sg = ctxt->rw_sg_table.sgl; |
|
for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) { |
|
seg_len = min_t(unsigned int, len, |
|
PAGE_SIZE - info->ri_pageoff); |
|
|
|
if (!info->ri_pageoff) |
|
head->rc_page_count++; |
|
|
|
sg_set_page(sg, rqstp->rq_pages[info->ri_pageno], |
|
seg_len, info->ri_pageoff); |
|
sg = sg_next(sg); |
|
|
|
info->ri_pageoff += seg_len; |
|
if (info->ri_pageoff == PAGE_SIZE) { |
|
info->ri_pageno++; |
|
info->ri_pageoff = 0; |
|
} |
|
len -= seg_len; |
|
|
|
/* Safety check */ |
|
if (len && |
|
&rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end) |
|
goto out_overrun; |
|
} |
|
|
|
ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, segment->rs_offset, |
|
segment->rs_handle, DMA_FROM_DEVICE); |
|
if (ret < 0) |
|
return -EIO; |
|
percpu_counter_inc(&svcrdma_stat_read); |
|
|
|
list_add(&ctxt->rw_list, &cc->cc_rwctxts); |
|
cc->cc_sqecount += ret; |
|
return 0; |
|
|
|
out_overrun: |
|
trace_svcrdma_page_overrun_err(cc->cc_rdma, rqstp, info->ri_pageno); |
|
return -EINVAL; |
|
} |
|
|
|
/** |
|
* svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk |
|
* @info: context for ongoing I/O |
|
* @chunk: Read chunk to pull |
|
* |
|
* Return values: |
|
* %0: the Read WR chain was constructed successfully |
|
* %-EINVAL: there were not enough resources to finish |
|
* %-ENOMEM: allocating a local resources failed |
|
* %-EIO: a DMA mapping error occurred |
|
*/ |
|
static int svc_rdma_build_read_chunk(struct svc_rdma_read_info *info, |
|
const struct svc_rdma_chunk *chunk) |
|
{ |
|
const struct svc_rdma_segment *segment; |
|
int ret; |
|
|
|
ret = -EINVAL; |
|
pcl_for_each_segment(segment, chunk) { |
|
ret = svc_rdma_build_read_segment(info, segment); |
|
if (ret < 0) |
|
break; |
|
info->ri_totalbytes += segment->rs_length; |
|
} |
|
return ret; |
|
} |
|
|
|
/** |
|
* svc_rdma_copy_inline_range - Copy part of the inline content into pages |
|
* @info: context for RDMA Reads |
|
* @offset: offset into the Receive buffer of region to copy |
|
* @remaining: length of region to copy |
|
* |
|
* Take a page at a time from rqstp->rq_pages and copy the inline |
|
* content from the Receive buffer into that page. Update |
|
* info->ri_pageno and info->ri_pageoff so that the next RDMA Read |
|
* result will land contiguously with the copied content. |
|
* |
|
* Return values: |
|
* %0: Inline content was successfully copied |
|
* %-EINVAL: offset or length was incorrect |
|
*/ |
|
static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info, |
|
unsigned int offset, |
|
unsigned int remaining) |
|
{ |
|
struct svc_rdma_recv_ctxt *head = info->ri_readctxt; |
|
unsigned char *dst, *src = head->rc_recv_buf; |
|
struct svc_rqst *rqstp = info->ri_rqst; |
|
unsigned int page_no, numpages; |
|
|
|
numpages = PAGE_ALIGN(info->ri_pageoff + remaining) >> PAGE_SHIFT; |
|
for (page_no = 0; page_no < numpages; page_no++) { |
|
unsigned int page_len; |
|
|
|
page_len = min_t(unsigned int, remaining, |
|
PAGE_SIZE - info->ri_pageoff); |
|
|
|
if (!info->ri_pageoff) |
|
head->rc_page_count++; |
|
|
|
dst = page_address(rqstp->rq_pages[info->ri_pageno]); |
|
memcpy(dst + info->ri_pageno, src + offset, page_len); |
|
|
|
info->ri_totalbytes += page_len; |
|
info->ri_pageoff += page_len; |
|
if (info->ri_pageoff == PAGE_SIZE) { |
|
info->ri_pageno++; |
|
info->ri_pageoff = 0; |
|
} |
|
remaining -= page_len; |
|
offset += page_len; |
|
} |
|
|
|
return -EINVAL; |
|
} |
|
|
|
/** |
|
* svc_rdma_read_multiple_chunks - Construct RDMA Reads to pull data item Read chunks |
|
* @info: context for RDMA Reads |
|
* |
|
* The chunk data lands in rqstp->rq_arg as a series of contiguous pages, |
|
* like an incoming TCP call. |
|
* |
|
* Return values: |
|
* %0: RDMA Read WQEs were successfully built |
|
* %-EINVAL: client provided too many chunks or segments, |
|
* %-ENOMEM: rdma_rw context pool was exhausted, |
|
* %-ENOTCONN: posting failed (connection is lost), |
|
* %-EIO: rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *info) |
|
{ |
|
struct svc_rdma_recv_ctxt *head = info->ri_readctxt; |
|
const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; |
|
struct xdr_buf *buf = &info->ri_rqst->rq_arg; |
|
struct svc_rdma_chunk *chunk, *next; |
|
unsigned int start, length; |
|
int ret; |
|
|
|
start = 0; |
|
chunk = pcl_first_chunk(pcl); |
|
length = chunk->ch_position; |
|
ret = svc_rdma_copy_inline_range(info, start, length); |
|
if (ret < 0) |
|
return ret; |
|
|
|
pcl_for_each_chunk(chunk, pcl) { |
|
ret = svc_rdma_build_read_chunk(info, chunk); |
|
if (ret < 0) |
|
return ret; |
|
|
|
next = pcl_next_chunk(pcl, chunk); |
|
if (!next) |
|
break; |
|
|
|
start += length; |
|
length = next->ch_position - info->ri_totalbytes; |
|
ret = svc_rdma_copy_inline_range(info, start, length); |
|
if (ret < 0) |
|
return ret; |
|
} |
|
|
|
start += length; |
|
length = head->rc_byte_len - start; |
|
ret = svc_rdma_copy_inline_range(info, start, length); |
|
if (ret < 0) |
|
return ret; |
|
|
|
buf->len += info->ri_totalbytes; |
|
buf->buflen += info->ri_totalbytes; |
|
|
|
buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); |
|
buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); |
|
buf->pages = &info->ri_rqst->rq_pages[1]; |
|
buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; |
|
return 0; |
|
} |
|
|
|
/** |
|
* svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks |
|
* @info: context for RDMA Reads |
|
* |
|
* The chunk data lands in the page list of rqstp->rq_arg.pages. |
|
* |
|
* Currently NFSD does not look at the rqstp->rq_arg.tail[0] kvec. |
|
* Therefore, XDR round-up of the Read chunk and trailing |
|
* inline content must both be added at the end of the pagelist. |
|
* |
|
* Return values: |
|
* %0: RDMA Read WQEs were successfully built |
|
* %-EINVAL: client provided too many chunks or segments, |
|
* %-ENOMEM: rdma_rw context pool was exhausted, |
|
* %-ENOTCONN: posting failed (connection is lost), |
|
* %-EIO: rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
static int svc_rdma_read_data_item(struct svc_rdma_read_info *info) |
|
{ |
|
struct svc_rdma_recv_ctxt *head = info->ri_readctxt; |
|
struct xdr_buf *buf = &info->ri_rqst->rq_arg; |
|
struct svc_rdma_chunk *chunk; |
|
unsigned int length; |
|
int ret; |
|
|
|
chunk = pcl_first_chunk(&head->rc_read_pcl); |
|
ret = svc_rdma_build_read_chunk(info, chunk); |
|
if (ret < 0) |
|
goto out; |
|
|
|
/* Split the Receive buffer between the head and tail |
|
* buffers at Read chunk's position. XDR roundup of the |
|
* chunk is not included in either the pagelist or in |
|
* the tail. |
|
*/ |
|
buf->tail[0].iov_base = buf->head[0].iov_base + chunk->ch_position; |
|
buf->tail[0].iov_len = buf->head[0].iov_len - chunk->ch_position; |
|
buf->head[0].iov_len = chunk->ch_position; |
|
|
|
/* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2). |
|
* |
|
* If the client already rounded up the chunk length, the |
|
* length does not change. Otherwise, the length of the page |
|
* list is increased to include XDR round-up. |
|
* |
|
* Currently these chunks always start at page offset 0, |
|
* thus the rounded-up length never crosses a page boundary. |
|
*/ |
|
buf->pages = &info->ri_rqst->rq_pages[0]; |
|
length = xdr_align_size(chunk->ch_length); |
|
buf->page_len = length; |
|
buf->len += length; |
|
buf->buflen += length; |
|
|
|
out: |
|
return ret; |
|
} |
|
|
|
/** |
|
* svc_rdma_read_chunk_range - Build RDMA Read WQEs for portion of a chunk |
|
* @info: context for RDMA Reads |
|
* @chunk: parsed Call chunk to pull |
|
* @offset: offset of region to pull |
|
* @length: length of region to pull |
|
* |
|
* Return values: |
|
* %0: RDMA Read WQEs were successfully built |
|
* %-EINVAL: there were not enough resources to finish |
|
* %-ENOMEM: rdma_rw context pool was exhausted, |
|
* %-ENOTCONN: posting failed (connection is lost), |
|
* %-EIO: rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info, |
|
const struct svc_rdma_chunk *chunk, |
|
unsigned int offset, unsigned int length) |
|
{ |
|
const struct svc_rdma_segment *segment; |
|
int ret; |
|
|
|
ret = -EINVAL; |
|
pcl_for_each_segment(segment, chunk) { |
|
struct svc_rdma_segment dummy; |
|
|
|
if (offset > segment->rs_length) { |
|
offset -= segment->rs_length; |
|
continue; |
|
} |
|
|
|
dummy.rs_handle = segment->rs_handle; |
|
dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; |
|
dummy.rs_offset = segment->rs_offset + offset; |
|
|
|
ret = svc_rdma_build_read_segment(info, &dummy); |
|
if (ret < 0) |
|
break; |
|
|
|
info->ri_totalbytes += dummy.rs_length; |
|
length -= dummy.rs_length; |
|
offset = 0; |
|
} |
|
return ret; |
|
} |
|
|
|
/** |
|
* svc_rdma_read_call_chunk - Build RDMA Read WQEs to pull a Long Message |
|
* @info: context for RDMA Reads |
|
* |
|
* Return values: |
|
* %0: RDMA Read WQEs were successfully built |
|
* %-EINVAL: there were not enough resources to finish |
|
* %-ENOMEM: rdma_rw context pool was exhausted, |
|
* %-ENOTCONN: posting failed (connection is lost), |
|
* %-EIO: rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info) |
|
{ |
|
struct svc_rdma_recv_ctxt *head = info->ri_readctxt; |
|
const struct svc_rdma_chunk *call_chunk = |
|
pcl_first_chunk(&head->rc_call_pcl); |
|
const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; |
|
struct svc_rdma_chunk *chunk, *next; |
|
unsigned int start, length; |
|
int ret; |
|
|
|
if (pcl_is_empty(pcl)) |
|
return svc_rdma_build_read_chunk(info, call_chunk); |
|
|
|
start = 0; |
|
chunk = pcl_first_chunk(pcl); |
|
length = chunk->ch_position; |
|
ret = svc_rdma_read_chunk_range(info, call_chunk, start, length); |
|
if (ret < 0) |
|
return ret; |
|
|
|
pcl_for_each_chunk(chunk, pcl) { |
|
ret = svc_rdma_build_read_chunk(info, chunk); |
|
if (ret < 0) |
|
return ret; |
|
|
|
next = pcl_next_chunk(pcl, chunk); |
|
if (!next) |
|
break; |
|
|
|
start += length; |
|
length = next->ch_position - info->ri_totalbytes; |
|
ret = svc_rdma_read_chunk_range(info, call_chunk, |
|
start, length); |
|
if (ret < 0) |
|
return ret; |
|
} |
|
|
|
start += length; |
|
length = call_chunk->ch_length - start; |
|
return svc_rdma_read_chunk_range(info, call_chunk, start, length); |
|
} |
|
|
|
/** |
|
* svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message |
|
* @info: context for RDMA Reads |
|
* |
|
* The start of the data lands in the first page just after the |
|
* Transport header, and the rest lands in rqstp->rq_arg.pages. |
|
* |
|
* Assumptions: |
|
* - A PZRC is never sent in an RDMA_MSG message, though it's |
|
* allowed by spec. |
|
* |
|
* Return values: |
|
* %0: RDMA Read WQEs were successfully built |
|
* %-EINVAL: client provided too many chunks or segments, |
|
* %-ENOMEM: rdma_rw context pool was exhausted, |
|
* %-ENOTCONN: posting failed (connection is lost), |
|
* %-EIO: rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
static noinline int svc_rdma_read_special(struct svc_rdma_read_info *info) |
|
{ |
|
struct xdr_buf *buf = &info->ri_rqst->rq_arg; |
|
int ret; |
|
|
|
ret = svc_rdma_read_call_chunk(info); |
|
if (ret < 0) |
|
goto out; |
|
|
|
buf->len += info->ri_totalbytes; |
|
buf->buflen += info->ri_totalbytes; |
|
|
|
buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]); |
|
buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes); |
|
buf->pages = &info->ri_rqst->rq_pages[1]; |
|
buf->page_len = info->ri_totalbytes - buf->head[0].iov_len; |
|
|
|
out: |
|
return ret; |
|
} |
|
|
|
/** |
|
* svc_rdma_process_read_list - Pull list of Read chunks from the client |
|
* @rdma: controlling RDMA transport |
|
* @rqstp: set of pages to use as Read sink buffers |
|
* @head: pages under I/O collect here |
|
* |
|
* The RPC/RDMA protocol assumes that the upper layer's XDR decoders |
|
* pull each Read chunk as they decode an incoming RPC message. |
|
* |
|
* On Linux, however, the server needs to have a fully-constructed RPC |
|
* message in rqstp->rq_arg when there is a positive return code from |
|
* ->xpo_recvfrom. So the Read list is safety-checked immediately when |
|
* it is received, then here the whole Read list is pulled all at once. |
|
* The ingress RPC message is fully reconstructed once all associated |
|
* RDMA Reads have completed. |
|
* |
|
* Return values: |
|
* %1: all needed RDMA Reads were posted successfully, |
|
* %-EINVAL: client provided too many chunks or segments, |
|
* %-ENOMEM: rdma_rw context pool was exhausted, |
|
* %-ENOTCONN: posting failed (connection is lost), |
|
* %-EIO: rdma_rw initialization failed (DMA mapping, etc). |
|
*/ |
|
int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, |
|
struct svc_rqst *rqstp, |
|
struct svc_rdma_recv_ctxt *head) |
|
{ |
|
struct svc_rdma_read_info *info; |
|
struct svc_rdma_chunk_ctxt *cc; |
|
int ret; |
|
|
|
info = svc_rdma_read_info_alloc(rdma); |
|
if (!info) |
|
return -ENOMEM; |
|
cc = &info->ri_cc; |
|
info->ri_rqst = rqstp; |
|
info->ri_readctxt = head; |
|
info->ri_pageno = 0; |
|
info->ri_pageoff = 0; |
|
info->ri_totalbytes = 0; |
|
|
|
if (pcl_is_empty(&head->rc_call_pcl)) { |
|
if (head->rc_read_pcl.cl_count == 1) |
|
ret = svc_rdma_read_data_item(info); |
|
else |
|
ret = svc_rdma_read_multiple_chunks(info); |
|
} else |
|
ret = svc_rdma_read_special(info); |
|
if (ret < 0) |
|
goto out_err; |
|
|
|
trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount); |
|
init_completion(&cc->cc_done); |
|
ret = svc_rdma_post_chunk_ctxt(cc); |
|
if (ret < 0) |
|
goto out_err; |
|
|
|
ret = 1; |
|
wait_for_completion(&cc->cc_done); |
|
if (cc->cc_status != IB_WC_SUCCESS) |
|
ret = -EIO; |
|
|
|
/* rq_respages starts after the last arg page */ |
|
rqstp->rq_respages = &rqstp->rq_pages[head->rc_page_count]; |
|
rqstp->rq_next_page = rqstp->rq_respages + 1; |
|
|
|
/* Ensure svc_rdma_recv_ctxt_put() does not try to release pages */ |
|
head->rc_page_count = 0; |
|
|
|
out_err: |
|
svc_rdma_read_info_free(info); |
|
return ret; |
|
}
|
|
|