mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2057 lines
54 KiB
2057 lines
54 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* |
|
* Copyright (C) 2017, Microsoft Corporation. |
|
* Copyright (C) 2018, LG Electronics. |
|
* |
|
* Author(s): Long Li <[email protected]>, |
|
* Hyunchul Lee <[email protected]> |
|
* |
|
* This program is free software; you can redistribute it and/or modify |
|
* it under the terms of the GNU General Public License as published by |
|
* the Free Software Foundation; either version 2 of the License, or |
|
* (at your option) any later version. |
|
* |
|
* This program is distributed in the hope that it will be useful, |
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See |
|
* the GNU General Public License for more details. |
|
*/ |
|
|
|
#define SUBMOD_NAME "smb_direct" |
|
|
|
#include <linux/kthread.h> |
|
#include <linux/list.h> |
|
#include <linux/mempool.h> |
|
#include <linux/highmem.h> |
|
#include <linux/scatterlist.h> |
|
#include <rdma/ib_verbs.h> |
|
#include <rdma/rdma_cm.h> |
|
#include <rdma/rw.h> |
|
|
|
#include "glob.h" |
|
#include "connection.h" |
|
#include "smb_common.h" |
|
#include "smbstatus.h" |
|
#include "transport_rdma.h" |
|
|
|
#define SMB_DIRECT_PORT 5445 |
|
|
|
#define SMB_DIRECT_VERSION_LE cpu_to_le16(0x0100) |
|
|
|
/* SMB_DIRECT negotiation timeout in seconds */ |
|
#define SMB_DIRECT_NEGOTIATE_TIMEOUT 120 |
|
|
|
#define SMB_DIRECT_MAX_SEND_SGES 8 |
|
#define SMB_DIRECT_MAX_RECV_SGES 1 |
|
|
|
/* |
|
* Default maximum number of RDMA read/write outstanding on this connection |
|
* This value is possibly decreased during QP creation on hardware limit |
|
*/ |
|
#define SMB_DIRECT_CM_INITIATOR_DEPTH 8 |
|
|
|
/* Maximum number of retries on data transfer operations */ |
|
#define SMB_DIRECT_CM_RETRY 6 |
|
/* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */ |
|
#define SMB_DIRECT_CM_RNR_RETRY 0 |
|
|
|
/* |
|
* User configurable initial values per SMB_DIRECT transport connection |
|
* as defined in [MS-SMBD] 3.1.1.1 |
|
* Those may change after a SMB_DIRECT negotiation |
|
*/ |
|
/* The local peer's maximum number of credits to grant to the peer */ |
|
static int smb_direct_receive_credit_max = 255; |
|
|
|
/* The remote peer's credit request of local peer */ |
|
static int smb_direct_send_credit_target = 255; |
|
|
|
/* The maximum single message size can be sent to remote peer */ |
|
static int smb_direct_max_send_size = 8192; |
|
|
|
/* The maximum fragmented upper-layer payload receive size supported */ |
|
static int smb_direct_max_fragmented_recv_size = 1024 * 1024; |
|
|
|
/* The maximum single-message size which can be received */ |
|
static int smb_direct_max_receive_size = 8192; |
|
|
|
static int smb_direct_max_read_write_size = 1024 * 1024; |
|
|
|
static int smb_direct_max_outstanding_rw_ops = 8; |
|
|
|
static struct smb_direct_listener { |
|
struct rdma_cm_id *cm_id; |
|
} smb_direct_listener; |
|
|
|
static struct workqueue_struct *smb_direct_wq; |
|
|
|
enum smb_direct_status { |
|
SMB_DIRECT_CS_NEW = 0, |
|
SMB_DIRECT_CS_CONNECTED, |
|
SMB_DIRECT_CS_DISCONNECTING, |
|
SMB_DIRECT_CS_DISCONNECTED, |
|
}; |
|
|
|
struct smb_direct_transport { |
|
struct ksmbd_transport transport; |
|
|
|
enum smb_direct_status status; |
|
bool full_packet_received; |
|
wait_queue_head_t wait_status; |
|
|
|
struct rdma_cm_id *cm_id; |
|
struct ib_cq *send_cq; |
|
struct ib_cq *recv_cq; |
|
struct ib_pd *pd; |
|
struct ib_qp *qp; |
|
|
|
int max_send_size; |
|
int max_recv_size; |
|
int max_fragmented_send_size; |
|
int max_fragmented_recv_size; |
|
int max_rdma_rw_size; |
|
|
|
spinlock_t reassembly_queue_lock; |
|
struct list_head reassembly_queue; |
|
int reassembly_data_length; |
|
int reassembly_queue_length; |
|
int first_entry_offset; |
|
wait_queue_head_t wait_reassembly_queue; |
|
|
|
spinlock_t receive_credit_lock; |
|
int recv_credits; |
|
int count_avail_recvmsg; |
|
int recv_credit_max; |
|
int recv_credit_target; |
|
|
|
spinlock_t recvmsg_queue_lock; |
|
struct list_head recvmsg_queue; |
|
|
|
spinlock_t empty_recvmsg_queue_lock; |
|
struct list_head empty_recvmsg_queue; |
|
|
|
int send_credit_target; |
|
atomic_t send_credits; |
|
spinlock_t lock_new_recv_credits; |
|
int new_recv_credits; |
|
atomic_t rw_avail_ops; |
|
|
|
wait_queue_head_t wait_send_credits; |
|
wait_queue_head_t wait_rw_avail_ops; |
|
|
|
mempool_t *sendmsg_mempool; |
|
struct kmem_cache *sendmsg_cache; |
|
mempool_t *recvmsg_mempool; |
|
struct kmem_cache *recvmsg_cache; |
|
|
|
wait_queue_head_t wait_send_payload_pending; |
|
atomic_t send_payload_pending; |
|
wait_queue_head_t wait_send_pending; |
|
atomic_t send_pending; |
|
|
|
struct delayed_work post_recv_credits_work; |
|
struct work_struct send_immediate_work; |
|
struct work_struct disconnect_work; |
|
|
|
bool negotiation_requested; |
|
}; |
|
|
|
#define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport)) |
|
|
|
enum { |
|
SMB_DIRECT_MSG_NEGOTIATE_REQ = 0, |
|
SMB_DIRECT_MSG_DATA_TRANSFER |
|
}; |
|
|
|
static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops; |
|
|
|
struct smb_direct_send_ctx { |
|
struct list_head msg_list; |
|
int wr_cnt; |
|
bool need_invalidate_rkey; |
|
unsigned int remote_key; |
|
}; |
|
|
|
struct smb_direct_sendmsg { |
|
struct smb_direct_transport *transport; |
|
struct ib_send_wr wr; |
|
struct list_head list; |
|
int num_sge; |
|
struct ib_sge sge[SMB_DIRECT_MAX_SEND_SGES]; |
|
struct ib_cqe cqe; |
|
u8 packet[]; |
|
}; |
|
|
|
struct smb_direct_recvmsg { |
|
struct smb_direct_transport *transport; |
|
struct list_head list; |
|
int type; |
|
struct ib_sge sge; |
|
struct ib_cqe cqe; |
|
bool first_segment; |
|
u8 packet[]; |
|
}; |
|
|
|
struct smb_direct_rdma_rw_msg { |
|
struct smb_direct_transport *t; |
|
struct ib_cqe cqe; |
|
struct completion *completion; |
|
struct rdma_rw_ctx rw_ctx; |
|
struct sg_table sgt; |
|
struct scatterlist sg_list[0]; |
|
}; |
|
|
|
static inline int get_buf_page_count(void *buf, int size) |
|
{ |
|
return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) - |
|
(uintptr_t)buf / PAGE_SIZE; |
|
} |
|
|
|
static void smb_direct_destroy_pools(struct smb_direct_transport *transport); |
|
static void smb_direct_post_recv_credits(struct work_struct *work); |
|
static int smb_direct_post_send_data(struct smb_direct_transport *t, |
|
struct smb_direct_send_ctx *send_ctx, |
|
struct kvec *iov, int niov, |
|
int remaining_data_length); |
|
|
|
static inline struct smb_direct_transport * |
|
smb_trans_direct_transfort(struct ksmbd_transport *t) |
|
{ |
|
return container_of(t, struct smb_direct_transport, transport); |
|
} |
|
|
|
static inline void |
|
*smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg) |
|
{ |
|
return (void *)recvmsg->packet; |
|
} |
|
|
|
static inline bool is_receive_credit_post_required(int receive_credits, |
|
int avail_recvmsg_count) |
|
{ |
|
return receive_credits <= (smb_direct_receive_credit_max >> 3) && |
|
avail_recvmsg_count >= (receive_credits >> 2); |
|
} |
|
|
|
static struct |
|
smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t) |
|
{ |
|
struct smb_direct_recvmsg *recvmsg = NULL; |
|
|
|
spin_lock(&t->recvmsg_queue_lock); |
|
if (!list_empty(&t->recvmsg_queue)) { |
|
recvmsg = list_first_entry(&t->recvmsg_queue, |
|
struct smb_direct_recvmsg, |
|
list); |
|
list_del(&recvmsg->list); |
|
} |
|
spin_unlock(&t->recvmsg_queue_lock); |
|
return recvmsg; |
|
} |
|
|
|
static void put_recvmsg(struct smb_direct_transport *t, |
|
struct smb_direct_recvmsg *recvmsg) |
|
{ |
|
ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, |
|
recvmsg->sge.length, DMA_FROM_DEVICE); |
|
|
|
spin_lock(&t->recvmsg_queue_lock); |
|
list_add(&recvmsg->list, &t->recvmsg_queue); |
|
spin_unlock(&t->recvmsg_queue_lock); |
|
} |
|
|
|
static struct |
|
smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t) |
|
{ |
|
struct smb_direct_recvmsg *recvmsg = NULL; |
|
|
|
spin_lock(&t->empty_recvmsg_queue_lock); |
|
if (!list_empty(&t->empty_recvmsg_queue)) { |
|
recvmsg = list_first_entry(&t->empty_recvmsg_queue, |
|
struct smb_direct_recvmsg, list); |
|
list_del(&recvmsg->list); |
|
} |
|
spin_unlock(&t->empty_recvmsg_queue_lock); |
|
return recvmsg; |
|
} |
|
|
|
static void put_empty_recvmsg(struct smb_direct_transport *t, |
|
struct smb_direct_recvmsg *recvmsg) |
|
{ |
|
ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, |
|
recvmsg->sge.length, DMA_FROM_DEVICE); |
|
|
|
spin_lock(&t->empty_recvmsg_queue_lock); |
|
list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue); |
|
spin_unlock(&t->empty_recvmsg_queue_lock); |
|
} |
|
|
|
static void enqueue_reassembly(struct smb_direct_transport *t, |
|
struct smb_direct_recvmsg *recvmsg, |
|
int data_length) |
|
{ |
|
spin_lock(&t->reassembly_queue_lock); |
|
list_add_tail(&recvmsg->list, &t->reassembly_queue); |
|
t->reassembly_queue_length++; |
|
/* |
|
* Make sure reassembly_data_length is updated after list and |
|
* reassembly_queue_length are updated. On the dequeue side |
|
* reassembly_data_length is checked without a lock to determine |
|
* if reassembly_queue_length and list is up to date |
|
*/ |
|
virt_wmb(); |
|
t->reassembly_data_length += data_length; |
|
spin_unlock(&t->reassembly_queue_lock); |
|
} |
|
|
|
static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t) |
|
{ |
|
if (!list_empty(&t->reassembly_queue)) |
|
return list_first_entry(&t->reassembly_queue, |
|
struct smb_direct_recvmsg, list); |
|
else |
|
return NULL; |
|
} |
|
|
|
static void smb_direct_disconnect_rdma_work(struct work_struct *work) |
|
{ |
|
struct smb_direct_transport *t = |
|
container_of(work, struct smb_direct_transport, |
|
disconnect_work); |
|
|
|
if (t->status == SMB_DIRECT_CS_CONNECTED) { |
|
t->status = SMB_DIRECT_CS_DISCONNECTING; |
|
rdma_disconnect(t->cm_id); |
|
} |
|
} |
|
|
|
static void |
|
smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t) |
|
{ |
|
if (t->status == SMB_DIRECT_CS_CONNECTED) |
|
queue_work(smb_direct_wq, &t->disconnect_work); |
|
} |
|
|
|
static void smb_direct_send_immediate_work(struct work_struct *work) |
|
{ |
|
struct smb_direct_transport *t = container_of(work, |
|
struct smb_direct_transport, send_immediate_work); |
|
|
|
if (t->status != SMB_DIRECT_CS_CONNECTED) |
|
return; |
|
|
|
smb_direct_post_send_data(t, NULL, NULL, 0, 0); |
|
} |
|
|
|
static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) |
|
{ |
|
struct smb_direct_transport *t; |
|
struct ksmbd_conn *conn; |
|
|
|
t = kzalloc(sizeof(*t), GFP_KERNEL); |
|
if (!t) |
|
return NULL; |
|
|
|
t->cm_id = cm_id; |
|
cm_id->context = t; |
|
|
|
t->status = SMB_DIRECT_CS_NEW; |
|
init_waitqueue_head(&t->wait_status); |
|
|
|
spin_lock_init(&t->reassembly_queue_lock); |
|
INIT_LIST_HEAD(&t->reassembly_queue); |
|
t->reassembly_data_length = 0; |
|
t->reassembly_queue_length = 0; |
|
init_waitqueue_head(&t->wait_reassembly_queue); |
|
init_waitqueue_head(&t->wait_send_credits); |
|
init_waitqueue_head(&t->wait_rw_avail_ops); |
|
|
|
spin_lock_init(&t->receive_credit_lock); |
|
spin_lock_init(&t->recvmsg_queue_lock); |
|
INIT_LIST_HEAD(&t->recvmsg_queue); |
|
|
|
spin_lock_init(&t->empty_recvmsg_queue_lock); |
|
INIT_LIST_HEAD(&t->empty_recvmsg_queue); |
|
|
|
init_waitqueue_head(&t->wait_send_payload_pending); |
|
atomic_set(&t->send_payload_pending, 0); |
|
init_waitqueue_head(&t->wait_send_pending); |
|
atomic_set(&t->send_pending, 0); |
|
|
|
spin_lock_init(&t->lock_new_recv_credits); |
|
|
|
INIT_DELAYED_WORK(&t->post_recv_credits_work, |
|
smb_direct_post_recv_credits); |
|
INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work); |
|
INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work); |
|
|
|
conn = ksmbd_conn_alloc(); |
|
if (!conn) |
|
goto err; |
|
conn->transport = KSMBD_TRANS(t); |
|
KSMBD_TRANS(t)->conn = conn; |
|
KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops; |
|
return t; |
|
err: |
|
kfree(t); |
|
return NULL; |
|
} |
|
|
|
static void free_transport(struct smb_direct_transport *t) |
|
{ |
|
struct smb_direct_recvmsg *recvmsg; |
|
|
|
wake_up_interruptible(&t->wait_send_credits); |
|
|
|
ksmbd_debug(RDMA, "wait for all send posted to IB to finish\n"); |
|
wait_event(t->wait_send_payload_pending, |
|
atomic_read(&t->send_payload_pending) == 0); |
|
wait_event(t->wait_send_pending, |
|
atomic_read(&t->send_pending) == 0); |
|
|
|
cancel_work_sync(&t->disconnect_work); |
|
cancel_delayed_work_sync(&t->post_recv_credits_work); |
|
cancel_work_sync(&t->send_immediate_work); |
|
|
|
if (t->qp) { |
|
ib_drain_qp(t->qp); |
|
ib_destroy_qp(t->qp); |
|
} |
|
|
|
ksmbd_debug(RDMA, "drain the reassembly queue\n"); |
|
do { |
|
spin_lock(&t->reassembly_queue_lock); |
|
recvmsg = get_first_reassembly(t); |
|
if (recvmsg) { |
|
list_del(&recvmsg->list); |
|
spin_unlock(&t->reassembly_queue_lock); |
|
put_recvmsg(t, recvmsg); |
|
} else { |
|
spin_unlock(&t->reassembly_queue_lock); |
|
} |
|
} while (recvmsg); |
|
t->reassembly_data_length = 0; |
|
|
|
if (t->send_cq) |
|
ib_free_cq(t->send_cq); |
|
if (t->recv_cq) |
|
ib_free_cq(t->recv_cq); |
|
if (t->pd) |
|
ib_dealloc_pd(t->pd); |
|
if (t->cm_id) |
|
rdma_destroy_id(t->cm_id); |
|
|
|
smb_direct_destroy_pools(t); |
|
ksmbd_conn_free(KSMBD_TRANS(t)->conn); |
|
kfree(t); |
|
} |
|
|
|
static struct smb_direct_sendmsg |
|
*smb_direct_alloc_sendmsg(struct smb_direct_transport *t) |
|
{ |
|
struct smb_direct_sendmsg *msg; |
|
|
|
msg = mempool_alloc(t->sendmsg_mempool, GFP_KERNEL); |
|
if (!msg) |
|
return ERR_PTR(-ENOMEM); |
|
msg->transport = t; |
|
INIT_LIST_HEAD(&msg->list); |
|
msg->num_sge = 0; |
|
return msg; |
|
} |
|
|
|
static void smb_direct_free_sendmsg(struct smb_direct_transport *t, |
|
struct smb_direct_sendmsg *msg) |
|
{ |
|
int i; |
|
|
|
if (msg->num_sge > 0) { |
|
ib_dma_unmap_single(t->cm_id->device, |
|
msg->sge[0].addr, msg->sge[0].length, |
|
DMA_TO_DEVICE); |
|
for (i = 1; i < msg->num_sge; i++) |
|
ib_dma_unmap_page(t->cm_id->device, |
|
msg->sge[i].addr, msg->sge[i].length, |
|
DMA_TO_DEVICE); |
|
} |
|
mempool_free(msg, t->sendmsg_mempool); |
|
} |
|
|
|
static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg) |
|
{ |
|
switch (recvmsg->type) { |
|
case SMB_DIRECT_MSG_DATA_TRANSFER: { |
|
struct smb_direct_data_transfer *req = |
|
(struct smb_direct_data_transfer *)recvmsg->packet; |
|
struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet |
|
+ le32_to_cpu(req->data_offset) - 4); |
|
ksmbd_debug(RDMA, |
|
"CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n", |
|
le16_to_cpu(req->credits_granted), |
|
le16_to_cpu(req->credits_requested), |
|
req->data_length, req->remaining_data_length, |
|
hdr->ProtocolId, hdr->Command); |
|
break; |
|
} |
|
case SMB_DIRECT_MSG_NEGOTIATE_REQ: { |
|
struct smb_direct_negotiate_req *req = |
|
(struct smb_direct_negotiate_req *)recvmsg->packet; |
|
ksmbd_debug(RDMA, |
|
"MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n", |
|
le16_to_cpu(req->min_version), |
|
le16_to_cpu(req->max_version), |
|
le16_to_cpu(req->credits_requested), |
|
le32_to_cpu(req->preferred_send_size), |
|
le32_to_cpu(req->max_receive_size), |
|
le32_to_cpu(req->max_fragmented_size)); |
|
if (le16_to_cpu(req->min_version) > 0x0100 || |
|
le16_to_cpu(req->max_version) < 0x0100) |
|
return -EOPNOTSUPP; |
|
if (le16_to_cpu(req->credits_requested) <= 0 || |
|
le32_to_cpu(req->max_receive_size) <= 128 || |
|
le32_to_cpu(req->max_fragmented_size) <= |
|
128 * 1024) |
|
return -ECONNABORTED; |
|
|
|
break; |
|
} |
|
default: |
|
return -EINVAL; |
|
} |
|
return 0; |
|
} |
|
|
|
static void recv_done(struct ib_cq *cq, struct ib_wc *wc) |
|
{ |
|
struct smb_direct_recvmsg *recvmsg; |
|
struct smb_direct_transport *t; |
|
|
|
recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe); |
|
t = recvmsg->transport; |
|
|
|
if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { |
|
if (wc->status != IB_WC_WR_FLUSH_ERR) { |
|
pr_err("Recv error. status='%s (%d)' opcode=%d\n", |
|
ib_wc_status_msg(wc->status), wc->status, |
|
wc->opcode); |
|
smb_direct_disconnect_rdma_connection(t); |
|
} |
|
put_empty_recvmsg(t, recvmsg); |
|
return; |
|
} |
|
|
|
ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n", |
|
ib_wc_status_msg(wc->status), wc->status, |
|
wc->opcode); |
|
|
|
ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr, |
|
recvmsg->sge.length, DMA_FROM_DEVICE); |
|
|
|
switch (recvmsg->type) { |
|
case SMB_DIRECT_MSG_NEGOTIATE_REQ: |
|
t->negotiation_requested = true; |
|
t->full_packet_received = true; |
|
wake_up_interruptible(&t->wait_status); |
|
break; |
|
case SMB_DIRECT_MSG_DATA_TRANSFER: { |
|
struct smb_direct_data_transfer *data_transfer = |
|
(struct smb_direct_data_transfer *)recvmsg->packet; |
|
int data_length = le32_to_cpu(data_transfer->data_length); |
|
int avail_recvmsg_count, receive_credits; |
|
|
|
if (data_length) { |
|
if (t->full_packet_received) |
|
recvmsg->first_segment = true; |
|
|
|
if (le32_to_cpu(data_transfer->remaining_data_length)) |
|
t->full_packet_received = false; |
|
else |
|
t->full_packet_received = true; |
|
|
|
enqueue_reassembly(t, recvmsg, data_length); |
|
wake_up_interruptible(&t->wait_reassembly_queue); |
|
|
|
spin_lock(&t->receive_credit_lock); |
|
receive_credits = --(t->recv_credits); |
|
avail_recvmsg_count = t->count_avail_recvmsg; |
|
spin_unlock(&t->receive_credit_lock); |
|
} else { |
|
put_empty_recvmsg(t, recvmsg); |
|
|
|
spin_lock(&t->receive_credit_lock); |
|
receive_credits = --(t->recv_credits); |
|
avail_recvmsg_count = ++(t->count_avail_recvmsg); |
|
spin_unlock(&t->receive_credit_lock); |
|
} |
|
|
|
t->recv_credit_target = |
|
le16_to_cpu(data_transfer->credits_requested); |
|
atomic_add(le16_to_cpu(data_transfer->credits_granted), |
|
&t->send_credits); |
|
|
|
if (le16_to_cpu(data_transfer->flags) & |
|
SMB_DIRECT_RESPONSE_REQUESTED) |
|
queue_work(smb_direct_wq, &t->send_immediate_work); |
|
|
|
if (atomic_read(&t->send_credits) > 0) |
|
wake_up_interruptible(&t->wait_send_credits); |
|
|
|
if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count)) |
|
mod_delayed_work(smb_direct_wq, |
|
&t->post_recv_credits_work, 0); |
|
break; |
|
} |
|
default: |
|
break; |
|
} |
|
} |
|
|
|
static int smb_direct_post_recv(struct smb_direct_transport *t, |
|
struct smb_direct_recvmsg *recvmsg) |
|
{ |
|
struct ib_recv_wr wr; |
|
int ret; |
|
|
|
recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device, |
|
recvmsg->packet, t->max_recv_size, |
|
DMA_FROM_DEVICE); |
|
ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr); |
|
if (ret) |
|
return ret; |
|
recvmsg->sge.length = t->max_recv_size; |
|
recvmsg->sge.lkey = t->pd->local_dma_lkey; |
|
recvmsg->cqe.done = recv_done; |
|
|
|
wr.wr_cqe = &recvmsg->cqe; |
|
wr.next = NULL; |
|
wr.sg_list = &recvmsg->sge; |
|
wr.num_sge = 1; |
|
|
|
ret = ib_post_recv(t->qp, &wr, NULL); |
|
if (ret) { |
|
pr_err("Can't post recv: %d\n", ret); |
|
ib_dma_unmap_single(t->cm_id->device, |
|
recvmsg->sge.addr, recvmsg->sge.length, |
|
DMA_FROM_DEVICE); |
|
smb_direct_disconnect_rdma_connection(t); |
|
return ret; |
|
} |
|
return ret; |
|
} |
|
|
|
static int smb_direct_read(struct ksmbd_transport *t, char *buf, |
|
unsigned int size) |
|
{ |
|
struct smb_direct_recvmsg *recvmsg; |
|
struct smb_direct_data_transfer *data_transfer; |
|
int to_copy, to_read, data_read, offset; |
|
u32 data_length, remaining_data_length, data_offset; |
|
int rc; |
|
struct smb_direct_transport *st = smb_trans_direct_transfort(t); |
|
|
|
again: |
|
if (st->status != SMB_DIRECT_CS_CONNECTED) { |
|
pr_err("disconnected\n"); |
|
return -ENOTCONN; |
|
} |
|
|
|
/* |
|
* No need to hold the reassembly queue lock all the time as we are |
|
* the only one reading from the front of the queue. The transport |
|
* may add more entries to the back of the queue at the same time |
|
*/ |
|
if (st->reassembly_data_length >= size) { |
|
int queue_length; |
|
int queue_removed = 0; |
|
|
|
/* |
|
* Need to make sure reassembly_data_length is read before |
|
* reading reassembly_queue_length and calling |
|
* get_first_reassembly. This call is lock free |
|
* as we never read at the end of the queue which are being |
|
* updated in SOFTIRQ as more data is received |
|
*/ |
|
virt_rmb(); |
|
queue_length = st->reassembly_queue_length; |
|
data_read = 0; |
|
to_read = size; |
|
offset = st->first_entry_offset; |
|
while (data_read < size) { |
|
recvmsg = get_first_reassembly(st); |
|
data_transfer = smb_direct_recvmsg_payload(recvmsg); |
|
data_length = le32_to_cpu(data_transfer->data_length); |
|
remaining_data_length = |
|
le32_to_cpu(data_transfer->remaining_data_length); |
|
data_offset = le32_to_cpu(data_transfer->data_offset); |
|
|
|
/* |
|
* The upper layer expects RFC1002 length at the |
|
* beginning of the payload. Return it to indicate |
|
* the total length of the packet. This minimize the |
|
* change to upper layer packet processing logic. This |
|
* will be eventually remove when an intermediate |
|
* transport layer is added |
|
*/ |
|
if (recvmsg->first_segment && size == 4) { |
|
unsigned int rfc1002_len = |
|
data_length + remaining_data_length; |
|
*((__be32 *)buf) = cpu_to_be32(rfc1002_len); |
|
data_read = 4; |
|
recvmsg->first_segment = false; |
|
ksmbd_debug(RDMA, |
|
"returning rfc1002 length %d\n", |
|
rfc1002_len); |
|
goto read_rfc1002_done; |
|
} |
|
|
|
to_copy = min_t(int, data_length - offset, to_read); |
|
memcpy(buf + data_read, (char *)data_transfer + data_offset + offset, |
|
to_copy); |
|
|
|
/* move on to the next buffer? */ |
|
if (to_copy == data_length - offset) { |
|
queue_length--; |
|
/* |
|
* No need to lock if we are not at the |
|
* end of the queue |
|
*/ |
|
if (queue_length) { |
|
list_del(&recvmsg->list); |
|
} else { |
|
spin_lock_irq(&st->reassembly_queue_lock); |
|
list_del(&recvmsg->list); |
|
spin_unlock_irq(&st->reassembly_queue_lock); |
|
} |
|
queue_removed++; |
|
put_recvmsg(st, recvmsg); |
|
offset = 0; |
|
} else { |
|
offset += to_copy; |
|
} |
|
|
|
to_read -= to_copy; |
|
data_read += to_copy; |
|
} |
|
|
|
spin_lock_irq(&st->reassembly_queue_lock); |
|
st->reassembly_data_length -= data_read; |
|
st->reassembly_queue_length -= queue_removed; |
|
spin_unlock_irq(&st->reassembly_queue_lock); |
|
|
|
spin_lock(&st->receive_credit_lock); |
|
st->count_avail_recvmsg += queue_removed; |
|
if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) { |
|
spin_unlock(&st->receive_credit_lock); |
|
mod_delayed_work(smb_direct_wq, |
|
&st->post_recv_credits_work, 0); |
|
} else { |
|
spin_unlock(&st->receive_credit_lock); |
|
} |
|
|
|
st->first_entry_offset = offset; |
|
ksmbd_debug(RDMA, |
|
"returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n", |
|
data_read, st->reassembly_data_length, |
|
st->first_entry_offset); |
|
read_rfc1002_done: |
|
return data_read; |
|
} |
|
|
|
ksmbd_debug(RDMA, "wait_event on more data\n"); |
|
rc = wait_event_interruptible(st->wait_reassembly_queue, |
|
st->reassembly_data_length >= size || |
|
st->status != SMB_DIRECT_CS_CONNECTED); |
|
if (rc) |
|
return -EINTR; |
|
|
|
goto again; |
|
} |
|
|
|
static void smb_direct_post_recv_credits(struct work_struct *work) |
|
{ |
|
struct smb_direct_transport *t = container_of(work, |
|
struct smb_direct_transport, post_recv_credits_work.work); |
|
struct smb_direct_recvmsg *recvmsg; |
|
int receive_credits, credits = 0; |
|
int ret; |
|
int use_free = 1; |
|
|
|
spin_lock(&t->receive_credit_lock); |
|
receive_credits = t->recv_credits; |
|
spin_unlock(&t->receive_credit_lock); |
|
|
|
if (receive_credits < t->recv_credit_target) { |
|
while (true) { |
|
if (use_free) |
|
recvmsg = get_free_recvmsg(t); |
|
else |
|
recvmsg = get_empty_recvmsg(t); |
|
if (!recvmsg) { |
|
if (use_free) { |
|
use_free = 0; |
|
continue; |
|
} else { |
|
break; |
|
} |
|
} |
|
|
|
recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER; |
|
recvmsg->first_segment = false; |
|
|
|
ret = smb_direct_post_recv(t, recvmsg); |
|
if (ret) { |
|
pr_err("Can't post recv: %d\n", ret); |
|
put_recvmsg(t, recvmsg); |
|
break; |
|
} |
|
credits++; |
|
} |
|
} |
|
|
|
spin_lock(&t->receive_credit_lock); |
|
t->recv_credits += credits; |
|
t->count_avail_recvmsg -= credits; |
|
spin_unlock(&t->receive_credit_lock); |
|
|
|
spin_lock(&t->lock_new_recv_credits); |
|
t->new_recv_credits += credits; |
|
spin_unlock(&t->lock_new_recv_credits); |
|
|
|
if (credits) |
|
queue_work(smb_direct_wq, &t->send_immediate_work); |
|
} |
|
|
|
static void send_done(struct ib_cq *cq, struct ib_wc *wc) |
|
{ |
|
struct smb_direct_sendmsg *sendmsg, *sibling; |
|
struct smb_direct_transport *t; |
|
struct list_head *pos, *prev, *end; |
|
|
|
sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe); |
|
t = sendmsg->transport; |
|
|
|
ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n", |
|
ib_wc_status_msg(wc->status), wc->status, |
|
wc->opcode); |
|
|
|
if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { |
|
pr_err("Send error. status='%s (%d)', opcode=%d\n", |
|
ib_wc_status_msg(wc->status), wc->status, |
|
wc->opcode); |
|
smb_direct_disconnect_rdma_connection(t); |
|
} |
|
|
|
if (sendmsg->num_sge > 1) { |
|
if (atomic_dec_and_test(&t->send_payload_pending)) |
|
wake_up(&t->wait_send_payload_pending); |
|
} else { |
|
if (atomic_dec_and_test(&t->send_pending)) |
|
wake_up(&t->wait_send_pending); |
|
} |
|
|
|
/* iterate and free the list of messages in reverse. the list's head |
|
* is invalid. |
|
*/ |
|
for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next; |
|
prev != end; pos = prev, prev = prev->prev) { |
|
sibling = container_of(pos, struct smb_direct_sendmsg, list); |
|
smb_direct_free_sendmsg(t, sibling); |
|
} |
|
|
|
sibling = container_of(pos, struct smb_direct_sendmsg, list); |
|
smb_direct_free_sendmsg(t, sibling); |
|
} |
|
|
|
static int manage_credits_prior_sending(struct smb_direct_transport *t) |
|
{ |
|
int new_credits; |
|
|
|
spin_lock(&t->lock_new_recv_credits); |
|
new_credits = t->new_recv_credits; |
|
t->new_recv_credits = 0; |
|
spin_unlock(&t->lock_new_recv_credits); |
|
|
|
return new_credits; |
|
} |
|
|
|
static int smb_direct_post_send(struct smb_direct_transport *t, |
|
struct ib_send_wr *wr) |
|
{ |
|
int ret; |
|
|
|
if (wr->num_sge > 1) |
|
atomic_inc(&t->send_payload_pending); |
|
else |
|
atomic_inc(&t->send_pending); |
|
|
|
ret = ib_post_send(t->qp, wr, NULL); |
|
if (ret) { |
|
pr_err("failed to post send: %d\n", ret); |
|
if (wr->num_sge > 1) { |
|
if (atomic_dec_and_test(&t->send_payload_pending)) |
|
wake_up(&t->wait_send_payload_pending); |
|
} else { |
|
if (atomic_dec_and_test(&t->send_pending)) |
|
wake_up(&t->wait_send_pending); |
|
} |
|
smb_direct_disconnect_rdma_connection(t); |
|
} |
|
return ret; |
|
} |
|
|
|
static void smb_direct_send_ctx_init(struct smb_direct_transport *t, |
|
struct smb_direct_send_ctx *send_ctx, |
|
bool need_invalidate_rkey, |
|
unsigned int remote_key) |
|
{ |
|
INIT_LIST_HEAD(&send_ctx->msg_list); |
|
send_ctx->wr_cnt = 0; |
|
send_ctx->need_invalidate_rkey = need_invalidate_rkey; |
|
send_ctx->remote_key = remote_key; |
|
} |
|
|
|
static int smb_direct_flush_send_list(struct smb_direct_transport *t, |
|
struct smb_direct_send_ctx *send_ctx, |
|
bool is_last) |
|
{ |
|
struct smb_direct_sendmsg *first, *last; |
|
int ret; |
|
|
|
if (list_empty(&send_ctx->msg_list)) |
|
return 0; |
|
|
|
first = list_first_entry(&send_ctx->msg_list, |
|
struct smb_direct_sendmsg, |
|
list); |
|
last = list_last_entry(&send_ctx->msg_list, |
|
struct smb_direct_sendmsg, |
|
list); |
|
|
|
last->wr.send_flags = IB_SEND_SIGNALED; |
|
last->wr.wr_cqe = &last->cqe; |
|
if (is_last && send_ctx->need_invalidate_rkey) { |
|
last->wr.opcode = IB_WR_SEND_WITH_INV; |
|
last->wr.ex.invalidate_rkey = send_ctx->remote_key; |
|
} |
|
|
|
ret = smb_direct_post_send(t, &first->wr); |
|
if (!ret) { |
|
smb_direct_send_ctx_init(t, send_ctx, |
|
send_ctx->need_invalidate_rkey, |
|
send_ctx->remote_key); |
|
} else { |
|
atomic_add(send_ctx->wr_cnt, &t->send_credits); |
|
wake_up(&t->wait_send_credits); |
|
list_for_each_entry_safe(first, last, &send_ctx->msg_list, |
|
list) { |
|
smb_direct_free_sendmsg(t, first); |
|
} |
|
} |
|
return ret; |
|
} |
|
|
|
static int wait_for_credits(struct smb_direct_transport *t, |
|
wait_queue_head_t *waitq, atomic_t *credits) |
|
{ |
|
int ret; |
|
|
|
do { |
|
if (atomic_dec_return(credits) >= 0) |
|
return 0; |
|
|
|
atomic_inc(credits); |
|
ret = wait_event_interruptible(*waitq, |
|
atomic_read(credits) > 0 || |
|
t->status != SMB_DIRECT_CS_CONNECTED); |
|
|
|
if (t->status != SMB_DIRECT_CS_CONNECTED) |
|
return -ENOTCONN; |
|
else if (ret < 0) |
|
return ret; |
|
} while (true); |
|
} |
|
|
|
static int wait_for_send_credits(struct smb_direct_transport *t, |
|
struct smb_direct_send_ctx *send_ctx) |
|
{ |
|
int ret; |
|
|
|
if (send_ctx && |
|
(send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) { |
|
ret = smb_direct_flush_send_list(t, send_ctx, false); |
|
if (ret) |
|
return ret; |
|
} |
|
|
|
return wait_for_credits(t, &t->wait_send_credits, &t->send_credits); |
|
} |
|
|
|
static int smb_direct_create_header(struct smb_direct_transport *t, |
|
int size, int remaining_data_length, |
|
struct smb_direct_sendmsg **sendmsg_out) |
|
{ |
|
struct smb_direct_sendmsg *sendmsg; |
|
struct smb_direct_data_transfer *packet; |
|
int header_length; |
|
int ret; |
|
|
|
sendmsg = smb_direct_alloc_sendmsg(t); |
|
if (IS_ERR(sendmsg)) |
|
return PTR_ERR(sendmsg); |
|
|
|
/* Fill in the packet header */ |
|
packet = (struct smb_direct_data_transfer *)sendmsg->packet; |
|
packet->credits_requested = cpu_to_le16(t->send_credit_target); |
|
packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t)); |
|
|
|
packet->flags = 0; |
|
packet->reserved = 0; |
|
if (!size) |
|
packet->data_offset = 0; |
|
else |
|
packet->data_offset = cpu_to_le32(24); |
|
packet->data_length = cpu_to_le32(size); |
|
packet->remaining_data_length = cpu_to_le32(remaining_data_length); |
|
packet->padding = 0; |
|
|
|
ksmbd_debug(RDMA, |
|
"credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n", |
|
le16_to_cpu(packet->credits_requested), |
|
le16_to_cpu(packet->credits_granted), |
|
le32_to_cpu(packet->data_offset), |
|
le32_to_cpu(packet->data_length), |
|
le32_to_cpu(packet->remaining_data_length)); |
|
|
|
/* Map the packet to DMA */ |
|
header_length = sizeof(struct smb_direct_data_transfer); |
|
/* If this is a packet without payload, don't send padding */ |
|
if (!size) |
|
header_length = |
|
offsetof(struct smb_direct_data_transfer, padding); |
|
|
|
sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device, |
|
(void *)packet, |
|
header_length, |
|
DMA_TO_DEVICE); |
|
ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr); |
|
if (ret) { |
|
smb_direct_free_sendmsg(t, sendmsg); |
|
return ret; |
|
} |
|
|
|
sendmsg->num_sge = 1; |
|
sendmsg->sge[0].length = header_length; |
|
sendmsg->sge[0].lkey = t->pd->local_dma_lkey; |
|
|
|
*sendmsg_out = sendmsg; |
|
return 0; |
|
} |
|
|
|
static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries) |
|
{ |
|
bool high = is_vmalloc_addr(buf); |
|
struct page *page; |
|
int offset, len; |
|
int i = 0; |
|
|
|
if (nentries < get_buf_page_count(buf, size)) |
|
return -EINVAL; |
|
|
|
offset = offset_in_page(buf); |
|
buf -= offset; |
|
while (size > 0) { |
|
len = min_t(int, PAGE_SIZE - offset, size); |
|
if (high) |
|
page = vmalloc_to_page(buf); |
|
else |
|
page = kmap_to_page(buf); |
|
|
|
if (!sg_list) |
|
return -EINVAL; |
|
sg_set_page(sg_list, page, len, offset); |
|
sg_list = sg_next(sg_list); |
|
|
|
buf += PAGE_SIZE; |
|
size -= len; |
|
offset = 0; |
|
i++; |
|
} |
|
return i; |
|
} |
|
|
|
static int get_mapped_sg_list(struct ib_device *device, void *buf, int size, |
|
struct scatterlist *sg_list, int nentries, |
|
enum dma_data_direction dir) |
|
{ |
|
int npages; |
|
|
|
npages = get_sg_list(buf, size, sg_list, nentries); |
|
if (npages <= 0) |
|
return -EINVAL; |
|
return ib_dma_map_sg(device, sg_list, npages, dir); |
|
} |
|
|
|
static int post_sendmsg(struct smb_direct_transport *t, |
|
struct smb_direct_send_ctx *send_ctx, |
|
struct smb_direct_sendmsg *msg) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < msg->num_sge; i++) |
|
ib_dma_sync_single_for_device(t->cm_id->device, |
|
msg->sge[i].addr, msg->sge[i].length, |
|
DMA_TO_DEVICE); |
|
|
|
msg->cqe.done = send_done; |
|
msg->wr.opcode = IB_WR_SEND; |
|
msg->wr.sg_list = &msg->sge[0]; |
|
msg->wr.num_sge = msg->num_sge; |
|
msg->wr.next = NULL; |
|
|
|
if (send_ctx) { |
|
msg->wr.wr_cqe = NULL; |
|
msg->wr.send_flags = 0; |
|
if (!list_empty(&send_ctx->msg_list)) { |
|
struct smb_direct_sendmsg *last; |
|
|
|
last = list_last_entry(&send_ctx->msg_list, |
|
struct smb_direct_sendmsg, |
|
list); |
|
last->wr.next = &msg->wr; |
|
} |
|
list_add_tail(&msg->list, &send_ctx->msg_list); |
|
send_ctx->wr_cnt++; |
|
return 0; |
|
} |
|
|
|
msg->wr.wr_cqe = &msg->cqe; |
|
msg->wr.send_flags = IB_SEND_SIGNALED; |
|
return smb_direct_post_send(t, &msg->wr); |
|
} |
|
|
|
static int smb_direct_post_send_data(struct smb_direct_transport *t, |
|
struct smb_direct_send_ctx *send_ctx, |
|
struct kvec *iov, int niov, |
|
int remaining_data_length) |
|
{ |
|
int i, j, ret; |
|
struct smb_direct_sendmsg *msg; |
|
int data_length; |
|
struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1]; |
|
|
|
ret = wait_for_send_credits(t, send_ctx); |
|
if (ret) |
|
return ret; |
|
|
|
data_length = 0; |
|
for (i = 0; i < niov; i++) |
|
data_length += iov[i].iov_len; |
|
|
|
ret = smb_direct_create_header(t, data_length, remaining_data_length, |
|
&msg); |
|
if (ret) { |
|
atomic_inc(&t->send_credits); |
|
return ret; |
|
} |
|
|
|
for (i = 0; i < niov; i++) { |
|
struct ib_sge *sge; |
|
int sg_cnt; |
|
|
|
sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1); |
|
sg_cnt = get_mapped_sg_list(t->cm_id->device, |
|
iov[i].iov_base, iov[i].iov_len, |
|
sg, SMB_DIRECT_MAX_SEND_SGES - 1, |
|
DMA_TO_DEVICE); |
|
if (sg_cnt <= 0) { |
|
pr_err("failed to map buffer\n"); |
|
ret = -ENOMEM; |
|
goto err; |
|
} else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) { |
|
pr_err("buffer not fitted into sges\n"); |
|
ret = -E2BIG; |
|
ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt, |
|
DMA_TO_DEVICE); |
|
goto err; |
|
} |
|
|
|
for (j = 0; j < sg_cnt; j++) { |
|
sge = &msg->sge[msg->num_sge]; |
|
sge->addr = sg_dma_address(&sg[j]); |
|
sge->length = sg_dma_len(&sg[j]); |
|
sge->lkey = t->pd->local_dma_lkey; |
|
msg->num_sge++; |
|
} |
|
} |
|
|
|
ret = post_sendmsg(t, send_ctx, msg); |
|
if (ret) |
|
goto err; |
|
return 0; |
|
err: |
|
smb_direct_free_sendmsg(t, msg); |
|
atomic_inc(&t->send_credits); |
|
return ret; |
|
} |
|
|
|
static int smb_direct_writev(struct ksmbd_transport *t, |
|
struct kvec *iov, int niovs, int buflen, |
|
bool need_invalidate, unsigned int remote_key) |
|
{ |
|
struct smb_direct_transport *st = smb_trans_direct_transfort(t); |
|
int remaining_data_length; |
|
int start, i, j; |
|
int max_iov_size = st->max_send_size - |
|
sizeof(struct smb_direct_data_transfer); |
|
int ret; |
|
struct kvec vec; |
|
struct smb_direct_send_ctx send_ctx; |
|
|
|
if (st->status != SMB_DIRECT_CS_CONNECTED) |
|
return -ENOTCONN; |
|
|
|
//FIXME: skip RFC1002 header.. |
|
buflen -= 4; |
|
iov[0].iov_base += 4; |
|
iov[0].iov_len -= 4; |
|
|
|
remaining_data_length = buflen; |
|
ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen); |
|
|
|
smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key); |
|
start = i = 0; |
|
buflen = 0; |
|
while (true) { |
|
buflen += iov[i].iov_len; |
|
if (buflen > max_iov_size) { |
|
if (i > start) { |
|
remaining_data_length -= |
|
(buflen - iov[i].iov_len); |
|
ret = smb_direct_post_send_data(st, &send_ctx, |
|
&iov[start], i - start, |
|
remaining_data_length); |
|
if (ret) |
|
goto done; |
|
} else { |
|
/* iov[start] is too big, break it */ |
|
int nvec = (buflen + max_iov_size - 1) / |
|
max_iov_size; |
|
|
|
for (j = 0; j < nvec; j++) { |
|
vec.iov_base = |
|
(char *)iov[start].iov_base + |
|
j * max_iov_size; |
|
vec.iov_len = |
|
min_t(int, max_iov_size, |
|
buflen - max_iov_size * j); |
|
remaining_data_length -= vec.iov_len; |
|
ret = smb_direct_post_send_data(st, &send_ctx, &vec, 1, |
|
remaining_data_length); |
|
if (ret) |
|
goto done; |
|
} |
|
i++; |
|
if (i == niovs) |
|
break; |
|
} |
|
start = i; |
|
buflen = 0; |
|
} else { |
|
i++; |
|
if (i == niovs) { |
|
/* send out all remaining vecs */ |
|
remaining_data_length -= buflen; |
|
ret = smb_direct_post_send_data(st, &send_ctx, |
|
&iov[start], i - start, |
|
remaining_data_length); |
|
if (ret) |
|
goto done; |
|
break; |
|
} |
|
} |
|
} |
|
|
|
done: |
|
ret = smb_direct_flush_send_list(st, &send_ctx, true); |
|
|
|
/* |
|
* As an optimization, we don't wait for individual I/O to finish |
|
* before sending the next one. |
|
* Send them all and wait for pending send count to get to 0 |
|
* that means all the I/Os have been out and we are good to return |
|
*/ |
|
|
|
wait_event(st->wait_send_payload_pending, |
|
atomic_read(&st->send_payload_pending) == 0); |
|
return ret; |
|
} |
|
|
|
static void read_write_done(struct ib_cq *cq, struct ib_wc *wc, |
|
enum dma_data_direction dir) |
|
{ |
|
struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe, |
|
struct smb_direct_rdma_rw_msg, cqe); |
|
struct smb_direct_transport *t = msg->t; |
|
|
|
if (wc->status != IB_WC_SUCCESS) { |
|
pr_err("read/write error. opcode = %d, status = %s(%d)\n", |
|
wc->opcode, ib_wc_status_msg(wc->status), wc->status); |
|
smb_direct_disconnect_rdma_connection(t); |
|
} |
|
|
|
if (atomic_inc_return(&t->rw_avail_ops) > 0) |
|
wake_up(&t->wait_rw_avail_ops); |
|
|
|
rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port, |
|
msg->sg_list, msg->sgt.nents, dir); |
|
sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); |
|
complete(msg->completion); |
|
kfree(msg); |
|
} |
|
|
|
static void read_done(struct ib_cq *cq, struct ib_wc *wc) |
|
{ |
|
read_write_done(cq, wc, DMA_FROM_DEVICE); |
|
} |
|
|
|
static void write_done(struct ib_cq *cq, struct ib_wc *wc) |
|
{ |
|
read_write_done(cq, wc, DMA_TO_DEVICE); |
|
} |
|
|
|
static int smb_direct_rdma_xmit(struct smb_direct_transport *t, void *buf, |
|
int buf_len, u32 remote_key, u64 remote_offset, |
|
u32 remote_len, bool is_read) |
|
{ |
|
struct smb_direct_rdma_rw_msg *msg; |
|
int ret; |
|
DECLARE_COMPLETION_ONSTACK(completion); |
|
struct ib_send_wr *first_wr = NULL; |
|
|
|
ret = wait_for_credits(t, &t->wait_rw_avail_ops, &t->rw_avail_ops); |
|
if (ret < 0) |
|
return ret; |
|
|
|
/* TODO: mempool */ |
|
msg = kmalloc(offsetof(struct smb_direct_rdma_rw_msg, sg_list) + |
|
sizeof(struct scatterlist) * SG_CHUNK_SIZE, GFP_KERNEL); |
|
if (!msg) { |
|
atomic_inc(&t->rw_avail_ops); |
|
return -ENOMEM; |
|
} |
|
|
|
msg->sgt.sgl = &msg->sg_list[0]; |
|
ret = sg_alloc_table_chained(&msg->sgt, |
|
get_buf_page_count(buf, buf_len), |
|
msg->sg_list, SG_CHUNK_SIZE); |
|
if (ret) { |
|
atomic_inc(&t->rw_avail_ops); |
|
kfree(msg); |
|
return -ENOMEM; |
|
} |
|
|
|
ret = get_sg_list(buf, buf_len, msg->sgt.sgl, msg->sgt.orig_nents); |
|
if (ret <= 0) { |
|
pr_err("failed to get pages\n"); |
|
goto err; |
|
} |
|
|
|
ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port, |
|
msg->sg_list, get_buf_page_count(buf, buf_len), |
|
0, remote_offset, remote_key, |
|
is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE); |
|
if (ret < 0) { |
|
pr_err("failed to init rdma_rw_ctx: %d\n", ret); |
|
goto err; |
|
} |
|
|
|
msg->t = t; |
|
msg->cqe.done = is_read ? read_done : write_done; |
|
msg->completion = &completion; |
|
first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port, |
|
&msg->cqe, NULL); |
|
|
|
ret = ib_post_send(t->qp, first_wr, NULL); |
|
if (ret) { |
|
pr_err("failed to post send wr: %d\n", ret); |
|
goto err; |
|
} |
|
|
|
wait_for_completion(&completion); |
|
return 0; |
|
|
|
err: |
|
atomic_inc(&t->rw_avail_ops); |
|
if (first_wr) |
|
rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port, |
|
msg->sg_list, msg->sgt.nents, |
|
is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE); |
|
sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); |
|
kfree(msg); |
|
return ret; |
|
} |
|
|
|
static int smb_direct_rdma_write(struct ksmbd_transport *t, void *buf, |
|
unsigned int buflen, u32 remote_key, |
|
u64 remote_offset, u32 remote_len) |
|
{ |
|
return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen, |
|
remote_key, remote_offset, |
|
remote_len, false); |
|
} |
|
|
|
static int smb_direct_rdma_read(struct ksmbd_transport *t, void *buf, |
|
unsigned int buflen, u32 remote_key, |
|
u64 remote_offset, u32 remote_len) |
|
{ |
|
return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen, |
|
remote_key, remote_offset, |
|
remote_len, true); |
|
} |
|
|
|
static void smb_direct_disconnect(struct ksmbd_transport *t) |
|
{ |
|
struct smb_direct_transport *st = smb_trans_direct_transfort(t); |
|
|
|
ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id); |
|
|
|
smb_direct_disconnect_rdma_work(&st->disconnect_work); |
|
wait_event_interruptible(st->wait_status, |
|
st->status == SMB_DIRECT_CS_DISCONNECTED); |
|
free_transport(st); |
|
} |
|
|
|
static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, |
|
struct rdma_cm_event *event) |
|
{ |
|
struct smb_direct_transport *t = cm_id->context; |
|
|
|
ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n", |
|
cm_id, rdma_event_msg(event->event), event->event); |
|
|
|
switch (event->event) { |
|
case RDMA_CM_EVENT_ESTABLISHED: { |
|
t->status = SMB_DIRECT_CS_CONNECTED; |
|
wake_up_interruptible(&t->wait_status); |
|
break; |
|
} |
|
case RDMA_CM_EVENT_DEVICE_REMOVAL: |
|
case RDMA_CM_EVENT_DISCONNECTED: { |
|
t->status = SMB_DIRECT_CS_DISCONNECTED; |
|
wake_up_interruptible(&t->wait_status); |
|
wake_up_interruptible(&t->wait_reassembly_queue); |
|
wake_up(&t->wait_send_credits); |
|
break; |
|
} |
|
case RDMA_CM_EVENT_CONNECT_ERROR: { |
|
t->status = SMB_DIRECT_CS_DISCONNECTED; |
|
wake_up_interruptible(&t->wait_status); |
|
break; |
|
} |
|
default: |
|
pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n", |
|
cm_id, rdma_event_msg(event->event), |
|
event->event); |
|
break; |
|
} |
|
return 0; |
|
} |
|
|
|
static void smb_direct_qpair_handler(struct ib_event *event, void *context) |
|
{ |
|
struct smb_direct_transport *t = context; |
|
|
|
ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n", |
|
t->cm_id, ib_event_msg(event->event), event->event); |
|
|
|
switch (event->event) { |
|
case IB_EVENT_CQ_ERR: |
|
case IB_EVENT_QP_FATAL: |
|
smb_direct_disconnect_rdma_connection(t); |
|
break; |
|
default: |
|
break; |
|
} |
|
} |
|
|
|
static int smb_direct_send_negotiate_response(struct smb_direct_transport *t, |
|
int failed) |
|
{ |
|
struct smb_direct_sendmsg *sendmsg; |
|
struct smb_direct_negotiate_resp *resp; |
|
int ret; |
|
|
|
sendmsg = smb_direct_alloc_sendmsg(t); |
|
if (IS_ERR(sendmsg)) |
|
return -ENOMEM; |
|
|
|
resp = (struct smb_direct_negotiate_resp *)sendmsg->packet; |
|
if (failed) { |
|
memset(resp, 0, sizeof(*resp)); |
|
resp->min_version = cpu_to_le16(0x0100); |
|
resp->max_version = cpu_to_le16(0x0100); |
|
resp->status = STATUS_NOT_SUPPORTED; |
|
} else { |
|
resp->status = STATUS_SUCCESS; |
|
resp->min_version = SMB_DIRECT_VERSION_LE; |
|
resp->max_version = SMB_DIRECT_VERSION_LE; |
|
resp->negotiated_version = SMB_DIRECT_VERSION_LE; |
|
resp->reserved = 0; |
|
resp->credits_requested = |
|
cpu_to_le16(t->send_credit_target); |
|
resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t)); |
|
resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size); |
|
resp->preferred_send_size = cpu_to_le32(t->max_send_size); |
|
resp->max_receive_size = cpu_to_le32(t->max_recv_size); |
|
resp->max_fragmented_size = |
|
cpu_to_le32(t->max_fragmented_recv_size); |
|
} |
|
|
|
sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device, |
|
(void *)resp, sizeof(*resp), |
|
DMA_TO_DEVICE); |
|
ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr); |
|
if (ret) { |
|
smb_direct_free_sendmsg(t, sendmsg); |
|
return ret; |
|
} |
|
|
|
sendmsg->num_sge = 1; |
|
sendmsg->sge[0].length = sizeof(*resp); |
|
sendmsg->sge[0].lkey = t->pd->local_dma_lkey; |
|
|
|
ret = post_sendmsg(t, NULL, sendmsg); |
|
if (ret) { |
|
smb_direct_free_sendmsg(t, sendmsg); |
|
return ret; |
|
} |
|
|
|
wait_event(t->wait_send_pending, |
|
atomic_read(&t->send_pending) == 0); |
|
return 0; |
|
} |
|
|
|
static int smb_direct_accept_client(struct smb_direct_transport *t) |
|
{ |
|
struct rdma_conn_param conn_param; |
|
struct ib_port_immutable port_immutable; |
|
u32 ird_ord_hdr[2]; |
|
int ret; |
|
|
|
memset(&conn_param, 0, sizeof(conn_param)); |
|
conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom, |
|
SMB_DIRECT_CM_INITIATOR_DEPTH); |
|
conn_param.responder_resources = 0; |
|
|
|
t->cm_id->device->ops.get_port_immutable(t->cm_id->device, |
|
t->cm_id->port_num, |
|
&port_immutable); |
|
if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) { |
|
ird_ord_hdr[0] = conn_param.responder_resources; |
|
ird_ord_hdr[1] = 1; |
|
conn_param.private_data = ird_ord_hdr; |
|
conn_param.private_data_len = sizeof(ird_ord_hdr); |
|
} else { |
|
conn_param.private_data = NULL; |
|
conn_param.private_data_len = 0; |
|
} |
|
conn_param.retry_count = SMB_DIRECT_CM_RETRY; |
|
conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY; |
|
conn_param.flow_control = 0; |
|
|
|
ret = rdma_accept(t->cm_id, &conn_param); |
|
if (ret) { |
|
pr_err("error at rdma_accept: %d\n", ret); |
|
return ret; |
|
} |
|
|
|
wait_event_interruptible(t->wait_status, |
|
t->status != SMB_DIRECT_CS_NEW); |
|
if (t->status != SMB_DIRECT_CS_CONNECTED) |
|
return -ENOTCONN; |
|
return 0; |
|
} |
|
|
|
static int smb_direct_negotiate(struct smb_direct_transport *t) |
|
{ |
|
int ret; |
|
struct smb_direct_recvmsg *recvmsg; |
|
struct smb_direct_negotiate_req *req; |
|
|
|
recvmsg = get_free_recvmsg(t); |
|
if (!recvmsg) |
|
return -ENOMEM; |
|
recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ; |
|
|
|
ret = smb_direct_post_recv(t, recvmsg); |
|
if (ret) { |
|
pr_err("Can't post recv: %d\n", ret); |
|
goto out; |
|
} |
|
|
|
t->negotiation_requested = false; |
|
ret = smb_direct_accept_client(t); |
|
if (ret) { |
|
pr_err("Can't accept client\n"); |
|
goto out; |
|
} |
|
|
|
smb_direct_post_recv_credits(&t->post_recv_credits_work.work); |
|
|
|
ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n"); |
|
ret = wait_event_interruptible_timeout(t->wait_status, |
|
t->negotiation_requested || |
|
t->status == SMB_DIRECT_CS_DISCONNECTED, |
|
SMB_DIRECT_NEGOTIATE_TIMEOUT * HZ); |
|
if (ret <= 0 || t->status == SMB_DIRECT_CS_DISCONNECTED) { |
|
ret = ret < 0 ? ret : -ETIMEDOUT; |
|
goto out; |
|
} |
|
|
|
ret = smb_direct_check_recvmsg(recvmsg); |
|
if (ret == -ECONNABORTED) |
|
goto out; |
|
|
|
req = (struct smb_direct_negotiate_req *)recvmsg->packet; |
|
t->max_recv_size = min_t(int, t->max_recv_size, |
|
le32_to_cpu(req->preferred_send_size)); |
|
t->max_send_size = min_t(int, t->max_send_size, |
|
le32_to_cpu(req->max_receive_size)); |
|
t->max_fragmented_send_size = |
|
le32_to_cpu(req->max_fragmented_size); |
|
|
|
ret = smb_direct_send_negotiate_response(t, ret); |
|
out: |
|
if (recvmsg) |
|
put_recvmsg(t, recvmsg); |
|
return ret; |
|
} |
|
|
|
static int smb_direct_init_params(struct smb_direct_transport *t, |
|
struct ib_qp_cap *cap) |
|
{ |
|
struct ib_device *device = t->cm_id->device; |
|
int max_send_sges, max_pages, max_rw_wrs, max_send_wrs; |
|
|
|
/* need 2 more sge. because a SMB_DIRECT header will be mapped, |
|
* and maybe a send buffer could be not page aligned. |
|
*/ |
|
t->max_send_size = smb_direct_max_send_size; |
|
max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 2; |
|
if (max_send_sges > SMB_DIRECT_MAX_SEND_SGES) { |
|
pr_err("max_send_size %d is too large\n", t->max_send_size); |
|
return -EINVAL; |
|
} |
|
|
|
/* |
|
* allow smb_direct_max_outstanding_rw_ops of in-flight RDMA |
|
* read/writes. HCA guarantees at least max_send_sge of sges for |
|
* a RDMA read/write work request, and if memory registration is used, |
|
* we need reg_mr, local_inv wrs for each read/write. |
|
*/ |
|
t->max_rdma_rw_size = smb_direct_max_read_write_size; |
|
max_pages = DIV_ROUND_UP(t->max_rdma_rw_size, PAGE_SIZE) + 1; |
|
max_rw_wrs = DIV_ROUND_UP(max_pages, SMB_DIRECT_MAX_SEND_SGES); |
|
max_rw_wrs += rdma_rw_mr_factor(device, t->cm_id->port_num, |
|
max_pages) * 2; |
|
max_rw_wrs *= smb_direct_max_outstanding_rw_ops; |
|
|
|
max_send_wrs = smb_direct_send_credit_target + max_rw_wrs; |
|
if (max_send_wrs > device->attrs.max_cqe || |
|
max_send_wrs > device->attrs.max_qp_wr) { |
|
pr_err("consider lowering send_credit_target = %d, or max_outstanding_rw_ops = %d\n", |
|
smb_direct_send_credit_target, |
|
smb_direct_max_outstanding_rw_ops); |
|
pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", |
|
device->attrs.max_cqe, device->attrs.max_qp_wr); |
|
return -EINVAL; |
|
} |
|
|
|
if (smb_direct_receive_credit_max > device->attrs.max_cqe || |
|
smb_direct_receive_credit_max > device->attrs.max_qp_wr) { |
|
pr_err("consider lowering receive_credit_max = %d\n", |
|
smb_direct_receive_credit_max); |
|
pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n", |
|
device->attrs.max_cqe, device->attrs.max_qp_wr); |
|
return -EINVAL; |
|
} |
|
|
|
if (device->attrs.max_send_sge < SMB_DIRECT_MAX_SEND_SGES) { |
|
pr_err("warning: device max_send_sge = %d too small\n", |
|
device->attrs.max_send_sge); |
|
return -EINVAL; |
|
} |
|
if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) { |
|
pr_err("warning: device max_recv_sge = %d too small\n", |
|
device->attrs.max_recv_sge); |
|
return -EINVAL; |
|
} |
|
|
|
t->recv_credits = 0; |
|
t->count_avail_recvmsg = 0; |
|
|
|
t->recv_credit_max = smb_direct_receive_credit_max; |
|
t->recv_credit_target = 10; |
|
t->new_recv_credits = 0; |
|
|
|
t->send_credit_target = smb_direct_send_credit_target; |
|
atomic_set(&t->send_credits, 0); |
|
atomic_set(&t->rw_avail_ops, smb_direct_max_outstanding_rw_ops); |
|
|
|
t->max_send_size = smb_direct_max_send_size; |
|
t->max_recv_size = smb_direct_max_receive_size; |
|
t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size; |
|
|
|
cap->max_send_wr = max_send_wrs; |
|
cap->max_recv_wr = t->recv_credit_max; |
|
cap->max_send_sge = SMB_DIRECT_MAX_SEND_SGES; |
|
cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES; |
|
cap->max_inline_data = 0; |
|
cap->max_rdma_ctxs = 0; |
|
return 0; |
|
} |
|
|
|
static void smb_direct_destroy_pools(struct smb_direct_transport *t) |
|
{ |
|
struct smb_direct_recvmsg *recvmsg; |
|
|
|
while ((recvmsg = get_free_recvmsg(t))) |
|
mempool_free(recvmsg, t->recvmsg_mempool); |
|
while ((recvmsg = get_empty_recvmsg(t))) |
|
mempool_free(recvmsg, t->recvmsg_mempool); |
|
|
|
mempool_destroy(t->recvmsg_mempool); |
|
t->recvmsg_mempool = NULL; |
|
|
|
kmem_cache_destroy(t->recvmsg_cache); |
|
t->recvmsg_cache = NULL; |
|
|
|
mempool_destroy(t->sendmsg_mempool); |
|
t->sendmsg_mempool = NULL; |
|
|
|
kmem_cache_destroy(t->sendmsg_cache); |
|
t->sendmsg_cache = NULL; |
|
} |
|
|
|
static int smb_direct_create_pools(struct smb_direct_transport *t) |
|
{ |
|
char name[80]; |
|
int i; |
|
struct smb_direct_recvmsg *recvmsg; |
|
|
|
snprintf(name, sizeof(name), "smb_direct_rqst_pool_%p", t); |
|
t->sendmsg_cache = kmem_cache_create(name, |
|
sizeof(struct smb_direct_sendmsg) + |
|
sizeof(struct smb_direct_negotiate_resp), |
|
0, SLAB_HWCACHE_ALIGN, NULL); |
|
if (!t->sendmsg_cache) |
|
return -ENOMEM; |
|
|
|
t->sendmsg_mempool = mempool_create(t->send_credit_target, |
|
mempool_alloc_slab, mempool_free_slab, |
|
t->sendmsg_cache); |
|
if (!t->sendmsg_mempool) |
|
goto err; |
|
|
|
snprintf(name, sizeof(name), "smb_direct_resp_%p", t); |
|
t->recvmsg_cache = kmem_cache_create(name, |
|
sizeof(struct smb_direct_recvmsg) + |
|
t->max_recv_size, |
|
0, SLAB_HWCACHE_ALIGN, NULL); |
|
if (!t->recvmsg_cache) |
|
goto err; |
|
|
|
t->recvmsg_mempool = |
|
mempool_create(t->recv_credit_max, mempool_alloc_slab, |
|
mempool_free_slab, t->recvmsg_cache); |
|
if (!t->recvmsg_mempool) |
|
goto err; |
|
|
|
INIT_LIST_HEAD(&t->recvmsg_queue); |
|
|
|
for (i = 0; i < t->recv_credit_max; i++) { |
|
recvmsg = mempool_alloc(t->recvmsg_mempool, GFP_KERNEL); |
|
if (!recvmsg) |
|
goto err; |
|
recvmsg->transport = t; |
|
list_add(&recvmsg->list, &t->recvmsg_queue); |
|
} |
|
t->count_avail_recvmsg = t->recv_credit_max; |
|
|
|
return 0; |
|
err: |
|
smb_direct_destroy_pools(t); |
|
return -ENOMEM; |
|
} |
|
|
|
static int smb_direct_create_qpair(struct smb_direct_transport *t, |
|
struct ib_qp_cap *cap) |
|
{ |
|
int ret; |
|
struct ib_qp_init_attr qp_attr; |
|
|
|
t->pd = ib_alloc_pd(t->cm_id->device, 0); |
|
if (IS_ERR(t->pd)) { |
|
pr_err("Can't create RDMA PD\n"); |
|
ret = PTR_ERR(t->pd); |
|
t->pd = NULL; |
|
return ret; |
|
} |
|
|
|
t->send_cq = ib_alloc_cq(t->cm_id->device, t, |
|
t->send_credit_target, 0, IB_POLL_WORKQUEUE); |
|
if (IS_ERR(t->send_cq)) { |
|
pr_err("Can't create RDMA send CQ\n"); |
|
ret = PTR_ERR(t->send_cq); |
|
t->send_cq = NULL; |
|
goto err; |
|
} |
|
|
|
t->recv_cq = ib_alloc_cq(t->cm_id->device, t, |
|
cap->max_send_wr + cap->max_rdma_ctxs, |
|
0, IB_POLL_WORKQUEUE); |
|
if (IS_ERR(t->recv_cq)) { |
|
pr_err("Can't create RDMA recv CQ\n"); |
|
ret = PTR_ERR(t->recv_cq); |
|
t->recv_cq = NULL; |
|
goto err; |
|
} |
|
|
|
memset(&qp_attr, 0, sizeof(qp_attr)); |
|
qp_attr.event_handler = smb_direct_qpair_handler; |
|
qp_attr.qp_context = t; |
|
qp_attr.cap = *cap; |
|
qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; |
|
qp_attr.qp_type = IB_QPT_RC; |
|
qp_attr.send_cq = t->send_cq; |
|
qp_attr.recv_cq = t->recv_cq; |
|
qp_attr.port_num = ~0; |
|
|
|
ret = rdma_create_qp(t->cm_id, t->pd, &qp_attr); |
|
if (ret) { |
|
pr_err("Can't create RDMA QP: %d\n", ret); |
|
goto err; |
|
} |
|
|
|
t->qp = t->cm_id->qp; |
|
t->cm_id->event_handler = smb_direct_cm_handler; |
|
|
|
return 0; |
|
err: |
|
if (t->qp) { |
|
ib_destroy_qp(t->qp); |
|
t->qp = NULL; |
|
} |
|
if (t->recv_cq) { |
|
ib_destroy_cq(t->recv_cq); |
|
t->recv_cq = NULL; |
|
} |
|
if (t->send_cq) { |
|
ib_destroy_cq(t->send_cq); |
|
t->send_cq = NULL; |
|
} |
|
if (t->pd) { |
|
ib_dealloc_pd(t->pd); |
|
t->pd = NULL; |
|
} |
|
return ret; |
|
} |
|
|
|
static int smb_direct_prepare(struct ksmbd_transport *t) |
|
{ |
|
struct smb_direct_transport *st = smb_trans_direct_transfort(t); |
|
int ret; |
|
struct ib_qp_cap qp_cap; |
|
|
|
ret = smb_direct_init_params(st, &qp_cap); |
|
if (ret) { |
|
pr_err("Can't configure RDMA parameters\n"); |
|
return ret; |
|
} |
|
|
|
ret = smb_direct_create_pools(st); |
|
if (ret) { |
|
pr_err("Can't init RDMA pool: %d\n", ret); |
|
return ret; |
|
} |
|
|
|
ret = smb_direct_create_qpair(st, &qp_cap); |
|
if (ret) { |
|
pr_err("Can't accept RDMA client: %d\n", ret); |
|
return ret; |
|
} |
|
|
|
ret = smb_direct_negotiate(st); |
|
if (ret) { |
|
pr_err("Can't negotiate: %d\n", ret); |
|
return ret; |
|
} |
|
|
|
st->status = SMB_DIRECT_CS_CONNECTED; |
|
return 0; |
|
} |
|
|
|
static bool rdma_frwr_is_supported(struct ib_device_attr *attrs) |
|
{ |
|
if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) |
|
return false; |
|
if (attrs->max_fast_reg_page_list_len == 0) |
|
return false; |
|
return true; |
|
} |
|
|
|
static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id) |
|
{ |
|
struct smb_direct_transport *t; |
|
|
|
if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) { |
|
ksmbd_debug(RDMA, |
|
"Fast Registration Work Requests is not supported. device capabilities=%llx\n", |
|
new_cm_id->device->attrs.device_cap_flags); |
|
return -EPROTONOSUPPORT; |
|
} |
|
|
|
t = alloc_transport(new_cm_id); |
|
if (!t) |
|
return -ENOMEM; |
|
|
|
KSMBD_TRANS(t)->handler = kthread_run(ksmbd_conn_handler_loop, |
|
KSMBD_TRANS(t)->conn, "ksmbd:r%u", |
|
SMB_DIRECT_PORT); |
|
if (IS_ERR(KSMBD_TRANS(t)->handler)) { |
|
int ret = PTR_ERR(KSMBD_TRANS(t)->handler); |
|
|
|
pr_err("Can't start thread\n"); |
|
free_transport(t); |
|
return ret; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static int smb_direct_listen_handler(struct rdma_cm_id *cm_id, |
|
struct rdma_cm_event *event) |
|
{ |
|
switch (event->event) { |
|
case RDMA_CM_EVENT_CONNECT_REQUEST: { |
|
int ret = smb_direct_handle_connect_request(cm_id); |
|
|
|
if (ret) { |
|
pr_err("Can't create transport: %d\n", ret); |
|
return ret; |
|
} |
|
|
|
ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n", |
|
cm_id); |
|
break; |
|
} |
|
default: |
|
pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n", |
|
cm_id, rdma_event_msg(event->event), event->event); |
|
break; |
|
} |
|
return 0; |
|
} |
|
|
|
static int smb_direct_listen(int port) |
|
{ |
|
int ret; |
|
struct rdma_cm_id *cm_id; |
|
struct sockaddr_in sin = { |
|
.sin_family = AF_INET, |
|
.sin_addr.s_addr = htonl(INADDR_ANY), |
|
.sin_port = htons(port), |
|
}; |
|
|
|
cm_id = rdma_create_id(&init_net, smb_direct_listen_handler, |
|
&smb_direct_listener, RDMA_PS_TCP, IB_QPT_RC); |
|
if (IS_ERR(cm_id)) { |
|
pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id)); |
|
return PTR_ERR(cm_id); |
|
} |
|
|
|
ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin); |
|
if (ret) { |
|
pr_err("Can't bind: %d\n", ret); |
|
goto err; |
|
} |
|
|
|
smb_direct_listener.cm_id = cm_id; |
|
|
|
ret = rdma_listen(cm_id, 10); |
|
if (ret) { |
|
pr_err("Can't listen: %d\n", ret); |
|
goto err; |
|
} |
|
return 0; |
|
err: |
|
smb_direct_listener.cm_id = NULL; |
|
rdma_destroy_id(cm_id); |
|
return ret; |
|
} |
|
|
|
int ksmbd_rdma_init(void) |
|
{ |
|
int ret; |
|
|
|
smb_direct_listener.cm_id = NULL; |
|
|
|
/* When a client is running out of send credits, the credits are |
|
* granted by the server's sending a packet using this queue. |
|
* This avoids the situation that a clients cannot send packets |
|
* for lack of credits |
|
*/ |
|
smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq", |
|
WQ_HIGHPRI | WQ_MEM_RECLAIM, 0); |
|
if (!smb_direct_wq) |
|
return -ENOMEM; |
|
|
|
ret = smb_direct_listen(SMB_DIRECT_PORT); |
|
if (ret) { |
|
destroy_workqueue(smb_direct_wq); |
|
smb_direct_wq = NULL; |
|
pr_err("Can't listen: %d\n", ret); |
|
return ret; |
|
} |
|
|
|
ksmbd_debug(RDMA, "init RDMA listener. cm_id=%p\n", |
|
smb_direct_listener.cm_id); |
|
return 0; |
|
} |
|
|
|
int ksmbd_rdma_destroy(void) |
|
{ |
|
if (smb_direct_listener.cm_id) |
|
rdma_destroy_id(smb_direct_listener.cm_id); |
|
smb_direct_listener.cm_id = NULL; |
|
|
|
if (smb_direct_wq) { |
|
flush_workqueue(smb_direct_wq); |
|
destroy_workqueue(smb_direct_wq); |
|
smb_direct_wq = NULL; |
|
} |
|
return 0; |
|
} |
|
|
|
bool ksmbd_rdma_capable_netdev(struct net_device *netdev) |
|
{ |
|
struct ib_device *ibdev; |
|
bool rdma_capable = false; |
|
|
|
ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN); |
|
if (ibdev) { |
|
if (rdma_frwr_is_supported(&ibdev->attrs)) |
|
rdma_capable = true; |
|
ib_device_put(ibdev); |
|
} |
|
return rdma_capable; |
|
} |
|
|
|
static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = { |
|
.prepare = smb_direct_prepare, |
|
.disconnect = smb_direct_disconnect, |
|
.writev = smb_direct_writev, |
|
.read = smb_direct_read, |
|
.rdma_read = smb_direct_rdma_read, |
|
.rdma_write = smb_direct_rdma_write, |
|
};
|
|
|