mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
5532 lines
157 KiB
5532 lines
157 KiB
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) |
|
/* |
|
* Copyright(c) 2018 - 2020 Intel Corporation. |
|
* |
|
*/ |
|
|
|
#include "hfi.h" |
|
#include "qp.h" |
|
#include "rc.h" |
|
#include "verbs.h" |
|
#include "tid_rdma.h" |
|
#include "exp_rcv.h" |
|
#include "trace.h" |
|
|
|
/** |
|
* DOC: TID RDMA READ protocol |
|
* |
|
* This is an end-to-end protocol at the hfi1 level between two nodes that |
|
* improves performance by avoiding data copy on the requester side. It |
|
* converts a qualified RDMA READ request into a TID RDMA READ request on |
|
* the requester side and thereafter handles the request and response |
|
* differently. To be qualified, the RDMA READ request should meet the |
|
* following: |
|
* -- The total data length should be greater than 256K; |
|
* -- The total data length should be a multiple of 4K page size; |
|
* -- Each local scatter-gather entry should be 4K page aligned; |
|
* -- Each local scatter-gather entry should be a multiple of 4K page size; |
|
*/ |
|
|
|
#define RCV_TID_FLOW_TABLE_CTRL_FLOW_VALID_SMASK BIT_ULL(32) |
|
#define RCV_TID_FLOW_TABLE_CTRL_HDR_SUPP_EN_SMASK BIT_ULL(33) |
|
#define RCV_TID_FLOW_TABLE_CTRL_KEEP_AFTER_SEQ_ERR_SMASK BIT_ULL(34) |
|
#define RCV_TID_FLOW_TABLE_CTRL_KEEP_ON_GEN_ERR_SMASK BIT_ULL(35) |
|
#define RCV_TID_FLOW_TABLE_STATUS_SEQ_MISMATCH_SMASK BIT_ULL(37) |
|
#define RCV_TID_FLOW_TABLE_STATUS_GEN_MISMATCH_SMASK BIT_ULL(38) |
|
|
|
/* Maximum number of packets within a flow generation. */ |
|
#define MAX_TID_FLOW_PSN BIT(HFI1_KDETH_BTH_SEQ_SHIFT) |
|
|
|
#define GENERATION_MASK 0xFFFFF |
|
|
|
static u32 mask_generation(u32 a) |
|
{ |
|
return a & GENERATION_MASK; |
|
} |
|
|
|
/* Reserved generation value to set to unused flows for kernel contexts */ |
|
#define KERN_GENERATION_RESERVED mask_generation(U32_MAX) |
|
|
|
/* |
|
* J_KEY for kernel contexts when TID RDMA is used. |
|
* See generate_jkey() in hfi.h for more information. |
|
*/ |
|
#define TID_RDMA_JKEY 32 |
|
#define HFI1_KERNEL_MIN_JKEY HFI1_ADMIN_JKEY_RANGE |
|
#define HFI1_KERNEL_MAX_JKEY (2 * HFI1_ADMIN_JKEY_RANGE - 1) |
|
|
|
/* Maximum number of segments in flight per QP request. */ |
|
#define TID_RDMA_MAX_READ_SEGS_PER_REQ 6 |
|
#define TID_RDMA_MAX_WRITE_SEGS_PER_REQ 4 |
|
#define MAX_REQ max_t(u16, TID_RDMA_MAX_READ_SEGS_PER_REQ, \ |
|
TID_RDMA_MAX_WRITE_SEGS_PER_REQ) |
|
#define MAX_FLOWS roundup_pow_of_two(MAX_REQ + 1) |
|
|
|
#define MAX_EXPECTED_PAGES (MAX_EXPECTED_BUFFER / PAGE_SIZE) |
|
|
|
#define TID_RDMA_DESTQP_FLOW_SHIFT 11 |
|
#define TID_RDMA_DESTQP_FLOW_MASK 0x1f |
|
|
|
#define TID_OPFN_QP_CTXT_MASK 0xff |
|
#define TID_OPFN_QP_CTXT_SHIFT 56 |
|
#define TID_OPFN_QP_KDETH_MASK 0xff |
|
#define TID_OPFN_QP_KDETH_SHIFT 48 |
|
#define TID_OPFN_MAX_LEN_MASK 0x7ff |
|
#define TID_OPFN_MAX_LEN_SHIFT 37 |
|
#define TID_OPFN_TIMEOUT_MASK 0x1f |
|
#define TID_OPFN_TIMEOUT_SHIFT 32 |
|
#define TID_OPFN_RESERVED_MASK 0x3f |
|
#define TID_OPFN_RESERVED_SHIFT 26 |
|
#define TID_OPFN_URG_MASK 0x1 |
|
#define TID_OPFN_URG_SHIFT 25 |
|
#define TID_OPFN_VER_MASK 0x7 |
|
#define TID_OPFN_VER_SHIFT 22 |
|
#define TID_OPFN_JKEY_MASK 0x3f |
|
#define TID_OPFN_JKEY_SHIFT 16 |
|
#define TID_OPFN_MAX_READ_MASK 0x3f |
|
#define TID_OPFN_MAX_READ_SHIFT 10 |
|
#define TID_OPFN_MAX_WRITE_MASK 0x3f |
|
#define TID_OPFN_MAX_WRITE_SHIFT 4 |
|
|
|
/* |
|
* OPFN TID layout |
|
* |
|
* 63 47 31 15 |
|
* NNNNNNNNKKKKKKKK MMMMMMMMMMMTTTTT DDDDDDUVVVJJJJJJ RRRRRRWWWWWWCCCC |
|
* 3210987654321098 7654321098765432 1098765432109876 5432109876543210 |
|
* N - the context Number |
|
* K - the Kdeth_qp |
|
* M - Max_len |
|
* T - Timeout |
|
* D - reserveD |
|
* V - version |
|
* U - Urg capable |
|
* J - Jkey |
|
* R - max_Read |
|
* W - max_Write |
|
* C - Capcode |
|
*/ |
|
|
|
static void tid_rdma_trigger_resume(struct work_struct *work); |
|
static void hfi1_kern_exp_rcv_free_flows(struct tid_rdma_request *req); |
|
static int hfi1_kern_exp_rcv_alloc_flows(struct tid_rdma_request *req, |
|
gfp_t gfp); |
|
static void hfi1_init_trdma_req(struct rvt_qp *qp, |
|
struct tid_rdma_request *req); |
|
static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx); |
|
static void hfi1_tid_timeout(struct timer_list *t); |
|
static void hfi1_add_tid_reap_timer(struct rvt_qp *qp); |
|
static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp); |
|
static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp); |
|
static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp); |
|
static void hfi1_tid_retry_timeout(struct timer_list *t); |
|
static int make_tid_rdma_ack(struct rvt_qp *qp, |
|
struct ib_other_headers *ohdr, |
|
struct hfi1_pkt_state *ps); |
|
static void hfi1_do_tid_send(struct rvt_qp *qp); |
|
static u32 read_r_next_psn(struct hfi1_devdata *dd, u8 ctxt, u8 fidx); |
|
static void tid_rdma_rcv_err(struct hfi1_packet *packet, |
|
struct ib_other_headers *ohdr, |
|
struct rvt_qp *qp, u32 psn, int diff, bool fecn); |
|
static void update_r_next_psn_fecn(struct hfi1_packet *packet, |
|
struct hfi1_qp_priv *priv, |
|
struct hfi1_ctxtdata *rcd, |
|
struct tid_rdma_flow *flow, |
|
bool fecn); |
|
|
|
static void validate_r_tid_ack(struct hfi1_qp_priv *priv) |
|
{ |
|
if (priv->r_tid_ack == HFI1_QP_WQE_INVALID) |
|
priv->r_tid_ack = priv->r_tid_tail; |
|
} |
|
|
|
static void tid_rdma_schedule_ack(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
priv->s_flags |= RVT_S_ACK_PENDING; |
|
hfi1_schedule_tid_send(qp); |
|
} |
|
|
|
static void tid_rdma_trigger_ack(struct rvt_qp *qp) |
|
{ |
|
validate_r_tid_ack(qp->priv); |
|
tid_rdma_schedule_ack(qp); |
|
} |
|
|
|
static u64 tid_rdma_opfn_encode(struct tid_rdma_params *p) |
|
{ |
|
return |
|
(((u64)p->qp & TID_OPFN_QP_CTXT_MASK) << |
|
TID_OPFN_QP_CTXT_SHIFT) | |
|
((((u64)p->qp >> 16) & TID_OPFN_QP_KDETH_MASK) << |
|
TID_OPFN_QP_KDETH_SHIFT) | |
|
(((u64)((p->max_len >> PAGE_SHIFT) - 1) & |
|
TID_OPFN_MAX_LEN_MASK) << TID_OPFN_MAX_LEN_SHIFT) | |
|
(((u64)p->timeout & TID_OPFN_TIMEOUT_MASK) << |
|
TID_OPFN_TIMEOUT_SHIFT) | |
|
(((u64)p->urg & TID_OPFN_URG_MASK) << TID_OPFN_URG_SHIFT) | |
|
(((u64)p->jkey & TID_OPFN_JKEY_MASK) << TID_OPFN_JKEY_SHIFT) | |
|
(((u64)p->max_read & TID_OPFN_MAX_READ_MASK) << |
|
TID_OPFN_MAX_READ_SHIFT) | |
|
(((u64)p->max_write & TID_OPFN_MAX_WRITE_MASK) << |
|
TID_OPFN_MAX_WRITE_SHIFT); |
|
} |
|
|
|
static void tid_rdma_opfn_decode(struct tid_rdma_params *p, u64 data) |
|
{ |
|
p->max_len = (((data >> TID_OPFN_MAX_LEN_SHIFT) & |
|
TID_OPFN_MAX_LEN_MASK) + 1) << PAGE_SHIFT; |
|
p->jkey = (data >> TID_OPFN_JKEY_SHIFT) & TID_OPFN_JKEY_MASK; |
|
p->max_write = (data >> TID_OPFN_MAX_WRITE_SHIFT) & |
|
TID_OPFN_MAX_WRITE_MASK; |
|
p->max_read = (data >> TID_OPFN_MAX_READ_SHIFT) & |
|
TID_OPFN_MAX_READ_MASK; |
|
p->qp = |
|
((((data >> TID_OPFN_QP_KDETH_SHIFT) & TID_OPFN_QP_KDETH_MASK) |
|
<< 16) | |
|
((data >> TID_OPFN_QP_CTXT_SHIFT) & TID_OPFN_QP_CTXT_MASK)); |
|
p->urg = (data >> TID_OPFN_URG_SHIFT) & TID_OPFN_URG_MASK; |
|
p->timeout = (data >> TID_OPFN_TIMEOUT_SHIFT) & TID_OPFN_TIMEOUT_MASK; |
|
} |
|
|
|
void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
p->qp = (RVT_KDETH_QP_PREFIX << 16) | priv->rcd->ctxt; |
|
p->max_len = TID_RDMA_MAX_SEGMENT_SIZE; |
|
p->jkey = priv->rcd->jkey; |
|
p->max_read = TID_RDMA_MAX_READ_SEGS_PER_REQ; |
|
p->max_write = TID_RDMA_MAX_WRITE_SEGS_PER_REQ; |
|
p->timeout = qp->timeout; |
|
p->urg = is_urg_masked(priv->rcd); |
|
} |
|
|
|
bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
*data = tid_rdma_opfn_encode(&priv->tid_rdma.local); |
|
return true; |
|
} |
|
|
|
bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct tid_rdma_params *remote, *old; |
|
bool ret = true; |
|
|
|
old = rcu_dereference_protected(priv->tid_rdma.remote, |
|
lockdep_is_held(&priv->opfn.lock)); |
|
data &= ~0xfULL; |
|
/* |
|
* If data passed in is zero, return true so as not to continue the |
|
* negotiation process |
|
*/ |
|
if (!data || !HFI1_CAP_IS_KSET(TID_RDMA)) |
|
goto null; |
|
/* |
|
* If kzalloc fails, return false. This will result in: |
|
* * at the requester a new OPFN request being generated to retry |
|
* the negotiation |
|
* * at the responder, 0 being returned to the requester so as to |
|
* disable TID RDMA at both the requester and the responder |
|
*/ |
|
remote = kzalloc(sizeof(*remote), GFP_ATOMIC); |
|
if (!remote) { |
|
ret = false; |
|
goto null; |
|
} |
|
|
|
tid_rdma_opfn_decode(remote, data); |
|
priv->tid_timer_timeout_jiffies = |
|
usecs_to_jiffies((((4096UL * (1UL << remote->timeout)) / |
|
1000UL) << 3) * 7); |
|
trace_hfi1_opfn_param(qp, 0, &priv->tid_rdma.local); |
|
trace_hfi1_opfn_param(qp, 1, remote); |
|
rcu_assign_pointer(priv->tid_rdma.remote, remote); |
|
/* |
|
* A TID RDMA READ request's segment size is not equal to |
|
* remote->max_len only when the request's data length is smaller |
|
* than remote->max_len. In that case, there will be only one segment. |
|
* Therefore, when priv->pkts_ps is used to calculate req->cur_seg |
|
* during retry, it will lead to req->cur_seg = 0, which is exactly |
|
* what is expected. |
|
*/ |
|
priv->pkts_ps = (u16)rvt_div_mtu(qp, remote->max_len); |
|
priv->timeout_shift = ilog2(priv->pkts_ps - 1) + 1; |
|
goto free; |
|
null: |
|
RCU_INIT_POINTER(priv->tid_rdma.remote, NULL); |
|
priv->timeout_shift = 0; |
|
free: |
|
if (old) |
|
kfree_rcu(old, rcu_head); |
|
return ret; |
|
} |
|
|
|
bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data) |
|
{ |
|
bool ret; |
|
|
|
ret = tid_rdma_conn_reply(qp, *data); |
|
*data = 0; |
|
/* |
|
* If tid_rdma_conn_reply() returns error, set *data as 0 to indicate |
|
* TID RDMA could not be enabled. This will result in TID RDMA being |
|
* disabled at the requester too. |
|
*/ |
|
if (ret) |
|
(void)tid_rdma_conn_req(qp, data); |
|
return ret; |
|
} |
|
|
|
void tid_rdma_conn_error(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct tid_rdma_params *old; |
|
|
|
old = rcu_dereference_protected(priv->tid_rdma.remote, |
|
lockdep_is_held(&priv->opfn.lock)); |
|
RCU_INIT_POINTER(priv->tid_rdma.remote, NULL); |
|
if (old) |
|
kfree_rcu(old, rcu_head); |
|
} |
|
|
|
/* This is called at context initialization time */ |
|
int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit) |
|
{ |
|
if (reinit) |
|
return 0; |
|
|
|
BUILD_BUG_ON(TID_RDMA_JKEY < HFI1_KERNEL_MIN_JKEY); |
|
BUILD_BUG_ON(TID_RDMA_JKEY > HFI1_KERNEL_MAX_JKEY); |
|
rcd->jkey = TID_RDMA_JKEY; |
|
hfi1_set_ctxt_jkey(rcd->dd, rcd, rcd->jkey); |
|
return hfi1_alloc_ctxt_rcv_groups(rcd); |
|
} |
|
|
|
/** |
|
* qp_to_rcd - determine the receive context used by a qp |
|
* @rdi: rvt dev struct |
|
* @qp: the qp |
|
* |
|
* This routine returns the receive context associated |
|
* with a a qp's qpn. |
|
* |
|
* Returns the context. |
|
*/ |
|
static struct hfi1_ctxtdata *qp_to_rcd(struct rvt_dev_info *rdi, |
|
struct rvt_qp *qp) |
|
{ |
|
struct hfi1_ibdev *verbs_dev = container_of(rdi, |
|
struct hfi1_ibdev, |
|
rdi); |
|
struct hfi1_devdata *dd = container_of(verbs_dev, |
|
struct hfi1_devdata, |
|
verbs_dev); |
|
unsigned int ctxt; |
|
|
|
if (qp->ibqp.qp_num == 0) |
|
ctxt = 0; |
|
else |
|
ctxt = hfi1_get_qp_map(dd, qp->ibqp.qp_num >> dd->qos_shift); |
|
return dd->rcd[ctxt]; |
|
} |
|
|
|
int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, |
|
struct ib_qp_init_attr *init_attr) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
int i, ret; |
|
|
|
qpriv->rcd = qp_to_rcd(rdi, qp); |
|
|
|
spin_lock_init(&qpriv->opfn.lock); |
|
INIT_WORK(&qpriv->opfn.opfn_work, opfn_send_conn_request); |
|
INIT_WORK(&qpriv->tid_rdma.trigger_work, tid_rdma_trigger_resume); |
|
qpriv->flow_state.psn = 0; |
|
qpriv->flow_state.index = RXE_NUM_TID_FLOWS; |
|
qpriv->flow_state.last_index = RXE_NUM_TID_FLOWS; |
|
qpriv->flow_state.generation = KERN_GENERATION_RESERVED; |
|
qpriv->s_state = TID_OP(WRITE_RESP); |
|
qpriv->s_tid_cur = HFI1_QP_WQE_INVALID; |
|
qpriv->s_tid_head = HFI1_QP_WQE_INVALID; |
|
qpriv->s_tid_tail = HFI1_QP_WQE_INVALID; |
|
qpriv->rnr_nak_state = TID_RNR_NAK_INIT; |
|
qpriv->r_tid_head = HFI1_QP_WQE_INVALID; |
|
qpriv->r_tid_tail = HFI1_QP_WQE_INVALID; |
|
qpriv->r_tid_ack = HFI1_QP_WQE_INVALID; |
|
qpriv->r_tid_alloc = HFI1_QP_WQE_INVALID; |
|
atomic_set(&qpriv->n_requests, 0); |
|
atomic_set(&qpriv->n_tid_requests, 0); |
|
timer_setup(&qpriv->s_tid_timer, hfi1_tid_timeout, 0); |
|
timer_setup(&qpriv->s_tid_retry_timer, hfi1_tid_retry_timeout, 0); |
|
INIT_LIST_HEAD(&qpriv->tid_wait); |
|
|
|
if (init_attr->qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) { |
|
struct hfi1_devdata *dd = qpriv->rcd->dd; |
|
|
|
qpriv->pages = kzalloc_node(TID_RDMA_MAX_PAGES * |
|
sizeof(*qpriv->pages), |
|
GFP_KERNEL, dd->node); |
|
if (!qpriv->pages) |
|
return -ENOMEM; |
|
for (i = 0; i < qp->s_size; i++) { |
|
struct hfi1_swqe_priv *priv; |
|
struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); |
|
|
|
priv = kzalloc_node(sizeof(*priv), GFP_KERNEL, |
|
dd->node); |
|
if (!priv) |
|
return -ENOMEM; |
|
|
|
hfi1_init_trdma_req(qp, &priv->tid_req); |
|
priv->tid_req.e.swqe = wqe; |
|
wqe->priv = priv; |
|
} |
|
for (i = 0; i < rvt_max_atomic(rdi); i++) { |
|
struct hfi1_ack_priv *priv; |
|
|
|
priv = kzalloc_node(sizeof(*priv), GFP_KERNEL, |
|
dd->node); |
|
if (!priv) |
|
return -ENOMEM; |
|
|
|
hfi1_init_trdma_req(qp, &priv->tid_req); |
|
priv->tid_req.e.ack = &qp->s_ack_queue[i]; |
|
|
|
ret = hfi1_kern_exp_rcv_alloc_flows(&priv->tid_req, |
|
GFP_KERNEL); |
|
if (ret) { |
|
kfree(priv); |
|
return ret; |
|
} |
|
qp->s_ack_queue[i].priv = priv; |
|
} |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct rvt_swqe *wqe; |
|
u32 i; |
|
|
|
if (qp->ibqp.qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) { |
|
for (i = 0; i < qp->s_size; i++) { |
|
wqe = rvt_get_swqe_ptr(qp, i); |
|
kfree(wqe->priv); |
|
wqe->priv = NULL; |
|
} |
|
for (i = 0; i < rvt_max_atomic(rdi); i++) { |
|
struct hfi1_ack_priv *priv = qp->s_ack_queue[i].priv; |
|
|
|
if (priv) |
|
hfi1_kern_exp_rcv_free_flows(&priv->tid_req); |
|
kfree(priv); |
|
qp->s_ack_queue[i].priv = NULL; |
|
} |
|
cancel_work_sync(&qpriv->opfn.opfn_work); |
|
kfree(qpriv->pages); |
|
qpriv->pages = NULL; |
|
} |
|
} |
|
|
|
/* Flow and tid waiter functions */ |
|
/** |
|
* DOC: lock ordering |
|
* |
|
* There are two locks involved with the queuing |
|
* routines: the qp s_lock and the exp_lock. |
|
* |
|
* Since the tid space allocation is called from |
|
* the send engine, the qp s_lock is already held. |
|
* |
|
* The allocation routines will get the exp_lock. |
|
* |
|
* The first_qp() call is provided to allow the head of |
|
* the rcd wait queue to be fetched under the exp_lock and |
|
* followed by a drop of the exp_lock. |
|
* |
|
* Any qp in the wait list will have the qp reference count held |
|
* to hold the qp in memory. |
|
*/ |
|
|
|
/* |
|
* return head of rcd wait list |
|
* |
|
* Must hold the exp_lock. |
|
* |
|
* Get a reference to the QP to hold the QP in memory. |
|
* |
|
* The caller must release the reference when the local |
|
* is no longer being used. |
|
*/ |
|
static struct rvt_qp *first_qp(struct hfi1_ctxtdata *rcd, |
|
struct tid_queue *queue) |
|
__must_hold(&rcd->exp_lock) |
|
{ |
|
struct hfi1_qp_priv *priv; |
|
|
|
lockdep_assert_held(&rcd->exp_lock); |
|
priv = list_first_entry_or_null(&queue->queue_head, |
|
struct hfi1_qp_priv, |
|
tid_wait); |
|
if (!priv) |
|
return NULL; |
|
rvt_get_qp(priv->owner); |
|
return priv->owner; |
|
} |
|
|
|
/** |
|
* kernel_tid_waiters - determine rcd wait |
|
* @rcd: the receive context |
|
* @queue: the queue to operate on |
|
* @qp: the head of the qp being processed |
|
* |
|
* This routine will return false IFF |
|
* the list is NULL or the head of the |
|
* list is the indicated qp. |
|
* |
|
* Must hold the qp s_lock and the exp_lock. |
|
* |
|
* Return: |
|
* false if either of the conditions below are satisfied: |
|
* 1. The list is empty or |
|
* 2. The indicated qp is at the head of the list and the |
|
* HFI1_S_WAIT_TID_SPACE bit is set in qp->s_flags. |
|
* true is returned otherwise. |
|
*/ |
|
static bool kernel_tid_waiters(struct hfi1_ctxtdata *rcd, |
|
struct tid_queue *queue, struct rvt_qp *qp) |
|
__must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) |
|
{ |
|
struct rvt_qp *fqp; |
|
bool ret = true; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
lockdep_assert_held(&rcd->exp_lock); |
|
fqp = first_qp(rcd, queue); |
|
if (!fqp || (fqp == qp && (qp->s_flags & HFI1_S_WAIT_TID_SPACE))) |
|
ret = false; |
|
rvt_put_qp(fqp); |
|
return ret; |
|
} |
|
|
|
/** |
|
* dequeue_tid_waiter - dequeue the qp from the list |
|
* @rcd: the receive context |
|
* @queue: the queue to operate on |
|
* @qp: the qp to remove the wait list |
|
* |
|
* This routine removes the indicated qp from the |
|
* wait list if it is there. |
|
* |
|
* This should be done after the hardware flow and |
|
* tid array resources have been allocated. |
|
* |
|
* Must hold the qp s_lock and the rcd exp_lock. |
|
* |
|
* It assumes the s_lock to protect the s_flags |
|
* field and to reliably test the HFI1_S_WAIT_TID_SPACE flag. |
|
*/ |
|
static void dequeue_tid_waiter(struct hfi1_ctxtdata *rcd, |
|
struct tid_queue *queue, struct rvt_qp *qp) |
|
__must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
lockdep_assert_held(&rcd->exp_lock); |
|
if (list_empty(&priv->tid_wait)) |
|
return; |
|
list_del_init(&priv->tid_wait); |
|
qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; |
|
queue->dequeue++; |
|
rvt_put_qp(qp); |
|
} |
|
|
|
/** |
|
* queue_qp_for_tid_wait - suspend QP on tid space |
|
* @rcd: the receive context |
|
* @queue: the queue to operate on |
|
* @qp: the qp |
|
* |
|
* The qp is inserted at the tail of the rcd |
|
* wait queue and the HFI1_S_WAIT_TID_SPACE s_flag is set. |
|
* |
|
* Must hold the qp s_lock and the exp_lock. |
|
*/ |
|
static void queue_qp_for_tid_wait(struct hfi1_ctxtdata *rcd, |
|
struct tid_queue *queue, struct rvt_qp *qp) |
|
__must_hold(&rcd->exp_lock) __must_hold(&qp->s_lock) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
lockdep_assert_held(&rcd->exp_lock); |
|
if (list_empty(&priv->tid_wait)) { |
|
qp->s_flags |= HFI1_S_WAIT_TID_SPACE; |
|
list_add_tail(&priv->tid_wait, &queue->queue_head); |
|
priv->tid_enqueue = ++queue->enqueue; |
|
rcd->dd->verbs_dev.n_tidwait++; |
|
trace_hfi1_qpsleep(qp, HFI1_S_WAIT_TID_SPACE); |
|
rvt_get_qp(qp); |
|
} |
|
} |
|
|
|
/** |
|
* __trigger_tid_waiter - trigger tid waiter |
|
* @qp: the qp |
|
* |
|
* This is a private entrance to schedule the qp |
|
* assuming the caller is holding the qp->s_lock. |
|
*/ |
|
static void __trigger_tid_waiter(struct rvt_qp *qp) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
lockdep_assert_held(&qp->s_lock); |
|
if (!(qp->s_flags & HFI1_S_WAIT_TID_SPACE)) |
|
return; |
|
trace_hfi1_qpwakeup(qp, HFI1_S_WAIT_TID_SPACE); |
|
hfi1_schedule_send(qp); |
|
} |
|
|
|
/** |
|
* tid_rdma_schedule_tid_wakeup - schedule wakeup for a qp |
|
* @qp: the qp |
|
* |
|
* trigger a schedule or a waiting qp in a deadlock |
|
* safe manner. The qp reference is held prior |
|
* to this call via first_qp(). |
|
* |
|
* If the qp trigger was already scheduled (!rval) |
|
* the the reference is dropped, otherwise the resume |
|
* or the destroy cancel will dispatch the reference. |
|
*/ |
|
static void tid_rdma_schedule_tid_wakeup(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv; |
|
struct hfi1_ibport *ibp; |
|
struct hfi1_pportdata *ppd; |
|
struct hfi1_devdata *dd; |
|
bool rval; |
|
|
|
if (!qp) |
|
return; |
|
|
|
priv = qp->priv; |
|
ibp = to_iport(qp->ibqp.device, qp->port_num); |
|
ppd = ppd_from_ibp(ibp); |
|
dd = dd_from_ibdev(qp->ibqp.device); |
|
|
|
rval = queue_work_on(priv->s_sde ? |
|
priv->s_sde->cpu : |
|
cpumask_first(cpumask_of_node(dd->node)), |
|
ppd->hfi1_wq, |
|
&priv->tid_rdma.trigger_work); |
|
if (!rval) |
|
rvt_put_qp(qp); |
|
} |
|
|
|
/** |
|
* tid_rdma_trigger_resume - field a trigger work request |
|
* @work: the work item |
|
* |
|
* Complete the off qp trigger processing by directly |
|
* calling the progress routine. |
|
*/ |
|
static void tid_rdma_trigger_resume(struct work_struct *work) |
|
{ |
|
struct tid_rdma_qp_params *tr; |
|
struct hfi1_qp_priv *priv; |
|
struct rvt_qp *qp; |
|
|
|
tr = container_of(work, struct tid_rdma_qp_params, trigger_work); |
|
priv = container_of(tr, struct hfi1_qp_priv, tid_rdma); |
|
qp = priv->owner; |
|
spin_lock_irq(&qp->s_lock); |
|
if (qp->s_flags & HFI1_S_WAIT_TID_SPACE) { |
|
spin_unlock_irq(&qp->s_lock); |
|
hfi1_do_send(priv->owner, true); |
|
} else { |
|
spin_unlock_irq(&qp->s_lock); |
|
} |
|
rvt_put_qp(qp); |
|
} |
|
|
|
/* |
|
* tid_rdma_flush_wait - unwind any tid space wait |
|
* |
|
* This is called when resetting a qp to |
|
* allow a destroy or reset to get rid |
|
* of any tid space linkage and reference counts. |
|
*/ |
|
static void _tid_rdma_flush_wait(struct rvt_qp *qp, struct tid_queue *queue) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
struct hfi1_qp_priv *priv; |
|
|
|
if (!qp) |
|
return; |
|
lockdep_assert_held(&qp->s_lock); |
|
priv = qp->priv; |
|
qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; |
|
spin_lock(&priv->rcd->exp_lock); |
|
if (!list_empty(&priv->tid_wait)) { |
|
list_del_init(&priv->tid_wait); |
|
qp->s_flags &= ~HFI1_S_WAIT_TID_SPACE; |
|
queue->dequeue++; |
|
rvt_put_qp(qp); |
|
} |
|
spin_unlock(&priv->rcd->exp_lock); |
|
} |
|
|
|
void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
_tid_rdma_flush_wait(qp, &priv->rcd->flow_queue); |
|
_tid_rdma_flush_wait(qp, &priv->rcd->rarr_queue); |
|
} |
|
|
|
/* Flow functions */ |
|
/** |
|
* kern_reserve_flow - allocate a hardware flow |
|
* @rcd: the context to use for allocation |
|
* @last: the index of the preferred flow. Use RXE_NUM_TID_FLOWS to |
|
* signify "don't care". |
|
* |
|
* Use a bit mask based allocation to reserve a hardware |
|
* flow for use in receiving KDETH data packets. If a preferred flow is |
|
* specified the function will attempt to reserve that flow again, if |
|
* available. |
|
* |
|
* The exp_lock must be held. |
|
* |
|
* Return: |
|
* On success: a value postive value between 0 and RXE_NUM_TID_FLOWS - 1 |
|
* On failure: -EAGAIN |
|
*/ |
|
static int kern_reserve_flow(struct hfi1_ctxtdata *rcd, int last) |
|
__must_hold(&rcd->exp_lock) |
|
{ |
|
int nr; |
|
|
|
/* Attempt to reserve the preferred flow index */ |
|
if (last >= 0 && last < RXE_NUM_TID_FLOWS && |
|
!test_and_set_bit(last, &rcd->flow_mask)) |
|
return last; |
|
|
|
nr = ffz(rcd->flow_mask); |
|
BUILD_BUG_ON(RXE_NUM_TID_FLOWS >= |
|
(sizeof(rcd->flow_mask) * BITS_PER_BYTE)); |
|
if (nr > (RXE_NUM_TID_FLOWS - 1)) |
|
return -EAGAIN; |
|
set_bit(nr, &rcd->flow_mask); |
|
return nr; |
|
} |
|
|
|
static void kern_set_hw_flow(struct hfi1_ctxtdata *rcd, u32 generation, |
|
u32 flow_idx) |
|
{ |
|
u64 reg; |
|
|
|
reg = ((u64)generation << HFI1_KDETH_BTH_SEQ_SHIFT) | |
|
RCV_TID_FLOW_TABLE_CTRL_FLOW_VALID_SMASK | |
|
RCV_TID_FLOW_TABLE_CTRL_KEEP_AFTER_SEQ_ERR_SMASK | |
|
RCV_TID_FLOW_TABLE_CTRL_KEEP_ON_GEN_ERR_SMASK | |
|
RCV_TID_FLOW_TABLE_STATUS_SEQ_MISMATCH_SMASK | |
|
RCV_TID_FLOW_TABLE_STATUS_GEN_MISMATCH_SMASK; |
|
|
|
if (generation != KERN_GENERATION_RESERVED) |
|
reg |= RCV_TID_FLOW_TABLE_CTRL_HDR_SUPP_EN_SMASK; |
|
|
|
write_uctxt_csr(rcd->dd, rcd->ctxt, |
|
RCV_TID_FLOW_TABLE + 8 * flow_idx, reg); |
|
} |
|
|
|
static u32 kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, u32 flow_idx) |
|
__must_hold(&rcd->exp_lock) |
|
{ |
|
u32 generation = rcd->flows[flow_idx].generation; |
|
|
|
kern_set_hw_flow(rcd, generation, flow_idx); |
|
return generation; |
|
} |
|
|
|
static u32 kern_flow_generation_next(u32 gen) |
|
{ |
|
u32 generation = mask_generation(gen + 1); |
|
|
|
if (generation == KERN_GENERATION_RESERVED) |
|
generation = mask_generation(generation + 1); |
|
return generation; |
|
} |
|
|
|
static void kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, u32 flow_idx) |
|
__must_hold(&rcd->exp_lock) |
|
{ |
|
rcd->flows[flow_idx].generation = |
|
kern_flow_generation_next(rcd->flows[flow_idx].generation); |
|
kern_set_hw_flow(rcd, KERN_GENERATION_RESERVED, flow_idx); |
|
} |
|
|
|
int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; |
|
struct tid_flow_state *fs = &qpriv->flow_state; |
|
struct rvt_qp *fqp; |
|
unsigned long flags; |
|
int ret = 0; |
|
|
|
/* The QP already has an allocated flow */ |
|
if (fs->index != RXE_NUM_TID_FLOWS) |
|
return ret; |
|
|
|
spin_lock_irqsave(&rcd->exp_lock, flags); |
|
if (kernel_tid_waiters(rcd, &rcd->flow_queue, qp)) |
|
goto queue; |
|
|
|
ret = kern_reserve_flow(rcd, fs->last_index); |
|
if (ret < 0) |
|
goto queue; |
|
fs->index = ret; |
|
fs->last_index = fs->index; |
|
|
|
/* Generation received in a RESYNC overrides default flow generation */ |
|
if (fs->generation != KERN_GENERATION_RESERVED) |
|
rcd->flows[fs->index].generation = fs->generation; |
|
fs->generation = kern_setup_hw_flow(rcd, fs->index); |
|
fs->psn = 0; |
|
dequeue_tid_waiter(rcd, &rcd->flow_queue, qp); |
|
/* get head before dropping lock */ |
|
fqp = first_qp(rcd, &rcd->flow_queue); |
|
spin_unlock_irqrestore(&rcd->exp_lock, flags); |
|
|
|
tid_rdma_schedule_tid_wakeup(fqp); |
|
return 0; |
|
queue: |
|
queue_qp_for_tid_wait(rcd, &rcd->flow_queue, qp); |
|
spin_unlock_irqrestore(&rcd->exp_lock, flags); |
|
return -EAGAIN; |
|
} |
|
|
|
void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; |
|
struct tid_flow_state *fs = &qpriv->flow_state; |
|
struct rvt_qp *fqp; |
|
unsigned long flags; |
|
|
|
if (fs->index >= RXE_NUM_TID_FLOWS) |
|
return; |
|
spin_lock_irqsave(&rcd->exp_lock, flags); |
|
kern_clear_hw_flow(rcd, fs->index); |
|
clear_bit(fs->index, &rcd->flow_mask); |
|
fs->index = RXE_NUM_TID_FLOWS; |
|
fs->psn = 0; |
|
fs->generation = KERN_GENERATION_RESERVED; |
|
|
|
/* get head before dropping lock */ |
|
fqp = first_qp(rcd, &rcd->flow_queue); |
|
spin_unlock_irqrestore(&rcd->exp_lock, flags); |
|
|
|
if (fqp == qp) { |
|
__trigger_tid_waiter(fqp); |
|
rvt_put_qp(fqp); |
|
} else { |
|
tid_rdma_schedule_tid_wakeup(fqp); |
|
} |
|
} |
|
|
|
void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < RXE_NUM_TID_FLOWS; i++) { |
|
rcd->flows[i].generation = mask_generation(prandom_u32()); |
|
kern_set_hw_flow(rcd, KERN_GENERATION_RESERVED, i); |
|
} |
|
} |
|
|
|
/* TID allocation functions */ |
|
static u8 trdma_pset_order(struct tid_rdma_pageset *s) |
|
{ |
|
u8 count = s->count; |
|
|
|
return ilog2(count) + 1; |
|
} |
|
|
|
/** |
|
* tid_rdma_find_phys_blocks_4k - get groups base on mr info |
|
* @flow: overall info for a TID RDMA segment |
|
* @pages: pointer to an array of page structs |
|
* @npages: number of pages |
|
* @list: page set array to return |
|
* |
|
* This routine returns the number of groups associated with |
|
* the current sge information. This implementation is based |
|
* on the expected receive find_phys_blocks() adjusted to |
|
* use the MR information vs. the pfn. |
|
* |
|
* Return: |
|
* the number of RcvArray entries |
|
*/ |
|
static u32 tid_rdma_find_phys_blocks_4k(struct tid_rdma_flow *flow, |
|
struct page **pages, |
|
u32 npages, |
|
struct tid_rdma_pageset *list) |
|
{ |
|
u32 pagecount, pageidx, setcount = 0, i; |
|
void *vaddr, *this_vaddr; |
|
|
|
if (!npages) |
|
return 0; |
|
|
|
/* |
|
* Look for sets of physically contiguous pages in the user buffer. |
|
* This will allow us to optimize Expected RcvArray entry usage by |
|
* using the bigger supported sizes. |
|
*/ |
|
vaddr = page_address(pages[0]); |
|
trace_hfi1_tid_flow_page(flow->req->qp, flow, 0, 0, 0, vaddr); |
|
for (pageidx = 0, pagecount = 1, i = 1; i <= npages; i++) { |
|
this_vaddr = i < npages ? page_address(pages[i]) : NULL; |
|
trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 0, 0, |
|
this_vaddr); |
|
/* |
|
* If the vaddr's are not sequential, pages are not physically |
|
* contiguous. |
|
*/ |
|
if (this_vaddr != (vaddr + PAGE_SIZE)) { |
|
/* |
|
* At this point we have to loop over the set of |
|
* physically contiguous pages and break them down it |
|
* sizes supported by the HW. |
|
* There are two main constraints: |
|
* 1. The max buffer size is MAX_EXPECTED_BUFFER. |
|
* If the total set size is bigger than that |
|
* program only a MAX_EXPECTED_BUFFER chunk. |
|
* 2. The buffer size has to be a power of two. If |
|
* it is not, round down to the closes power of |
|
* 2 and program that size. |
|
*/ |
|
while (pagecount) { |
|
int maxpages = pagecount; |
|
u32 bufsize = pagecount * PAGE_SIZE; |
|
|
|
if (bufsize > MAX_EXPECTED_BUFFER) |
|
maxpages = |
|
MAX_EXPECTED_BUFFER >> |
|
PAGE_SHIFT; |
|
else if (!is_power_of_2(bufsize)) |
|
maxpages = |
|
rounddown_pow_of_two(bufsize) >> |
|
PAGE_SHIFT; |
|
|
|
list[setcount].idx = pageidx; |
|
list[setcount].count = maxpages; |
|
trace_hfi1_tid_pageset(flow->req->qp, setcount, |
|
list[setcount].idx, |
|
list[setcount].count); |
|
pagecount -= maxpages; |
|
pageidx += maxpages; |
|
setcount++; |
|
} |
|
pageidx = i; |
|
pagecount = 1; |
|
vaddr = this_vaddr; |
|
} else { |
|
vaddr += PAGE_SIZE; |
|
pagecount++; |
|
} |
|
} |
|
/* insure we always return an even number of sets */ |
|
if (setcount & 1) |
|
list[setcount++].count = 0; |
|
return setcount; |
|
} |
|
|
|
/** |
|
* tid_flush_pages - dump out pages into pagesets |
|
* @list: list of pagesets |
|
* @idx: pointer to current page index |
|
* @pages: number of pages to dump |
|
* @sets: current number of pagesset |
|
* |
|
* This routine flushes out accumuated pages. |
|
* |
|
* To insure an even number of sets the |
|
* code may add a filler. |
|
* |
|
* This can happen with when pages is not |
|
* a power of 2 or pages is a power of 2 |
|
* less than the maximum pages. |
|
* |
|
* Return: |
|
* The new number of sets |
|
*/ |
|
|
|
static u32 tid_flush_pages(struct tid_rdma_pageset *list, |
|
u32 *idx, u32 pages, u32 sets) |
|
{ |
|
while (pages) { |
|
u32 maxpages = pages; |
|
|
|
if (maxpages > MAX_EXPECTED_PAGES) |
|
maxpages = MAX_EXPECTED_PAGES; |
|
else if (!is_power_of_2(maxpages)) |
|
maxpages = rounddown_pow_of_two(maxpages); |
|
list[sets].idx = *idx; |
|
list[sets++].count = maxpages; |
|
*idx += maxpages; |
|
pages -= maxpages; |
|
} |
|
/* might need a filler */ |
|
if (sets & 1) |
|
list[sets++].count = 0; |
|
return sets; |
|
} |
|
|
|
/** |
|
* tid_rdma_find_phys_blocks_8k - get groups base on mr info |
|
* @flow: overall info for a TID RDMA segment |
|
* @pages: pointer to an array of page structs |
|
* @npages: number of pages |
|
* @list: page set array to return |
|
* |
|
* This routine parses an array of pages to compute pagesets |
|
* in an 8k compatible way. |
|
* |
|
* pages are tested two at a time, i, i + 1 for contiguous |
|
* pages and i - 1 and i contiguous pages. |
|
* |
|
* If any condition is false, any accumlated pages are flushed and |
|
* v0,v1 are emitted as separate PAGE_SIZE pagesets |
|
* |
|
* Otherwise, the current 8k is totaled for a future flush. |
|
* |
|
* Return: |
|
* The number of pagesets |
|
* list set with the returned number of pagesets |
|
* |
|
*/ |
|
static u32 tid_rdma_find_phys_blocks_8k(struct tid_rdma_flow *flow, |
|
struct page **pages, |
|
u32 npages, |
|
struct tid_rdma_pageset *list) |
|
{ |
|
u32 idx, sets = 0, i; |
|
u32 pagecnt = 0; |
|
void *v0, *v1, *vm1; |
|
|
|
if (!npages) |
|
return 0; |
|
for (idx = 0, i = 0, vm1 = NULL; i < npages; i += 2) { |
|
/* get a new v0 */ |
|
v0 = page_address(pages[i]); |
|
trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 0, v0); |
|
v1 = i + 1 < npages ? |
|
page_address(pages[i + 1]) : NULL; |
|
trace_hfi1_tid_flow_page(flow->req->qp, flow, i, 1, 1, v1); |
|
/* compare i, i + 1 vaddr */ |
|
if (v1 != (v0 + PAGE_SIZE)) { |
|
/* flush out pages */ |
|
sets = tid_flush_pages(list, &idx, pagecnt, sets); |
|
/* output v0,v1 as two pagesets */ |
|
list[sets].idx = idx++; |
|
list[sets++].count = 1; |
|
if (v1) { |
|
list[sets].count = 1; |
|
list[sets++].idx = idx++; |
|
} else { |
|
list[sets++].count = 0; |
|
} |
|
vm1 = NULL; |
|
pagecnt = 0; |
|
continue; |
|
} |
|
/* i,i+1 consecutive, look at i-1,i */ |
|
if (vm1 && v0 != (vm1 + PAGE_SIZE)) { |
|
/* flush out pages */ |
|
sets = tid_flush_pages(list, &idx, pagecnt, sets); |
|
pagecnt = 0; |
|
} |
|
/* pages will always be a multiple of 8k */ |
|
pagecnt += 2; |
|
/* save i-1 */ |
|
vm1 = v1; |
|
/* move to next pair */ |
|
} |
|
/* dump residual pages at end */ |
|
sets = tid_flush_pages(list, &idx, npages - idx, sets); |
|
/* by design cannot be odd sets */ |
|
WARN_ON(sets & 1); |
|
return sets; |
|
} |
|
|
|
/* |
|
* Find pages for one segment of a sge array represented by @ss. The function |
|
* does not check the sge, the sge must have been checked for alignment with a |
|
* prior call to hfi1_kern_trdma_ok. Other sge checking is done as part of |
|
* rvt_lkey_ok and rvt_rkey_ok. Also, the function only modifies the local sge |
|
* copy maintained in @ss->sge, the original sge is not modified. |
|
* |
|
* Unlike IB RDMA WRITE, we can't decrement ss->num_sge here because we are not |
|
* releasing the MR reference count at the same time. Otherwise, we'll "leak" |
|
* references to the MR. This difference requires that we keep track of progress |
|
* into the sg_list. This is done by the cur_seg cursor in the tid_rdma_request |
|
* structure. |
|
*/ |
|
static u32 kern_find_pages(struct tid_rdma_flow *flow, |
|
struct page **pages, |
|
struct rvt_sge_state *ss, bool *last) |
|
{ |
|
struct tid_rdma_request *req = flow->req; |
|
struct rvt_sge *sge = &ss->sge; |
|
u32 length = flow->req->seg_len; |
|
u32 len = PAGE_SIZE; |
|
u32 i = 0; |
|
|
|
while (length && req->isge < ss->num_sge) { |
|
pages[i++] = virt_to_page(sge->vaddr); |
|
|
|
sge->vaddr += len; |
|
sge->length -= len; |
|
sge->sge_length -= len; |
|
if (!sge->sge_length) { |
|
if (++req->isge < ss->num_sge) |
|
*sge = ss->sg_list[req->isge - 1]; |
|
} else if (sge->length == 0 && sge->mr->lkey) { |
|
if (++sge->n >= RVT_SEGSZ) { |
|
++sge->m; |
|
sge->n = 0; |
|
} |
|
sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr; |
|
sge->length = sge->mr->map[sge->m]->segs[sge->n].length; |
|
} |
|
length -= len; |
|
} |
|
|
|
flow->length = flow->req->seg_len - length; |
|
*last = req->isge == ss->num_sge ? false : true; |
|
return i; |
|
} |
|
|
|
static void dma_unmap_flow(struct tid_rdma_flow *flow) |
|
{ |
|
struct hfi1_devdata *dd; |
|
int i; |
|
struct tid_rdma_pageset *pset; |
|
|
|
dd = flow->req->rcd->dd; |
|
for (i = 0, pset = &flow->pagesets[0]; i < flow->npagesets; |
|
i++, pset++) { |
|
if (pset->count && pset->addr) { |
|
dma_unmap_page(&dd->pcidev->dev, |
|
pset->addr, |
|
PAGE_SIZE * pset->count, |
|
DMA_FROM_DEVICE); |
|
pset->mapped = 0; |
|
} |
|
} |
|
} |
|
|
|
static int dma_map_flow(struct tid_rdma_flow *flow, struct page **pages) |
|
{ |
|
int i; |
|
struct hfi1_devdata *dd = flow->req->rcd->dd; |
|
struct tid_rdma_pageset *pset; |
|
|
|
for (i = 0, pset = &flow->pagesets[0]; i < flow->npagesets; |
|
i++, pset++) { |
|
if (pset->count) { |
|
pset->addr = dma_map_page(&dd->pcidev->dev, |
|
pages[pset->idx], |
|
0, |
|
PAGE_SIZE * pset->count, |
|
DMA_FROM_DEVICE); |
|
|
|
if (dma_mapping_error(&dd->pcidev->dev, pset->addr)) { |
|
dma_unmap_flow(flow); |
|
return -ENOMEM; |
|
} |
|
pset->mapped = 1; |
|
} |
|
} |
|
return 0; |
|
} |
|
|
|
static inline bool dma_mapped(struct tid_rdma_flow *flow) |
|
{ |
|
return !!flow->pagesets[0].mapped; |
|
} |
|
|
|
/* |
|
* Get pages pointers and identify contiguous physical memory chunks for a |
|
* segment. All segments are of length flow->req->seg_len. |
|
*/ |
|
static int kern_get_phys_blocks(struct tid_rdma_flow *flow, |
|
struct page **pages, |
|
struct rvt_sge_state *ss, bool *last) |
|
{ |
|
u8 npages; |
|
|
|
/* Reuse previously computed pagesets, if any */ |
|
if (flow->npagesets) { |
|
trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head, |
|
flow); |
|
if (!dma_mapped(flow)) |
|
return dma_map_flow(flow, pages); |
|
return 0; |
|
} |
|
|
|
npages = kern_find_pages(flow, pages, ss, last); |
|
|
|
if (flow->req->qp->pmtu == enum_to_mtu(OPA_MTU_4096)) |
|
flow->npagesets = |
|
tid_rdma_find_phys_blocks_4k(flow, pages, npages, |
|
flow->pagesets); |
|
else |
|
flow->npagesets = |
|
tid_rdma_find_phys_blocks_8k(flow, pages, npages, |
|
flow->pagesets); |
|
|
|
return dma_map_flow(flow, pages); |
|
} |
|
|
|
static inline void kern_add_tid_node(struct tid_rdma_flow *flow, |
|
struct hfi1_ctxtdata *rcd, char *s, |
|
struct tid_group *grp, u8 cnt) |
|
{ |
|
struct kern_tid_node *node = &flow->tnode[flow->tnode_cnt++]; |
|
|
|
WARN_ON_ONCE(flow->tnode_cnt >= |
|
(TID_RDMA_MAX_SEGMENT_SIZE >> PAGE_SHIFT)); |
|
if (WARN_ON_ONCE(cnt & 1)) |
|
dd_dev_err(rcd->dd, |
|
"unexpected odd allocation cnt %u map 0x%x used %u", |
|
cnt, grp->map, grp->used); |
|
|
|
node->grp = grp; |
|
node->map = grp->map; |
|
node->cnt = cnt; |
|
trace_hfi1_tid_node_add(flow->req->qp, s, flow->tnode_cnt - 1, |
|
grp->base, grp->map, grp->used, cnt); |
|
} |
|
|
|
/* |
|
* Try to allocate pageset_count TID's from TID groups for a context |
|
* |
|
* This function allocates TID's without moving groups between lists or |
|
* modifying grp->map. This is done as follows, being cogizant of the lists |
|
* between which the TID groups will move: |
|
* 1. First allocate complete groups of 8 TID's since this is more efficient, |
|
* these groups will move from group->full without affecting used |
|
* 2. If more TID's are needed allocate from used (will move from used->full or |
|
* stay in used) |
|
* 3. If we still don't have the required number of TID's go back and look again |
|
* at a complete group (will move from group->used) |
|
*/ |
|
static int kern_alloc_tids(struct tid_rdma_flow *flow) |
|
{ |
|
struct hfi1_ctxtdata *rcd = flow->req->rcd; |
|
struct hfi1_devdata *dd = rcd->dd; |
|
u32 ngroups, pageidx = 0; |
|
struct tid_group *group = NULL, *used; |
|
u8 use; |
|
|
|
flow->tnode_cnt = 0; |
|
ngroups = flow->npagesets / dd->rcv_entries.group_size; |
|
if (!ngroups) |
|
goto used_list; |
|
|
|
/* First look at complete groups */ |
|
list_for_each_entry(group, &rcd->tid_group_list.list, list) { |
|
kern_add_tid_node(flow, rcd, "complete groups", group, |
|
group->size); |
|
|
|
pageidx += group->size; |
|
if (!--ngroups) |
|
break; |
|
} |
|
|
|
if (pageidx >= flow->npagesets) |
|
goto ok; |
|
|
|
used_list: |
|
/* Now look at partially used groups */ |
|
list_for_each_entry(used, &rcd->tid_used_list.list, list) { |
|
use = min_t(u32, flow->npagesets - pageidx, |
|
used->size - used->used); |
|
kern_add_tid_node(flow, rcd, "used groups", used, use); |
|
|
|
pageidx += use; |
|
if (pageidx >= flow->npagesets) |
|
goto ok; |
|
} |
|
|
|
/* |
|
* Look again at a complete group, continuing from where we left. |
|
* However, if we are at the head, we have reached the end of the |
|
* complete groups list from the first loop above |
|
*/ |
|
if (group && &group->list == &rcd->tid_group_list.list) |
|
goto bail_eagain; |
|
group = list_prepare_entry(group, &rcd->tid_group_list.list, |
|
list); |
|
if (list_is_last(&group->list, &rcd->tid_group_list.list)) |
|
goto bail_eagain; |
|
group = list_next_entry(group, list); |
|
use = min_t(u32, flow->npagesets - pageidx, group->size); |
|
kern_add_tid_node(flow, rcd, "complete continue", group, use); |
|
pageidx += use; |
|
if (pageidx >= flow->npagesets) |
|
goto ok; |
|
bail_eagain: |
|
trace_hfi1_msg_alloc_tids(flow->req->qp, " insufficient tids: needed ", |
|
(u64)flow->npagesets); |
|
return -EAGAIN; |
|
ok: |
|
return 0; |
|
} |
|
|
|
static void kern_program_rcv_group(struct tid_rdma_flow *flow, int grp_num, |
|
u32 *pset_idx) |
|
{ |
|
struct hfi1_ctxtdata *rcd = flow->req->rcd; |
|
struct hfi1_devdata *dd = rcd->dd; |
|
struct kern_tid_node *node = &flow->tnode[grp_num]; |
|
struct tid_group *grp = node->grp; |
|
struct tid_rdma_pageset *pset; |
|
u32 pmtu_pg = flow->req->qp->pmtu >> PAGE_SHIFT; |
|
u32 rcventry, npages = 0, pair = 0, tidctrl; |
|
u8 i, cnt = 0; |
|
|
|
for (i = 0; i < grp->size; i++) { |
|
rcventry = grp->base + i; |
|
|
|
if (node->map & BIT(i) || cnt >= node->cnt) { |
|
rcv_array_wc_fill(dd, rcventry); |
|
continue; |
|
} |
|
pset = &flow->pagesets[(*pset_idx)++]; |
|
if (pset->count) { |
|
hfi1_put_tid(dd, rcventry, PT_EXPECTED, |
|
pset->addr, trdma_pset_order(pset)); |
|
} else { |
|
hfi1_put_tid(dd, rcventry, PT_INVALID, 0, 0); |
|
} |
|
npages += pset->count; |
|
|
|
rcventry -= rcd->expected_base; |
|
tidctrl = pair ? 0x3 : rcventry & 0x1 ? 0x2 : 0x1; |
|
/* |
|
* A single TID entry will be used to use a rcvarr pair (with |
|
* tidctrl 0x3), if ALL these are true (a) the bit pos is even |
|
* (b) the group map shows current and the next bits as free |
|
* indicating two consecutive rcvarry entries are available (c) |
|
* we actually need 2 more entries |
|
*/ |
|
pair = !(i & 0x1) && !((node->map >> i) & 0x3) && |
|
node->cnt >= cnt + 2; |
|
if (!pair) { |
|
if (!pset->count) |
|
tidctrl = 0x1; |
|
flow->tid_entry[flow->tidcnt++] = |
|
EXP_TID_SET(IDX, rcventry >> 1) | |
|
EXP_TID_SET(CTRL, tidctrl) | |
|
EXP_TID_SET(LEN, npages); |
|
trace_hfi1_tid_entry_alloc(/* entry */ |
|
flow->req->qp, flow->tidcnt - 1, |
|
flow->tid_entry[flow->tidcnt - 1]); |
|
|
|
/* Efficient DIV_ROUND_UP(npages, pmtu_pg) */ |
|
flow->npkts += (npages + pmtu_pg - 1) >> ilog2(pmtu_pg); |
|
npages = 0; |
|
} |
|
|
|
if (grp->used == grp->size - 1) |
|
tid_group_move(grp, &rcd->tid_used_list, |
|
&rcd->tid_full_list); |
|
else if (!grp->used) |
|
tid_group_move(grp, &rcd->tid_group_list, |
|
&rcd->tid_used_list); |
|
|
|
grp->used++; |
|
grp->map |= BIT(i); |
|
cnt++; |
|
} |
|
} |
|
|
|
static void kern_unprogram_rcv_group(struct tid_rdma_flow *flow, int grp_num) |
|
{ |
|
struct hfi1_ctxtdata *rcd = flow->req->rcd; |
|
struct hfi1_devdata *dd = rcd->dd; |
|
struct kern_tid_node *node = &flow->tnode[grp_num]; |
|
struct tid_group *grp = node->grp; |
|
u32 rcventry; |
|
u8 i, cnt = 0; |
|
|
|
for (i = 0; i < grp->size; i++) { |
|
rcventry = grp->base + i; |
|
|
|
if (node->map & BIT(i) || cnt >= node->cnt) { |
|
rcv_array_wc_fill(dd, rcventry); |
|
continue; |
|
} |
|
|
|
hfi1_put_tid(dd, rcventry, PT_INVALID, 0, 0); |
|
|
|
grp->used--; |
|
grp->map &= ~BIT(i); |
|
cnt++; |
|
|
|
if (grp->used == grp->size - 1) |
|
tid_group_move(grp, &rcd->tid_full_list, |
|
&rcd->tid_used_list); |
|
else if (!grp->used) |
|
tid_group_move(grp, &rcd->tid_used_list, |
|
&rcd->tid_group_list); |
|
} |
|
if (WARN_ON_ONCE(cnt & 1)) { |
|
struct hfi1_ctxtdata *rcd = flow->req->rcd; |
|
struct hfi1_devdata *dd = rcd->dd; |
|
|
|
dd_dev_err(dd, "unexpected odd free cnt %u map 0x%x used %u", |
|
cnt, grp->map, grp->used); |
|
} |
|
} |
|
|
|
static void kern_program_rcvarray(struct tid_rdma_flow *flow) |
|
{ |
|
u32 pset_idx = 0; |
|
int i; |
|
|
|
flow->npkts = 0; |
|
flow->tidcnt = 0; |
|
for (i = 0; i < flow->tnode_cnt; i++) |
|
kern_program_rcv_group(flow, i, &pset_idx); |
|
trace_hfi1_tid_flow_alloc(flow->req->qp, flow->req->setup_head, flow); |
|
} |
|
|
|
/** |
|
* hfi1_kern_exp_rcv_setup() - setup TID's and flow for one segment of a |
|
* TID RDMA request |
|
* |
|
* @req: TID RDMA request for which the segment/flow is being set up |
|
* @ss: sge state, maintains state across successive segments of a sge |
|
* @last: set to true after the last sge segment has been processed |
|
* |
|
* This function |
|
* (1) finds a free flow entry in the flow circular buffer |
|
* (2) finds pages and continuous physical chunks constituing one segment |
|
* of an sge |
|
* (3) allocates TID group entries for those chunks |
|
* (4) programs rcvarray entries in the hardware corresponding to those |
|
* TID's |
|
* (5) computes a tidarray with formatted TID entries which can be sent |
|
* to the sender |
|
* (6) Reserves and programs HW flows. |
|
* (7) It also manages queing the QP when TID/flow resources are not |
|
* available. |
|
* |
|
* @req points to struct tid_rdma_request of which the segments are a part. The |
|
* function uses qp, rcd and seg_len members of @req. In the absence of errors, |
|
* req->flow_idx is the index of the flow which has been prepared in this |
|
* invocation of function call. With flow = &req->flows[req->flow_idx], |
|
* flow->tid_entry contains the TID array which the sender can use for TID RDMA |
|
* sends and flow->npkts contains number of packets required to send the |
|
* segment. |
|
* |
|
* hfi1_check_sge_align should be called prior to calling this function and if |
|
* it signals error TID RDMA cannot be used for this sge and this function |
|
* should not be called. |
|
* |
|
* For the queuing, caller must hold the flow->req->qp s_lock from the send |
|
* engine and the function will procure the exp_lock. |
|
* |
|
* Return: |
|
* The function returns -EAGAIN if sufficient number of TID/flow resources to |
|
* map the segment could not be allocated. In this case the function should be |
|
* called again with previous arguments to retry the TID allocation. There are |
|
* no other error returns. The function returns 0 on success. |
|
*/ |
|
int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req, |
|
struct rvt_sge_state *ss, bool *last) |
|
__must_hold(&req->qp->s_lock) |
|
{ |
|
struct tid_rdma_flow *flow = &req->flows[req->setup_head]; |
|
struct hfi1_ctxtdata *rcd = req->rcd; |
|
struct hfi1_qp_priv *qpriv = req->qp->priv; |
|
unsigned long flags; |
|
struct rvt_qp *fqp; |
|
u16 clear_tail = req->clear_tail; |
|
|
|
lockdep_assert_held(&req->qp->s_lock); |
|
/* |
|
* We return error if either (a) we don't have space in the flow |
|
* circular buffer, or (b) we already have max entries in the buffer. |
|
* Max entries depend on the type of request we are processing and the |
|
* negotiated TID RDMA parameters. |
|
*/ |
|
if (!CIRC_SPACE(req->setup_head, clear_tail, MAX_FLOWS) || |
|
CIRC_CNT(req->setup_head, clear_tail, MAX_FLOWS) >= |
|
req->n_flows) |
|
return -EINVAL; |
|
|
|
/* |
|
* Get pages, identify contiguous physical memory chunks for the segment |
|
* If we can not determine a DMA address mapping we will treat it just |
|
* like if we ran out of space above. |
|
*/ |
|
if (kern_get_phys_blocks(flow, qpriv->pages, ss, last)) { |
|
hfi1_wait_kmem(flow->req->qp); |
|
return -ENOMEM; |
|
} |
|
|
|
spin_lock_irqsave(&rcd->exp_lock, flags); |
|
if (kernel_tid_waiters(rcd, &rcd->rarr_queue, flow->req->qp)) |
|
goto queue; |
|
|
|
/* |
|
* At this point we know the number of pagesets and hence the number of |
|
* TID's to map the segment. Allocate the TID's from the TID groups. If |
|
* we cannot allocate the required number we exit and try again later |
|
*/ |
|
if (kern_alloc_tids(flow)) |
|
goto queue; |
|
/* |
|
* Finally program the TID entries with the pagesets, compute the |
|
* tidarray and enable the HW flow |
|
*/ |
|
kern_program_rcvarray(flow); |
|
|
|
/* |
|
* Setup the flow state with relevant information. |
|
* This information is used for tracking the sequence of data packets |
|
* for the segment. |
|
* The flow is setup here as this is the most accurate time and place |
|
* to do so. Doing at a later time runs the risk of the flow data in |
|
* qpriv getting out of sync. |
|
*/ |
|
memset(&flow->flow_state, 0x0, sizeof(flow->flow_state)); |
|
flow->idx = qpriv->flow_state.index; |
|
flow->flow_state.generation = qpriv->flow_state.generation; |
|
flow->flow_state.spsn = qpriv->flow_state.psn; |
|
flow->flow_state.lpsn = flow->flow_state.spsn + flow->npkts - 1; |
|
flow->flow_state.r_next_psn = |
|
full_flow_psn(flow, flow->flow_state.spsn); |
|
qpriv->flow_state.psn += flow->npkts; |
|
|
|
dequeue_tid_waiter(rcd, &rcd->rarr_queue, flow->req->qp); |
|
/* get head before dropping lock */ |
|
fqp = first_qp(rcd, &rcd->rarr_queue); |
|
spin_unlock_irqrestore(&rcd->exp_lock, flags); |
|
tid_rdma_schedule_tid_wakeup(fqp); |
|
|
|
req->setup_head = (req->setup_head + 1) & (MAX_FLOWS - 1); |
|
return 0; |
|
queue: |
|
queue_qp_for_tid_wait(rcd, &rcd->rarr_queue, flow->req->qp); |
|
spin_unlock_irqrestore(&rcd->exp_lock, flags); |
|
return -EAGAIN; |
|
} |
|
|
|
static void hfi1_tid_rdma_reset_flow(struct tid_rdma_flow *flow) |
|
{ |
|
flow->npagesets = 0; |
|
} |
|
|
|
/* |
|
* This function is called after one segment has been successfully sent to |
|
* release the flow and TID HW/SW resources for that segment. The segments for a |
|
* TID RDMA request are setup and cleared in FIFO order which is managed using a |
|
* circular buffer. |
|
*/ |
|
int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req) |
|
__must_hold(&req->qp->s_lock) |
|
{ |
|
struct tid_rdma_flow *flow = &req->flows[req->clear_tail]; |
|
struct hfi1_ctxtdata *rcd = req->rcd; |
|
unsigned long flags; |
|
int i; |
|
struct rvt_qp *fqp; |
|
|
|
lockdep_assert_held(&req->qp->s_lock); |
|
/* Exit if we have nothing in the flow circular buffer */ |
|
if (!CIRC_CNT(req->setup_head, req->clear_tail, MAX_FLOWS)) |
|
return -EINVAL; |
|
|
|
spin_lock_irqsave(&rcd->exp_lock, flags); |
|
|
|
for (i = 0; i < flow->tnode_cnt; i++) |
|
kern_unprogram_rcv_group(flow, i); |
|
/* To prevent double unprogramming */ |
|
flow->tnode_cnt = 0; |
|
/* get head before dropping lock */ |
|
fqp = first_qp(rcd, &rcd->rarr_queue); |
|
spin_unlock_irqrestore(&rcd->exp_lock, flags); |
|
|
|
dma_unmap_flow(flow); |
|
|
|
hfi1_tid_rdma_reset_flow(flow); |
|
req->clear_tail = (req->clear_tail + 1) & (MAX_FLOWS - 1); |
|
|
|
if (fqp == req->qp) { |
|
__trigger_tid_waiter(fqp); |
|
rvt_put_qp(fqp); |
|
} else { |
|
tid_rdma_schedule_tid_wakeup(fqp); |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* This function is called to release all the tid entries for |
|
* a request. |
|
*/ |
|
void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req) |
|
__must_hold(&req->qp->s_lock) |
|
{ |
|
/* Use memory barrier for proper ordering */ |
|
while (CIRC_CNT(req->setup_head, req->clear_tail, MAX_FLOWS)) { |
|
if (hfi1_kern_exp_rcv_clear(req)) |
|
break; |
|
} |
|
} |
|
|
|
/** |
|
* hfi1_kern_exp_rcv_free_flows - free priviously allocated flow information |
|
* @req: the tid rdma request to be cleaned |
|
*/ |
|
static void hfi1_kern_exp_rcv_free_flows(struct tid_rdma_request *req) |
|
{ |
|
kfree(req->flows); |
|
req->flows = NULL; |
|
} |
|
|
|
/** |
|
* __trdma_clean_swqe - clean up for large sized QPs |
|
* @qp: the queue patch |
|
* @wqe: the send wqe |
|
*/ |
|
void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) |
|
{ |
|
struct hfi1_swqe_priv *p = wqe->priv; |
|
|
|
hfi1_kern_exp_rcv_free_flows(&p->tid_req); |
|
} |
|
|
|
/* |
|
* This can be called at QP create time or in the data path. |
|
*/ |
|
static int hfi1_kern_exp_rcv_alloc_flows(struct tid_rdma_request *req, |
|
gfp_t gfp) |
|
{ |
|
struct tid_rdma_flow *flows; |
|
int i; |
|
|
|
if (likely(req->flows)) |
|
return 0; |
|
flows = kmalloc_node(MAX_FLOWS * sizeof(*flows), gfp, |
|
req->rcd->numa_id); |
|
if (!flows) |
|
return -ENOMEM; |
|
/* mini init */ |
|
for (i = 0; i < MAX_FLOWS; i++) { |
|
flows[i].req = req; |
|
flows[i].npagesets = 0; |
|
flows[i].pagesets[0].mapped = 0; |
|
flows[i].resync_npkts = 0; |
|
} |
|
req->flows = flows; |
|
return 0; |
|
} |
|
|
|
static void hfi1_init_trdma_req(struct rvt_qp *qp, |
|
struct tid_rdma_request *req) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
|
|
/* |
|
* Initialize various TID RDMA request variables. |
|
* These variables are "static", which is why they |
|
* can be pre-initialized here before the WRs has |
|
* even been submitted. |
|
* However, non-NULL values for these variables do not |
|
* imply that this WQE has been enabled for TID RDMA. |
|
* Drivers should check the WQE's opcode to determine |
|
* if a request is a TID RDMA one or not. |
|
*/ |
|
req->qp = qp; |
|
req->rcd = qpriv->rcd; |
|
} |
|
|
|
u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry, |
|
void *context, int vl, int mode, u64 data) |
|
{ |
|
struct hfi1_devdata *dd = context; |
|
|
|
return dd->verbs_dev.n_tidwait; |
|
} |
|
|
|
static struct tid_rdma_flow *find_flow_ib(struct tid_rdma_request *req, |
|
u32 psn, u16 *fidx) |
|
{ |
|
u16 head, tail; |
|
struct tid_rdma_flow *flow; |
|
|
|
head = req->setup_head; |
|
tail = req->clear_tail; |
|
for ( ; CIRC_CNT(head, tail, MAX_FLOWS); |
|
tail = CIRC_NEXT(tail, MAX_FLOWS)) { |
|
flow = &req->flows[tail]; |
|
if (cmp_psn(psn, flow->flow_state.ib_spsn) >= 0 && |
|
cmp_psn(psn, flow->flow_state.ib_lpsn) <= 0) { |
|
if (fidx) |
|
*fidx = tail; |
|
return flow; |
|
} |
|
} |
|
return NULL; |
|
} |
|
|
|
/* TID RDMA READ functions */ |
|
u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, |
|
struct ib_other_headers *ohdr, u32 *bth1, |
|
u32 *bth2, u32 *len) |
|
{ |
|
struct tid_rdma_request *req = wqe_to_tid_req(wqe); |
|
struct tid_rdma_flow *flow = &req->flows[req->flow_idx]; |
|
struct rvt_qp *qp = req->qp; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct hfi1_swqe_priv *wpriv = wqe->priv; |
|
struct tid_rdma_read_req *rreq = &ohdr->u.tid_rdma.r_req; |
|
struct tid_rdma_params *remote; |
|
u32 req_len = 0; |
|
void *req_addr = NULL; |
|
|
|
/* This is the IB psn used to send the request */ |
|
*bth2 = mask_psn(flow->flow_state.ib_spsn + flow->pkt); |
|
trace_hfi1_tid_flow_build_read_pkt(qp, req->flow_idx, flow); |
|
|
|
/* TID Entries for TID RDMA READ payload */ |
|
req_addr = &flow->tid_entry[flow->tid_idx]; |
|
req_len = sizeof(*flow->tid_entry) * |
|
(flow->tidcnt - flow->tid_idx); |
|
|
|
memset(&ohdr->u.tid_rdma.r_req, 0, sizeof(ohdr->u.tid_rdma.r_req)); |
|
wpriv->ss.sge.vaddr = req_addr; |
|
wpriv->ss.sge.sge_length = req_len; |
|
wpriv->ss.sge.length = wpriv->ss.sge.sge_length; |
|
/* |
|
* We can safely zero these out. Since the first SGE covers the |
|
* entire packet, nothing else should even look at the MR. |
|
*/ |
|
wpriv->ss.sge.mr = NULL; |
|
wpriv->ss.sge.m = 0; |
|
wpriv->ss.sge.n = 0; |
|
|
|
wpriv->ss.sg_list = NULL; |
|
wpriv->ss.total_len = wpriv->ss.sge.sge_length; |
|
wpriv->ss.num_sge = 1; |
|
|
|
/* Construct the TID RDMA READ REQ packet header */ |
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
|
|
KDETH_RESET(rreq->kdeth0, KVER, 0x1); |
|
KDETH_RESET(rreq->kdeth1, JKEY, remote->jkey); |
|
rreq->reth.vaddr = cpu_to_be64(wqe->rdma_wr.remote_addr + |
|
req->cur_seg * req->seg_len + flow->sent); |
|
rreq->reth.rkey = cpu_to_be32(wqe->rdma_wr.rkey); |
|
rreq->reth.length = cpu_to_be32(*len); |
|
rreq->tid_flow_psn = |
|
cpu_to_be32((flow->flow_state.generation << |
|
HFI1_KDETH_BTH_SEQ_SHIFT) | |
|
((flow->flow_state.spsn + flow->pkt) & |
|
HFI1_KDETH_BTH_SEQ_MASK)); |
|
rreq->tid_flow_qp = |
|
cpu_to_be32(qpriv->tid_rdma.local.qp | |
|
((flow->idx & TID_RDMA_DESTQP_FLOW_MASK) << |
|
TID_RDMA_DESTQP_FLOW_SHIFT) | |
|
qpriv->rcd->ctxt); |
|
rreq->verbs_qp = cpu_to_be32(qp->remote_qpn); |
|
*bth1 &= ~RVT_QPN_MASK; |
|
*bth1 |= remote->qp; |
|
*bth2 |= IB_BTH_REQ_ACK; |
|
rcu_read_unlock(); |
|
|
|
/* We are done with this segment */ |
|
flow->sent += *len; |
|
req->cur_seg++; |
|
qp->s_state = TID_OP(READ_REQ); |
|
req->ack_pending++; |
|
req->flow_idx = (req->flow_idx + 1) & (MAX_FLOWS - 1); |
|
qpriv->pending_tid_r_segs++; |
|
qp->s_num_rd_atomic++; |
|
|
|
/* Set the TID RDMA READ request payload size */ |
|
*len = req_len; |
|
|
|
return sizeof(ohdr->u.tid_rdma.r_req) / sizeof(u32); |
|
} |
|
|
|
/* |
|
* @len: contains the data length to read upon entry and the read request |
|
* payload length upon exit. |
|
*/ |
|
u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, |
|
struct ib_other_headers *ohdr, u32 *bth1, |
|
u32 *bth2, u32 *len) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_rdma_request *req = wqe_to_tid_req(wqe); |
|
struct tid_rdma_flow *flow = NULL; |
|
u32 hdwords = 0; |
|
bool last; |
|
bool retry = true; |
|
u32 npkts = rvt_div_round_up_mtu(qp, *len); |
|
|
|
trace_hfi1_tid_req_build_read_req(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
/* |
|
* Check sync conditions. Make sure that there are no pending |
|
* segments before freeing the flow. |
|
*/ |
|
sync_check: |
|
if (req->state == TID_REQUEST_SYNC) { |
|
if (qpriv->pending_tid_r_segs) |
|
goto done; |
|
|
|
hfi1_kern_clear_hw_flow(req->rcd, qp); |
|
qpriv->s_flags &= ~HFI1_R_TID_SW_PSN; |
|
req->state = TID_REQUEST_ACTIVE; |
|
} |
|
|
|
/* |
|
* If the request for this segment is resent, the tid resources should |
|
* have been allocated before. In this case, req->flow_idx should |
|
* fall behind req->setup_head. |
|
*/ |
|
if (req->flow_idx == req->setup_head) { |
|
retry = false; |
|
if (req->state == TID_REQUEST_RESEND) { |
|
/* |
|
* This is the first new segment for a request whose |
|
* earlier segments have been re-sent. We need to |
|
* set up the sge pointer correctly. |
|
*/ |
|
restart_sge(&qp->s_sge, wqe, req->s_next_psn, |
|
qp->pmtu); |
|
req->isge = 0; |
|
req->state = TID_REQUEST_ACTIVE; |
|
} |
|
|
|
/* |
|
* Check sync. The last PSN of each generation is reserved for |
|
* RESYNC. |
|
*/ |
|
if ((qpriv->flow_state.psn + npkts) > MAX_TID_FLOW_PSN - 1) { |
|
req->state = TID_REQUEST_SYNC; |
|
goto sync_check; |
|
} |
|
|
|
/* Allocate the flow if not yet */ |
|
if (hfi1_kern_setup_hw_flow(qpriv->rcd, qp)) |
|
goto done; |
|
|
|
/* |
|
* The following call will advance req->setup_head after |
|
* allocating the tid entries. |
|
*/ |
|
if (hfi1_kern_exp_rcv_setup(req, &qp->s_sge, &last)) { |
|
req->state = TID_REQUEST_QUEUED; |
|
|
|
/* |
|
* We don't have resources for this segment. The QP has |
|
* already been queued. |
|
*/ |
|
goto done; |
|
} |
|
} |
|
|
|
/* req->flow_idx should only be one slot behind req->setup_head */ |
|
flow = &req->flows[req->flow_idx]; |
|
flow->pkt = 0; |
|
flow->tid_idx = 0; |
|
flow->sent = 0; |
|
if (!retry) { |
|
/* Set the first and last IB PSN for the flow in use.*/ |
|
flow->flow_state.ib_spsn = req->s_next_psn; |
|
flow->flow_state.ib_lpsn = |
|
flow->flow_state.ib_spsn + flow->npkts - 1; |
|
} |
|
|
|
/* Calculate the next segment start psn.*/ |
|
req->s_next_psn += flow->npkts; |
|
|
|
/* Build the packet header */ |
|
hdwords = hfi1_build_tid_rdma_read_packet(wqe, ohdr, bth1, bth2, len); |
|
done: |
|
return hdwords; |
|
} |
|
|
|
/* |
|
* Validate and accept the TID RDMA READ request parameters. |
|
* Return 0 if the request is accepted successfully; |
|
* Return 1 otherwise. |
|
*/ |
|
static int tid_rdma_rcv_read_request(struct rvt_qp *qp, |
|
struct rvt_ack_entry *e, |
|
struct hfi1_packet *packet, |
|
struct ib_other_headers *ohdr, |
|
u32 bth0, u32 psn, u64 vaddr, u32 len) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
u32 flow_psn, i, tidlen = 0, pktlen, tlen; |
|
|
|
req = ack_to_tid_req(e); |
|
|
|
/* Validate the payload first */ |
|
flow = &req->flows[req->setup_head]; |
|
|
|
/* payload length = packet length - (header length + ICRC length) */ |
|
pktlen = packet->tlen - (packet->hlen + 4); |
|
if (pktlen > sizeof(flow->tid_entry)) |
|
return 1; |
|
memcpy(flow->tid_entry, packet->ebuf, pktlen); |
|
flow->tidcnt = pktlen / sizeof(*flow->tid_entry); |
|
|
|
/* |
|
* Walk the TID_ENTRY list to make sure we have enough space for a |
|
* complete segment. Also calculate the number of required packets. |
|
*/ |
|
flow->npkts = rvt_div_round_up_mtu(qp, len); |
|
for (i = 0; i < flow->tidcnt; i++) { |
|
trace_hfi1_tid_entry_rcv_read_req(qp, i, |
|
flow->tid_entry[i]); |
|
tlen = EXP_TID_GET(flow->tid_entry[i], LEN); |
|
if (!tlen) |
|
return 1; |
|
|
|
/* |
|
* For tid pair (tidctr == 3), the buffer size of the pair |
|
* should be the sum of the buffer size described by each |
|
* tid entry. However, only the first entry needs to be |
|
* specified in the request (see WFR HAS Section 8.5.7.1). |
|
*/ |
|
tidlen += tlen; |
|
} |
|
if (tidlen * PAGE_SIZE < len) |
|
return 1; |
|
|
|
/* Empty the flow array */ |
|
req->clear_tail = req->setup_head; |
|
flow->pkt = 0; |
|
flow->tid_idx = 0; |
|
flow->tid_offset = 0; |
|
flow->sent = 0; |
|
flow->tid_qpn = be32_to_cpu(ohdr->u.tid_rdma.r_req.tid_flow_qp); |
|
flow->idx = (flow->tid_qpn >> TID_RDMA_DESTQP_FLOW_SHIFT) & |
|
TID_RDMA_DESTQP_FLOW_MASK; |
|
flow_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.r_req.tid_flow_psn)); |
|
flow->flow_state.generation = flow_psn >> HFI1_KDETH_BTH_SEQ_SHIFT; |
|
flow->flow_state.spsn = flow_psn & HFI1_KDETH_BTH_SEQ_MASK; |
|
flow->length = len; |
|
|
|
flow->flow_state.lpsn = flow->flow_state.spsn + |
|
flow->npkts - 1; |
|
flow->flow_state.ib_spsn = psn; |
|
flow->flow_state.ib_lpsn = flow->flow_state.ib_spsn + flow->npkts - 1; |
|
|
|
trace_hfi1_tid_flow_rcv_read_req(qp, req->setup_head, flow); |
|
/* Set the initial flow index to the current flow. */ |
|
req->flow_idx = req->setup_head; |
|
|
|
/* advance circular buffer head */ |
|
req->setup_head = (req->setup_head + 1) & (MAX_FLOWS - 1); |
|
|
|
/* |
|
* Compute last PSN for request. |
|
*/ |
|
e->opcode = (bth0 >> 24) & 0xff; |
|
e->psn = psn; |
|
e->lpsn = psn + flow->npkts - 1; |
|
e->sent = 0; |
|
|
|
req->n_flows = qpriv->tid_rdma.local.max_read; |
|
req->state = TID_REQUEST_ACTIVE; |
|
req->cur_seg = 0; |
|
req->comp_seg = 0; |
|
req->ack_seg = 0; |
|
req->isge = 0; |
|
req->seg_len = qpriv->tid_rdma.local.max_len; |
|
req->total_len = len; |
|
req->total_segs = 1; |
|
req->r_flow_psn = e->psn; |
|
|
|
trace_hfi1_tid_req_rcv_read_req(qp, 0, e->opcode, e->psn, e->lpsn, |
|
req); |
|
return 0; |
|
} |
|
|
|
static int tid_rdma_rcv_error(struct hfi1_packet *packet, |
|
struct ib_other_headers *ohdr, |
|
struct rvt_qp *qp, u32 psn, int diff) |
|
{ |
|
struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); |
|
struct hfi1_ctxtdata *rcd = ((struct hfi1_qp_priv *)qp->priv)->rcd; |
|
struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct rvt_ack_entry *e; |
|
struct tid_rdma_request *req; |
|
unsigned long flags; |
|
u8 prev; |
|
bool old_req; |
|
|
|
trace_hfi1_rsp_tid_rcv_error(qp, psn); |
|
trace_hfi1_tid_rdma_rcv_err(qp, 0, psn, diff); |
|
if (diff > 0) { |
|
/* sequence error */ |
|
if (!qp->r_nak_state) { |
|
ibp->rvp.n_rc_seqnak++; |
|
qp->r_nak_state = IB_NAK_PSN_ERROR; |
|
qp->r_ack_psn = qp->r_psn; |
|
rc_defered_ack(rcd, qp); |
|
} |
|
goto done; |
|
} |
|
|
|
ibp->rvp.n_rc_dupreq++; |
|
|
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
e = find_prev_entry(qp, psn, &prev, NULL, &old_req); |
|
if (!e || (e->opcode != TID_OP(READ_REQ) && |
|
e->opcode != TID_OP(WRITE_REQ))) |
|
goto unlock; |
|
|
|
req = ack_to_tid_req(e); |
|
req->r_flow_psn = psn; |
|
trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, e->lpsn, req); |
|
if (e->opcode == TID_OP(READ_REQ)) { |
|
struct ib_reth *reth; |
|
u32 len; |
|
u32 rkey; |
|
u64 vaddr; |
|
int ok; |
|
u32 bth0; |
|
|
|
reth = &ohdr->u.tid_rdma.r_req.reth; |
|
/* |
|
* The requester always restarts from the start of the original |
|
* request. |
|
*/ |
|
len = be32_to_cpu(reth->length); |
|
if (psn != e->psn || len != req->total_len) |
|
goto unlock; |
|
|
|
release_rdma_sge_mr(e); |
|
|
|
rkey = be32_to_cpu(reth->rkey); |
|
vaddr = get_ib_reth_vaddr(reth); |
|
|
|
qp->r_len = len; |
|
ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, |
|
IB_ACCESS_REMOTE_READ); |
|
if (unlikely(!ok)) |
|
goto unlock; |
|
|
|
/* |
|
* If all the response packets for the current request have |
|
* been sent out and this request is complete (old_request |
|
* == false) and the TID flow may be unusable (the |
|
* req->clear_tail is advanced). However, when an earlier |
|
* request is received, this request will not be complete any |
|
* more (qp->s_tail_ack_queue is moved back, see below). |
|
* Consequently, we need to update the TID flow info everytime |
|
* a duplicate request is received. |
|
*/ |
|
bth0 = be32_to_cpu(ohdr->bth[0]); |
|
if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn, |
|
vaddr, len)) |
|
goto unlock; |
|
|
|
/* |
|
* True if the request is already scheduled (between |
|
* qp->s_tail_ack_queue and qp->r_head_ack_queue); |
|
*/ |
|
if (old_req) |
|
goto unlock; |
|
} else { |
|
struct flow_state *fstate; |
|
bool schedule = false; |
|
u8 i; |
|
|
|
if (req->state == TID_REQUEST_RESEND) { |
|
req->state = TID_REQUEST_RESEND_ACTIVE; |
|
} else if (req->state == TID_REQUEST_INIT_RESEND) { |
|
req->state = TID_REQUEST_INIT; |
|
schedule = true; |
|
} |
|
|
|
/* |
|
* True if the request is already scheduled (between |
|
* qp->s_tail_ack_queue and qp->r_head_ack_queue). |
|
* Also, don't change requests, which are at the SYNC |
|
* point and haven't generated any responses yet. |
|
* There is nothing to retransmit for them yet. |
|
*/ |
|
if (old_req || req->state == TID_REQUEST_INIT || |
|
(req->state == TID_REQUEST_SYNC && !req->cur_seg)) { |
|
for (i = prev + 1; ; i++) { |
|
if (i > rvt_size_atomic(&dev->rdi)) |
|
i = 0; |
|
if (i == qp->r_head_ack_queue) |
|
break; |
|
e = &qp->s_ack_queue[i]; |
|
req = ack_to_tid_req(e); |
|
if (e->opcode == TID_OP(WRITE_REQ) && |
|
req->state == TID_REQUEST_INIT) |
|
req->state = TID_REQUEST_INIT_RESEND; |
|
} |
|
/* |
|
* If the state of the request has been changed, |
|
* the first leg needs to get scheduled in order to |
|
* pick up the change. Otherwise, normal response |
|
* processing should take care of it. |
|
*/ |
|
if (!schedule) |
|
goto unlock; |
|
} |
|
|
|
/* |
|
* If there is no more allocated segment, just schedule the qp |
|
* without changing any state. |
|
*/ |
|
if (req->clear_tail == req->setup_head) |
|
goto schedule; |
|
/* |
|
* If this request has sent responses for segments, which have |
|
* not received data yet (flow_idx != clear_tail), the flow_idx |
|
* pointer needs to be adjusted so the same responses can be |
|
* re-sent. |
|
*/ |
|
if (CIRC_CNT(req->flow_idx, req->clear_tail, MAX_FLOWS)) { |
|
fstate = &req->flows[req->clear_tail].flow_state; |
|
qpriv->pending_tid_w_segs -= |
|
CIRC_CNT(req->flow_idx, req->clear_tail, |
|
MAX_FLOWS); |
|
req->flow_idx = |
|
CIRC_ADD(req->clear_tail, |
|
delta_psn(psn, fstate->resp_ib_psn), |
|
MAX_FLOWS); |
|
qpriv->pending_tid_w_segs += |
|
delta_psn(psn, fstate->resp_ib_psn); |
|
/* |
|
* When flow_idx == setup_head, we've gotten a duplicate |
|
* request for a segment, which has not been allocated |
|
* yet. In that case, don't adjust this request. |
|
* However, we still want to go through the loop below |
|
* to adjust all subsequent requests. |
|
*/ |
|
if (CIRC_CNT(req->setup_head, req->flow_idx, |
|
MAX_FLOWS)) { |
|
req->cur_seg = delta_psn(psn, e->psn); |
|
req->state = TID_REQUEST_RESEND_ACTIVE; |
|
} |
|
} |
|
|
|
for (i = prev + 1; ; i++) { |
|
/* |
|
* Look at everything up to and including |
|
* s_tail_ack_queue |
|
*/ |
|
if (i > rvt_size_atomic(&dev->rdi)) |
|
i = 0; |
|
if (i == qp->r_head_ack_queue) |
|
break; |
|
e = &qp->s_ack_queue[i]; |
|
req = ack_to_tid_req(e); |
|
trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, |
|
e->lpsn, req); |
|
if (e->opcode != TID_OP(WRITE_REQ) || |
|
req->cur_seg == req->comp_seg || |
|
req->state == TID_REQUEST_INIT || |
|
req->state == TID_REQUEST_INIT_RESEND) { |
|
if (req->state == TID_REQUEST_INIT) |
|
req->state = TID_REQUEST_INIT_RESEND; |
|
continue; |
|
} |
|
qpriv->pending_tid_w_segs -= |
|
CIRC_CNT(req->flow_idx, |
|
req->clear_tail, |
|
MAX_FLOWS); |
|
req->flow_idx = req->clear_tail; |
|
req->state = TID_REQUEST_RESEND; |
|
req->cur_seg = req->comp_seg; |
|
} |
|
qpriv->s_flags &= ~HFI1_R_TID_WAIT_INTERLCK; |
|
} |
|
/* Re-process old requests.*/ |
|
if (qp->s_acked_ack_queue == qp->s_tail_ack_queue) |
|
qp->s_acked_ack_queue = prev; |
|
qp->s_tail_ack_queue = prev; |
|
/* |
|
* Since the qp->s_tail_ack_queue is modified, the |
|
* qp->s_ack_state must be changed to re-initialize |
|
* qp->s_ack_rdma_sge; Otherwise, we will end up in |
|
* wrong memory region. |
|
*/ |
|
qp->s_ack_state = OP(ACKNOWLEDGE); |
|
schedule: |
|
/* |
|
* It's possible to receive a retry psn that is earlier than an RNRNAK |
|
* psn. In this case, the rnrnak state should be cleared. |
|
*/ |
|
if (qpriv->rnr_nak_state) { |
|
qp->s_nak_state = 0; |
|
qpriv->rnr_nak_state = TID_RNR_NAK_INIT; |
|
qp->r_psn = e->lpsn + 1; |
|
hfi1_tid_write_alloc_resources(qp, true); |
|
} |
|
|
|
qp->r_state = e->opcode; |
|
qp->r_nak_state = 0; |
|
qp->s_flags |= RVT_S_RESP_PENDING; |
|
hfi1_schedule_send(qp); |
|
unlock: |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
done: |
|
return 1; |
|
} |
|
|
|
void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet) |
|
{ |
|
/* HANDLER FOR TID RDMA READ REQUEST packet (Responder side)*/ |
|
|
|
/* |
|
* 1. Verify TID RDMA READ REQ as per IB_OPCODE_RC_RDMA_READ |
|
* (see hfi1_rc_rcv()) |
|
* 2. Put TID RDMA READ REQ into the response queueu (s_ack_queue) |
|
* - Setup struct tid_rdma_req with request info |
|
* - Initialize struct tid_rdma_flow info; |
|
* - Copy TID entries; |
|
* 3. Set the qp->s_ack_state. |
|
* 4. Set RVT_S_RESP_PENDING in s_flags. |
|
* 5. Kick the send engine (hfi1_schedule_send()) |
|
*/ |
|
struct hfi1_ctxtdata *rcd = packet->rcd; |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); |
|
struct ib_other_headers *ohdr = packet->ohdr; |
|
struct rvt_ack_entry *e; |
|
unsigned long flags; |
|
struct ib_reth *reth; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
u32 bth0, psn, len, rkey; |
|
bool fecn; |
|
u8 next; |
|
u64 vaddr; |
|
int diff; |
|
u8 nack_state = IB_NAK_INVALID_REQUEST; |
|
|
|
bth0 = be32_to_cpu(ohdr->bth[0]); |
|
if (hfi1_ruc_check_hdr(ibp, packet)) |
|
return; |
|
|
|
fecn = process_ecn(qp, packet); |
|
psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
trace_hfi1_rsp_rcv_tid_read_req(qp, psn); |
|
|
|
if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) |
|
rvt_comm_est(qp); |
|
|
|
if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) |
|
goto nack_inv; |
|
|
|
reth = &ohdr->u.tid_rdma.r_req.reth; |
|
vaddr = be64_to_cpu(reth->vaddr); |
|
len = be32_to_cpu(reth->length); |
|
/* The length needs to be in multiples of PAGE_SIZE */ |
|
if (!len || len & ~PAGE_MASK || len > qpriv->tid_rdma.local.max_len) |
|
goto nack_inv; |
|
|
|
diff = delta_psn(psn, qp->r_psn); |
|
if (unlikely(diff)) { |
|
tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn); |
|
return; |
|
} |
|
|
|
/* We've verified the request, insert it into the ack queue. */ |
|
next = qp->r_head_ack_queue + 1; |
|
if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) |
|
next = 0; |
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
if (unlikely(next == qp->s_tail_ack_queue)) { |
|
if (!qp->s_ack_queue[next].sent) { |
|
nack_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; |
|
goto nack_inv_unlock; |
|
} |
|
update_ack_queue(qp, next); |
|
} |
|
e = &qp->s_ack_queue[qp->r_head_ack_queue]; |
|
release_rdma_sge_mr(e); |
|
|
|
rkey = be32_to_cpu(reth->rkey); |
|
qp->r_len = len; |
|
|
|
if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr, |
|
rkey, IB_ACCESS_REMOTE_READ))) |
|
goto nack_acc; |
|
|
|
/* Accept the request parameters */ |
|
if (tid_rdma_rcv_read_request(qp, e, packet, ohdr, bth0, psn, vaddr, |
|
len)) |
|
goto nack_inv_unlock; |
|
|
|
qp->r_state = e->opcode; |
|
qp->r_nak_state = 0; |
|
/* |
|
* We need to increment the MSN here instead of when we |
|
* finish sending the result since a duplicate request would |
|
* increment it more than once. |
|
*/ |
|
qp->r_msn++; |
|
qp->r_psn += e->lpsn - e->psn + 1; |
|
|
|
qp->r_head_ack_queue = next; |
|
|
|
/* |
|
* For all requests other than TID WRITE which are added to the ack |
|
* queue, qpriv->r_tid_alloc follows qp->r_head_ack_queue. It is ok to |
|
* do this because of interlocks between these and TID WRITE |
|
* requests. The same change has also been made in hfi1_rc_rcv(). |
|
*/ |
|
qpriv->r_tid_alloc = qp->r_head_ack_queue; |
|
|
|
/* Schedule the send tasklet. */ |
|
qp->s_flags |= RVT_S_RESP_PENDING; |
|
if (fecn) |
|
qp->s_flags |= RVT_S_ECN; |
|
hfi1_schedule_send(qp); |
|
|
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
return; |
|
|
|
nack_inv_unlock: |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
nack_inv: |
|
rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); |
|
qp->r_nak_state = nack_state; |
|
qp->r_ack_psn = qp->r_psn; |
|
/* Queue NAK for later */ |
|
rc_defered_ack(rcd, qp); |
|
return; |
|
nack_acc: |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); |
|
qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; |
|
qp->r_ack_psn = qp->r_psn; |
|
} |
|
|
|
u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, |
|
struct ib_other_headers *ohdr, u32 *bth0, |
|
u32 *bth1, u32 *bth2, u32 *len, bool *last) |
|
{ |
|
struct hfi1_ack_priv *epriv = e->priv; |
|
struct tid_rdma_request *req = &epriv->tid_req; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_rdma_flow *flow = &req->flows[req->clear_tail]; |
|
u32 tidentry = flow->tid_entry[flow->tid_idx]; |
|
u32 tidlen = EXP_TID_GET(tidentry, LEN) << PAGE_SHIFT; |
|
struct tid_rdma_read_resp *resp = &ohdr->u.tid_rdma.r_rsp; |
|
u32 next_offset, om = KDETH_OM_LARGE; |
|
bool last_pkt; |
|
u32 hdwords = 0; |
|
struct tid_rdma_params *remote; |
|
|
|
*len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset); |
|
flow->sent += *len; |
|
next_offset = flow->tid_offset + *len; |
|
last_pkt = (flow->sent >= flow->length); |
|
|
|
trace_hfi1_tid_entry_build_read_resp(qp, flow->tid_idx, tidentry); |
|
trace_hfi1_tid_flow_build_read_resp(qp, req->clear_tail, flow); |
|
|
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
if (!remote) { |
|
rcu_read_unlock(); |
|
goto done; |
|
} |
|
KDETH_RESET(resp->kdeth0, KVER, 0x1); |
|
KDETH_SET(resp->kdeth0, SH, !last_pkt); |
|
KDETH_SET(resp->kdeth0, INTR, !!(!last_pkt && remote->urg)); |
|
KDETH_SET(resp->kdeth0, TIDCTRL, EXP_TID_GET(tidentry, CTRL)); |
|
KDETH_SET(resp->kdeth0, TID, EXP_TID_GET(tidentry, IDX)); |
|
KDETH_SET(resp->kdeth0, OM, om == KDETH_OM_LARGE); |
|
KDETH_SET(resp->kdeth0, OFFSET, flow->tid_offset / om); |
|
KDETH_RESET(resp->kdeth1, JKEY, remote->jkey); |
|
resp->verbs_qp = cpu_to_be32(qp->remote_qpn); |
|
rcu_read_unlock(); |
|
|
|
resp->aeth = rvt_compute_aeth(qp); |
|
resp->verbs_psn = cpu_to_be32(mask_psn(flow->flow_state.ib_spsn + |
|
flow->pkt)); |
|
|
|
*bth0 = TID_OP(READ_RESP) << 24; |
|
*bth1 = flow->tid_qpn; |
|
*bth2 = mask_psn(((flow->flow_state.spsn + flow->pkt++) & |
|
HFI1_KDETH_BTH_SEQ_MASK) | |
|
(flow->flow_state.generation << |
|
HFI1_KDETH_BTH_SEQ_SHIFT)); |
|
*last = last_pkt; |
|
if (last_pkt) |
|
/* Advance to next flow */ |
|
req->clear_tail = (req->clear_tail + 1) & |
|
(MAX_FLOWS - 1); |
|
|
|
if (next_offset >= tidlen) { |
|
flow->tid_offset = 0; |
|
flow->tid_idx++; |
|
} else { |
|
flow->tid_offset = next_offset; |
|
} |
|
|
|
hdwords = sizeof(ohdr->u.tid_rdma.r_rsp) / sizeof(u32); |
|
|
|
done: |
|
return hdwords; |
|
} |
|
|
|
static inline struct tid_rdma_request * |
|
find_tid_request(struct rvt_qp *qp, u32 psn, enum ib_wr_opcode opcode) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
struct rvt_swqe *wqe; |
|
struct tid_rdma_request *req = NULL; |
|
u32 i, end; |
|
|
|
end = qp->s_cur + 1; |
|
if (end == qp->s_size) |
|
end = 0; |
|
for (i = qp->s_acked; i != end;) { |
|
wqe = rvt_get_swqe_ptr(qp, i); |
|
if (cmp_psn(psn, wqe->psn) >= 0 && |
|
cmp_psn(psn, wqe->lpsn) <= 0) { |
|
if (wqe->wr.opcode == opcode) |
|
req = wqe_to_tid_req(wqe); |
|
break; |
|
} |
|
if (++i == qp->s_size) |
|
i = 0; |
|
} |
|
|
|
return req; |
|
} |
|
|
|
void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet) |
|
{ |
|
/* HANDLER FOR TID RDMA READ RESPONSE packet (Requestor side */ |
|
|
|
/* |
|
* 1. Find matching SWQE |
|
* 2. Check that the entire segment has been read. |
|
* 3. Remove HFI1_S_WAIT_TID_RESP from s_flags. |
|
* 4. Free the TID flow resources. |
|
* 5. Kick the send engine (hfi1_schedule_send()) |
|
*/ |
|
struct ib_other_headers *ohdr = packet->ohdr; |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct hfi1_ctxtdata *rcd = packet->rcd; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
u32 opcode, aeth; |
|
bool fecn; |
|
unsigned long flags; |
|
u32 kpsn, ipsn; |
|
|
|
trace_hfi1_sender_rcv_tid_read_resp(qp); |
|
fecn = process_ecn(qp, packet); |
|
kpsn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
aeth = be32_to_cpu(ohdr->u.tid_rdma.r_rsp.aeth); |
|
opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff; |
|
|
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
ipsn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.r_rsp.verbs_psn)); |
|
req = find_tid_request(qp, ipsn, IB_WR_TID_RDMA_READ); |
|
if (unlikely(!req)) |
|
goto ack_op_err; |
|
|
|
flow = &req->flows[req->clear_tail]; |
|
/* When header suppression is disabled */ |
|
if (cmp_psn(ipsn, flow->flow_state.ib_lpsn)) { |
|
update_r_next_psn_fecn(packet, priv, rcd, flow, fecn); |
|
|
|
if (cmp_psn(kpsn, flow->flow_state.r_next_psn)) |
|
goto ack_done; |
|
flow->flow_state.r_next_psn = mask_psn(kpsn + 1); |
|
/* |
|
* Copy the payload to destination buffer if this packet is |
|
* delivered as an eager packet due to RSM rule and FECN. |
|
* The RSM rule selects FECN bit in BTH and SH bit in |
|
* KDETH header and therefore will not match the last |
|
* packet of each segment that has SH bit cleared. |
|
*/ |
|
if (fecn && packet->etype == RHF_RCV_TYPE_EAGER) { |
|
struct rvt_sge_state ss; |
|
u32 len; |
|
u32 tlen = packet->tlen; |
|
u16 hdrsize = packet->hlen; |
|
u8 pad = packet->pad; |
|
u8 extra_bytes = pad + packet->extra_byte + |
|
(SIZE_OF_CRC << 2); |
|
u32 pmtu = qp->pmtu; |
|
|
|
if (unlikely(tlen != (hdrsize + pmtu + extra_bytes))) |
|
goto ack_op_err; |
|
len = restart_sge(&ss, req->e.swqe, ipsn, pmtu); |
|
if (unlikely(len < pmtu)) |
|
goto ack_op_err; |
|
rvt_copy_sge(qp, &ss, packet->payload, pmtu, false, |
|
false); |
|
/* Raise the sw sequence check flag for next packet */ |
|
priv->s_flags |= HFI1_R_TID_SW_PSN; |
|
} |
|
|
|
goto ack_done; |
|
} |
|
flow->flow_state.r_next_psn = mask_psn(kpsn + 1); |
|
req->ack_pending--; |
|
priv->pending_tid_r_segs--; |
|
qp->s_num_rd_atomic--; |
|
if ((qp->s_flags & RVT_S_WAIT_FENCE) && |
|
!qp->s_num_rd_atomic) { |
|
qp->s_flags &= ~(RVT_S_WAIT_FENCE | |
|
RVT_S_WAIT_ACK); |
|
hfi1_schedule_send(qp); |
|
} |
|
if (qp->s_flags & RVT_S_WAIT_RDMAR) { |
|
qp->s_flags &= ~(RVT_S_WAIT_RDMAR | RVT_S_WAIT_ACK); |
|
hfi1_schedule_send(qp); |
|
} |
|
|
|
trace_hfi1_ack(qp, ipsn); |
|
trace_hfi1_tid_req_rcv_read_resp(qp, 0, req->e.swqe->wr.opcode, |
|
req->e.swqe->psn, req->e.swqe->lpsn, |
|
req); |
|
trace_hfi1_tid_flow_rcv_read_resp(qp, req->clear_tail, flow); |
|
|
|
/* Release the tid resources */ |
|
hfi1_kern_exp_rcv_clear(req); |
|
|
|
if (!do_rc_ack(qp, aeth, ipsn, opcode, 0, rcd)) |
|
goto ack_done; |
|
|
|
/* If not done yet, build next read request */ |
|
if (++req->comp_seg >= req->total_segs) { |
|
priv->tid_r_comp++; |
|
req->state = TID_REQUEST_COMPLETE; |
|
} |
|
|
|
/* |
|
* Clear the hw flow under two conditions: |
|
* 1. This request is a sync point and it is complete; |
|
* 2. Current request is completed and there are no more requests. |
|
*/ |
|
if ((req->state == TID_REQUEST_SYNC && |
|
req->comp_seg == req->cur_seg) || |
|
priv->tid_r_comp == priv->tid_r_reqs) { |
|
hfi1_kern_clear_hw_flow(priv->rcd, qp); |
|
priv->s_flags &= ~HFI1_R_TID_SW_PSN; |
|
if (req->state == TID_REQUEST_SYNC) |
|
req->state = TID_REQUEST_ACTIVE; |
|
} |
|
|
|
hfi1_schedule_send(qp); |
|
goto ack_done; |
|
|
|
ack_op_err: |
|
/* |
|
* The test indicates that the send engine has finished its cleanup |
|
* after sending the request and it's now safe to put the QP into error |
|
* state. However, if the wqe queue is empty (qp->s_acked == qp->s_tail |
|
* == qp->s_head), it would be unsafe to complete the wqe pointed by |
|
* qp->s_acked here. Putting the qp into error state will safely flush |
|
* all remaining requests. |
|
*/ |
|
if (qp->s_last == qp->s_acked) |
|
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); |
|
|
|
ack_done: |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
} |
|
|
|
void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
u32 n = qp->s_acked; |
|
struct rvt_swqe *wqe; |
|
struct tid_rdma_request *req; |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
/* Free any TID entries */ |
|
while (n != qp->s_tail) { |
|
wqe = rvt_get_swqe_ptr(qp, n); |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { |
|
req = wqe_to_tid_req(wqe); |
|
hfi1_kern_exp_rcv_clear_all(req); |
|
} |
|
|
|
if (++n == qp->s_size) |
|
n = 0; |
|
} |
|
/* Free flow */ |
|
hfi1_kern_clear_hw_flow(priv->rcd, qp); |
|
} |
|
|
|
static bool tid_rdma_tid_err(struct hfi1_packet *packet, u8 rcv_type) |
|
{ |
|
struct rvt_qp *qp = packet->qp; |
|
|
|
if (rcv_type >= RHF_RCV_TYPE_IB) |
|
goto done; |
|
|
|
spin_lock(&qp->s_lock); |
|
|
|
/* |
|
* We've ran out of space in the eager buffer. |
|
* Eagerly received KDETH packets which require space in the |
|
* Eager buffer (packet that have payload) are TID RDMA WRITE |
|
* response packets. In this case, we have to re-transmit the |
|
* TID RDMA WRITE request. |
|
*/ |
|
if (rcv_type == RHF_RCV_TYPE_EAGER) { |
|
hfi1_restart_rc(qp, qp->s_last_psn + 1, 1); |
|
hfi1_schedule_send(qp); |
|
} |
|
|
|
/* Since no payload is delivered, just drop the packet */ |
|
spin_unlock(&qp->s_lock); |
|
done: |
|
return true; |
|
} |
|
|
|
static void restart_tid_rdma_read_req(struct hfi1_ctxtdata *rcd, |
|
struct rvt_qp *qp, struct rvt_swqe *wqe) |
|
{ |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
|
|
/* Start from the right segment */ |
|
qp->r_flags |= RVT_R_RDMAR_SEQ; |
|
req = wqe_to_tid_req(wqe); |
|
flow = &req->flows[req->clear_tail]; |
|
hfi1_restart_rc(qp, flow->flow_state.ib_spsn, 0); |
|
if (list_empty(&qp->rspwait)) { |
|
qp->r_flags |= RVT_R_RSP_SEND; |
|
rvt_get_qp(qp); |
|
list_add_tail(&qp->rspwait, &rcd->qp_wait_list); |
|
} |
|
} |
|
|
|
/* |
|
* Handle the KDETH eflags for TID RDMA READ response. |
|
* |
|
* Return true if the last packet for a segment has been received and it is |
|
* time to process the response normally; otherwise, return true. |
|
* |
|
* The caller must hold the packet->qp->r_lock and the rcu_read_lock. |
|
*/ |
|
static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd, |
|
struct hfi1_packet *packet, u8 rcv_type, |
|
u8 rte, u32 psn, u32 ibpsn) |
|
__must_hold(&packet->qp->r_lock) __must_hold(RCU) |
|
{ |
|
struct hfi1_pportdata *ppd = rcd->ppd; |
|
struct hfi1_devdata *dd = ppd->dd; |
|
struct hfi1_ibport *ibp; |
|
struct rvt_swqe *wqe; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
u32 ack_psn; |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
bool ret = true; |
|
int diff = 0; |
|
u32 fpsn; |
|
|
|
lockdep_assert_held(&qp->r_lock); |
|
trace_hfi1_rsp_read_kdeth_eflags(qp, ibpsn); |
|
trace_hfi1_sender_read_kdeth_eflags(qp); |
|
trace_hfi1_tid_read_sender_kdeth_eflags(qp, 0); |
|
spin_lock(&qp->s_lock); |
|
/* If the psn is out of valid range, drop the packet */ |
|
if (cmp_psn(ibpsn, qp->s_last_psn) < 0 || |
|
cmp_psn(ibpsn, qp->s_psn) > 0) |
|
goto s_unlock; |
|
|
|
/* |
|
* Note that NAKs implicitly ACK outstanding SEND and RDMA write |
|
* requests and implicitly NAK RDMA read and atomic requests issued |
|
* before the NAK'ed request. |
|
*/ |
|
ack_psn = ibpsn - 1; |
|
wqe = rvt_get_swqe_ptr(qp, qp->s_acked); |
|
ibp = to_iport(qp->ibqp.device, qp->port_num); |
|
|
|
/* Complete WQEs that the PSN finishes. */ |
|
while ((int)delta_psn(ack_psn, wqe->lpsn) >= 0) { |
|
/* |
|
* If this request is a RDMA read or atomic, and the NACK is |
|
* for a later operation, this NACK NAKs the RDMA read or |
|
* atomic. |
|
*/ |
|
if (wqe->wr.opcode == IB_WR_RDMA_READ || |
|
wqe->wr.opcode == IB_WR_TID_RDMA_READ || |
|
wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || |
|
wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { |
|
/* Retry this request. */ |
|
if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { |
|
qp->r_flags |= RVT_R_RDMAR_SEQ; |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { |
|
restart_tid_rdma_read_req(rcd, qp, |
|
wqe); |
|
} else { |
|
hfi1_restart_rc(qp, qp->s_last_psn + 1, |
|
0); |
|
if (list_empty(&qp->rspwait)) { |
|
qp->r_flags |= RVT_R_RSP_SEND; |
|
rvt_get_qp(qp); |
|
list_add_tail(/* wait */ |
|
&qp->rspwait, |
|
&rcd->qp_wait_list); |
|
} |
|
} |
|
} |
|
/* |
|
* No need to process the NAK since we are |
|
* restarting an earlier request. |
|
*/ |
|
break; |
|
} |
|
|
|
wqe = do_rc_completion(qp, wqe, ibp); |
|
if (qp->s_acked == qp->s_tail) |
|
goto s_unlock; |
|
} |
|
|
|
if (qp->s_acked == qp->s_tail) |
|
goto s_unlock; |
|
|
|
/* Handle the eflags for the request */ |
|
if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) |
|
goto s_unlock; |
|
|
|
req = wqe_to_tid_req(wqe); |
|
trace_hfi1_tid_req_read_kdeth_eflags(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
switch (rcv_type) { |
|
case RHF_RCV_TYPE_EXPECTED: |
|
switch (rte) { |
|
case RHF_RTE_EXPECTED_FLOW_SEQ_ERR: |
|
/* |
|
* On the first occurrence of a Flow Sequence error, |
|
* the flag TID_FLOW_SW_PSN is set. |
|
* |
|
* After that, the flow is *not* reprogrammed and the |
|
* protocol falls back to SW PSN checking. This is done |
|
* to prevent continuous Flow Sequence errors for any |
|
* packets that could be still in the fabric. |
|
*/ |
|
flow = &req->flows[req->clear_tail]; |
|
trace_hfi1_tid_flow_read_kdeth_eflags(qp, |
|
req->clear_tail, |
|
flow); |
|
if (priv->s_flags & HFI1_R_TID_SW_PSN) { |
|
diff = cmp_psn(psn, |
|
flow->flow_state.r_next_psn); |
|
if (diff > 0) { |
|
/* Drop the packet.*/ |
|
goto s_unlock; |
|
} else if (diff < 0) { |
|
/* |
|
* If a response packet for a restarted |
|
* request has come back, reset the |
|
* restart flag. |
|
*/ |
|
if (qp->r_flags & RVT_R_RDMAR_SEQ) |
|
qp->r_flags &= |
|
~RVT_R_RDMAR_SEQ; |
|
|
|
/* Drop the packet.*/ |
|
goto s_unlock; |
|
} |
|
|
|
/* |
|
* If SW PSN verification is successful and |
|
* this is the last packet in the segment, tell |
|
* the caller to process it as a normal packet. |
|
*/ |
|
fpsn = full_flow_psn(flow, |
|
flow->flow_state.lpsn); |
|
if (cmp_psn(fpsn, psn) == 0) { |
|
ret = false; |
|
if (qp->r_flags & RVT_R_RDMAR_SEQ) |
|
qp->r_flags &= |
|
~RVT_R_RDMAR_SEQ; |
|
} |
|
flow->flow_state.r_next_psn = |
|
mask_psn(psn + 1); |
|
} else { |
|
u32 last_psn; |
|
|
|
last_psn = read_r_next_psn(dd, rcd->ctxt, |
|
flow->idx); |
|
flow->flow_state.r_next_psn = last_psn; |
|
priv->s_flags |= HFI1_R_TID_SW_PSN; |
|
/* |
|
* If no request has been restarted yet, |
|
* restart the current one. |
|
*/ |
|
if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) |
|
restart_tid_rdma_read_req(rcd, qp, |
|
wqe); |
|
} |
|
|
|
break; |
|
|
|
case RHF_RTE_EXPECTED_FLOW_GEN_ERR: |
|
/* |
|
* Since the TID flow is able to ride through |
|
* generation mismatch, drop this stale packet. |
|
*/ |
|
break; |
|
|
|
default: |
|
break; |
|
} |
|
break; |
|
|
|
case RHF_RCV_TYPE_ERROR: |
|
switch (rte) { |
|
case RHF_RTE_ERROR_OP_CODE_ERR: |
|
case RHF_RTE_ERROR_KHDR_MIN_LEN_ERR: |
|
case RHF_RTE_ERROR_KHDR_HCRC_ERR: |
|
case RHF_RTE_ERROR_KHDR_KVER_ERR: |
|
case RHF_RTE_ERROR_CONTEXT_ERR: |
|
case RHF_RTE_ERROR_KHDR_TID_ERR: |
|
default: |
|
break; |
|
} |
|
break; |
|
default: |
|
break; |
|
} |
|
s_unlock: |
|
spin_unlock(&qp->s_lock); |
|
return ret; |
|
} |
|
|
|
bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd, |
|
struct hfi1_pportdata *ppd, |
|
struct hfi1_packet *packet) |
|
{ |
|
struct hfi1_ibport *ibp = &ppd->ibport_data; |
|
struct hfi1_devdata *dd = ppd->dd; |
|
struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; |
|
u8 rcv_type = rhf_rcv_type(packet->rhf); |
|
u8 rte = rhf_rcv_type_err(packet->rhf); |
|
struct ib_header *hdr = packet->hdr; |
|
struct ib_other_headers *ohdr = NULL; |
|
int lnh = be16_to_cpu(hdr->lrh[0]) & 3; |
|
u16 lid = be16_to_cpu(hdr->lrh[1]); |
|
u8 opcode; |
|
u32 qp_num, psn, ibpsn; |
|
struct rvt_qp *qp; |
|
struct hfi1_qp_priv *qpriv; |
|
unsigned long flags; |
|
bool ret = true; |
|
struct rvt_ack_entry *e; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
int diff = 0; |
|
|
|
trace_hfi1_msg_handle_kdeth_eflags(NULL, "Kdeth error: rhf ", |
|
packet->rhf); |
|
if (packet->rhf & RHF_ICRC_ERR) |
|
return ret; |
|
|
|
packet->ohdr = &hdr->u.oth; |
|
ohdr = packet->ohdr; |
|
trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf))); |
|
|
|
/* Get the destination QP number. */ |
|
qp_num = be32_to_cpu(ohdr->u.tid_rdma.r_rsp.verbs_qp) & |
|
RVT_QPN_MASK; |
|
if (lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) |
|
goto drop; |
|
|
|
psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff; |
|
|
|
rcu_read_lock(); |
|
qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); |
|
if (!qp) |
|
goto rcu_unlock; |
|
|
|
packet->qp = qp; |
|
|
|
/* Check for valid receive state. */ |
|
spin_lock_irqsave(&qp->r_lock, flags); |
|
if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { |
|
ibp->rvp.n_pkt_drops++; |
|
goto r_unlock; |
|
} |
|
|
|
if (packet->rhf & RHF_TID_ERR) { |
|
/* For TIDERR and RC QPs preemptively schedule a NAK */ |
|
u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ |
|
|
|
/* Sanity check packet */ |
|
if (tlen < 24) |
|
goto r_unlock; |
|
|
|
/* |
|
* Check for GRH. We should never get packets with GRH in this |
|
* path. |
|
*/ |
|
if (lnh == HFI1_LRH_GRH) |
|
goto r_unlock; |
|
|
|
if (tid_rdma_tid_err(packet, rcv_type)) |
|
goto r_unlock; |
|
} |
|
|
|
/* handle TID RDMA READ */ |
|
if (opcode == TID_OP(READ_RESP)) { |
|
ibpsn = be32_to_cpu(ohdr->u.tid_rdma.r_rsp.verbs_psn); |
|
ibpsn = mask_psn(ibpsn); |
|
ret = handle_read_kdeth_eflags(rcd, packet, rcv_type, rte, psn, |
|
ibpsn); |
|
goto r_unlock; |
|
} |
|
|
|
/* |
|
* qp->s_tail_ack_queue points to the rvt_ack_entry currently being |
|
* processed. These a completed sequentially so we can be sure that |
|
* the pointer will not change until the entire request has completed. |
|
*/ |
|
spin_lock(&qp->s_lock); |
|
qpriv = qp->priv; |
|
if (qpriv->r_tid_tail == HFI1_QP_WQE_INVALID || |
|
qpriv->r_tid_tail == qpriv->r_tid_head) |
|
goto unlock; |
|
e = &qp->s_ack_queue[qpriv->r_tid_tail]; |
|
if (e->opcode != TID_OP(WRITE_REQ)) |
|
goto unlock; |
|
req = ack_to_tid_req(e); |
|
if (req->comp_seg == req->cur_seg) |
|
goto unlock; |
|
flow = &req->flows[req->clear_tail]; |
|
trace_hfi1_eflags_err_write(qp, rcv_type, rte, psn); |
|
trace_hfi1_rsp_handle_kdeth_eflags(qp, psn); |
|
trace_hfi1_tid_write_rsp_handle_kdeth_eflags(qp); |
|
trace_hfi1_tid_req_handle_kdeth_eflags(qp, 0, e->opcode, e->psn, |
|
e->lpsn, req); |
|
trace_hfi1_tid_flow_handle_kdeth_eflags(qp, req->clear_tail, flow); |
|
|
|
switch (rcv_type) { |
|
case RHF_RCV_TYPE_EXPECTED: |
|
switch (rte) { |
|
case RHF_RTE_EXPECTED_FLOW_SEQ_ERR: |
|
if (!(qpriv->s_flags & HFI1_R_TID_SW_PSN)) { |
|
qpriv->s_flags |= HFI1_R_TID_SW_PSN; |
|
flow->flow_state.r_next_psn = |
|
read_r_next_psn(dd, rcd->ctxt, |
|
flow->idx); |
|
qpriv->r_next_psn_kdeth = |
|
flow->flow_state.r_next_psn; |
|
goto nak_psn; |
|
} else { |
|
/* |
|
* If the received PSN does not match the next |
|
* expected PSN, NAK the packet. |
|
* However, only do that if we know that the a |
|
* NAK has already been sent. Otherwise, this |
|
* mismatch could be due to packets that were |
|
* already in flight. |
|
*/ |
|
diff = cmp_psn(psn, |
|
flow->flow_state.r_next_psn); |
|
if (diff > 0) |
|
goto nak_psn; |
|
else if (diff < 0) |
|
break; |
|
|
|
qpriv->s_nak_state = 0; |
|
/* |
|
* If SW PSN verification is successful and this |
|
* is the last packet in the segment, tell the |
|
* caller to process it as a normal packet. |
|
*/ |
|
if (psn == full_flow_psn(flow, |
|
flow->flow_state.lpsn)) |
|
ret = false; |
|
flow->flow_state.r_next_psn = |
|
mask_psn(psn + 1); |
|
qpriv->r_next_psn_kdeth = |
|
flow->flow_state.r_next_psn; |
|
} |
|
break; |
|
|
|
case RHF_RTE_EXPECTED_FLOW_GEN_ERR: |
|
goto nak_psn; |
|
|
|
default: |
|
break; |
|
} |
|
break; |
|
|
|
case RHF_RCV_TYPE_ERROR: |
|
switch (rte) { |
|
case RHF_RTE_ERROR_OP_CODE_ERR: |
|
case RHF_RTE_ERROR_KHDR_MIN_LEN_ERR: |
|
case RHF_RTE_ERROR_KHDR_HCRC_ERR: |
|
case RHF_RTE_ERROR_KHDR_KVER_ERR: |
|
case RHF_RTE_ERROR_CONTEXT_ERR: |
|
case RHF_RTE_ERROR_KHDR_TID_ERR: |
|
default: |
|
break; |
|
} |
|
break; |
|
default: |
|
break; |
|
} |
|
|
|
unlock: |
|
spin_unlock(&qp->s_lock); |
|
r_unlock: |
|
spin_unlock_irqrestore(&qp->r_lock, flags); |
|
rcu_unlock: |
|
rcu_read_unlock(); |
|
drop: |
|
return ret; |
|
nak_psn: |
|
ibp->rvp.n_rc_seqnak++; |
|
if (!qpriv->s_nak_state) { |
|
qpriv->s_nak_state = IB_NAK_PSN_ERROR; |
|
/* We are NAK'ing the next expected PSN */ |
|
qpriv->s_nak_psn = mask_psn(flow->flow_state.r_next_psn); |
|
tid_rdma_trigger_ack(qp); |
|
} |
|
goto unlock; |
|
} |
|
|
|
/* |
|
* "Rewind" the TID request information. |
|
* This means that we reset the state back to ACTIVE, |
|
* find the proper flow, set the flow index to that flow, |
|
* and reset the flow information. |
|
*/ |
|
void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, |
|
u32 *bth2) |
|
{ |
|
struct tid_rdma_request *req = wqe_to_tid_req(wqe); |
|
struct tid_rdma_flow *flow; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
int diff, delta_pkts; |
|
u32 tididx = 0, i; |
|
u16 fidx; |
|
|
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { |
|
*bth2 = mask_psn(qp->s_psn); |
|
flow = find_flow_ib(req, *bth2, &fidx); |
|
if (!flow) { |
|
trace_hfi1_msg_tid_restart_req(/* msg */ |
|
qp, "!!!!!! Could not find flow to restart: bth2 ", |
|
(u64)*bth2); |
|
trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, |
|
wqe->psn, wqe->lpsn, |
|
req); |
|
return; |
|
} |
|
} else { |
|
fidx = req->acked_tail; |
|
flow = &req->flows[fidx]; |
|
*bth2 = mask_psn(req->r_ack_psn); |
|
} |
|
|
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) |
|
delta_pkts = delta_psn(*bth2, flow->flow_state.ib_spsn); |
|
else |
|
delta_pkts = delta_psn(*bth2, |
|
full_flow_psn(flow, |
|
flow->flow_state.spsn)); |
|
|
|
trace_hfi1_tid_flow_restart_req(qp, fidx, flow); |
|
diff = delta_pkts + flow->resync_npkts; |
|
|
|
flow->sent = 0; |
|
flow->pkt = 0; |
|
flow->tid_idx = 0; |
|
flow->tid_offset = 0; |
|
if (diff) { |
|
for (tididx = 0; tididx < flow->tidcnt; tididx++) { |
|
u32 tidentry = flow->tid_entry[tididx], tidlen, |
|
tidnpkts, npkts; |
|
|
|
flow->tid_offset = 0; |
|
tidlen = EXP_TID_GET(tidentry, LEN) * PAGE_SIZE; |
|
tidnpkts = rvt_div_round_up_mtu(qp, tidlen); |
|
npkts = min_t(u32, diff, tidnpkts); |
|
flow->pkt += npkts; |
|
flow->sent += (npkts == tidnpkts ? tidlen : |
|
npkts * qp->pmtu); |
|
flow->tid_offset += npkts * qp->pmtu; |
|
diff -= npkts; |
|
if (!diff) |
|
break; |
|
} |
|
} |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { |
|
rvt_skip_sge(&qpriv->tid_ss, (req->cur_seg * req->seg_len) + |
|
flow->sent, 0); |
|
/* |
|
* Packet PSN is based on flow_state.spsn + flow->pkt. However, |
|
* during a RESYNC, the generation is incremented and the |
|
* sequence is reset to 0. Since we've adjusted the npkts in the |
|
* flow and the SGE has been sufficiently advanced, we have to |
|
* adjust flow->pkt in order to calculate the correct PSN. |
|
*/ |
|
flow->pkt -= flow->resync_npkts; |
|
} |
|
|
|
if (flow->tid_offset == |
|
EXP_TID_GET(flow->tid_entry[tididx], LEN) * PAGE_SIZE) { |
|
tididx++; |
|
flow->tid_offset = 0; |
|
} |
|
flow->tid_idx = tididx; |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) |
|
/* Move flow_idx to correct index */ |
|
req->flow_idx = fidx; |
|
else |
|
req->clear_tail = fidx; |
|
|
|
trace_hfi1_tid_flow_restart_req(qp, fidx, flow); |
|
trace_hfi1_tid_req_restart_req(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
req->state = TID_REQUEST_ACTIVE; |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { |
|
/* Reset all the flows that we are going to resend */ |
|
fidx = CIRC_NEXT(fidx, MAX_FLOWS); |
|
i = qpriv->s_tid_tail; |
|
do { |
|
for (; CIRC_CNT(req->setup_head, fidx, MAX_FLOWS); |
|
fidx = CIRC_NEXT(fidx, MAX_FLOWS)) { |
|
req->flows[fidx].sent = 0; |
|
req->flows[fidx].pkt = 0; |
|
req->flows[fidx].tid_idx = 0; |
|
req->flows[fidx].tid_offset = 0; |
|
req->flows[fidx].resync_npkts = 0; |
|
} |
|
if (i == qpriv->s_tid_cur) |
|
break; |
|
do { |
|
i = (++i == qp->s_size ? 0 : i); |
|
wqe = rvt_get_swqe_ptr(qp, i); |
|
} while (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE); |
|
req = wqe_to_tid_req(wqe); |
|
req->cur_seg = req->ack_seg; |
|
fidx = req->acked_tail; |
|
/* Pull req->clear_tail back */ |
|
req->clear_tail = fidx; |
|
} while (1); |
|
} |
|
} |
|
|
|
void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp) |
|
{ |
|
int i, ret; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_flow_state *fs; |
|
|
|
if (qp->ibqp.qp_type != IB_QPT_RC || !HFI1_CAP_IS_KSET(TID_RDMA)) |
|
return; |
|
|
|
/* |
|
* First, clear the flow to help prevent any delayed packets from |
|
* being delivered. |
|
*/ |
|
fs = &qpriv->flow_state; |
|
if (fs->index != RXE_NUM_TID_FLOWS) |
|
hfi1_kern_clear_hw_flow(qpriv->rcd, qp); |
|
|
|
for (i = qp->s_acked; i != qp->s_head;) { |
|
struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, i); |
|
|
|
if (++i == qp->s_size) |
|
i = 0; |
|
/* Free only locally allocated TID entries */ |
|
if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) |
|
continue; |
|
do { |
|
struct hfi1_swqe_priv *priv = wqe->priv; |
|
|
|
ret = hfi1_kern_exp_rcv_clear(&priv->tid_req); |
|
} while (!ret); |
|
} |
|
for (i = qp->s_acked_ack_queue; i != qp->r_head_ack_queue;) { |
|
struct rvt_ack_entry *e = &qp->s_ack_queue[i]; |
|
|
|
if (++i == rvt_max_atomic(ib_to_rvt(qp->ibqp.device))) |
|
i = 0; |
|
/* Free only locally allocated TID entries */ |
|
if (e->opcode != TID_OP(WRITE_REQ)) |
|
continue; |
|
do { |
|
struct hfi1_ack_priv *priv = e->priv; |
|
|
|
ret = hfi1_kern_exp_rcv_clear(&priv->tid_req); |
|
} while (!ret); |
|
} |
|
} |
|
|
|
bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe) |
|
{ |
|
struct rvt_swqe *prev; |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
u32 s_prev; |
|
struct tid_rdma_request *req; |
|
|
|
s_prev = (qp->s_cur == 0 ? qp->s_size : qp->s_cur) - 1; |
|
prev = rvt_get_swqe_ptr(qp, s_prev); |
|
|
|
switch (wqe->wr.opcode) { |
|
case IB_WR_SEND: |
|
case IB_WR_SEND_WITH_IMM: |
|
case IB_WR_SEND_WITH_INV: |
|
case IB_WR_ATOMIC_CMP_AND_SWP: |
|
case IB_WR_ATOMIC_FETCH_AND_ADD: |
|
case IB_WR_RDMA_WRITE: |
|
case IB_WR_RDMA_WRITE_WITH_IMM: |
|
switch (prev->wr.opcode) { |
|
case IB_WR_TID_RDMA_WRITE: |
|
req = wqe_to_tid_req(prev); |
|
if (req->ack_seg != req->total_segs) |
|
goto interlock; |
|
break; |
|
default: |
|
break; |
|
} |
|
break; |
|
case IB_WR_RDMA_READ: |
|
if (prev->wr.opcode != IB_WR_TID_RDMA_WRITE) |
|
break; |
|
fallthrough; |
|
case IB_WR_TID_RDMA_READ: |
|
switch (prev->wr.opcode) { |
|
case IB_WR_RDMA_READ: |
|
if (qp->s_acked != qp->s_cur) |
|
goto interlock; |
|
break; |
|
case IB_WR_TID_RDMA_WRITE: |
|
req = wqe_to_tid_req(prev); |
|
if (req->ack_seg != req->total_segs) |
|
goto interlock; |
|
break; |
|
default: |
|
break; |
|
} |
|
break; |
|
default: |
|
break; |
|
} |
|
return false; |
|
|
|
interlock: |
|
priv->s_flags |= HFI1_S_TID_WAIT_INTERLCK; |
|
return true; |
|
} |
|
|
|
/* Does @sge meet the alignment requirements for tid rdma? */ |
|
static inline bool hfi1_check_sge_align(struct rvt_qp *qp, |
|
struct rvt_sge *sge, int num_sge) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < num_sge; i++, sge++) { |
|
trace_hfi1_sge_check_align(qp, i, sge); |
|
if ((u64)sge->vaddr & ~PAGE_MASK || |
|
sge->sge_length & ~PAGE_MASK) |
|
return false; |
|
} |
|
return true; |
|
} |
|
|
|
void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe) |
|
{ |
|
struct hfi1_qp_priv *qpriv = (struct hfi1_qp_priv *)qp->priv; |
|
struct hfi1_swqe_priv *priv = wqe->priv; |
|
struct tid_rdma_params *remote; |
|
enum ib_wr_opcode new_opcode; |
|
bool do_tid_rdma = false; |
|
struct hfi1_pportdata *ppd = qpriv->rcd->ppd; |
|
|
|
if ((rdma_ah_get_dlid(&qp->remote_ah_attr) & ~((1 << ppd->lmc) - 1)) == |
|
ppd->lid) |
|
return; |
|
if (qpriv->hdr_type != HFI1_PKT_TYPE_9B) |
|
return; |
|
|
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
/* |
|
* If TID RDMA is disabled by the negotiation, don't |
|
* use it. |
|
*/ |
|
if (!remote) |
|
goto exit; |
|
|
|
if (wqe->wr.opcode == IB_WR_RDMA_READ) { |
|
if (hfi1_check_sge_align(qp, &wqe->sg_list[0], |
|
wqe->wr.num_sge)) { |
|
new_opcode = IB_WR_TID_RDMA_READ; |
|
do_tid_rdma = true; |
|
} |
|
} else if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { |
|
/* |
|
* TID RDMA is enabled for this RDMA WRITE request iff: |
|
* 1. The remote address is page-aligned, |
|
* 2. The length is larger than the minimum segment size, |
|
* 3. The length is page-multiple. |
|
*/ |
|
if (!(wqe->rdma_wr.remote_addr & ~PAGE_MASK) && |
|
!(wqe->length & ~PAGE_MASK)) { |
|
new_opcode = IB_WR_TID_RDMA_WRITE; |
|
do_tid_rdma = true; |
|
} |
|
} |
|
|
|
if (do_tid_rdma) { |
|
if (hfi1_kern_exp_rcv_alloc_flows(&priv->tid_req, GFP_ATOMIC)) |
|
goto exit; |
|
wqe->wr.opcode = new_opcode; |
|
priv->tid_req.seg_len = |
|
min_t(u32, remote->max_len, wqe->length); |
|
priv->tid_req.total_segs = |
|
DIV_ROUND_UP(wqe->length, priv->tid_req.seg_len); |
|
/* Compute the last PSN of the request */ |
|
wqe->lpsn = wqe->psn; |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { |
|
priv->tid_req.n_flows = remote->max_read; |
|
qpriv->tid_r_reqs++; |
|
wqe->lpsn += rvt_div_round_up_mtu(qp, wqe->length) - 1; |
|
} else { |
|
wqe->lpsn += priv->tid_req.total_segs - 1; |
|
atomic_inc(&qpriv->n_requests); |
|
} |
|
|
|
priv->tid_req.cur_seg = 0; |
|
priv->tid_req.comp_seg = 0; |
|
priv->tid_req.ack_seg = 0; |
|
priv->tid_req.state = TID_REQUEST_INACTIVE; |
|
/* |
|
* Reset acked_tail. |
|
* TID RDMA READ does not have ACKs so it does not |
|
* update the pointer. We have to reset it so TID RDMA |
|
* WRITE does not get confused. |
|
*/ |
|
priv->tid_req.acked_tail = priv->tid_req.setup_head; |
|
trace_hfi1_tid_req_setup_tid_wqe(qp, 1, wqe->wr.opcode, |
|
wqe->psn, wqe->lpsn, |
|
&priv->tid_req); |
|
} |
|
exit: |
|
rcu_read_unlock(); |
|
} |
|
|
|
/* TID RDMA WRITE functions */ |
|
|
|
u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, |
|
struct ib_other_headers *ohdr, |
|
u32 *bth1, u32 *bth2, u32 *len) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_rdma_request *req = wqe_to_tid_req(wqe); |
|
struct tid_rdma_params *remote; |
|
|
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
/* |
|
* Set the number of flow to be used based on negotiated |
|
* parameters. |
|
*/ |
|
req->n_flows = remote->max_write; |
|
req->state = TID_REQUEST_ACTIVE; |
|
|
|
KDETH_RESET(ohdr->u.tid_rdma.w_req.kdeth0, KVER, 0x1); |
|
KDETH_RESET(ohdr->u.tid_rdma.w_req.kdeth1, JKEY, remote->jkey); |
|
ohdr->u.tid_rdma.w_req.reth.vaddr = |
|
cpu_to_be64(wqe->rdma_wr.remote_addr + (wqe->length - *len)); |
|
ohdr->u.tid_rdma.w_req.reth.rkey = |
|
cpu_to_be32(wqe->rdma_wr.rkey); |
|
ohdr->u.tid_rdma.w_req.reth.length = cpu_to_be32(*len); |
|
ohdr->u.tid_rdma.w_req.verbs_qp = cpu_to_be32(qp->remote_qpn); |
|
*bth1 &= ~RVT_QPN_MASK; |
|
*bth1 |= remote->qp; |
|
qp->s_state = TID_OP(WRITE_REQ); |
|
qp->s_flags |= HFI1_S_WAIT_TID_RESP; |
|
*bth2 |= IB_BTH_REQ_ACK; |
|
*len = 0; |
|
|
|
rcu_read_unlock(); |
|
return sizeof(ohdr->u.tid_rdma.w_req) / sizeof(u32); |
|
} |
|
|
|
static u32 hfi1_compute_tid_rdma_flow_wt(struct rvt_qp *qp) |
|
{ |
|
/* |
|
* Heuristic for computing the RNR timeout when waiting on the flow |
|
* queue. Rather than a computationaly expensive exact estimate of when |
|
* a flow will be available, we assume that if a QP is at position N in |
|
* the flow queue it has to wait approximately (N + 1) * (number of |
|
* segments between two sync points). The rationale for this is that |
|
* flows are released and recycled at each sync point. |
|
*/ |
|
return (MAX_TID_FLOW_PSN * qp->pmtu) >> TID_RDMA_SEGMENT_SHIFT; |
|
} |
|
|
|
static u32 position_in_queue(struct hfi1_qp_priv *qpriv, |
|
struct tid_queue *queue) |
|
{ |
|
return qpriv->tid_enqueue - queue->dequeue; |
|
} |
|
|
|
/* |
|
* @qp: points to rvt_qp context. |
|
* @to_seg: desired RNR timeout in segments. |
|
* Return: index of the next highest timeout in the ib_hfi1_rnr_table[] |
|
*/ |
|
static u32 hfi1_compute_tid_rnr_timeout(struct rvt_qp *qp, u32 to_seg) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
u64 timeout; |
|
u32 bytes_per_us; |
|
u8 i; |
|
|
|
bytes_per_us = active_egress_rate(qpriv->rcd->ppd) / 8; |
|
timeout = (to_seg * TID_RDMA_MAX_SEGMENT_SIZE) / bytes_per_us; |
|
/* |
|
* Find the next highest value in the RNR table to the required |
|
* timeout. This gives the responder some padding. |
|
*/ |
|
for (i = 1; i <= IB_AETH_CREDIT_MASK; i++) |
|
if (rvt_rnr_tbl_to_usec(i) >= timeout) |
|
return i; |
|
return 0; |
|
} |
|
|
|
/* |
|
* Central place for resource allocation at TID write responder, |
|
* is called from write_req and write_data interrupt handlers as |
|
* well as the send thread when a queued QP is scheduled for |
|
* resource allocation. |
|
* |
|
* Iterates over (a) segments of a request and then (b) queued requests |
|
* themselves to allocate resources for up to local->max_write |
|
* segments across multiple requests. Stop allocating when we |
|
* hit a sync point, resume allocating after data packets at |
|
* sync point have been received. |
|
* |
|
* Resource allocation and sending of responses is decoupled. The |
|
* request/segment which are being allocated and sent are as follows. |
|
* Resources are allocated for: |
|
* [request: qpriv->r_tid_alloc, segment: req->alloc_seg] |
|
* The send thread sends: |
|
* [request: qp->s_tail_ack_queue, segment:req->cur_seg] |
|
*/ |
|
static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx) |
|
{ |
|
struct tid_rdma_request *req; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct hfi1_ctxtdata *rcd = qpriv->rcd; |
|
struct tid_rdma_params *local = &qpriv->tid_rdma.local; |
|
struct rvt_ack_entry *e; |
|
u32 npkts, to_seg; |
|
bool last; |
|
int ret = 0; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
|
|
while (1) { |
|
trace_hfi1_rsp_tid_write_alloc_res(qp, 0); |
|
trace_hfi1_tid_write_rsp_alloc_res(qp); |
|
/* |
|
* Don't allocate more segments if a RNR NAK has already been |
|
* scheduled to avoid messing up qp->r_psn: the RNR NAK will |
|
* be sent only when all allocated segments have been sent. |
|
* However, if more segments are allocated before that, TID RDMA |
|
* WRITE RESP packets will be sent out for these new segments |
|
* before the RNR NAK packet. When the requester receives the |
|
* RNR NAK packet, it will restart with qp->s_last_psn + 1, |
|
* which does not match qp->r_psn and will be dropped. |
|
* Consequently, the requester will exhaust its retries and |
|
* put the qp into error state. |
|
*/ |
|
if (qpriv->rnr_nak_state == TID_RNR_NAK_SEND) |
|
break; |
|
|
|
/* No requests left to process */ |
|
if (qpriv->r_tid_alloc == qpriv->r_tid_head) { |
|
/* If all data has been received, clear the flow */ |
|
if (qpriv->flow_state.index < RXE_NUM_TID_FLOWS && |
|
!qpriv->alloc_w_segs) { |
|
hfi1_kern_clear_hw_flow(rcd, qp); |
|
qpriv->s_flags &= ~HFI1_R_TID_SW_PSN; |
|
} |
|
break; |
|
} |
|
|
|
e = &qp->s_ack_queue[qpriv->r_tid_alloc]; |
|
if (e->opcode != TID_OP(WRITE_REQ)) |
|
goto next_req; |
|
req = ack_to_tid_req(e); |
|
trace_hfi1_tid_req_write_alloc_res(qp, 0, e->opcode, e->psn, |
|
e->lpsn, req); |
|
/* Finished allocating for all segments of this request */ |
|
if (req->alloc_seg >= req->total_segs) |
|
goto next_req; |
|
|
|
/* Can allocate only a maximum of local->max_write for a QP */ |
|
if (qpriv->alloc_w_segs >= local->max_write) |
|
break; |
|
|
|
/* Don't allocate at a sync point with data packets pending */ |
|
if (qpriv->sync_pt && qpriv->alloc_w_segs) |
|
break; |
|
|
|
/* All data received at the sync point, continue */ |
|
if (qpriv->sync_pt && !qpriv->alloc_w_segs) { |
|
hfi1_kern_clear_hw_flow(rcd, qp); |
|
qpriv->sync_pt = false; |
|
qpriv->s_flags &= ~HFI1_R_TID_SW_PSN; |
|
} |
|
|
|
/* Allocate flow if we don't have one */ |
|
if (qpriv->flow_state.index >= RXE_NUM_TID_FLOWS) { |
|
ret = hfi1_kern_setup_hw_flow(qpriv->rcd, qp); |
|
if (ret) { |
|
to_seg = hfi1_compute_tid_rdma_flow_wt(qp) * |
|
position_in_queue(qpriv, |
|
&rcd->flow_queue); |
|
break; |
|
} |
|
} |
|
|
|
npkts = rvt_div_round_up_mtu(qp, req->seg_len); |
|
|
|
/* |
|
* We are at a sync point if we run out of KDETH PSN space. |
|
* Last PSN of every generation is reserved for RESYNC. |
|
*/ |
|
if (qpriv->flow_state.psn + npkts > MAX_TID_FLOW_PSN - 1) { |
|
qpriv->sync_pt = true; |
|
break; |
|
} |
|
|
|
/* |
|
* If overtaking req->acked_tail, send an RNR NAK. Because the |
|
* QP is not queued in this case, and the issue can only be |
|
* caused by a delay in scheduling the second leg which we |
|
* cannot estimate, we use a rather arbitrary RNR timeout of |
|
* (MAX_FLOWS / 2) segments |
|
*/ |
|
if (!CIRC_SPACE(req->setup_head, req->acked_tail, |
|
MAX_FLOWS)) { |
|
ret = -EAGAIN; |
|
to_seg = MAX_FLOWS >> 1; |
|
tid_rdma_trigger_ack(qp); |
|
break; |
|
} |
|
|
|
/* Try to allocate rcv array / TID entries */ |
|
ret = hfi1_kern_exp_rcv_setup(req, &req->ss, &last); |
|
if (ret == -EAGAIN) |
|
to_seg = position_in_queue(qpriv, &rcd->rarr_queue); |
|
if (ret) |
|
break; |
|
|
|
qpriv->alloc_w_segs++; |
|
req->alloc_seg++; |
|
continue; |
|
next_req: |
|
/* Begin processing the next request */ |
|
if (++qpriv->r_tid_alloc > |
|
rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) |
|
qpriv->r_tid_alloc = 0; |
|
} |
|
|
|
/* |
|
* Schedule an RNR NAK to be sent if (a) flow or rcv array allocation |
|
* has failed (b) we are called from the rcv handler interrupt context |
|
* (c) an RNR NAK has not already been scheduled |
|
*/ |
|
if (ret == -EAGAIN && intr_ctx && !qp->r_nak_state) |
|
goto send_rnr_nak; |
|
|
|
return; |
|
|
|
send_rnr_nak: |
|
lockdep_assert_held(&qp->r_lock); |
|
|
|
/* Set r_nak_state to prevent unrelated events from generating NAK's */ |
|
qp->r_nak_state = hfi1_compute_tid_rnr_timeout(qp, to_seg) | IB_RNR_NAK; |
|
|
|
/* Pull back r_psn to the segment being RNR NAK'd */ |
|
qp->r_psn = e->psn + req->alloc_seg; |
|
qp->r_ack_psn = qp->r_psn; |
|
/* |
|
* Pull back r_head_ack_queue to the ack entry following the request |
|
* being RNR NAK'd. This allows resources to be allocated to the request |
|
* if the queued QP is scheduled. |
|
*/ |
|
qp->r_head_ack_queue = qpriv->r_tid_alloc + 1; |
|
if (qp->r_head_ack_queue > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) |
|
qp->r_head_ack_queue = 0; |
|
qpriv->r_tid_head = qp->r_head_ack_queue; |
|
/* |
|
* These send side fields are used in make_rc_ack(). They are set in |
|
* hfi1_send_rc_ack() but must be set here before dropping qp->s_lock |
|
* for consistency |
|
*/ |
|
qp->s_nak_state = qp->r_nak_state; |
|
qp->s_ack_psn = qp->r_ack_psn; |
|
/* |
|
* Clear the ACK PENDING flag to prevent unwanted ACK because we |
|
* have modified qp->s_ack_psn here. |
|
*/ |
|
qp->s_flags &= ~(RVT_S_ACK_PENDING); |
|
|
|
trace_hfi1_rsp_tid_write_alloc_res(qp, qp->r_psn); |
|
/* |
|
* qpriv->rnr_nak_state is used to determine when the scheduled RNR NAK |
|
* has actually been sent. qp->s_flags RVT_S_ACK_PENDING bit cannot be |
|
* used for this because qp->s_lock is dropped before calling |
|
* hfi1_send_rc_ack() leading to inconsistency between the receive |
|
* interrupt handlers and the send thread in make_rc_ack() |
|
*/ |
|
qpriv->rnr_nak_state = TID_RNR_NAK_SEND; |
|
|
|
/* |
|
* Schedule RNR NAK to be sent. RNR NAK's are scheduled from the receive |
|
* interrupt handlers but will be sent from the send engine behind any |
|
* previous responses that may have been scheduled |
|
*/ |
|
rc_defered_ack(rcd, qp); |
|
} |
|
|
|
void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet) |
|
{ |
|
/* HANDLER FOR TID RDMA WRITE REQUEST packet (Responder side)*/ |
|
|
|
/* |
|
* 1. Verify TID RDMA WRITE REQ as per IB_OPCODE_RC_RDMA_WRITE_FIRST |
|
* (see hfi1_rc_rcv()) |
|
* - Don't allow 0-length requests. |
|
* 2. Put TID RDMA WRITE REQ into the response queueu (s_ack_queue) |
|
* - Setup struct tid_rdma_req with request info |
|
* - Prepare struct tid_rdma_flow array? |
|
* 3. Set the qp->s_ack_state as state diagram in design doc. |
|
* 4. Set RVT_S_RESP_PENDING in s_flags. |
|
* 5. Kick the send engine (hfi1_schedule_send()) |
|
*/ |
|
struct hfi1_ctxtdata *rcd = packet->rcd; |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); |
|
struct ib_other_headers *ohdr = packet->ohdr; |
|
struct rvt_ack_entry *e; |
|
unsigned long flags; |
|
struct ib_reth *reth; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_rdma_request *req; |
|
u32 bth0, psn, len, rkey, num_segs; |
|
bool fecn; |
|
u8 next; |
|
u64 vaddr; |
|
int diff; |
|
|
|
bth0 = be32_to_cpu(ohdr->bth[0]); |
|
if (hfi1_ruc_check_hdr(ibp, packet)) |
|
return; |
|
|
|
fecn = process_ecn(qp, packet); |
|
psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
trace_hfi1_rsp_rcv_tid_write_req(qp, psn); |
|
|
|
if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) |
|
rvt_comm_est(qp); |
|
|
|
if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) |
|
goto nack_inv; |
|
|
|
reth = &ohdr->u.tid_rdma.w_req.reth; |
|
vaddr = be64_to_cpu(reth->vaddr); |
|
len = be32_to_cpu(reth->length); |
|
|
|
num_segs = DIV_ROUND_UP(len, qpriv->tid_rdma.local.max_len); |
|
diff = delta_psn(psn, qp->r_psn); |
|
if (unlikely(diff)) { |
|
tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn); |
|
return; |
|
} |
|
|
|
/* |
|
* The resent request which was previously RNR NAK'd is inserted at the |
|
* location of the original request, which is one entry behind |
|
* r_head_ack_queue |
|
*/ |
|
if (qpriv->rnr_nak_state) |
|
qp->r_head_ack_queue = qp->r_head_ack_queue ? |
|
qp->r_head_ack_queue - 1 : |
|
rvt_size_atomic(ib_to_rvt(qp->ibqp.device)); |
|
|
|
/* We've verified the request, insert it into the ack queue. */ |
|
next = qp->r_head_ack_queue + 1; |
|
if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) |
|
next = 0; |
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
if (unlikely(next == qp->s_acked_ack_queue)) { |
|
if (!qp->s_ack_queue[next].sent) |
|
goto nack_inv_unlock; |
|
update_ack_queue(qp, next); |
|
} |
|
e = &qp->s_ack_queue[qp->r_head_ack_queue]; |
|
req = ack_to_tid_req(e); |
|
|
|
/* Bring previously RNR NAK'd request back to life */ |
|
if (qpriv->rnr_nak_state) { |
|
qp->r_nak_state = 0; |
|
qp->s_nak_state = 0; |
|
qpriv->rnr_nak_state = TID_RNR_NAK_INIT; |
|
qp->r_psn = e->lpsn + 1; |
|
req->state = TID_REQUEST_INIT; |
|
goto update_head; |
|
} |
|
|
|
release_rdma_sge_mr(e); |
|
|
|
/* The length needs to be in multiples of PAGE_SIZE */ |
|
if (!len || len & ~PAGE_MASK) |
|
goto nack_inv_unlock; |
|
|
|
rkey = be32_to_cpu(reth->rkey); |
|
qp->r_len = len; |
|
|
|
if (e->opcode == TID_OP(WRITE_REQ) && |
|
(req->setup_head != req->clear_tail || |
|
req->clear_tail != req->acked_tail)) |
|
goto nack_inv_unlock; |
|
|
|
if (unlikely(!rvt_rkey_ok(qp, &e->rdma_sge, qp->r_len, vaddr, |
|
rkey, IB_ACCESS_REMOTE_WRITE))) |
|
goto nack_acc; |
|
|
|
qp->r_psn += num_segs - 1; |
|
|
|
e->opcode = (bth0 >> 24) & 0xff; |
|
e->psn = psn; |
|
e->lpsn = qp->r_psn; |
|
e->sent = 0; |
|
|
|
req->n_flows = min_t(u16, num_segs, qpriv->tid_rdma.local.max_write); |
|
req->state = TID_REQUEST_INIT; |
|
req->cur_seg = 0; |
|
req->comp_seg = 0; |
|
req->ack_seg = 0; |
|
req->alloc_seg = 0; |
|
req->isge = 0; |
|
req->seg_len = qpriv->tid_rdma.local.max_len; |
|
req->total_len = len; |
|
req->total_segs = num_segs; |
|
req->r_flow_psn = e->psn; |
|
req->ss.sge = e->rdma_sge; |
|
req->ss.num_sge = 1; |
|
|
|
req->flow_idx = req->setup_head; |
|
req->clear_tail = req->setup_head; |
|
req->acked_tail = req->setup_head; |
|
|
|
qp->r_state = e->opcode; |
|
qp->r_nak_state = 0; |
|
/* |
|
* We need to increment the MSN here instead of when we |
|
* finish sending the result since a duplicate request would |
|
* increment it more than once. |
|
*/ |
|
qp->r_msn++; |
|
qp->r_psn++; |
|
|
|
trace_hfi1_tid_req_rcv_write_req(qp, 0, e->opcode, e->psn, e->lpsn, |
|
req); |
|
|
|
if (qpriv->r_tid_tail == HFI1_QP_WQE_INVALID) { |
|
qpriv->r_tid_tail = qp->r_head_ack_queue; |
|
} else if (qpriv->r_tid_tail == qpriv->r_tid_head) { |
|
struct tid_rdma_request *ptr; |
|
|
|
e = &qp->s_ack_queue[qpriv->r_tid_tail]; |
|
ptr = ack_to_tid_req(e); |
|
|
|
if (e->opcode != TID_OP(WRITE_REQ) || |
|
ptr->comp_seg == ptr->total_segs) { |
|
if (qpriv->r_tid_tail == qpriv->r_tid_ack) |
|
qpriv->r_tid_ack = qp->r_head_ack_queue; |
|
qpriv->r_tid_tail = qp->r_head_ack_queue; |
|
} |
|
} |
|
update_head: |
|
qp->r_head_ack_queue = next; |
|
qpriv->r_tid_head = qp->r_head_ack_queue; |
|
|
|
hfi1_tid_write_alloc_resources(qp, true); |
|
trace_hfi1_tid_write_rsp_rcv_req(qp); |
|
|
|
/* Schedule the send tasklet. */ |
|
qp->s_flags |= RVT_S_RESP_PENDING; |
|
if (fecn) |
|
qp->s_flags |= RVT_S_ECN; |
|
hfi1_schedule_send(qp); |
|
|
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
return; |
|
|
|
nack_inv_unlock: |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
nack_inv: |
|
rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); |
|
qp->r_nak_state = IB_NAK_INVALID_REQUEST; |
|
qp->r_ack_psn = qp->r_psn; |
|
/* Queue NAK for later */ |
|
rc_defered_ack(rcd, qp); |
|
return; |
|
nack_acc: |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); |
|
qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; |
|
qp->r_ack_psn = qp->r_psn; |
|
} |
|
|
|
u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, |
|
struct ib_other_headers *ohdr, u32 *bth1, |
|
u32 bth2, u32 *len, |
|
struct rvt_sge_state **ss) |
|
{ |
|
struct hfi1_ack_priv *epriv = e->priv; |
|
struct tid_rdma_request *req = &epriv->tid_req; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_rdma_flow *flow = NULL; |
|
u32 resp_len = 0, hdwords = 0; |
|
void *resp_addr = NULL; |
|
struct tid_rdma_params *remote; |
|
|
|
trace_hfi1_tid_req_build_write_resp(qp, 0, e->opcode, e->psn, e->lpsn, |
|
req); |
|
trace_hfi1_tid_write_rsp_build_resp(qp); |
|
trace_hfi1_rsp_build_tid_write_resp(qp, bth2); |
|
flow = &req->flows[req->flow_idx]; |
|
switch (req->state) { |
|
default: |
|
/* |
|
* Try to allocate resources here in case QP was queued and was |
|
* later scheduled when resources became available |
|
*/ |
|
hfi1_tid_write_alloc_resources(qp, false); |
|
|
|
/* We've already sent everything which is ready */ |
|
if (req->cur_seg >= req->alloc_seg) |
|
goto done; |
|
|
|
/* |
|
* Resources can be assigned but responses cannot be sent in |
|
* rnr_nak state, till the resent request is received |
|
*/ |
|
if (qpriv->rnr_nak_state == TID_RNR_NAK_SENT) |
|
goto done; |
|
|
|
req->state = TID_REQUEST_ACTIVE; |
|
trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow); |
|
req->flow_idx = CIRC_NEXT(req->flow_idx, MAX_FLOWS); |
|
hfi1_add_tid_reap_timer(qp); |
|
break; |
|
|
|
case TID_REQUEST_RESEND_ACTIVE: |
|
case TID_REQUEST_RESEND: |
|
trace_hfi1_tid_flow_build_write_resp(qp, req->flow_idx, flow); |
|
req->flow_idx = CIRC_NEXT(req->flow_idx, MAX_FLOWS); |
|
if (!CIRC_CNT(req->setup_head, req->flow_idx, MAX_FLOWS)) |
|
req->state = TID_REQUEST_ACTIVE; |
|
|
|
hfi1_mod_tid_reap_timer(qp); |
|
break; |
|
} |
|
flow->flow_state.resp_ib_psn = bth2; |
|
resp_addr = (void *)flow->tid_entry; |
|
resp_len = sizeof(*flow->tid_entry) * flow->tidcnt; |
|
req->cur_seg++; |
|
|
|
memset(&ohdr->u.tid_rdma.w_rsp, 0, sizeof(ohdr->u.tid_rdma.w_rsp)); |
|
epriv->ss.sge.vaddr = resp_addr; |
|
epriv->ss.sge.sge_length = resp_len; |
|
epriv->ss.sge.length = epriv->ss.sge.sge_length; |
|
/* |
|
* We can safely zero these out. Since the first SGE covers the |
|
* entire packet, nothing else should even look at the MR. |
|
*/ |
|
epriv->ss.sge.mr = NULL; |
|
epriv->ss.sge.m = 0; |
|
epriv->ss.sge.n = 0; |
|
|
|
epriv->ss.sg_list = NULL; |
|
epriv->ss.total_len = epriv->ss.sge.sge_length; |
|
epriv->ss.num_sge = 1; |
|
|
|
*ss = &epriv->ss; |
|
*len = epriv->ss.total_len; |
|
|
|
/* Construct the TID RDMA WRITE RESP packet header */ |
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
|
|
KDETH_RESET(ohdr->u.tid_rdma.w_rsp.kdeth0, KVER, 0x1); |
|
KDETH_RESET(ohdr->u.tid_rdma.w_rsp.kdeth1, JKEY, remote->jkey); |
|
ohdr->u.tid_rdma.w_rsp.aeth = rvt_compute_aeth(qp); |
|
ohdr->u.tid_rdma.w_rsp.tid_flow_psn = |
|
cpu_to_be32((flow->flow_state.generation << |
|
HFI1_KDETH_BTH_SEQ_SHIFT) | |
|
(flow->flow_state.spsn & |
|
HFI1_KDETH_BTH_SEQ_MASK)); |
|
ohdr->u.tid_rdma.w_rsp.tid_flow_qp = |
|
cpu_to_be32(qpriv->tid_rdma.local.qp | |
|
((flow->idx & TID_RDMA_DESTQP_FLOW_MASK) << |
|
TID_RDMA_DESTQP_FLOW_SHIFT) | |
|
qpriv->rcd->ctxt); |
|
ohdr->u.tid_rdma.w_rsp.verbs_qp = cpu_to_be32(qp->remote_qpn); |
|
*bth1 = remote->qp; |
|
rcu_read_unlock(); |
|
hdwords = sizeof(ohdr->u.tid_rdma.w_rsp) / sizeof(u32); |
|
qpriv->pending_tid_w_segs++; |
|
done: |
|
return hdwords; |
|
} |
|
|
|
static void hfi1_add_tid_reap_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
if (!(qpriv->s_flags & HFI1_R_TID_RSC_TIMER)) { |
|
qpriv->s_flags |= HFI1_R_TID_RSC_TIMER; |
|
qpriv->s_tid_timer.expires = jiffies + |
|
qpriv->tid_timer_timeout_jiffies; |
|
add_timer(&qpriv->s_tid_timer); |
|
} |
|
} |
|
|
|
static void hfi1_mod_tid_reap_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
qpriv->s_flags |= HFI1_R_TID_RSC_TIMER; |
|
mod_timer(&qpriv->s_tid_timer, jiffies + |
|
qpriv->tid_timer_timeout_jiffies); |
|
} |
|
|
|
static int hfi1_stop_tid_reap_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
int rval = 0; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
if (qpriv->s_flags & HFI1_R_TID_RSC_TIMER) { |
|
rval = del_timer(&qpriv->s_tid_timer); |
|
qpriv->s_flags &= ~HFI1_R_TID_RSC_TIMER; |
|
} |
|
return rval; |
|
} |
|
|
|
void hfi1_del_tid_reap_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
|
|
del_timer_sync(&qpriv->s_tid_timer); |
|
qpriv->s_flags &= ~HFI1_R_TID_RSC_TIMER; |
|
} |
|
|
|
static void hfi1_tid_timeout(struct timer_list *t) |
|
{ |
|
struct hfi1_qp_priv *qpriv = from_timer(qpriv, t, s_tid_timer); |
|
struct rvt_qp *qp = qpriv->owner; |
|
struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device); |
|
unsigned long flags; |
|
u32 i; |
|
|
|
spin_lock_irqsave(&qp->r_lock, flags); |
|
spin_lock(&qp->s_lock); |
|
if (qpriv->s_flags & HFI1_R_TID_RSC_TIMER) { |
|
dd_dev_warn(dd_from_ibdev(qp->ibqp.device), "[QP%u] %s %d\n", |
|
qp->ibqp.qp_num, __func__, __LINE__); |
|
trace_hfi1_msg_tid_timeout(/* msg */ |
|
qp, "resource timeout = ", |
|
(u64)qpriv->tid_timer_timeout_jiffies); |
|
hfi1_stop_tid_reap_timer(qp); |
|
/* |
|
* Go though the entire ack queue and clear any outstanding |
|
* HW flow and RcvArray resources. |
|
*/ |
|
hfi1_kern_clear_hw_flow(qpriv->rcd, qp); |
|
for (i = 0; i < rvt_max_atomic(rdi); i++) { |
|
struct tid_rdma_request *req = |
|
ack_to_tid_req(&qp->s_ack_queue[i]); |
|
|
|
hfi1_kern_exp_rcv_clear_all(req); |
|
} |
|
spin_unlock(&qp->s_lock); |
|
if (qp->ibqp.event_handler) { |
|
struct ib_event ev; |
|
|
|
ev.device = qp->ibqp.device; |
|
ev.element.qp = &qp->ibqp; |
|
ev.event = IB_EVENT_QP_FATAL; |
|
qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); |
|
} |
|
rvt_rc_error(qp, IB_WC_RESP_TIMEOUT_ERR); |
|
goto unlock_r_lock; |
|
} |
|
spin_unlock(&qp->s_lock); |
|
unlock_r_lock: |
|
spin_unlock_irqrestore(&qp->r_lock, flags); |
|
} |
|
|
|
void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet) |
|
{ |
|
/* HANDLER FOR TID RDMA WRITE RESPONSE packet (Requestor side */ |
|
|
|
/* |
|
* 1. Find matching SWQE |
|
* 2. Check that TIDENTRY array has enough space for a complete |
|
* segment. If not, put QP in error state. |
|
* 3. Save response data in struct tid_rdma_req and struct tid_rdma_flow |
|
* 4. Remove HFI1_S_WAIT_TID_RESP from s_flags. |
|
* 5. Set qp->s_state |
|
* 6. Kick the send engine (hfi1_schedule_send()) |
|
*/ |
|
struct ib_other_headers *ohdr = packet->ohdr; |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct hfi1_ctxtdata *rcd = packet->rcd; |
|
struct rvt_swqe *wqe; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
enum ib_wc_status status; |
|
u32 opcode, aeth, psn, flow_psn, i, tidlen = 0, pktlen; |
|
bool fecn; |
|
unsigned long flags; |
|
|
|
fecn = process_ecn(qp, packet); |
|
psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
aeth = be32_to_cpu(ohdr->u.tid_rdma.w_rsp.aeth); |
|
opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff; |
|
|
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
|
|
/* Ignore invalid responses */ |
|
if (cmp_psn(psn, qp->s_next_psn) >= 0) |
|
goto ack_done; |
|
|
|
/* Ignore duplicate responses. */ |
|
if (unlikely(cmp_psn(psn, qp->s_last_psn) <= 0)) |
|
goto ack_done; |
|
|
|
if (unlikely(qp->s_acked == qp->s_tail)) |
|
goto ack_done; |
|
|
|
/* |
|
* If we are waiting for a particular packet sequence number |
|
* due to a request being resent, check for it. Otherwise, |
|
* ensure that we haven't missed anything. |
|
*/ |
|
if (qp->r_flags & RVT_R_RDMAR_SEQ) { |
|
if (cmp_psn(psn, qp->s_last_psn + 1) != 0) |
|
goto ack_done; |
|
qp->r_flags &= ~RVT_R_RDMAR_SEQ; |
|
} |
|
|
|
wqe = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); |
|
if (unlikely(wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)) |
|
goto ack_op_err; |
|
|
|
req = wqe_to_tid_req(wqe); |
|
/* |
|
* If we've lost ACKs and our acked_tail pointer is too far |
|
* behind, don't overwrite segments. Just drop the packet and |
|
* let the reliability protocol take care of it. |
|
*/ |
|
if (!CIRC_SPACE(req->setup_head, req->acked_tail, MAX_FLOWS)) |
|
goto ack_done; |
|
|
|
/* |
|
* The call to do_rc_ack() should be last in the chain of |
|
* packet checks because it will end up updating the QP state. |
|
* Therefore, anything that would prevent the packet from |
|
* being accepted as a successful response should be prior |
|
* to it. |
|
*/ |
|
if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) |
|
goto ack_done; |
|
|
|
trace_hfi1_ack(qp, psn); |
|
|
|
flow = &req->flows[req->setup_head]; |
|
flow->pkt = 0; |
|
flow->tid_idx = 0; |
|
flow->tid_offset = 0; |
|
flow->sent = 0; |
|
flow->resync_npkts = 0; |
|
flow->tid_qpn = be32_to_cpu(ohdr->u.tid_rdma.w_rsp.tid_flow_qp); |
|
flow->idx = (flow->tid_qpn >> TID_RDMA_DESTQP_FLOW_SHIFT) & |
|
TID_RDMA_DESTQP_FLOW_MASK; |
|
flow_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.w_rsp.tid_flow_psn)); |
|
flow->flow_state.generation = flow_psn >> HFI1_KDETH_BTH_SEQ_SHIFT; |
|
flow->flow_state.spsn = flow_psn & HFI1_KDETH_BTH_SEQ_MASK; |
|
flow->flow_state.resp_ib_psn = psn; |
|
flow->length = min_t(u32, req->seg_len, |
|
(wqe->length - (req->comp_seg * req->seg_len))); |
|
|
|
flow->npkts = rvt_div_round_up_mtu(qp, flow->length); |
|
flow->flow_state.lpsn = flow->flow_state.spsn + |
|
flow->npkts - 1; |
|
/* payload length = packet length - (header length + ICRC length) */ |
|
pktlen = packet->tlen - (packet->hlen + 4); |
|
if (pktlen > sizeof(flow->tid_entry)) { |
|
status = IB_WC_LOC_LEN_ERR; |
|
goto ack_err; |
|
} |
|
memcpy(flow->tid_entry, packet->ebuf, pktlen); |
|
flow->tidcnt = pktlen / sizeof(*flow->tid_entry); |
|
trace_hfi1_tid_flow_rcv_write_resp(qp, req->setup_head, flow); |
|
|
|
req->comp_seg++; |
|
trace_hfi1_tid_write_sender_rcv_resp(qp, 0); |
|
/* |
|
* Walk the TID_ENTRY list to make sure we have enough space for a |
|
* complete segment. |
|
*/ |
|
for (i = 0; i < flow->tidcnt; i++) { |
|
trace_hfi1_tid_entry_rcv_write_resp(/* entry */ |
|
qp, i, flow->tid_entry[i]); |
|
if (!EXP_TID_GET(flow->tid_entry[i], LEN)) { |
|
status = IB_WC_LOC_LEN_ERR; |
|
goto ack_err; |
|
} |
|
tidlen += EXP_TID_GET(flow->tid_entry[i], LEN); |
|
} |
|
if (tidlen * PAGE_SIZE < flow->length) { |
|
status = IB_WC_LOC_LEN_ERR; |
|
goto ack_err; |
|
} |
|
|
|
trace_hfi1_tid_req_rcv_write_resp(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
/* |
|
* If this is the first response for this request, set the initial |
|
* flow index to the current flow. |
|
*/ |
|
if (!cmp_psn(psn, wqe->psn)) { |
|
req->r_last_acked = mask_psn(wqe->psn - 1); |
|
/* Set acked flow index to head index */ |
|
req->acked_tail = req->setup_head; |
|
} |
|
|
|
/* advance circular buffer head */ |
|
req->setup_head = CIRC_NEXT(req->setup_head, MAX_FLOWS); |
|
req->state = TID_REQUEST_ACTIVE; |
|
|
|
/* |
|
* If all responses for this TID RDMA WRITE request have been received |
|
* advance the pointer to the next one. |
|
* Since TID RDMA requests could be mixed in with regular IB requests, |
|
* they might not appear sequentially in the queue. Therefore, the |
|
* next request needs to be "found". |
|
*/ |
|
if (qpriv->s_tid_cur != qpriv->s_tid_head && |
|
req->comp_seg == req->total_segs) { |
|
for (i = qpriv->s_tid_cur + 1; ; i++) { |
|
if (i == qp->s_size) |
|
i = 0; |
|
wqe = rvt_get_swqe_ptr(qp, i); |
|
if (i == qpriv->s_tid_head) |
|
break; |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) |
|
break; |
|
} |
|
qpriv->s_tid_cur = i; |
|
} |
|
qp->s_flags &= ~HFI1_S_WAIT_TID_RESP; |
|
hfi1_schedule_tid_send(qp); |
|
goto ack_done; |
|
|
|
ack_op_err: |
|
status = IB_WC_LOC_QP_OP_ERR; |
|
ack_err: |
|
rvt_error_qp(qp, status); |
|
ack_done: |
|
if (fecn) |
|
qp->s_flags |= RVT_S_ECN; |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
} |
|
|
|
bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe, |
|
struct ib_other_headers *ohdr, |
|
u32 *bth1, u32 *bth2, u32 *len) |
|
{ |
|
struct tid_rdma_request *req = wqe_to_tid_req(wqe); |
|
struct tid_rdma_flow *flow = &req->flows[req->clear_tail]; |
|
struct tid_rdma_params *remote; |
|
struct rvt_qp *qp = req->qp; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
u32 tidentry = flow->tid_entry[flow->tid_idx]; |
|
u32 tidlen = EXP_TID_GET(tidentry, LEN) << PAGE_SHIFT; |
|
struct tid_rdma_write_data *wd = &ohdr->u.tid_rdma.w_data; |
|
u32 next_offset, om = KDETH_OM_LARGE; |
|
bool last_pkt; |
|
|
|
if (!tidlen) { |
|
hfi1_trdma_send_complete(qp, wqe, IB_WC_REM_INV_RD_REQ_ERR); |
|
rvt_error_qp(qp, IB_WC_REM_INV_RD_REQ_ERR); |
|
} |
|
|
|
*len = min_t(u32, qp->pmtu, tidlen - flow->tid_offset); |
|
flow->sent += *len; |
|
next_offset = flow->tid_offset + *len; |
|
last_pkt = (flow->tid_idx == (flow->tidcnt - 1) && |
|
next_offset >= tidlen) || (flow->sent >= flow->length); |
|
trace_hfi1_tid_entry_build_write_data(qp, flow->tid_idx, tidentry); |
|
trace_hfi1_tid_flow_build_write_data(qp, req->clear_tail, flow); |
|
|
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
KDETH_RESET(wd->kdeth0, KVER, 0x1); |
|
KDETH_SET(wd->kdeth0, SH, !last_pkt); |
|
KDETH_SET(wd->kdeth0, INTR, !!(!last_pkt && remote->urg)); |
|
KDETH_SET(wd->kdeth0, TIDCTRL, EXP_TID_GET(tidentry, CTRL)); |
|
KDETH_SET(wd->kdeth0, TID, EXP_TID_GET(tidentry, IDX)); |
|
KDETH_SET(wd->kdeth0, OM, om == KDETH_OM_LARGE); |
|
KDETH_SET(wd->kdeth0, OFFSET, flow->tid_offset / om); |
|
KDETH_RESET(wd->kdeth1, JKEY, remote->jkey); |
|
wd->verbs_qp = cpu_to_be32(qp->remote_qpn); |
|
rcu_read_unlock(); |
|
|
|
*bth1 = flow->tid_qpn; |
|
*bth2 = mask_psn(((flow->flow_state.spsn + flow->pkt++) & |
|
HFI1_KDETH_BTH_SEQ_MASK) | |
|
(flow->flow_state.generation << |
|
HFI1_KDETH_BTH_SEQ_SHIFT)); |
|
if (last_pkt) { |
|
/* PSNs are zero-based, so +1 to count number of packets */ |
|
if (flow->flow_state.lpsn + 1 + |
|
rvt_div_round_up_mtu(qp, req->seg_len) > |
|
MAX_TID_FLOW_PSN) |
|
req->state = TID_REQUEST_SYNC; |
|
*bth2 |= IB_BTH_REQ_ACK; |
|
} |
|
|
|
if (next_offset >= tidlen) { |
|
flow->tid_offset = 0; |
|
flow->tid_idx++; |
|
} else { |
|
flow->tid_offset = next_offset; |
|
} |
|
return last_pkt; |
|
} |
|
|
|
void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet) |
|
{ |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct hfi1_ctxtdata *rcd = priv->rcd; |
|
struct ib_other_headers *ohdr = packet->ohdr; |
|
struct rvt_ack_entry *e; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
|
unsigned long flags; |
|
u32 psn, next; |
|
u8 opcode; |
|
bool fecn; |
|
|
|
fecn = process_ecn(qp, packet); |
|
psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff; |
|
|
|
/* |
|
* All error handling should be done by now. If we are here, the packet |
|
* is either good or been accepted by the error handler. |
|
*/ |
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
e = &qp->s_ack_queue[priv->r_tid_tail]; |
|
req = ack_to_tid_req(e); |
|
flow = &req->flows[req->clear_tail]; |
|
if (cmp_psn(psn, full_flow_psn(flow, flow->flow_state.lpsn))) { |
|
update_r_next_psn_fecn(packet, priv, rcd, flow, fecn); |
|
|
|
if (cmp_psn(psn, flow->flow_state.r_next_psn)) |
|
goto send_nak; |
|
|
|
flow->flow_state.r_next_psn = mask_psn(psn + 1); |
|
/* |
|
* Copy the payload to destination buffer if this packet is |
|
* delivered as an eager packet due to RSM rule and FECN. |
|
* The RSM rule selects FECN bit in BTH and SH bit in |
|
* KDETH header and therefore will not match the last |
|
* packet of each segment that has SH bit cleared. |
|
*/ |
|
if (fecn && packet->etype == RHF_RCV_TYPE_EAGER) { |
|
struct rvt_sge_state ss; |
|
u32 len; |
|
u32 tlen = packet->tlen; |
|
u16 hdrsize = packet->hlen; |
|
u8 pad = packet->pad; |
|
u8 extra_bytes = pad + packet->extra_byte + |
|
(SIZE_OF_CRC << 2); |
|
u32 pmtu = qp->pmtu; |
|
|
|
if (unlikely(tlen != (hdrsize + pmtu + extra_bytes))) |
|
goto send_nak; |
|
len = req->comp_seg * req->seg_len; |
|
len += delta_psn(psn, |
|
full_flow_psn(flow, flow->flow_state.spsn)) * |
|
pmtu; |
|
if (unlikely(req->total_len - len < pmtu)) |
|
goto send_nak; |
|
|
|
/* |
|
* The e->rdma_sge field is set when TID RDMA WRITE REQ |
|
* is first received and is never modified thereafter. |
|
*/ |
|
ss.sge = e->rdma_sge; |
|
ss.sg_list = NULL; |
|
ss.num_sge = 1; |
|
ss.total_len = req->total_len; |
|
rvt_skip_sge(&ss, len, false); |
|
rvt_copy_sge(qp, &ss, packet->payload, pmtu, false, |
|
false); |
|
/* Raise the sw sequence check flag for next packet */ |
|
priv->r_next_psn_kdeth = mask_psn(psn + 1); |
|
priv->s_flags |= HFI1_R_TID_SW_PSN; |
|
} |
|
goto exit; |
|
} |
|
flow->flow_state.r_next_psn = mask_psn(psn + 1); |
|
hfi1_kern_exp_rcv_clear(req); |
|
priv->alloc_w_segs--; |
|
rcd->flows[flow->idx].psn = psn & HFI1_KDETH_BTH_SEQ_MASK; |
|
req->comp_seg++; |
|
priv->s_nak_state = 0; |
|
|
|
/* |
|
* Release the flow if one of the following conditions has been met: |
|
* - The request has reached a sync point AND all outstanding |
|
* segments have been completed, or |
|
* - The entire request is complete and there are no more requests |
|
* (of any kind) in the queue. |
|
*/ |
|
trace_hfi1_rsp_rcv_tid_write_data(qp, psn); |
|
trace_hfi1_tid_req_rcv_write_data(qp, 0, e->opcode, e->psn, e->lpsn, |
|
req); |
|
trace_hfi1_tid_write_rsp_rcv_data(qp); |
|
validate_r_tid_ack(priv); |
|
|
|
if (opcode == TID_OP(WRITE_DATA_LAST)) { |
|
release_rdma_sge_mr(e); |
|
for (next = priv->r_tid_tail + 1; ; next++) { |
|
if (next > rvt_size_atomic(&dev->rdi)) |
|
next = 0; |
|
if (next == priv->r_tid_head) |
|
break; |
|
e = &qp->s_ack_queue[next]; |
|
if (e->opcode == TID_OP(WRITE_REQ)) |
|
break; |
|
} |
|
priv->r_tid_tail = next; |
|
if (++qp->s_acked_ack_queue > rvt_size_atomic(&dev->rdi)) |
|
qp->s_acked_ack_queue = 0; |
|
} |
|
|
|
hfi1_tid_write_alloc_resources(qp, true); |
|
|
|
/* |
|
* If we need to generate more responses, schedule the |
|
* send engine. |
|
*/ |
|
if (req->cur_seg < req->total_segs || |
|
qp->s_tail_ack_queue != qp->r_head_ack_queue) { |
|
qp->s_flags |= RVT_S_RESP_PENDING; |
|
hfi1_schedule_send(qp); |
|
} |
|
|
|
priv->pending_tid_w_segs--; |
|
if (priv->s_flags & HFI1_R_TID_RSC_TIMER) { |
|
if (priv->pending_tid_w_segs) |
|
hfi1_mod_tid_reap_timer(req->qp); |
|
else |
|
hfi1_stop_tid_reap_timer(req->qp); |
|
} |
|
|
|
done: |
|
tid_rdma_schedule_ack(qp); |
|
exit: |
|
priv->r_next_psn_kdeth = flow->flow_state.r_next_psn; |
|
if (fecn) |
|
qp->s_flags |= RVT_S_ECN; |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
return; |
|
|
|
send_nak: |
|
if (!priv->s_nak_state) { |
|
priv->s_nak_state = IB_NAK_PSN_ERROR; |
|
priv->s_nak_psn = flow->flow_state.r_next_psn; |
|
tid_rdma_trigger_ack(qp); |
|
} |
|
goto done; |
|
} |
|
|
|
static bool hfi1_tid_rdma_is_resync_psn(u32 psn) |
|
{ |
|
return (bool)((psn & HFI1_KDETH_BTH_SEQ_MASK) == |
|
HFI1_KDETH_BTH_SEQ_MASK); |
|
} |
|
|
|
u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e, |
|
struct ib_other_headers *ohdr, u16 iflow, |
|
u32 *bth1, u32 *bth2) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_flow_state *fs = &qpriv->flow_state; |
|
struct tid_rdma_request *req = ack_to_tid_req(e); |
|
struct tid_rdma_flow *flow = &req->flows[iflow]; |
|
struct tid_rdma_params *remote; |
|
|
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth1, JKEY, remote->jkey); |
|
ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn); |
|
*bth1 = remote->qp; |
|
rcu_read_unlock(); |
|
|
|
if (qpriv->resync) { |
|
*bth2 = mask_psn((fs->generation << |
|
HFI1_KDETH_BTH_SEQ_SHIFT) - 1); |
|
ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp); |
|
} else if (qpriv->s_nak_state) { |
|
*bth2 = mask_psn(qpriv->s_nak_psn); |
|
ohdr->u.tid_rdma.ack.aeth = |
|
cpu_to_be32((qp->r_msn & IB_MSN_MASK) | |
|
(qpriv->s_nak_state << |
|
IB_AETH_CREDIT_SHIFT)); |
|
} else { |
|
*bth2 = full_flow_psn(flow, flow->flow_state.lpsn); |
|
ohdr->u.tid_rdma.ack.aeth = rvt_compute_aeth(qp); |
|
} |
|
KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth0, KVER, 0x1); |
|
ohdr->u.tid_rdma.ack.tid_flow_qp = |
|
cpu_to_be32(qpriv->tid_rdma.local.qp | |
|
((flow->idx & TID_RDMA_DESTQP_FLOW_MASK) << |
|
TID_RDMA_DESTQP_FLOW_SHIFT) | |
|
qpriv->rcd->ctxt); |
|
|
|
ohdr->u.tid_rdma.ack.tid_flow_psn = 0; |
|
ohdr->u.tid_rdma.ack.verbs_psn = |
|
cpu_to_be32(flow->flow_state.resp_ib_psn); |
|
|
|
if (qpriv->resync) { |
|
/* |
|
* If the PSN before the current expect KDETH PSN is the |
|
* RESYNC PSN, then we never received a good TID RDMA WRITE |
|
* DATA packet after a previous RESYNC. |
|
* In this case, the next expected KDETH PSN stays the same. |
|
*/ |
|
if (hfi1_tid_rdma_is_resync_psn(qpriv->r_next_psn_kdeth - 1)) { |
|
ohdr->u.tid_rdma.ack.tid_flow_psn = |
|
cpu_to_be32(qpriv->r_next_psn_kdeth_save); |
|
} else { |
|
/* |
|
* Because the KDETH PSNs jump during a RESYNC, it's |
|
* not possible to infer (or compute) the previous value |
|
* of r_next_psn_kdeth in the case of back-to-back |
|
* RESYNC packets. Therefore, we save it. |
|
*/ |
|
qpriv->r_next_psn_kdeth_save = |
|
qpriv->r_next_psn_kdeth - 1; |
|
ohdr->u.tid_rdma.ack.tid_flow_psn = |
|
cpu_to_be32(qpriv->r_next_psn_kdeth_save); |
|
qpriv->r_next_psn_kdeth = mask_psn(*bth2 + 1); |
|
} |
|
qpriv->resync = false; |
|
} |
|
|
|
return sizeof(ohdr->u.tid_rdma.ack) / sizeof(u32); |
|
} |
|
|
|
void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet) |
|
{ |
|
struct ib_other_headers *ohdr = packet->ohdr; |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct rvt_swqe *wqe; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
u32 aeth, psn, req_psn, ack_psn, flpsn, resync_psn, ack_kpsn; |
|
unsigned long flags; |
|
u16 fidx; |
|
|
|
trace_hfi1_tid_write_sender_rcv_tid_ack(qp, 0); |
|
process_ecn(qp, packet); |
|
psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
aeth = be32_to_cpu(ohdr->u.tid_rdma.ack.aeth); |
|
req_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.ack.verbs_psn)); |
|
resync_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.ack.tid_flow_psn)); |
|
|
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
trace_hfi1_rcv_tid_ack(qp, aeth, psn, req_psn, resync_psn); |
|
|
|
/* If we are waiting for an ACK to RESYNC, drop any other packets */ |
|
if ((qp->s_flags & HFI1_S_WAIT_HALT) && |
|
cmp_psn(psn, qpriv->s_resync_psn)) |
|
goto ack_op_err; |
|
|
|
ack_psn = req_psn; |
|
if (hfi1_tid_rdma_is_resync_psn(psn)) |
|
ack_kpsn = resync_psn; |
|
else |
|
ack_kpsn = psn; |
|
if (aeth >> 29) { |
|
ack_psn--; |
|
ack_kpsn--; |
|
} |
|
|
|
if (unlikely(qp->s_acked == qp->s_tail)) |
|
goto ack_op_err; |
|
|
|
wqe = rvt_get_swqe_ptr(qp, qp->s_acked); |
|
|
|
if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) |
|
goto ack_op_err; |
|
|
|
req = wqe_to_tid_req(wqe); |
|
trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
flow = &req->flows[req->acked_tail]; |
|
trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); |
|
|
|
/* Drop stale ACK/NAK */ |
|
if (cmp_psn(psn, full_flow_psn(flow, flow->flow_state.spsn)) < 0 || |
|
cmp_psn(req_psn, flow->flow_state.resp_ib_psn) < 0) |
|
goto ack_op_err; |
|
|
|
while (cmp_psn(ack_kpsn, |
|
full_flow_psn(flow, flow->flow_state.lpsn)) >= 0 && |
|
req->ack_seg < req->cur_seg) { |
|
req->ack_seg++; |
|
/* advance acked segment pointer */ |
|
req->acked_tail = CIRC_NEXT(req->acked_tail, MAX_FLOWS); |
|
req->r_last_acked = flow->flow_state.resp_ib_psn; |
|
trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
if (req->ack_seg == req->total_segs) { |
|
req->state = TID_REQUEST_COMPLETE; |
|
wqe = do_rc_completion(qp, wqe, |
|
to_iport(qp->ibqp.device, |
|
qp->port_num)); |
|
trace_hfi1_sender_rcv_tid_ack(qp); |
|
atomic_dec(&qpriv->n_tid_requests); |
|
if (qp->s_acked == qp->s_tail) |
|
break; |
|
if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) |
|
break; |
|
req = wqe_to_tid_req(wqe); |
|
} |
|
flow = &req->flows[req->acked_tail]; |
|
trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); |
|
} |
|
|
|
trace_hfi1_tid_req_rcv_tid_ack(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
switch (aeth >> 29) { |
|
case 0: /* ACK */ |
|
if (qpriv->s_flags & RVT_S_WAIT_ACK) |
|
qpriv->s_flags &= ~RVT_S_WAIT_ACK; |
|
if (!hfi1_tid_rdma_is_resync_psn(psn)) { |
|
/* Check if there is any pending TID ACK */ |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && |
|
req->ack_seg < req->cur_seg) |
|
hfi1_mod_tid_retry_timer(qp); |
|
else |
|
hfi1_stop_tid_retry_timer(qp); |
|
hfi1_schedule_send(qp); |
|
} else { |
|
u32 spsn, fpsn, last_acked, generation; |
|
struct tid_rdma_request *rptr; |
|
|
|
/* ACK(RESYNC) */ |
|
hfi1_stop_tid_retry_timer(qp); |
|
/* Allow new requests (see hfi1_make_tid_rdma_pkt) */ |
|
qp->s_flags &= ~HFI1_S_WAIT_HALT; |
|
/* |
|
* Clear RVT_S_SEND_ONE flag in case that the TID RDMA |
|
* ACK is received after the TID retry timer is fired |
|
* again. In this case, do not send any more TID |
|
* RESYNC request or wait for any more TID ACK packet. |
|
*/ |
|
qpriv->s_flags &= ~RVT_S_SEND_ONE; |
|
hfi1_schedule_send(qp); |
|
|
|
if ((qp->s_acked == qpriv->s_tid_tail && |
|
req->ack_seg == req->total_segs) || |
|
qp->s_acked == qp->s_tail) { |
|
qpriv->s_state = TID_OP(WRITE_DATA_LAST); |
|
goto done; |
|
} |
|
|
|
if (req->ack_seg == req->comp_seg) { |
|
qpriv->s_state = TID_OP(WRITE_DATA); |
|
goto done; |
|
} |
|
|
|
/* |
|
* The PSN to start with is the next PSN after the |
|
* RESYNC PSN. |
|
*/ |
|
psn = mask_psn(psn + 1); |
|
generation = psn >> HFI1_KDETH_BTH_SEQ_SHIFT; |
|
spsn = 0; |
|
|
|
/* |
|
* Update to the correct WQE when we get an ACK(RESYNC) |
|
* in the middle of a request. |
|
*/ |
|
if (delta_psn(ack_psn, wqe->lpsn)) |
|
wqe = rvt_get_swqe_ptr(qp, qp->s_acked); |
|
req = wqe_to_tid_req(wqe); |
|
flow = &req->flows[req->acked_tail]; |
|
/* |
|
* RESYNC re-numbers the PSN ranges of all remaining |
|
* segments. Also, PSN's start from 0 in the middle of a |
|
* segment and the first segment size is less than the |
|
* default number of packets. flow->resync_npkts is used |
|
* to track the number of packets from the start of the |
|
* real segment to the point of 0 PSN after the RESYNC |
|
* in order to later correctly rewind the SGE. |
|
*/ |
|
fpsn = full_flow_psn(flow, flow->flow_state.spsn); |
|
req->r_ack_psn = psn; |
|
/* |
|
* If resync_psn points to the last flow PSN for a |
|
* segment and the new segment (likely from a new |
|
* request) starts with a new generation number, we |
|
* need to adjust resync_psn accordingly. |
|
*/ |
|
if (flow->flow_state.generation != |
|
(resync_psn >> HFI1_KDETH_BTH_SEQ_SHIFT)) |
|
resync_psn = mask_psn(fpsn - 1); |
|
flow->resync_npkts += |
|
delta_psn(mask_psn(resync_psn + 1), fpsn); |
|
/* |
|
* Renumber all packet sequence number ranges |
|
* based on the new generation. |
|
*/ |
|
last_acked = qp->s_acked; |
|
rptr = req; |
|
while (1) { |
|
/* start from last acked segment */ |
|
for (fidx = rptr->acked_tail; |
|
CIRC_CNT(rptr->setup_head, fidx, |
|
MAX_FLOWS); |
|
fidx = CIRC_NEXT(fidx, MAX_FLOWS)) { |
|
u32 lpsn; |
|
u32 gen; |
|
|
|
flow = &rptr->flows[fidx]; |
|
gen = flow->flow_state.generation; |
|
if (WARN_ON(gen == generation && |
|
flow->flow_state.spsn != |
|
spsn)) |
|
continue; |
|
lpsn = flow->flow_state.lpsn; |
|
lpsn = full_flow_psn(flow, lpsn); |
|
flow->npkts = |
|
delta_psn(lpsn, |
|
mask_psn(resync_psn) |
|
); |
|
flow->flow_state.generation = |
|
generation; |
|
flow->flow_state.spsn = spsn; |
|
flow->flow_state.lpsn = |
|
flow->flow_state.spsn + |
|
flow->npkts - 1; |
|
flow->pkt = 0; |
|
spsn += flow->npkts; |
|
resync_psn += flow->npkts; |
|
trace_hfi1_tid_flow_rcv_tid_ack(qp, |
|
fidx, |
|
flow); |
|
} |
|
if (++last_acked == qpriv->s_tid_cur + 1) |
|
break; |
|
if (last_acked == qp->s_size) |
|
last_acked = 0; |
|
wqe = rvt_get_swqe_ptr(qp, last_acked); |
|
rptr = wqe_to_tid_req(wqe); |
|
} |
|
req->cur_seg = req->ack_seg; |
|
qpriv->s_tid_tail = qp->s_acked; |
|
qpriv->s_state = TID_OP(WRITE_REQ); |
|
hfi1_schedule_tid_send(qp); |
|
} |
|
done: |
|
qpriv->s_retry = qp->s_retry_cnt; |
|
break; |
|
|
|
case 3: /* NAK */ |
|
hfi1_stop_tid_retry_timer(qp); |
|
switch ((aeth >> IB_AETH_CREDIT_SHIFT) & |
|
IB_AETH_CREDIT_MASK) { |
|
case 0: /* PSN sequence error */ |
|
if (!req->flows) |
|
break; |
|
flow = &req->flows[req->acked_tail]; |
|
flpsn = full_flow_psn(flow, flow->flow_state.lpsn); |
|
if (cmp_psn(psn, flpsn) > 0) |
|
break; |
|
trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, |
|
flow); |
|
req->r_ack_psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
req->cur_seg = req->ack_seg; |
|
qpriv->s_tid_tail = qp->s_acked; |
|
qpriv->s_state = TID_OP(WRITE_REQ); |
|
qpriv->s_retry = qp->s_retry_cnt; |
|
hfi1_schedule_tid_send(qp); |
|
break; |
|
|
|
default: |
|
break; |
|
} |
|
break; |
|
|
|
default: |
|
break; |
|
} |
|
|
|
ack_op_err: |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
} |
|
|
|
void hfi1_add_tid_retry_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct ib_qp *ibqp = &qp->ibqp; |
|
struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
if (!(priv->s_flags & HFI1_S_TID_RETRY_TIMER)) { |
|
priv->s_flags |= HFI1_S_TID_RETRY_TIMER; |
|
priv->s_tid_retry_timer.expires = jiffies + |
|
priv->tid_retry_timeout_jiffies + rdi->busy_jiffies; |
|
add_timer(&priv->s_tid_retry_timer); |
|
} |
|
} |
|
|
|
static void hfi1_mod_tid_retry_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct ib_qp *ibqp = &qp->ibqp; |
|
struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device); |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
priv->s_flags |= HFI1_S_TID_RETRY_TIMER; |
|
mod_timer(&priv->s_tid_retry_timer, jiffies + |
|
priv->tid_retry_timeout_jiffies + rdi->busy_jiffies); |
|
} |
|
|
|
static int hfi1_stop_tid_retry_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
int rval = 0; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
if (priv->s_flags & HFI1_S_TID_RETRY_TIMER) { |
|
rval = del_timer(&priv->s_tid_retry_timer); |
|
priv->s_flags &= ~HFI1_S_TID_RETRY_TIMER; |
|
} |
|
return rval; |
|
} |
|
|
|
void hfi1_del_tid_retry_timer(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
del_timer_sync(&priv->s_tid_retry_timer); |
|
priv->s_flags &= ~HFI1_S_TID_RETRY_TIMER; |
|
} |
|
|
|
static void hfi1_tid_retry_timeout(struct timer_list *t) |
|
{ |
|
struct hfi1_qp_priv *priv = from_timer(priv, t, s_tid_retry_timer); |
|
struct rvt_qp *qp = priv->owner; |
|
struct rvt_swqe *wqe; |
|
unsigned long flags; |
|
struct tid_rdma_request *req; |
|
|
|
spin_lock_irqsave(&qp->r_lock, flags); |
|
spin_lock(&qp->s_lock); |
|
trace_hfi1_tid_write_sender_retry_timeout(qp, 0); |
|
if (priv->s_flags & HFI1_S_TID_RETRY_TIMER) { |
|
hfi1_stop_tid_retry_timer(qp); |
|
if (!priv->s_retry) { |
|
trace_hfi1_msg_tid_retry_timeout(/* msg */ |
|
qp, |
|
"Exhausted retries. Tid retry timeout = ", |
|
(u64)priv->tid_retry_timeout_jiffies); |
|
|
|
wqe = rvt_get_swqe_ptr(qp, qp->s_acked); |
|
hfi1_trdma_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); |
|
rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); |
|
} else { |
|
wqe = rvt_get_swqe_ptr(qp, qp->s_acked); |
|
req = wqe_to_tid_req(wqe); |
|
trace_hfi1_tid_req_tid_retry_timeout(/* req */ |
|
qp, 0, wqe->wr.opcode, wqe->psn, wqe->lpsn, req); |
|
|
|
priv->s_flags &= ~RVT_S_WAIT_ACK; |
|
/* Only send one packet (the RESYNC) */ |
|
priv->s_flags |= RVT_S_SEND_ONE; |
|
/* |
|
* No additional request shall be made by this QP until |
|
* the RESYNC has been complete. |
|
*/ |
|
qp->s_flags |= HFI1_S_WAIT_HALT; |
|
priv->s_state = TID_OP(RESYNC); |
|
priv->s_retry--; |
|
hfi1_schedule_tid_send(qp); |
|
} |
|
} |
|
spin_unlock(&qp->s_lock); |
|
spin_unlock_irqrestore(&qp->r_lock, flags); |
|
} |
|
|
|
u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, |
|
struct ib_other_headers *ohdr, u32 *bth1, |
|
u32 *bth2, u16 fidx) |
|
{ |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct tid_rdma_params *remote; |
|
struct tid_rdma_request *req = wqe_to_tid_req(wqe); |
|
struct tid_rdma_flow *flow = &req->flows[fidx]; |
|
u32 generation; |
|
|
|
rcu_read_lock(); |
|
remote = rcu_dereference(qpriv->tid_rdma.remote); |
|
KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth1, JKEY, remote->jkey); |
|
ohdr->u.tid_rdma.ack.verbs_qp = cpu_to_be32(qp->remote_qpn); |
|
*bth1 = remote->qp; |
|
rcu_read_unlock(); |
|
|
|
generation = kern_flow_generation_next(flow->flow_state.generation); |
|
*bth2 = mask_psn((generation << HFI1_KDETH_BTH_SEQ_SHIFT) - 1); |
|
qpriv->s_resync_psn = *bth2; |
|
*bth2 |= IB_BTH_REQ_ACK; |
|
KDETH_RESET(ohdr->u.tid_rdma.ack.kdeth0, KVER, 0x1); |
|
|
|
return sizeof(ohdr->u.tid_rdma.resync) / sizeof(u32); |
|
} |
|
|
|
void hfi1_rc_rcv_tid_rdma_resync(struct hfi1_packet *packet) |
|
{ |
|
struct ib_other_headers *ohdr = packet->ohdr; |
|
struct rvt_qp *qp = packet->qp; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct hfi1_ctxtdata *rcd = qpriv->rcd; |
|
struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
|
struct rvt_ack_entry *e; |
|
struct tid_rdma_request *req; |
|
struct tid_rdma_flow *flow; |
|
struct tid_flow_state *fs = &qpriv->flow_state; |
|
u32 psn, generation, idx, gen_next; |
|
bool fecn; |
|
unsigned long flags; |
|
|
|
fecn = process_ecn(qp, packet); |
|
psn = mask_psn(be32_to_cpu(ohdr->bth[2])); |
|
|
|
generation = mask_psn(psn + 1) >> HFI1_KDETH_BTH_SEQ_SHIFT; |
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
|
|
gen_next = (fs->generation == KERN_GENERATION_RESERVED) ? |
|
generation : kern_flow_generation_next(fs->generation); |
|
/* |
|
* RESYNC packet contains the "next" generation and can only be |
|
* from the current or previous generations |
|
*/ |
|
if (generation != mask_generation(gen_next - 1) && |
|
generation != gen_next) |
|
goto bail; |
|
/* Already processing a resync */ |
|
if (qpriv->resync) |
|
goto bail; |
|
|
|
spin_lock(&rcd->exp_lock); |
|
if (fs->index >= RXE_NUM_TID_FLOWS) { |
|
/* |
|
* If we don't have a flow, save the generation so it can be |
|
* applied when a new flow is allocated |
|
*/ |
|
fs->generation = generation; |
|
} else { |
|
/* Reprogram the QP flow with new generation */ |
|
rcd->flows[fs->index].generation = generation; |
|
fs->generation = kern_setup_hw_flow(rcd, fs->index); |
|
} |
|
fs->psn = 0; |
|
/* |
|
* Disable SW PSN checking since a RESYNC is equivalent to a |
|
* sync point and the flow has/will be reprogrammed |
|
*/ |
|
qpriv->s_flags &= ~HFI1_R_TID_SW_PSN; |
|
trace_hfi1_tid_write_rsp_rcv_resync(qp); |
|
|
|
/* |
|
* Reset all TID flow information with the new generation. |
|
* This is done for all requests and segments after the |
|
* last received segment |
|
*/ |
|
for (idx = qpriv->r_tid_tail; ; idx++) { |
|
u16 flow_idx; |
|
|
|
if (idx > rvt_size_atomic(&dev->rdi)) |
|
idx = 0; |
|
e = &qp->s_ack_queue[idx]; |
|
if (e->opcode == TID_OP(WRITE_REQ)) { |
|
req = ack_to_tid_req(e); |
|
trace_hfi1_tid_req_rcv_resync(qp, 0, e->opcode, e->psn, |
|
e->lpsn, req); |
|
|
|
/* start from last unacked segment */ |
|
for (flow_idx = req->clear_tail; |
|
CIRC_CNT(req->setup_head, flow_idx, |
|
MAX_FLOWS); |
|
flow_idx = CIRC_NEXT(flow_idx, MAX_FLOWS)) { |
|
u32 lpsn; |
|
u32 next; |
|
|
|
flow = &req->flows[flow_idx]; |
|
lpsn = full_flow_psn(flow, |
|
flow->flow_state.lpsn); |
|
next = flow->flow_state.r_next_psn; |
|
flow->npkts = delta_psn(lpsn, next - 1); |
|
flow->flow_state.generation = fs->generation; |
|
flow->flow_state.spsn = fs->psn; |
|
flow->flow_state.lpsn = |
|
flow->flow_state.spsn + flow->npkts - 1; |
|
flow->flow_state.r_next_psn = |
|
full_flow_psn(flow, |
|
flow->flow_state.spsn); |
|
fs->psn += flow->npkts; |
|
trace_hfi1_tid_flow_rcv_resync(qp, flow_idx, |
|
flow); |
|
} |
|
} |
|
if (idx == qp->s_tail_ack_queue) |
|
break; |
|
} |
|
|
|
spin_unlock(&rcd->exp_lock); |
|
qpriv->resync = true; |
|
/* RESYNC request always gets a TID RDMA ACK. */ |
|
qpriv->s_nak_state = 0; |
|
tid_rdma_trigger_ack(qp); |
|
bail: |
|
if (fecn) |
|
qp->s_flags |= RVT_S_ECN; |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
} |
|
|
|
/* |
|
* Call this function when the last TID RDMA WRITE DATA packet for a request |
|
* is built. |
|
*/ |
|
static void update_tid_tail(struct rvt_qp *qp) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
u32 i; |
|
struct rvt_swqe *wqe; |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
/* Can't move beyond s_tid_cur */ |
|
if (priv->s_tid_tail == priv->s_tid_cur) |
|
return; |
|
for (i = priv->s_tid_tail + 1; ; i++) { |
|
if (i == qp->s_size) |
|
i = 0; |
|
|
|
if (i == priv->s_tid_cur) |
|
break; |
|
wqe = rvt_get_swqe_ptr(qp, i); |
|
if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) |
|
break; |
|
} |
|
priv->s_tid_tail = i; |
|
priv->s_state = TID_OP(WRITE_RESP); |
|
} |
|
|
|
int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps) |
|
__must_hold(&qp->s_lock) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct rvt_swqe *wqe; |
|
u32 bth1 = 0, bth2 = 0, hwords = 5, len, middle = 0; |
|
struct ib_other_headers *ohdr; |
|
struct rvt_sge_state *ss = &qp->s_sge; |
|
struct rvt_ack_entry *e = &qp->s_ack_queue[qp->s_tail_ack_queue]; |
|
struct tid_rdma_request *req = ack_to_tid_req(e); |
|
bool last = false; |
|
u8 opcode = TID_OP(WRITE_DATA); |
|
|
|
lockdep_assert_held(&qp->s_lock); |
|
trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0); |
|
/* |
|
* Prioritize the sending of the requests and responses over the |
|
* sending of the TID RDMA data packets. |
|
*/ |
|
if (((atomic_read(&priv->n_tid_requests) < HFI1_TID_RDMA_WRITE_CNT) && |
|
atomic_read(&priv->n_requests) && |
|
!(qp->s_flags & (RVT_S_BUSY | RVT_S_WAIT_ACK | |
|
HFI1_S_ANY_WAIT_IO))) || |
|
(e->opcode == TID_OP(WRITE_REQ) && req->cur_seg < req->alloc_seg && |
|
!(qp->s_flags & (RVT_S_BUSY | HFI1_S_ANY_WAIT_IO)))) { |
|
struct iowait_work *iowork; |
|
|
|
iowork = iowait_get_ib_work(&priv->s_iowait); |
|
ps->s_txreq = get_waiting_verbs_txreq(iowork); |
|
if (ps->s_txreq || hfi1_make_rc_req(qp, ps)) { |
|
priv->s_flags |= HFI1_S_TID_BUSY_SET; |
|
return 1; |
|
} |
|
} |
|
|
|
ps->s_txreq = get_txreq(ps->dev, qp); |
|
if (!ps->s_txreq) |
|
goto bail_no_tx; |
|
|
|
ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth; |
|
|
|
if ((priv->s_flags & RVT_S_ACK_PENDING) && |
|
make_tid_rdma_ack(qp, ohdr, ps)) |
|
return 1; |
|
|
|
/* |
|
* Bail out if we can't send data. |
|
* Be reminded that this check must been done after the call to |
|
* make_tid_rdma_ack() because the responding QP could be in |
|
* RTR state where it can send TID RDMA ACK, not TID RDMA WRITE DATA. |
|
*/ |
|
if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) |
|
goto bail; |
|
|
|
if (priv->s_flags & RVT_S_WAIT_ACK) |
|
goto bail; |
|
|
|
/* Check whether there is anything to do. */ |
|
if (priv->s_tid_tail == HFI1_QP_WQE_INVALID) |
|
goto bail; |
|
wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); |
|
req = wqe_to_tid_req(wqe); |
|
trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, wqe->psn, |
|
wqe->lpsn, req); |
|
switch (priv->s_state) { |
|
case TID_OP(WRITE_REQ): |
|
case TID_OP(WRITE_RESP): |
|
priv->tid_ss.sge = wqe->sg_list[0]; |
|
priv->tid_ss.sg_list = wqe->sg_list + 1; |
|
priv->tid_ss.num_sge = wqe->wr.num_sge; |
|
priv->tid_ss.total_len = wqe->length; |
|
|
|
if (priv->s_state == TID_OP(WRITE_REQ)) |
|
hfi1_tid_rdma_restart_req(qp, wqe, &bth2); |
|
priv->s_state = TID_OP(WRITE_DATA); |
|
fallthrough; |
|
|
|
case TID_OP(WRITE_DATA): |
|
/* |
|
* 1. Check whether TID RDMA WRITE RESP available. |
|
* 2. If no: |
|
* 2.1 If have more segments and no TID RDMA WRITE RESP, |
|
* set HFI1_S_WAIT_TID_RESP |
|
* 2.2 Return indicating no progress made. |
|
* 3. If yes: |
|
* 3.1 Build TID RDMA WRITE DATA packet. |
|
* 3.2 If last packet in segment: |
|
* 3.2.1 Change KDETH header bits |
|
* 3.2.2 Advance RESP pointers. |
|
* 3.3 Return indicating progress made. |
|
*/ |
|
trace_hfi1_sender_make_tid_pkt(qp); |
|
trace_hfi1_tid_write_sender_make_tid_pkt(qp, 0); |
|
wqe = rvt_get_swqe_ptr(qp, priv->s_tid_tail); |
|
req = wqe_to_tid_req(wqe); |
|
len = wqe->length; |
|
|
|
if (!req->comp_seg || req->cur_seg == req->comp_seg) |
|
goto bail; |
|
|
|
trace_hfi1_tid_req_make_tid_pkt(qp, 0, wqe->wr.opcode, |
|
wqe->psn, wqe->lpsn, req); |
|
last = hfi1_build_tid_rdma_packet(wqe, ohdr, &bth1, &bth2, |
|
&len); |
|
|
|
if (last) { |
|
/* move pointer to next flow */ |
|
req->clear_tail = CIRC_NEXT(req->clear_tail, |
|
MAX_FLOWS); |
|
if (++req->cur_seg < req->total_segs) { |
|
if (!CIRC_CNT(req->setup_head, req->clear_tail, |
|
MAX_FLOWS)) |
|
qp->s_flags |= HFI1_S_WAIT_TID_RESP; |
|
} else { |
|
priv->s_state = TID_OP(WRITE_DATA_LAST); |
|
opcode = TID_OP(WRITE_DATA_LAST); |
|
|
|
/* Advance the s_tid_tail now */ |
|
update_tid_tail(qp); |
|
} |
|
} |
|
hwords += sizeof(ohdr->u.tid_rdma.w_data) / sizeof(u32); |
|
ss = &priv->tid_ss; |
|
break; |
|
|
|
case TID_OP(RESYNC): |
|
trace_hfi1_sender_make_tid_pkt(qp); |
|
/* Use generation from the most recently received response */ |
|
wqe = rvt_get_swqe_ptr(qp, priv->s_tid_cur); |
|
req = wqe_to_tid_req(wqe); |
|
/* If no responses for this WQE look at the previous one */ |
|
if (!req->comp_seg) { |
|
wqe = rvt_get_swqe_ptr(qp, |
|
(!priv->s_tid_cur ? qp->s_size : |
|
priv->s_tid_cur) - 1); |
|
req = wqe_to_tid_req(wqe); |
|
} |
|
hwords += hfi1_build_tid_rdma_resync(qp, wqe, ohdr, &bth1, |
|
&bth2, |
|
CIRC_PREV(req->setup_head, |
|
MAX_FLOWS)); |
|
ss = NULL; |
|
len = 0; |
|
opcode = TID_OP(RESYNC); |
|
break; |
|
|
|
default: |
|
goto bail; |
|
} |
|
if (priv->s_flags & RVT_S_SEND_ONE) { |
|
priv->s_flags &= ~RVT_S_SEND_ONE; |
|
priv->s_flags |= RVT_S_WAIT_ACK; |
|
bth2 |= IB_BTH_REQ_ACK; |
|
} |
|
qp->s_len -= len; |
|
ps->s_txreq->hdr_dwords = hwords; |
|
ps->s_txreq->sde = priv->s_sde; |
|
ps->s_txreq->ss = ss; |
|
ps->s_txreq->s_cur_size = len; |
|
hfi1_make_ruc_header(qp, ohdr, (opcode << 24), bth1, bth2, |
|
middle, ps); |
|
return 1; |
|
bail: |
|
hfi1_put_txreq(ps->s_txreq); |
|
bail_no_tx: |
|
ps->s_txreq = NULL; |
|
priv->s_flags &= ~RVT_S_BUSY; |
|
/* |
|
* If we didn't get a txreq, the QP will be woken up later to try |
|
* again, set the flags to the the wake up which work item to wake |
|
* up. |
|
* (A better algorithm should be found to do this and generalize the |
|
* sleep/wakeup flags.) |
|
*/ |
|
iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_TID); |
|
return 0; |
|
} |
|
|
|
static int make_tid_rdma_ack(struct rvt_qp *qp, |
|
struct ib_other_headers *ohdr, |
|
struct hfi1_pkt_state *ps) |
|
{ |
|
struct rvt_ack_entry *e; |
|
struct hfi1_qp_priv *qpriv = qp->priv; |
|
struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
|
u32 hwords, next; |
|
u32 len = 0; |
|
u32 bth1 = 0, bth2 = 0; |
|
int middle = 0; |
|
u16 flow; |
|
struct tid_rdma_request *req, *nreq; |
|
|
|
trace_hfi1_tid_write_rsp_make_tid_ack(qp); |
|
/* Don't send an ACK if we aren't supposed to. */ |
|
if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) |
|
goto bail; |
|
|
|
/* header size in 32-bit words LRH+BTH = (8+12)/4. */ |
|
hwords = 5; |
|
|
|
e = &qp->s_ack_queue[qpriv->r_tid_ack]; |
|
req = ack_to_tid_req(e); |
|
/* |
|
* In the RESYNC case, we are exactly one segment past the |
|
* previously sent ack or at the previously sent NAK. So to send |
|
* the resync ack, we go back one segment (which might be part of |
|
* the previous request) and let the do-while loop execute again. |
|
* The advantage of executing the do-while loop is that any data |
|
* received after the previous ack is automatically acked in the |
|
* RESYNC ack. It turns out that for the do-while loop we only need |
|
* to pull back qpriv->r_tid_ack, not the segment |
|
* indices/counters. The scheme works even if the previous request |
|
* was not a TID WRITE request. |
|
*/ |
|
if (qpriv->resync) { |
|
if (!req->ack_seg || req->ack_seg == req->total_segs) |
|
qpriv->r_tid_ack = !qpriv->r_tid_ack ? |
|
rvt_size_atomic(&dev->rdi) : |
|
qpriv->r_tid_ack - 1; |
|
e = &qp->s_ack_queue[qpriv->r_tid_ack]; |
|
req = ack_to_tid_req(e); |
|
} |
|
|
|
trace_hfi1_rsp_make_tid_ack(qp, e->psn); |
|
trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn, |
|
req); |
|
/* |
|
* If we've sent all the ACKs that we can, we are done |
|
* until we get more segments... |
|
*/ |
|
if (!qpriv->s_nak_state && !qpriv->resync && |
|
req->ack_seg == req->comp_seg) |
|
goto bail; |
|
|
|
do { |
|
/* |
|
* To deal with coalesced ACKs, the acked_tail pointer |
|
* into the flow array is used. The distance between it |
|
* and the clear_tail is the number of flows that are |
|
* being ACK'ed. |
|
*/ |
|
req->ack_seg += |
|
/* Get up-to-date value */ |
|
CIRC_CNT(req->clear_tail, req->acked_tail, |
|
MAX_FLOWS); |
|
/* Advance acked index */ |
|
req->acked_tail = req->clear_tail; |
|
|
|
/* |
|
* req->clear_tail points to the segment currently being |
|
* received. So, when sending an ACK, the previous |
|
* segment is being ACK'ed. |
|
*/ |
|
flow = CIRC_PREV(req->acked_tail, MAX_FLOWS); |
|
if (req->ack_seg != req->total_segs) |
|
break; |
|
req->state = TID_REQUEST_COMPLETE; |
|
|
|
next = qpriv->r_tid_ack + 1; |
|
if (next > rvt_size_atomic(&dev->rdi)) |
|
next = 0; |
|
qpriv->r_tid_ack = next; |
|
if (qp->s_ack_queue[next].opcode != TID_OP(WRITE_REQ)) |
|
break; |
|
nreq = ack_to_tid_req(&qp->s_ack_queue[next]); |
|
if (!nreq->comp_seg || nreq->ack_seg == nreq->comp_seg) |
|
break; |
|
|
|
/* Move to the next ack entry now */ |
|
e = &qp->s_ack_queue[qpriv->r_tid_ack]; |
|
req = ack_to_tid_req(e); |
|
} while (1); |
|
|
|
/* |
|
* At this point qpriv->r_tid_ack == qpriv->r_tid_tail but e and |
|
* req could be pointing at the previous ack queue entry |
|
*/ |
|
if (qpriv->s_nak_state || |
|
(qpriv->resync && |
|
!hfi1_tid_rdma_is_resync_psn(qpriv->r_next_psn_kdeth - 1) && |
|
(cmp_psn(qpriv->r_next_psn_kdeth - 1, |
|
full_flow_psn(&req->flows[flow], |
|
req->flows[flow].flow_state.lpsn)) > 0))) { |
|
/* |
|
* A NAK will implicitly acknowledge all previous TID RDMA |
|
* requests. Therefore, we NAK with the req->acked_tail |
|
* segment for the request at qpriv->r_tid_ack (same at |
|
* this point as the req->clear_tail segment for the |
|
* qpriv->r_tid_tail request) |
|
*/ |
|
e = &qp->s_ack_queue[qpriv->r_tid_ack]; |
|
req = ack_to_tid_req(e); |
|
flow = req->acked_tail; |
|
} else if (req->ack_seg == req->total_segs && |
|
qpriv->s_flags & HFI1_R_TID_WAIT_INTERLCK) |
|
qpriv->s_flags &= ~HFI1_R_TID_WAIT_INTERLCK; |
|
|
|
trace_hfi1_tid_write_rsp_make_tid_ack(qp); |
|
trace_hfi1_tid_req_make_tid_ack(qp, 0, e->opcode, e->psn, e->lpsn, |
|
req); |
|
hwords += hfi1_build_tid_rdma_write_ack(qp, e, ohdr, flow, &bth1, |
|
&bth2); |
|
len = 0; |
|
qpriv->s_flags &= ~RVT_S_ACK_PENDING; |
|
ps->s_txreq->hdr_dwords = hwords; |
|
ps->s_txreq->sde = qpriv->s_sde; |
|
ps->s_txreq->s_cur_size = len; |
|
ps->s_txreq->ss = NULL; |
|
hfi1_make_ruc_header(qp, ohdr, (TID_OP(ACK) << 24), bth1, bth2, middle, |
|
ps); |
|
ps->s_txreq->txreq.flags |= SDMA_TXREQ_F_VIP; |
|
return 1; |
|
bail: |
|
/* |
|
* Ensure s_rdma_ack_cnt changes are committed prior to resetting |
|
* RVT_S_RESP_PENDING |
|
*/ |
|
smp_wmb(); |
|
qpriv->s_flags &= ~RVT_S_ACK_PENDING; |
|
return 0; |
|
} |
|
|
|
static int hfi1_send_tid_ok(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
return !(priv->s_flags & RVT_S_BUSY || |
|
qp->s_flags & HFI1_S_ANY_WAIT_IO) && |
|
(verbs_txreq_queued(iowait_get_tid_work(&priv->s_iowait)) || |
|
(priv->s_flags & RVT_S_RESP_PENDING) || |
|
!(qp->s_flags & HFI1_S_ANY_TID_WAIT_SEND)); |
|
} |
|
|
|
void _hfi1_do_tid_send(struct work_struct *work) |
|
{ |
|
struct iowait_work *w = container_of(work, struct iowait_work, iowork); |
|
struct rvt_qp *qp = iowait_to_qp(w->iow); |
|
|
|
hfi1_do_tid_send(qp); |
|
} |
|
|
|
static void hfi1_do_tid_send(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_pkt_state ps; |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
ps.dev = to_idev(qp->ibqp.device); |
|
ps.ibp = to_iport(qp->ibqp.device, qp->port_num); |
|
ps.ppd = ppd_from_ibp(ps.ibp); |
|
ps.wait = iowait_get_tid_work(&priv->s_iowait); |
|
ps.in_thread = false; |
|
ps.timeout_int = qp->timeout_jiffies / 8; |
|
|
|
trace_hfi1_rc_do_tid_send(qp, false); |
|
spin_lock_irqsave(&qp->s_lock, ps.flags); |
|
|
|
/* Return if we are already busy processing a work request. */ |
|
if (!hfi1_send_tid_ok(qp)) { |
|
if (qp->s_flags & HFI1_S_ANY_WAIT_IO) |
|
iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_TID); |
|
spin_unlock_irqrestore(&qp->s_lock, ps.flags); |
|
return; |
|
} |
|
|
|
priv->s_flags |= RVT_S_BUSY; |
|
|
|
ps.timeout = jiffies + ps.timeout_int; |
|
ps.cpu = priv->s_sde ? priv->s_sde->cpu : |
|
cpumask_first(cpumask_of_node(ps.ppd->dd->node)); |
|
ps.pkts_sent = false; |
|
|
|
/* insure a pre-built packet is handled */ |
|
ps.s_txreq = get_waiting_verbs_txreq(ps.wait); |
|
do { |
|
/* Check for a constructed packet to be sent. */ |
|
if (ps.s_txreq) { |
|
if (priv->s_flags & HFI1_S_TID_BUSY_SET) { |
|
qp->s_flags |= RVT_S_BUSY; |
|
ps.wait = iowait_get_ib_work(&priv->s_iowait); |
|
} |
|
spin_unlock_irqrestore(&qp->s_lock, ps.flags); |
|
|
|
/* |
|
* If the packet cannot be sent now, return and |
|
* the send tasklet will be woken up later. |
|
*/ |
|
if (hfi1_verbs_send(qp, &ps)) |
|
return; |
|
|
|
/* allow other tasks to run */ |
|
if (hfi1_schedule_send_yield(qp, &ps, true)) |
|
return; |
|
|
|
spin_lock_irqsave(&qp->s_lock, ps.flags); |
|
if (priv->s_flags & HFI1_S_TID_BUSY_SET) { |
|
qp->s_flags &= ~RVT_S_BUSY; |
|
priv->s_flags &= ~HFI1_S_TID_BUSY_SET; |
|
ps.wait = iowait_get_tid_work(&priv->s_iowait); |
|
if (iowait_flag_set(&priv->s_iowait, |
|
IOWAIT_PENDING_IB)) |
|
hfi1_schedule_send(qp); |
|
} |
|
} |
|
} while (hfi1_make_tid_rdma_pkt(qp, &ps)); |
|
iowait_starve_clear(ps.pkts_sent, &priv->s_iowait); |
|
spin_unlock_irqrestore(&qp->s_lock, ps.flags); |
|
} |
|
|
|
static bool _hfi1_schedule_tid_send(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct hfi1_ibport *ibp = |
|
to_iport(qp->ibqp.device, qp->port_num); |
|
struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); |
|
struct hfi1_devdata *dd = ppd->dd; |
|
|
|
if ((dd->flags & HFI1_SHUTDOWN)) |
|
return true; |
|
|
|
return iowait_tid_schedule(&priv->s_iowait, ppd->hfi1_wq, |
|
priv->s_sde ? |
|
priv->s_sde->cpu : |
|
cpumask_first(cpumask_of_node(dd->node))); |
|
} |
|
|
|
/** |
|
* hfi1_schedule_tid_send - schedule progress on TID RDMA state machine |
|
* @qp: the QP |
|
* |
|
* This schedules qp progress on the TID RDMA state machine. Caller |
|
* should hold the s_lock. |
|
* Unlike hfi1_schedule_send(), this cannot use hfi1_send_ok() because |
|
* the two state machines can step on each other with respect to the |
|
* RVT_S_BUSY flag. |
|
* Therefore, a modified test is used. |
|
* @return true if the second leg is scheduled; |
|
* false if the second leg is not scheduled. |
|
*/ |
|
bool hfi1_schedule_tid_send(struct rvt_qp *qp) |
|
{ |
|
lockdep_assert_held(&qp->s_lock); |
|
if (hfi1_send_tid_ok(qp)) { |
|
/* |
|
* The following call returns true if the qp is not on the |
|
* queue and false if the qp is already on the queue before |
|
* this call. Either way, the qp will be on the queue when the |
|
* call returns. |
|
*/ |
|
_hfi1_schedule_tid_send(qp); |
|
return true; |
|
} |
|
if (qp->s_flags & HFI1_S_ANY_WAIT_IO) |
|
iowait_set_flag(&((struct hfi1_qp_priv *)qp->priv)->s_iowait, |
|
IOWAIT_PENDING_TID); |
|
return false; |
|
} |
|
|
|
bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e) |
|
{ |
|
struct rvt_ack_entry *prev; |
|
struct tid_rdma_request *req; |
|
struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
u32 s_prev; |
|
|
|
s_prev = qp->s_tail_ack_queue == 0 ? rvt_size_atomic(&dev->rdi) : |
|
(qp->s_tail_ack_queue - 1); |
|
prev = &qp->s_ack_queue[s_prev]; |
|
|
|
if ((e->opcode == TID_OP(READ_REQ) || |
|
e->opcode == OP(RDMA_READ_REQUEST)) && |
|
prev->opcode == TID_OP(WRITE_REQ)) { |
|
req = ack_to_tid_req(prev); |
|
if (req->ack_seg != req->total_segs) { |
|
priv->s_flags |= HFI1_R_TID_WAIT_INTERLCK; |
|
return true; |
|
} |
|
} |
|
return false; |
|
} |
|
|
|
static u32 read_r_next_psn(struct hfi1_devdata *dd, u8 ctxt, u8 fidx) |
|
{ |
|
u64 reg; |
|
|
|
/* |
|
* The only sane way to get the amount of |
|
* progress is to read the HW flow state. |
|
*/ |
|
reg = read_uctxt_csr(dd, ctxt, RCV_TID_FLOW_TABLE + (8 * fidx)); |
|
return mask_psn(reg); |
|
} |
|
|
|
static void tid_rdma_rcv_err(struct hfi1_packet *packet, |
|
struct ib_other_headers *ohdr, |
|
struct rvt_qp *qp, u32 psn, int diff, bool fecn) |
|
{ |
|
unsigned long flags; |
|
|
|
tid_rdma_rcv_error(packet, ohdr, qp, psn, diff); |
|
if (fecn) { |
|
spin_lock_irqsave(&qp->s_lock, flags); |
|
qp->s_flags |= RVT_S_ECN; |
|
spin_unlock_irqrestore(&qp->s_lock, flags); |
|
} |
|
} |
|
|
|
static void update_r_next_psn_fecn(struct hfi1_packet *packet, |
|
struct hfi1_qp_priv *priv, |
|
struct hfi1_ctxtdata *rcd, |
|
struct tid_rdma_flow *flow, |
|
bool fecn) |
|
{ |
|
/* |
|
* If a start/middle packet is delivered here due to |
|
* RSM rule and FECN, we need to update the r_next_psn. |
|
*/ |
|
if (fecn && packet->etype == RHF_RCV_TYPE_EAGER && |
|
!(priv->s_flags & HFI1_R_TID_SW_PSN)) { |
|
struct hfi1_devdata *dd = rcd->dd; |
|
|
|
flow->flow_state.r_next_psn = |
|
read_r_next_psn(dd, rcd->ctxt, flow->idx); |
|
} |
|
}
|
|
|