mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
323 lines
8.5 KiB
323 lines
8.5 KiB
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) |
|
/* |
|
* Copyright(c) 2018 Intel Corporation. |
|
* |
|
*/ |
|
#include "hfi.h" |
|
#include "trace.h" |
|
#include "qp.h" |
|
#include "opfn.h" |
|
|
|
#define IB_BTHE_E BIT(IB_BTHE_E_SHIFT) |
|
|
|
#define OPFN_CODE(code) BIT((code) - 1) |
|
#define OPFN_MASK(code) OPFN_CODE(STL_VERBS_EXTD_##code) |
|
|
|
struct hfi1_opfn_type { |
|
bool (*request)(struct rvt_qp *qp, u64 *data); |
|
bool (*response)(struct rvt_qp *qp, u64 *data); |
|
bool (*reply)(struct rvt_qp *qp, u64 data); |
|
void (*error)(struct rvt_qp *qp); |
|
}; |
|
|
|
static struct hfi1_opfn_type hfi1_opfn_handlers[STL_VERBS_EXTD_MAX] = { |
|
[STL_VERBS_EXTD_TID_RDMA] = { |
|
.request = tid_rdma_conn_req, |
|
.response = tid_rdma_conn_resp, |
|
.reply = tid_rdma_conn_reply, |
|
.error = tid_rdma_conn_error, |
|
}, |
|
}; |
|
|
|
static struct workqueue_struct *opfn_wq; |
|
|
|
static void opfn_schedule_conn_request(struct rvt_qp *qp); |
|
|
|
static bool hfi1_opfn_extended(u32 bth1) |
|
{ |
|
return !!(bth1 & IB_BTHE_E); |
|
} |
|
|
|
static void opfn_conn_request(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct ib_atomic_wr wr; |
|
u16 mask, capcode; |
|
struct hfi1_opfn_type *extd; |
|
u64 data; |
|
unsigned long flags; |
|
int ret = 0; |
|
|
|
trace_hfi1_opfn_state_conn_request(qp); |
|
spin_lock_irqsave(&priv->opfn.lock, flags); |
|
/* |
|
* Exit if the extended bit is not set, or if nothing is requested, or |
|
* if we have completed all requests, or if a previous request is in |
|
* progress |
|
*/ |
|
if (!priv->opfn.extended || !priv->opfn.requested || |
|
priv->opfn.requested == priv->opfn.completed || priv->opfn.curr) |
|
goto done; |
|
|
|
mask = priv->opfn.requested & ~priv->opfn.completed; |
|
capcode = ilog2(mask & ~(mask - 1)) + 1; |
|
if (capcode >= STL_VERBS_EXTD_MAX) { |
|
priv->opfn.completed |= OPFN_CODE(capcode); |
|
goto done; |
|
} |
|
|
|
extd = &hfi1_opfn_handlers[capcode]; |
|
if (!extd || !extd->request || !extd->request(qp, &data)) { |
|
/* |
|
* Either there is no handler for this capability or the request |
|
* packet could not be generated. Either way, mark it as done so |
|
* we don't keep attempting to complete it. |
|
*/ |
|
priv->opfn.completed |= OPFN_CODE(capcode); |
|
goto done; |
|
} |
|
|
|
trace_hfi1_opfn_data_conn_request(qp, capcode, data); |
|
data = (data & ~0xf) | capcode; |
|
|
|
memset(&wr, 0, sizeof(wr)); |
|
wr.wr.opcode = IB_WR_OPFN; |
|
wr.remote_addr = HFI1_VERBS_E_ATOMIC_VADDR; |
|
wr.compare_add = data; |
|
|
|
priv->opfn.curr = capcode; /* A new request is now in progress */ |
|
/* Drop opfn.lock before calling ib_post_send() */ |
|
spin_unlock_irqrestore(&priv->opfn.lock, flags); |
|
|
|
ret = ib_post_send(&qp->ibqp, &wr.wr, NULL); |
|
if (ret) |
|
goto err; |
|
trace_hfi1_opfn_state_conn_request(qp); |
|
return; |
|
err: |
|
trace_hfi1_msg_opfn_conn_request(qp, "ib_ost_send failed: ret = ", |
|
(u64)ret); |
|
spin_lock_irqsave(&priv->opfn.lock, flags); |
|
/* |
|
* In case of an unexpected error return from ib_post_send |
|
* clear opfn.curr and reschedule to try again |
|
*/ |
|
priv->opfn.curr = STL_VERBS_EXTD_NONE; |
|
opfn_schedule_conn_request(qp); |
|
done: |
|
spin_unlock_irqrestore(&priv->opfn.lock, flags); |
|
} |
|
|
|
void opfn_send_conn_request(struct work_struct *work) |
|
{ |
|
struct hfi1_opfn_data *od; |
|
struct hfi1_qp_priv *qpriv; |
|
|
|
od = container_of(work, struct hfi1_opfn_data, opfn_work); |
|
qpriv = container_of(od, struct hfi1_qp_priv, opfn); |
|
|
|
opfn_conn_request(qpriv->owner); |
|
} |
|
|
|
/* |
|
* When QP s_lock is held in the caller, the OPFN request must be scheduled |
|
* to a different workqueue to avoid double locking QP s_lock in call to |
|
* ib_post_send in opfn_conn_request |
|
*/ |
|
static void opfn_schedule_conn_request(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
trace_hfi1_opfn_state_sched_conn_request(qp); |
|
queue_work(opfn_wq, &priv->opfn.opfn_work); |
|
} |
|
|
|
void opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e, |
|
struct ib_atomic_eth *ateth) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
u64 data = be64_to_cpu(ateth->compare_data); |
|
struct hfi1_opfn_type *extd; |
|
u8 capcode; |
|
unsigned long flags; |
|
|
|
trace_hfi1_opfn_state_conn_response(qp); |
|
capcode = data & 0xf; |
|
trace_hfi1_opfn_data_conn_response(qp, capcode, data); |
|
if (!capcode || capcode >= STL_VERBS_EXTD_MAX) |
|
return; |
|
|
|
extd = &hfi1_opfn_handlers[capcode]; |
|
|
|
if (!extd || !extd->response) { |
|
e->atomic_data = capcode; |
|
return; |
|
} |
|
|
|
spin_lock_irqsave(&priv->opfn.lock, flags); |
|
if (priv->opfn.completed & OPFN_CODE(capcode)) { |
|
/* |
|
* We are receiving a request for a feature that has already |
|
* been negotiated. This may mean that the other side has reset |
|
*/ |
|
priv->opfn.completed &= ~OPFN_CODE(capcode); |
|
if (extd->error) |
|
extd->error(qp); |
|
} |
|
|
|
if (extd->response(qp, &data)) |
|
priv->opfn.completed |= OPFN_CODE(capcode); |
|
e->atomic_data = (data & ~0xf) | capcode; |
|
trace_hfi1_opfn_state_conn_response(qp); |
|
spin_unlock_irqrestore(&priv->opfn.lock, flags); |
|
} |
|
|
|
void opfn_conn_reply(struct rvt_qp *qp, u64 data) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct hfi1_opfn_type *extd; |
|
u8 capcode; |
|
unsigned long flags; |
|
|
|
trace_hfi1_opfn_state_conn_reply(qp); |
|
capcode = data & 0xf; |
|
trace_hfi1_opfn_data_conn_reply(qp, capcode, data); |
|
if (!capcode || capcode >= STL_VERBS_EXTD_MAX) |
|
return; |
|
|
|
spin_lock_irqsave(&priv->opfn.lock, flags); |
|
/* |
|
* Either there is no previous request or the reply is not for the |
|
* current request |
|
*/ |
|
if (!priv->opfn.curr || capcode != priv->opfn.curr) |
|
goto done; |
|
|
|
extd = &hfi1_opfn_handlers[capcode]; |
|
|
|
if (!extd || !extd->reply) |
|
goto clear; |
|
|
|
if (extd->reply(qp, data)) |
|
priv->opfn.completed |= OPFN_CODE(capcode); |
|
clear: |
|
/* |
|
* Clear opfn.curr to indicate that the previous request is no longer in |
|
* progress |
|
*/ |
|
priv->opfn.curr = STL_VERBS_EXTD_NONE; |
|
trace_hfi1_opfn_state_conn_reply(qp); |
|
done: |
|
spin_unlock_irqrestore(&priv->opfn.lock, flags); |
|
} |
|
|
|
void opfn_conn_error(struct rvt_qp *qp) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
struct hfi1_opfn_type *extd = NULL; |
|
unsigned long flags; |
|
u16 capcode; |
|
|
|
trace_hfi1_opfn_state_conn_error(qp); |
|
trace_hfi1_msg_opfn_conn_error(qp, "error. qp state ", (u64)qp->state); |
|
/* |
|
* The QP has gone into the Error state. We have to invalidate all |
|
* negotiated feature, including the one in progress (if any). The RC |
|
* QP handling will clean the WQE for the connection request. |
|
*/ |
|
spin_lock_irqsave(&priv->opfn.lock, flags); |
|
while (priv->opfn.completed) { |
|
capcode = priv->opfn.completed & ~(priv->opfn.completed - 1); |
|
extd = &hfi1_opfn_handlers[ilog2(capcode) + 1]; |
|
if (extd->error) |
|
extd->error(qp); |
|
priv->opfn.completed &= ~OPFN_CODE(capcode); |
|
} |
|
priv->opfn.extended = 0; |
|
priv->opfn.requested = 0; |
|
priv->opfn.curr = STL_VERBS_EXTD_NONE; |
|
spin_unlock_irqrestore(&priv->opfn.lock, flags); |
|
} |
|
|
|
void opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask) |
|
{ |
|
struct ib_qp *ibqp = &qp->ibqp; |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
unsigned long flags; |
|
|
|
if (attr_mask & IB_QP_RETRY_CNT) |
|
priv->s_retry = attr->retry_cnt; |
|
|
|
spin_lock_irqsave(&priv->opfn.lock, flags); |
|
if (ibqp->qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) { |
|
struct tid_rdma_params *local = &priv->tid_rdma.local; |
|
|
|
if (attr_mask & IB_QP_TIMEOUT) |
|
priv->tid_retry_timeout_jiffies = qp->timeout_jiffies; |
|
if (qp->pmtu == enum_to_mtu(OPA_MTU_4096) || |
|
qp->pmtu == enum_to_mtu(OPA_MTU_8192)) { |
|
tid_rdma_opfn_init(qp, local); |
|
/* |
|
* We only want to set the OPFN requested bit when the |
|
* QP transitions to RTS. |
|
*/ |
|
if (attr_mask & IB_QP_STATE && |
|
attr->qp_state == IB_QPS_RTS) { |
|
priv->opfn.requested |= OPFN_MASK(TID_RDMA); |
|
/* |
|
* If the QP is transitioning to RTS and the |
|
* opfn.completed for TID RDMA has already been |
|
* set, the QP is being moved *back* into RTS. |
|
* We can now renegotiate the TID RDMA |
|
* parameters. |
|
*/ |
|
if (priv->opfn.completed & |
|
OPFN_MASK(TID_RDMA)) { |
|
priv->opfn.completed &= |
|
~OPFN_MASK(TID_RDMA); |
|
/* |
|
* Since the opfn.completed bit was |
|
* already set, it is safe to assume |
|
* that the opfn.extended is also set. |
|
*/ |
|
opfn_schedule_conn_request(qp); |
|
} |
|
} |
|
} else { |
|
memset(local, 0, sizeof(*local)); |
|
} |
|
} |
|
spin_unlock_irqrestore(&priv->opfn.lock, flags); |
|
} |
|
|
|
void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1) |
|
{ |
|
struct hfi1_qp_priv *priv = qp->priv; |
|
|
|
if (!priv->opfn.extended && hfi1_opfn_extended(bth1) && |
|
HFI1_CAP_IS_KSET(OPFN)) { |
|
priv->opfn.extended = 1; |
|
if (qp->state == IB_QPS_RTS) |
|
opfn_conn_request(qp); |
|
} |
|
} |
|
|
|
int opfn_init(void) |
|
{ |
|
opfn_wq = alloc_workqueue("hfi_opfn", |
|
WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | |
|
WQ_MEM_RECLAIM, |
|
HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES); |
|
if (!opfn_wq) |
|
return -ENOMEM; |
|
|
|
return 0; |
|
} |
|
|
|
void opfn_exit(void) |
|
{ |
|
if (opfn_wq) { |
|
destroy_workqueue(opfn_wq); |
|
opfn_wq = NULL; |
|
} |
|
}
|
|
|