| From add55daff850e7237b3bd00b336ab403b1184a4c Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Wed, 1 Dec 2021 17:15:09 -0600 |
| Subject: RDMA/irdma: Don't arm the CQ more than two times if no CE for this CQ |
| |
| From: Tatyana Nikolova <tatyana.e.nikolova@intel.com> |
| |
| [ Upstream commit 10467ce09fefa2e74359f5b2ab1efb8909402f19 ] |
| |
| Completion events (CEs) are lost if the application is allowed to arm the |
| CQ more than two times when no new CE for this CQ has been generated by |
| the HW. |
| |
| Check if arming has been done for the CQ and if not, arm the CQ for any |
| event otherwise promote to arm the CQ for any event only when the last arm |
| event was solicited. |
| |
| Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs") |
| Link: https://lore.kernel.org/r/20211201231509.1930-2-shiraz.saleem@intel.com |
| Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com> |
| Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com> |
| Signed-off-by: Jason Gunthorpe <jgg@nvidia.com> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| drivers/infiniband/hw/irdma/hw.c | 2 ++ |
| drivers/infiniband/hw/irdma/main.h | 1 + |
| drivers/infiniband/hw/irdma/utils.c | 15 +++++++++++++++ |
| drivers/infiniband/hw/irdma/verbs.c | 23 ++++++++++++++++++----- |
| drivers/infiniband/hw/irdma/verbs.h | 2 ++ |
| 5 files changed, 38 insertions(+), 5 deletions(-) |
| |
| diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c |
| index 7f58a1b9bb4f8..aa119441eb45c 100644 |
| --- a/drivers/infiniband/hw/irdma/hw.c |
| +++ b/drivers/infiniband/hw/irdma/hw.c |
| @@ -60,6 +60,8 @@ static void irdma_iwarp_ce_handler(struct irdma_sc_cq *iwcq) |
| { |
| struct irdma_cq *cq = iwcq->back_cq; |
| |
| + if (!cq->user_mode) |
| + cq->armed = false; |
| if (cq->ibcq.comp_handler) |
| cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); |
| } |
| diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h |
| index b678fe712447e..8b215f3cee891 100644 |
| --- a/drivers/infiniband/hw/irdma/main.h |
| +++ b/drivers/infiniband/hw/irdma/main.h |
| @@ -541,6 +541,7 @@ int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, |
| void (*callback_fcn)(struct irdma_cqp_request *cqp_request), |
| void *cb_param); |
| void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request); |
| +bool irdma_cq_empty(struct irdma_cq *iwcq); |
| int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event, |
| void *ptr); |
| int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event, |
| diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c |
| index 1bbf23689de48..feebfe6bf31ad 100644 |
| --- a/drivers/infiniband/hw/irdma/utils.c |
| +++ b/drivers/infiniband/hw/irdma/utils.c |
| @@ -2531,3 +2531,18 @@ void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) |
| ibevent.element.qp = &iwqp->ibqp; |
| iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); |
| } |
| + |
| +bool irdma_cq_empty(struct irdma_cq *iwcq) |
| +{ |
| + struct irdma_cq_uk *ukcq; |
| + u64 qword3; |
| + __le64 *cqe; |
| + u8 polarity; |
| + |
| + ukcq = &iwcq->sc_cq.cq_uk; |
| + cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); |
| + get_64bit_val(cqe, 24, &qword3); |
| + polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); |
| + |
| + return polarity != ukcq->polarity; |
| +} |
| diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c |
| index 102dc9342f2a2..8bbc4620a97a2 100644 |
| --- a/drivers/infiniband/hw/irdma/verbs.c |
| +++ b/drivers/infiniband/hw/irdma/verbs.c |
| @@ -3604,18 +3604,31 @@ static int irdma_req_notify_cq(struct ib_cq *ibcq, |
| struct irdma_cq *iwcq; |
| struct irdma_cq_uk *ukcq; |
| unsigned long flags; |
| - enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT; |
| + enum irdma_cmpl_notify cq_notify; |
| + bool promo_event = false; |
| + int ret = 0; |
| |
| + cq_notify = notify_flags == IB_CQ_SOLICITED ? |
| + IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT; |
| iwcq = to_iwcq(ibcq); |
| ukcq = &iwcq->sc_cq.cq_uk; |
| - if (notify_flags == IB_CQ_SOLICITED) |
| - cq_notify = IRDMA_CQ_COMPL_SOLICITED; |
| |
| spin_lock_irqsave(&iwcq->lock, flags); |
| - irdma_uk_cq_request_notification(ukcq, cq_notify); |
| + /* Only promote to arm the CQ for any event if the last arm event was solicited. */ |
| + if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED) |
| + promo_event = true; |
| + |
| + if (!iwcq->armed || promo_event) { |
| + iwcq->armed = true; |
| + iwcq->last_notify = cq_notify; |
| + irdma_uk_cq_request_notification(ukcq, cq_notify); |
| + } |
| + |
| + if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && !irdma_cq_empty(iwcq)) |
| + ret = 1; |
| spin_unlock_irqrestore(&iwcq->lock, flags); |
| |
| - return 0; |
| + return ret; |
| } |
| |
| static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, |
| diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h |
| index 5c244cd321a3a..d0fdef8d09ead 100644 |
| --- a/drivers/infiniband/hw/irdma/verbs.h |
| +++ b/drivers/infiniband/hw/irdma/verbs.h |
| @@ -110,6 +110,8 @@ struct irdma_cq { |
| u16 cq_size; |
| u16 cq_num; |
| bool user_mode; |
| + bool armed; |
| + enum irdma_cmpl_notify last_notify; |
| u32 polled_cmpls; |
| u32 cq_mem_size; |
| struct irdma_dma_mem kmem; |
| -- |
| 2.33.0 |
| |