| From ecd5d8957c5bbfd6a8fa3b11ef72e36e0c8a8e77 Mon Sep 17 00:00:00 2001 |
| From: Chuck Lever <chuck.lever@oracle.com> |
| Date: Mon, 26 Aug 2019 13:12:57 -0400 |
| Subject: [PATCH] xprtrdma: Send Queue size grows after a reconnect |
| |
| commit 98ef77d1aaa7a2f4e1b2a721faa084222021fda7 upstream. |
| |
| Eli Dorfman reports that after a series of idle disconnects, an |
| RPC/RDMA transport becomes unusable (rdma_create_qp returns |
| -ENOMEM). Problem was tracked down to increasing Send Queue size |
| after each reconnect. |
| |
| The rdma_create_qp() API does not promise to leave its @qp_init_attr |
| parameter unaltered. In fact, some drivers do modify one or more of |
| its fields. Thus our calls to rdma_create_qp must use a fresh copy |
| of ib_qp_init_attr each time. |
| |
| This fix is appropriate for kernels dating back to late 2007, though |
| it will have to be adapted, as the connect code has changed over the |
| years. |
| |
| Reported-by: Eli Dorfman <eli@vastdata.com> |
| Signed-off-by: Chuck Lever <chuck.lever@oracle.com> |
| Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com> |
| Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com> |
| |
| diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c |
| index 84bb37924540..3f67c395845d 100644 |
| --- a/net/sunrpc/xprtrdma/verbs.c |
| +++ b/net/sunrpc/xprtrdma/verbs.c |
| @@ -607,10 +607,10 @@ void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt) |
| * Unlike a normal reconnection, a fresh PD and a new set |
| * of MRs and buffers is needed. |
| */ |
| -static int |
| -rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, |
| - struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) |
| +static int rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, |
| + struct ib_qp_init_attr *qp_init_attr) |
| { |
| + struct rpcrdma_ia *ia = &r_xprt->rx_ia; |
| int rc, err; |
| |
| trace_xprtrdma_reinsert(r_xprt); |
| @@ -627,7 +627,7 @@ rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, |
| } |
| |
| rc = -ENETUNREACH; |
| - err = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); |
| + err = rdma_create_qp(ia->ri_id, ia->ri_pd, qp_init_attr); |
| if (err) { |
| pr_err("rpcrdma: rdma_create_qp returned %d\n", err); |
| goto out3; |
| @@ -644,16 +644,16 @@ rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt, |
| return rc; |
| } |
| |
| -static int |
| -rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, struct rpcrdma_ep *ep, |
| - struct rpcrdma_ia *ia) |
| +static int rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, |
| + struct ib_qp_init_attr *qp_init_attr) |
| { |
| + struct rpcrdma_ia *ia = &r_xprt->rx_ia; |
| struct rdma_cm_id *id, *old; |
| int err, rc; |
| |
| trace_xprtrdma_reconnect(r_xprt); |
| |
| - rpcrdma_ep_disconnect(ep, ia); |
| + rpcrdma_ep_disconnect(&r_xprt->rx_ep, ia); |
| |
| rc = -EHOSTUNREACH; |
| id = rpcrdma_create_id(r_xprt, ia); |
| @@ -675,7 +675,7 @@ rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, struct rpcrdma_ep *ep, |
| goto out_destroy; |
| } |
| |
| - err = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); |
| + err = rdma_create_qp(id, ia->ri_pd, qp_init_attr); |
| if (err) |
| goto out_destroy; |
| |
| @@ -700,25 +700,27 @@ rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) |
| struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt, |
| rx_ia); |
| struct rpc_xprt *xprt = &r_xprt->rx_xprt; |
| + struct ib_qp_init_attr qp_init_attr; |
| int rc; |
| |
| retry: |
| + memcpy(&qp_init_attr, &ep->rep_attr, sizeof(qp_init_attr)); |
| switch (ep->rep_connected) { |
| case 0: |
| dprintk("RPC: %s: connecting...\n", __func__); |
| - rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); |
| + rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &qp_init_attr); |
| if (rc) { |
| rc = -ENETUNREACH; |
| goto out_noupdate; |
| } |
| break; |
| case -ENODEV: |
| - rc = rpcrdma_ep_recreate_xprt(r_xprt, ep, ia); |
| + rc = rpcrdma_ep_recreate_xprt(r_xprt, &qp_init_attr); |
| if (rc) |
| goto out_noupdate; |
| break; |
| default: |
| - rc = rpcrdma_ep_reconnect(r_xprt, ep, ia); |
| + rc = rpcrdma_ep_reconnect(r_xprt, &qp_init_attr); |
| if (rc) |
| goto out; |
| } |
| -- |
| 2.7.4 |
| |