[PATCH AUTOSEL 5.1 006/375] io_uring: use cpu_online() to check p->sq_thread_cpu instead of cpu_possible()
diff --git a/m b/m
index da906a8..5029ecc 100644
--- a/m
+++ b/m
@@ -4,55 +4,48 @@
 X-Spam-Level: 
 X-Spam-Status: No, score=-9.1 required=3.0 tests=DKIMWL_WL_HIGH,DKIM_SIGNED,
 	DKIM_VALID,DKIM_VALID_AU,INCLUDES_PATCH,MAILING_LIST_MULTI,SIGNED_OFF_BY,
-	SPF_HELO_NONE,SPF_PASS,USER_AGENT_GIT autolearn=unavailable
+	SPF_HELO_NONE,SPF_PASS,URIBL_BLOCKED,USER_AGENT_GIT autolearn=unavailable
 	autolearn_force=no version=3.4.0
 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99])
-	by smtp.lore.kernel.org (Postfix) with ESMTP id 70BCDC282CE
-	for <linux-block@archiver.kernel.org>; Wed, 22 May 2019 20:03:46 +0000 (UTC)
+	by smtp.lore.kernel.org (Postfix) with ESMTP id F012FC282DD
+	for <linux-block@archiver.kernel.org>; Wed, 22 May 2019 20:04:34 +0000 (UTC)
 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67])
-	by mail.kernel.org (Postfix) with ESMTP id 41D2F20863
-	for <linux-block@archiver.kernel.org>; Wed, 22 May 2019 20:03:46 +0000 (UTC)
+	by mail.kernel.org (Postfix) with ESMTP id B7DCE20821
+	for <linux-block@archiver.kernel.org>; Wed, 22 May 2019 20:04:34 +0000 (UTC)
 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org;
-	s=default; t=1558555426;
-	bh=BqFREttlpvHpkKLt6hvNGva987bDyHxt91ItdyW53v8=;
+	s=default; t=1558555474;
+	bh=doNg1gXw6eyhwf7280nikzntpuqn8BhVuwpCh91jyuA=;
 	h=From:To:Cc:Subject:Date:In-Reply-To:References:List-ID:From;
-	b=Cw8VaK7EKdmV3bEuFHLe5LeyNaCRT0lnc0UmwDMFr108XUctbaALOMuhepMrz+yD0
-	 uNoDwQ+pgvHm1UpBWqEzKjfqlUAF0thgn8eF7XHZD6U8ue9WTb+MEkZgaQdz7SQFTX
-	 Mm8Pcx4RflYiP3MhW/hVMzXe15qF5+mqN4rGdy8g=
+	b=yuseDm604ggk0d1SdshJMlVASCj1jpJShsrpFH2POC9532LsxTF2pR3cQY8kak/qj
+	 9KSLEXMGrF4SLfvXZFCVEiiE5P/qLrcwPOnPU8EDZSdV4JSE67EhkJj4E2zHOUqpbZ
+	 SR9mAWaY8KpBHkJpioyxsynWULHRYsEqLivqHoco=
 Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
-        id S1730588AbfEVUDT (ORCPT <rfc822;linux-block@archiver.kernel.org>);
-        Wed, 22 May 2019 16:03:19 -0400
-Received: from mail.kernel.org ([198.145.29.99]:42360 "EHLO mail.kernel.org"
+        id S1729863AbfEVTVY (ORCPT <rfc822;linux-block@archiver.kernel.org>);
+        Wed, 22 May 2019 15:21:24 -0400
+Received: from mail.kernel.org ([198.145.29.99]:41792 "EHLO mail.kernel.org"
         rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
-        id S1729971AbfEVTVo (ORCPT <rfc822;linux-block@vger.kernel.org>);
-        Wed, 22 May 2019 15:21:44 -0400
+        id S1729790AbfEVTVX (ORCPT <rfc822;linux-block@vger.kernel.org>);
+        Wed, 22 May 2019 15:21:23 -0400
 Received: from sasha-vm.mshome.net (c-73-47-72-35.hsd1.nh.comcast.net [73.47.72.35])
         (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits))
         (No client certificate requested)
-        by mail.kernel.org (Postfix) with ESMTPSA id 4D6172173C;
-        Wed, 22 May 2019 19:21:42 +0000 (UTC)
+        by mail.kernel.org (Postfix) with ESMTPSA id 0DFD821841;
+        Wed, 22 May 2019 19:21:21 +0000 (UTC)
 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org;
-        s=default; t=1558552903;
-        bh=BqFREttlpvHpkKLt6hvNGva987bDyHxt91ItdyW53v8=;
+        s=default; t=1558552882;
+        bh=doNg1gXw6eyhwf7280nikzntpuqn8BhVuwpCh91jyuA=;
         h=From:To:Cc:Subject:Date:In-Reply-To:References:From;
-        b=wpuLr7FFOa2BNtBSNcTMjCqG6Yy9M7ZJLQG6zx86kgEJ5H+zLc9BUUcwhh5uFUIqE
-         gPxFDABqkrbQ6FBLNp9x54Z4FIlmNHZYjxSZFDcY1A4wLQpoevPWbrQ0qfgni7P57+
-         9xmXZGwBAvAK619qhYDTx4NDA3K4pLNgDfs8TpD8=
+        b=Yv5Viyw/JPFlMiGQY3QDwNVJMTJE9bwA0Jxb1HzkIdrry6rulvzV/FqGfFAqQvmqJ
+         21M63oxpODoH15MttF2Zs9tSaoLpo3wqPTZ+JV5V9j4XSIYUEPqmTmNPvGEr5bgMHu
+         h1IVlQNVnDFW7WMWHCIoT/CegtgwezVV4a+M8c3A=
 From:   Sasha Levin <sashal@kernel.org>
 To:     linux-kernel@vger.kernel.org, stable@vger.kernel.org
-Cc:     Ming Lei <ming.lei@redhat.com>,
-        Dongli Zhang <dongli.zhang@oracle.com>,
-        James Smart <james.smart@broadcom.com>,
-        linux-scsi@vger.kernel.org,
-        "Martin K . Petersen" <martin.petersen@oracle.com>,
-        Christoph Hellwig <hch@lst.de>,
-        "James E . J . Bottomley" <jejb@linux.vnet.ibm.com>,
-        Bart Van Assche <bvanassche@acm.org>,
+Cc:     Shenghui Wang <shhuiw@foxmail.com>, Jeff Moyer <jmoyer@redhat.com>,
         Jens Axboe <axboe@kernel.dk>, Sasha Levin <sashal@kernel.org>,
-        linux-block@vger.kernel.org
-Subject: [PATCH AUTOSEL 5.1 020/375] blk-mq: grab .q_usage_counter when queuing request from plug code path
-Date:   Wed, 22 May 2019 15:15:20 -0400
-Message-Id: <20190522192115.22666-20-sashal@kernel.org>
+        linux-block@vger.kernel.org, linux-fsdevel@vger.kernel.org
+Subject: [PATCH AUTOSEL 5.1 006/375] io_uring: use cpu_online() to check p->sq_thread_cpu instead of cpu_possible()
+Date:   Wed, 22 May 2019 15:15:06 -0400
+Message-Id: <20190522192115.22666-6-sashal@kernel.org>
 X-Mailer: git-send-email 2.20.1
 In-Reply-To: <20190522192115.22666-1-sashal@kernel.org>
 References: <20190522192115.22666-1-sashal@kernel.org>
@@ -65,91 +58,55 @@
 List-ID: <linux-block.vger.kernel.org>
 X-Mailing-List: linux-block@vger.kernel.org
 
-From: Ming Lei <ming.lei@redhat.com>
+From: Shenghui Wang <shhuiw@foxmail.com>
 
-[ Upstream commit e87eb301bee183d82bb3d04bd71b6660889a2588 ]
+[ Upstream commit 7889f44dd9cee15aff1c3f7daf81ca4dfed48fc7 ]
 
-Just like aio/io_uring, we need to grab 2 refcount for queuing one
-request, one is for submission, another is for completion.
+This issue is found by running liburing/test/io_uring_setup test.
 
-If the request isn't queued from plug code path, the refcount grabbed
-in generic_make_request() serves for submission. In theroy, this
-refcount should have been released after the sumission(async run queue)
-is done. blk_freeze_queue() works with blk_sync_queue() together
-for avoiding race between cleanup queue and IO submission, given async
-run queue activities are canceled because hctx->run_work is scheduled with
-the refcount held, so it is fine to not hold the refcount when
-running the run queue work function for dispatch IO.
+When test run, the testcase "attempt to bind to invalid cpu" would not
+pass with messages like:
+   io_uring_setup(1, 0xbfc2f7c8), \
+flags: IORING_SETUP_SQPOLL|IORING_SETUP_SQ_AFF, \
+resv: 0x00000000 0x00000000 0x00000000 0x00000000 0x00000000, \
+sq_thread_cpu: 2
+   expected -1, got 3
+   FAIL
 
-However, if request is staggered into plug list, and finally queued
-from plug code path, the refcount in submission side is actually missed.
-And we may start to run queue after queue is removed because the queue's
-kobject refcount isn't guaranteed to be grabbed in flushing plug list
-context, then kernel oops is triggered, see the following race:
+On my system, there is:
+   CPU(s) possible : 0-3
+   CPU(s) online   : 0-1
+   CPU(s) offline  : 2-3
+   CPU(s) present  : 0-1
 
-blk_mq_flush_plug_list():
-        blk_mq_sched_insert_requests()
-                insert requests to sw queue or scheduler queue
-                blk_mq_run_hw_queue
+The sq_thread_cpu 2 is offline on my system, so the bind should fail.
+But cpu_possible() will pass the check. We shouldn't be able to bind
+to an offline cpu. Use cpu_online() to do the check.
 
-Because of concurrent run queue, all requests inserted above may be
-completed before calling the above blk_mq_run_hw_queue. Then queue can
-be freed during the above blk_mq_run_hw_queue().
+After the change, the testcase run as expected: EINVAL will be returned
+for cpu offlined.
 
-Fixes the issue by grab .q_usage_counter before calling
-blk_mq_sched_insert_requests() in blk_mq_flush_plug_list(). This way is
-safe because the queue is absolutely alive before inserting request.
-
-Cc: Dongli Zhang <dongli.zhang@oracle.com>
-Cc: James Smart <james.smart@broadcom.com>
-Cc: linux-scsi@vger.kernel.org,
-Cc: Martin K . Petersen <martin.petersen@oracle.com>,
-Cc: Christoph Hellwig <hch@lst.de>,
-Cc: James E . J . Bottomley <jejb@linux.vnet.ibm.com>,
-Reviewed-by: Bart Van Assche <bvanassche@acm.org>
-Tested-by: James Smart <james.smart@broadcom.com>
-Signed-off-by: Ming Lei <ming.lei@redhat.com>
+Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
+Signed-off-by: Shenghui Wang <shhuiw@foxmail.com>
 Signed-off-by: Jens Axboe <axboe@kernel.dk>
 Signed-off-by: Sasha Levin <sashal@kernel.org>
 ---
- block/blk-mq-sched.c | 12 +++++++++++-
- 1 file changed, 11 insertions(+), 1 deletion(-)
+ fs/io_uring.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
 
-diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
-index aa6bc5c026438..c59babca6857a 100644
---- a/block/blk-mq-sched.c
-+++ b/block/blk-mq-sched.c
-@@ -413,6 +413,14 @@ void blk_mq_sched_insert_requests(struct blk_mq_hw_ctx *hctx,
- 				  struct list_head *list, bool run_queue_async)
- {
- 	struct elevator_queue *e;
-+	struct request_queue *q = hctx->queue;
-+
-+	/*
-+	 * blk_mq_sched_insert_requests() is called from flush plug
-+	 * context only, and hold one usage counter to prevent queue
-+	 * from being released.
-+	 */
-+	percpu_ref_get(&q->q_usage_counter);
+diff --git a/fs/io_uring.c b/fs/io_uring.c
+index 84efb8956734f..30a5687a17b65 100644
+--- a/fs/io_uring.c
++++ b/fs/io_uring.c
+@@ -2334,7 +2334,7 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx,
+ 							nr_cpu_ids);
  
- 	e = hctx->queue->elevator;
- 	if (e && e->type->ops.insert_requests)
-@@ -426,12 +434,14 @@ void blk_mq_sched_insert_requests(struct blk_mq_hw_ctx *hctx,
- 		if (!hctx->dispatch_busy && !e && !run_queue_async) {
- 			blk_mq_try_issue_list_directly(hctx, list);
- 			if (list_empty(list))
--				return;
-+				goto out;
- 		}
- 		blk_mq_insert_requests(hctx, ctx, list);
- 	}
+ 			ret = -EINVAL;
+-			if (!cpu_possible(cpu))
++			if (!cpu_online(cpu))
+ 				goto err;
  
- 	blk_mq_run_hw_queue(hctx, run_queue_async);
-+ out:
-+	percpu_ref_put(&q->q_usage_counter);
- }
- 
- static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
+ 			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
 -- 
 2.20.1