| From b96487432f031362e59edeed1bd9703d998d3995 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Tue, 26 Jan 2021 19:53:36 +0800 |
| Subject: dma-iommu: use static-key to minimize the impact in the fast-path |
| |
| From: Lianbo Jiang <lijiang@redhat.com> |
| |
| [ Upstream commit a8e8af35c9f4f75f981a95488c7066d31bac4bef ] |
| |
| Let's move out the is_kdump_kernel() check from iommu_dma_deferred_attach() |
| to iommu_dma_init(), and use the static-key in the fast-path to minimize |
| the impact in the normal case. |
| |
| Co-developed-by: Robin Murphy <robin.murphy@arm.com> |
| Signed-off-by: Lianbo Jiang <lijiang@redhat.com> |
| Signed-off-by: Robin Murphy <robin.murphy@arm.com> |
| Link: https://lore.kernel.org/r/20210126115337.20068-2-lijiang@redhat.com |
| Signed-off-by: Joerg Roedel <jroedel@suse.de> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| drivers/iommu/dma-iommu.c | 17 +++++++++++------ |
| 1 file changed, 11 insertions(+), 6 deletions(-) |
| |
| diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c |
| index 00fbc591a142..07e7b2f3ba27 100644 |
| --- a/drivers/iommu/dma-iommu.c |
| +++ b/drivers/iommu/dma-iommu.c |
| @@ -51,6 +51,8 @@ struct iommu_dma_cookie { |
| struct iommu_domain *fq_domain; |
| }; |
| |
| +static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled); |
| + |
| void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, |
| struct iommu_domain *domain) |
| { |
| @@ -389,9 +391,6 @@ static int iommu_dma_deferred_attach(struct device *dev, |
| { |
| const struct iommu_ops *ops = domain->ops; |
| |
| - if (!is_kdump_kernel()) |
| - return 0; |
| - |
| if (unlikely(ops->is_attach_deferred && |
| ops->is_attach_deferred(domain, dev))) |
| return iommu_attach_device(domain, dev); |
| @@ -536,7 +535,8 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, |
| size_t iova_off = iova_offset(iovad, phys); |
| dma_addr_t iova; |
| |
| - if (unlikely(iommu_dma_deferred_attach(dev, domain))) |
| + if (static_branch_unlikely(&iommu_deferred_attach_enabled) && |
| + iommu_dma_deferred_attach(dev, domain)) |
| return DMA_MAPPING_ERROR; |
| |
| size = iova_align(iovad, size + iova_off); |
| @@ -694,7 +694,8 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size, |
| |
| *dma_handle = DMA_MAPPING_ERROR; |
| |
| - if (unlikely(iommu_dma_deferred_attach(dev, domain))) |
| + if (static_branch_unlikely(&iommu_deferred_attach_enabled) && |
| + iommu_dma_deferred_attach(dev, domain)) |
| return NULL; |
| |
| min_size = alloc_sizes & -alloc_sizes; |
| @@ -977,7 +978,8 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, |
| unsigned long mask = dma_get_seg_boundary(dev); |
| int i; |
| |
| - if (unlikely(iommu_dma_deferred_attach(dev, domain))) |
| + if (static_branch_unlikely(&iommu_deferred_attach_enabled) && |
| + iommu_dma_deferred_attach(dev, domain)) |
| return 0; |
| |
| if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) |
| @@ -1425,6 +1427,9 @@ void iommu_dma_compose_msi_msg(struct msi_desc *desc, |
| |
| static int iommu_dma_init(void) |
| { |
| + if (is_kdump_kernel()) |
| + static_branch_enable(&iommu_deferred_attach_enabled); |
| + |
| return iova_cache_get(); |
| } |
| arch_initcall(iommu_dma_init); |
| -- |
| 2.30.2 |
| |