| From 6dd35f45b8dac827b6f9dd86f5aca6436cdd2410 Mon Sep 17 00:00:00 2001 |
| From: Will Deacon <will.deacon@arm.com> |
| Date: Wed, 5 Feb 2014 17:49:34 +0000 |
| Subject: iommu/arm-smmu: fix table flushing during initial allocations |
| |
| From: Will Deacon <will.deacon@arm.com> |
| |
| commit 6dd35f45b8dac827b6f9dd86f5aca6436cdd2410 upstream. |
| |
| Now that we populate page tables as we traverse them ("iommu/arm-smmu: |
| fix pud/pmd entry fill sequence"), we need to ensure that we flush out |
| our zeroed tables after initial allocation, to prevent speculative TLB |
| fills using bogus data. |
| |
| This patch adds additional calls to arm_smmu_flush_pgtable during |
| initial table allocation, and moves the dsb required by coherent table |
| walkers into the helper. |
| |
| Signed-off-by: Will Deacon <will.deacon@arm.com> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| drivers/iommu/arm-smmu.c | 51 ++++++++++++++++++++++++----------------------- |
| 1 file changed, 27 insertions(+), 24 deletions(-) |
| |
| --- a/drivers/iommu/arm-smmu.c |
| +++ b/drivers/iommu/arm-smmu.c |
| @@ -78,7 +78,6 @@ |
| |
| #define ARM_SMMU_PTE_CONT_SIZE (PAGE_SIZE * ARM_SMMU_PTE_CONT_ENTRIES) |
| #define ARM_SMMU_PTE_CONT_MASK (~(ARM_SMMU_PTE_CONT_SIZE - 1)) |
| -#define ARM_SMMU_PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(pte_t)) |
| |
| /* Stage-1 PTE */ |
| #define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6) |
| @@ -631,6 +630,28 @@ static irqreturn_t arm_smmu_global_fault |
| return IRQ_HANDLED; |
| } |
| |
| +static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr, |
| + size_t size) |
| +{ |
| + unsigned long offset = (unsigned long)addr & ~PAGE_MASK; |
| + |
| + |
| + /* Ensure new page tables are visible to the hardware walker */ |
| + if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) { |
| + dsb(); |
| + } else { |
| + /* |
| + * If the SMMU can't walk tables in the CPU caches, treat them |
| + * like non-coherent DMA since we need to flush the new entries |
| + * all the way out to memory. There's no possibility of |
| + * recursion here as the SMMU table walker will not be wired |
| + * through another SMMU. |
| + */ |
| + dma_map_page(smmu->dev, virt_to_page(addr), offset, size, |
| + DMA_TO_DEVICE); |
| + } |
| +} |
| + |
| static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) |
| { |
| u32 reg; |
| @@ -714,6 +735,8 @@ static void arm_smmu_init_context_bank(s |
| } |
| |
| /* TTBR0 */ |
| + arm_smmu_flush_pgtable(smmu, root_cfg->pgd, |
| + PTRS_PER_PGD * sizeof(pgd_t)); |
| reg = __pa(root_cfg->pgd); |
| writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); |
| reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32; |
| @@ -1176,23 +1199,6 @@ static void arm_smmu_detach_dev(struct i |
| arm_smmu_domain_remove_master(smmu_domain, master); |
| } |
| |
| -static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr, |
| - size_t size) |
| -{ |
| - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; |
| - |
| - /* |
| - * If the SMMU can't walk tables in the CPU caches, treat them |
| - * like non-coherent DMA since we need to flush the new entries |
| - * all the way out to memory. There's no possibility of recursion |
| - * here as the SMMU table walker will not be wired through another |
| - * SMMU. |
| - */ |
| - if (!(smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)) |
| - dma_map_page(smmu->dev, virt_to_page(addr), offset, size, |
| - DMA_TO_DEVICE); |
| -} |
| - |
| static bool arm_smmu_pte_is_contiguous_range(unsigned long addr, |
| unsigned long end) |
| { |
| @@ -1213,8 +1219,7 @@ static int arm_smmu_alloc_init_pte(struc |
| if (!table) |
| return -ENOMEM; |
| |
| - arm_smmu_flush_pgtable(smmu, page_address(table), |
| - ARM_SMMU_PTE_HWTABLE_SIZE); |
| + arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE); |
| if (!pgtable_page_ctor(table)) { |
| __free_page(table); |
| return -ENOMEM; |
| @@ -1318,6 +1323,7 @@ static int arm_smmu_alloc_init_pmd(struc |
| if (!pmd) |
| return -ENOMEM; |
| |
| + arm_smmu_flush_pgtable(smmu, pmd, PAGE_SIZE); |
| pud_populate(NULL, pud, pmd); |
| arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud)); |
| |
| @@ -1350,6 +1356,7 @@ static int arm_smmu_alloc_init_pud(struc |
| if (!pud) |
| return -ENOMEM; |
| |
| + arm_smmu_flush_pgtable(smmu, pud, PAGE_SIZE); |
| pgd_populate(NULL, pgd, pud); |
| arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd)); |
| |
| @@ -1418,10 +1425,6 @@ static int arm_smmu_handle_mapping(struc |
| out_unlock: |
| spin_unlock(&smmu_domain->lock); |
| |
| - /* Ensure new page tables are visible to the hardware walker */ |
| - if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) |
| - dsb(); |
| - |
| return ret; |
| } |
| |