| From cf6f0b2a460992f6cf2a151842a24e3fd3700374 Mon Sep 17 00:00:00 2001 |
| From: Sasha Levin <sashal@kernel.org> |
| Date: Mon, 4 May 2020 14:54:12 +0200 |
| Subject: iommu/amd: Update Device Table in increase_address_space() |
| |
| From: Joerg Roedel <jroedel@suse.de> |
| |
| [ Upstream commit 19c6978fba68a2cdedee7d55fb8c3063d47982d9 ] |
| |
| The Device Table needs to be updated before the new page-table root |
| can be published in domain->pt_root. Otherwise a concurrent call to |
| fetch_pte might fetch a PTE which is not reachable through the Device |
| Table Entry. |
| |
| Fixes: 92d420ec028d ("iommu/amd: Relax locking in dma_ops path") |
| Reported-by: Qian Cai <cai@lca.pw> |
| Signed-off-by: Joerg Roedel <jroedel@suse.de> |
| Tested-by: Qian Cai <cai@lca.pw> |
| Link: https://lore.kernel.org/r/20200504125413.16798-5-joro@8bytes.org |
| Signed-off-by: Joerg Roedel <jroedel@suse.de> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| drivers/iommu/amd_iommu.c | 49 ++++++++++++++++++++++++++++----------- |
| 1 file changed, 36 insertions(+), 13 deletions(-) |
| |
| diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c |
| index 28229a38af4d2..500d0a8c966fc 100644 |
| --- a/drivers/iommu/amd_iommu.c |
| +++ b/drivers/iommu/amd_iommu.c |
| @@ -101,6 +101,8 @@ struct kmem_cache *amd_iommu_irq_cache; |
| static void update_domain(struct protection_domain *domain); |
| static int protection_domain_init(struct protection_domain *domain); |
| static void detach_device(struct device *dev); |
| +static void update_and_flush_device_table(struct protection_domain *domain, |
| + struct domain_pgtable *pgtable); |
| |
| /**************************************************************************** |
| * |
| @@ -1461,8 +1463,16 @@ static bool increase_address_space(struct protection_domain *domain, |
| |
| *pte = PM_LEVEL_PDE(pgtable.mode, iommu_virt_to_phys(pgtable.root)); |
| |
| - root = amd_iommu_domain_encode_pgtable(pte, pgtable.mode + 1); |
| + pgtable.root = pte; |
| + pgtable.mode += 1; |
| + update_and_flush_device_table(domain, &pgtable); |
| + domain_flush_complete(domain); |
| |
| + /* |
| + * Device Table needs to be updated and flushed before the new root can |
| + * be published. |
| + */ |
| + root = amd_iommu_domain_encode_pgtable(pte, pgtable.mode); |
| atomic64_set(&domain->pt_root, root); |
| |
| ret = true; |
| @@ -1882,19 +1892,17 @@ static bool dma_ops_domain(struct protection_domain *domain) |
| } |
| |
| static void set_dte_entry(u16 devid, struct protection_domain *domain, |
| + struct domain_pgtable *pgtable, |
| bool ats, bool ppr) |
| { |
| - struct domain_pgtable pgtable; |
| u64 pte_root = 0; |
| u64 flags = 0; |
| u32 old_domid; |
| |
| - amd_iommu_domain_get_pgtable(domain, &pgtable); |
| + if (pgtable->mode != PAGE_MODE_NONE) |
| + pte_root = iommu_virt_to_phys(pgtable->root); |
| |
| - if (pgtable.mode != PAGE_MODE_NONE) |
| - pte_root = iommu_virt_to_phys(pgtable.root); |
| - |
| - pte_root |= (pgtable.mode & DEV_ENTRY_MODE_MASK) |
| + pte_root |= (pgtable->mode & DEV_ENTRY_MODE_MASK) |
| << DEV_ENTRY_MODE_SHIFT; |
| pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV; |
| |
| @@ -1967,6 +1975,7 @@ static void clear_dte_entry(u16 devid) |
| static void do_attach(struct iommu_dev_data *dev_data, |
| struct protection_domain *domain) |
| { |
| + struct domain_pgtable pgtable; |
| struct amd_iommu *iommu; |
| bool ats; |
| |
| @@ -1982,7 +1991,9 @@ static void do_attach(struct iommu_dev_data *dev_data, |
| domain->dev_cnt += 1; |
| |
| /* Update device table */ |
| - set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2); |
| + amd_iommu_domain_get_pgtable(domain, &pgtable); |
| + set_dte_entry(dev_data->devid, domain, &pgtable, |
| + ats, dev_data->iommu_v2); |
| clone_aliases(dev_data->pdev); |
| |
| device_flush_dte(dev_data); |
| @@ -2293,22 +2304,34 @@ static int amd_iommu_domain_get_attr(struct iommu_domain *domain, |
| * |
| *****************************************************************************/ |
| |
| -static void update_device_table(struct protection_domain *domain) |
| +static void update_device_table(struct protection_domain *domain, |
| + struct domain_pgtable *pgtable) |
| { |
| struct iommu_dev_data *dev_data; |
| |
| list_for_each_entry(dev_data, &domain->dev_list, list) { |
| - set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled, |
| - dev_data->iommu_v2); |
| + set_dte_entry(dev_data->devid, domain, pgtable, |
| + dev_data->ats.enabled, dev_data->iommu_v2); |
| clone_aliases(dev_data->pdev); |
| } |
| } |
| |
| +static void update_and_flush_device_table(struct protection_domain *domain, |
| + struct domain_pgtable *pgtable) |
| +{ |
| + update_device_table(domain, pgtable); |
| + domain_flush_devices(domain); |
| +} |
| + |
| static void update_domain(struct protection_domain *domain) |
| { |
| - update_device_table(domain); |
| + struct domain_pgtable pgtable; |
| |
| - domain_flush_devices(domain); |
| + /* Update device table */ |
| + amd_iommu_domain_get_pgtable(domain, &pgtable); |
| + update_and_flush_device_table(domain, &pgtable); |
| + |
| + /* Flush domain TLB(s) and wait for completion */ |
| domain_flush_tlb_pde(domain); |
| } |
| |
| -- |
| 2.20.1 |
| |