| From 249b18a8f81b80bf5bfd23a2c6a24df339600b3b Mon Sep 17 00:00:00 2001 |
| From: Boris Ostrovsky <boris.ostrovsky@oracle.com> |
| Date: Thu, 5 Dec 2019 01:30:51 +0000 |
| Subject: [PATCH] x86/kvm: Cache gfn to pfn translation |
| |
| commit 917248144db5d7320655dbb41d3af0b8a0f3d589 upstream. |
| |
| __kvm_map_gfn()'s call to gfn_to_pfn_memslot() is |
| * relatively expensive |
| * in certain cases (such as when done from atomic context) cannot be called |
| |
| Stashing gfn-to-pfn mapping should help with both cases. |
| |
| This is part of CVE-2019-3016. |
| |
| Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com> |
| Reviewed-by: Joao Martins <joao.m.martins@oracle.com> |
| Cc: stable@vger.kernel.org |
| Signed-off-by: Paolo Bonzini <pbonzini@redhat.com> |
| Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com> |
| |
| diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h |
| index 4512200c33d2..4c8704f579d4 100644 |
| --- a/arch/x86/include/asm/kvm_host.h |
| +++ b/arch/x86/include/asm/kvm_host.h |
| @@ -676,6 +676,7 @@ struct kvm_vcpu_arch { |
| u64 last_steal; |
| struct gfn_to_hva_cache stime; |
| struct kvm_steal_time steal; |
| + struct gfn_to_pfn_cache cache; |
| } st; |
| |
| u64 tsc_offset; |
| diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c |
| index 46f854c0183e..0449ca02f227 100644 |
| --- a/arch/x86/kvm/x86.c |
| +++ b/arch/x86/kvm/x86.c |
| @@ -8894,6 +8894,9 @@ static void fx_init(struct kvm_vcpu *vcpu) |
| void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) |
| { |
| void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask; |
| + struct gfn_to_pfn_cache *cache = &vcpu->arch.st.cache; |
| + |
| + kvm_release_pfn(cache->pfn, cache->dirty, cache); |
| |
| kvmclock_reset(vcpu); |
| |
| @@ -9555,11 +9558,18 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, |
| |
| void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) |
| { |
| + struct kvm_vcpu *vcpu; |
| + int i; |
| + |
| /* |
| * memslots->generation has been incremented. |
| * mmio generation may have reached its maximum value. |
| */ |
| kvm_mmu_invalidate_mmio_sptes(kvm, gen); |
| + |
| + /* Force re-initialization of steal_time cache */ |
| + kvm_for_each_vcpu(i, vcpu, kvm) |
| + kvm_vcpu_kick(vcpu); |
| } |
| |
| int kvm_arch_prepare_memory_region(struct kvm *kvm, |
| diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h |
| index aa92d553567e..b79ea2e46a1c 100644 |
| --- a/include/linux/kvm_host.h |
| +++ b/include/linux/kvm_host.h |
| @@ -727,6 +727,7 @@ void kvm_set_pfn_dirty(kvm_pfn_t pfn); |
| void kvm_set_pfn_accessed(kvm_pfn_t pfn); |
| void kvm_get_pfn(kvm_pfn_t pfn); |
| |
| +void kvm_release_pfn(kvm_pfn_t pfn, bool dirty, struct gfn_to_pfn_cache *cache); |
| int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, |
| int len); |
| int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data, |
| @@ -757,10 +758,12 @@ struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn |
| kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn); |
| kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn); |
| int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map); |
| -int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map); |
| +int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, |
| + struct gfn_to_pfn_cache *cache, bool atomic); |
| struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn); |
| void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty); |
| -int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty); |
| +int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map, |
| + struct gfn_to_pfn_cache *cache, bool dirty, bool atomic); |
| unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); |
| unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); |
| int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, |
| diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h |
| index bde5374ae021..2382cb58969d 100644 |
| --- a/include/linux/kvm_types.h |
| +++ b/include/linux/kvm_types.h |
| @@ -18,7 +18,7 @@ struct kvm_memslots; |
| |
| enum kvm_mr_change; |
| |
| -#include <asm/types.h> |
| +#include <linux/types.h> |
| |
| /* |
| * Address types: |
| @@ -49,4 +49,11 @@ struct gfn_to_hva_cache { |
| struct kvm_memory_slot *memslot; |
| }; |
| |
| +struct gfn_to_pfn_cache { |
| + u64 generation; |
| + gfn_t gfn; |
| + kvm_pfn_t pfn; |
| + bool dirty; |
| +}; |
| + |
| #endif /* __KVM_TYPES_H__ */ |
| diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c |
| index 73ef2a326a6d..f19943bfa362 100644 |
| --- a/virt/kvm/kvm_main.c |
| +++ b/virt/kvm/kvm_main.c |
| @@ -1791,27 +1791,72 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) |
| } |
| EXPORT_SYMBOL_GPL(gfn_to_page); |
| |
| +void kvm_release_pfn(kvm_pfn_t pfn, bool dirty, struct gfn_to_pfn_cache *cache) |
| +{ |
| + if (pfn == 0) |
| + return; |
| + |
| + if (cache) |
| + cache->pfn = cache->gfn = 0; |
| + |
| + if (dirty) |
| + kvm_release_pfn_dirty(pfn); |
| + else |
| + kvm_release_pfn_clean(pfn); |
| +} |
| + |
| +static void kvm_cache_gfn_to_pfn(struct kvm_memory_slot *slot, gfn_t gfn, |
| + struct gfn_to_pfn_cache *cache, u64 gen) |
| +{ |
| + kvm_release_pfn(cache->pfn, cache->dirty, cache); |
| + |
| + cache->pfn = gfn_to_pfn_memslot(slot, gfn); |
| + cache->gfn = gfn; |
| + cache->dirty = false; |
| + cache->generation = gen; |
| +} |
| + |
| static int __kvm_map_gfn(struct kvm_memslots *slots, gfn_t gfn, |
| - struct kvm_host_map *map) |
| + struct kvm_host_map *map, |
| + struct gfn_to_pfn_cache *cache, |
| + bool atomic) |
| { |
| kvm_pfn_t pfn; |
| void *hva = NULL; |
| struct page *page = KVM_UNMAPPED_PAGE; |
| struct kvm_memory_slot *slot = __gfn_to_memslot(slots, gfn); |
| + u64 gen = slots->generation; |
| |
| if (!map) |
| return -EINVAL; |
| |
| - pfn = gfn_to_pfn_memslot(slot, gfn); |
| + if (cache) { |
| + if (!cache->pfn || cache->gfn != gfn || |
| + cache->generation != gen) { |
| + if (atomic) |
| + return -EAGAIN; |
| + kvm_cache_gfn_to_pfn(slot, gfn, cache, gen); |
| + } |
| + pfn = cache->pfn; |
| + } else { |
| + if (atomic) |
| + return -EAGAIN; |
| + pfn = gfn_to_pfn_memslot(slot, gfn); |
| + } |
| if (is_error_noslot_pfn(pfn)) |
| return -EINVAL; |
| |
| if (pfn_valid(pfn)) { |
| page = pfn_to_page(pfn); |
| - hva = kmap(page); |
| + if (atomic) |
| + hva = kmap_atomic(page); |
| + else |
| + hva = kmap(page); |
| #ifdef CONFIG_HAS_IOMEM |
| - } else { |
| + } else if (!atomic) { |
| hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB); |
| + } else { |
| + return -EINVAL; |
| #endif |
| } |
| |
| @@ -1826,20 +1871,25 @@ static int __kvm_map_gfn(struct kvm_memslots *slots, gfn_t gfn, |
| return 0; |
| } |
| |
| -int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map) |
| +int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, |
| + struct gfn_to_pfn_cache *cache, bool atomic) |
| { |
| - return __kvm_map_gfn(kvm_memslots(vcpu->kvm), gfn, map); |
| + return __kvm_map_gfn(kvm_memslots(vcpu->kvm), gfn, map, |
| + cache, atomic); |
| } |
| EXPORT_SYMBOL_GPL(kvm_map_gfn); |
| |
| int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map) |
| { |
| - return __kvm_map_gfn(kvm_vcpu_memslots(vcpu), gfn, map); |
| + return __kvm_map_gfn(kvm_vcpu_memslots(vcpu), gfn, map, |
| + NULL, false); |
| } |
| EXPORT_SYMBOL_GPL(kvm_vcpu_map); |
| |
| static void __kvm_unmap_gfn(struct kvm_memory_slot *memslot, |
| - struct kvm_host_map *map, bool dirty) |
| + struct kvm_host_map *map, |
| + struct gfn_to_pfn_cache *cache, |
| + bool dirty, bool atomic) |
| { |
| if (!map) |
| return; |
| @@ -1847,34 +1897,44 @@ static void __kvm_unmap_gfn(struct kvm_memory_slot *memslot, |
| if (!map->hva) |
| return; |
| |
| - if (map->page != KVM_UNMAPPED_PAGE) |
| - kunmap(map->page); |
| + if (map->page != KVM_UNMAPPED_PAGE) { |
| + if (atomic) |
| + kunmap_atomic(map->hva); |
| + else |
| + kunmap(map->page); |
| + } |
| #ifdef CONFIG_HAS_IOMEM |
| - else |
| + else if (!atomic) |
| memunmap(map->hva); |
| + else |
| + WARN_ONCE(1, "Unexpected unmapping in atomic context"); |
| #endif |
| |
| - if (dirty) { |
| + if (dirty) |
| mark_page_dirty_in_slot(memslot, map->gfn); |
| - kvm_release_pfn_dirty(map->pfn); |
| - } else { |
| - kvm_release_pfn_clean(map->pfn); |
| - } |
| + |
| + if (cache) |
| + cache->dirty |= dirty; |
| + else |
| + kvm_release_pfn(map->pfn, dirty, NULL); |
| |
| map->hva = NULL; |
| map->page = NULL; |
| } |
| |
| -int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty) |
| +int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map, |
| + struct gfn_to_pfn_cache *cache, bool dirty, bool atomic) |
| { |
| - __kvm_unmap_gfn(gfn_to_memslot(vcpu->kvm, map->gfn), map, dirty); |
| + __kvm_unmap_gfn(gfn_to_memslot(vcpu->kvm, map->gfn), map, |
| + cache, dirty, atomic); |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(kvm_unmap_gfn); |
| |
| void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty) |
| { |
| - __kvm_unmap_gfn(kvm_vcpu_gfn_to_memslot(vcpu, map->gfn), map, dirty); |
| + __kvm_unmap_gfn(kvm_vcpu_gfn_to_memslot(vcpu, map->gfn), map, NULL, |
| + dirty, false); |
| } |
| EXPORT_SYMBOL_GPL(kvm_vcpu_unmap); |
| |
| -- |
| 2.7.4 |
| |