| // SPDX-License-Identifier: GPL-2.0-only |
| /* |
| * Kernel-based Virtual Machine driver for Linux |
| * |
| * AMD SVM support |
| * |
| * Copyright (C) 2006 Qumranet, Inc. |
| * Copyright 2010 Red Hat, Inc. and/or its affiliates. |
| * |
| * Authors: |
| * Yaniv Kamay <yaniv@qumranet.com> |
| * Avi Kivity <avi@qumranet.com> |
| */ |
| |
| #define pr_fmt(fmt) "SVM: " fmt |
| |
| #include <linux/kvm_types.h> |
| #include <linux/hashtable.h> |
| #include <linux/amd-iommu.h> |
| #include <linux/kvm_host.h> |
| |
| #include <asm/irq_remapping.h> |
| |
| #include "trace.h" |
| #include "lapic.h" |
| #include "x86.h" |
| #include "irq.h" |
| #include "svm.h" |
| |
| /* enable / disable AVIC */ |
| int avic; |
| #ifdef CONFIG_X86_LOCAL_APIC |
| module_param(avic, int, S_IRUGO); |
| #endif |
| |
| /* AVIC GATAG is encoded using VM and VCPU IDs */ |
| #define AVIC_VCPU_ID_BITS 8 |
| #define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) |
| |
| #define AVIC_VM_ID_BITS 24 |
| #define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) |
| #define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) |
| |
| #define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ |
| (y & AVIC_VCPU_ID_MASK)) |
| #define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) |
| #define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) |
| |
| /* Note: |
| * This hash table is used to map VM_ID to a struct kvm_svm, |
| * when handling AMD IOMMU GALOG notification to schedule in |
| * a particular vCPU. |
| */ |
| #define SVM_VM_DATA_HASH_BITS 8 |
| static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); |
| static u32 next_vm_id = 0; |
| static bool next_vm_id_wrapped = 0; |
| static DEFINE_SPINLOCK(svm_vm_data_hash_lock); |
| |
| /* |
| * This is a wrapper of struct amd_iommu_ir_data. |
| */ |
| struct amd_svm_iommu_ir { |
| struct list_head node; /* Used by SVM for per-vcpu ir_list */ |
| void *data; /* Storing pointer to struct amd_ir_data */ |
| }; |
| |
| |
| /* Note: |
| * This function is called from IOMMU driver to notify |
| * SVM to schedule in a particular vCPU of a particular VM. |
| */ |
| int avic_ga_log_notifier(u32 ga_tag) |
| { |
| unsigned long flags; |
| struct kvm_svm *kvm_svm; |
| struct kvm_vcpu *vcpu = NULL; |
| u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); |
| u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); |
| |
| pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); |
| trace_kvm_avic_ga_log(vm_id, vcpu_id); |
| |
| spin_lock_irqsave(&svm_vm_data_hash_lock, flags); |
| hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { |
| if (kvm_svm->avic_vm_id != vm_id) |
| continue; |
| vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); |
| break; |
| } |
| spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); |
| |
| /* Note: |
| * At this point, the IOMMU should have already set the pending |
| * bit in the vAPIC backing page. So, we just need to schedule |
| * in the vcpu. |
| */ |
| if (vcpu) |
| kvm_vcpu_wake_up(vcpu); |
| |
| return 0; |
| } |
| |
| void avic_vm_destroy(struct kvm *kvm) |
| { |
| unsigned long flags; |
| struct kvm_svm *kvm_svm = to_kvm_svm(kvm); |
| |
| if (!avic) |
| return; |
| |
| if (kvm_svm->avic_logical_id_table_page) |
| __free_page(kvm_svm->avic_logical_id_table_page); |
| if (kvm_svm->avic_physical_id_table_page) |
| __free_page(kvm_svm->avic_physical_id_table_page); |
| |
| spin_lock_irqsave(&svm_vm_data_hash_lock, flags); |
| hash_del(&kvm_svm->hnode); |
| spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); |
| } |
| |
| int avic_vm_init(struct kvm *kvm) |
| { |
| unsigned long flags; |
| int err = -ENOMEM; |
| struct kvm_svm *kvm_svm = to_kvm_svm(kvm); |
| struct kvm_svm *k2; |
| struct page *p_page; |
| struct page *l_page; |
| u32 vm_id; |
| |
| if (!avic) |
| return 0; |
| |
| /* Allocating physical APIC ID table (4KB) */ |
| p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); |
| if (!p_page) |
| goto free_avic; |
| |
| kvm_svm->avic_physical_id_table_page = p_page; |
| |
| /* Allocating logical APIC ID table (4KB) */ |
| l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); |
| if (!l_page) |
| goto free_avic; |
| |
| kvm_svm->avic_logical_id_table_page = l_page; |
| |
| spin_lock_irqsave(&svm_vm_data_hash_lock, flags); |
| again: |
| vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; |
| if (vm_id == 0) { /* id is 1-based, zero is not okay */ |
| next_vm_id_wrapped = 1; |
| goto again; |
| } |
| /* Is it still in use? Only possible if wrapped at least once */ |
| if (next_vm_id_wrapped) { |
| hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { |
| if (k2->avic_vm_id == vm_id) |
| goto again; |
| } |
| } |
| kvm_svm->avic_vm_id = vm_id; |
| hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); |
| spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); |
| |
| return 0; |
| |
| free_avic: |
| avic_vm_destroy(kvm); |
| return err; |
| } |
| |
| void avic_init_vmcb(struct vcpu_svm *svm) |
| { |
| struct vmcb *vmcb = svm->vmcb; |
| struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); |
| phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); |
| phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); |
| phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); |
| |
| vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; |
| vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; |
| vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; |
| vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; |
| if (kvm_apicv_activated(svm->vcpu.kvm)) |
| vmcb->control.int_ctl |= AVIC_ENABLE_MASK; |
| else |
| vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; |
| } |
| |
| static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, |
| unsigned int index) |
| { |
| u64 *avic_physical_id_table; |
| struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); |
| |
| if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) |
| return NULL; |
| |
| avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); |
| |
| return &avic_physical_id_table[index]; |
| } |
| |
| /** |
| * Note: |
| * AVIC hardware walks the nested page table to check permissions, |
| * but does not use the SPA address specified in the leaf page |
| * table entry since it uses address in the AVIC_BACKING_PAGE pointer |
| * field of the VMCB. Therefore, we set up the |
| * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. |
| */ |
| static int avic_update_access_page(struct kvm *kvm, bool activate) |
| { |
| int ret = 0; |
| |
| mutex_lock(&kvm->slots_lock); |
| /* |
| * During kvm_destroy_vm(), kvm_pit_set_reinject() could trigger |
| * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT |
| * memory region. So, we need to ensure that kvm->mm == current->mm. |
| */ |
| if ((kvm->arch.apic_access_page_done == activate) || |
| (kvm->mm != current->mm)) |
| goto out; |
| |
| ret = __x86_set_memory_region(kvm, |
| APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, |
| APIC_DEFAULT_PHYS_BASE, |
| activate ? PAGE_SIZE : 0); |
| if (ret) |
| goto out; |
| |
| kvm->arch.apic_access_page_done = activate; |
| out: |
| mutex_unlock(&kvm->slots_lock); |
| return ret; |
| } |
| |
| static int avic_init_backing_page(struct kvm_vcpu *vcpu) |
| { |
| u64 *entry, new_entry; |
| int id = vcpu->vcpu_id; |
| struct vcpu_svm *svm = to_svm(vcpu); |
| |
| if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) |
| return -EINVAL; |
| |
| if (!svm->vcpu.arch.apic->regs) |
| return -EINVAL; |
| |
| if (kvm_apicv_activated(vcpu->kvm)) { |
| int ret; |
| |
| ret = avic_update_access_page(vcpu->kvm, true); |
| if (ret) |
| return ret; |
| } |
| |
| svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs); |
| |
| /* Setting AVIC backing page address in the phy APIC ID table */ |
| entry = avic_get_physical_id_entry(vcpu, id); |
| if (!entry) |
| return -EINVAL; |
| |
| new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & |
| AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | |
| AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); |
| WRITE_ONCE(*entry, new_entry); |
| |
| svm->avic_physical_id_cache = entry; |
| |
| return 0; |
| } |
| |
| int avic_incomplete_ipi_interception(struct vcpu_svm *svm) |
| { |
| u32 icrh = svm->vmcb->control.exit_info_1 >> 32; |
| u32 icrl = svm->vmcb->control.exit_info_1; |
| u32 id = svm->vmcb->control.exit_info_2 >> 32; |
| u32 index = svm->vmcb->control.exit_info_2 & 0xFF; |
| struct kvm_lapic *apic = svm->vcpu.arch.apic; |
| |
| trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index); |
| |
| switch (id) { |
| case AVIC_IPI_FAILURE_INVALID_INT_TYPE: |
| /* |
| * AVIC hardware handles the generation of |
| * IPIs when the specified Message Type is Fixed |
| * (also known as fixed delivery mode) and |
| * the Trigger Mode is edge-triggered. The hardware |
| * also supports self and broadcast delivery modes |
| * specified via the Destination Shorthand(DSH) |
| * field of the ICRL. Logical and physical APIC ID |
| * formats are supported. All other IPI types cause |
| * a #VMEXIT, which needs to emulated. |
| */ |
| kvm_lapic_reg_write(apic, APIC_ICR2, icrh); |
| kvm_lapic_reg_write(apic, APIC_ICR, icrl); |
| break; |
| case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: { |
| int i; |
| struct kvm_vcpu *vcpu; |
| struct kvm *kvm = svm->vcpu.kvm; |
| struct kvm_lapic *apic = svm->vcpu.arch.apic; |
| |
| /* |
| * At this point, we expect that the AVIC HW has already |
| * set the appropriate IRR bits on the valid target |
| * vcpus. So, we just need to kick the appropriate vcpu. |
| */ |
| kvm_for_each_vcpu(i, vcpu, kvm) { |
| bool m = kvm_apic_match_dest(vcpu, apic, |
| icrl & APIC_SHORT_MASK, |
| GET_APIC_DEST_FIELD(icrh), |
| icrl & APIC_DEST_MASK); |
| |
| if (m && !avic_vcpu_is_running(vcpu)) |
| kvm_vcpu_wake_up(vcpu); |
| } |
| break; |
| } |
| case AVIC_IPI_FAILURE_INVALID_TARGET: |
| break; |
| case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: |
| WARN_ONCE(1, "Invalid backing page\n"); |
| break; |
| default: |
| pr_err("Unknown IPI interception\n"); |
| } |
| |
| return 1; |
| } |
| |
| static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) |
| { |
| struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); |
| int index; |
| u32 *logical_apic_id_table; |
| int dlid = GET_APIC_LOGICAL_ID(ldr); |
| |
| if (!dlid) |
| return NULL; |
| |
| if (flat) { /* flat */ |
| index = ffs(dlid) - 1; |
| if (index > 7) |
| return NULL; |
| } else { /* cluster */ |
| int cluster = (dlid & 0xf0) >> 4; |
| int apic = ffs(dlid & 0x0f) - 1; |
| |
| if ((apic < 0) || (apic > 7) || |
| (cluster >= 0xf)) |
| return NULL; |
| index = (cluster << 2) + apic; |
| } |
| |
| logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); |
| |
| return &logical_apic_id_table[index]; |
| } |
| |
| static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) |
| { |
| bool flat; |
| u32 *entry, new_entry; |
| |
| flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; |
| entry = avic_get_logical_id_entry(vcpu, ldr, flat); |
| if (!entry) |
| return -EINVAL; |
| |
| new_entry = READ_ONCE(*entry); |
| new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; |
| new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); |
| new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; |
| WRITE_ONCE(*entry, new_entry); |
| |
| return 0; |
| } |
| |
| static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_svm *svm = to_svm(vcpu); |
| bool flat = svm->dfr_reg == APIC_DFR_FLAT; |
| u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); |
| |
| if (entry) |
| clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); |
| } |
| |
| static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) |
| { |
| int ret = 0; |
| struct vcpu_svm *svm = to_svm(vcpu); |
| u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); |
| u32 id = kvm_xapic_id(vcpu->arch.apic); |
| |
| if (ldr == svm->ldr_reg) |
| return 0; |
| |
| avic_invalidate_logical_id_entry(vcpu); |
| |
| if (ldr) |
| ret = avic_ldr_write(vcpu, id, ldr); |
| |
| if (!ret) |
| svm->ldr_reg = ldr; |
| |
| return ret; |
| } |
| |
| static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) |
| { |
| u64 *old, *new; |
| struct vcpu_svm *svm = to_svm(vcpu); |
| u32 id = kvm_xapic_id(vcpu->arch.apic); |
| |
| if (vcpu->vcpu_id == id) |
| return 0; |
| |
| old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); |
| new = avic_get_physical_id_entry(vcpu, id); |
| if (!new || !old) |
| return 1; |
| |
| /* We need to move physical_id_entry to new offset */ |
| *new = *old; |
| *old = 0ULL; |
| to_svm(vcpu)->avic_physical_id_cache = new; |
| |
| /* |
| * Also update the guest physical APIC ID in the logical |
| * APIC ID table entry if already setup the LDR. |
| */ |
| if (svm->ldr_reg) |
| avic_handle_ldr_update(vcpu); |
| |
| return 0; |
| } |
| |
| static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_svm *svm = to_svm(vcpu); |
| u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); |
| |
| if (svm->dfr_reg == dfr) |
| return; |
| |
| avic_invalidate_logical_id_entry(vcpu); |
| svm->dfr_reg = dfr; |
| } |
| |
| static int avic_unaccel_trap_write(struct vcpu_svm *svm) |
| { |
| struct kvm_lapic *apic = svm->vcpu.arch.apic; |
| u32 offset = svm->vmcb->control.exit_info_1 & |
| AVIC_UNACCEL_ACCESS_OFFSET_MASK; |
| |
| switch (offset) { |
| case APIC_ID: |
| if (avic_handle_apic_id_update(&svm->vcpu)) |
| return 0; |
| break; |
| case APIC_LDR: |
| if (avic_handle_ldr_update(&svm->vcpu)) |
| return 0; |
| break; |
| case APIC_DFR: |
| avic_handle_dfr_update(&svm->vcpu); |
| break; |
| default: |
| break; |
| } |
| |
| kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); |
| |
| return 1; |
| } |
| |
| static bool is_avic_unaccelerated_access_trap(u32 offset) |
| { |
| bool ret = false; |
| |
| switch (offset) { |
| case APIC_ID: |
| case APIC_EOI: |
| case APIC_RRR: |
| case APIC_LDR: |
| case APIC_DFR: |
| case APIC_SPIV: |
| case APIC_ESR: |
| case APIC_ICR: |
| case APIC_LVTT: |
| case APIC_LVTTHMR: |
| case APIC_LVTPC: |
| case APIC_LVT0: |
| case APIC_LVT1: |
| case APIC_LVTERR: |
| case APIC_TMICT: |
| case APIC_TDCR: |
| ret = true; |
| break; |
| default: |
| break; |
| } |
| return ret; |
| } |
| |
| int avic_unaccelerated_access_interception(struct vcpu_svm *svm) |
| { |
| int ret = 0; |
| u32 offset = svm->vmcb->control.exit_info_1 & |
| AVIC_UNACCEL_ACCESS_OFFSET_MASK; |
| u32 vector = svm->vmcb->control.exit_info_2 & |
| AVIC_UNACCEL_ACCESS_VECTOR_MASK; |
| bool write = (svm->vmcb->control.exit_info_1 >> 32) & |
| AVIC_UNACCEL_ACCESS_WRITE_MASK; |
| bool trap = is_avic_unaccelerated_access_trap(offset); |
| |
| trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset, |
| trap, write, vector); |
| if (trap) { |
| /* Handling Trap */ |
| WARN_ONCE(!write, "svm: Handling trap read.\n"); |
| ret = avic_unaccel_trap_write(svm); |
| } else { |
| /* Handling Fault */ |
| ret = kvm_emulate_instruction(&svm->vcpu, 0); |
| } |
| |
| return ret; |
| } |
| |
| int avic_init_vcpu(struct vcpu_svm *svm) |
| { |
| int ret; |
| struct kvm_vcpu *vcpu = &svm->vcpu; |
| |
| if (!avic || !irqchip_in_kernel(vcpu->kvm)) |
| return 0; |
| |
| ret = avic_init_backing_page(&svm->vcpu); |
| if (ret) |
| return ret; |
| |
| INIT_LIST_HEAD(&svm->ir_list); |
| spin_lock_init(&svm->ir_list_lock); |
| svm->dfr_reg = APIC_DFR_FLAT; |
| |
| return ret; |
| } |
| |
| void avic_post_state_restore(struct kvm_vcpu *vcpu) |
| { |
| if (avic_handle_apic_id_update(vcpu) != 0) |
| return; |
| avic_handle_dfr_update(vcpu); |
| avic_handle_ldr_update(vcpu); |
| } |
| |
| void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate) |
| { |
| if (!avic || !lapic_in_kernel(vcpu)) |
| return; |
| |
| srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); |
| kvm_request_apicv_update(vcpu->kvm, activate, |
| APICV_INHIBIT_REASON_IRQWIN); |
| vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); |
| } |
| |
| void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu) |
| { |
| return; |
| } |
| |
| void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) |
| { |
| } |
| |
| void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) |
| { |
| } |
| |
| static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) |
| { |
| int ret = 0; |
| unsigned long flags; |
| struct amd_svm_iommu_ir *ir; |
| struct vcpu_svm *svm = to_svm(vcpu); |
| |
| if (!kvm_arch_has_assigned_device(vcpu->kvm)) |
| return 0; |
| |
| /* |
| * Here, we go through the per-vcpu ir_list to update all existing |
| * interrupt remapping table entry targeting this vcpu. |
| */ |
| spin_lock_irqsave(&svm->ir_list_lock, flags); |
| |
| if (list_empty(&svm->ir_list)) |
| goto out; |
| |
| list_for_each_entry(ir, &svm->ir_list, node) { |
| if (activate) |
| ret = amd_iommu_activate_guest_mode(ir->data); |
| else |
| ret = amd_iommu_deactivate_guest_mode(ir->data); |
| if (ret) |
| break; |
| } |
| out: |
| spin_unlock_irqrestore(&svm->ir_list_lock, flags); |
| return ret; |
| } |
| |
| void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) |
| { |
| struct vcpu_svm *svm = to_svm(vcpu); |
| struct vmcb *vmcb = svm->vmcb; |
| bool activated = kvm_vcpu_apicv_active(vcpu); |
| |
| if (!avic) |
| return; |
| |
| if (activated) { |
| /** |
| * During AVIC temporary deactivation, guest could update |
| * APIC ID, DFR and LDR registers, which would not be trapped |
| * by avic_unaccelerated_access_interception(). In this case, |
| * we need to check and update the AVIC logical APIC ID table |
| * accordingly before re-activating. |
| */ |
| avic_post_state_restore(vcpu); |
| vmcb->control.int_ctl |= AVIC_ENABLE_MASK; |
| } else { |
| vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; |
| } |
| vmcb_mark_dirty(vmcb, VMCB_AVIC); |
| |
| svm_set_pi_irte_mode(vcpu, activated); |
| } |
| |
| void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) |
| { |
| return; |
| } |
| |
| int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) |
| { |
| if (!vcpu->arch.apicv_active) |
| return -1; |
| |
| kvm_lapic_set_irr(vec, vcpu->arch.apic); |
| smp_mb__after_atomic(); |
| |
| if (avic_vcpu_is_running(vcpu)) { |
| int cpuid = vcpu->cpu; |
| |
| if (cpuid != get_cpu()) |
| wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid)); |
| put_cpu(); |
| } else |
| kvm_vcpu_wake_up(vcpu); |
| |
| return 0; |
| } |
| |
| bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) |
| { |
| return false; |
| } |
| |
| static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) |
| { |
| unsigned long flags; |
| struct amd_svm_iommu_ir *cur; |
| |
| spin_lock_irqsave(&svm->ir_list_lock, flags); |
| list_for_each_entry(cur, &svm->ir_list, node) { |
| if (cur->data != pi->ir_data) |
| continue; |
| list_del(&cur->node); |
| kfree(cur); |
| break; |
| } |
| spin_unlock_irqrestore(&svm->ir_list_lock, flags); |
| } |
| |
| static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) |
| { |
| int ret = 0; |
| unsigned long flags; |
| struct amd_svm_iommu_ir *ir; |
| |
| /** |
| * In some cases, the existing irte is updaed and re-set, |
| * so we need to check here if it's already been * added |
| * to the ir_list. |
| */ |
| if (pi->ir_data && (pi->prev_ga_tag != 0)) { |
| struct kvm *kvm = svm->vcpu.kvm; |
| u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); |
| struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); |
| struct vcpu_svm *prev_svm; |
| |
| if (!prev_vcpu) { |
| ret = -EINVAL; |
| goto out; |
| } |
| |
| prev_svm = to_svm(prev_vcpu); |
| svm_ir_list_del(prev_svm, pi); |
| } |
| |
| /** |
| * Allocating new amd_iommu_pi_data, which will get |
| * add to the per-vcpu ir_list. |
| */ |
| ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); |
| if (!ir) { |
| ret = -ENOMEM; |
| goto out; |
| } |
| ir->data = pi->ir_data; |
| |
| spin_lock_irqsave(&svm->ir_list_lock, flags); |
| list_add(&ir->node, &svm->ir_list); |
| spin_unlock_irqrestore(&svm->ir_list_lock, flags); |
| out: |
| return ret; |
| } |
| |
| /** |
| * Note: |
| * The HW cannot support posting multicast/broadcast |
| * interrupts to a vCPU. So, we still use legacy interrupt |
| * remapping for these kind of interrupts. |
| * |
| * For lowest-priority interrupts, we only support |
| * those with single CPU as the destination, e.g. user |
| * configures the interrupts via /proc/irq or uses |
| * irqbalance to make the interrupts single-CPU. |
| */ |
| static int |
| get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, |
| struct vcpu_data *vcpu_info, struct vcpu_svm **svm) |
| { |
| struct kvm_lapic_irq irq; |
| struct kvm_vcpu *vcpu = NULL; |
| |
| kvm_set_msi_irq(kvm, e, &irq); |
| |
| if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || |
| !kvm_irq_is_postable(&irq)) { |
| pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", |
| __func__, irq.vector); |
| return -1; |
| } |
| |
| pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, |
| irq.vector); |
| *svm = to_svm(vcpu); |
| vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); |
| vcpu_info->vector = irq.vector; |
| |
| return 0; |
| } |
| |
| /* |
| * svm_update_pi_irte - set IRTE for Posted-Interrupts |
| * |
| * @kvm: kvm |
| * @host_irq: host irq of the interrupt |
| * @guest_irq: gsi of the interrupt |
| * @set: set or unset PI |
| * returns 0 on success, < 0 on failure |
| */ |
| int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, |
| uint32_t guest_irq, bool set) |
| { |
| struct kvm_kernel_irq_routing_entry *e; |
| struct kvm_irq_routing_table *irq_rt; |
| int idx, ret = 0; |
| |
| if (!kvm_arch_has_assigned_device(kvm) || |
| !irq_remapping_cap(IRQ_POSTING_CAP)) |
| return 0; |
| |
| pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", |
| __func__, host_irq, guest_irq, set); |
| |
| idx = srcu_read_lock(&kvm->irq_srcu); |
| irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); |
| |
| if (guest_irq >= irq_rt->nr_rt_entries || |
| hlist_empty(&irq_rt->map[guest_irq])) { |
| pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", |
| guest_irq, irq_rt->nr_rt_entries); |
| goto out; |
| } |
| |
| hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { |
| struct vcpu_data vcpu_info; |
| struct vcpu_svm *svm = NULL; |
| |
| if (e->type != KVM_IRQ_ROUTING_MSI) |
| continue; |
| |
| /** |
| * Here, we setup with legacy mode in the following cases: |
| * 1. When cannot target interrupt to a specific vcpu. |
| * 2. Unsetting posted interrupt. |
| * 3. APIC virtialization is disabled for the vcpu. |
| * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) |
| */ |
| if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && |
| kvm_vcpu_apicv_active(&svm->vcpu)) { |
| struct amd_iommu_pi_data pi; |
| |
| /* Try to enable guest_mode in IRTE */ |
| pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & |
| AVIC_HPA_MASK); |
| pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, |
| svm->vcpu.vcpu_id); |
| pi.is_guest_mode = true; |
| pi.vcpu_data = &vcpu_info; |
| ret = irq_set_vcpu_affinity(host_irq, &pi); |
| |
| /** |
| * Here, we successfully setting up vcpu affinity in |
| * IOMMU guest mode. Now, we need to store the posted |
| * interrupt information in a per-vcpu ir_list so that |
| * we can reference to them directly when we update vcpu |
| * scheduling information in IOMMU irte. |
| */ |
| if (!ret && pi.is_guest_mode) |
| svm_ir_list_add(svm, &pi); |
| } else { |
| /* Use legacy mode in IRTE */ |
| struct amd_iommu_pi_data pi; |
| |
| /** |
| * Here, pi is used to: |
| * - Tell IOMMU to use legacy mode for this interrupt. |
| * - Retrieve ga_tag of prior interrupt remapping data. |
| */ |
| pi.prev_ga_tag = 0; |
| pi.is_guest_mode = false; |
| ret = irq_set_vcpu_affinity(host_irq, &pi); |
| |
| /** |
| * Check if the posted interrupt was previously |
| * setup with the guest_mode by checking if the ga_tag |
| * was cached. If so, we need to clean up the per-vcpu |
| * ir_list. |
| */ |
| if (!ret && pi.prev_ga_tag) { |
| int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); |
| struct kvm_vcpu *vcpu; |
| |
| vcpu = kvm_get_vcpu_by_id(kvm, id); |
| if (vcpu) |
| svm_ir_list_del(to_svm(vcpu), &pi); |
| } |
| } |
| |
| if (!ret && svm) { |
| trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, |
| e->gsi, vcpu_info.vector, |
| vcpu_info.pi_desc_addr, set); |
| } |
| |
| if (ret < 0) { |
| pr_err("%s: failed to update PI IRTE\n", __func__); |
| goto out; |
| } |
| } |
| |
| ret = 0; |
| out: |
| srcu_read_unlock(&kvm->irq_srcu, idx); |
| return ret; |
| } |
| |
| bool svm_check_apicv_inhibit_reasons(ulong bit) |
| { |
| ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | |
| BIT(APICV_INHIBIT_REASON_HYPERV) | |
| BIT(APICV_INHIBIT_REASON_NESTED) | |
| BIT(APICV_INHIBIT_REASON_IRQWIN) | |
| BIT(APICV_INHIBIT_REASON_PIT_REINJ) | |
| BIT(APICV_INHIBIT_REASON_X2APIC); |
| |
| return supported & BIT(bit); |
| } |
| |
| void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate) |
| { |
| avic_update_access_page(kvm, activate); |
| } |
| |
| static inline int |
| avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) |
| { |
| int ret = 0; |
| unsigned long flags; |
| struct amd_svm_iommu_ir *ir; |
| struct vcpu_svm *svm = to_svm(vcpu); |
| |
| if (!kvm_arch_has_assigned_device(vcpu->kvm)) |
| return 0; |
| |
| /* |
| * Here, we go through the per-vcpu ir_list to update all existing |
| * interrupt remapping table entry targeting this vcpu. |
| */ |
| spin_lock_irqsave(&svm->ir_list_lock, flags); |
| |
| if (list_empty(&svm->ir_list)) |
| goto out; |
| |
| list_for_each_entry(ir, &svm->ir_list, node) { |
| ret = amd_iommu_update_ga(cpu, r, ir->data); |
| if (ret) |
| break; |
| } |
| out: |
| spin_unlock_irqrestore(&svm->ir_list_lock, flags); |
| return ret; |
| } |
| |
| void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) |
| { |
| u64 entry; |
| int h_physical_id = kvm_cpu_get_apicid(cpu); |
| struct vcpu_svm *svm = to_svm(vcpu); |
| |
| if (!kvm_vcpu_apicv_active(vcpu)) |
| return; |
| |
| if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) |
| return; |
| |
| entry = READ_ONCE(*(svm->avic_physical_id_cache)); |
| WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); |
| |
| entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; |
| entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); |
| |
| entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; |
| if (svm->avic_is_running) |
| entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; |
| |
| WRITE_ONCE(*(svm->avic_physical_id_cache), entry); |
| avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, |
| svm->avic_is_running); |
| } |
| |
| void avic_vcpu_put(struct kvm_vcpu *vcpu) |
| { |
| u64 entry; |
| struct vcpu_svm *svm = to_svm(vcpu); |
| |
| if (!kvm_vcpu_apicv_active(vcpu)) |
| return; |
| |
| entry = READ_ONCE(*(svm->avic_physical_id_cache)); |
| if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) |
| avic_update_iommu_vcpu_affinity(vcpu, -1, 0); |
| |
| entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; |
| WRITE_ONCE(*(svm->avic_physical_id_cache), entry); |
| } |
| |
| /** |
| * This function is called during VCPU halt/unhalt. |
| */ |
| static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) |
| { |
| struct vcpu_svm *svm = to_svm(vcpu); |
| |
| svm->avic_is_running = is_run; |
| if (is_run) |
| avic_vcpu_load(vcpu, vcpu->cpu); |
| else |
| avic_vcpu_put(vcpu); |
| } |
| |
| void svm_vcpu_blocking(struct kvm_vcpu *vcpu) |
| { |
| avic_set_running(vcpu, false); |
| } |
| |
| void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) |
| { |
| if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) |
| kvm_vcpu_update_apicv(vcpu); |
| avic_set_running(vcpu, true); |
| } |