/*
 * qemu/kvm integration
 *
 * Copyright (C) 2006-2008 Qumranet Technologies
 *
 * Licensed under the terms of the GNU GPL version 2 or higher.
 */
#include "config.h"
#include "config-host.h"

#include <assert.h>
#include <string.h>
#include "hw/hw.h"
#include "sysemu.h"
#include "qemu-common.h"
#include "console.h"
#include "block.h"
#include "compatfd.h"
#include "gdbstub.h"

#include "qemu-kvm.h"
#include "libkvm.h"

#include <pthread.h>
#include <sys/utsname.h>
#include <sys/syscall.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
#include "compatfd.h"
#include <sys/prctl.h>

#define false 0
#define true 1

#ifndef PR_MCE_KILL
#define PR_MCE_KILL 33
#endif

#ifndef BUS_MCEERR_AR
#define BUS_MCEERR_AR 4
#endif
#ifndef BUS_MCEERR_AO
#define BUS_MCEERR_AO 5
#endif

#define EXPECTED_KVM_API_VERSION 12

#if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
#error libkvm: userspace and kernel version mismatch
#endif

int kvm_allowed = 1;
int kvm_irqchip = 1;
int kvm_pit = 1;
int kvm_pit_reinject = 1;
int kvm_nested = 0;


KVMState *kvm_state;
kvm_context_t kvm_context;

pthread_mutex_t qemu_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t qemu_vcpu_cond = PTHREAD_COND_INITIALIZER;
pthread_cond_t qemu_system_cond = PTHREAD_COND_INITIALIZER;
pthread_cond_t qemu_pause_cond = PTHREAD_COND_INITIALIZER;
pthread_cond_t qemu_work_cond = PTHREAD_COND_INITIALIZER;
__thread CPUState *current_env;

static int qemu_system_ready;

#define SIG_IPI (SIGRTMIN+4)

pthread_t io_thread;
static int io_thread_fd = -1;
static int io_thread_sigfd = -1;

static CPUState *kvm_debug_cpu_requested;

static uint64_t phys_ram_size;

#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
/* The list of ioperm_data */
static QLIST_HEAD(, ioperm_data) ioperm_head;
#endif

//#define DEBUG_MEMREG
#ifdef	DEBUG_MEMREG
#define DPRINTF(fmt, args...) \
	do { fprintf(stderr, "%s:%d " fmt , __func__, __LINE__, ##args); } while (0)
#else
#define DPRINTF(fmt, args...) do {} while (0)
#endif

#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))

int kvm_abi = EXPECTED_KVM_API_VERSION;
int kvm_page_size;

#ifdef KVM_CAP_SET_GUEST_DEBUG
static int kvm_debug(CPUState *env,
                     struct kvm_debug_exit_arch *arch_info)
{
    int handle = kvm_arch_debug(arch_info);

    if (handle) {
        kvm_debug_cpu_requested = env;
        env->stopped = 1;
    }
    return handle;
}
#endif

static int handle_unhandled(uint64_t reason)
{
    fprintf(stderr, "kvm: unhandled exit %" PRIx64 "\n", reason);
    return -EINVAL;
}


static inline void set_gsi(kvm_context_t kvm, unsigned int gsi)
{
    uint32_t *bitmap = kvm->used_gsi_bitmap;

    if (gsi < kvm->max_gsi)
        bitmap[gsi / 32] |= 1U << (gsi % 32);
    else
        DPRINTF("Invalid GSI %u\n", gsi);
}

static inline void clear_gsi(kvm_context_t kvm, unsigned int gsi)
{
    uint32_t *bitmap = kvm->used_gsi_bitmap;

    if (gsi < kvm->max_gsi)
        bitmap[gsi / 32] &= ~(1U << (gsi % 32));
    else
        DPRINTF("Invalid GSI %u\n", gsi);
}

struct slot_info {
    unsigned long phys_addr;
    unsigned long len;
    unsigned long userspace_addr;
    unsigned flags;
    int logging_count;
};

struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];

static void init_slots(void)
{
    int i;

    for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
        slots[i].len = 0;
}

static int get_free_slot(kvm_context_t kvm)
{
    int i;
    int tss_ext;

#if defined(KVM_CAP_SET_TSS_ADDR) && !defined(__s390__)
    tss_ext = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
#else
    tss_ext = 0;
#endif

    /*
     * on older kernels where the set tss ioctl is not supprted we must save
     * slot 0 to hold the extended memory, as the vmx will use the last 3
     * pages of this slot.
     */
    if (tss_ext > 0)
        i = 0;
    else
        i = 1;

    for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
        if (!slots[i].len)
            return i;
    return -1;
}

static void register_slot(int slot, unsigned long phys_addr,
                          unsigned long len, unsigned long userspace_addr,
                          unsigned flags)
{
    slots[slot].phys_addr = phys_addr;
    slots[slot].len = len;
    slots[slot].userspace_addr = userspace_addr;
    slots[slot].flags = flags;
}

static void free_slot(int slot)
{
    slots[slot].len = 0;
    slots[slot].logging_count = 0;
}

static int get_slot(unsigned long phys_addr)
{
    int i;

    for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
        if (slots[i].len && slots[i].phys_addr <= phys_addr &&
            (slots[i].phys_addr + slots[i].len - 1) >= phys_addr)
            return i;
    }
    return -1;
}

/* Returns -1 if this slot is not totally contained on any other,
 * and the number of the slot otherwise */
static int get_container_slot(uint64_t phys_addr, unsigned long size)
{
    int i;

    for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
        if (slots[i].len && slots[i].phys_addr <= phys_addr &&
            (slots[i].phys_addr + slots[i].len) >= phys_addr + size)
            return i;
    return -1;
}

int kvm_is_containing_region(kvm_context_t kvm, unsigned long phys_addr,
                             unsigned long size)
{
    int slot = get_container_slot(phys_addr, size);
    if (slot == -1)
        return 0;
    return 1;
}

/*
 * dirty pages logging control
 */
static int kvm_dirty_pages_log_change(kvm_context_t kvm,
                                      unsigned long phys_addr, unsigned flags,
                                      unsigned mask)
{
    int r = -1;
    int slot = get_slot(phys_addr);

    if (slot == -1) {
        fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
        return 1;
    }

    flags = (slots[slot].flags & ~mask) | flags;
    if (flags == slots[slot].flags)
        return 0;
    slots[slot].flags = flags;

    {
        struct kvm_userspace_memory_region mem = {
            .slot = slot,
            .memory_size = slots[slot].len,
            .guest_phys_addr = slots[slot].phys_addr,
            .userspace_addr = slots[slot].userspace_addr,
            .flags = slots[slot].flags,
        };


        DPRINTF("slot %d start %llx len %llx flags %x\n",
                mem.slot, mem.guest_phys_addr, mem.memory_size, mem.flags);
        r = kvm_vm_ioctl(kvm_state, KVM_SET_USER_MEMORY_REGION, &mem);
        if (r < 0)
            fprintf(stderr, "%s: %m\n", __FUNCTION__);
    }
    return r;
}

static int kvm_dirty_pages_log_change_all(kvm_context_t kvm,
                                          int (*change)(kvm_context_t kvm,
                                                        uint64_t start,
                                                        uint64_t len))
{
    int i, r;

    for (i = r = 0; i < KVM_MAX_NUM_MEM_REGIONS && r == 0; i++) {
        if (slots[i].len)
            r = change(kvm, slots[i].phys_addr, slots[i].len);
    }
    return r;
}

int kvm_dirty_pages_log_enable_slot(kvm_context_t kvm, uint64_t phys_addr,
                                    uint64_t len)
{
    int slot = get_slot(phys_addr);

    DPRINTF("start %" PRIx64 " len %" PRIx64 "\n", phys_addr, len);
    if (slot == -1) {
        fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
        return -EINVAL;
    }

    if (slots[slot].logging_count++)
        return 0;

    return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
                                      KVM_MEM_LOG_DIRTY_PAGES,
                                      KVM_MEM_LOG_DIRTY_PAGES);
}

int kvm_dirty_pages_log_disable_slot(kvm_context_t kvm, uint64_t phys_addr,
                                     uint64_t len)
{
    int slot = get_slot(phys_addr);

    if (slot == -1) {
        fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
        return -EINVAL;
    }

    if (--slots[slot].logging_count)
        return 0;

    return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr, 0,
                                      KVM_MEM_LOG_DIRTY_PAGES);
}

/**
 * Enable dirty page logging for all memory regions
 */
int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
{
    if (kvm->dirty_pages_log_all)
        return 0;
    kvm->dirty_pages_log_all = 1;
    return kvm_dirty_pages_log_change_all(kvm, kvm_dirty_pages_log_enable_slot);
}

/**
 * Enable dirty page logging only for memory regions that were created with
 *     dirty logging enabled (disable for all other memory regions).
 */
int kvm_dirty_pages_log_reset(kvm_context_t kvm)
{
    if (!kvm->dirty_pages_log_all)
        return 0;
    kvm->dirty_pages_log_all = 0;
    return kvm_dirty_pages_log_change_all(kvm,
                                          kvm_dirty_pages_log_disable_slot);
}


static int kvm_create_context(void);

int kvm_init(int smp_cpus)
{
    int fd;
    int r, gsi_count;


    fd = open("/dev/kvm", O_RDWR);
    if (fd == -1) {
        perror("open /dev/kvm");
        return -1;
    }
    r = ioctl(fd, KVM_GET_API_VERSION, 0);
    if (r == -1) {
        fprintf(stderr,
                "kvm kernel version too old: "
                "KVM_GET_API_VERSION ioctl not supported\n");
        goto out_close;
    }
    if (r < EXPECTED_KVM_API_VERSION) {
        fprintf(stderr, "kvm kernel version too old: "
                "We expect API version %d or newer, but got "
                "version %d\n", EXPECTED_KVM_API_VERSION, r);
        goto out_close;
    }
    if (r > EXPECTED_KVM_API_VERSION) {
        fprintf(stderr, "kvm userspace version too old\n");
        goto out_close;
    }
    kvm_abi = r;
    kvm_page_size = getpagesize();
    kvm_state = qemu_mallocz(sizeof(*kvm_state));
    kvm_context = &kvm_state->kvm_context;

    kvm_state->fd = fd;
    kvm_state->vmfd = -1;
    kvm_context->opaque = cpu_single_env;
    kvm_context->dirty_pages_log_all = 0;
    kvm_context->no_irqchip_creation = 0;
    kvm_context->no_pit_creation = 0;

#ifdef KVM_CAP_SET_GUEST_DEBUG
    QTAILQ_INIT(&kvm_state->kvm_sw_breakpoints);
#endif

    gsi_count = kvm_get_gsi_count(kvm_context);
    if (gsi_count > 0) {
        int gsi_bits, i;

        /* Round up so we can search ints using ffs */
        gsi_bits = ALIGN(gsi_count, 32);
        kvm_context->used_gsi_bitmap = qemu_mallocz(gsi_bits / 8);
        kvm_context->max_gsi = gsi_bits;

        /* Mark any over-allocated bits as already in use */
        for (i = gsi_count; i < gsi_bits; i++)
            set_gsi(kvm_context, i);
    }

    pthread_mutex_lock(&qemu_mutex);
    return kvm_create_context();

  out_close:
    close(fd);
    return -1;
}

static void kvm_finalize(KVMState *s)
{
    /* FIXME
       if (kvm->vcpu_fd[0] != -1)
           close(kvm->vcpu_fd[0]);
       if (kvm->vm_fd != -1)
           close(kvm->vm_fd);
     */
    close(s->fd);
    free(s);
}

void kvm_disable_irqchip_creation(kvm_context_t kvm)
{
    kvm->no_irqchip_creation = 1;
}

void kvm_disable_pit_creation(kvm_context_t kvm)
{
    kvm->no_pit_creation = 1;
}

static void kvm_create_vcpu(CPUState *env, int id)
{
    long mmap_size;
    int r;

    r = kvm_vm_ioctl(kvm_state, KVM_CREATE_VCPU, id);
    if (r < 0) {
        fprintf(stderr, "kvm_create_vcpu: %m\n");
        fprintf(stderr, "Failed to create vCPU. Check the -smp parameter.\n");
        goto err;
    }

    env->kvm_fd = r;
    env->kvm_state = kvm_state;

    mmap_size = kvm_ioctl(kvm_state, KVM_GET_VCPU_MMAP_SIZE, 0);
    if (mmap_size < 0) {
        fprintf(stderr, "get vcpu mmap size: %m\n");
        goto err_fd;
    }
    env->kvm_run =
        mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, env->kvm_fd,
             0);
    if (env->kvm_run == MAP_FAILED) {
        fprintf(stderr, "mmap vcpu area: %m\n");
        goto err_fd;
    }

    return;
  err_fd:
    close(env->kvm_fd);
  err:
    /* We're no good with semi-broken states. */
    abort();
}

static int kvm_set_boot_vcpu_id(kvm_context_t kvm, uint32_t id)
{
#ifdef KVM_CAP_SET_BOOT_CPU_ID
    int r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_SET_BOOT_CPU_ID);
    if (r > 0)
        return kvm_vm_ioctl(kvm_state, KVM_SET_BOOT_CPU_ID, id);
    return -ENOSYS;
#else
    return -ENOSYS;
#endif
}

int kvm_create_vm(kvm_context_t kvm)
{
    int fd;
#ifdef KVM_CAP_IRQ_ROUTING
    kvm->irq_routes = qemu_mallocz(sizeof(*kvm->irq_routes));
    kvm->nr_allocated_irq_routes = 0;
#endif

    fd = kvm_ioctl(kvm_state, KVM_CREATE_VM, 0);
    if (fd < 0) {
        fprintf(stderr, "kvm_create_vm: %m\n");
        return -1;
    }
    kvm_state->vmfd = fd;
    return 0;
}

static int kvm_create_default_phys_mem(kvm_context_t kvm,
                                       unsigned long phys_mem_bytes,
                                       void **vm_mem)
{
#ifdef KVM_CAP_USER_MEMORY
    int r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
    if (r > 0)
        return 0;
    fprintf(stderr,
            "Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported\n");
#else
#error Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported
#endif
    return -1;
}

void kvm_create_irqchip(kvm_context_t kvm)
{
    int r;

    kvm->irqchip_in_kernel = 0;
#ifdef KVM_CAP_IRQCHIP
    if (!kvm->no_irqchip_creation) {
        r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
        if (r > 0) {            /* kernel irqchip supported */
            r = kvm_vm_ioctl(kvm_state, KVM_CREATE_IRQCHIP);
            if (r >= 0) {
                kvm->irqchip_inject_ioctl = KVM_IRQ_LINE;
#if defined(KVM_CAP_IRQ_INJECT_STATUS) && defined(KVM_IRQ_LINE_STATUS)
                r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION,
                              KVM_CAP_IRQ_INJECT_STATUS);
                if (r > 0)
                    kvm->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
#endif
                kvm->irqchip_in_kernel = 1;
            } else
                fprintf(stderr, "Create kernel PIC irqchip failed\n");
        }
    }
#endif
    kvm_state->irqchip_in_kernel = kvm->irqchip_in_kernel;
}

int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
{
    int r;

    r = kvm_create_vm(kvm);
    if (r < 0)
        return r;
    r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
    if (r < 0)
        return r;
    init_slots();
    r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
    if (r < 0)
        return r;
    kvm_create_irqchip(kvm);

    return 0;
}


int kvm_register_phys_mem(kvm_context_t kvm,
                          unsigned long phys_start, void *userspace_addr,
                          unsigned long len, int log)
{

    struct kvm_userspace_memory_region memory = {
        .memory_size = len,
        .guest_phys_addr = phys_start,
        .userspace_addr = (unsigned long) (uintptr_t) userspace_addr,
        .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
    };
    int r;

    memory.slot = get_free_slot(kvm);
    DPRINTF
        ("memory: gpa: %llx, size: %llx, uaddr: %llx, slot: %x, flags: %x\n",
         memory.guest_phys_addr, memory.memory_size, memory.userspace_addr,
         memory.slot, memory.flags);
    r = kvm_vm_ioctl(kvm_state, KVM_SET_USER_MEMORY_REGION, &memory);
    if (r < 0) {
        fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(-r));
        return -1;
    }
    register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
                  memory.userspace_addr, memory.flags);
    return 0;
}


/* destroy/free a whole slot.
 * phys_start, len and slot are the params passed to kvm_create_phys_mem()
 */
void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
                          unsigned long len)
{
    int slot;
    int r;
    struct kvm_userspace_memory_region memory = {
        .memory_size = 0,
        .guest_phys_addr = phys_start,
        .userspace_addr = 0,
        .flags = 0,
    };

    slot = get_slot(phys_start);

    if ((slot >= KVM_MAX_NUM_MEM_REGIONS) || (slot == -1)) {
        fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n", __FUNCTION__,
                slot);
        return;
    }
    if (phys_start != slots[slot].phys_addr) {
        fprintf(stderr,
                "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
                __FUNCTION__, phys_start, slots[slot].phys_addr);
        phys_start = slots[slot].phys_addr;
    }

    memory.slot = slot;
    DPRINTF("slot %d start %llx len %llx flags %x\n",
            memory.slot, memory.guest_phys_addr, memory.memory_size,
            memory.flags);
    r = kvm_vm_ioctl(kvm_state, KVM_SET_USER_MEMORY_REGION, &memory);
    if (r < 0) {
        fprintf(stderr, "destroy_userspace_phys_mem: %s", strerror(-r));
        return;
    }

    free_slot(memory.slot);
}

void kvm_unregister_memory_area(kvm_context_t kvm, uint64_t phys_addr,
                                unsigned long size)
{

    int slot = get_container_slot(phys_addr, size);

    if (slot != -1) {
        DPRINTF("Unregistering memory region %" PRIx64 " (%lx)\n", phys_addr, size);
        kvm_destroy_phys_mem(kvm, phys_addr, size);
        return;
    }
}

static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
{
    int r;
    struct kvm_dirty_log log = {
        .slot = slot,
    };

    log.dirty_bitmap = buf;

    r = kvm_vm_ioctl(kvm_state, ioctl_num, &log);
    if (r < 0)
        return r;
    return 0;
}

int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
{
    int slot;

    slot = get_slot(phys_addr);
    return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
}

int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
                              unsigned long len, void *opaque,
                              int (*cb)(unsigned long start,
                                        unsigned long len, void *bitmap,
                                        void *opaque))
{
    int i;
    int r;
    unsigned long end_addr = phys_addr + len;
    void *buf;

    for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
        if ((slots[i].len && (uint64_t) slots[i].phys_addr >= phys_addr)
            && ((uint64_t) slots[i].phys_addr + slots[i].len <= end_addr)) {
            buf = qemu_malloc(BITMAP_SIZE(slots[i].len));
            r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
            if (r) {
                qemu_free(buf);
                return r;
            }
            r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
            qemu_free(buf);
            if (r)
                return r;
        }
    }
    return 0;
}

#ifdef KVM_CAP_IRQCHIP

int kvm_set_irq_level(kvm_context_t kvm, int irq, int level, int *status)
{
    struct kvm_irq_level event;
    int r;

    if (!kvm->irqchip_in_kernel)
        return 0;
    event.level = level;
    event.irq = irq;
    r = kvm_vm_ioctl(kvm_state, kvm->irqchip_inject_ioctl, &event);
    if (r < 0)
        perror("kvm_set_irq_level");

    if (status) {
#ifdef KVM_CAP_IRQ_INJECT_STATUS
        *status =
            (kvm->irqchip_inject_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
#else
        *status = 1;
#endif
    }

    return 1;
}

int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
{
    int r;

    if (!kvm->irqchip_in_kernel)
        return 0;
    r = kvm_vm_ioctl(kvm_state, KVM_GET_IRQCHIP, chip);
    if (r < 0) {
        perror("kvm_get_irqchip\n");
    }
    return r;
}

int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
{
    int r;

    if (!kvm->irqchip_in_kernel)
        return 0;
    r = kvm_vm_ioctl(kvm_state, KVM_SET_IRQCHIP, chip);
    if (r < 0) {
        perror("kvm_set_irqchip\n");
    }
    return r;
}

#endif

static int handle_debug(CPUState *env)
{
#ifdef KVM_CAP_SET_GUEST_DEBUG
    struct kvm_run *run = env->kvm_run;

    return kvm_debug(env, &run->debug.arch);
#else
    return 0;
#endif
}

int kvm_get_regs(CPUState *env, struct kvm_regs *regs)
{
    return kvm_vcpu_ioctl(env, KVM_GET_REGS, regs);
}

int kvm_set_regs(CPUState *env, struct kvm_regs *regs)
{
    return kvm_vcpu_ioctl(env, KVM_SET_REGS, regs);
}

int kvm_get_fpu(CPUState *env, struct kvm_fpu *fpu)
{
    return kvm_vcpu_ioctl(env, KVM_GET_FPU, fpu);
}

int kvm_set_fpu(CPUState *env, struct kvm_fpu *fpu)
{
    return kvm_vcpu_ioctl(env, KVM_SET_FPU, fpu);
}

int kvm_get_sregs(CPUState *env, struct kvm_sregs *sregs)
{
    return kvm_vcpu_ioctl(env, KVM_GET_SREGS, sregs);
}

int kvm_set_sregs(CPUState *env, struct kvm_sregs *sregs)
{
    return kvm_vcpu_ioctl(env, KVM_SET_SREGS, sregs);
}

#ifdef KVM_CAP_MP_STATE
int kvm_get_mpstate(CPUState *env, struct kvm_mp_state *mp_state)
{
    int r;

    r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
    if (r > 0)
        return kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, mp_state);
    return -ENOSYS;
}

int kvm_set_mpstate(CPUState *env, struct kvm_mp_state *mp_state)
{
    int r;

    r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
    if (r > 0)
        return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, mp_state);
    return -ENOSYS;
}
#endif

static int handle_mmio(CPUState *env)
{
    unsigned long addr = env->kvm_run->mmio.phys_addr;
    struct kvm_run *kvm_run = env->kvm_run;
    void *data = kvm_run->mmio.data;

    /* hack: Red Hat 7.1 generates these weird accesses. */
    if ((addr > 0xa0000 - 4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
        return 0;

    cpu_physical_memory_rw(addr, data, kvm_run->mmio.len, kvm_run->mmio.is_write);
    return 0;
}

int handle_io_window(kvm_context_t kvm)
{
    return 1;
}

int handle_shutdown(kvm_context_t kvm, CPUState *env)
{
    /* stop the current vcpu from going back to guest mode */
    env->stopped = 1;

    qemu_system_reset_request();
    return 1;
}

static inline void push_nmi(kvm_context_t kvm)
{
#ifdef KVM_CAP_USER_NMI
    kvm_arch_push_nmi(kvm->opaque);
#endif                          /* KVM_CAP_USER_NMI */
}

void post_kvm_run(kvm_context_t kvm, CPUState *env)
{
    pthread_mutex_lock(&qemu_mutex);
    kvm_arch_post_run(env, env->kvm_run);
    cpu_single_env = env;
}

int pre_kvm_run(kvm_context_t kvm, CPUState *env)
{
    kvm_arch_pre_run(env, env->kvm_run);

    if (env->kvm_cpu_state.regs_modified) {
        kvm_arch_put_registers(env);
        env->kvm_cpu_state.regs_modified = 0;
    }

    pthread_mutex_unlock(&qemu_mutex);
    return 0;
}

int kvm_is_ready_for_interrupt_injection(CPUState *env)
{
    return env->kvm_run->ready_for_interrupt_injection;
}

static int kvm_handle_internal_error(kvm_context_t kvm,
                                     CPUState *env,
                                     struct kvm_run *run)
{
    fprintf(stderr, "KVM internal error. Suberror: %d\n",
            run->internal.suberror);
#ifdef KVM_CAP_INTERNAL_ERROR_DATA
    if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
        int i;

        for (i = 0; i < run->internal.ndata; ++i) {
            fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
                    i, (uint64_t)run->internal.data[i]);
        }
    }
#endif
    kvm_show_regs(env);
    if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION)
        fprintf(stderr, "emulation failure, check dmesg for details\n");
    vm_stop(0);
    return 1;
}

int kvm_run(CPUState *env)
{
    int r;
    kvm_context_t kvm = &env->kvm_state->kvm_context;
    struct kvm_run *run = env->kvm_run;
    int fd = env->kvm_fd;

  again:
    push_nmi(kvm);
#if !defined(__s390__)
    if (!kvm->irqchip_in_kernel)
        run->request_interrupt_window = kvm_arch_try_push_interrupts(env);
#endif

    r = pre_kvm_run(kvm, env);
    if (r)
        return r;
    r = ioctl(fd, KVM_RUN, 0);

    if (r == -1 && errno != EINTR && errno != EAGAIN) {
        r = -errno;
        post_kvm_run(kvm, env);
        fprintf(stderr, "kvm_run: %s\n", strerror(-r));
        return r;
    }

    post_kvm_run(kvm, env);

#if defined(KVM_CAP_COALESCED_MMIO)
    if (kvm_state->coalesced_mmio) {
        struct kvm_coalesced_mmio_ring *ring =
            (void *) run + kvm_state->coalesced_mmio * PAGE_SIZE;
        while (ring->first != ring->last) {
            cpu_physical_memory_rw(ring->coalesced_mmio[ring->first].phys_addr,
                           &ring->coalesced_mmio[ring->first].data[0],
                           ring->coalesced_mmio[ring->first].len, 1);
            smp_wmb();
            ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
        }
    }
#endif

#if !defined(__s390__)
    if (r == -1) {
        r = handle_io_window(kvm);
        goto more;
    }
#endif
    if (1) {
        switch (run->exit_reason) {
        case KVM_EXIT_UNKNOWN:
            r = handle_unhandled(run->hw.hardware_exit_reason);
            break;
        case KVM_EXIT_FAIL_ENTRY:
            r = handle_unhandled(run->fail_entry.hardware_entry_failure_reason);
            break;
        case KVM_EXIT_EXCEPTION:
            fprintf(stderr, "exception %d (%x)\n", run->ex.exception,
                    run->ex.error_code);
            kvm_show_regs(env);
            kvm_show_code(env);
            abort();
            break;
        case KVM_EXIT_IO:
            r = kvm_handle_io(run->io.port,
                                (uint8_t *)run + run->io.data_offset,
                                run->io.direction,
                                run->io.size,
                                run->io.count);
            r = 0;
            break;
        case KVM_EXIT_DEBUG:
            r = handle_debug(env);
            break;
        case KVM_EXIT_MMIO:
            r = handle_mmio(env);
            break;
        case KVM_EXIT_HLT:
            r = kvm_arch_halt(env);
            break;
        case KVM_EXIT_IRQ_WINDOW_OPEN:
            break;
        case KVM_EXIT_SHUTDOWN:
            r = handle_shutdown(kvm, env);
            break;
#if defined(__s390__)
        case KVM_EXIT_S390_SIEIC:
            r = kvm_s390_handle_intercept(kvm, env, run);
            break;
        case KVM_EXIT_S390_RESET:
            r = kvm_s390_handle_reset(kvm, env, run);
            break;
#endif
	case KVM_EXIT_INTERNAL_ERROR:
            r = kvm_handle_internal_error(kvm, env, run);
	    break;
        default:
            if (kvm_arch_run(env)) {
                fprintf(stderr, "unhandled vm exit: 0x%x\n", run->exit_reason);
                kvm_show_regs(env);
                abort();
            }
            break;
        }
    }
  more:
    if (!r)
        goto again;
    return r;
}

int kvm_inject_irq(CPUState *env, unsigned irq)
{
    struct kvm_interrupt intr;

    intr.irq = irq;
    return kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
}

#ifdef KVM_CAP_SET_GUEST_DEBUG
int kvm_set_guest_debug(CPUState *env, struct kvm_guest_debug *dbg)
{
    return kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, dbg);
}
#endif

int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
{
    struct kvm_signal_mask *sigmask;
    int r;

    if (!sigset) {
        return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
    }
    sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));

    sigmask->len = 8;
    memcpy(sigmask->sigset, sigset, sizeof(*sigset));
    r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
    free(sigmask);
    return r;
}

int kvm_pit_in_kernel(kvm_context_t kvm)
{
    return kvm->pit_in_kernel;
}

int kvm_inject_nmi(CPUState *env)
{
#ifdef KVM_CAP_USER_NMI
    return kvm_vcpu_ioctl(env, KVM_NMI);
#else
    return -ENOSYS;
#endif
}

int kvm_init_coalesced_mmio(kvm_context_t kvm)
{
    int r = 0;
    kvm_state->coalesced_mmio = 0;
#ifdef KVM_CAP_COALESCED_MMIO
    r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_COALESCED_MMIO);
    if (r > 0) {
        kvm_state->coalesced_mmio = r;
        return 0;
    }
#endif
    return r;
}

#ifdef KVM_CAP_DEVICE_ASSIGNMENT
int kvm_assign_pci_device(kvm_context_t kvm,
                          struct kvm_assigned_pci_dev *assigned_dev)
{
    return kvm_vm_ioctl(kvm_state, KVM_ASSIGN_PCI_DEVICE, assigned_dev);
}

static int kvm_old_assign_irq(kvm_context_t kvm,
                              struct kvm_assigned_irq *assigned_irq)
{
    return kvm_vm_ioctl(kvm_state, KVM_ASSIGN_IRQ, assigned_irq);
}

#ifdef KVM_CAP_ASSIGN_DEV_IRQ
int kvm_assign_irq(kvm_context_t kvm, struct kvm_assigned_irq *assigned_irq)
{
    int ret;

    ret = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_ASSIGN_DEV_IRQ);
    if (ret > 0) {
        return kvm_vm_ioctl(kvm_state, KVM_ASSIGN_DEV_IRQ, assigned_irq);
    }

    return kvm_old_assign_irq(kvm, assigned_irq);
}

int kvm_deassign_irq(kvm_context_t kvm, struct kvm_assigned_irq *assigned_irq)
{
    return kvm_vm_ioctl(kvm_state, KVM_DEASSIGN_DEV_IRQ, assigned_irq);
}
#else
int kvm_assign_irq(kvm_context_t kvm, struct kvm_assigned_irq *assigned_irq)
{
    return kvm_old_assign_irq(kvm, assigned_irq);
}
#endif
#endif

#ifdef KVM_CAP_DEVICE_DEASSIGNMENT
int kvm_deassign_pci_device(kvm_context_t kvm,
                            struct kvm_assigned_pci_dev *assigned_dev)
{
    return kvm_vm_ioctl(kvm_state, KVM_DEASSIGN_PCI_DEVICE, assigned_dev);
}
#endif

int kvm_destroy_memory_region_works(kvm_context_t kvm)
{
    int ret = 0;

#ifdef KVM_CAP_DESTROY_MEMORY_REGION_WORKS
    ret =
        kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION,
                  KVM_CAP_DESTROY_MEMORY_REGION_WORKS);
    if (ret <= 0)
        ret = 0;
#endif
    return ret;
}

int kvm_reinject_control(kvm_context_t kvm, int pit_reinject)
{
#ifdef KVM_CAP_REINJECT_CONTROL
    int r;
    struct kvm_reinject_control control;

    control.pit_reinject = pit_reinject;

    r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_REINJECT_CONTROL);
    if (r > 0) {
        return kvm_vm_ioctl(kvm_state, KVM_REINJECT_CONTROL, &control);
    }
#endif
    return -ENOSYS;
}

int kvm_has_gsi_routing(kvm_context_t kvm)
{
    int r = 0;

#ifdef KVM_CAP_IRQ_ROUTING
    r = kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
#endif
    return r;
}

int kvm_get_gsi_count(kvm_context_t kvm)
{
#ifdef KVM_CAP_IRQ_ROUTING
    return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
#else
    return -EINVAL;
#endif
}

int kvm_clear_gsi_routes(kvm_context_t kvm)
{
#ifdef KVM_CAP_IRQ_ROUTING
    kvm->irq_routes->nr = 0;
    return 0;
#else
    return -EINVAL;
#endif
}

int kvm_add_routing_entry(kvm_context_t kvm,
                          struct kvm_irq_routing_entry *entry)
{
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing *z;
    struct kvm_irq_routing_entry *new;
    int n, size;

    if (kvm->irq_routes->nr == kvm->nr_allocated_irq_routes) {
        n = kvm->nr_allocated_irq_routes * 2;
        if (n < 64)
            n = 64;
        size = sizeof(struct kvm_irq_routing);
        size += n * sizeof(*new);
        z = realloc(kvm->irq_routes, size);
        if (!z)
            return -ENOMEM;
        kvm->nr_allocated_irq_routes = n;
        kvm->irq_routes = z;
    }
    n = kvm->irq_routes->nr++;
    new = &kvm->irq_routes->entries[n];
    memset(new, 0, sizeof(*new));
    new->gsi = entry->gsi;
    new->type = entry->type;
    new->flags = entry->flags;
    new->u = entry->u;

    set_gsi(kvm, entry->gsi);

    return 0;
#else
    return -ENOSYS;
#endif
}

int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
{
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing_entry e;

    e.gsi = gsi;
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
    e.flags = 0;
    e.u.irqchip.irqchip = irqchip;
    e.u.irqchip.pin = pin;
    return kvm_add_routing_entry(kvm, &e);
#else
    return -ENOSYS;
#endif
}

int kvm_del_routing_entry(kvm_context_t kvm,
                          struct kvm_irq_routing_entry *entry)
{
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing_entry *e, *p;
    int i, gsi, found = 0;

    gsi = entry->gsi;

    for (i = 0; i < kvm->irq_routes->nr; ++i) {
        e = &kvm->irq_routes->entries[i];
        if (e->type == entry->type && e->gsi == gsi) {
            switch (e->type) {
            case KVM_IRQ_ROUTING_IRQCHIP:{
                    if (e->u.irqchip.irqchip ==
                        entry->u.irqchip.irqchip
                        && e->u.irqchip.pin == entry->u.irqchip.pin) {
                        p = &kvm->irq_routes->entries[--kvm->irq_routes->nr];
                        *e = *p;
                        found = 1;
                    }
                    break;
                }
            case KVM_IRQ_ROUTING_MSI:{
                    if (e->u.msi.address_lo ==
                        entry->u.msi.address_lo
                        && e->u.msi.address_hi ==
                        entry->u.msi.address_hi
                        && e->u.msi.data == entry->u.msi.data) {
                        p = &kvm->irq_routes->entries[--kvm->irq_routes->nr];
                        *e = *p;
                        found = 1;
                    }
                    break;
                }
            default:
                break;
            }
            if (found) {
                /* If there are no other users of this GSI
                 * mark it available in the bitmap */
                for (i = 0; i < kvm->irq_routes->nr; i++) {
                    e = &kvm->irq_routes->entries[i];
                    if (e->gsi == gsi)
                        break;
                }
                if (i == kvm->irq_routes->nr)
                    clear_gsi(kvm, gsi);

                return 0;
            }
        }
    }
    return -ESRCH;
#else
    return -ENOSYS;
#endif
}

int kvm_update_routing_entry(kvm_context_t kvm,
                             struct kvm_irq_routing_entry *entry,
                             struct kvm_irq_routing_entry *newentry)
{
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing_entry *e;
    int i;

    if (entry->gsi != newentry->gsi || entry->type != newentry->type) {
        return -EINVAL;
    }

    for (i = 0; i < kvm->irq_routes->nr; ++i) {
        e = &kvm->irq_routes->entries[i];
        if (e->type != entry->type || e->gsi != entry->gsi) {
            continue;
        }
        switch (e->type) {
        case KVM_IRQ_ROUTING_IRQCHIP:
            if (e->u.irqchip.irqchip == entry->u.irqchip.irqchip &&
                e->u.irqchip.pin == entry->u.irqchip.pin) {
                memcpy(&e->u.irqchip, &newentry->u.irqchip,
                       sizeof e->u.irqchip);
                return 0;
            }
            break;
        case KVM_IRQ_ROUTING_MSI:
            if (e->u.msi.address_lo == entry->u.msi.address_lo &&
                e->u.msi.address_hi == entry->u.msi.address_hi &&
                e->u.msi.data == entry->u.msi.data) {
                memcpy(&e->u.msi, &newentry->u.msi, sizeof e->u.msi);
                return 0;
            }
            break;
        default:
            break;
        }
    }
    return -ESRCH;
#else
    return -ENOSYS;
#endif
}

int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
{
#ifdef KVM_CAP_IRQ_ROUTING
    struct kvm_irq_routing_entry e;

    e.gsi = gsi;
    e.type = KVM_IRQ_ROUTING_IRQCHIP;
    e.flags = 0;
    e.u.irqchip.irqchip = irqchip;
    e.u.irqchip.pin = pin;
    return kvm_del_routing_entry(kvm, &e);
#else
    return -ENOSYS;
#endif
}

int kvm_commit_irq_routes(kvm_context_t kvm)
{
#ifdef KVM_CAP_IRQ_ROUTING
    kvm->irq_routes->flags = 0;
    return kvm_vm_ioctl(kvm_state, KVM_SET_GSI_ROUTING, kvm->irq_routes);
#else
    return -ENOSYS;
#endif
}

int kvm_get_irq_route_gsi(kvm_context_t kvm)
{
    int i, bit;
    uint32_t *buf = kvm->used_gsi_bitmap;

    /* Return the lowest unused GSI in the bitmap */
    for (i = 0; i < kvm->max_gsi / 32; i++) {
        bit = ffs(~buf[i]);
        if (!bit)
            continue;

        return bit - 1 + i * 32;
    }

    return -ENOSPC;
}

#ifdef KVM_CAP_DEVICE_MSIX
int kvm_assign_set_msix_nr(kvm_context_t kvm,
                           struct kvm_assigned_msix_nr *msix_nr)
{
    return kvm_vm_ioctl(kvm_state, KVM_ASSIGN_SET_MSIX_NR, msix_nr);
}

int kvm_assign_set_msix_entry(kvm_context_t kvm,
                              struct kvm_assigned_msix_entry *entry)
{
    return kvm_vm_ioctl(kvm_state, KVM_ASSIGN_SET_MSIX_ENTRY, entry);
}
#endif

#if defined(KVM_CAP_IRQFD) && defined(CONFIG_EVENTFD)

#include <sys/eventfd.h>

static int _kvm_irqfd(kvm_context_t kvm, int fd, int gsi, int flags)
{
    struct kvm_irqfd data = {
        .fd = fd,
        .gsi = gsi,
        .flags = flags,
    };

    return kvm_vm_ioctl(kvm_state, KVM_IRQFD, &data);
}

int kvm_irqfd(kvm_context_t kvm, int gsi, int flags)
{
    int r;
    int fd;

    if (!kvm_check_extension(kvm_state, KVM_CAP_IRQFD))
        return -ENOENT;

    fd = eventfd(0, 0);
    if (fd < 0)
        return -errno;

    r = _kvm_irqfd(kvm, fd, gsi, 0);
    if (r < 0) {
        close(fd);
        return -errno;
    }

    return fd;
}

#else                           /* KVM_CAP_IRQFD */

int kvm_irqfd(kvm_context_t kvm, int gsi, int flags)
{
    return -ENOSYS;
}

#endif                          /* KVM_CAP_IRQFD */
static inline unsigned long kvm_get_thread_id(void)
{
    return syscall(SYS_gettid);
}

static void qemu_cond_wait(pthread_cond_t *cond)
{
    CPUState *env = cpu_single_env;

    pthread_cond_wait(cond, &qemu_mutex);
    cpu_single_env = env;
}

static void sig_ipi_handler(int n)
{
}

static void hardware_memory_error(void)
{
    fprintf(stderr, "Hardware memory error!\n");
    exit(1);
}

static void sigbus_reraise(void)
{
    sigset_t set;
    struct sigaction action;

    memset(&action, 0, sizeof(action));
    action.sa_handler = SIG_DFL;
    if (!sigaction(SIGBUS, &action, NULL)) {
        raise(SIGBUS);
        sigemptyset(&set);
        sigaddset(&set, SIGBUS);
        sigprocmask(SIG_UNBLOCK, &set, NULL);
    }
    perror("Failed to re-raise SIGBUS!\n");
    abort();
}

static void sigbus_handler(int n, struct qemu_signalfd_siginfo *siginfo,
                           void *ctx)
{
#if defined(KVM_CAP_MCE) && defined(TARGET_I386)
    if (first_cpu->mcg_cap && siginfo->ssi_addr
        && siginfo->ssi_code == BUS_MCEERR_AO) {
        uint64_t status;
        unsigned long paddr;
        CPUState *cenv;

        /* Hope we are lucky for AO MCE */
        if (do_qemu_ram_addr_from_host((void *)(intptr_t)siginfo->ssi_addr,
				       &paddr)) {
            fprintf(stderr, "Hardware memory error for memory used by "
                    "QEMU itself instead of guest system!: %llx\n",
                    (unsigned long long)siginfo->ssi_addr);
            return;
        }
        status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
            | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
            | 0xc0;
        kvm_inject_x86_mce(first_cpu, 9, status,
                           MCG_STATUS_MCIP | MCG_STATUS_RIPV, paddr,
                           (MCM_ADDR_PHYS << 6) | 0xc, 1);
        for (cenv = first_cpu->next_cpu; cenv != NULL; cenv = cenv->next_cpu)
            kvm_inject_x86_mce(cenv, 1, MCI_STATUS_VAL | MCI_STATUS_UC,
                               MCG_STATUS_MCIP | MCG_STATUS_RIPV, 0, 0, 1);
    } else
#endif
    {
        if (siginfo->ssi_code == BUS_MCEERR_AO)
            return;
        else if (siginfo->ssi_code == BUS_MCEERR_AR)
            hardware_memory_error();
        else
            sigbus_reraise();
    }
}

static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
{
    struct qemu_work_item wi;

    if (env == current_env) {
        func(data);
        return;
    }

    wi.func = func;
    wi.data = data;
    if (!env->kvm_cpu_state.queued_work_first)
        env->kvm_cpu_state.queued_work_first = &wi;
    else
        env->kvm_cpu_state.queued_work_last->next = &wi;
    env->kvm_cpu_state.queued_work_last = &wi;
    wi.next = NULL;
    wi.done = false;

    pthread_kill(env->kvm_cpu_state.thread, SIG_IPI);
    while (!wi.done)
        qemu_cond_wait(&qemu_work_cond);
}

void kvm_arch_get_registers(CPUState *env)
{
	kvm_arch_save_regs(env);
}

static void do_kvm_cpu_synchronize_state(void *_env)
{
    CPUState *env = _env;
    if (!env->kvm_cpu_state.regs_modified) {
        kvm_arch_get_registers(env);
        env->kvm_cpu_state.regs_modified = 1;
    }
}

void kvm_cpu_synchronize_state(CPUState *env)
{
    if (!env->kvm_cpu_state.regs_modified)
        on_vcpu(env, do_kvm_cpu_synchronize_state, env);
}

static void inject_interrupt(void *data)
{
    cpu_interrupt(current_env, (long) data);
}

void kvm_inject_interrupt(CPUState *env, int mask)
{
    on_vcpu(env, inject_interrupt, (void *) (long) mask);
}

void kvm_update_interrupt_request(CPUState *env)
{
    int signal = 0;

    if (env) {
        if (!current_env || !current_env->created)
            signal = 1;
        /*
         * Testing for created here is really redundant
         */
        if (current_env && current_env->created &&
            env != current_env && !env->kvm_cpu_state.signalled)
            signal = 1;

        if (signal) {
            env->kvm_cpu_state.signalled = 1;
            if (env->kvm_cpu_state.thread)
                pthread_kill(env->kvm_cpu_state.thread, SIG_IPI);
        }
    }
}

static void kvm_do_load_registers(void *_env)
{
    CPUState *env = _env;

    kvm_arch_load_regs(env);
}

void kvm_load_registers(CPUState *env)
{
    if (kvm_enabled() && qemu_system_ready)
        on_vcpu(env, kvm_do_load_registers, env);
}

static void kvm_do_save_registers(void *_env)
{
    CPUState *env = _env;

    kvm_arch_save_regs(env);
}

void kvm_save_registers(CPUState *env)
{
    if (kvm_enabled())
        on_vcpu(env, kvm_do_save_registers, env);
}

static void kvm_do_load_mpstate(void *_env)
{
    CPUState *env = _env;

    kvm_arch_load_mpstate(env);
}

void kvm_load_mpstate(CPUState *env)
{
    if (kvm_enabled() && qemu_system_ready && kvm_vcpu_inited(env))
        on_vcpu(env, kvm_do_load_mpstate, env);
}

static void kvm_do_save_mpstate(void *_env)
{
    CPUState *env = _env;

    kvm_arch_save_mpstate(env);
#ifdef KVM_CAP_MP_STATE
    if (kvm_irqchip_in_kernel())
        env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
#endif
}

void kvm_save_mpstate(CPUState *env)
{
    if (kvm_enabled())
        on_vcpu(env, kvm_do_save_mpstate, env);
}

int kvm_cpu_exec(CPUState *env)
{
    int r;

    r = kvm_run(env);
    if (r < 0) {
        printf("kvm_run returned %d\n", r);
        vm_stop(0);
    }

    return 0;
}

static int is_cpu_stopped(CPUState *env)
{
    return !vm_running || env->stopped;
}

static void flush_queued_work(CPUState *env)
{
    struct qemu_work_item *wi;

    if (!env->kvm_cpu_state.queued_work_first)
        return;

    while ((wi = env->kvm_cpu_state.queued_work_first)) {
        env->kvm_cpu_state.queued_work_first = wi->next;
        wi->func(wi->data);
        wi->done = true;
    }
    env->kvm_cpu_state.queued_work_last = NULL;
    pthread_cond_broadcast(&qemu_work_cond);
}

static void kvm_on_sigbus(CPUState *env, siginfo_t *siginfo)
{
#if defined(KVM_CAP_MCE) && defined(TARGET_I386)
    struct kvm_x86_mce mce = {
            .bank = 9,
    };
    unsigned long paddr;
    int r;

    if (env->mcg_cap && siginfo->si_addr
        && (siginfo->si_code == BUS_MCEERR_AR
            || siginfo->si_code == BUS_MCEERR_AO)) {
        if (siginfo->si_code == BUS_MCEERR_AR) {
            /* Fake an Intel architectural Data Load SRAR UCR */
            mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
                | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
                | MCI_STATUS_AR | 0x134;
            mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
            mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV;
        } else {
            /* Fake an Intel architectural Memory scrubbing UCR */
            mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
                | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
                | 0xc0;
            mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
            mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV;
        }
        if (do_qemu_ram_addr_from_host((void *)siginfo->si_addr, &paddr)) {
            fprintf(stderr, "Hardware memory error for memory used by "
                    "QEMU itself instaed of guest system!\n");
            /* Hope we are lucky for AO MCE */
            if (siginfo->si_code == BUS_MCEERR_AO)
                return;
            else
                hardware_memory_error();
        }
        mce.addr = paddr;
        r = kvm_set_mce(env, &mce);
        if (r < 0) {
            fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
            abort();
        }
    } else
#endif
    {
        if (siginfo->si_code == BUS_MCEERR_AO)
            return;
        else if (siginfo->si_code == BUS_MCEERR_AR)
            hardware_memory_error();
        else
            sigbus_reraise();
    }
}

static void kvm_main_loop_wait(CPUState *env, int timeout)
{
    struct timespec ts;
    int r, e;
    siginfo_t siginfo;
    sigset_t waitset;
    sigset_t chkset;

    ts.tv_sec = timeout / 1000;
    ts.tv_nsec = (timeout % 1000) * 1000000;
    sigemptyset(&waitset);
    sigaddset(&waitset, SIG_IPI);
    sigaddset(&waitset, SIGBUS);

    do {
        pthread_mutex_unlock(&qemu_mutex);

        r = sigtimedwait(&waitset, &siginfo, &ts);
        e = errno;

        pthread_mutex_lock(&qemu_mutex);

        if (r == -1 && !(e == EAGAIN || e == EINTR)) {
            printf("sigtimedwait: %s\n", strerror(e));
            exit(1);
        }

        switch (r) {
        case SIGBUS:
            kvm_on_sigbus(env, &siginfo);
            break;
        default:
            break;
        }

        r = sigpending(&chkset);
        if (r == -1) {
            printf("sigpending: %s\n", strerror(e));
            exit(1);
        }
    } while (sigismember(&chkset, SIG_IPI) || sigismember(&chkset, SIGBUS));

    cpu_single_env = env;
    flush_queued_work(env);

    if (env->stop) {
        env->stop = 0;
        env->stopped = 1;
        pthread_cond_signal(&qemu_pause_cond);
    }

    env->kvm_cpu_state.signalled = 0;
}

static int all_threads_paused(void)
{
    CPUState *penv = first_cpu;

    while (penv) {
        if (penv->stop)
            return 0;
        penv = (CPUState *) penv->next_cpu;
    }

    return 1;
}

static void pause_all_threads(void)
{
    CPUState *penv = first_cpu;

    while (penv) {
        if (penv != cpu_single_env) {
            penv->stop = 1;
            pthread_kill(penv->kvm_cpu_state.thread, SIG_IPI);
        } else {
            penv->stop = 0;
            penv->stopped = 1;
            cpu_exit(penv);
        }
        penv = (CPUState *) penv->next_cpu;
    }

    while (!all_threads_paused())
        qemu_cond_wait(&qemu_pause_cond);
}

static void resume_all_threads(void)
{
    CPUState *penv = first_cpu;

    assert(!cpu_single_env);

    while (penv) {
        penv->stop = 0;
        penv->stopped = 0;
        pthread_kill(penv->kvm_cpu_state.thread, SIG_IPI);
        penv = (CPUState *) penv->next_cpu;
    }
}

static void kvm_vm_state_change_handler(void *context, int running, int reason)
{
    if (running)
        resume_all_threads();
    else
        pause_all_threads();
}

static void setup_kernel_sigmask(CPUState *env)
{
    sigset_t set;

    sigemptyset(&set);
    sigaddset(&set, SIGUSR2);
    sigaddset(&set, SIGIO);
    sigaddset(&set, SIGALRM);
    sigprocmask(SIG_BLOCK, &set, NULL);

    sigprocmask(SIG_BLOCK, NULL, &set);
    sigdelset(&set, SIG_IPI);
    sigdelset(&set, SIGBUS);

    kvm_set_signal_mask(env, &set);
}

static void qemu_kvm_system_reset(void)
{
    CPUState *penv = first_cpu;

    pause_all_threads();

    qemu_system_reset();

    while (penv) {
        kvm_arch_cpu_reset(penv);
        penv = (CPUState *) penv->next_cpu;
    }

    resume_all_threads();
}

static void process_irqchip_events(CPUState *env)
{
    kvm_arch_process_irqchip_events(env);
    if (kvm_arch_has_work(env))
        env->halted = 0;
}

static int kvm_main_loop_cpu(CPUState *env)
{
    while (1) {
        int run_cpu = !is_cpu_stopped(env);
        if (run_cpu && !kvm_irqchip_in_kernel()) {
            process_irqchip_events(env);
            run_cpu = !env->halted;
        }
        if (run_cpu) {
            kvm_cpu_exec(env);
            kvm_main_loop_wait(env, 0);
        } else {
            kvm_main_loop_wait(env, 1000);
        }
    }
    pthread_mutex_unlock(&qemu_mutex);
    return 0;
}

static void *ap_main_loop(void *_env)
{
    CPUState *env = _env;
    sigset_t signals;
#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
    struct ioperm_data *data = NULL;
#endif

    current_env = env;
    env->thread_id = kvm_get_thread_id();
    sigfillset(&signals);
    sigprocmask(SIG_BLOCK, &signals, NULL);
    kvm_create_vcpu(env, env->cpu_index);

#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
    /* do ioperm for io ports of assigned devices */
    QLIST_FOREACH(data, &ioperm_head, entries)
        on_vcpu(env, kvm_arch_do_ioperm, data);
#endif

    setup_kernel_sigmask(env);

    pthread_mutex_lock(&qemu_mutex);
    cpu_single_env = env;

    kvm_arch_init_vcpu(env);

    kvm_arch_load_regs(env);

    /* signal VCPU creation */
    current_env->created = 1;
    pthread_cond_signal(&qemu_vcpu_cond);

    /* and wait for machine initialization */
    while (!qemu_system_ready)
        qemu_cond_wait(&qemu_system_cond);

    /* re-initialize cpu_single_env after re-acquiring qemu_mutex */
    cpu_single_env = env;

    kvm_main_loop_cpu(env);
    return NULL;
}

void kvm_init_vcpu(CPUState *env)
{
    pthread_create(&env->kvm_cpu_state.thread, NULL, ap_main_loop, env);

    while (env->created == 0)
        qemu_cond_wait(&qemu_vcpu_cond);
}

int kvm_vcpu_inited(CPUState *env)
{
    return env->created;
}

#ifdef TARGET_I386
void kvm_hpet_disable_kpit(void)
{
    struct kvm_pit_state2 ps2;

    kvm_get_pit2(kvm_context, &ps2);
    ps2.flags |= KVM_PIT_FLAGS_HPET_LEGACY;
    kvm_set_pit2(kvm_context, &ps2);
}

void kvm_hpet_enable_kpit(void)
{
    struct kvm_pit_state2 ps2;

    kvm_get_pit2(kvm_context, &ps2);
    ps2.flags &= ~KVM_PIT_FLAGS_HPET_LEGACY;
    kvm_set_pit2(kvm_context, &ps2);
}
#endif

int kvm_init_ap(void)
{
    struct sigaction action;

    qemu_add_vm_change_state_handler(kvm_vm_state_change_handler, NULL);

    signal(SIG_IPI, sig_ipi_handler);

    memset(&action, 0, sizeof(action));
    action.sa_flags = SA_SIGINFO;
    action.sa_sigaction = (void (*)(int, siginfo_t*, void*))sigbus_handler;
    sigaction(SIGBUS, &action, NULL);
    prctl(PR_MCE_KILL, 1, 1);
    return 0;
}

void qemu_kvm_notify_work(void)
{
    uint64_t value = 1;
    char buffer[8];
    size_t offset = 0;

    if (io_thread_fd == -1)
        return;

    memcpy(buffer, &value, sizeof(value));

    while (offset < 8) {
        ssize_t len;

        len = write(io_thread_fd, buffer + offset, 8 - offset);
        if (len == -1 && errno == EINTR)
            continue;

        /* In case we have a pipe, there is not reason to insist writing
         * 8 bytes
         */
        if (len == -1 && errno == EAGAIN)
            break;

        if (len <= 0)
            break;

        offset += len;
    }
}

/* If we have signalfd, we mask out the signals we want to handle and then
 * use signalfd to listen for them.  We rely on whatever the current signal
 * handler is to dispatch the signals when we receive them.
 */

static void sigfd_handler(void *opaque)
{
    int fd = (unsigned long) opaque;
    struct qemu_signalfd_siginfo info;
    struct sigaction action;
    ssize_t len;

    while (1) {
        do {
            len = read(fd, &info, sizeof(info));
        } while (len == -1 && errno == EINTR);

        if (len == -1 && errno == EAGAIN)
            break;

        if (len != sizeof(info)) {
            printf("read from sigfd returned %zd: %m\n", len);
            return;
        }

        sigaction(info.ssi_signo, NULL, &action);
        if ((action.sa_flags & SA_SIGINFO) && action.sa_sigaction)
            action.sa_sigaction(info.ssi_signo,
                                (siginfo_t *)&info, NULL);
	else if (action.sa_handler)
            action.sa_handler(info.ssi_signo);

    }
}

/* Used to break IO thread out of select */
static void io_thread_wakeup(void *opaque)
{
    int fd = (unsigned long) opaque;
    char buffer[4096];

    /* Drain the pipe/(eventfd) */
    while (1) {
        ssize_t len;

        len = read(fd, buffer, sizeof(buffer));
        if (len == -1 && errno == EINTR)
            continue;

        if (len <= 0)
            break;
    }
}

int kvm_main_loop(void)
{
    int fds[2];
    sigset_t mask;
    int sigfd;

    io_thread = pthread_self();
    qemu_system_ready = 1;

    if (qemu_eventfd(fds) == -1) {
        fprintf(stderr, "failed to create eventfd\n");
        return -errno;
    }

    fcntl(fds[0], F_SETFL, O_NONBLOCK);
    fcntl(fds[1], F_SETFL, O_NONBLOCK);

    qemu_set_fd_handler2(fds[0], NULL, io_thread_wakeup, NULL,
                         (void *)(unsigned long) fds[0]);

    io_thread_fd = fds[1];

    sigemptyset(&mask);
    sigaddset(&mask, SIGIO);
    sigaddset(&mask, SIGALRM);
    sigaddset(&mask, SIGBUS);
    sigprocmask(SIG_BLOCK, &mask, NULL);

    sigfd = qemu_signalfd(&mask);
    if (sigfd == -1) {
        fprintf(stderr, "failed to create signalfd\n");
        return -errno;
    }

    fcntl(sigfd, F_SETFL, O_NONBLOCK);

    qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
                         (void *)(unsigned long) sigfd);

    pthread_cond_broadcast(&qemu_system_cond);

    io_thread_sigfd = sigfd;
    cpu_single_env = NULL;

    while (1) {
        main_loop_wait(1000);
        if (qemu_shutdown_requested()) {
            if (qemu_no_shutdown()) {
                vm_stop(0);
            } else
                break;
        } else if (qemu_powerdown_requested())
            qemu_irq_raise(qemu_system_powerdown);
        else if (qemu_reset_requested())
            qemu_kvm_system_reset();
        else if (kvm_debug_cpu_requested) {
            gdb_set_stop_cpu(kvm_debug_cpu_requested);
            vm_stop(EXCP_DEBUG);
            kvm_debug_cpu_requested = NULL;
        }
    }

    pause_all_threads();
    pthread_mutex_unlock(&qemu_mutex);

    return 0;
}

#ifdef TARGET_I386
static int destroy_region_works = 0;
#endif


#if !defined(TARGET_I386)
int kvm_arch_init_irq_routing(void)
{
    return 0;
}
#endif

extern int no_hpet;

static int kvm_create_context(void)
{
    int r;

    if (!kvm_irqchip) {
        kvm_disable_irqchip_creation(kvm_context);
    }
    if (!kvm_pit) {
        kvm_disable_pit_creation(kvm_context);
    }
    if (kvm_create(kvm_context, 0, NULL) < 0) {
        kvm_finalize(kvm_state);
        return -1;
    }
    r = kvm_arch_qemu_create_context();
    if (r < 0)
        kvm_finalize(kvm_state);
    if (kvm_pit && !kvm_pit_reinject) {
        if (kvm_reinject_control(kvm_context, 0)) {
            fprintf(stderr, "failure to disable in-kernel PIT reinjection\n");
            return -1;
        }
    }
#ifdef TARGET_I386
    destroy_region_works = kvm_destroy_memory_region_works(kvm_context);
#endif

    r = kvm_arch_init_irq_routing();
    if (r < 0) {
        return r;
    }

    kvm_state->vcpu_events = 0;
#ifdef KVM_CAP_VCPU_EVENTS
    kvm_state->vcpu_events = kvm_check_extension(kvm_state, KVM_CAP_VCPU_EVENTS);
#endif

    kvm_init_ap();
    if (kvm_irqchip) {
        if (!qemu_kvm_has_gsi_routing()) {
            irq0override = 0;
#ifdef TARGET_I386
            /* if kernel can't do irq routing, interrupt source
             * override 0->2 can not be set up as required by hpet,
             * so disable hpet.
             */
            no_hpet = 1;
        } else if (!qemu_kvm_has_pit_state2()) {
            no_hpet = 1;
        }
#else
        }
#endif
    }

    return 0;
}

#ifdef TARGET_I386
static int must_use_aliases_source(target_phys_addr_t addr)
{
    if (destroy_region_works)
        return false;
    if (addr == 0xa0000 || addr == 0xa8000)
        return true;
    return false;
}

static int must_use_aliases_target(target_phys_addr_t addr)
{
    if (destroy_region_works)
        return false;
    if (addr >= 0xe0000000 && addr < 0x100000000ull)
        return true;
    return false;
}

static struct mapping {
    target_phys_addr_t phys;
    ram_addr_t ram;
    ram_addr_t len;
} mappings[50];
static int nr_mappings;

static struct mapping *find_ram_mapping(ram_addr_t ram_addr)
{
    struct mapping *p;

    for (p = mappings; p < mappings + nr_mappings; ++p) {
        if (p->ram <= ram_addr && ram_addr < p->ram + p->len) {
            return p;
        }
    }
    return NULL;
}

static struct mapping *find_mapping(target_phys_addr_t start_addr)
{
    struct mapping *p;

    for (p = mappings; p < mappings + nr_mappings; ++p) {
        if (p->phys <= start_addr && start_addr < p->phys + p->len) {
            return p;
        }
    }
    return NULL;
}

static void drop_mapping(target_phys_addr_t start_addr)
{
    struct mapping *p = find_mapping(start_addr);

    if (p)
        *p = mappings[--nr_mappings];
}
#endif

void kvm_set_phys_mem(target_phys_addr_t start_addr, ram_addr_t size,
                      ram_addr_t phys_offset)
{
    int r = 0;
    unsigned long area_flags;
#ifdef TARGET_I386
    struct mapping *p;
#endif

    if (start_addr + size > phys_ram_size) {
        phys_ram_size = start_addr + size;
    }

    phys_offset &= ~IO_MEM_ROM;
    area_flags = phys_offset & ~TARGET_PAGE_MASK;

    if (area_flags != IO_MEM_RAM) {
#ifdef TARGET_I386
        if (must_use_aliases_source(start_addr)) {
            kvm_destroy_memory_alias(kvm_context, start_addr);
            return;
        }
        if (must_use_aliases_target(start_addr))
            return;
#endif
        while (size > 0) {
            p = find_mapping(start_addr);
            if (p) {
                kvm_unregister_memory_area(kvm_context, p->phys, p->len);
                drop_mapping(p->phys);
            }
            start_addr += TARGET_PAGE_SIZE;
            if (size > TARGET_PAGE_SIZE) {
                size -= TARGET_PAGE_SIZE;
            } else {
                size = 0;
            }
        }
        return;
    }

    r = kvm_is_containing_region(kvm_context, start_addr, size);
    if (r)
        return;

    if (area_flags >= TLB_MMIO)
        return;

#ifdef TARGET_I386
    if (must_use_aliases_source(start_addr)) {
        p = find_ram_mapping(phys_offset);
        if (p) {
            kvm_create_memory_alias(kvm_context, start_addr, size,
                                    p->phys + (phys_offset - p->ram));
        }
        return;
    }
#endif

    r = kvm_register_phys_mem(kvm_context, start_addr,
                              qemu_get_ram_ptr(phys_offset), size, 0);
    if (r < 0) {
        printf("kvm_cpu_register_physical_memory: failed\n");
        exit(1);
    }
#ifdef TARGET_I386
    drop_mapping(start_addr);
    p = &mappings[nr_mappings++];
    p->phys = start_addr;
    p->ram = phys_offset;
    p->len = size;
#endif

    return;
}

int kvm_setup_guest_memory(void *area, unsigned long size)
{
    int ret = 0;

#ifdef MADV_DONTFORK
    if (kvm_enabled() && !kvm_has_sync_mmu())
        ret = madvise(area, size, MADV_DONTFORK);
#endif

    if (ret)
        perror("madvise");

    return ret;
}

#ifdef KVM_CAP_SET_GUEST_DEBUG

struct kvm_set_guest_debug_data {
    struct kvm_guest_debug dbg;
    int err;
};

static void kvm_invoke_set_guest_debug(void *data)
{
    struct kvm_set_guest_debug_data *dbg_data = data;

    if (cpu_single_env->kvm_cpu_state.regs_modified) {
        kvm_arch_put_registers(cpu_single_env);
        cpu_single_env->kvm_cpu_state.regs_modified = 0;
    }
    dbg_data->err =
        kvm_set_guest_debug(cpu_single_env,
                            &dbg_data->dbg);
}

int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
{
    struct kvm_set_guest_debug_data data;

    data.dbg.control = 0;
    if (env->singlestep_enabled)
        data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;

    kvm_arch_update_guest_debug(env, &data.dbg);
    data.dbg.control |= reinject_trap;

    on_vcpu(env, kvm_invoke_set_guest_debug, &data);
    return data.err;
}

#endif

/*
 * dirty pages logging
 */
/* FIXME: use unsigned long pointer instead of unsigned char */
unsigned char *kvm_dirty_bitmap = NULL;
int kvm_physical_memory_set_dirty_tracking(int enable)
{
    int r = 0;

    if (!kvm_enabled())
        return 0;

    if (enable) {
        if (!kvm_dirty_bitmap) {
            unsigned bitmap_size = BITMAP_SIZE(phys_ram_size);
            kvm_dirty_bitmap = qemu_malloc(bitmap_size);
            r = kvm_dirty_pages_log_enable_all(kvm_context);
        }
    } else {
        if (kvm_dirty_bitmap) {
            r = kvm_dirty_pages_log_reset(kvm_context);
            qemu_free(kvm_dirty_bitmap);
            kvm_dirty_bitmap = NULL;
        }
    }
    return r;
}

/* get kvm's dirty pages bitmap and update qemu's */
static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
                                         unsigned char *bitmap,
                                         unsigned long offset,
                                         unsigned long mem_size)
{
    unsigned int i, j, n = 0;
    unsigned char c;
    unsigned long page_number, addr, addr1;
    ram_addr_t ram_addr;
    unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;

    /* 
     * bitmap-traveling is faster than memory-traveling (for addr...) 
     * especially when most of the memory is not dirty.
     */
    for (i = 0; i < len; i++) {
        c = bitmap[i];
        while (c > 0) {
            j = ffsl(c) - 1;
            c &= ~(1u << j);
            page_number = i * 8 + j;
            addr1 = page_number * TARGET_PAGE_SIZE;
            addr = offset + addr1;
            ram_addr = cpu_get_physical_page_desc(addr);
            cpu_physical_memory_set_dirty(ram_addr);
            n++;
        }
    }
    return 0;
}

static int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
                                   void *bitmap, void *opaque)
{
    return kvm_get_dirty_pages_log_range(start, bitmap, start, len);
}

/* 
 * get kvm's dirty pages bitmap and update qemu's
 * we only care about physical ram, which resides in slots 0 and 3
 */
int kvm_update_dirty_pages_log(void)
{
    int r = 0;


    r = kvm_get_dirty_pages_range(kvm_context, 0, -1UL, NULL,
                                  kvm_get_dirty_bitmap_cb);
    return r;
}

void kvm_qemu_log_memory(target_phys_addr_t start, target_phys_addr_t size,
                         int log)
{
    if (log)
        kvm_dirty_pages_log_enable_slot(kvm_context, start, size);
    else {
#ifdef TARGET_I386
        if (must_use_aliases_target(start))
            return;
#endif
        kvm_dirty_pages_log_disable_slot(kvm_context, start, size);
    }
}

#ifdef KVM_CAP_IRQCHIP

int kvm_set_irq(int irq, int level, int *status)
{
    return kvm_set_irq_level(kvm_context, irq, level, status);
}

#endif

int qemu_kvm_get_dirty_pages(unsigned long phys_addr, void *buf)
{
    return kvm_get_dirty_pages(kvm_context, phys_addr, buf);
}

void kvm_mutex_unlock(void)
{
    assert(!cpu_single_env);
    pthread_mutex_unlock(&qemu_mutex);
}

void kvm_mutex_lock(void)
{
    pthread_mutex_lock(&qemu_mutex);
    cpu_single_env = NULL;
}

void qemu_mutex_unlock_iothread(void)
{
    if (kvm_enabled())
        kvm_mutex_unlock();
}

void qemu_mutex_lock_iothread(void)
{
    if (kvm_enabled())
        kvm_mutex_lock();
}

#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
void kvm_add_ioperm_data(struct ioperm_data *data)
{
    QLIST_INSERT_HEAD(&ioperm_head, data, entries);
}

void kvm_remove_ioperm_data(unsigned long start_port, unsigned long num)
{
    struct ioperm_data *data;

    data = QLIST_FIRST(&ioperm_head);
    while (data) {
        struct ioperm_data *next = QLIST_NEXT(data, entries);

        if (data->start_port == start_port && data->num == num) {
            QLIST_REMOVE(data, entries);
            qemu_free(data);
        }

        data = next;
    }
}

void kvm_ioperm(CPUState *env, void *data)
{
    if (kvm_enabled() && qemu_system_ready)
        on_vcpu(env, kvm_arch_do_ioperm, data);
}

#endif

int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
                                   target_phys_addr_t end_addr)
{
#ifndef TARGET_IA64

#ifdef TARGET_I386
    if (must_use_aliases_source(start_addr))
        return 0;
#endif

    kvm_get_dirty_pages_range(kvm_context, start_addr,
                              end_addr - start_addr, NULL,
                              kvm_get_dirty_bitmap_cb);
#endif
    return 0;
}

int kvm_log_start(target_phys_addr_t phys_addr, target_phys_addr_t len)
{
#ifdef TARGET_I386
    if (must_use_aliases_source(phys_addr))
        return 0;
#endif

#ifndef TARGET_IA64
    kvm_qemu_log_memory(phys_addr, len, 1);
#endif
    return 0;
}

int kvm_log_stop(target_phys_addr_t phys_addr, target_phys_addr_t len)
{
#ifdef TARGET_I386
    if (must_use_aliases_source(phys_addr))
        return 0;
#endif

#ifndef TARGET_IA64
    kvm_qemu_log_memory(phys_addr, len, 0);
#endif
    return 0;
}

int kvm_set_boot_cpu_id(uint32_t id)
{
    return kvm_set_boot_vcpu_id(kvm_context, id);
}

#ifdef TARGET_I386
#ifdef KVM_CAP_MCE
struct kvm_x86_mce_data {
    CPUState *env;
    struct kvm_x86_mce *mce;
    int abort_on_error;
};

static void kvm_do_inject_x86_mce(void *_data)
{
    struct kvm_x86_mce_data *data = _data;
    int r;

    r = kvm_set_mce(data->env, data->mce);
    if (r < 0) {
        perror("kvm_set_mce FAILED");
        if (data->abort_on_error)
            abort();
    }
}
#endif

void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,
                        uint64_t mcg_status, uint64_t addr, uint64_t misc,
                        int abort_on_error)
{
#ifdef KVM_CAP_MCE
    struct kvm_x86_mce mce = {
        .bank = bank,
        .status = status,
        .mcg_status = mcg_status,
        .addr = addr,
        .misc = misc,
    };
    struct kvm_x86_mce_data data = {
        .env = cenv,
        .mce = &mce,
        .abort_on_error = abort_on_error,
    };

    if (!cenv->mcg_cap) {
        fprintf(stderr, "MCE support is not enabled!\n");
        return;
    }
    on_vcpu(cenv, kvm_do_inject_x86_mce, &data);
#else
    if (abort_on_error)
        abort();
#endif
}
#endif