qemu-kvm-x86.c

/*
 * qemu/kvm integration, x86 specific code
 *
 * Copyright (C) 2006-2008 Qumranet Technologies
 *
 * Licensed under the terms of the GNU GPL version 2 or higher.
 */

#include "config.h"
#include "config-host.h"

#include <string.h>
#include "hw/hw.h"
#include "gdbstub.h"
#include <sys/io.h>

#include "qemu-kvm.h"
#include "libkvm.h"
#include <pthread.h>
#include <sys/utsname.h>
#include <linux/kvm_para.h>
#include <sys/ioctl.h>

#include "kvm.h"
#include "hw/apic.h"

#define MSR_IA32_TSC		0x10

static struct kvm_msr_list *kvm_msr_list;
extern unsigned int kvm_shadow_memory;
static int kvm_has_msr_star;
static int kvm_has_vm_hsave_pa;

static int lm_capable_kernel;

int kvm_set_tss_addr(kvm_context_t kvm, unsigned long addr)
{
	int r;
        /*
         * Tell fw_cfg to notify the BIOS to reserve the range.
         */
        if (e820_add_entry(addr, 0x4000, E820_RESERVED) < 0) {
            perror("e820_add_entry() table is full");
            exit(1);
        }

	r = kvm_vm_ioctl(kvm_state, KVM_SET_TSS_ADDR, addr);
	if (r < 0) {
		fprintf(stderr, "kvm_set_tss_addr: %m\n");
		return r;
	}
	return 0;
}

static int kvm_init_tss(kvm_context_t kvm)
{
	int r;

	r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
	if (r > 0) {
		/*
		 * this address is 3 pages before the bios, and the bios should present
		 * as unavaible memory
		 */
		r = kvm_set_tss_addr(kvm, 0xfeffd000);
		if (r < 0) {
			fprintf(stderr, "kvm_init_tss: unable to set tss addr\n");
			return r;
		}
	} else {
		fprintf(stderr, "kvm does not support KVM_CAP_SET_TSS_ADDR\n");
	}
	return 0;
}

static int kvm_set_identity_map_addr(kvm_context_t kvm, uint64_t addr)
{
#ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
	int r;

	r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_SET_IDENTITY_MAP_ADDR);
	if (r > 0) {
		r = kvm_vm_ioctl(kvm_state, KVM_SET_IDENTITY_MAP_ADDR, &addr);
		if (r == -1) {
			fprintf(stderr, "kvm_set_identity_map_addr: %m\n");
			return -errno;
		}
		return 0;
	}
#endif
	return -ENOSYS;
}

static int kvm_init_identity_map_page(kvm_context_t kvm)
{
#ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
	int r;

	r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_SET_IDENTITY_MAP_ADDR);
	if (r > 0) {
		/*
		 * this address is 4 pages before the bios, and the bios should present
		 * as unavaible memory
		 */
		r = kvm_set_identity_map_addr(kvm, 0xfeffc000);
		if (r < 0) {
			fprintf(stderr, "kvm_init_identity_map_page: "
				"unable to set identity mapping addr\n");
			return r;
		}

	}
#endif
	return 0;
}

static int kvm_create_pit(kvm_context_t kvm)
{
#ifdef KVM_CAP_PIT
	int r;

	kvm_state->pit_in_kernel = 0;
	if (!kvm->no_pit_creation) {
		r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_PIT);
		if (r > 0) {
			r = kvm_vm_ioctl(kvm_state, KVM_CREATE_PIT);
			if (r >= 0)
				kvm_state->pit_in_kernel = 1;
			else {
				fprintf(stderr, "Create kernel PIC irqchip failed\n");
				return r;
			}
		}
	}
#endif
	return 0;
}

int kvm_arch_create(kvm_context_t kvm, unsigned long phys_mem_bytes,
			void **vm_mem)
{
	int r = 0;

	r = kvm_init_tss(kvm);
	if (r < 0)
		return r;

	r = kvm_init_identity_map_page(kvm);
	if (r < 0)
		return r;

	r = kvm_create_pit(kvm);
	if (r < 0)
		return r;

	r = kvm_init_coalesced_mmio(kvm);
	if (r < 0)
		return r;

	return 0;
}

#ifdef KVM_EXIT_TPR_ACCESS

static int kvm_handle_tpr_access(CPUState *env)
{
	struct kvm_run *run = env->kvm_run;
	kvm_tpr_access_report(env,
                         run->tpr_access.rip,
                         run->tpr_access.is_write);
    return 0;
}


int kvm_enable_vapic(CPUState *env, uint64_t vapic)
{
	struct kvm_vapic_addr va = {
		.vapic_addr = vapic,
	};

	return kvm_vcpu_ioctl(env, KVM_SET_VAPIC_ADDR, &va);
}

#endif

int kvm_arch_run(CPUState *env)
{
	int r = 0;
	struct kvm_run *run = env->kvm_run;


	switch (run->exit_reason) {
#ifdef KVM_EXIT_SET_TPR
		case KVM_EXIT_SET_TPR:
			break;
#endif
#ifdef KVM_EXIT_TPR_ACCESS
		case KVM_EXIT_TPR_ACCESS:
			r = kvm_handle_tpr_access(env);
			break;
#endif
		default:
			r = 1;
			break;
	}

	return r;
}

#ifdef KVM_CAP_IRQCHIP

int kvm_get_lapic(CPUState *env, struct kvm_lapic_state *s)
{
	int r = 0;

	if (!kvm_irqchip_in_kernel())
		return r;

	r = kvm_vcpu_ioctl(env, KVM_GET_LAPIC, s);
	if (r < 0)
		fprintf(stderr, "KVM_GET_LAPIC failed\n");
	return r;
}

int kvm_set_lapic(CPUState *env, struct kvm_lapic_state *s)
{
	int r = 0;

	if (!kvm_irqchip_in_kernel())
		return 0;

	r = kvm_vcpu_ioctl(env, KVM_SET_LAPIC, s);

	if (r < 0)
		fprintf(stderr, "KVM_SET_LAPIC failed\n");
	return r;
}

#endif

#ifdef KVM_CAP_PIT

int kvm_get_pit(kvm_context_t kvm, struct kvm_pit_state *s)
{
	if (!kvm_pit_in_kernel())
		return 0;
	return kvm_vm_ioctl(kvm_state, KVM_GET_PIT, s);
}

int kvm_set_pit(kvm_context_t kvm, struct kvm_pit_state *s)
{
	if (!kvm_pit_in_kernel())
		return 0;
	return kvm_vm_ioctl(kvm_state, KVM_SET_PIT, s);
}

#ifdef KVM_CAP_PIT_STATE2
int kvm_get_pit2(kvm_context_t kvm, struct kvm_pit_state2 *ps2)
{
	if (!kvm_pit_in_kernel())
		return 0;
	return kvm_vm_ioctl(kvm_state, KVM_GET_PIT2, ps2);
}

int kvm_set_pit2(kvm_context_t kvm, struct kvm_pit_state2 *ps2)
{
	if (!kvm_pit_in_kernel())
		return 0;
	return kvm_vm_ioctl(kvm_state, KVM_SET_PIT2, ps2);
}

#endif
#endif

int kvm_has_pit_state2(kvm_context_t kvm)
{
	int r = 0;

#ifdef KVM_CAP_PIT_STATE2
	r = kvm_check_extension(kvm_state, KVM_CAP_PIT_STATE2);
#endif
	return r;
}

void kvm_show_code(CPUState *env)
{
#define SHOW_CODE_LEN 50
	struct kvm_regs regs;
	struct kvm_sregs sregs;
	int r, n;
	int back_offset;
	unsigned char code;
	char code_str[SHOW_CODE_LEN * 3 + 1];
	unsigned long rip;

	r = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
	if (r < 0 ) {
		perror("KVM_GET_SREGS");
		return;
	}
	r = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
	if (r < 0) {
		perror("KVM_GET_REGS");
		return;
	}
	rip = sregs.cs.base + regs.rip;
	back_offset = regs.rip;
	if (back_offset > 20)
	    back_offset = 20;
	*code_str = 0;
	for (n = -back_offset; n < SHOW_CODE_LEN-back_offset; ++n) {
		if (n == 0)
			strcat(code_str, " -->");
		cpu_physical_memory_rw(rip + n, &code, 1, 1);
		sprintf(code_str + strlen(code_str), " %02x", code);
	}
	fprintf(stderr, "code:%s\n", code_str);
}


/*
 * Returns available msr list.  User must free.
 */
static struct kvm_msr_list *kvm_get_msr_list(void)
{
	struct kvm_msr_list sizer, *msrs;
	int r;

	sizer.nmsrs = 0;
	r = kvm_ioctl(kvm_state, KVM_GET_MSR_INDEX_LIST, &sizer);
	if (r < 0 && r != -E2BIG)
		return NULL;
	/* Old kernel modules had a bug and could write beyond the provided
	   memory. Allocate at least a safe amount of 1K. */
	msrs = qemu_malloc(MAX(1024, sizeof(*msrs) +
				       sizer.nmsrs * sizeof(*msrs->indices)));

	msrs->nmsrs = sizer.nmsrs;
	r = kvm_ioctl(kvm_state, KVM_GET_MSR_INDEX_LIST, msrs);
	if (r < 0) {
		free(msrs);
		errno = r;
		return NULL;
	}
	return msrs;
}

int kvm_get_msrs(CPUState *env, struct kvm_msr_entry *msrs, int n)
{
    struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
    int r;

    kmsrs->nmsrs = n;
    memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
    r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs);
    memcpy(msrs, kmsrs->entries, n * sizeof *msrs);
    free(kmsrs);
    return r;
}

int kvm_set_msrs(CPUState *env, struct kvm_msr_entry *msrs, int n)
{
    struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
    int r;

    kmsrs->nmsrs = n;
    memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
    r = kvm_vcpu_ioctl(env, KVM_SET_MSRS, kmsrs);
    free(kmsrs);
    return r;
}

int kvm_get_mce_cap_supported(kvm_context_t kvm, uint64_t *mce_cap,
                              int *max_banks)
{
#ifdef KVM_CAP_MCE
    int r;

    r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_MCE);
    if (r > 0) {
        *max_banks = r;
        return kvm_ioctl(kvm_state, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
    }
#endif
    return -ENOSYS;
}

int kvm_setup_mce(CPUState *env, uint64_t *mcg_cap)
{
#ifdef KVM_CAP_MCE
    return kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, mcg_cap);
#else
    return -ENOSYS;
#endif
}

int kvm_set_mce(CPUState *env, struct kvm_x86_mce *m)
{
#ifdef KVM_CAP_MCE
    return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m);
#else
    return -ENOSYS;
#endif
}

static void print_seg(FILE *file, const char *name, struct kvm_segment *seg)
{
	fprintf(stderr,
		"%s %04x (%08llx/%08x p %d dpl %d db %d s %d type %x l %d"
		" g %d avl %d)\n",
		name, seg->selector, seg->base, seg->limit, seg->present,
		seg->dpl, seg->db, seg->s, seg->type, seg->l, seg->g,
		seg->avl);
}

static void print_dt(FILE *file, const char *name, struct kvm_dtable *dt)
{
	fprintf(stderr, "%s %llx/%x\n", name, dt->base, dt->limit);
}

void kvm_show_regs(CPUState *env)
{
	struct kvm_regs regs;
	struct kvm_sregs sregs;
	int r;

	r = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
	if (r < 0) {
		perror("KVM_GET_REGS");
		return;
	}
	fprintf(stderr,
		"rax %016llx rbx %016llx rcx %016llx rdx %016llx\n"
		"rsi %016llx rdi %016llx rsp %016llx rbp %016llx\n"
		"r8  %016llx r9  %016llx r10 %016llx r11 %016llx\n"
		"r12 %016llx r13 %016llx r14 %016llx r15 %016llx\n"
		"rip %016llx rflags %08llx\n",
		regs.rax, regs.rbx, regs.rcx, regs.rdx,
		regs.rsi, regs.rdi, regs.rsp, regs.rbp,
		regs.r8,  regs.r9,  regs.r10, regs.r11,
		regs.r12, regs.r13, regs.r14, regs.r15,
		regs.rip, regs.rflags);
	r = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
	if (r < 0) {
		perror("KVM_GET_SREGS");
		return;
	}
	print_seg(stderr, "cs", &sregs.cs);
	print_seg(stderr, "ds", &sregs.ds);
	print_seg(stderr, "es", &sregs.es);
	print_seg(stderr, "ss", &sregs.ss);
	print_seg(stderr, "fs", &sregs.fs);
	print_seg(stderr, "gs", &sregs.gs);
	print_seg(stderr, "tr", &sregs.tr);
	print_seg(stderr, "ldt", &sregs.ldt);
	print_dt(stderr, "gdt", &sregs.gdt);
	print_dt(stderr, "idt", &sregs.idt);
	fprintf(stderr, "cr0 %llx cr2 %llx cr3 %llx cr4 %llx cr8 %llx"
		" efer %llx\n",
		sregs.cr0, sregs.cr2, sregs.cr3, sregs.cr4, sregs.cr8,
		sregs.efer);
}

static void kvm_set_cr8(CPUState *env, uint64_t cr8)
{
	env->kvm_run->cr8 = cr8;
}

int kvm_setup_cpuid(CPUState *env, int nent,
		    struct kvm_cpuid_entry *entries)
{
	struct kvm_cpuid *cpuid;
	int r;

	cpuid = qemu_malloc(sizeof(*cpuid) + nent * sizeof(*entries));

	cpuid->nent = nent;
	memcpy(cpuid->entries, entries, nent * sizeof(*entries));
	r = kvm_vcpu_ioctl(env, KVM_SET_CPUID, cpuid);

	free(cpuid);
	return r;
}

int kvm_setup_cpuid2(CPUState *env, int nent,
		     struct kvm_cpuid_entry2 *entries)
{
	struct kvm_cpuid2 *cpuid;
	int r;

	cpuid = qemu_malloc(sizeof(*cpuid) + nent * sizeof(*entries));

	cpuid->nent = nent;
	memcpy(cpuid->entries, entries, nent * sizeof(*entries));
	r = kvm_vcpu_ioctl(env, KVM_SET_CPUID2, cpuid);
	free(cpuid);
	return r;
}

int kvm_set_shadow_pages(kvm_context_t kvm, unsigned int nrshadow_pages)
{
#ifdef KVM_CAP_MMU_SHADOW_CACHE_CONTROL
	int r;

	r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION,
		  KVM_CAP_MMU_SHADOW_CACHE_CONTROL);
	if (r > 0) {
		r = kvm_vm_ioctl(kvm_state, KVM_SET_NR_MMU_PAGES, nrshadow_pages);
		if (r < 0) {
			fprintf(stderr, "kvm_set_shadow_pages: %m\n");
			return r;
		}
		return 0;
	}
#endif
	return -1;
}

int kvm_get_shadow_pages(kvm_context_t kvm, unsigned int *nrshadow_pages)
{
#ifdef KVM_CAP_MMU_SHADOW_CACHE_CONTROL
	int r;

	r = kvm_ioctl(kvm_state, KVM_CHECK_EXTENSION,
		  KVM_CAP_MMU_SHADOW_CACHE_CONTROL);
	if (r > 0) {
		*nrshadow_pages = kvm_vm_ioctl(kvm_state, KVM_GET_NR_MMU_PAGES);
		return 0;
	}
#endif
	return -1;
}

#ifdef KVM_CAP_VAPIC
static int kvm_enable_tpr_access_reporting(CPUState *env)
{
    int r;
    struct kvm_tpr_access_ctl tac = { .enabled = 1 };

    r = kvm_ioctl(env->kvm_state, KVM_CHECK_EXTENSION, KVM_CAP_VAPIC);
    if (r <= 0)
        return -ENOSYS;
    return kvm_vcpu_ioctl(env, KVM_TPR_ACCESS_REPORTING, &tac);
}
#endif

#ifdef KVM_CAP_ADJUST_CLOCK
static struct kvm_clock_data kvmclock_data;

static void kvmclock_pre_save(void *opaque)
{
    struct kvm_clock_data *cl = opaque;

    kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, cl);
}

static int kvmclock_post_load(void *opaque, int version_id)
{
    struct kvm_clock_data *cl = opaque;

    return kvm_vm_ioctl(kvm_state, KVM_SET_CLOCK, cl);
}

static const VMStateDescription vmstate_kvmclock= {
    .name = "kvmclock",
    .version_id = 1,
    .minimum_version_id = 1,
    .minimum_version_id_old = 1,
    .pre_save = kvmclock_pre_save,
    .post_load = kvmclock_post_load,
    .fields      = (VMStateField []) {
        VMSTATE_U64(clock, struct kvm_clock_data),
        VMSTATE_END_OF_LIST()
    }
};
#endif

int kvm_arch_qemu_create_context(void)
{
    int i, r;
    struct utsname utsname;

    uname(&utsname);
    lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;

    if (kvm_shadow_memory)
        kvm_set_shadow_pages(kvm_context, kvm_shadow_memory);

    kvm_msr_list = kvm_get_msr_list();
    if (!kvm_msr_list)
		return -1;
    for (i = 0; i < kvm_msr_list->nmsrs; ++i) {
	if (kvm_msr_list->indices[i] == MSR_STAR)
	    kvm_has_msr_star = 1;
        if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA)
            kvm_has_vm_hsave_pa = 1;
    }

#ifdef KVM_CAP_ADJUST_CLOCK
    if (kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK))
        vmstate_register(NULL, 0, &vmstate_kvmclock, &kvmclock_data);
#endif

    r = kvm_set_boot_cpu_id(0);
    if (r < 0 && r != -ENOSYS) {
        return r;
    }

    return 0;
}

/* returns 0 on success, non-0 on failure */
static int get_msr_entry(struct kvm_msr_entry *entry, CPUState *env)
{
        switch (entry->index) {
        case MSR_IA32_SYSENTER_CS:
            env->sysenter_cs  = entry->data;
            break;
        case MSR_IA32_SYSENTER_ESP:
            env->sysenter_esp = entry->data;
            break;
        case MSR_IA32_SYSENTER_EIP:
            env->sysenter_eip = entry->data;
            break;
        case MSR_STAR:
            env->star         = entry->data;
            break;
#ifdef TARGET_X86_64
        case MSR_CSTAR:
            env->cstar        = entry->data;
            break;
        case MSR_KERNELGSBASE:
            env->kernelgsbase = entry->data;
            break;
        case MSR_FMASK:
            env->fmask        = entry->data;
            break;
        case MSR_LSTAR:
            env->lstar        = entry->data;
            break;
#endif
        case MSR_IA32_TSC:
            env->tsc          = entry->data;
            break;
        case MSR_VM_HSAVE_PA:
            env->vm_hsave     = entry->data;
            break;
        case MSR_KVM_SYSTEM_TIME:
            env->system_time_msr = entry->data;
            break;
        case MSR_KVM_WALL_CLOCK:
            env->wall_clock_msr = entry->data;
            break;
#ifdef KVM_CAP_MCE
        case MSR_MCG_STATUS:
            env->mcg_status = entry->data;
            break;
        case MSR_MCG_CTL:
            env->mcg_ctl = entry->data;
            break;
#endif
        default:
#ifdef KVM_CAP_MCE
            if (entry->index >= MSR_MC0_CTL && \
                entry->index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
                env->mce_banks[entry->index - MSR_MC0_CTL] = entry->data;
                break;
            }
#endif
            printf("Warning unknown msr index 0x%x\n", entry->index);
            return 1;
        }
        return 0;
}

static void kvm_arch_save_mpstate(CPUState *env)
{
#ifdef KVM_CAP_MP_STATE
    int r;
    struct kvm_mp_state mp_state;

    r = kvm_get_mpstate(env, &mp_state);
    if (r < 0) {
        env->mp_state = -1;
    } else {
        env->mp_state = mp_state.mp_state;
        if (kvm_irqchip_in_kernel()) {
            env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
        }
    }
#else
    env->mp_state = -1;
#endif
}

static void kvm_arch_load_mpstate(CPUState *env)
{
#ifdef KVM_CAP_MP_STATE
    struct kvm_mp_state mp_state;

    /*
     * -1 indicates that the host did not support GET_MP_STATE ioctl,
     *  so don't touch it.
     */
    if (env->mp_state != -1) {
        mp_state.mp_state = env->mp_state;
        kvm_set_mpstate(env, &mp_state);
    }
#endif
}

static void kvm_reset_mpstate(CPUState *env)
{
#ifdef KVM_CAP_MP_STATE
    if (kvm_check_extension(kvm_state, KVM_CAP_MP_STATE)) {
        if (kvm_irqchip_in_kernel()) {
            env->mp_state = cpu_is_bsp(env) ? KVM_MP_STATE_RUNNABLE :
                                              KVM_MP_STATE_UNINITIALIZED;
        } else {
            env->mp_state = KVM_MP_STATE_RUNNABLE;
        }
    }
#endif
}

static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
{
    lhs->selector = rhs->selector;
    lhs->base = rhs->base;
    lhs->limit = rhs->limit;
    lhs->type = 3;
    lhs->present = 1;
    lhs->dpl = 3;
    lhs->db = 0;
    lhs->s = 1;
    lhs->l = 0;
    lhs->g = 0;
    lhs->avl = 0;
    lhs->unusable = 0;
}

static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
{
    unsigned flags = rhs->flags;
    lhs->selector = rhs->selector;
    lhs->base = rhs->base;
    lhs->limit = rhs->limit;
    lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
    lhs->present = (flags & DESC_P_MASK) != 0;
    lhs->dpl = rhs->selector & 3;
    lhs->db = (flags >> DESC_B_SHIFT) & 1;
    lhs->s = (flags & DESC_S_MASK) != 0;
    lhs->l = (flags >> DESC_L_SHIFT) & 1;
    lhs->g = (flags & DESC_G_MASK) != 0;
    lhs->avl = (flags & DESC_AVL_MASK) != 0;
    lhs->unusable = 0;
}

static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
{
    lhs->selector = rhs->selector;
    lhs->base = rhs->base;
    lhs->limit = rhs->limit;
    lhs->flags =
	(rhs->type << DESC_TYPE_SHIFT)
	| (rhs->present * DESC_P_MASK)
	| (rhs->dpl << DESC_DPL_SHIFT)
	| (rhs->db << DESC_B_SHIFT)
	| (rhs->s * DESC_S_MASK)
	| (rhs->l << DESC_L_SHIFT)
	| (rhs->g * DESC_G_MASK)
	| (rhs->avl * DESC_AVL_MASK);
}

#define XSAVE_CWD_RIP     2
#define XSAVE_CWD_RDP     4
#define XSAVE_MXCSR       6
#define XSAVE_ST_SPACE    8
#define XSAVE_XMM_SPACE   40
#define XSAVE_XSTATE_BV   128
#define XSAVE_YMMH_SPACE  144

void kvm_arch_load_regs(CPUState *env, int level)
{
    struct kvm_regs regs;
    struct kvm_fpu fpu;
    struct kvm_sregs sregs;
    struct kvm_msr_entry msrs[100];
    int rc, n, i;

    assert(kvm_cpu_is_stopped(env) || env->thread_id == kvm_get_thread_id());

    regs.rax = env->regs[R_EAX];
    regs.rbx = env->regs[R_EBX];
    regs.rcx = env->regs[R_ECX];
    regs.rdx = env->regs[R_EDX];
    regs.rsi = env->regs[R_ESI];
    regs.rdi = env->regs[R_EDI];
    regs.rsp = env->regs[R_ESP];
    regs.rbp = env->regs[R_EBP];
#ifdef TARGET_X86_64
    regs.r8 = env->regs[8];
    regs.r9 = env->regs[9];
    regs.r10 = env->regs[10];
    regs.r11 = env->regs[11];
    regs.r12 = env->regs[12];
    regs.r13 = env->regs[13];
    regs.r14 = env->regs[14];
    regs.r15 = env->regs[15];
#endif

    regs.rflags = env->eflags;
    regs.rip = env->eip;

    kvm_set_regs(env, &regs);

#ifdef KVM_CAP_XSAVE
    if (kvm_check_extension(kvm_state, KVM_CAP_XSAVE)) {
        struct kvm_xsave* xsave;

        uint16_t cwd, swd, twd, fop;

        xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
        memset(xsave, 0, sizeof(struct kvm_xsave));
        cwd = swd = twd = fop = 0;
        swd = env->fpus & ~(7 << 11);
        swd |= (env->fpstt & 7) << 11;
        cwd = env->fpuc;
        for (i = 0; i < 8; ++i)
            twd |= (!env->fptags[i]) << i;
        xsave->region[0] = (uint32_t)(swd << 16) + cwd;
        xsave->region[1] = (uint32_t)(fop << 16) + twd;
        memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
                sizeof env->fpregs);
        memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
                sizeof env->xmm_regs);
        xsave->region[XSAVE_MXCSR] = env->mxcsr;
        *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
        memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
                sizeof env->ymmh_regs);
        kvm_set_xsave(env, xsave);
        if (kvm_check_extension(kvm_state, KVM_CAP_XCRS)) {
            struct kvm_xcrs xcrs;

            xcrs.nr_xcrs = 1;
            xcrs.flags = 0;
            xcrs.xcrs[0].xcr = 0;
            xcrs.xcrs[0].value = env->xcr0;
            kvm_set_xcrs(env, &xcrs);
        }
        qemu_free(xsave);
    } else {
#endif
        memset(&fpu, 0, sizeof fpu);
        fpu.fsw = env->fpus & ~(7 << 11);
        fpu.fsw |= (env->fpstt & 7) << 11;
        fpu.fcw = env->fpuc;
        for (i = 0; i < 8; ++i)
            fpu.ftwx |= (!env->fptags[i]) << i;
        memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
        memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
        fpu.mxcsr = env->mxcsr;
        kvm_set_fpu(env, &fpu);
#ifdef KVM_CAP_XSAVE
    }
#endif

    memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
    if (env->interrupt_injected >= 0) {
        sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
                (uint64_t)1 << (env->interrupt_injected % 64);
    }

    if ((env->eflags & VM_MASK)) {
	    set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
	    set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
	    set_v8086_seg(&sregs.es, &env->segs[R_ES]);
	    set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
	    set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
	    set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
    } else {
	    set_seg(&sregs.cs, &env->segs[R_CS]);
	    set_seg(&sregs.ds, &env->segs[R_DS]);
	    set_seg(&sregs.es, &env->segs[R_ES]);
	    set_seg(&sregs.fs, &env->segs[R_FS]);
	    set_seg(&sregs.gs, &env->segs[R_GS]);
	    set_seg(&sregs.ss, &env->segs[R_SS]);

	    if (env->cr[0] & CR0_PE_MASK) {
		/* force ss cpl to cs cpl */
		sregs.ss.selector = (sregs.ss.selector & ~3) |
			(sregs.cs.selector & 3);
		sregs.ss.dpl = sregs.ss.selector & 3;
	    }
    }

    set_seg(&sregs.tr, &env->tr);
    set_seg(&sregs.ldt, &env->ldt);

    sregs.idt.limit = env->idt.limit;
    sregs.idt.base = env->idt.base;
    sregs.gdt.limit = env->gdt.limit;
    sregs.gdt.base = env->gdt.base;

    sregs.cr0 = env->cr[0];
    sregs.cr2 = env->cr[2];
    sregs.cr3 = env->cr[3];
    sregs.cr4 = env->cr[4];

    sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
    sregs.apic_base = cpu_get_apic_base(env->apic_state);

    sregs.efer = env->efer;

    kvm_set_sregs(env, &sregs);

    /* msrs */
    n = 0;
    /* Remember to increase msrs size if you add new registers below */
    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS,  env->sysenter_cs);
    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
    if (kvm_has_msr_star)
	kvm_msr_entry_set(&msrs[n++], MSR_STAR,              env->star);
    if (kvm_has_vm_hsave_pa)
        kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
#ifdef TARGET_X86_64
    if (lm_capable_kernel) {
        kvm_msr_entry_set(&msrs[n++], MSR_CSTAR,             env->cstar);
        kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE,      env->kernelgsbase);
        kvm_msr_entry_set(&msrs[n++], MSR_FMASK,             env->fmask);
        kvm_msr_entry_set(&msrs[n++], MSR_LSTAR  ,           env->lstar);
    }
#endif
    if (level == KVM_PUT_FULL_STATE) {
        /*
         * KVM is yet unable to synchronize TSC values of multiple VCPUs on
         * writeback. Until this is fixed, we only write the offset to SMP
         * guests after migration, desynchronizing the VCPUs, but avoiding
         * huge jump-backs that would occur without any writeback at all.
         */
        if (smp_cpus == 1 || env->tsc != 0) {
            kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
        }
        kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME, env->system_time_msr);
        kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
    }
#ifdef KVM_CAP_MCE
    if (env->mcg_cap) {
        if (level == KVM_PUT_RESET_STATE)
            kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
        else if (level == KVM_PUT_FULL_STATE) {
            kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
            kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
            for (i = 0; i < (env->mcg_cap & 0xff); i++)
                kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
        }
    }
#endif

    rc = kvm_set_msrs(env, msrs, n);
    if (rc == -1)
        perror("kvm_set_msrs FAILED");

    if (level >= KVM_PUT_RESET_STATE) {
        kvm_arch_load_mpstate(env);
        kvm_load_lapic(env);
    }
    if (level == KVM_PUT_FULL_STATE) {
        if (env->kvm_vcpu_update_vapic)
            kvm_tpr_enable_vapic(env);
    }

    kvm_put_vcpu_events(env, level);
    kvm_put_debugregs(env);

    /* must be last */
    kvm_guest_debug_workarounds(env);
}

void kvm_arch_save_regs(CPUState *env)
{
    struct kvm_regs regs;
    struct kvm_fpu fpu;
    struct kvm_sregs sregs;
    struct kvm_msr_entry msrs[100];
    uint32_t hflags;
    uint32_t i, n, rc, bit;

    assert(kvm_cpu_is_stopped(env) || env->thread_id == kvm_get_thread_id());

    kvm_get_regs(env, &regs);

    env->regs[R_EAX] = regs.rax;
    env->regs[R_EBX] = regs.rbx;
    env->regs[R_ECX] = regs.rcx;
    env->regs[R_EDX] = regs.rdx;
    env->regs[R_ESI] = regs.rsi;
    env->regs[R_EDI] = regs.rdi;
    env->regs[R_ESP] = regs.rsp;
    env->regs[R_EBP] = regs.rbp;
#ifdef TARGET_X86_64
    env->regs[8] = regs.r8;
    env->regs[9] = regs.r9;
    env->regs[10] = regs.r10;
    env->regs[11] = regs.r11;
    env->regs[12] = regs.r12;
    env->regs[13] = regs.r13;
    env->regs[14] = regs.r14;
    env->regs[15] = regs.r15;
#endif

    env->eflags = regs.rflags;
    env->eip = regs.rip;

#ifdef KVM_CAP_XSAVE
    if (kvm_check_extension(kvm_state, KVM_CAP_XSAVE)) {
        struct kvm_xsave* xsave;
        uint16_t cwd, swd, twd, fop;
        xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
        kvm_get_xsave(env, xsave);
        cwd = (uint16_t)xsave->region[0];
        swd = (uint16_t)(xsave->region[0] >> 16);
        twd = (uint16_t)xsave->region[1];
        fop = (uint16_t)(xsave->region[1] >> 16);
        env->fpstt = (swd >> 11) & 7;
        env->fpus = swd;
        env->fpuc = cwd;
        for (i = 0; i < 8; ++i)
            env->fptags[i] = !((twd >> i) & 1);
        env->mxcsr = xsave->region[XSAVE_MXCSR];
        memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
                sizeof env->fpregs);
        memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
                sizeof env->xmm_regs);
        env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
        memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
                sizeof env->ymmh_regs);
        if (kvm_check_extension(kvm_state, KVM_CAP_XCRS)) {
            struct kvm_xcrs xcrs;

            kvm_get_xcrs(env, &xcrs);
            if (xcrs.xcrs[0].xcr == 0)
                env->xcr0 = xcrs.xcrs[0].value;
        }
        qemu_free(xsave);
    } else {
#endif
        kvm_get_fpu(env, &fpu);
        env->fpstt = (fpu.fsw >> 11) & 7;
        env->fpus = fpu.fsw;
        env->fpuc = fpu.fcw;
        for (i = 0; i < 8; ++i)
            env->fptags[i] = !((fpu.ftwx >> i) & 1);
        memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
        memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
        env->mxcsr = fpu.mxcsr;
#ifdef KVM_CAP_XSAVE
    }
#endif

    kvm_get_sregs(env, &sregs);

    /* There can only be one pending IRQ set in the bitmap at a time, so try
       to find it and save its number instead (-1 for none). */
    env->interrupt_injected = -1;
    for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
        if (sregs.interrupt_bitmap[i]) {
            bit = ctz64(sregs.interrupt_bitmap[i]);
            env->interrupt_injected = i * 64 + bit;
            break;
        }
    }

    get_seg(&env->segs[R_CS], &sregs.cs);
    get_seg(&env->segs[R_DS], &sregs.ds);
    get_seg(&env->segs[R_ES], &sregs.es);
    get_seg(&env->segs[R_FS], &sregs.fs);
    get_seg(&env->segs[R_GS], &sregs.gs);
    get_seg(&env->segs[R_SS], &sregs.ss);

    get_seg(&env->tr, &sregs.tr);
    get_seg(&env->ldt, &sregs.ldt);

    env->idt.limit = sregs.idt.limit;
    env->idt.base = sregs.idt.base;
    env->gdt.limit = sregs.gdt.limit;
    env->gdt.base = sregs.gdt.base;

    env->cr[0] = sregs.cr0;
    env->cr[2] = sregs.cr2;
    env->cr[3] = sregs.cr3;
    env->cr[4] = sregs.cr4;

    cpu_set_apic_base(env->apic_state, sregs.apic_base);

    env->efer = sregs.efer;
    //cpu_set_apic_tpr(env, sregs.cr8);

#define HFLAG_COPY_MASK ~( \
			HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
			HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
			HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
			HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)



    hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
    hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
    hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
	    (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
    hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
    hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
	    (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);

    if (env->efer & MSR_EFER_LMA) {
        hflags |= HF_LMA_MASK;
    }

    if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
        hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
    } else {
        hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
		(DESC_B_SHIFT - HF_CS32_SHIFT);
        hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
		(DESC_B_SHIFT - HF_SS32_SHIFT);
        if (!(env->cr[0] & CR0_PE_MASK) ||
                   (env->eflags & VM_MASK) ||
                   !(hflags & HF_CS32_MASK)) {
                hflags |= HF_ADDSEG_MASK;
            } else {
                hflags |= ((env->segs[R_DS].base |
                                env->segs[R_ES].base |
                                env->segs[R_SS].base) != 0) <<
                    HF_ADDSEG_SHIFT;
            }
    }
    env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;

    /* msrs */
    n = 0;
    /* Remember to increase msrs size if you add new registers below */
    msrs[n++].index = MSR_IA32_SYSENTER_CS;
    msrs[n++].index = MSR_IA32_SYSENTER_ESP;
    msrs[n++].index = MSR_IA32_SYSENTER_EIP;
    if (kvm_has_msr_star)
	msrs[n++].index = MSR_STAR;
    msrs[n++].index = MSR_IA32_TSC;
    if (kvm_has_vm_hsave_pa)
        msrs[n++].index = MSR_VM_HSAVE_PA;
#ifdef TARGET_X86_64
    if (lm_capable_kernel) {
        msrs[n++].index = MSR_CSTAR;
        msrs[n++].index = MSR_KERNELGSBASE;
        msrs[n++].index = MSR_FMASK;
        msrs[n++].index = MSR_LSTAR;
    }
#endif
    msrs[n++].index = MSR_KVM_SYSTEM_TIME;
    msrs[n++].index = MSR_KVM_WALL_CLOCK;

#ifdef KVM_CAP_MCE
    if (env->mcg_cap) {
        msrs[n++].index = MSR_MCG_STATUS;
        msrs[n++].index = MSR_MCG_CTL;
        for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++)
            msrs[n++].index = MSR_MC0_CTL + i;
    }
#endif

    rc = kvm_get_msrs(env, msrs, n);
    if (rc == -1) {
        perror("kvm_get_msrs FAILED");
    }
    else {
        n = rc; /* actual number of MSRs */
        for (i=0 ; i<n; i++) {
            if (get_msr_entry(&msrs[i], env))
                return;
        }
    }
    kvm_arch_save_mpstate(env);
    kvm_save_lapic(env);
    kvm_get_vcpu_events(env);
    kvm_get_debugregs(env);
}

static int _kvm_arch_init_vcpu(CPUState *env)
{
    kvm_arch_reset_vcpu(env);

#ifdef KVM_CAP_MCE
    if (((env->cpuid_version >> 8)&0xF) >= 6
        && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
        && kvm_check_extension(kvm_state, KVM_CAP_MCE) > 0) {
        uint64_t mcg_cap;
        int banks;

        if (kvm_get_mce_cap_supported(kvm_context, &mcg_cap, &banks))
            perror("kvm_get_mce_cap_supported FAILED");
        else {
            if (banks > MCE_BANKS_DEF)
                banks = MCE_BANKS_DEF;
            mcg_cap &= MCE_CAP_DEF;
            mcg_cap |= banks;
            if (kvm_setup_mce(env, &mcg_cap))
                perror("kvm_setup_mce FAILED");
            else
                env->mcg_cap = mcg_cap;
        }
    }
#endif

#ifdef KVM_EXIT_TPR_ACCESS
    kvm_enable_tpr_access_reporting(env);
#endif
    kvm_reset_mpstate(env);
    return 0;
}

int kvm_arch_halt(CPUState *env)
{

    if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
	  (env->eflags & IF_MASK)) &&
	!(env->interrupt_request & CPU_INTERRUPT_NMI)) {
            env->halted = 1;
    }
    return 1;
}

int kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
{
    if (!kvm_irqchip_in_kernel())
	kvm_set_cr8(env, cpu_get_apic_tpr(env->apic_state));
    return 0;
}

int kvm_arch_has_work(CPUState *env)
{
    if (((env->interrupt_request & CPU_INTERRUPT_HARD) &&
	 (env->eflags & IF_MASK)) ||
	(env->interrupt_request & CPU_INTERRUPT_NMI))
	return 1;
    return 0;
}

int kvm_arch_try_push_interrupts(void *opaque)
{
    CPUState *env = cpu_single_env;
    int r, irq;

    if (kvm_is_ready_for_interrupt_injection(env) &&
        (env->interrupt_request & CPU_INTERRUPT_HARD) &&
        (env->eflags & IF_MASK)) {
            env->interrupt_request &= ~CPU_INTERRUPT_HARD;
	    irq = cpu_get_pic_interrupt(env);
	    if (irq >= 0) {
		r = kvm_inject_irq(env, irq);
		if (r < 0)
		    printf("cpu %d fail inject %x\n", env->cpu_index, irq);
	    }
    }

    return (env->interrupt_request & CPU_INTERRUPT_HARD) != 0;
}

#ifdef KVM_CAP_USER_NMI
void kvm_arch_push_nmi(void *opaque)
{
    CPUState *env = cpu_single_env;
    int r;

    if (likely(!(env->interrupt_request & CPU_INTERRUPT_NMI)))
        return;

    env->interrupt_request &= ~CPU_INTERRUPT_NMI;
    r = kvm_inject_nmi(env);
    if (r < 0)
        printf("cpu %d fail inject NMI\n", env->cpu_index);
}
#endif /* KVM_CAP_USER_NMI */

static int kvm_reset_msrs(CPUState *env)
{
    struct {
        struct kvm_msrs info;
        struct kvm_msr_entry entries[100];
    } msr_data;
    int n;
    struct kvm_msr_entry *msrs = msr_data.entries;
    uint32_t index;
    uint64_t data;

    if (!kvm_msr_list)
        return -1;

    for (n = 0; n < kvm_msr_list->nmsrs; n++) {
        index = kvm_msr_list->indices[n];
        switch (index) {
        case MSR_PAT:
            data = 0x0007040600070406ULL;
            break;
        default:
            data = 0;
        }
        kvm_msr_entry_set(&msrs[n], index, data);
    }

    msr_data.info.nmsrs = n;

    return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
}


void kvm_arch_cpu_reset(CPUState *env)
{
    kvm_reset_msrs(env);
    kvm_arch_reset_vcpu(env);
    kvm_reset_mpstate(env);
}

#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
void kvm_arch_do_ioperm(void *_data)
{
    struct ioperm_data *data = _data;
    ioperm(data->start_port, data->num, data->turn_on);
}
#endif

/*
 * Setup x86 specific IRQ routing
 */
int kvm_arch_init_irq_routing(void)
{
    int i, r;

    if (kvm_irqchip && kvm_has_gsi_routing(kvm_context)) {
        kvm_clear_gsi_routes(kvm_context);
        for (i = 0; i < 8; ++i) {
            if (i == 2)
                continue;
            r = kvm_add_irq_route(kvm_context, i, KVM_IRQCHIP_PIC_MASTER, i);
            if (r < 0)
                return r;
        }
        for (i = 8; i < 16; ++i) {
            r = kvm_add_irq_route(kvm_context, i, KVM_IRQCHIP_PIC_SLAVE, i - 8);
            if (r < 0)
                return r;
        }
        for (i = 0; i < 24; ++i) {
            if (i == 0) {
                r = kvm_add_irq_route(kvm_context, i, KVM_IRQCHIP_IOAPIC, 2);
            } else if (i != 2) {
                r = kvm_add_irq_route(kvm_context, i, KVM_IRQCHIP_IOAPIC, i);
            }
            if (r < 0)
                return r;
        }
        kvm_commit_irq_routes(kvm_context);
    }
    return 0;
}

void kvm_arch_process_irqchip_events(CPUState *env)
{
    if (env->interrupt_request & CPU_INTERRUPT_INIT) {
        kvm_cpu_synchronize_state(env);
        do_cpu_init(env);
    }
    if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
        kvm_cpu_synchronize_state(env);
        do_cpu_sipi(env);
    }
}