x86/paravirt: abstract calling convention for pte_val/make_pte/etc.
Signed-off-by: Anders Kaseorg <anders.kaseorg@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index a7d2db9..6a80c92 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -464,60 +464,27 @@
static inline pte_t __pte(pteval_t val)
{
- pteval_t ret;
-
- if (sizeof(pteval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pteval_t,
- pv_mmu_ops.make_pte,
- val, (u64)val >> 32);
- else
- ret = PVOP_CALLEE1(pteval_t,
- pv_mmu_ops.make_pte,
- val);
-
- return (pte_t) { .pte = ret };
+ return (pte_t) { .pte = PVOP_LIKELY_IDENT(pteval_t,
+ pv_mmu_ops.make_pte,
+ val) };
}
static inline pteval_t pte_val(pte_t pte)
{
- pteval_t ret;
-
- if (sizeof(pteval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
- pte.pte, (u64)pte.pte >> 32);
- else
- ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
- pte.pte);
-
- return ret;
+ return PVOP_LIKELY_IDENT(pteval_t, pv_mmu_ops.pte_val,
+ pte.pte);
}
static inline pgd_t __pgd(pgdval_t val)
{
- pgdval_t ret;
-
- if (sizeof(pgdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
- val, (u64)val >> 32);
- else
- ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
- val);
-
- return (pgd_t) { ret };
+ return (pgd_t) { PVOP_LIKELY_IDENT(pgdval_t, pv_mmu_ops.make_pgd,
+ val) };
}
static inline pgdval_t pgd_val(pgd_t pgd)
{
- pgdval_t ret;
-
- if (sizeof(pgdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
- pgd.pgd, (u64)pgd.pgd >> 32);
- else
- ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
- pgd.pgd);
-
- return ret;
+ return PVOP_LIKELY_IDENT(pgdval_t, pv_mmu_ops.pgd_val,
+ pgd.pgd);
}
#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
@@ -589,30 +556,14 @@
#if PAGETABLE_LEVELS >= 3
static inline pmd_t __pmd(pmdval_t val)
{
- pmdval_t ret;
-
- if (sizeof(pmdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
- val, (u64)val >> 32);
- else
- ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
- val);
-
- return (pmd_t) { ret };
+ return (pmd_t) { PVOP_LIKELY_IDENT(pmdval_t, pv_mmu_ops.make_pmd,
+ val) };
}
static inline pmdval_t pmd_val(pmd_t pmd)
{
- pmdval_t ret;
-
- if (sizeof(pmdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
- pmd.pmd, (u64)pmd.pmd >> 32);
- else
- ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
- pmd.pmd);
-
- return ret;
+ return PVOP_LIKELY_IDENT(pmdval_t, pv_mmu_ops.pmd_val,
+ pmd.pmd);
}
static inline void set_pud(pud_t *pudp, pud_t pud)
@@ -629,30 +580,14 @@
#if PAGETABLE_LEVELS == 4
static inline pud_t __pud(pudval_t val)
{
- pudval_t ret;
-
- if (sizeof(pudval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
- val, (u64)val >> 32);
- else
- ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
- val);
-
- return (pud_t) { ret };
+ return (pud_t) { PVOP_LIKELY_IDENT(pudval_t, pv_mmu_ops.make_pud,
+ val) };
}
static inline pudval_t pud_val(pud_t pud)
{
- pudval_t ret;
-
- if (sizeof(pudval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
- pud.pud, (u64)pud.pud >> 32);
- else
- ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
- pud.pud);
-
- return ret;
+ return PVOP_LIKELY_IDENT(pudval_t, pv_mmu_ops.pud_val,
+ pud.pud);
}
static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
@@ -857,6 +792,11 @@
#define __PV_IS_CALLEE_SAVE(func) \
((struct paravirt_callee_save) { func })
+#define PV_NOT_IDENT_REGS_THUNK(func) PV_CALLEE_SAVE_REGS_THUNK(func)
+#define PV_NOT_IDENT(func) PV_CALLEE_SAVE(func)
+#define PV_IDENT_32 __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
+#define PV_IDENT_64 __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
+
static inline notrace unsigned long arch_local_save_flags(void)
{
return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl);
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 8e8b9a4..0b90b10 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -60,6 +60,8 @@
void *func;
};
+typedef struct paravirt_callee_save paravirt_likely_ident;
+
/* general info */
struct pv_info {
unsigned int kernel_rpl;
@@ -288,11 +290,11 @@
void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte);
- struct paravirt_callee_save pte_val;
- struct paravirt_callee_save make_pte;
+ paravirt_likely_ident pte_val;
+ paravirt_likely_ident make_pte;
- struct paravirt_callee_save pgd_val;
- struct paravirt_callee_save make_pgd;
+ paravirt_likely_ident pgd_val;
+ paravirt_likely_ident make_pgd;
#if PAGETABLE_LEVELS >= 3
#ifdef CONFIG_X86_PAE
@@ -305,12 +307,12 @@
void (*set_pud)(pud_t *pudp, pud_t pudval);
- struct paravirt_callee_save pmd_val;
- struct paravirt_callee_save make_pmd;
+ paravirt_likely_ident pmd_val;
+ paravirt_likely_ident make_pmd;
#if PAGETABLE_LEVELS == 4
- struct paravirt_callee_save pud_val;
- struct paravirt_callee_save make_pud;
+ paravirt_likely_ident pud_val;
+ paravirt_likely_ident make_pud;
void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
#endif /* PAGETABLE_LEVELS == 4 */
@@ -667,6 +669,12 @@
PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
#endif
+#define PVOP_LIKELY_IDENT(rettype, op, arg) \
+ (sizeof(rettype) > sizeof(long) ? \
+ PVOP_CALLEE2(rettype, op, arg, (u64)(arg) >> 32) : \
+ PVOP_CALLEE1(rettype, op, arg))
+
+
/* Lazy mode for batching updates / context switch */
enum paravirt_lazy_mode {
PARAVIRT_LAZY_NONE,
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..712688a 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -403,10 +403,10 @@
#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
/* 32-bit pagetable entries */
-#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
+#define PTE_IDENT PV_IDENT_32
#else
/* 64-bit pagetable entries */
-#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
+#define PTE_IDENT PV_IDENT_64
#endif
struct pv_mmu_ops pv_mmu_ops = {
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 87f6673..63510a9 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -427,13 +427,13 @@
return pte_mfn_to_pfn(pteval);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
+PV_NOT_IDENT_REGS_THUNK(xen_pte_val);
static pgdval_t xen_pgd_val(pgd_t pgd)
{
return pte_mfn_to_pfn(pgd.pgd);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
+PV_NOT_IDENT_REGS_THUNK(xen_pgd_val);
/*
* Xen's PAT setup is part of its ABI, though I assume entries 6 & 7
@@ -493,20 +493,20 @@
return native_make_pte(pte);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
+PV_NOT_IDENT_REGS_THUNK(xen_make_pte);
static pgd_t xen_make_pgd(pgdval_t pgd)
{
pgd = pte_pfn_to_mfn(pgd);
return native_make_pgd(pgd);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
+PV_NOT_IDENT_REGS_THUNK(xen_make_pgd);
static pmdval_t xen_pmd_val(pmd_t pmd)
{
return pte_mfn_to_pfn(pmd.pmd);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
+PV_NOT_IDENT_REGS_THUNK(xen_pmd_val);
static void xen_set_pud_hyper(pud_t *ptr, pud_t val)
{
@@ -566,14 +566,14 @@
pmd = pte_pfn_to_mfn(pmd);
return native_make_pmd(pmd);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
+PV_NOT_IDENT_REGS_THUNK(xen_make_pmd);
#if PAGETABLE_LEVELS == 4
static pudval_t xen_pud_val(pud_t pud)
{
return pte_mfn_to_pfn(pud.pud);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
+PV_NOT_IDENT_REGS_THUNK(xen_pud_val);
static pud_t xen_make_pud(pudval_t pud)
{
@@ -581,7 +581,7 @@
return native_make_pud(pud);
}
-PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
+PV_NOT_IDENT_REGS_THUNK(xen_make_pud);
static pgd_t *xen_get_user_pgd(pgd_t *pgd)
{
@@ -2023,11 +2023,11 @@
.ptep_modify_prot_start = __ptep_modify_prot_start,
.ptep_modify_prot_commit = __ptep_modify_prot_commit,
- .pte_val = PV_CALLEE_SAVE(xen_pte_val),
- .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
+ .pte_val = PV_NOT_IDENT(xen_pte_val),
+ .pgd_val = PV_NOT_IDENT(xen_pgd_val),
- .make_pte = PV_CALLEE_SAVE(xen_make_pte),
- .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
+ .make_pte = PV_NOT_IDENT(xen_make_pte),
+ .make_pgd = PV_NOT_IDENT(xen_make_pgd),
#ifdef CONFIG_X86_PAE
.set_pte_atomic = xen_set_pte_atomic,
@@ -2036,12 +2036,12 @@
#endif /* CONFIG_X86_PAE */
.set_pud = xen_set_pud_hyper,
- .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
- .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
+ .make_pmd = PV_NOT_IDENT(xen_make_pmd),
+ .pmd_val = PV_NOT_IDENT(xen_pmd_val),
#if PAGETABLE_LEVELS == 4
- .pud_val = PV_CALLEE_SAVE(xen_pud_val),
- .make_pud = PV_CALLEE_SAVE(xen_make_pud),
+ .pud_val = PV_NOT_IDENT(xen_pud_val),
+ .make_pud = PV_NOT_IDENT(xen_make_pud),
.set_pgd = xen_set_pgd_hyper,
.alloc_pud = xen_alloc_pmd_init,
