| From 436816bc3897ca0d8b5d486453013dc0e6ffbbdb Mon Sep 17 00:00:00 2001 |
| From: Peter Zijlstra <peterz@infradead.org> |
| Date: Fri, 3 Jul 2009 08:44:54 -0500 |
| Subject: [PATCH] mm: shrink the page frame to !-rt size |
| |
| commit 27909c87933670deead6ab74274cf61ebffad5ac in tip. |
| |
| He below is a boot-tested hack to shrink the page frame size back to |
| normal. |
| |
| Should be a net win since there should be many less PTE-pages than |
| page-frames. |
| |
| Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> |
| Signed-off-by: Thomas Gleixner <tglx@linutronix.de> |
| |
| diff --git a/include/linux/mm.h b/include/linux/mm.h |
| index 9817658..fc0f76f 100644 |
| --- a/include/linux/mm.h |
| +++ b/include/linux/mm.h |
| @@ -947,27 +947,85 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a |
| * overflow into the next struct page (as it might with DEBUG_SPINLOCK). |
| * When freeing, reset page->mapping so free_pages_check won't complain. |
| */ |
| +#ifndef CONFIG_PREEMPT_RT |
| + |
| #define __pte_lockptr(page) &((page)->ptl) |
| -#define pte_lock_init(_page) do { \ |
| - spin_lock_init(__pte_lockptr(_page)); \ |
| -} while (0) |
| + |
| +static inline struct page *pte_lock_init(struct page *page) |
| +{ |
| + spin_lock_init(__pte_lockptr(page)); |
| + return page; |
| +} |
| + |
| #define pte_lock_deinit(page) ((page)->mapping = NULL) |
| + |
| +#else /* PREEMPT_RT */ |
| + |
| +/* |
| + * On PREEMPT_RT the spinlock_t's are too large to embed in the |
| + * page frame, hence it only has a pointer and we need to dynamically |
| + * allocate the lock when we allocate PTE-pages. |
| + * |
| + * This is an overall win, since only a small fraction of the pages |
| + * will be PTE pages under normal circumstances. |
| + */ |
| + |
| +#define __pte_lockptr(page) ((page)->ptl) |
| + |
| +/* |
| + * Heinous hack, relies on the caller doing something like: |
| + * |
| + * pte = alloc_pages(PGALLOC_GFP, 0); |
| + * if (pte) |
| + * pgtable_page_ctor(pte); |
| + * return pte; |
| + * |
| + * This ensures we release the page and return NULL when the |
| + * lock allocation fails. |
| + */ |
| +static inline struct page *pte_lock_init(struct page *page) |
| +{ |
| + page->ptl = kmalloc(sizeof(spinlock_t), GFP_KERNEL); |
| + if (page->ptl) { |
| + spin_lock_init(__pte_lockptr(page)); |
| + } else { |
| + __free_page(page); |
| + page = NULL; |
| + } |
| + return page; |
| +} |
| + |
| +static inline void pte_lock_deinit(struct page *page) |
| +{ |
| + kfree(page->ptl); |
| + page->mapping = NULL; |
| +} |
| + |
| +#endif /* PREEMPT_RT */ |
| + |
| #define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));}) |
| #else /* !USE_SPLIT_PTLOCKS */ |
| /* |
| * We use mm->page_table_lock to guard all pagetable pages of the mm. |
| */ |
| -#define pte_lock_init(page) do {} while (0) |
| +static inline struct page *pte_lock_init(struct page *page) { return page; } |
| #define pte_lock_deinit(page) do {} while (0) |
| #define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;}) |
| #endif /* USE_SPLIT_PTLOCKS */ |
| |
| -static inline void pgtable_page_ctor(struct page *page) |
| +static inline struct page *__pgtable_page_ctor(struct page *page) |
| { |
| - pte_lock_init(page); |
| - inc_zone_page_state(page, NR_PAGETABLE); |
| + page = pte_lock_init(page); |
| + if (page) |
| + inc_zone_page_state(page, NR_PAGETABLE); |
| + return page; |
| } |
| |
| +#define pgtable_page_ctor(page) \ |
| +do { \ |
| + page = __pgtable_page_ctor(page); \ |
| +} while (0) |
| + |
| static inline void pgtable_page_dtor(struct page *page) |
| { |
| pte_lock_deinit(page); |
| diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h |
| index b1a31eb..bd17761 100644 |
| --- a/include/linux/mm_types.h |
| +++ b/include/linux/mm_types.h |
| @@ -69,7 +69,11 @@ struct page { |
| */ |
| }; |
| #if USE_SPLIT_PTLOCKS |
| +#ifndef CONFIG_PREEMPT_RT |
| spinlock_t ptl; |
| +#else |
| + spinlock_t *ptl; |
| +#endif |
| #endif |
| struct kmem_cache *slab; /* SLUB: Pointer to slab */ |
| struct page *first_page; /* Compound tail pages */ |
| -- |
| 1.7.1.1 |
| |