| From 29b0ff908a19b5d98e5a5d080a5b9f6d56ccdd5a Mon Sep 17 00:00:00 2001 |
| From: David S. Miller <davem@davemloft.net> |
| Date: Tue, 25 Aug 2009 16:47:46 -0700 |
| Subject: sparc64: Validate linear D-TLB misses. |
| |
| From: David S. Miller <davem@davemloft.net> |
| |
| [ Upstream commit d8ed1d43e17898761c7221014a15a4c7501d2ff3 ] |
| |
| When page alloc debugging is not enabled, we essentially accept any |
| virtual address for linear kernel TLB misses. But with kgdb, kernel |
| address probing, and other facilities we can try to access arbitrary |
| crap. |
| |
| So, make sure the address we miss on will translate to physical memory |
| that actually exists. |
| |
| In order to make this work we have to embed the valid address bitmap |
| into the kernel image. And in order to make that less expensive we |
| make an adjustment, in that the max physical memory address is |
| decreased to "1 << 41", even on the chips that support a 42-bit |
| physical address space. We can do this because bit 41 indicates |
| "I/O space" and thus covers non-memory ranges. |
| |
| The result of this is that: |
| |
| 1) kpte_linear_bitmap shrinks from 2K to 1K in size |
| |
| 2) we need 64K more for the valid address bitmap |
| |
| We can't let the valid address bitmap be dynamically allocated |
| once we start using it to validate TLB misses, otherwise we have |
| crazy issues to deal with wrt. recursive TLB misses and such. |
| |
| If we're in a TLB miss it could be the deepest trap level that's legal |
| inside of the cpu. So if we TLB miss referencing the bitmap, the cpu |
| will be out of trap levels and enter RED state. |
| |
| To guard against out-of-range accesses to the bitmap, we have to check |
| to make sure no bits in the physical address above bit 40 are set. We |
| could export and use last_valid_pfn for this check, but that's just an |
| unnecessary extra memory reference. |
| |
| On the plus side of all this, since we load all of these translations |
| into the special 4MB mapping TSB, and we check the TSB first for TLB |
| misses, there should be absolutely no real cost for these new checks |
| in the TLB miss path. |
| |
| Reported-by: heyongli@gmail.com |
| Signed-off-by: David S. Miller <davem@davemloft.net> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de> |
| --- |
| arch/sparc/include/asm/pgtable_64.h | 12 +++++++--- |
| arch/sparc/kernel/ktlb.S | 42 +++++++++++++++++++++++++++++++---- |
| arch/sparc/mm/init_64.c | 43 ++++++++++++++++++++---------------- |
| arch/sparc/mm/init_64.h | 7 ++++- |
| 4 files changed, 76 insertions(+), 28 deletions(-) |
| |
| --- a/arch/sparc/include/asm/pgtable_64.h |
| +++ b/arch/sparc/include/asm/pgtable_64.h |
| @@ -726,11 +726,17 @@ extern unsigned long pte_file(pte_t); |
| extern pte_t pgoff_to_pte(unsigned long); |
| #define PTE_FILE_MAX_BITS (64UL - PAGE_SHIFT - 1UL) |
| |
| -extern unsigned long *sparc64_valid_addr_bitmap; |
| +extern unsigned long sparc64_valid_addr_bitmap[]; |
| |
| /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ |
| -#define kern_addr_valid(addr) \ |
| - (test_bit(__pa((unsigned long)(addr))>>22, sparc64_valid_addr_bitmap)) |
| +static inline bool kern_addr_valid(unsigned long addr) |
| +{ |
| + unsigned long paddr = __pa(addr); |
| + |
| + if ((paddr >> 41UL) != 0UL) |
| + return false; |
| + return test_bit(paddr >> 22, sparc64_valid_addr_bitmap); |
| +} |
| |
| extern int page_in_phys_avail(unsigned long paddr); |
| |
| --- a/arch/sparc/kernel/ktlb.S |
| +++ b/arch/sparc/kernel/ktlb.S |
| @@ -151,12 +151,46 @@ kvmap_dtlb_4v: |
| * Must preserve %g1 and %g6 (TAG). |
| */ |
| kvmap_dtlb_tsb4m_miss: |
| - sethi %hi(kpte_linear_bitmap), %g2 |
| - or %g2, %lo(kpte_linear_bitmap), %g2 |
| + /* Clear the PAGE_OFFSET top virtual bits, shift |
| + * down to get PFN, and make sure PFN is in range. |
| + */ |
| + sllx %g4, 21, %g5 |
| |
| - /* Clear the PAGE_OFFSET top virtual bits, then shift |
| - * down to get a 256MB physical address index. |
| + /* Check to see if we know about valid memory at the 4MB |
| + * chunk this physical address will reside within. |
| */ |
| + srlx %g5, 21 + 41, %g2 |
| + brnz,pn %g2, kvmap_dtlb_longpath |
| + nop |
| + |
| + /* This unconditional branch and delay-slot nop gets patched |
| + * by the sethi sequence once the bitmap is properly setup. |
| + */ |
| + .globl valid_addr_bitmap_insn |
| +valid_addr_bitmap_insn: |
| + ba,pt %xcc, 2f |
| + nop |
| + .subsection 2 |
| + .globl valid_addr_bitmap_patch |
| +valid_addr_bitmap_patch: |
| + sethi %hi(sparc64_valid_addr_bitmap), %g7 |
| + or %g7, %lo(sparc64_valid_addr_bitmap), %g7 |
| + .previous |
| + |
| + srlx %g5, 21 + 22, %g2 |
| + srlx %g2, 6, %g5 |
| + and %g2, 63, %g2 |
| + sllx %g5, 3, %g5 |
| + ldx [%g7 + %g5], %g5 |
| + mov 1, %g7 |
| + sllx %g7, %g2, %g7 |
| + andcc %g5, %g7, %g0 |
| + be,pn %xcc, kvmap_dtlb_longpath |
| + |
| +2: sethi %hi(kpte_linear_bitmap), %g2 |
| + or %g2, %lo(kpte_linear_bitmap), %g2 |
| + |
| + /* Get the 256MB physical address index. */ |
| sllx %g4, 21, %g5 |
| mov 1, %g7 |
| srlx %g5, 21 + 28, %g5 |
| --- a/arch/sparc/mm/init_64.c |
| +++ b/arch/sparc/mm/init_64.c |
| @@ -145,7 +145,8 @@ static void __init read_obp_memory(const |
| cmp_p64, NULL); |
| } |
| |
| -unsigned long *sparc64_valid_addr_bitmap __read_mostly; |
| +unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES / |
| + sizeof(unsigned long)]; |
| EXPORT_SYMBOL(sparc64_valid_addr_bitmap); |
| |
| /* Kernel physical address base and size in bytes. */ |
| @@ -1876,7 +1877,7 @@ static int pavail_rescan_ents __initdata |
| * memory list again, and make sure it provides at least as much |
| * memory as 'pavail' does. |
| */ |
| -static void __init setup_valid_addr_bitmap_from_pavail(void) |
| +static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap) |
| { |
| int i; |
| |
| @@ -1899,8 +1900,7 @@ static void __init setup_valid_addr_bitm |
| |
| if (new_start <= old_start && |
| new_end >= (old_start + PAGE_SIZE)) { |
| - set_bit(old_start >> 22, |
| - sparc64_valid_addr_bitmap); |
| + set_bit(old_start >> 22, bitmap); |
| goto do_next_page; |
| } |
| } |
| @@ -1921,20 +1921,21 @@ static void __init setup_valid_addr_bitm |
| } |
| } |
| |
| +static void __init patch_tlb_miss_handler_bitmap(void) |
| +{ |
| + extern unsigned int valid_addr_bitmap_insn[]; |
| + extern unsigned int valid_addr_bitmap_patch[]; |
| + |
| + valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1]; |
| + mb(); |
| + valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0]; |
| + flushi(&valid_addr_bitmap_insn[0]); |
| +} |
| + |
| void __init mem_init(void) |
| { |
| unsigned long codepages, datapages, initpages; |
| unsigned long addr, last; |
| - int i; |
| - |
| - i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6); |
| - i += 1; |
| - sparc64_valid_addr_bitmap = (unsigned long *) alloc_bootmem(i << 3); |
| - if (sparc64_valid_addr_bitmap == NULL) { |
| - prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n"); |
| - prom_halt(); |
| - } |
| - memset(sparc64_valid_addr_bitmap, 0, i << 3); |
| |
| addr = PAGE_OFFSET + kern_base; |
| last = PAGE_ALIGN(kern_size) + addr; |
| @@ -1943,15 +1944,19 @@ void __init mem_init(void) |
| addr += PAGE_SIZE; |
| } |
| |
| - setup_valid_addr_bitmap_from_pavail(); |
| + setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap); |
| + patch_tlb_miss_handler_bitmap(); |
| |
| high_memory = __va(last_valid_pfn << PAGE_SHIFT); |
| |
| #ifdef CONFIG_NEED_MULTIPLE_NODES |
| - for_each_online_node(i) { |
| - if (NODE_DATA(i)->node_spanned_pages != 0) { |
| - totalram_pages += |
| - free_all_bootmem_node(NODE_DATA(i)); |
| + { |
| + int i; |
| + for_each_online_node(i) { |
| + if (NODE_DATA(i)->node_spanned_pages != 0) { |
| + totalram_pages += |
| + free_all_bootmem_node(NODE_DATA(i)); |
| + } |
| } |
| } |
| #else |
| --- a/arch/sparc/mm/init_64.h |
| +++ b/arch/sparc/mm/init_64.h |
| @@ -5,10 +5,13 @@ |
| * marked non-static so that assembler code can get at them. |
| */ |
| |
| -#define MAX_PHYS_ADDRESS (1UL << 42UL) |
| -#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) |
| +#define MAX_PHYS_ADDRESS (1UL << 41UL) |
| +#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) |
| #define KPTE_BITMAP_BYTES \ |
| ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8) |
| +#define VALID_ADDR_BITMAP_CHUNK_SZ (4UL * 1024UL * 1024UL) |
| +#define VALID_ADDR_BITMAP_BYTES \ |
| + ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8) |
| |
| extern unsigned long kern_linear_pte_xor[2]; |
| extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; |