| From f0a18819e261afc5fdbd8c5c6f9943123c5461ba Mon Sep 17 00:00:00 2001 |
| From: John David Anglin <dave.anglin@bell.net> |
| Date: Fri, 10 May 2013 23:21:38 +0000 |
| Subject: parisc: fix SMP races when updating PTE and TLB entries in entry.S |
| |
| From: John David Anglin <dave.anglin@bell.net> |
| |
| commit f0a18819e261afc5fdbd8c5c6f9943123c5461ba upstream. |
| |
| Currently, race conditions exist in the handling of TLB interruptions in |
| entry.S. In particular, dirty bit updates can be lost if an accessed |
| interruption occurs just after the dirty bit interruption on a different |
| cpu. Lost dirty bit updates result in user pages not being flushed and |
| general system instability. This change adds lock and unlock macros to |
| synchronize all PTE and TLB updates done in entry.S. As a result, |
| userspace stability is significantly improved. |
| |
| Signed-off-by: John David Anglin <dave.anglin@bell.net> |
| Signed-off-by: Helge Deller <deller@gmx.de> |
| Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
| |
| --- |
| arch/parisc/kernel/entry.S | 155 ++++++++++++++++++++++++--------------------- |
| 1 file changed, 83 insertions(+), 72 deletions(-) |
| |
| --- a/arch/parisc/kernel/entry.S |
| +++ b/arch/parisc/kernel/entry.S |
| @@ -444,9 +444,41 @@ |
| L2_ptep \pgd,\pte,\index,\va,\fault |
| .endm |
| |
| + /* Acquire pa_dbit_lock lock. */ |
| + .macro dbit_lock spc,tmp,tmp1 |
| +#ifdef CONFIG_SMP |
| + cmpib,COND(=),n 0,\spc,2f |
| + load32 PA(pa_dbit_lock),\tmp |
| +1: LDCW 0(\tmp),\tmp1 |
| + cmpib,COND(=) 0,\tmp1,1b |
| + nop |
| +2: |
| +#endif |
| + .endm |
| + |
| + /* Release pa_dbit_lock lock without reloading lock address. */ |
| + .macro dbit_unlock0 spc,tmp |
| +#ifdef CONFIG_SMP |
| + or,COND(=) %r0,\spc,%r0 |
| + stw \spc,0(\tmp) |
| +#endif |
| + .endm |
| + |
| + /* Release pa_dbit_lock lock. */ |
| + .macro dbit_unlock1 spc,tmp |
| +#ifdef CONFIG_SMP |
| + load32 PA(pa_dbit_lock),\tmp |
| + dbit_unlock0 \spc,\tmp |
| +#endif |
| + .endm |
| + |
| /* Set the _PAGE_ACCESSED bit of the PTE. Be clever and |
| * don't needlessly dirty the cache line if it was already set */ |
| - .macro update_ptep ptep,pte,tmp,tmp1 |
| + .macro update_ptep spc,ptep,pte,tmp,tmp1 |
| +#ifdef CONFIG_SMP |
| + or,COND(=) %r0,\spc,%r0 |
| + LDREG 0(\ptep),\pte |
| +#endif |
| ldi _PAGE_ACCESSED,\tmp1 |
| or \tmp1,\pte,\tmp |
| and,COND(<>) \tmp1,\pte,%r0 |
| @@ -455,7 +487,11 @@ |
| |
| /* Set the dirty bit (and accessed bit). No need to be |
| * clever, this is only used from the dirty fault */ |
| - .macro update_dirty ptep,pte,tmp |
| + .macro update_dirty spc,ptep,pte,tmp |
| +#ifdef CONFIG_SMP |
| + or,COND(=) %r0,\spc,%r0 |
| + LDREG 0(\ptep),\pte |
| +#endif |
| ldi _PAGE_ACCESSED|_PAGE_DIRTY,\tmp |
| or \tmp,\pte,\pte |
| STREG \pte,0(\ptep) |
| @@ -1103,11 +1139,13 @@ dtlb_miss_20w: |
| |
| L3_ptep ptp,pte,t0,va,dtlb_check_alias_20w |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| idtlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1127,11 +1165,13 @@ nadtlb_miss_20w: |
| |
| L3_ptep ptp,pte,t0,va,nadtlb_check_alias_20w |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| idtlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1153,7 +1193,8 @@ dtlb_miss_11: |
| |
| L2_ptep ptp,pte,t0,va,dtlb_check_alias_11 |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb_11 spc,pte,prot |
| |
| @@ -1164,6 +1205,7 @@ dtlb_miss_11: |
| idtlbp prot,(%sr1,va) |
| |
| mtsp t0, %sr1 /* Restore sr1 */ |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1184,7 +1226,8 @@ nadtlb_miss_11: |
| |
| L2_ptep ptp,pte,t0,va,nadtlb_check_alias_11 |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb_11 spc,pte,prot |
| |
| @@ -1196,6 +1239,7 @@ nadtlb_miss_11: |
| idtlbp prot,(%sr1,va) |
| |
| mtsp t0, %sr1 /* Restore sr1 */ |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1216,13 +1260,15 @@ dtlb_miss_20: |
| |
| L2_ptep ptp,pte,t0,va,dtlb_check_alias_20 |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| f_extend pte,t0 |
| |
| idtlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1242,13 +1288,15 @@ nadtlb_miss_20: |
| |
| L2_ptep ptp,pte,t0,va,nadtlb_check_alias_20 |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| f_extend pte,t0 |
| |
| idtlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1349,11 +1397,13 @@ itlb_miss_20w: |
| |
| L3_ptep ptp,pte,t0,va,itlb_fault |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| iitlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1371,11 +1421,13 @@ naitlb_miss_20w: |
| |
| L3_ptep ptp,pte,t0,va,naitlb_check_alias_20w |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| iitlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1397,7 +1449,8 @@ itlb_miss_11: |
| |
| L2_ptep ptp,pte,t0,va,itlb_fault |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb_11 spc,pte,prot |
| |
| @@ -1408,6 +1461,7 @@ itlb_miss_11: |
| iitlbp prot,(%sr1,va) |
| |
| mtsp t0, %sr1 /* Restore sr1 */ |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1419,7 +1473,8 @@ naitlb_miss_11: |
| |
| L2_ptep ptp,pte,t0,va,naitlb_check_alias_11 |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb_11 spc,pte,prot |
| |
| @@ -1430,6 +1485,7 @@ naitlb_miss_11: |
| iitlbp prot,(%sr1,va) |
| |
| mtsp t0, %sr1 /* Restore sr1 */ |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1451,13 +1507,15 @@ itlb_miss_20: |
| |
| L2_ptep ptp,pte,t0,va,itlb_fault |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| f_extend pte,t0 |
| |
| iitlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1469,13 +1527,15 @@ naitlb_miss_20: |
| |
| L2_ptep ptp,pte,t0,va,naitlb_check_alias_20 |
| |
| - update_ptep ptp,pte,t0,t1 |
| + dbit_lock spc,t0,t1 |
| + update_ptep spc,ptp,pte,t0,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| f_extend pte,t0 |
| |
| iitlbt pte,prot |
| + dbit_unlock1 spc,t0 |
| |
| rfir |
| nop |
| @@ -1499,29 +1559,13 @@ dbit_trap_20w: |
| |
| L3_ptep ptp,pte,t0,va,dbit_fault |
| |
| -#ifdef CONFIG_SMP |
| - cmpib,COND(=),n 0,spc,dbit_nolock_20w |
| - load32 PA(pa_dbit_lock),t0 |
| - |
| -dbit_spin_20w: |
| - LDCW 0(t0),t1 |
| - cmpib,COND(=) 0,t1,dbit_spin_20w |
| - nop |
| - |
| -dbit_nolock_20w: |
| -#endif |
| - update_dirty ptp,pte,t1 |
| + dbit_lock spc,t0,t1 |
| + update_dirty spc,ptp,pte,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| idtlbt pte,prot |
| -#ifdef CONFIG_SMP |
| - cmpib,COND(=),n 0,spc,dbit_nounlock_20w |
| - ldi 1,t1 |
| - stw t1,0(t0) |
| - |
| -dbit_nounlock_20w: |
| -#endif |
| + dbit_unlock0 spc,t0 |
| |
| rfir |
| nop |
| @@ -1535,18 +1579,8 @@ dbit_trap_11: |
| |
| L2_ptep ptp,pte,t0,va,dbit_fault |
| |
| -#ifdef CONFIG_SMP |
| - cmpib,COND(=),n 0,spc,dbit_nolock_11 |
| - load32 PA(pa_dbit_lock),t0 |
| - |
| -dbit_spin_11: |
| - LDCW 0(t0),t1 |
| - cmpib,= 0,t1,dbit_spin_11 |
| - nop |
| - |
| -dbit_nolock_11: |
| -#endif |
| - update_dirty ptp,pte,t1 |
| + dbit_lock spc,t0,t1 |
| + update_dirty spc,ptp,pte,t1 |
| |
| make_insert_tlb_11 spc,pte,prot |
| |
| @@ -1557,13 +1591,7 @@ dbit_nolock_11: |
| idtlbp prot,(%sr1,va) |
| |
| mtsp t1, %sr1 /* Restore sr1 */ |
| -#ifdef CONFIG_SMP |
| - cmpib,COND(=),n 0,spc,dbit_nounlock_11 |
| - ldi 1,t1 |
| - stw t1,0(t0) |
| - |
| -dbit_nounlock_11: |
| -#endif |
| + dbit_unlock0 spc,t0 |
| |
| rfir |
| nop |
| @@ -1575,32 +1603,15 @@ dbit_trap_20: |
| |
| L2_ptep ptp,pte,t0,va,dbit_fault |
| |
| -#ifdef CONFIG_SMP |
| - cmpib,COND(=),n 0,spc,dbit_nolock_20 |
| - load32 PA(pa_dbit_lock),t0 |
| - |
| -dbit_spin_20: |
| - LDCW 0(t0),t1 |
| - cmpib,= 0,t1,dbit_spin_20 |
| - nop |
| - |
| -dbit_nolock_20: |
| -#endif |
| - update_dirty ptp,pte,t1 |
| + dbit_lock spc,t0,t1 |
| + update_dirty spc,ptp,pte,t1 |
| |
| make_insert_tlb spc,pte,prot |
| |
| f_extend pte,t1 |
| |
| idtlbt pte,prot |
| - |
| -#ifdef CONFIG_SMP |
| - cmpib,COND(=),n 0,spc,dbit_nounlock_20 |
| - ldi 1,t1 |
| - stw t1,0(t0) |
| - |
| -dbit_nounlock_20: |
| -#endif |
| + dbit_unlock0 spc,t0 |
| |
| rfir |
| nop |