| /********************************************************************* |
| * |
| * (C) Copyright IBM Corp. 2010 |
| * |
| * This program is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License as published by the |
| * Free Software Foundation; either version 2 of the License, or (at your |
| * option) any later version. |
| * |
| * This program is distributed in the hope that it will be useful, but |
| * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY |
| * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * for more details. |
| * |
| * You should have received a copy of the GNU General Public License along |
| * with this program; if not, see <http://www.gnu.org/licenses>. |
| * |
| * Author: Chris Ward <tjcw@uk.ibm.com> |
| * |
| * Description: Blue Gene low-level driver copy_tofrom_user using |
| * BlueGene-specific hardware |
| * |
| * |
| ********************************************************************/ |
| #ifndef __BGP_MEMCPY_H__ |
| #define __BGP_MEMCPY_H__ |
| |
| |
| typedef struct |
| { |
| int use_dma ; |
| int use_fpu ; |
| int dma_threshold ; /* Use the BGP DMA unit for copy_tofrom_user this size or larger */ |
| int fpu_threshold ; /* Use the BGP FPU for copy_tofrom_user this size or larger */ |
| int verify_dma ; /* Whether to verify the copy (for diagnostics) */ |
| int verify_fpu ; /* Whether to verify the copy (for diagnostics) */ |
| int cycles_per_packet ; /* Estimate of number of cycles per packet, for local spin before looking at counters */ |
| int faults_until_disable ; /* Number of faults until we disable acceleration */ |
| int rate_observe_report_count ; /* Number of times out of 256 that the rate gets displayed */ |
| int handle_pagecrossing ; /* Whether the DMA version should attempt to handle page-boundary-crossings */ |
| int fpu_handle_pagecrossing_read ; /* Whether the FPU version should attempt to handle page-boundary-crossings on reads */ |
| int fpu_handle_pagecrossing_write ; /* Whether the FPU version should attempt to handle page-boundary-crossings on writes */ |
| int mask ; /* Whether to mask interrupts */ |
| int assist_active ; /* Whether to assist copypage and clearpages */ |
| /* int trace_count ; */ /* Number of trace records to cut before stopping */ |
| } bgp_memcpy_control_t ; |
| |
| extern bgp_memcpy_control_t bgp_memcpy_control ; |
| |
| enum { |
| k_copy_cause_fallback , |
| k_copy_verify_miscompares , |
| k_in_irq , |
| k_in_softirq , |
| |
| k_copy_verify_attempts , |
| k_copy_tofrom_user_calls , |
| k_copy_accelerate_successes , |
| k_copy_accelerate_rejects , |
| |
| k_copy_size_rejects , |
| k_copy_spanpage_rejects , |
| k_copy_crosspage_limitation_rejects , |
| k_copy_inconsistent_tlb_1_rejects , |
| |
| k_copy_inconsistent_tlb_2_rejects , |
| k_copy_no_counter_rejects , |
| k_copy_source_tlb_rejects , |
| k_copy_target_tlb_rejects , |
| |
| k_copy_source_rejects , |
| k_copy_target_rejects , |
| k_copy_unaligned_rejects , |
| k_copy_tlb_touches , |
| |
| k_copy_await_idle_zero , |
| k_copy_await_idle_low , |
| k_copy_await_idle_high , |
| k_copy_inconsistent_tlb_1_info , |
| |
| k_copy_inconsistent_tlb_2_info , |
| k_copy_segv_trap , |
| k_precopy_segv_trap , |
| k_postcopy_segv_trap , |
| |
| k_copy_statistics |
| }; |
| |
| /* The underlying assembler copy function, returns 0 iff it copies all the data */ |
| extern unsigned long __real__copy_tofrom_user(void *to, |
| const void __user *from, unsigned long size) ; |
| |
| extern unsigned int bgp_dma_memcpy_statistics[k_copy_statistics] ; |
| static inline void dma_memcpy_statistic(unsigned int X) |
| { |
| bgp_dma_memcpy_statistics[X] += 1 ; |
| } |
| |
| extern unsigned long bgp_dma_instrument_copy_tofrom_user(void *to, |
| const void *from, unsigned long size) ; |
| extern unsigned long bgp_fpu_instrument_copy_tofrom_user(void *to, |
| const void *from, unsigned long size) ; |
| |
| enum |
| { |
| k_diagnose = 1 |
| }; |
| /* Items to record about a copy op, for diagnosing faults */ |
| typedef struct |
| { |
| const void * vaddr ; |
| unsigned int tlb_v ; |
| unsigned int pageid ; |
| unsigned int xlat ; |
| unsigned int attrib ; |
| } tlb_t ; |
| |
| typedef struct |
| { |
| void * to_vaddr ; |
| const void * from_vaddr ; |
| unsigned int size ; |
| tlb_t a_tlb ; |
| tlb_t b_tlb ; |
| unsigned int a_raddress ; |
| unsigned int b_raddress ; |
| unsigned int from_check_pre ; |
| unsigned int to_check_pre ; |
| unsigned int from_check_post ; |
| unsigned int to_check_post ; |
| unsigned int frag_index ; |
| } copy_op_t ; |
| |
| static void diagnose_tlb(tlb_t *t) |
| { |
| unsigned int t0=t->pageid ; |
| unsigned int t1=t->xlat ; |
| unsigned int t2=t->attrib ; |
| TRACEN(k_t_request,"vaddr=%p tlb_v=0x%08x %08x-%08x-%08x ts=%d tid=0x%02x epn=0x%08x rpn=0x%01x-%08x size=%s WIMG=%d%d%d%d U=%d%d%d%d V=%d uxwr=%d sxwr=%d", |
| t->vaddr,t->tlb_v,t0,t1,t2, |
| (t0 & TLB0_TS) ? 1 : 0, |
| (t2 >> 22) & 0xff , |
| TLB0_EPN_1K(t0), |
| TLB1_ERPN(t1),TLB1_RPN_1K(t1), |
| TLB_SIZES[(t0 & 0xF0) >> 4], |
| (t2 & TLB2_W) ? 1 : 0, |
| (t2 & TLB2_I) ? 1 : 0, |
| (t2 & TLB2_M) ? 1 : 0, |
| (t2 & TLB2_G) ? 1 : 0, |
| (t2 & TLB2_U0) ? 1 : 0, |
| (t2 & TLB2_U1) ? 1 : 0, |
| (t2 & TLB2_U2) ? 1 : 0, |
| (t2 & TLB2_U3) ? 1 : 0, |
| (t0 & TLB0_V) ? 1 : 0, |
| (t2 >> 3) & 7, |
| t2 & 7 |
| ) ; |
| } |
| static void diagnose_faulty_copy(copy_op_t *c) __attribute__((unused)) ; |
| static void diagnose_faulty_copy(copy_op_t *c) |
| { |
| TRACEN(k_t_request,"from_vaddr=%p to_vaddr=%p size=0x%08x a_raddress=0x%08x b_raddress=0x%08x from_check_pre=0x%08x to_check_pre=0x%08x from_check_post=0x%08x to_check_post=0x%08x frag_index=%d", |
| c->from_vaddr,c->to_vaddr,c->size,c->a_raddress,c->b_raddress,c->from_check_pre,c->from_check_post,c->to_check_pre,c->to_check_post,c->frag_index) ; |
| diagnose_tlb(&c->a_tlb) ; |
| diagnose_tlb(&c->b_tlb) ; |
| } |
| |
| /* Find the real store address for a virtual address, by looking at the TLB and causing a TLB miss if needed */ |
| static unsigned int v_to_r_maybe(const void * vaddr,tlb_t *t) |
| { |
| unsigned int vaddr_int=(unsigned int)vaddr ; |
| int tlbx=search_tlb_v(vaddr_int) ; |
| int pageid=get_tlb_pageid(tlbx) ; |
| int xlat=get_tlb_xlat(tlbx) ; |
| int attrib=get_tlb_attrib(tlbx) ; |
| int tlbx1=search_tlb_v((unsigned int)vaddr) ; |
| if( k_diagnose) |
| { |
| t->vaddr = vaddr ; |
| t->tlb_v = tlbx1 ; |
| t->pageid = pageid ; |
| t->xlat = xlat ; |
| t->attrib = attrib ; |
| } |
| if( (tlbx == tlbx1) /* Translation didn't change under me due to e.g. interrupt */ |
| && ((pageid & TLB0_V) != 0) /* TLB is valid */ |
| && ((tlbx & 0x20000000) != 0) /* search_tlb_v sets this bit if it found a translation */ |
| ) |
| { |
| unsigned int epn = TLB0_EPN_1K(pageid) ; // virtual page for the TLB |
| unsigned int rpn = TLB1_RPN_1K(xlat) ; // real page for the TLB |
| unsigned int result = (vaddr_int-epn) + rpn ; |
| TRACEN(k_t_dmacopy,"vaddr=%p tlbx=0x%08x pageid=0x%08x xlat=0x%08x attrib=0x%08x epn=0x%08x rpn=0x%08x result=0x%08x", |
| vaddr,tlbx,pageid,xlat,attrib,epn,rpn,result) ; |
| return result ; |
| |
| } |
| else |
| { |
| TRACEN(k_t_dmacopy,"vaddr=%p tlbx=0x%08x pageid=0x%08x tlbx1=0x%08x unmapped", |
| vaddr,tlbx,pageid,tlbx1) ; |
| return (unsigned int) -1 ; // Not mapped |
| } |
| } |
| |
| #endif |