| /* SPDX-License-Identifier: GPL-2.0 */ |
| |
| /* |
| * template for memcpy and copy_user with SIMD |
| * |
| * $4: 8-byte misalignment of src when dest is 8-byte aligned |
| * $5: 32-byte misalignment of src when dest is 32-byte aligned |
| * $7: SIMD status |
| * 0: not in simd loop |
| * 1: in simd loop |
| * 2: in simd_u loop |
| * $16: latest dest, clobbered |
| * $17: latest src, clobbered |
| * $18: bytes left to copy |
| * |
| */ |
| |
| #define NC_STORE_THRESHOLD 2048 |
| |
| #define SAVE_SIMD_REGS \ |
| ldi $sp, -0x60($sp); \ |
| addl $sp, 0x1f, $23; \ |
| bic $23, 0x1f, $23; \ |
| vstd $f1, 0($23); \ |
| vstd $f2, 0x20($23); \ |
| ldi $7, 1 |
| |
| #define RESTORE_SIMD_REGS \ |
| addl $sp, 0x1f, $23; \ |
| bic $23, 0x1f, $23; \ |
| vldd $f1, 0($23); \ |
| vldd $f2, 0x20($23); \ |
| ldi $sp, 0x60($sp); \ |
| bis $31, $31, $7 |
| |
| #define SAVE_SIMD_U_REGS \ |
| ldi $sp, -0xc0($sp); \ |
| addl $sp, 0x1f, $23; \ |
| bic $23, 0x1f, $23; \ |
| vstd $f1, 0($23); \ |
| vstd $f2, 0x20($23); \ |
| vstd $f4, 0x40($23); \ |
| vstd $f5, 0x60($23); \ |
| vstd $f3, 0x80($23); \ |
| ldi $7, 2 |
| |
| #define RESTORE_SIMD_U_REGS \ |
| addl $sp, 0x1f, $23; \ |
| bic $23, 0x1f, $23; \ |
| vldd $f1, 0($23); \ |
| vldd $f2, 0x20($23); \ |
| vldd $f4, 0x40($23); \ |
| vldd $f5, 0x60($23); \ |
| vldd $f3, 0x80($23); \ |
| ldi $sp, 0xc0($sp); \ |
| bis $31, $31, $7 |
| |
| ble $18, $out |
| and $16, 7, $1 |
| beq $1, $dest_aligned_8 |
| |
| $byte_loop_head: |
| FIXUP_LDST( ldbu $2, 0($17) ) |
| FIXUP_LDST( stb $2, 0($16) ) |
| subl $18, 1, $18 |
| addl $17, 1, $17 |
| addl $16, 1, $16 |
| ble $18, $out |
| and $16, 7, $1 |
| bne $1, $byte_loop_head |
| |
| $dest_aligned_8: |
| and $17, 7, $4 |
| cmplt $18, 16, $1 |
| bne $1, $quad_loop_end |
| and $16, 31, $1 |
| beq $1, $dest_aligned_32 |
| cmplt $18, 64, $1 |
| bne $1, $simd_end |
| bne $4, $quad_u_loop_head |
| |
| $quad_loop_head: |
| FIXUP_LDST( ldl $2, 0($17) ) |
| FIXUP_LDST( stl $2, 0($16) ) |
| addl $16, 8, $16 |
| addl $17, 8, $17 |
| subl $18, 8, $18 |
| and $16, 31, $1 |
| beq $1, $dest_aligned_32 |
| br $31, $quad_loop_head |
| |
| $dest_aligned_32: |
| cmplt $18, 64, $1 |
| bne $1, $simd_end |
| and $17, 31, $5 |
| bne $5, $prep_simd_u_loop |
| |
| $prep_simd_loop: |
| SAVE_SIMD_REGS |
| ldi $1, NC_STORE_THRESHOLD($31) |
| cmple $18, $1, $1 |
| bne $1, $simd_loop |
| |
| .align 4 |
| $simd_loop_nc: |
| FIXUP_LDST( vldd $f1, 0($17) ) |
| FIXUP_LDST( vldd $f2, 32($17) ) |
| FIXUP_LDST( vstd_nc $f1, 0($16) ) |
| FIXUP_LDST( vstd_nc $f2, 32($16) ) |
| subl $18, 64, $18 |
| addl $17, 64, $17 |
| addl $16, 64, $16 |
| cmplt $18, 64, $1 |
| beq $1, $simd_loop_nc |
| memb # required for _nc store instructions |
| br $31, $simd_loop_end |
| |
| .align 4 |
| $simd_loop: |
| FIXUP_LDST( vldd $f1, 0($17) ) |
| FIXUP_LDST( vldd $f2, 32($17) ) |
| FIXUP_LDST( vstd $f1, 0($16) ) |
| FIXUP_LDST( vstd $f2, 32($16) ) |
| subl $18, 64, $18 |
| addl $17, 64, $17 |
| addl $16, 64, $16 |
| cmplt $18, 64, $1 |
| beq $1, $simd_loop |
| |
| $simd_loop_end: |
| cmplt $18, 32, $1 |
| bne $1, $no_more_simd |
| FIXUP_LDST( vldd $f1, 0($17) ) |
| FIXUP_LDST( vstd $f1, 0($16) ) |
| subl $18, 32, $18 |
| addl $17, 32, $17 |
| addl $16, 32, $16 |
| |
| $no_more_simd: |
| RESTORE_SIMD_REGS |
| |
| $simd_end: |
| ble $18, $out |
| cmplt $18, 16, $1 |
| bne $1, $quad_loop_end |
| bne $4, $prep_quad_u_loop_tail |
| |
| .align 4 |
| $quad_loop_tail: |
| FIXUP_LDST( ldl $2, 0($17) ) |
| FIXUP_LDST( ldl $3, 8($17) ) |
| FIXUP_LDST( stl $2, 0($16) ) |
| FIXUP_LDST( stl $3, 8($16) ) |
| subl $18, 16, $18 |
| addl $17, 16, $17 |
| addl $16, 16, $16 |
| cmplt $18, 16, $1 |
| beq $1, $quad_loop_tail |
| |
| $quad_loop_end: |
| ble $18, $out |
| cmplt $18, 8, $1 |
| bne $1, $byte_loop_tail |
| bne $4, $move_one_quad_u |
| |
| $move_one_quad: |
| FIXUP_LDST( ldl $2, 0($17) ) |
| FIXUP_LDST( stl $2, 0($16) ) |
| subl $18, 8, $18 |
| addl $17, 8, $17 |
| addl $16, 8, $16 |
| ble $18, $out |
| |
| .align 3 |
| $byte_loop_tail: |
| FIXUP_LDST( ldbu $2, 0($17) ) |
| FIXUP_LDST( stb $2, 0($16) ) |
| subl $18, 1, $18 |
| addl $17, 1, $17 |
| addl $16, 1, $16 |
| bgt $18, $byte_loop_tail |
| br $31, $out |
| |
| /* misaligned src and dst */ |
| $quad_u_loop_head: |
| FIXUP_LDST( ldl_u $2, 0($17) ) |
| FIXUP_LDST( ldl_u $3, 7($17) ) |
| extll $2, $4, $2 |
| exthl $3, $4, $3 |
| bis $2, $3, $2 |
| FIXUP_LDST( stl $2, 0($16) ) |
| addl $16, 8, $16 |
| addl $17, 8, $17 |
| subl $18, 8, $18 |
| and $16, 31, $1 |
| beq $1, $dest_aligned_32 |
| br $31, $quad_u_loop_head |
| |
| $prep_simd_u_loop: |
| SAVE_SIMD_U_REGS |
| andnot $17, 31, $3 |
| ldi $2, 256($31) |
| sll $5, 3, $1 |
| subl $2, $1, $2 |
| sll $1, 29, $1 |
| sll $2, 29, $2 |
| ifmovd $1, $f1 |
| ifmovd $2, $f2 |
| FIXUP_LDST( vldd $f4, 0($3) ) |
| ldi $1, NC_STORE_THRESHOLD($31) |
| cmple $18, $1, $1 |
| bne $1, $simd_u_loop |
| |
| .align 4 |
| $simd_u_loop_nc: |
| FIXUP_LDST( vldd $f5, 32($3) ) |
| srlow $f4, $f1, $f4 |
| sllow $f5, $f2, $f3 |
| vlogfc $f3, $f4, $f31, $f3 |
| FIXUP_LDST( vstd_nc $f3, 0($16) ) |
| FIXUP_LDST( vldd $f4, 64($3) ) |
| srlow $f5, $f1, $f5 |
| sllow $f4, $f2, $f3 |
| vlogfc $f5, $f3, $f31, $f5 |
| FIXUP_LDST( vstd_nc $f5, 32($16) ) |
| subl $18, 64, $18 |
| addl $3, 64, $3 |
| addl $16, 64, $16 |
| cmplt $18, 64, $1 |
| beq $1, $simd_u_loop_nc |
| memb # required for _nc store instructions |
| br $31, $simd_u_loop_end |
| |
| .align 4 |
| $simd_u_loop: |
| FIXUP_LDST( vldd $f5, 32($3) ) |
| srlow $f4, $f1, $f4 |
| sllow $f5, $f2, $f3 |
| vlogfc $f4, $f3, $f31, $f3 |
| FIXUP_LDST( vstd $f3, 0($16) ) |
| FIXUP_LDST( vldd $f4, 64($3) ) |
| srlow $f5, $f1, $f5 |
| sllow $f4, $f2, $f3 |
| vlogfc $f5, $f3, $f31, $f3 |
| FIXUP_LDST( vstd $f3, 32($16) ) |
| subl $18, 64, $18 |
| addl $3, 64, $3 |
| addl $16, 64, $16 |
| cmplt $18, 64, $1 |
| beq $1, $simd_u_loop |
| |
| $simd_u_loop_end: |
| cmplt $18, 32, $1 |
| bne $1, $no_more_simd_u |
| FIXUP_LDST( vldd $f5, 32($3) ) |
| srlow $f4, $f1, $f4 |
| sllow $f5, $f2, $f3 |
| vlogfc $f4, $f3, $f31, $f3 |
| FIXUP_LDST( vstd $f3, 0($16) ) |
| subl $18, 32, $18 |
| addl $3, 32, $3 |
| addl $16, 32, $16 |
| |
| $no_more_simd_u: |
| RESTORE_SIMD_U_REGS |
| bis $3, $5, $17 |
| br $31, $simd_end |
| |
| $prep_quad_u_loop_tail: |
| FIXUP_LDST( ldl_u $2, 0($17) ) |
| .align 4 |
| $quad_u_loop_tail: |
| FIXUP_LDST( ldl_u $3, 8($17) ) |
| extll $2, $4, $22 |
| exthl $3, $4, $23 |
| bis $22, $23, $22 |
| FIXUP_LDST( stl $22, 0($16) ) |
| FIXUP_LDST( ldl_u $2, 16($17) ) |
| extll $3, $4, $24 |
| exthl $2, $4, $25 |
| bis $24, $25, $24 |
| FIXUP_LDST( stl $24, 8($16) ) |
| subl $18, 16, $18 |
| addl $17, 16, $17 |
| addl $16, 16, $16 |
| cmplt $18, 16, $1 |
| beq $1, $quad_u_loop_tail |
| br $31, $quad_loop_end |
| |
| $move_one_quad_u: |
| FIXUP_LDST( ldl_u $2, 0($17) ) |
| FIXUP_LDST( ldl_u $3, 8($17) ) |
| extll $2, $4, $22 |
| exthl $3, $4, $23 |
| bis $22, $23, $22 |
| FIXUP_LDST( stl $22, 0($16) ) |
| subl $18, 8, $18 |
| addl $17, 8, $17 |
| addl $16, 8, $16 |
| ble $18, $out |
| br $31, $byte_loop_tail |
