| /* SPDX-License-Identifier: GPL-2.0 */ |
| /* |
| * Original implementation written by Andy Polyakov, @dot-asm. |
| * This is an adaptation of the original code for kernel use. |
| * |
| * Copyright (C) 2006-2019 CRYPTOGAMS by <appro@openssl.org>. All Rights Reserved. |
| */ |
| |
| #include <linux/linkage.h> |
| #include <asm/nospec-insn.h> |
| #include <asm/fpu-insn.h> |
| |
| #define SP %r15 |
| #define FRAME (16 * 8 + 4 * 8) |
| |
| .data |
| .balign 32 |
| |
| SYM_DATA_START_LOCAL(sigma) |
| .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral |
| .long 1,0,0,0 |
| .long 2,0,0,0 |
| .long 3,0,0,0 |
| .long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap |
| |
| .long 0,1,2,3 |
| .long 0x61707865,0x61707865,0x61707865,0x61707865 # smashed sigma |
| .long 0x3320646e,0x3320646e,0x3320646e,0x3320646e |
| .long 0x79622d32,0x79622d32,0x79622d32,0x79622d32 |
| .long 0x6b206574,0x6b206574,0x6b206574,0x6b206574 |
| SYM_DATA_END(sigma) |
| |
| .previous |
| |
| GEN_BR_THUNK %r14 |
| |
| .text |
| |
| ############################################################################# |
| # void chacha20_vx_4x(u8 *out, counst u8 *inp, size_t len, |
| # counst u32 *key, const u32 *counter) |
| |
| #define OUT %r2 |
| #define INP %r3 |
| #define LEN %r4 |
| #define KEY %r5 |
| #define COUNTER %r6 |
| |
| #define BEPERM %v31 |
| #define CTR %v26 |
| |
| #define K0 %v16 |
| #define K1 %v17 |
| #define K2 %v18 |
| #define K3 %v19 |
| |
| #define XA0 %v0 |
| #define XA1 %v1 |
| #define XA2 %v2 |
| #define XA3 %v3 |
| |
| #define XB0 %v4 |
| #define XB1 %v5 |
| #define XB2 %v6 |
| #define XB3 %v7 |
| |
| #define XC0 %v8 |
| #define XC1 %v9 |
| #define XC2 %v10 |
| #define XC3 %v11 |
| |
| #define XD0 %v12 |
| #define XD1 %v13 |
| #define XD2 %v14 |
| #define XD3 %v15 |
| |
| #define XT0 %v27 |
| #define XT1 %v28 |
| #define XT2 %v29 |
| #define XT3 %v30 |
| |
| SYM_FUNC_START(chacha20_vx_4x) |
| stmg %r6,%r7,6*8(SP) |
| |
| larl %r7,sigma |
| lhi %r0,10 |
| lhi %r1,0 |
| |
| VL K0,0,,%r7 # load sigma |
| VL K1,0,,KEY # load key |
| VL K2,16,,KEY |
| VL K3,0,,COUNTER # load counter |
| |
| VL BEPERM,0x40,,%r7 |
| VL CTR,0x50,,%r7 |
| |
| VLM XA0,XA3,0x60,%r7,4 # load [smashed] sigma |
| |
| VREPF XB0,K1,0 # smash the key |
| VREPF XB1,K1,1 |
| VREPF XB2,K1,2 |
| VREPF XB3,K1,3 |
| |
| VREPF XD0,K3,0 |
| VREPF XD1,K3,1 |
| VREPF XD2,K3,2 |
| VREPF XD3,K3,3 |
| VAF XD0,XD0,CTR |
| |
| VREPF XC0,K2,0 |
| VREPF XC1,K2,1 |
| VREPF XC2,K2,2 |
| VREPF XC3,K2,3 |
| |
| .Loop_4x: |
| VAF XA0,XA0,XB0 |
| VX XD0,XD0,XA0 |
| VERLLF XD0,XD0,16 |
| |
| VAF XA1,XA1,XB1 |
| VX XD1,XD1,XA1 |
| VERLLF XD1,XD1,16 |
| |
| VAF XA2,XA2,XB2 |
| VX XD2,XD2,XA2 |
| VERLLF XD2,XD2,16 |
| |
| VAF XA3,XA3,XB3 |
| VX XD3,XD3,XA3 |
| VERLLF XD3,XD3,16 |
| |
| VAF XC0,XC0,XD0 |
| VX XB0,XB0,XC0 |
| VERLLF XB0,XB0,12 |
| |
| VAF XC1,XC1,XD1 |
| VX XB1,XB1,XC1 |
| VERLLF XB1,XB1,12 |
| |
| VAF XC2,XC2,XD2 |
| VX XB2,XB2,XC2 |
| VERLLF XB2,XB2,12 |
| |
| VAF XC3,XC3,XD3 |
| VX XB3,XB3,XC3 |
| VERLLF XB3,XB3,12 |
| |
| VAF XA0,XA0,XB0 |
| VX XD0,XD0,XA0 |
| VERLLF XD0,XD0,8 |
| |
| VAF XA1,XA1,XB1 |
| VX XD1,XD1,XA1 |
| VERLLF XD1,XD1,8 |
| |
| VAF XA2,XA2,XB2 |
| VX XD2,XD2,XA2 |
| VERLLF XD2,XD2,8 |
| |
| VAF XA3,XA3,XB3 |
| VX XD3,XD3,XA3 |
| VERLLF XD3,XD3,8 |
| |
| VAF XC0,XC0,XD0 |
| VX XB0,XB0,XC0 |
| VERLLF XB0,XB0,7 |
| |
| VAF XC1,XC1,XD1 |
| VX XB1,XB1,XC1 |
| VERLLF XB1,XB1,7 |
| |
| VAF XC2,XC2,XD2 |
| VX XB2,XB2,XC2 |
| VERLLF XB2,XB2,7 |
| |
| VAF XC3,XC3,XD3 |
| VX XB3,XB3,XC3 |
| VERLLF XB3,XB3,7 |
| |
| VAF XA0,XA0,XB1 |
| VX XD3,XD3,XA0 |
| VERLLF XD3,XD3,16 |
| |
| VAF XA1,XA1,XB2 |
| VX XD0,XD0,XA1 |
| VERLLF XD0,XD0,16 |
| |
| VAF XA2,XA2,XB3 |
| VX XD1,XD1,XA2 |
| VERLLF XD1,XD1,16 |
| |
| VAF XA3,XA3,XB0 |
| VX XD2,XD2,XA3 |
| VERLLF XD2,XD2,16 |
| |
| VAF XC2,XC2,XD3 |
| VX XB1,XB1,XC2 |
| VERLLF XB1,XB1,12 |
| |
| VAF XC3,XC3,XD0 |
| VX XB2,XB2,XC3 |
| VERLLF XB2,XB2,12 |
| |
| VAF XC0,XC0,XD1 |
| VX XB3,XB3,XC0 |
| VERLLF XB3,XB3,12 |
| |
| VAF XC1,XC1,XD2 |
| VX XB0,XB0,XC1 |
| VERLLF XB0,XB0,12 |
| |
| VAF XA0,XA0,XB1 |
| VX XD3,XD3,XA0 |
| VERLLF XD3,XD3,8 |
| |
| VAF XA1,XA1,XB2 |
| VX XD0,XD0,XA1 |
| VERLLF XD0,XD0,8 |
| |
| VAF XA2,XA2,XB3 |
| VX XD1,XD1,XA2 |
| VERLLF XD1,XD1,8 |
| |
| VAF XA3,XA3,XB0 |
| VX XD2,XD2,XA3 |
| VERLLF XD2,XD2,8 |
| |
| VAF XC2,XC2,XD3 |
| VX XB1,XB1,XC2 |
| VERLLF XB1,XB1,7 |
| |
| VAF XC3,XC3,XD0 |
| VX XB2,XB2,XC3 |
| VERLLF XB2,XB2,7 |
| |
| VAF XC0,XC0,XD1 |
| VX XB3,XB3,XC0 |
| VERLLF XB3,XB3,7 |
| |
| VAF XC1,XC1,XD2 |
| VX XB0,XB0,XC1 |
| VERLLF XB0,XB0,7 |
| brct %r0,.Loop_4x |
| |
| VAF XD0,XD0,CTR |
| |
| VMRHF XT0,XA0,XA1 # transpose data |
| VMRHF XT1,XA2,XA3 |
| VMRLF XT2,XA0,XA1 |
| VMRLF XT3,XA2,XA3 |
| VPDI XA0,XT0,XT1,0b0000 |
| VPDI XA1,XT0,XT1,0b0101 |
| VPDI XA2,XT2,XT3,0b0000 |
| VPDI XA3,XT2,XT3,0b0101 |
| |
| VMRHF XT0,XB0,XB1 |
| VMRHF XT1,XB2,XB3 |
| VMRLF XT2,XB0,XB1 |
| VMRLF XT3,XB2,XB3 |
| VPDI XB0,XT0,XT1,0b0000 |
| VPDI XB1,XT0,XT1,0b0101 |
| VPDI XB2,XT2,XT3,0b0000 |
| VPDI XB3,XT2,XT3,0b0101 |
| |
| VMRHF XT0,XC0,XC1 |
| VMRHF XT1,XC2,XC3 |
| VMRLF XT2,XC0,XC1 |
| VMRLF XT3,XC2,XC3 |
| VPDI XC0,XT0,XT1,0b0000 |
| VPDI XC1,XT0,XT1,0b0101 |
| VPDI XC2,XT2,XT3,0b0000 |
| VPDI XC3,XT2,XT3,0b0101 |
| |
| VMRHF XT0,XD0,XD1 |
| VMRHF XT1,XD2,XD3 |
| VMRLF XT2,XD0,XD1 |
| VMRLF XT3,XD2,XD3 |
| VPDI XD0,XT0,XT1,0b0000 |
| VPDI XD1,XT0,XT1,0b0101 |
| VPDI XD2,XT2,XT3,0b0000 |
| VPDI XD3,XT2,XT3,0b0101 |
| |
| VAF XA0,XA0,K0 |
| VAF XB0,XB0,K1 |
| VAF XC0,XC0,K2 |
| VAF XD0,XD0,K3 |
| |
| VPERM XA0,XA0,XA0,BEPERM |
| VPERM XB0,XB0,XB0,BEPERM |
| VPERM XC0,XC0,XC0,BEPERM |
| VPERM XD0,XD0,XD0,BEPERM |
| |
| VLM XT0,XT3,0,INP,0 |
| |
| VX XT0,XT0,XA0 |
| VX XT1,XT1,XB0 |
| VX XT2,XT2,XC0 |
| VX XT3,XT3,XD0 |
| |
| VSTM XT0,XT3,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| |
| VAF XA0,XA1,K0 |
| VAF XB0,XB1,K1 |
| VAF XC0,XC1,K2 |
| VAF XD0,XD1,K3 |
| |
| VPERM XA0,XA0,XA0,BEPERM |
| VPERM XB0,XB0,XB0,BEPERM |
| VPERM XC0,XC0,XC0,BEPERM |
| VPERM XD0,XD0,XD0,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_4x |
| |
| VLM XT0,XT3,0,INP,0 |
| |
| VX XT0,XT0,XA0 |
| VX XT1,XT1,XB0 |
| VX XT2,XT2,XC0 |
| VX XT3,XT3,XD0 |
| |
| VSTM XT0,XT3,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| je .Ldone_4x |
| |
| VAF XA0,XA2,K0 |
| VAF XB0,XB2,K1 |
| VAF XC0,XC2,K2 |
| VAF XD0,XD2,K3 |
| |
| VPERM XA0,XA0,XA0,BEPERM |
| VPERM XB0,XB0,XB0,BEPERM |
| VPERM XC0,XC0,XC0,BEPERM |
| VPERM XD0,XD0,XD0,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_4x |
| |
| VLM XT0,XT3,0,INP,0 |
| |
| VX XT0,XT0,XA0 |
| VX XT1,XT1,XB0 |
| VX XT2,XT2,XC0 |
| VX XT3,XT3,XD0 |
| |
| VSTM XT0,XT3,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| je .Ldone_4x |
| |
| VAF XA0,XA3,K0 |
| VAF XB0,XB3,K1 |
| VAF XC0,XC3,K2 |
| VAF XD0,XD3,K3 |
| |
| VPERM XA0,XA0,XA0,BEPERM |
| VPERM XB0,XB0,XB0,BEPERM |
| VPERM XC0,XC0,XC0,BEPERM |
| VPERM XD0,XD0,XD0,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_4x |
| |
| VLM XT0,XT3,0,INP,0 |
| |
| VX XT0,XT0,XA0 |
| VX XT1,XT1,XB0 |
| VX XT2,XT2,XC0 |
| VX XT3,XT3,XD0 |
| |
| VSTM XT0,XT3,0,OUT,0 |
| |
| .Ldone_4x: |
| lmg %r6,%r7,6*8(SP) |
| BR_EX %r14 |
| |
| .Ltail_4x: |
| VLR XT0,XC0 |
| VLR XT1,XD0 |
| |
| VST XA0,8*8+0x00,,SP |
| VST XB0,8*8+0x10,,SP |
| VST XT0,8*8+0x20,,SP |
| VST XT1,8*8+0x30,,SP |
| |
| lghi %r1,0 |
| |
| .Loop_tail_4x: |
| llgc %r5,0(%r1,INP) |
| llgc %r6,8*8(%r1,SP) |
| xr %r6,%r5 |
| stc %r6,0(%r1,OUT) |
| la %r1,1(%r1) |
| brct LEN,.Loop_tail_4x |
| |
| lmg %r6,%r7,6*8(SP) |
| BR_EX %r14 |
| SYM_FUNC_END(chacha20_vx_4x) |
| |
| #undef OUT |
| #undef INP |
| #undef LEN |
| #undef KEY |
| #undef COUNTER |
| |
| #undef BEPERM |
| |
| #undef K0 |
| #undef K1 |
| #undef K2 |
| #undef K3 |
| |
| |
| ############################################################################# |
| # void chacha20_vx(u8 *out, counst u8 *inp, size_t len, |
| # counst u32 *key, const u32 *counter) |
| |
| #define OUT %r2 |
| #define INP %r3 |
| #define LEN %r4 |
| #define KEY %r5 |
| #define COUNTER %r6 |
| |
| #define BEPERM %v31 |
| |
| #define K0 %v27 |
| #define K1 %v24 |
| #define K2 %v25 |
| #define K3 %v26 |
| |
| #define A0 %v0 |
| #define B0 %v1 |
| #define C0 %v2 |
| #define D0 %v3 |
| |
| #define A1 %v4 |
| #define B1 %v5 |
| #define C1 %v6 |
| #define D1 %v7 |
| |
| #define A2 %v8 |
| #define B2 %v9 |
| #define C2 %v10 |
| #define D2 %v11 |
| |
| #define A3 %v12 |
| #define B3 %v13 |
| #define C3 %v14 |
| #define D3 %v15 |
| |
| #define A4 %v16 |
| #define B4 %v17 |
| #define C4 %v18 |
| #define D4 %v19 |
| |
| #define A5 %v20 |
| #define B5 %v21 |
| #define C5 %v22 |
| #define D5 %v23 |
| |
| #define T0 %v27 |
| #define T1 %v28 |
| #define T2 %v29 |
| #define T3 %v30 |
| |
| SYM_FUNC_START(chacha20_vx) |
| clgfi LEN,256 |
| jle chacha20_vx_4x |
| stmg %r6,%r7,6*8(SP) |
| |
| lghi %r1,-FRAME |
| lgr %r0,SP |
| la SP,0(%r1,SP) |
| stg %r0,0(SP) # back-chain |
| |
| larl %r7,sigma |
| lhi %r0,10 |
| |
| VLM K1,K2,0,KEY,0 # load key |
| VL K3,0,,COUNTER # load counter |
| |
| VLM K0,BEPERM,0,%r7,4 # load sigma, increments, ... |
| |
| .Loop_outer_vx: |
| VLR A0,K0 |
| VLR B0,K1 |
| VLR A1,K0 |
| VLR B1,K1 |
| VLR A2,K0 |
| VLR B2,K1 |
| VLR A3,K0 |
| VLR B3,K1 |
| VLR A4,K0 |
| VLR B4,K1 |
| VLR A5,K0 |
| VLR B5,K1 |
| |
| VLR D0,K3 |
| VAF D1,K3,T1 # K[3]+1 |
| VAF D2,K3,T2 # K[3]+2 |
| VAF D3,K3,T3 # K[3]+3 |
| VAF D4,D2,T2 # K[3]+4 |
| VAF D5,D2,T3 # K[3]+5 |
| |
| VLR C0,K2 |
| VLR C1,K2 |
| VLR C2,K2 |
| VLR C3,K2 |
| VLR C4,K2 |
| VLR C5,K2 |
| |
| VLR T1,D1 |
| VLR T2,D2 |
| VLR T3,D3 |
| |
| .Loop_vx: |
| VAF A0,A0,B0 |
| VAF A1,A1,B1 |
| VAF A2,A2,B2 |
| VAF A3,A3,B3 |
| VAF A4,A4,B4 |
| VAF A5,A5,B5 |
| VX D0,D0,A0 |
| VX D1,D1,A1 |
| VX D2,D2,A2 |
| VX D3,D3,A3 |
| VX D4,D4,A4 |
| VX D5,D5,A5 |
| VERLLF D0,D0,16 |
| VERLLF D1,D1,16 |
| VERLLF D2,D2,16 |
| VERLLF D3,D3,16 |
| VERLLF D4,D4,16 |
| VERLLF D5,D5,16 |
| |
| VAF C0,C0,D0 |
| VAF C1,C1,D1 |
| VAF C2,C2,D2 |
| VAF C3,C3,D3 |
| VAF C4,C4,D4 |
| VAF C5,C5,D5 |
| VX B0,B0,C0 |
| VX B1,B1,C1 |
| VX B2,B2,C2 |
| VX B3,B3,C3 |
| VX B4,B4,C4 |
| VX B5,B5,C5 |
| VERLLF B0,B0,12 |
| VERLLF B1,B1,12 |
| VERLLF B2,B2,12 |
| VERLLF B3,B3,12 |
| VERLLF B4,B4,12 |
| VERLLF B5,B5,12 |
| |
| VAF A0,A0,B0 |
| VAF A1,A1,B1 |
| VAF A2,A2,B2 |
| VAF A3,A3,B3 |
| VAF A4,A4,B4 |
| VAF A5,A5,B5 |
| VX D0,D0,A0 |
| VX D1,D1,A1 |
| VX D2,D2,A2 |
| VX D3,D3,A3 |
| VX D4,D4,A4 |
| VX D5,D5,A5 |
| VERLLF D0,D0,8 |
| VERLLF D1,D1,8 |
| VERLLF D2,D2,8 |
| VERLLF D3,D3,8 |
| VERLLF D4,D4,8 |
| VERLLF D5,D5,8 |
| |
| VAF C0,C0,D0 |
| VAF C1,C1,D1 |
| VAF C2,C2,D2 |
| VAF C3,C3,D3 |
| VAF C4,C4,D4 |
| VAF C5,C5,D5 |
| VX B0,B0,C0 |
| VX B1,B1,C1 |
| VX B2,B2,C2 |
| VX B3,B3,C3 |
| VX B4,B4,C4 |
| VX B5,B5,C5 |
| VERLLF B0,B0,7 |
| VERLLF B1,B1,7 |
| VERLLF B2,B2,7 |
| VERLLF B3,B3,7 |
| VERLLF B4,B4,7 |
| VERLLF B5,B5,7 |
| |
| VSLDB C0,C0,C0,8 |
| VSLDB C1,C1,C1,8 |
| VSLDB C2,C2,C2,8 |
| VSLDB C3,C3,C3,8 |
| VSLDB C4,C4,C4,8 |
| VSLDB C5,C5,C5,8 |
| VSLDB B0,B0,B0,4 |
| VSLDB B1,B1,B1,4 |
| VSLDB B2,B2,B2,4 |
| VSLDB B3,B3,B3,4 |
| VSLDB B4,B4,B4,4 |
| VSLDB B5,B5,B5,4 |
| VSLDB D0,D0,D0,12 |
| VSLDB D1,D1,D1,12 |
| VSLDB D2,D2,D2,12 |
| VSLDB D3,D3,D3,12 |
| VSLDB D4,D4,D4,12 |
| VSLDB D5,D5,D5,12 |
| |
| VAF A0,A0,B0 |
| VAF A1,A1,B1 |
| VAF A2,A2,B2 |
| VAF A3,A3,B3 |
| VAF A4,A4,B4 |
| VAF A5,A5,B5 |
| VX D0,D0,A0 |
| VX D1,D1,A1 |
| VX D2,D2,A2 |
| VX D3,D3,A3 |
| VX D4,D4,A4 |
| VX D5,D5,A5 |
| VERLLF D0,D0,16 |
| VERLLF D1,D1,16 |
| VERLLF D2,D2,16 |
| VERLLF D3,D3,16 |
| VERLLF D4,D4,16 |
| VERLLF D5,D5,16 |
| |
| VAF C0,C0,D0 |
| VAF C1,C1,D1 |
| VAF C2,C2,D2 |
| VAF C3,C3,D3 |
| VAF C4,C4,D4 |
| VAF C5,C5,D5 |
| VX B0,B0,C0 |
| VX B1,B1,C1 |
| VX B2,B2,C2 |
| VX B3,B3,C3 |
| VX B4,B4,C4 |
| VX B5,B5,C5 |
| VERLLF B0,B0,12 |
| VERLLF B1,B1,12 |
| VERLLF B2,B2,12 |
| VERLLF B3,B3,12 |
| VERLLF B4,B4,12 |
| VERLLF B5,B5,12 |
| |
| VAF A0,A0,B0 |
| VAF A1,A1,B1 |
| VAF A2,A2,B2 |
| VAF A3,A3,B3 |
| VAF A4,A4,B4 |
| VAF A5,A5,B5 |
| VX D0,D0,A0 |
| VX D1,D1,A1 |
| VX D2,D2,A2 |
| VX D3,D3,A3 |
| VX D4,D4,A4 |
| VX D5,D5,A5 |
| VERLLF D0,D0,8 |
| VERLLF D1,D1,8 |
| VERLLF D2,D2,8 |
| VERLLF D3,D3,8 |
| VERLLF D4,D4,8 |
| VERLLF D5,D5,8 |
| |
| VAF C0,C0,D0 |
| VAF C1,C1,D1 |
| VAF C2,C2,D2 |
| VAF C3,C3,D3 |
| VAF C4,C4,D4 |
| VAF C5,C5,D5 |
| VX B0,B0,C0 |
| VX B1,B1,C1 |
| VX B2,B2,C2 |
| VX B3,B3,C3 |
| VX B4,B4,C4 |
| VX B5,B5,C5 |
| VERLLF B0,B0,7 |
| VERLLF B1,B1,7 |
| VERLLF B2,B2,7 |
| VERLLF B3,B3,7 |
| VERLLF B4,B4,7 |
| VERLLF B5,B5,7 |
| |
| VSLDB C0,C0,C0,8 |
| VSLDB C1,C1,C1,8 |
| VSLDB C2,C2,C2,8 |
| VSLDB C3,C3,C3,8 |
| VSLDB C4,C4,C4,8 |
| VSLDB C5,C5,C5,8 |
| VSLDB B0,B0,B0,12 |
| VSLDB B1,B1,B1,12 |
| VSLDB B2,B2,B2,12 |
| VSLDB B3,B3,B3,12 |
| VSLDB B4,B4,B4,12 |
| VSLDB B5,B5,B5,12 |
| VSLDB D0,D0,D0,4 |
| VSLDB D1,D1,D1,4 |
| VSLDB D2,D2,D2,4 |
| VSLDB D3,D3,D3,4 |
| VSLDB D4,D4,D4,4 |
| VSLDB D5,D5,D5,4 |
| brct %r0,.Loop_vx |
| |
| VAF A0,A0,K0 |
| VAF B0,B0,K1 |
| VAF C0,C0,K2 |
| VAF D0,D0,K3 |
| VAF A1,A1,K0 |
| VAF D1,D1,T1 # +K[3]+1 |
| |
| VPERM A0,A0,A0,BEPERM |
| VPERM B0,B0,B0,BEPERM |
| VPERM C0,C0,C0,BEPERM |
| VPERM D0,D0,D0,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_vx |
| |
| VAF D2,D2,T2 # +K[3]+2 |
| VAF D3,D3,T3 # +K[3]+3 |
| VLM T0,T3,0,INP,0 |
| |
| VX A0,A0,T0 |
| VX B0,B0,T1 |
| VX C0,C0,T2 |
| VX D0,D0,T3 |
| |
| VLM K0,T3,0,%r7,4 # re-load sigma and increments |
| |
| VSTM A0,D0,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| je .Ldone_vx |
| |
| VAF B1,B1,K1 |
| VAF C1,C1,K2 |
| |
| VPERM A0,A1,A1,BEPERM |
| VPERM B0,B1,B1,BEPERM |
| VPERM C0,C1,C1,BEPERM |
| VPERM D0,D1,D1,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_vx |
| |
| VLM A1,D1,0,INP,0 |
| |
| VX A0,A0,A1 |
| VX B0,B0,B1 |
| VX C0,C0,C1 |
| VX D0,D0,D1 |
| |
| VSTM A0,D0,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| je .Ldone_vx |
| |
| VAF A2,A2,K0 |
| VAF B2,B2,K1 |
| VAF C2,C2,K2 |
| |
| VPERM A0,A2,A2,BEPERM |
| VPERM B0,B2,B2,BEPERM |
| VPERM C0,C2,C2,BEPERM |
| VPERM D0,D2,D2,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_vx |
| |
| VLM A1,D1,0,INP,0 |
| |
| VX A0,A0,A1 |
| VX B0,B0,B1 |
| VX C0,C0,C1 |
| VX D0,D0,D1 |
| |
| VSTM A0,D0,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| je .Ldone_vx |
| |
| VAF A3,A3,K0 |
| VAF B3,B3,K1 |
| VAF C3,C3,K2 |
| VAF D2,K3,T3 # K[3]+3 |
| |
| VPERM A0,A3,A3,BEPERM |
| VPERM B0,B3,B3,BEPERM |
| VPERM C0,C3,C3,BEPERM |
| VPERM D0,D3,D3,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_vx |
| |
| VAF D3,D2,T1 # K[3]+4 |
| VLM A1,D1,0,INP,0 |
| |
| VX A0,A0,A1 |
| VX B0,B0,B1 |
| VX C0,C0,C1 |
| VX D0,D0,D1 |
| |
| VSTM A0,D0,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| je .Ldone_vx |
| |
| VAF A4,A4,K0 |
| VAF B4,B4,K1 |
| VAF C4,C4,K2 |
| VAF D4,D4,D3 # +K[3]+4 |
| VAF D3,D3,T1 # K[3]+5 |
| VAF K3,D2,T3 # K[3]+=6 |
| |
| VPERM A0,A4,A4,BEPERM |
| VPERM B0,B4,B4,BEPERM |
| VPERM C0,C4,C4,BEPERM |
| VPERM D0,D4,D4,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_vx |
| |
| VLM A1,D1,0,INP,0 |
| |
| VX A0,A0,A1 |
| VX B0,B0,B1 |
| VX C0,C0,C1 |
| VX D0,D0,D1 |
| |
| VSTM A0,D0,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| aghi LEN,-0x40 |
| je .Ldone_vx |
| |
| VAF A5,A5,K0 |
| VAF B5,B5,K1 |
| VAF C5,C5,K2 |
| VAF D5,D5,D3 # +K[3]+5 |
| |
| VPERM A0,A5,A5,BEPERM |
| VPERM B0,B5,B5,BEPERM |
| VPERM C0,C5,C5,BEPERM |
| VPERM D0,D5,D5,BEPERM |
| |
| clgfi LEN,0x40 |
| jl .Ltail_vx |
| |
| VLM A1,D1,0,INP,0 |
| |
| VX A0,A0,A1 |
| VX B0,B0,B1 |
| VX C0,C0,C1 |
| VX D0,D0,D1 |
| |
| VSTM A0,D0,0,OUT,0 |
| |
| la INP,0x40(INP) |
| la OUT,0x40(OUT) |
| lhi %r0,10 |
| aghi LEN,-0x40 |
| jne .Loop_outer_vx |
| |
| .Ldone_vx: |
| lmg %r6,%r7,FRAME+6*8(SP) |
| la SP,FRAME(SP) |
| BR_EX %r14 |
| |
| .Ltail_vx: |
| VSTM A0,D0,8*8,SP,3 |
| lghi %r1,0 |
| |
| .Loop_tail_vx: |
| llgc %r5,0(%r1,INP) |
| llgc %r6,8*8(%r1,SP) |
| xr %r6,%r5 |
| stc %r6,0(%r1,OUT) |
| la %r1,1(%r1) |
| brct LEN,.Loop_tail_vx |
| |
| lmg %r6,%r7,FRAME+6*8(SP) |
| la SP,FRAME(SP) |
| BR_EX %r14 |
| SYM_FUNC_END(chacha20_vx) |
| |
| .previous |
