| /* SPDX-License-Identifier: GPL-2.0-only */ | |
| /* | |
| * Chaining mode wrappers for AES | |
| * | |
| * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> | |
| */ | |
| /* included by aes-ce.S and aes-neon.S */ | |
| .text | |
| .align 4 | |
| #ifndef MAX_STRIDE | |
| #define MAX_STRIDE 4 | |
| #endif | |
| #if MAX_STRIDE == 4 | |
| #define ST4(x...) x | |
| #define ST5(x...) | |
| #else | |
| #define ST4(x...) | |
| #define ST5(x...) x | |
| #endif | |
| SYM_FUNC_START_LOCAL(aes_encrypt_block4x) | |
| encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 | |
| ret | |
| SYM_FUNC_END(aes_encrypt_block4x) | |
| SYM_FUNC_START_LOCAL(aes_decrypt_block4x) | |
| decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 | |
| ret | |
| SYM_FUNC_END(aes_decrypt_block4x) | |
| #if MAX_STRIDE == 5 | |
| SYM_FUNC_START_LOCAL(aes_encrypt_block5x) | |
| encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 | |
| ret | |
| SYM_FUNC_END(aes_encrypt_block5x) | |
| SYM_FUNC_START_LOCAL(aes_decrypt_block5x) | |
| decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 | |
| ret | |
| SYM_FUNC_END(aes_decrypt_block5x) | |
| #endif | |
| /* | |
| * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, | |
| * int blocks) | |
| * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, | |
| * int blocks) | |
| */ | |
| AES_FUNC_START(aes_ecb_encrypt) | |
| frame_push 0 | |
| enc_prepare w3, x2, x5 | |
| .LecbencloopNx: | |
| subs w4, w4, #MAX_STRIDE | |
| bmi .Lecbenc1x | |
| ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ | |
| ST4( bl aes_encrypt_block4x ) | |
| ST5( ld1 {v4.16b}, [x1], #16 ) | |
| ST5( bl aes_encrypt_block5x ) | |
| st1 {v0.16b-v3.16b}, [x0], #64 | |
| ST5( st1 {v4.16b}, [x0], #16 ) | |
| b .LecbencloopNx | |
| .Lecbenc1x: | |
| adds w4, w4, #MAX_STRIDE | |
| beq .Lecbencout | |
| .Lecbencloop: | |
| ld1 {v0.16b}, [x1], #16 /* get next pt block */ | |
| encrypt_block v0, w3, x2, x5, w6 | |
| st1 {v0.16b}, [x0], #16 | |
| subs w4, w4, #1 | |
| bne .Lecbencloop | |
| .Lecbencout: | |
| frame_pop | |
| ret | |
| AES_FUNC_END(aes_ecb_encrypt) | |
| AES_FUNC_START(aes_ecb_decrypt) | |
| frame_push 0 | |
| dec_prepare w3, x2, x5 | |
| .LecbdecloopNx: | |
| subs w4, w4, #MAX_STRIDE | |
| bmi .Lecbdec1x | |
| ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ | |
| ST4( bl aes_decrypt_block4x ) | |
| ST5( ld1 {v4.16b}, [x1], #16 ) | |
| ST5( bl aes_decrypt_block5x ) | |
| st1 {v0.16b-v3.16b}, [x0], #64 | |
| ST5( st1 {v4.16b}, [x0], #16 ) | |
| b .LecbdecloopNx | |
| .Lecbdec1x: | |
| adds w4, w4, #MAX_STRIDE | |
| beq .Lecbdecout | |
| .Lecbdecloop: | |
| ld1 {v0.16b}, [x1], #16 /* get next ct block */ | |
| decrypt_block v0, w3, x2, x5, w6 | |
| st1 {v0.16b}, [x0], #16 | |
| subs w4, w4, #1 | |
| bne .Lecbdecloop | |
| .Lecbdecout: | |
| frame_pop | |
| ret | |
| AES_FUNC_END(aes_ecb_decrypt) | |
| /* | |
| * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, | |
| * int blocks, u8 iv[]) | |
| * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, | |
| * int blocks, u8 iv[]) | |
| * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], | |
| * int rounds, int blocks, u8 iv[], | |
| * u32 const rk2[]); | |
| * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], | |
| * int rounds, int blocks, u8 iv[], | |
| * u32 const rk2[]); | |
| */ | |
| AES_FUNC_START(aes_essiv_cbc_encrypt) | |
| ld1 {v4.16b}, [x5] /* get iv */ | |
| mov w8, #14 /* AES-256: 14 rounds */ | |
| enc_prepare w8, x6, x7 | |
| encrypt_block v4, w8, x6, x7, w9 | |
| enc_switch_key w3, x2, x6 | |
| b .Lcbcencloop4x | |
| AES_FUNC_START(aes_cbc_encrypt) | |
| ld1 {v4.16b}, [x5] /* get iv */ | |
| enc_prepare w3, x2, x6 | |
| .Lcbcencloop4x: | |
| subs w4, w4, #4 | |
| bmi .Lcbcenc1x | |
| ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ | |
| eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */ | |
| encrypt_block v0, w3, x2, x6, w7 | |
| eor v1.16b, v1.16b, v0.16b | |
| encrypt_block v1, w3, x2, x6, w7 | |
| eor v2.16b, v2.16b, v1.16b | |
| encrypt_block v2, w3, x2, x6, w7 | |
| eor v3.16b, v3.16b, v2.16b | |
| encrypt_block v3, w3, x2, x6, w7 | |
| st1 {v0.16b-v3.16b}, [x0], #64 | |
| mov v4.16b, v3.16b | |
| b .Lcbcencloop4x | |
| .Lcbcenc1x: | |
| adds w4, w4, #4 | |
| beq .Lcbcencout | |
| .Lcbcencloop: | |
| ld1 {v0.16b}, [x1], #16 /* get next pt block */ | |
| eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */ | |
| encrypt_block v4, w3, x2, x6, w7 | |
| st1 {v4.16b}, [x0], #16 | |
| subs w4, w4, #1 | |
| bne .Lcbcencloop | |
| .Lcbcencout: | |
| st1 {v4.16b}, [x5] /* return iv */ | |
| ret | |
| AES_FUNC_END(aes_cbc_encrypt) | |
| AES_FUNC_END(aes_essiv_cbc_encrypt) | |
| AES_FUNC_START(aes_essiv_cbc_decrypt) | |
| ld1 {cbciv.16b}, [x5] /* get iv */ | |
| mov w8, #14 /* AES-256: 14 rounds */ | |
| enc_prepare w8, x6, x7 | |
| encrypt_block cbciv, w8, x6, x7, w9 | |
| b .Lessivcbcdecstart | |
| AES_FUNC_START(aes_cbc_decrypt) | |
| ld1 {cbciv.16b}, [x5] /* get iv */ | |
| .Lessivcbcdecstart: | |
| frame_push 0 | |
| dec_prepare w3, x2, x6 | |
| .LcbcdecloopNx: | |
| subs w4, w4, #MAX_STRIDE | |
| bmi .Lcbcdec1x | |
| ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ | |
| #if MAX_STRIDE == 5 | |
| ld1 {v4.16b}, [x1], #16 /* get 1 ct block */ | |
| mov v5.16b, v0.16b | |
| mov v6.16b, v1.16b | |
| mov v7.16b, v2.16b | |
| bl aes_decrypt_block5x | |
| sub x1, x1, #32 | |
| eor v0.16b, v0.16b, cbciv.16b | |
| eor v1.16b, v1.16b, v5.16b | |
| ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */ | |
| ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ | |
| eor v2.16b, v2.16b, v6.16b | |
| eor v3.16b, v3.16b, v7.16b | |
| eor v4.16b, v4.16b, v5.16b | |
| #else | |
| mov v4.16b, v0.16b | |
| mov v5.16b, v1.16b | |
| mov v6.16b, v2.16b | |
| bl aes_decrypt_block4x | |
| sub x1, x1, #16 | |
| eor v0.16b, v0.16b, cbciv.16b | |
| eor v1.16b, v1.16b, v4.16b | |
| ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ | |
| eor v2.16b, v2.16b, v5.16b | |
| eor v3.16b, v3.16b, v6.16b | |
| #endif | |
| st1 {v0.16b-v3.16b}, [x0], #64 | |
| ST5( st1 {v4.16b}, [x0], #16 ) | |
| b .LcbcdecloopNx | |
| .Lcbcdec1x: | |
| adds w4, w4, #MAX_STRIDE | |
| beq .Lcbcdecout | |
| .Lcbcdecloop: | |
| ld1 {v1.16b}, [x1], #16 /* get next ct block */ | |
| mov v0.16b, v1.16b /* ...and copy to v0 */ | |
| decrypt_block v0, w3, x2, x6, w7 | |
| eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */ | |
| mov cbciv.16b, v1.16b /* ct is next iv */ | |
| st1 {v0.16b}, [x0], #16 | |
| subs w4, w4, #1 | |
| bne .Lcbcdecloop | |
| .Lcbcdecout: | |
| st1 {cbciv.16b}, [x5] /* return iv */ | |
| frame_pop | |
| ret | |
| AES_FUNC_END(aes_cbc_decrypt) | |
| AES_FUNC_END(aes_essiv_cbc_decrypt) | |
| /* | |
| * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], | |
| * int rounds, int bytes, u8 const iv[]) | |
| * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], | |
| * int rounds, int bytes, u8 const iv[]) | |
| */ | |
| AES_FUNC_START(aes_cbc_cts_encrypt) | |
| adr_l x8, .Lcts_permute_table | |
| sub x4, x4, #16 | |
| add x9, x8, #32 | |
| add x8, x8, x4 | |
| sub x9, x9, x4 | |
| ld1 {v3.16b}, [x8] | |
| ld1 {v4.16b}, [x9] | |
| ld1 {v0.16b}, [x1], x4 /* overlapping loads */ | |
| ld1 {v1.16b}, [x1] | |
| ld1 {v5.16b}, [x5] /* get iv */ | |
| enc_prepare w3, x2, x6 | |
| eor v0.16b, v0.16b, v5.16b /* xor with iv */ | |
| tbl v1.16b, {v1.16b}, v4.16b | |
| encrypt_block v0, w3, x2, x6, w7 | |
| eor v1.16b, v1.16b, v0.16b | |
| tbl v0.16b, {v0.16b}, v3.16b | |
| encrypt_block v1, w3, x2, x6, w7 | |
| add x4, x0, x4 | |
| st1 {v0.16b}, [x4] /* overlapping stores */ | |
| st1 {v1.16b}, [x0] | |
| ret | |
| AES_FUNC_END(aes_cbc_cts_encrypt) | |
| AES_FUNC_START(aes_cbc_cts_decrypt) | |
| adr_l x8, .Lcts_permute_table | |
| sub x4, x4, #16 | |
| add x9, x8, #32 | |
| add x8, x8, x4 | |
| sub x9, x9, x4 | |
| ld1 {v3.16b}, [x8] | |
| ld1 {v4.16b}, [x9] | |
| ld1 {v0.16b}, [x1], x4 /* overlapping loads */ | |
| ld1 {v1.16b}, [x1] | |
| ld1 {v5.16b}, [x5] /* get iv */ | |
| dec_prepare w3, x2, x6 | |
| decrypt_block v0, w3, x2, x6, w7 | |
| tbl v2.16b, {v0.16b}, v3.16b | |
| eor v2.16b, v2.16b, v1.16b | |
| tbx v0.16b, {v1.16b}, v4.16b | |
| decrypt_block v0, w3, x2, x6, w7 | |
| eor v0.16b, v0.16b, v5.16b /* xor with iv */ | |
| add x4, x0, x4 | |
| st1 {v2.16b}, [x4] /* overlapping stores */ | |
| st1 {v0.16b}, [x0] | |
| ret | |
| AES_FUNC_END(aes_cbc_cts_decrypt) | |
| .section ".rodata", "a" | |
| .align 6 | |
| .Lcts_permute_table: | |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff | |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff | |
| .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 | |
| .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf | |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff | |
| .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff | |
| .previous | |
| /* | |
| * This macro generates the code for CTR and XCTR mode. | |
| */ | |
| .macro ctr_encrypt xctr | |
| // Arguments | |
| OUT .req x0 | |
| IN .req x1 | |
| KEY .req x2 | |
| ROUNDS_W .req w3 | |
| BYTES_W .req w4 | |
| IV .req x5 | |
| BYTE_CTR_W .req w6 // XCTR only | |
| // Intermediate values | |
| CTR_W .req w11 // XCTR only | |
| CTR .req x11 // XCTR only | |
| IV_PART .req x12 | |
| BLOCKS .req x13 | |
| BLOCKS_W .req w13 | |
| frame_push 0 | |
| enc_prepare ROUNDS_W, KEY, IV_PART | |
| ld1 {vctr.16b}, [IV] | |
| /* | |
| * Keep 64 bits of the IV in a register. For CTR mode this lets us | |
| * easily increment the IV. For XCTR mode this lets us efficiently XOR | |
| * the 64-bit counter with the IV. | |
| */ | |
| .if \xctr | |
| umov IV_PART, vctr.d[0] | |
| lsr CTR_W, BYTE_CTR_W, #4 | |
| .else | |
| umov IV_PART, vctr.d[1] | |
| rev IV_PART, IV_PART | |
| .endif | |
| .LctrloopNx\xctr: | |
| add BLOCKS_W, BYTES_W, #15 | |
| sub BYTES_W, BYTES_W, #MAX_STRIDE << 4 | |
| lsr BLOCKS_W, BLOCKS_W, #4 | |
| mov w8, #MAX_STRIDE | |
| cmp BLOCKS_W, w8 | |
| csel BLOCKS_W, BLOCKS_W, w8, lt | |
| /* | |
| * Set up the counter values in v0-v{MAX_STRIDE-1}. | |
| * | |
| * If we are encrypting less than MAX_STRIDE blocks, the tail block | |
| * handling code expects the last keystream block to be in | |
| * v{MAX_STRIDE-1}. For example: if encrypting two blocks with | |
| * MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks. | |
| */ | |
| .if \xctr | |
| add CTR, CTR, BLOCKS | |
| .else | |
| adds IV_PART, IV_PART, BLOCKS | |
| .endif | |
| mov v0.16b, vctr.16b | |
| mov v1.16b, vctr.16b | |
| mov v2.16b, vctr.16b | |
| mov v3.16b, vctr.16b | |
| ST5( mov v4.16b, vctr.16b ) | |
| .if \xctr | |
| sub x6, CTR, #MAX_STRIDE - 1 | |
| sub x7, CTR, #MAX_STRIDE - 2 | |
| sub x8, CTR, #MAX_STRIDE - 3 | |
| sub x9, CTR, #MAX_STRIDE - 4 | |
| ST5( sub x10, CTR, #MAX_STRIDE - 5 ) | |
| eor x6, x6, IV_PART | |
| eor x7, x7, IV_PART | |
| eor x8, x8, IV_PART | |
| eor x9, x9, IV_PART | |
| ST5( eor x10, x10, IV_PART ) | |
| mov v0.d[0], x6 | |
| mov v1.d[0], x7 | |
| mov v2.d[0], x8 | |
| mov v3.d[0], x9 | |
| ST5( mov v4.d[0], x10 ) | |
| .else | |
| bcs 0f | |
| .subsection 1 | |
| /* | |
| * This subsection handles carries. | |
| * | |
| * Conditional branching here is allowed with respect to time | |
| * invariance since the branches are dependent on the IV instead | |
| * of the plaintext or key. This code is rarely executed in | |
| * practice anyway. | |
| */ | |
| /* Apply carry to outgoing counter. */ | |
| 0: umov x8, vctr.d[0] | |
| rev x8, x8 | |
| add x8, x8, #1 | |
| rev x8, x8 | |
| ins vctr.d[0], x8 | |
| /* | |
| * Apply carry to counter blocks if needed. | |
| * | |
| * Since the carry flag was set, we know 0 <= IV_PART < | |
| * MAX_STRIDE. Using the value of IV_PART we can determine how | |
| * many counter blocks need to be updated. | |
| */ | |
| cbz IV_PART, 2f | |
| adr x16, 1f | |
| sub x16, x16, IV_PART, lsl #3 | |
| br x16 | |
| bti c | |
| mov v0.d[0], vctr.d[0] | |
| bti c | |
| mov v1.d[0], vctr.d[0] | |
| bti c | |
| mov v2.d[0], vctr.d[0] | |
| bti c | |
| mov v3.d[0], vctr.d[0] | |
| ST5( bti c ) | |
| ST5( mov v4.d[0], vctr.d[0] ) | |
| 1: b 2f | |
| .previous | |
| 2: rev x7, IV_PART | |
| ins vctr.d[1], x7 | |
| sub x7, IV_PART, #MAX_STRIDE - 1 | |
| sub x8, IV_PART, #MAX_STRIDE - 2 | |
| sub x9, IV_PART, #MAX_STRIDE - 3 | |
| rev x7, x7 | |
| rev x8, x8 | |
| mov v1.d[1], x7 | |
| rev x9, x9 | |
| ST5( sub x10, IV_PART, #MAX_STRIDE - 4 ) | |
| mov v2.d[1], x8 | |
| ST5( rev x10, x10 ) | |
| mov v3.d[1], x9 | |
| ST5( mov v4.d[1], x10 ) | |
| .endif | |
| /* | |
| * If there are at least MAX_STRIDE blocks left, XOR the data with | |
| * keystream and store. Otherwise jump to tail handling. | |
| */ | |
| tbnz BYTES_W, #31, .Lctrtail\xctr | |
| ld1 {v5.16b-v7.16b}, [IN], #48 | |
| ST4( bl aes_encrypt_block4x ) | |
| ST5( bl aes_encrypt_block5x ) | |
| eor v0.16b, v5.16b, v0.16b | |
| ST4( ld1 {v5.16b}, [IN], #16 ) | |
| eor v1.16b, v6.16b, v1.16b | |
| ST5( ld1 {v5.16b-v6.16b}, [IN], #32 ) | |
| eor v2.16b, v7.16b, v2.16b | |
| eor v3.16b, v5.16b, v3.16b | |
| ST5( eor v4.16b, v6.16b, v4.16b ) | |
| st1 {v0.16b-v3.16b}, [OUT], #64 | |
| ST5( st1 {v4.16b}, [OUT], #16 ) | |
| cbz BYTES_W, .Lctrout\xctr | |
| b .LctrloopNx\xctr | |
| .Lctrout\xctr: | |
| .if !\xctr | |
| st1 {vctr.16b}, [IV] /* return next CTR value */ | |
| .endif | |
| frame_pop | |
| ret | |
| .Lctrtail\xctr: | |
| /* | |
| * Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext | |
| * | |
| * This code expects the last keystream block to be in v{MAX_STRIDE-1}. | |
| * For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and | |
| * v4 should have the next two counter blocks. | |
| * | |
| * This allows us to store the ciphertext by writing to overlapping | |
| * regions of memory. Any invalid ciphertext blocks get overwritten by | |
| * correctly computed blocks. This approach greatly simplifies the | |
| * logic for storing the ciphertext. | |
| */ | |
| mov x16, #16 | |
| ands w7, BYTES_W, #0xf | |
| csel x13, x7, x16, ne | |
| ST5( cmp BYTES_W, #64 - (MAX_STRIDE << 4)) | |
| ST5( csel x14, x16, xzr, gt ) | |
| cmp BYTES_W, #48 - (MAX_STRIDE << 4) | |
| csel x15, x16, xzr, gt | |
| cmp BYTES_W, #32 - (MAX_STRIDE << 4) | |
| csel x16, x16, xzr, gt | |
| cmp BYTES_W, #16 - (MAX_STRIDE << 4) | |
| adr_l x9, .Lcts_permute_table | |
| add x9, x9, x13 | |
| ble .Lctrtail1x\xctr | |
| ST5( ld1 {v5.16b}, [IN], x14 ) | |
| ld1 {v6.16b}, [IN], x15 | |
| ld1 {v7.16b}, [IN], x16 | |
| ST4( bl aes_encrypt_block4x ) | |
| ST5( bl aes_encrypt_block5x ) | |
| ld1 {v8.16b}, [IN], x13 | |
| ld1 {v9.16b}, [IN] | |
| ld1 {v10.16b}, [x9] | |
| ST4( eor v6.16b, v6.16b, v0.16b ) | |
| ST4( eor v7.16b, v7.16b, v1.16b ) | |
| ST4( tbl v3.16b, {v3.16b}, v10.16b ) | |
| ST4( eor v8.16b, v8.16b, v2.16b ) | |
| ST4( eor v9.16b, v9.16b, v3.16b ) | |
| ST5( eor v5.16b, v5.16b, v0.16b ) | |
| ST5( eor v6.16b, v6.16b, v1.16b ) | |
| ST5( tbl v4.16b, {v4.16b}, v10.16b ) | |
| ST5( eor v7.16b, v7.16b, v2.16b ) | |
| ST5( eor v8.16b, v8.16b, v3.16b ) | |
| ST5( eor v9.16b, v9.16b, v4.16b ) | |
| ST5( st1 {v5.16b}, [OUT], x14 ) | |
| st1 {v6.16b}, [OUT], x15 | |
| st1 {v7.16b}, [OUT], x16 | |
| add x13, x13, OUT | |
| st1 {v9.16b}, [x13] // overlapping stores | |
| st1 {v8.16b}, [OUT] | |
| b .Lctrout\xctr | |
| .Lctrtail1x\xctr: | |
| /* | |
| * Handle <= 16 bytes of plaintext | |
| * | |
| * This code always reads and writes 16 bytes. To avoid out of bounds | |
| * accesses, XCTR and CTR modes must use a temporary buffer when | |
| * encrypting/decrypting less than 16 bytes. | |
| * | |
| * This code is unusual in that it loads the input and stores the output | |
| * relative to the end of the buffers rather than relative to the start. | |
| * This causes unusual behaviour when encrypting/decrypting less than 16 | |
| * bytes; the end of the data is expected to be at the end of the | |
| * temporary buffer rather than the start of the data being at the start | |
| * of the temporary buffer. | |
| */ | |
| sub x8, x7, #16 | |
| csel x7, x7, x8, eq | |
| add IN, IN, x7 | |
| add OUT, OUT, x7 | |
| ld1 {v5.16b}, [IN] | |
| ld1 {v6.16b}, [OUT] | |
| ST5( mov v3.16b, v4.16b ) | |
| encrypt_block v3, ROUNDS_W, KEY, x8, w7 | |
| ld1 {v10.16b-v11.16b}, [x9] | |
| tbl v3.16b, {v3.16b}, v10.16b | |
| sshr v11.16b, v11.16b, #7 | |
| eor v5.16b, v5.16b, v3.16b | |
| bif v5.16b, v6.16b, v11.16b | |
| st1 {v5.16b}, [OUT] | |
| b .Lctrout\xctr | |
| // Arguments | |
| .unreq OUT | |
| .unreq IN | |
| .unreq KEY | |
| .unreq ROUNDS_W | |
| .unreq BYTES_W | |
| .unreq IV | |
| .unreq BYTE_CTR_W // XCTR only | |
| // Intermediate values | |
| .unreq CTR_W // XCTR only | |
| .unreq CTR // XCTR only | |
| .unreq IV_PART | |
| .unreq BLOCKS | |
| .unreq BLOCKS_W | |
| .endm | |
| /* | |
| * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, | |
| * int bytes, u8 ctr[]) | |
| * | |
| * The input and output buffers must always be at least 16 bytes even if | |
| * encrypting/decrypting less than 16 bytes. Otherwise out of bounds | |
| * accesses will occur. The data to be encrypted/decrypted is expected | |
| * to be at the end of this 16-byte temporary buffer rather than the | |
| * start. | |
| */ | |
| AES_FUNC_START(aes_ctr_encrypt) | |
| ctr_encrypt 0 | |
| AES_FUNC_END(aes_ctr_encrypt) | |
| /* | |
| * aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, | |
| * int bytes, u8 const iv[], int byte_ctr) | |
| * | |
| * The input and output buffers must always be at least 16 bytes even if | |
| * encrypting/decrypting less than 16 bytes. Otherwise out of bounds | |
| * accesses will occur. The data to be encrypted/decrypted is expected | |
| * to be at the end of this 16-byte temporary buffer rather than the | |
| * start. | |
| */ | |
| AES_FUNC_START(aes_xctr_encrypt) | |
| ctr_encrypt 1 | |
| AES_FUNC_END(aes_xctr_encrypt) | |
| /* | |
| * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, | |
| * int bytes, u8 const rk2[], u8 iv[], int first) | |
| * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, | |
| * int bytes, u8 const rk2[], u8 iv[], int first) | |
| */ | |
| .macro next_tweak, out, in, tmp | |
| sshr \tmp\().2d, \in\().2d, #63 | |
| and \tmp\().16b, \tmp\().16b, xtsmask.16b | |
| add \out\().2d, \in\().2d, \in\().2d | |
| ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8 | |
| eor \out\().16b, \out\().16b, \tmp\().16b | |
| .endm | |
| .macro xts_load_mask, tmp | |
| movi xtsmask.2s, #0x1 | |
| movi \tmp\().2s, #0x87 | |
| uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s | |
| .endm | |
| AES_FUNC_START(aes_xts_encrypt) | |
| frame_push 0 | |
| ld1 {v4.16b}, [x6] | |
| xts_load_mask v8 | |
| cbz w7, .Lxtsencnotfirst | |
| enc_prepare w3, x5, x8 | |
| xts_cts_skip_tw w7, .LxtsencNx | |
| encrypt_block v4, w3, x5, x8, w7 /* first tweak */ | |
| enc_switch_key w3, x2, x8 | |
| b .LxtsencNx | |
| .Lxtsencnotfirst: | |
| enc_prepare w3, x2, x8 | |
| .LxtsencloopNx: | |
| next_tweak v4, v4, v8 | |
| .LxtsencNx: | |
| subs w4, w4, #64 | |
| bmi .Lxtsenc1x | |
| ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ | |
| next_tweak v5, v4, v8 | |
| eor v0.16b, v0.16b, v4.16b | |
| next_tweak v6, v5, v8 | |
| eor v1.16b, v1.16b, v5.16b | |
| eor v2.16b, v2.16b, v6.16b | |
| next_tweak v7, v6, v8 | |
| eor v3.16b, v3.16b, v7.16b | |
| bl aes_encrypt_block4x | |
| eor v3.16b, v3.16b, v7.16b | |
| eor v0.16b, v0.16b, v4.16b | |
| eor v1.16b, v1.16b, v5.16b | |
| eor v2.16b, v2.16b, v6.16b | |
| st1 {v0.16b-v3.16b}, [x0], #64 | |
| mov v4.16b, v7.16b | |
| cbz w4, .Lxtsencret | |
| xts_reload_mask v8 | |
| b .LxtsencloopNx | |
| .Lxtsenc1x: | |
| adds w4, w4, #64 | |
| beq .Lxtsencout | |
| subs w4, w4, #16 | |
| bmi .LxtsencctsNx | |
| .Lxtsencloop: | |
| ld1 {v0.16b}, [x1], #16 | |
| .Lxtsencctsout: | |
| eor v0.16b, v0.16b, v4.16b | |
| encrypt_block v0, w3, x2, x8, w7 | |
| eor v0.16b, v0.16b, v4.16b | |
| cbz w4, .Lxtsencout | |
| subs w4, w4, #16 | |
| next_tweak v4, v4, v8 | |
| bmi .Lxtsenccts | |
| st1 {v0.16b}, [x0], #16 | |
| b .Lxtsencloop | |
| .Lxtsencout: | |
| st1 {v0.16b}, [x0] | |
| .Lxtsencret: | |
| st1 {v4.16b}, [x6] | |
| frame_pop | |
| ret | |
| .LxtsencctsNx: | |
| mov v0.16b, v3.16b | |
| sub x0, x0, #16 | |
| .Lxtsenccts: | |
| adr_l x8, .Lcts_permute_table | |
| add x1, x1, w4, sxtw /* rewind input pointer */ | |
| add w4, w4, #16 /* # bytes in final block */ | |
| add x9, x8, #32 | |
| add x8, x8, x4 | |
| sub x9, x9, x4 | |
| add x4, x0, x4 /* output address of final block */ | |
| ld1 {v1.16b}, [x1] /* load final block */ | |
| ld1 {v2.16b}, [x8] | |
| ld1 {v3.16b}, [x9] | |
| tbl v2.16b, {v0.16b}, v2.16b | |
| tbx v0.16b, {v1.16b}, v3.16b | |
| st1 {v2.16b}, [x4] /* overlapping stores */ | |
| mov w4, wzr | |
| b .Lxtsencctsout | |
| AES_FUNC_END(aes_xts_encrypt) | |
| AES_FUNC_START(aes_xts_decrypt) | |
| frame_push 0 | |
| /* subtract 16 bytes if we are doing CTS */ | |
| sub w8, w4, #0x10 | |
| tst w4, #0xf | |
| csel w4, w4, w8, eq | |
| ld1 {v4.16b}, [x6] | |
| xts_load_mask v8 | |
| xts_cts_skip_tw w7, .Lxtsdecskiptw | |
| cbz w7, .Lxtsdecnotfirst | |
| enc_prepare w3, x5, x8 | |
| encrypt_block v4, w3, x5, x8, w7 /* first tweak */ | |
| .Lxtsdecskiptw: | |
| dec_prepare w3, x2, x8 | |
| b .LxtsdecNx | |
| .Lxtsdecnotfirst: | |
| dec_prepare w3, x2, x8 | |
| .LxtsdecloopNx: | |
| next_tweak v4, v4, v8 | |
| .LxtsdecNx: | |
| subs w4, w4, #64 | |
| bmi .Lxtsdec1x | |
| ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ | |
| next_tweak v5, v4, v8 | |
| eor v0.16b, v0.16b, v4.16b | |
| next_tweak v6, v5, v8 | |
| eor v1.16b, v1.16b, v5.16b | |
| eor v2.16b, v2.16b, v6.16b | |
| next_tweak v7, v6, v8 | |
| eor v3.16b, v3.16b, v7.16b | |
| bl aes_decrypt_block4x | |
| eor v3.16b, v3.16b, v7.16b | |
| eor v0.16b, v0.16b, v4.16b | |
| eor v1.16b, v1.16b, v5.16b | |
| eor v2.16b, v2.16b, v6.16b | |
| st1 {v0.16b-v3.16b}, [x0], #64 | |
| mov v4.16b, v7.16b | |
| cbz w4, .Lxtsdecout | |
| xts_reload_mask v8 | |
| b .LxtsdecloopNx | |
| .Lxtsdec1x: | |
| adds w4, w4, #64 | |
| beq .Lxtsdecout | |
| subs w4, w4, #16 | |
| .Lxtsdecloop: | |
| ld1 {v0.16b}, [x1], #16 | |
| bmi .Lxtsdeccts | |
| .Lxtsdecctsout: | |
| eor v0.16b, v0.16b, v4.16b | |
| decrypt_block v0, w3, x2, x8, w7 | |
| eor v0.16b, v0.16b, v4.16b | |
| st1 {v0.16b}, [x0], #16 | |
| cbz w4, .Lxtsdecout | |
| subs w4, w4, #16 | |
| next_tweak v4, v4, v8 | |
| b .Lxtsdecloop | |
| .Lxtsdecout: | |
| st1 {v4.16b}, [x6] | |
| frame_pop | |
| ret | |
| .Lxtsdeccts: | |
| adr_l x8, .Lcts_permute_table | |
| add x1, x1, w4, sxtw /* rewind input pointer */ | |
| add w4, w4, #16 /* # bytes in final block */ | |
| add x9, x8, #32 | |
| add x8, x8, x4 | |
| sub x9, x9, x4 | |
| add x4, x0, x4 /* output address of final block */ | |
| next_tweak v5, v4, v8 | |
| ld1 {v1.16b}, [x1] /* load final block */ | |
| ld1 {v2.16b}, [x8] | |
| ld1 {v3.16b}, [x9] | |
| eor v0.16b, v0.16b, v5.16b | |
| decrypt_block v0, w3, x2, x8, w7 | |
| eor v0.16b, v0.16b, v5.16b | |
| tbl v2.16b, {v0.16b}, v2.16b | |
| tbx v0.16b, {v1.16b}, v3.16b | |
| st1 {v2.16b}, [x4] /* overlapping stores */ | |
| mov w4, wzr | |
| b .Lxtsdecctsout | |
| AES_FUNC_END(aes_xts_decrypt) | |
| #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) | |
| /* | |
| * void aes_mac_update(u8 const in[], u32 const rk[], int rounds, | |
| * size_t blocks, u8 dg[], int enc_before, | |
| * int enc_after); | |
| */ | |
| AES_FUNC_START(aes_mac_update) | |
| ld1 {v0.16b}, [x4] /* get dg */ | |
| enc_prepare w2, x1, x7 | |
| cbz w5, .Lmacloop4x | |
| encrypt_block v0, w2, x1, x7, w8 | |
| .Lmacloop4x: | |
| subs x3, x3, #4 | |
| bmi .Lmac1x | |
| ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */ | |
| eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ | |
| encrypt_block v0, w2, x1, x7, w8 | |
| eor v0.16b, v0.16b, v2.16b | |
| encrypt_block v0, w2, x1, x7, w8 | |
| eor v0.16b, v0.16b, v3.16b | |
| encrypt_block v0, w2, x1, x7, w8 | |
| eor v0.16b, v0.16b, v4.16b | |
| cmp x3, xzr | |
| csinv w5, w6, wzr, eq | |
| cbz w5, .Lmacout | |
| encrypt_block v0, w2, x1, x7, w8 | |
| st1 {v0.16b}, [x4] /* return dg */ | |
| b .Lmacloop4x | |
| .Lmac1x: | |
| add x3, x3, #4 | |
| .Lmacloop: | |
| cbz x3, .Lmacout | |
| ld1 {v1.16b}, [x0], #16 /* get next pt block */ | |
| eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ | |
| subs x3, x3, #1 | |
| csinv w5, w6, wzr, eq | |
| cbz w5, .Lmacout | |
| .Lmacenc: | |
| encrypt_block v0, w2, x1, x7, w8 | |
| b .Lmacloop | |
| .Lmacout: | |
| st1 {v0.16b}, [x4] /* return dg */ | |
| ret | |
| AES_FUNC_END(aes_mac_update) | |
| #endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ |