| /* |
| * Accelerated GHASH implementation with ARMv8 PMULL instructions. |
| * |
| * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org> |
| * |
| * Based on arch/x86/crypto/ghash-pmullni-intel_asm.S |
| * |
| * Copyright (c) 2009 Intel Corp. |
| * Author: Huang Ying <ying.huang@intel.com> |
| * Vinodh Gopal |
| * Erdinc Ozturk |
| * Deniz Karakoyunlu |
| * |
| * This program is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 as published |
| * by the Free Software Foundation. |
| */ |
| |
| #include <linux/linkage.h> |
| #include <asm/assembler.h> |
| |
| DATA .req v0 |
| SHASH .req v1 |
| IN1 .req v2 |
| T1 .req v2 |
| T2 .req v3 |
| T3 .req v4 |
| VZR .req v5 |
| |
| .text |
| .arch armv8-a+crypto |
| |
| /* |
| * void pmull_ghash_update(int blocks, u64 dg[], const char *src, |
| * struct ghash_key const *k, const char *head) |
| */ |
| ENTRY(pmull_ghash_update) |
| ld1 {DATA.16b}, [x1] |
| ld1 {SHASH.16b}, [x3] |
| eor VZR.16b, VZR.16b, VZR.16b |
| |
| /* do the head block first, if supplied */ |
| cbz x4, 0f |
| ld1 {IN1.2d}, [x4] |
| b 1f |
| |
| 0: ld1 {IN1.2d}, [x2], #16 |
| sub w0, w0, #1 |
| 1: ext IN1.16b, IN1.16b, IN1.16b, #8 |
| CPU_LE( rev64 IN1.16b, IN1.16b ) |
| eor DATA.16b, DATA.16b, IN1.16b |
| |
| /* multiply DATA by SHASH in GF(2^128) */ |
| ext T2.16b, DATA.16b, DATA.16b, #8 |
| ext T3.16b, SHASH.16b, SHASH.16b, #8 |
| eor T2.16b, T2.16b, DATA.16b |
| eor T3.16b, T3.16b, SHASH.16b |
| |
| pmull2 T1.1q, SHASH.2d, DATA.2d // a1 * b1 |
| pmull DATA.1q, SHASH.1d, DATA.1d // a0 * b0 |
| pmull T2.1q, T2.1d, T3.1d // (a1 + a0)(b1 + b0) |
| eor T2.16b, T2.16b, T1.16b // (a0 * b1) + (a1 * b0) |
| eor T2.16b, T2.16b, DATA.16b |
| |
| ext T3.16b, VZR.16b, T2.16b, #8 |
| ext T2.16b, T2.16b, VZR.16b, #8 |
| eor DATA.16b, DATA.16b, T3.16b |
| eor T1.16b, T1.16b, T2.16b // <T1:DATA> is result of |
| // carry-less multiplication |
| |
| /* first phase of the reduction */ |
| shl T3.2d, DATA.2d, #1 |
| eor T3.16b, T3.16b, DATA.16b |
| shl T3.2d, T3.2d, #5 |
| eor T3.16b, T3.16b, DATA.16b |
| shl T3.2d, T3.2d, #57 |
| ext T2.16b, VZR.16b, T3.16b, #8 |
| ext T3.16b, T3.16b, VZR.16b, #8 |
| eor DATA.16b, DATA.16b, T2.16b |
| eor T1.16b, T1.16b, T3.16b |
| |
| /* second phase of the reduction */ |
| ushr T2.2d, DATA.2d, #5 |
| eor T2.16b, T2.16b, DATA.16b |
| ushr T2.2d, T2.2d, #1 |
| eor T2.16b, T2.16b, DATA.16b |
| ushr T2.2d, T2.2d, #1 |
| eor T1.16b, T1.16b, T2.16b |
| eor DATA.16b, DATA.16b, T1.16b |
| |
| cbnz w0, 0b |
| |
| st1 {DATA.16b}, [x1] |
| ret |
| ENDPROC(pmull_ghash_update) |
