-
Notifications
You must be signed in to change notification settings - Fork 5.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge tag 'pull-crypto-20230915' of https://gitlab.com/rth7680/qemu i…
…nto staging Unify implementation of carry-less multiply. Accelerate carry-less multiply for 64x64->128. # -----BEGIN PGP SIGNATURE----- # # iQFRBAABCgA7FiEEekgeeIaLTbaoWgXAZN846K9+IV8FAmUEiPodHHJpY2hhcmQu # aGVuZGVyc29uQGxpbmFyby5vcmcACgkQZN846K9+IV/akgf/XkiIeErWJr1YXSbS # YPQtCsDAfIrqn3RiyQ2uwSn2eeuwVqTFFPGER04YegRDK8dyO874JBfvOwmBT70J # I/aU8Z4BbRyNu9nfaCtFMlXQH9KArAKcAds1PnshfcnI5T2yBloZ1sAU97IuJFZk # Uuz96H60+ohc4wzaUiPqPhXQStgZeSYwwAJB0s25DhCckdea0udRCAJ1tQTVpxkM # wIFef1SHPoM6DtMzFKHLLUH6VivSlHjqx8GqFusa7pVqfQyDzNBfwvDl1F/bkE07 # yTocQEkV3QnZvIplhqUxAaZXIFZr9BNk7bDimMjHW6z3pNPN3T8zRn4trNjxbgPV # jqzAtg== # =8nnk # -----END PGP SIGNATURE----- # gpg: Signature made Fri 15 Sep 2023 12:40:26 EDT # gpg: using RSA key 7A481E78868B4DB6A85A05C064DF38E8AF7E215F # gpg: issuer "richard.henderson@linaro.org" # gpg: Good signature from "Richard Henderson <richard.henderson@linaro.org>" [full] # Primary key fingerprint: 7A48 1E78 868B 4DB6 A85A 05C0 64DF 38E8 AF7E 215F * tag 'pull-crypto-20230915' of https://gitlab.com/rth7680/qemu: host/include/aarch64: Implement clmul.h host/include/i386: Implement clmul.h target/ppc: Use clmul_64 target/s390x: Use clmul_64 target/i386: Use clmul_64 target/arm: Use clmul_64 crypto: Add generic 64-bit carry-less multiply routine target/ppc: Use clmul_32* routines target/s390x: Use clmul_32* routines target/arm: Use clmul_32* routines crypto: Add generic 32-bit carry-less multiply routines target/ppc: Use clmul_16* routines target/s390x: Use clmul_16* routines target/arm: Use clmul_16* routines crypto: Add generic 16-bit carry-less multiply routines target/ppc: Use clmul_8* routines target/s390x: Use clmul_8* routines target/arm: Use clmul_8* routines crypto: Add generic 8-bit carry-less multiply routines Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
- Loading branch information
Showing
18 changed files
with
434 additions
and
286 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,111 @@ | ||
/* | ||
* Carry-less multiply operations. | ||
* SPDX-License-Identifier: GPL-2.0-or-later | ||
* | ||
* Copyright (C) 2023 Linaro, Ltd. | ||
*/ | ||
|
||
#include "qemu/osdep.h" | ||
#include "crypto/clmul.h" | ||
|
||
uint64_t clmul_8x8_low(uint64_t n, uint64_t m) | ||
{ | ||
uint64_t r = 0; | ||
|
||
for (int i = 0; i < 8; ++i) { | ||
uint64_t mask = (n & 0x0101010101010101ull) * 0xff; | ||
r ^= m & mask; | ||
m = (m << 1) & 0xfefefefefefefefeull; | ||
n >>= 1; | ||
} | ||
return r; | ||
} | ||
|
||
static uint64_t clmul_8x4_even_int(uint64_t n, uint64_t m) | ||
{ | ||
uint64_t r = 0; | ||
|
||
for (int i = 0; i < 8; ++i) { | ||
uint64_t mask = (n & 0x0001000100010001ull) * 0xffff; | ||
r ^= m & mask; | ||
n >>= 1; | ||
m <<= 1; | ||
} | ||
return r; | ||
} | ||
|
||
uint64_t clmul_8x4_even(uint64_t n, uint64_t m) | ||
{ | ||
n &= 0x00ff00ff00ff00ffull; | ||
m &= 0x00ff00ff00ff00ffull; | ||
return clmul_8x4_even_int(n, m); | ||
} | ||
|
||
uint64_t clmul_8x4_odd(uint64_t n, uint64_t m) | ||
{ | ||
return clmul_8x4_even(n >> 8, m >> 8); | ||
} | ||
|
||
static uint64_t unpack_8_to_16(uint64_t x) | ||
{ | ||
return (x & 0x000000ff) | ||
| ((x & 0x0000ff00) << 8) | ||
| ((x & 0x00ff0000) << 16) | ||
| ((x & 0xff000000) << 24); | ||
} | ||
|
||
uint64_t clmul_8x4_packed(uint32_t n, uint32_t m) | ||
{ | ||
return clmul_8x4_even_int(unpack_8_to_16(n), unpack_8_to_16(m)); | ||
} | ||
|
||
uint64_t clmul_16x2_even(uint64_t n, uint64_t m) | ||
{ | ||
uint64_t r = 0; | ||
|
||
n &= 0x0000ffff0000ffffull; | ||
m &= 0x0000ffff0000ffffull; | ||
|
||
for (int i = 0; i < 16; ++i) { | ||
uint64_t mask = (n & 0x0000000100000001ull) * 0xffffffffull; | ||
r ^= m & mask; | ||
n >>= 1; | ||
m <<= 1; | ||
} | ||
return r; | ||
} | ||
|
||
uint64_t clmul_16x2_odd(uint64_t n, uint64_t m) | ||
{ | ||
return clmul_16x2_even(n >> 16, m >> 16); | ||
} | ||
|
||
uint64_t clmul_32(uint32_t n, uint32_t m32) | ||
{ | ||
uint64_t r = 0; | ||
uint64_t m = m32; | ||
|
||
for (int i = 0; i < 32; ++i) { | ||
r ^= n & 1 ? m : 0; | ||
n >>= 1; | ||
m <<= 1; | ||
} | ||
return r; | ||
} | ||
|
||
Int128 clmul_64_gen(uint64_t n, uint64_t m) | ||
{ | ||
uint64_t rl = 0, rh = 0; | ||
|
||
/* Bit 0 can only influence the low 64-bit result. */ | ||
if (n & 1) { | ||
rl = m; | ||
} | ||
|
||
for (int i = 1; i < 64; ++i) { | ||
uint64_t mask = -((n >> i) & 1); | ||
rl ^= (m << i) & mask; | ||
rh ^= (m >> (64 - i)) & mask; | ||
} | ||
return int128_make128(rl, rh); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
/* | ||
* AArch64 specific clmul acceleration. | ||
* SPDX-License-Identifier: GPL-2.0-or-later | ||
*/ | ||
|
||
#ifndef AARCH64_HOST_CRYPTO_CLMUL_H | ||
#define AARCH64_HOST_CRYPTO_CLMUL_H | ||
|
||
#include "host/cpuinfo.h" | ||
#include <arm_neon.h> | ||
|
||
/* | ||
* 64x64->128 pmull is available with FEAT_PMULL. | ||
* Both FEAT_AES and FEAT_PMULL are covered under the same macro. | ||
*/ | ||
#ifdef __ARM_FEATURE_AES | ||
# define HAVE_CLMUL_ACCEL true | ||
#else | ||
# define HAVE_CLMUL_ACCEL likely(cpuinfo & CPUINFO_PMULL) | ||
#endif | ||
#if !defined(__ARM_FEATURE_AES) && defined(CONFIG_ARM_AES_BUILTIN) | ||
# define ATTR_CLMUL_ACCEL __attribute__((target("+crypto"))) | ||
#else | ||
# define ATTR_CLMUL_ACCEL | ||
#endif | ||
|
||
static inline Int128 ATTR_CLMUL_ACCEL | ||
clmul_64_accel(uint64_t n, uint64_t m) | ||
{ | ||
union { poly128_t v; Int128 s; } u; | ||
|
||
#ifdef CONFIG_ARM_AES_BUILTIN | ||
u.v = vmull_p64((poly64_t)n, (poly64_t)m); | ||
#else | ||
asm(".arch_extension aes\n\t" | ||
"pmull %0.1q, %1.1d, %2.1d" : "=w"(u.v) : "w"(n), "w"(m)); | ||
#endif | ||
return u.s; | ||
} | ||
|
||
#endif /* AARCH64_HOST_CRYPTO_CLMUL_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
/* | ||
* No host specific carry-less multiply acceleration. | ||
* SPDX-License-Identifier: GPL-2.0-or-later | ||
*/ | ||
|
||
#ifndef GENERIC_HOST_CRYPTO_CLMUL_H | ||
#define GENERIC_HOST_CRYPTO_CLMUL_H | ||
|
||
#define HAVE_CLMUL_ACCEL false | ||
#define ATTR_CLMUL_ACCEL | ||
|
||
Int128 clmul_64_accel(uint64_t, uint64_t) | ||
QEMU_ERROR("unsupported accel"); | ||
|
||
#endif /* GENERIC_HOST_CRYPTO_CLMUL_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
/* | ||
* x86 specific clmul acceleration. | ||
* SPDX-License-Identifier: GPL-2.0-or-later | ||
*/ | ||
|
||
#ifndef X86_HOST_CRYPTO_CLMUL_H | ||
#define X86_HOST_CRYPTO_CLMUL_H | ||
|
||
#include "host/cpuinfo.h" | ||
#include <immintrin.h> | ||
|
||
#if defined(__PCLMUL__) | ||
# define HAVE_CLMUL_ACCEL true | ||
# define ATTR_CLMUL_ACCEL | ||
#else | ||
# define HAVE_CLMUL_ACCEL likely(cpuinfo & CPUINFO_PCLMUL) | ||
# define ATTR_CLMUL_ACCEL __attribute__((target("pclmul"))) | ||
#endif | ||
|
||
static inline Int128 ATTR_CLMUL_ACCEL | ||
clmul_64_accel(uint64_t n, uint64_t m) | ||
{ | ||
union { __m128i v; Int128 s; } u; | ||
|
||
u.v = _mm_clmulepi64_si128(_mm_set_epi64x(0, n), _mm_set_epi64x(0, m), 0); | ||
return u.s; | ||
} | ||
|
||
#endif /* X86_HOST_CRYPTO_CLMUL_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
#include "host/include/i386/host/crypto/clmul.h" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
/* | ||
* Carry-less multiply operations. | ||
* SPDX-License-Identifier: GPL-2.0-or-later | ||
* | ||
* Copyright (C) 2023 Linaro, Ltd. | ||
*/ | ||
|
||
#ifndef CRYPTO_CLMUL_H | ||
#define CRYPTO_CLMUL_H | ||
|
||
#include "qemu/int128.h" | ||
#include "host/crypto/clmul.h" | ||
|
||
/** | ||
* clmul_8x8_low: | ||
* | ||
* Perform eight 8x8->8 carry-less multiplies. | ||
*/ | ||
uint64_t clmul_8x8_low(uint64_t, uint64_t); | ||
|
||
/** | ||
* clmul_8x4_even: | ||
* | ||
* Perform four 8x8->16 carry-less multiplies. | ||
* The odd bytes of the inputs are ignored. | ||
*/ | ||
uint64_t clmul_8x4_even(uint64_t, uint64_t); | ||
|
||
/** | ||
* clmul_8x4_odd: | ||
* | ||
* Perform four 8x8->16 carry-less multiplies. | ||
* The even bytes of the inputs are ignored. | ||
*/ | ||
uint64_t clmul_8x4_odd(uint64_t, uint64_t); | ||
|
||
/** | ||
* clmul_8x4_packed: | ||
* | ||
* Perform four 8x8->16 carry-less multiplies. | ||
*/ | ||
uint64_t clmul_8x4_packed(uint32_t, uint32_t); | ||
|
||
/** | ||
* clmul_16x2_even: | ||
* | ||
* Perform two 16x16->32 carry-less multiplies. | ||
* The odd words of the inputs are ignored. | ||
*/ | ||
uint64_t clmul_16x2_even(uint64_t, uint64_t); | ||
|
||
/** | ||
* clmul_16x2_odd: | ||
* | ||
* Perform two 16x16->32 carry-less multiplies. | ||
* The even words of the inputs are ignored. | ||
*/ | ||
uint64_t clmul_16x2_odd(uint64_t, uint64_t); | ||
|
||
/** | ||
* clmul_32: | ||
* | ||
* Perform a 32x32->64 carry-less multiply. | ||
*/ | ||
uint64_t clmul_32(uint32_t, uint32_t); | ||
|
||
/** | ||
* clmul_64: | ||
* | ||
* Perform a 64x64->128 carry-less multiply. | ||
*/ | ||
Int128 clmul_64_gen(uint64_t, uint64_t); | ||
|
||
static inline Int128 clmul_64(uint64_t a, uint64_t b) | ||
{ | ||
if (HAVE_CLMUL_ACCEL) { | ||
return clmul_64_accel(a, b); | ||
} else { | ||
return clmul_64_gen(a, b); | ||
} | ||
} | ||
|
||
#endif /* CRYPTO_CLMUL_H */ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.