diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a3d736bc..5e91a12ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: matrix: include: - name: "Linux / OpenSSL 1.1.0" - command: make -f misc/docker-ci.mk CMAKE_ARGS='-DOPENSSL_ROOT_DIR=-DOPENSSL_ROOT_DIR=/opt/openssl-1.1.0' CONTAINER_NAME='h2oserver/h2o-ci:ubuntu1604' + command: make -f misc/docker-ci.mk CMAKE_ARGS='-DOPENSSL_ROOT_DIR=-DOPENSSL_ROOT_DIR=/opt/openssl-1.1.0 -DWITH_FUSION=OFF' CONTAINER_NAME='h2oserver/h2o-ci:ubuntu1604' - name: "Linux / OpenSSL 1.1.1" command: make -f misc/docker-ci.mk - name: "Linux / OpenSSL 1.1.1 + ASan & UBSan" diff --git a/CMakeLists.txt b/CMakeLists.txt index 8bce0dff8..d2cb4c03b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,14 +148,14 @@ ENDIF () IF (WITH_FUSION) ADD_LIBRARY(picotls-fusion lib/fusion.c) - SET_TARGET_PROPERTIES(picotls-fusion PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul") + SET_TARGET_PROPERTIES(picotls-fusion PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul -mvaes -mvpclmulqdq") TARGET_LINK_LIBRARIES(picotls-fusion picotls-core) ADD_EXECUTABLE(test-fusion.t deps/picotest/picotest.c lib/picotls.c t/fusion.c) TARGET_LINK_LIBRARIES(test-fusion.t picotls-minicrypto) - SET_TARGET_PROPERTIES(test-fusion.t PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul") + SET_TARGET_PROPERTIES(test-fusion.t PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul -mvaes -mvpclmulqdq") IF (WITH_DTRACE) ADD_DEPENDENCIES(test-fusion.t generate-picotls-probes) ENDIF () diff --git a/include/picotls.h b/include/picotls.h index cad1206cb..f7e36b10b 100644 --- a/include/picotls.h +++ b/include/picotls.h @@ -70,6 +70,10 @@ extern "C" { #define PTLS_FUZZ_HANDSHAKE 0 #endif +#ifndef PTLS_SIZEOF_CACHE_LINE +#define PTLS_SIZEOF_CACHE_LINE 64 +#endif + #define PTLS_HELLO_RANDOM_SIZE 32 #define PTLS_AES128_KEY_SIZE 16 @@ -392,6 +396,10 @@ typedef const struct st_ptls_aead_algorithm_t { * size of the tag */ size_t tag_size; + /** + * if encrypted bytes are going to be written using non-temporal store instructions (i.e., skip cache) + */ + unsigned non_temporal : 1; /** * size of memory allocated for ptls_aead_context_t. AEAD implementations can set this value to something greater than * sizeof(ptls_aead_context_t) and stuff additional data at the bottom of the struct. diff --git a/include/picotls/fusion.h b/include/picotls/fusion.h index 332dd93ab..0c7f211f5 100644 --- a/include/picotls/fusion.h +++ b/include/picotls/fusion.h @@ -28,19 +28,24 @@ extern "C" { #include #include +#include #include "../picotls.h" #define PTLS_FUSION_AES128_ROUNDS 10 #define PTLS_FUSION_AES256_ROUNDS 14 typedef struct ptls_fusion_aesecb_context { - __m128i keys[PTLS_FUSION_AES256_ROUNDS + 1]; + union { + __m128i m128[PTLS_FUSION_AES256_ROUNDS + 1]; + __m256i m256[PTLS_FUSION_AES256_ROUNDS + 1]; + } keys; unsigned rounds; -} ptls_fusion_aesecb_context_t; + uint8_t aesni256; +} __attribute__((aligned(32))) ptls_fusion_aesecb_context_t; typedef struct ptls_fusion_aesgcm_context ptls_fusion_aesgcm_context_t; -void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size); +void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int avx256); void ptls_fusion_aesecb_dispose(ptls_fusion_aesecb_context_t *ctx); void ptls_fusion_aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, void *dst, const void *src); @@ -84,8 +89,15 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *aad, size_t aadlen, const void *tag); +/** + * A boolean flag indicating if vaes and vpclmulqdq (256-bit crypto instructions) should be used. This flag is set automatically + * when `ptls_fusion_is_supported_by_cpu` is called. Users can update the flag to enforce behavior. Engines that do not have support + * for these 256-bit instructions will continue using the 128-bit ones, even when this flag is set. + */ +extern int ptls_fusion_can_aesni256; extern ptls_cipher_algorithm_t ptls_fusion_aes128ctr, ptls_fusion_aes256ctr; extern ptls_aead_algorithm_t ptls_fusion_aes128gcm, ptls_fusion_aes256gcm; +extern ptls_aead_algorithm_t ptls_non_temporal_aes128gcm, ptls_non_temporal_aes256gcm; /** * Returns a boolean indicating if fusion can be used. diff --git a/lib/cifra/aes128.c b/lib/cifra/aes128.c index 9fac84507..34d353b39 100644 --- a/lib/cifra/aes128.c +++ b/lib/cifra/aes128.c @@ -55,6 +55,7 @@ ptls_aead_algorithm_t ptls_minicrypto_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context_t), aead_aes128gcm_setup_crypto}; ptls_cipher_suite_t ptls_minicrypto_aes128gcmsha256 = {.id = PTLS_CIPHER_SUITE_AES_128_GCM_SHA256, diff --git a/lib/cifra/aes256.c b/lib/cifra/aes256.c index 1d604fc23..ae3ac7c9b 100644 --- a/lib/cifra/aes256.c +++ b/lib/cifra/aes256.c @@ -55,6 +55,7 @@ ptls_aead_algorithm_t ptls_minicrypto_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context_t), aead_aes256gcm_setup_crypto}; ptls_cipher_suite_t ptls_minicrypto_aes256gcmsha384 = {.id = PTLS_CIPHER_SUITE_AES_256_GCM_SHA384, diff --git a/lib/cifra/chacha20.c b/lib/cifra/chacha20.c index 2e3584348..4a3f5098f 100644 --- a/lib/cifra/chacha20.c +++ b/lib/cifra/chacha20.c @@ -226,6 +226,7 @@ ptls_aead_algorithm_t ptls_minicrypto_chacha20poly1305 = {"CHACHA20-POLY1305", PTLS_CHACHA20_KEY_SIZE, PTLS_CHACHA20POLY1305_IV_SIZE, PTLS_CHACHA20POLY1305_TAG_SIZE, + 0, sizeof(struct chacha20poly1305_context_t), aead_chacha20poly1305_setup_crypto}; ptls_cipher_suite_t ptls_minicrypto_chacha20poly1305sha256 = {.id = PTLS_CIPHER_SUITE_CHACHA20_POLY1305_SHA256, diff --git a/lib/fusion.c b/lib/fusion.c index e0d977cb4..6c1afa104 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -17,7 +17,7 @@ * All other work, including modifications to the `transformH` function is * covered by the following MIT license: * - * Copyright (c) 2020 Fastly, Kazuho Oku + * Copyright (c) 2020-2022 Fastly, Kazuho Oku * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -48,16 +48,49 @@ #include "picotls.h" #include "picotls/fusion.h" +#if defined(__clang__) +#if __has_feature(address_sanitizer) +#define NO_SANITIZE_ADDRESS __attribute__((no_sanitize("address"))) +#endif +#elif __SANITIZE_ADDRESS__ /* gcc */ +#define NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#endif +#ifndef NO_SANITIZE_ADDRESS +#define NO_SANITIZE_ADDRESS +#endif + +#ifdef _WINDOWS +#define aligned_alloc(a, s) _aligned_malloc((s), (a)) +#endif + struct ptls_fusion_aesgcm_context { ptls_fusion_aesecb_context_t ecb; size_t capacity; size_t ghash_cnt; - struct ptls_fusion_aesgcm_ghash_precompute { +}; + +struct ptls_fusion_aesgcm_context128 { + struct ptls_fusion_aesgcm_context super; + struct ptls_fusion_aesgcm_ghash_precompute128 { __m128i H; __m128i r; } ghash[0]; }; +struct ptls_fusion_aesgcm_context256 { + struct ptls_fusion_aesgcm_context super; + union ptls_fusion_aesgcm_ghash_precompute256 { + struct { + __m128i H[2]; + __m128i r[2]; + }; + struct { + __m256i Hx2; + __m256i rx2; + }; + } ghash[0]; +}; + struct ctr_context { ptls_cipher_context_t super; ptls_fusion_aesecb_context_t fusion; @@ -76,10 +109,14 @@ struct aesgcm_context { static const uint64_t poly_[2] __attribute__((aligned(16))) = {1, 0xc200000000000000}; #define poly (*(__m128i *)poly_) -static const uint8_t bswap8_[16] __attribute__((aligned(16))) = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; -#define bswap8 (*(__m128i *)bswap8_) -static const uint8_t one8_[16] __attribute__((aligned(16))) = {1}; -#define one8 (*(__m128i *)one8_) +static const uint8_t byteswap_[32] __attribute__((aligned(32))) = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; +#define byteswap128 (*(__m128i *)byteswap_) +#define byteswap256 (*(__m256i *)byteswap_) +static const uint8_t one_[16] __attribute__((aligned(16))) = {1}; +#define one8 (*(__m128i *)one_) +static const uint8_t incr128x2_[32] __attribute__((aligned(32))) = {2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}; +#define incr128x2 (*(__m256i *)incr128x2_) /* This function is covered by the Apache License and the MIT License. The origin is crypto/modes/asm/ghash-x86_64.pl of openssl * at commit 33388b4. */ @@ -144,89 +181,210 @@ static __m128i gfmul(__m128i x, __m128i y) return _mm_xor_si128(hi, lo); } -struct ptls_fusion_gfmul_state { +static inline __m128i gfmul_do_reduce(__m128i hi, __m128i lo, __m128i mid) +{ + mid = _mm_xor_si128(mid, hi); + mid = _mm_xor_si128(mid, lo); + lo = _mm_xor_si128(lo, _mm_slli_si128(mid, 8)); + hi = _mm_xor_si128(hi, _mm_srli_si128(mid, 8)); + + /* fast reduction, using https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf */ + __m128i r = _mm_clmulepi64_si128(lo, poly, 0x10); + lo = _mm_shuffle_epi32(lo, 78); + lo = _mm_xor_si128(lo, r); + r = _mm_clmulepi64_si128(lo, poly, 0x10); + lo = _mm_shuffle_epi32(lo, 78); + lo = _mm_xor_si128(lo, r); + lo = _mm_xor_si128(hi, lo); + + return lo; +} + +struct ptls_fusion_gfmul_state128 { __m128i hi, lo, mid; }; -static inline void gfmul_onestep(struct ptls_fusion_gfmul_state *gstate, __m128i X, - struct ptls_fusion_aesgcm_ghash_precompute *precompute) +#if defined(__GNUC__) && !defined(__clang__) +static inline __m128i xor128(__m128i x, __m128i y) { - X = _mm_shuffle_epi8(X, bswap8); - __m128i t = _mm_clmulepi64_si128(precompute->H, X, 0x00); - gstate->lo = _mm_xor_si128(gstate->lo, t); - t = _mm_clmulepi64_si128(precompute->H, X, 0x11); - gstate->hi = _mm_xor_si128(gstate->hi, t); - t = _mm_shuffle_epi32(X, 78); - t = _mm_xor_si128(t, X); - t = _mm_clmulepi64_si128(precompute->r, t, 0x00); - gstate->mid = _mm_xor_si128(gstate->mid, t); + __m128i ret; + __asm__("vpxor %2, %1, %0" : "=x"(ret) : "x"(x), "xm"(y)); + return ret; } +#else +#define xor128 _mm_xor_si128 +#endif -static inline __m128i gfmul_final(struct ptls_fusion_gfmul_state *gstate, __m128i ek0) +static inline void gfmul_do_step128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { - /* finish multiplication */ - gstate->mid = _mm_xor_si128(gstate->mid, gstate->hi); - gstate->mid = _mm_xor_si128(gstate->mid, gstate->lo); - gstate->lo = _mm_xor_si128(gstate->lo, _mm_slli_si128(gstate->mid, 8)); - gstate->hi = _mm_xor_si128(gstate->hi, _mm_srli_si128(gstate->mid, 8)); + __m128i t1 = _mm_clmulepi64_si128(precompute->H, X, 0x00); + __m128i t2 = _mm_clmulepi64_si128(precompute->H, X, 0x11); + __m128i t3 = _mm_shuffle_epi32(X, 78); + t3 = _mm_xor_si128(t3, X); + t3 = _mm_clmulepi64_si128(precompute->r, t3, 0x00); + gstate->lo = xor128(gstate->lo, t1); + gstate->hi = xor128(gstate->hi, t2); + gstate->mid = xor128(gstate->mid, t3); +} - /* fast reduction, using https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf */ - __m128i r = _mm_clmulepi64_si128(gstate->lo, poly, 0x10); - gstate->lo = _mm_shuffle_epi32(gstate->lo, 78); - gstate->lo = _mm_xor_si128(gstate->lo, r); - r = _mm_clmulepi64_si128(gstate->lo, poly, 0x10); - gstate->lo = _mm_shuffle_epi32(gstate->lo, 78); - gstate->lo = _mm_xor_si128(gstate->lo, r); - __m128i tag = _mm_xor_si128(gstate->hi, gstate->lo); - tag = _mm_shuffle_epi8(tag, bswap8); +#undef xor128 + +static inline void gfmul_firststep128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) +{ + X = _mm_shuffle_epi8(X, byteswap128); + X = _mm_xor_si128(gstate->lo, X); + gstate->lo = _mm_setzero_si128(); + gstate->hi = _mm_setzero_si128(); + gstate->mid = _mm_setzero_si128(); + gfmul_do_step128(gstate, X, precompute); +} + +static inline void gfmul_nextstep128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) +{ + X = _mm_shuffle_epi8(X, byteswap128); + gfmul_do_step128(gstate, X, precompute); +} + +static inline void gfmul_reduce128(struct ptls_fusion_gfmul_state128 *gstate) +{ + gstate->lo = gfmul_do_reduce(gstate->hi, gstate->lo, gstate->mid); +} + +static inline __m128i gfmul_get_tag128(struct ptls_fusion_gfmul_state128 *gstate, __m128i ek0) +{ + __m128i tag = _mm_shuffle_epi8(gstate->lo, byteswap128); tag = _mm_xor_si128(tag, ek0); + return tag; +} + +struct ptls_fusion_gfmul_state256 { + __m256i hi, lo, mid; +}; + +static inline void gfmul_do_step256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, + union ptls_fusion_aesgcm_ghash_precompute256 *precompute) +{ + __m256i t = _mm256_clmulepi64_epi128(precompute->Hx2, X, 0x00); + gstate->lo = _mm256_xor_si256(gstate->lo, t); + t = _mm256_clmulepi64_epi128(precompute->Hx2, X, 0x11); + gstate->hi = _mm256_xor_si256(gstate->hi, t); + t = _mm256_shuffle_epi32(X, 78); + t = _mm256_xor_si256(t, X); + t = _mm256_clmulepi64_epi128(precompute->rx2, t, 0x00); + gstate->mid = _mm256_xor_si256(gstate->mid, t); +} + +static inline void gfmul_firststep256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, int half, + union ptls_fusion_aesgcm_ghash_precompute256 *precompute) +{ + X = _mm256_shuffle_epi8(X, byteswap256); + X = _mm256_xor_si256(gstate->lo, X); + if (half) + X = _mm256_permute2f128_si256(X, X, 0x08); + gstate->lo = _mm256_setzero_si256(); + gstate->hi = _mm256_setzero_si256(); + gstate->mid = _mm256_setzero_si256(); + gfmul_do_step256(gstate, X, precompute); +} + +static inline void gfmul_nextstep256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, + union ptls_fusion_aesgcm_ghash_precompute256 *precompute) +{ + X = _mm256_shuffle_epi8(X, byteswap256); + gfmul_do_step256(gstate, X, precompute); +} +static inline void gfmul_reduce256(struct ptls_fusion_gfmul_state256 *gstate) +{ +#define XOR_256TO128(y) _mm_xor_si128(_mm256_castsi256_si128(y), _mm256_extractf128_si256((y), 1)) + __m128i hi = XOR_256TO128(gstate->hi); + __m128i lo = XOR_256TO128(gstate->lo); + __m128i mid = XOR_256TO128(gstate->mid); +#undef XOR_256TO128 + + lo = gfmul_do_reduce(hi, lo, mid); + gstate->lo = _mm256_castsi128_si256(lo); +} + +static inline __m128i gfmul_get_tag256(struct ptls_fusion_gfmul_state256 *gstate, __m128i ek0) +{ + __m128i tag = _mm_shuffle_epi8(_mm256_castsi256_si128(gstate->lo), byteswap128); + tag = _mm_xor_si128(tag, ek0); return tag; } static inline __m128i aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, __m128i v) { - size_t i; +#define ROUNDKEY(i) (ctx->aesni256 ? _mm256_castsi256_si128(ctx->keys.m256[i]) : ctx->keys.m128[i]) - v = _mm_xor_si128(v, ctx->keys[0]); - for (i = 1; i < ctx->rounds; ++i) - v = _mm_aesenc_si128(v, ctx->keys[i]); - v = _mm_aesenclast_si128(v, ctx->keys[i]); + v = _mm_xor_si128(v, ROUNDKEY(0)); + for (size_t i = 1; i < ctx->rounds; ++i) + v = _mm_aesenc_si128(v, ROUNDKEY(i)); + v = _mm_aesenclast_si128(v, ROUNDKEY(ctx->rounds)); return v; + +#undef ROUNDKEY } -static const uint8_t loadn_mask[31] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; +// 32-bytes of 0xff followed by 31-bytes of 0x00 +static const uint8_t loadn_mask[63] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; static const uint8_t loadn_shuffle[31] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // first 16 bytes map to byte offsets 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; // latter 15 bytes map to zero -#if defined(__clang__) -#if __has_feature(address_sanitizer) -__attribute__((no_sanitize("address"))) -#endif -#elif __SANITIZE_ADDRESS__ /* gcc */ -__attribute__((no_sanitize_address)) -#endif -static inline __m128i loadn(const void *p, size_t l) +NO_SANITIZE_ADDRESS +static inline __m128i loadn_end_of_page(const void *p, size_t l) { - __m128i v, mask = _mm_loadu_si128((__m128i *)(loadn_mask + 16 - l)); + uintptr_t shift = (uintptr_t)p & 15; + __m128i pattern = _mm_loadu_si128((const __m128i *)(loadn_shuffle + shift)); + return _mm_shuffle_epi8(_mm_load_si128((const __m128i *)((uintptr_t)p - shift)), pattern); +} + +NO_SANITIZE_ADDRESS +static inline __m128i loadn128(const void *p, size_t l) +{ + __m128i v, mask = _mm_loadu_si128((__m128i *)(loadn_mask + 32 - l)); uintptr_t mod4k = (uintptr_t)p % 4096; - if (PTLS_LIKELY(mod4k <= 4080) || mod4k + l > 4096) { + if (PTLS_LIKELY(mod4k <= 4096 - 16) || mod4k + l > 4096) { v = _mm_loadu_si128(p); } else { - uintptr_t shift = (uintptr_t)p & 15; - __m128i pattern = _mm_loadu_si128((const __m128i *)(loadn_shuffle + shift)); - v = _mm_shuffle_epi8(_mm_load_si128((const __m128i *)((uintptr_t)p - shift)), pattern); + v = loadn_end_of_page(p, l); } v = _mm_and_si128(v, mask); + return v; } -static inline void storen(void *_p, size_t l, __m128i v) +NO_SANITIZE_ADDRESS +static inline __m256i loadn256(const void *p, size_t l) +{ + __m256i v, mask = _mm256_loadu_si256((__m256i *)(loadn_mask + 32 - l)); + uintptr_t mod4k = (uintptr_t)p % 4096; + + if (PTLS_LIKELY(mod4k < 4096 - 32) || mod4k + l > 4096) { + v = _mm256_loadu_si256(p); + } else if (l > 16) { + __m128i first16 = _mm_loadu_si128(p), second16 = loadn128((uint8_t *)p + 16, l - 16); + v = _mm256_permute2f128_si256(_mm256_castsi128_si256(first16), _mm256_castsi128_si256(second16), 0x20); + } else if (l == 16) { + v = _mm256_castsi128_si256(_mm_loadu_si128(p)); + } else { + v = _mm256_castsi128_si256(loadn_end_of_page(p, l)); + } + v = _mm256_and_si256(v, mask); + + return v; +} + +static inline void storen128(void *_p, size_t l, __m128i v) { uint8_t buf[16], *p = _p; @@ -236,25 +394,25 @@ static inline void storen(void *_p, size_t l, __m128i v) p[i] = buf[i]; } -void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const void *input, size_t inlen, __m128i ctr, +void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *_ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *_aad, size_t aadlen, ptls_aead_supplementary_encryption_t *supp) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ do { \ ctr = _mm_add_epi64(ctr, one8); \ - bits0 = _mm_shuffle_epi8(ctr, bswap8); \ + bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits1 = _mm_shuffle_epi8(ctr, bswap8); \ + bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits2 = _mm_shuffle_epi8(ctr, bswap8); \ + bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits3 = _mm_shuffle_epi8(ctr, bswap8); \ + bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits4 = _mm_shuffle_epi8(ctr, bswap8); \ + bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ if (PTLS_LIKELY(srclen > 16 * 5)) { \ ctr = _mm_add_epi64(ctr, one8); \ - bits5 = _mm_shuffle_epi8(ctr, bswap8); \ + bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ } else { \ if ((state & STATE_EK0_BEEN_FED) == 0) { \ bits5 = ek0; \ @@ -262,11 +420,11 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, } \ if ((state & STATE_SUPP_USED) != 0 && srclen <= 16 * 4 && (const __m128i *)supp->input + 1 <= dst_ghash) { \ bits4 = _mm_loadu_si128(supp->input); \ - bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys; \ + bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys.m128; \ state |= STATE_SUPP_IN_PROCESS; \ } \ } \ - __m128i k = ctx->ecb.keys[0]; \ + __m128i k = ctx->super.ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ @@ -278,7 +436,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -290,7 +448,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ @@ -299,11 +457,12 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, bits5 = _mm_aesenclast_si128(bits5, k); \ } while (0) + struct ptls_fusion_aesgcm_context128 *ctx = (void *)_ctx; __m128i ek0, bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); - const __m128i *bits4keys = ctx->ecb.keys; /* is changed to supp->ctx.keys when calcurating suppout */ - struct ptls_fusion_gfmul_state gstate = {0}; + const __m128i *bits4keys = ctx->super.ecb.keys.m128; /* is changed to supp->ctx.keys when calcurating suppout */ + struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; - __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), bswap8); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); // src and dst are updated after the chunk is processed const __m128i *src = input; @@ -313,7 +472,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const __m128i *aad = _aad, *dst_ghash = dst; size_t dst_ghashlen = srclen; - struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (srclen + 15) / 16 + 1; + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (srclen + 15) / 16 + 1; #define STATE_EK0_BEEN_FED 0x3 #define STATE_EK0_INCOMPLETE 0x2 @@ -324,7 +483,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* build counter */ ctr = _mm_insert_epi32(ctr, 1, 0); - ek0 = _mm_shuffle_epi8(ctr, bswap8); + ek0 = _mm_shuffle_epi8(ctr, byteswap128); /* start preparing AES */ AESECB6_INIT(); @@ -337,7 +496,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (PTLS_LIKELY(aadlen < 16)) { if (aadlen != 0) { - gdatabuf[gdata_cnt++] = loadn(aad, aadlen); + gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; } goto MainLoop; @@ -354,9 +513,9 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, size_t i; for (i = 2; i < gdata_cnt + 2; ++i) { AESECB6_UPDATE(i); - gfmul_onestep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } - for (; i < ctx->ecb.rounds; ++i) + for (; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); AESECB6_FINAL(i); @@ -404,7 +563,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, #undef APPLY goto ApplyEnd; ApplyRemainder: - storen(dst, srclen, _mm_xor_si128(loadn(src, srclen), bits0)); + storen128(dst, srclen, _mm_xor_si128(loadn128(src, srclen), bits0)); dst = (__m128i *)((uint8_t *)dst + srclen); srclen = 0; ApplyEnd:; @@ -422,7 +581,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (aadlen < 16) { if (aadlen != 0) { - gdatabuf[gdata_cnt++] = loadn(aad, aadlen); + gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; } goto GdataFillDST; @@ -442,7 +601,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (dst_ghashlen < 16) { if (dst_ghashlen != 0) { - gdatabuf[gdata_cnt++] = loadn(dst_ghash, dst_ghashlen); + gdatabuf[gdata_cnt++] = loadn128(dst_ghash, dst_ghashlen); dst_ghashlen = 0; } if (gdata_cnt < 6) @@ -465,15 +624,16 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, */ assert(STATE_EK0_READY()); for (size_t i = 0; i < gdata_cnt; ++i) - gfmul_onestep(&gstate, gdatabuf[i], --ghash_precompute); + gfmul_nextstep128(&gstate, gdatabuf[i], --ghash_precompute); - _mm_storeu_si128(dst, gfmul_final(&gstate, ek0)); + gfmul_reduce128(&gstate); + _mm_storeu_si128(dst, gfmul_get_tag128(&gstate, ek0)); /* Finish the calculation of supplemental vector. Done at the very last, because the sample might cover the GCM tag. */ if ((state & STATE_SUPP_USED) != 0) { size_t i; if ((state & STATE_SUPP_IN_PROCESS) == 0) { - bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys; + bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys.m128; bits4 = _mm_xor_si128(_mm_loadu_si128(supp->input), bits4keys[0]); i = 1; } else { @@ -481,7 +641,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, } do { bits4 = _mm_aesenc_si128(bits4, bits4keys[i++]); - } while (i != ctx->ecb.rounds); + } while (i != ctx->super.ecb.rounds); bits4 = _mm_aesenclast_si128(bits4, bits4keys[i]); _mm_storeu_si128((__m128i *)supp->output, bits4); } @@ -494,15 +654,16 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, #undef STATE_SUPP_IN_PROCESS } -int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const void *input, size_t inlen, __m128i ctr, +int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *_ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *_aad, size_t aadlen, const void *tag) { + struct ptls_fusion_aesgcm_context128 *ctx = (void *)_ctx; __m128i ek0 = _mm_setzero_si128(), bits0, bits1 = _mm_setzero_si128(), bits2 = _mm_setzero_si128(), bits3 = _mm_setzero_si128(), bits4 = _mm_setzero_si128(), bits5 = _mm_setzero_si128(); - struct ptls_fusion_gfmul_state gstate = {0}; + struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; - __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), bswap8); - struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (inlen + 15) / 16 + 1; + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (inlen + 15) / 16 + 1; const __m128i *gdata; // points to the elements fed into GHASH size_t gdata_cnt; @@ -513,7 +674,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* schedule ek0 and suppkey */ ctr = _mm_add_epi64(ctr, one8); - bits0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, bswap8), ctx->ecb.keys[0]); + bits0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), ctx->super.ecb.keys.m128[0]); ++nondata_aes_cnt; #define STATE_IS_FIRST_RUN 0x1 @@ -530,7 +691,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (aadlen < 16) { if (aadlen != 0) { - gdatabuf[gdata_cnt++] = loadn(aad, aadlen); + gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; ++nondata_aes_cnt; } @@ -552,7 +713,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (src_ghashlen < 16) { if (src_ghashlen != 0) { - gdatabuf[gdata_cnt++] = loadn(src_ghash, src_ghashlen); + gdatabuf[gdata_cnt++] = loadn128(src_ghash, src_ghashlen); src_ghash = (__m128i *)((uint8_t *)src_ghash + src_ghashlen); src_ghashlen = 0; } @@ -574,21 +735,21 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, #define INIT_BITS(n, keys) \ case n: \ ctr = _mm_add_epi64(ctr, one8); \ - bits##n = _mm_xor_si128(_mm_shuffle_epi8(ctr, bswap8), keys[0]); + bits##n = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), keys.m128[0]); InitAllBits: - INIT_BITS(0, ctx->ecb.keys); - INIT_BITS(1, ctx->ecb.keys); - INIT_BITS(2, ctx->ecb.keys); - INIT_BITS(3, ctx->ecb.keys); - INIT_BITS(4, ctx->ecb.keys); - INIT_BITS(5, ctx->ecb.keys); + INIT_BITS(0, ctx->super.ecb.keys); + INIT_BITS(1, ctx->super.ecb.keys); + INIT_BITS(2, ctx->super.ecb.keys); + INIT_BITS(3, ctx->super.ecb.keys); + INIT_BITS(4, ctx->super.ecb.keys); + INIT_BITS(5, ctx->super.ecb.keys); #undef INIT_BITS } { /* run aes and ghash */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -600,11 +761,11 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, size_t aesi; for (aesi = 1; aesi <= gdata_cnt; ++aesi) { AESECB6_UPDATE(aesi); - gfmul_onestep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } - for (; aesi < ctx->ecb.rounds; ++aesi) + for (; aesi < ctx->super.ecb.rounds; ++aesi) AESECB6_UPDATE(aesi); - __m128i k = ctx->ecb.keys[aesi]; + __m128i k = ctx->super.ecb.keys.m128[aesi]; bits0 = _mm_aesenclast_si128(bits0, k); bits1 = _mm_aesenclast_si128(bits1, k); bits2 = _mm_aesenclast_si128(bits2, k); @@ -659,7 +820,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, if (src_aeslen == 16) { _mm_storeu_si128(dst, _mm_xor_si128(_mm_loadu_si128(src_aes), bits0)); } else if (src_aeslen != 0) { - storen(dst, src_aeslen, _mm_xor_si128(loadn(src_aes, src_aeslen), bits0)); + storen128(dst, src_aeslen, _mm_xor_si128(loadn128(src_aes, src_aeslen), bits0)); } assert((state & STATE_IS_FIRST_RUN) == 0); @@ -667,10 +828,11 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* the only case where AES operation is complete and GHASH is not is when the application of AC is remaining */ if ((state & STATE_GHASH_HAS_MORE) != 0) { assert(ghash_precompute - 1 == ctx->ghash); - gfmul_onestep(&gstate, ac, --ghash_precompute); + gfmul_nextstep128(&gstate, ac, --ghash_precompute); } - __m128i calctag = gfmul_final(&gstate, ek0); + gfmul_reduce128(&gstate); + __m128i calctag = gfmul_get_tag128(&gstate, ek0); return _mm_movemask_epi8(_mm_cmpeq_epi8(calctag, _mm_loadu_si128(tag))) == 0xffff; @@ -689,7 +851,7 @@ static __m128i expand_key(__m128i key, __m128i temp) return key; } -void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size) +void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int aesni256) { assert(is_enc && "decryption is not supported (yet)"); @@ -706,37 +868,48 @@ void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, cons assert(!"invalid key size; AES128 / AES256 are supported"); break; } + ctx->aesni256 = aesni256; - ctx->keys[i++] = _mm_loadu_si128((__m128i *)key); + /* load and expand keys using keys.m128 */ + ctx->keys.m128[i++] = _mm_loadu_si128((__m128i *)key); if (key_size == 32) - ctx->keys[i++] = _mm_loadu_si128((__m128i *)key + 1); - + ctx->keys.m128[i++] = _mm_loadu_si128((__m128i *)key + 1); + while (1) { #define EXPAND(R) \ - do { \ - ctx->keys[i] = expand_key(ctx->keys[i - key_size / 16], \ - _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys[i - 1], R), _MM_SHUFFLE(3, 3, 3, 3))); \ + { \ + ctx->keys.m128[i] = \ + expand_key(ctx->keys.m128[i - key_size / 16], \ + _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys.m128[i - 1], R), _MM_SHUFFLE(3, 3, 3, 3))); \ if (i == ctx->rounds) \ - goto Done; \ + break; \ ++i; \ if (key_size > 24) { \ - ctx->keys[i] = expand_key(ctx->keys[i - key_size / 16], \ - _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys[i - 1], R), _MM_SHUFFLE(2, 2, 2, 2))); \ + ctx->keys.m128[i] = \ + expand_key(ctx->keys.m128[i - key_size / 16], \ + _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys.m128[i - 1], R), _MM_SHUFFLE(2, 2, 2, 2))); \ ++i; \ } \ - } while (0) - EXPAND(0x1); - EXPAND(0x2); - EXPAND(0x4); - EXPAND(0x8); - EXPAND(0x10); - EXPAND(0x20); - EXPAND(0x40); - EXPAND(0x80); - EXPAND(0x1b); - EXPAND(0x36); + } + EXPAND(0x1); + EXPAND(0x2); + EXPAND(0x4); + EXPAND(0x8); + EXPAND(0x10); + EXPAND(0x20); + EXPAND(0x40); + EXPAND(0x80); + EXPAND(0x1b); + EXPAND(0x36); #undef EXPAND -Done: - assert(i == ctx->rounds); + } + + /* convert to keys.m256 if aesni256 is used */ + if (ctx->aesni256) { + size_t i = ctx->rounds; + do { + ctx->keys.m256[i] = _mm256_broadcastsi128_si256(ctx->keys.m128[i]); + } while (i-- != 0); + } } void ptls_fusion_aesecb_dispose(ptls_fusion_aesecb_context_t *ctx) @@ -762,31 +935,70 @@ static size_t aesgcm_calc_ghash_cnt(size_t capacity) static void setup_one_ghash_entry(ptls_fusion_aesgcm_context_t *ctx) { - if (ctx->ghash_cnt != 0) - ctx->ghash[ctx->ghash_cnt].H = gfmul(ctx->ghash[ctx->ghash_cnt - 1].H, ctx->ghash[0].H); + __m128i *H, *r, *Hprev, H0; + + if (ctx->ecb.aesni256) { + struct ptls_fusion_aesgcm_context256 *ctx256 = (void *)ctx; +#define GET_SLOT(i, mem) (&ctx256->ghash[(i) / 2].mem[(i) % 2 == 0]) + H = GET_SLOT(ctx->ghash_cnt, H); + r = GET_SLOT(ctx->ghash_cnt, r); + Hprev = ctx->ghash_cnt == 0 ? NULL : GET_SLOT(ctx->ghash_cnt - 1, H); +#undef GET_SLOT + H0 = ctx256->ghash[0].H[1]; + } else { + struct ptls_fusion_aesgcm_context128 *ctx128 = (void *)ctx; + H = &ctx128->ghash[ctx->ghash_cnt].H; + r = &ctx128->ghash[ctx->ghash_cnt].r; + Hprev = ctx->ghash_cnt == 0 ? NULL : &ctx128->ghash[ctx->ghash_cnt - 1].H; + H0 = ctx128->ghash[0].H; + } - __m128i r = _mm_shuffle_epi32(ctx->ghash[ctx->ghash_cnt].H, 78); - r = _mm_xor_si128(r, ctx->ghash[ctx->ghash_cnt].H); - ctx->ghash[ctx->ghash_cnt].r = r; + if (Hprev != NULL) + *H = gfmul(*Hprev, H0); + + *r = _mm_shuffle_epi32(*H, 78); + *r = _mm_xor_si128(*r, *H); ++ctx->ghash_cnt; } -ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_new(const void *key, size_t key_size, size_t capacity) +static size_t calc_aesgcm_context_size(size_t *ghash_cnt, int aesni256) +{ + size_t sz; + + if (aesni256) { + if (*ghash_cnt % 2 != 0) + ++*ghash_cnt; + sz = offsetof(struct ptls_fusion_aesgcm_context256, ghash) + + sizeof(union ptls_fusion_aesgcm_ghash_precompute256) * *ghash_cnt / 2; + } else { + sz = offsetof(struct ptls_fusion_aesgcm_context128, ghash) + + sizeof(struct ptls_fusion_aesgcm_ghash_precompute128) * *ghash_cnt; + } + return sz; +} + +static ptls_fusion_aesgcm_context_t *new_aesgcm(const void *key, size_t key_size, size_t capacity, int aesni256) { ptls_fusion_aesgcm_context_t *ctx; - size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity); + size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity), ctx_size = calc_aesgcm_context_size(&ghash_cnt, aesni256); - if ((ctx = malloc(sizeof(*ctx) + sizeof(ctx->ghash[0]) * ghash_cnt)) == NULL) + if ((ctx = aligned_alloc(32, ctx_size)) == NULL) return NULL; - ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size); + ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size, aesni256); ctx->capacity = capacity; - ctx->ghash[0].H = aesecb_encrypt(&ctx->ecb, _mm_setzero_si128()); - ctx->ghash[0].H = _mm_shuffle_epi8(ctx->ghash[0].H, bswap8); - ctx->ghash[0].H = transformH(ctx->ghash[0].H); + __m128i H0 = aesecb_encrypt(&ctx->ecb, _mm_setzero_si128()); + H0 = _mm_shuffle_epi8(H0, byteswap128); + H0 = transformH(H0); + if (ctx->ecb.aesni256) { + ((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash[0].H[1] = H0; + } else { + ((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash[0].H = H0; + } + ctx->ghash_cnt = 0; while (ctx->ghash_cnt < ghash_cnt) setup_one_ghash_entry(ctx); @@ -794,6 +1006,11 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_new(const void *key, size_t key return ctx; } +ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_new(const void *key, size_t key_size, size_t capacity) +{ + return new_aesgcm(key, key_size, capacity, 0); +} + ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm_context_t *ctx, size_t capacity) { size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity); @@ -801,8 +1018,13 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm if (ghash_cnt <= ctx->ghash_cnt) return ctx; - if ((ctx = realloc(ctx, sizeof(*ctx) + sizeof(ctx->ghash[0]) * ghash_cnt)) == NULL) + size_t ctx_size = calc_aesgcm_context_size(&ghash_cnt, ctx->ecb.aesni256); + ptls_fusion_aesgcm_context_t *newp; + if ((newp = aligned_alloc(32, ctx_size)) == NULL) return NULL; + memcpy(newp, ctx, ctx_size); + free(ctx); + ctx = newp; ctx->capacity = capacity; while (ghash_cnt < ctx->ghash_cnt) @@ -813,9 +1035,9 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm void ptls_fusion_aesgcm_free(ptls_fusion_aesgcm_context_t *ctx) { - ptls_clear_memory(ctx->ghash, sizeof(ctx->ghash[0]) * ctx->ghash_cnt); - ctx->ghash_cnt = 0; - ptls_fusion_aesecb_dispose(&ctx->ecb); + ptls_clear_memory(ctx, calc_aesgcm_context_size(&ctx->ghash_cnt, ctx->ecb.aesni256)); + /* skip ptls_fusion_aesecb_dispose, based on the knowledge that it does not allocate memory elsewhere */ + free(ctx); } @@ -842,7 +1064,7 @@ static void ctr_transform(ptls_cipher_context_t *_ctx, void *output, const void ctx->is_ready = 0; if (len < 16) { - storen(output, len, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), loadn(input, len))); + storen128(output, len, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), loadn128(input, len))); } else { _mm_storeu_si128(output, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), _mm_loadu_si128(input))); } @@ -855,7 +1077,7 @@ static int aesctr_setup(ptls_cipher_context_t *_ctx, int is_enc, const void *key ctx->super.do_dispose = ctr_dispose; ctx->super.do_init = ctr_init; ctx->super.do_transform = ctr_transform; - ptls_fusion_aesecb_init(&ctx->fusion, 1, key, key_size); + ptls_fusion_aesecb_init(&ctx->fusion, 1, key, key_size, 0 /* probably we do not need aesni256 for CTR? */); ctx->is_ready = 0; return 0; @@ -940,8 +1162,8 @@ static size_t aead_do_decrypt(ptls_aead_context_t *_ctx, void *output, const voi static inline void aesgcm_xor_iv(ptls_aead_context_t *_ctx, const void *_bytes, size_t len) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; - __m128i xor_mask = loadn(_bytes, len); - xor_mask = _mm_shuffle_epi8(xor_mask, bswap8); + __m128i xor_mask = loadn128(_bytes, len); + xor_mask = _mm_shuffle_epi8(xor_mask, byteswap128); ctx->static_iv = _mm_xor_si128(ctx->static_iv, xor_mask); } @@ -949,8 +1171,8 @@ static int aesgcm_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; - ctx->static_iv = loadn(iv, PTLS_AESGCM_IV_SIZE); - ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, bswap8); + ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); + ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); if (key == NULL) return 0; @@ -963,7 +1185,7 @@ static int aesgcm_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, ctx->super.do_encrypt_v = aead_do_encrypt_v; ctx->super.do_decrypt = aead_do_decrypt; - ctx->aesgcm = ptls_fusion_aesgcm_new(key, key_size, 1500 /* assume ordinary packet size */); + ctx->aesgcm = new_aesgcm(key, key_size, 1500 /* assume ordinary packet size */, 0 /* no support for aesni256 yet */); return 0; } @@ -978,6 +1200,7 @@ static int aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key return aesgcm_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); } +int ptls_fusion_can_aesni256 = 0; ptls_cipher_algorithm_t ptls_fusion_aes128ctr = {"AES128-CTR", PTLS_AES128_KEY_SIZE, 1, // block size @@ -998,6 +1221,7 @@ ptls_aead_algorithm_t ptls_fusion_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context), aes128gcm_setup}; ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", @@ -1008,9 +1232,928 @@ ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context), aes256gcm_setup}; +static inline size_t calc_total_length(ptls_iovec_t *input, size_t incnt) +{ + size_t totlen = 0; + for (size_t i = 0; i < incnt; ++i) + totlen += input[i].len; + return totlen; +} + +static inline void reduce_aad128(struct ptls_fusion_gfmul_state128 *gstate, struct ptls_fusion_aesgcm_ghash_precompute128 *ghash, + const void *_aad, size_t aadlen) +{ + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute; + const uint8_t *aad = _aad; + + while (PTLS_UNLIKELY(aadlen >= 6 * 16)) { + ghash_precompute = ghash + 6; + gfmul_firststep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + for (int i = 1; i < 6; ++i) { + gfmul_nextstep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + } + gfmul_reduce128(gstate); + } + + if (PTLS_LIKELY(aadlen != 0)) { + ghash_precompute = ghash + (aadlen + 15) / 16; + if (PTLS_UNLIKELY(aadlen >= 16)) { + gfmul_firststep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + while (aadlen >= 16) { + gfmul_nextstep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + } + if (PTLS_LIKELY(aadlen != 0)) + gfmul_nextstep128(gstate, loadn128(aad, aadlen), --ghash_precompute); + } else { + gfmul_firststep128(gstate, loadn128(aad, aadlen), --ghash_precompute); + } + assert(ghash == ghash_precompute); + gfmul_reduce128(gstate); + } +} + +NO_SANITIZE_ADDRESS +static inline uint8_t *load_preceding_unaligned(uint8_t *encbuf, uint8_t **output) +{ + uint8_t *encp; + + if ((encp = encbuf + ((uintptr_t)*output & 63)) != encbuf) { + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(*output - (encp - encbuf)))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(*output - (encp - encbuf) + 32))); + *output -= encp - encbuf; + } + + return encp; +} + +NO_SANITIZE_ADDRESS +static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const uint8_t *end) +{ + /* Write in 64-byte chunks, using NT store instructions. Last partial block, if any, is written to cache, as that cache line + * would likely be read when the next TLS record is being built. */ + + for (; end - src >= 64; dst += 64, src += 64) { + _mm256_stream_si256((void *)dst, _mm256_load_si256((void *)src)); + _mm256_stream_si256((void *)(dst + 32), _mm256_load_si256((void *)(src + 32))); + } + _mm_sfence(); /* weakly ordered writes have to be synced before being passed to NIC */ + if (src != end) { + for (; end - src >= 16; dst += 16, src += 16) + _mm_store_si128((void *)dst, _mm_load_si128((void *)src)); + if (src != end) + storen128((void *)dst, end - src, loadn128((void *)src, end - src)); + } +} + +NO_SANITIZE_ADDRESS +static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, + uint64_t seq, const void *aad, size_t aadlen) +{ +/* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ +#define AESECB6_INIT() \ + do { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ + if (PTLS_LIKELY(srclen > 16 * 5) || src_vecleft != 0) { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ + } else { \ + bits5 = ek0; \ + state |= STATE_EK0_READY; \ + } \ + __m128i k = ctx->super.ecb.keys.m128[0]; \ + bits0 = _mm_xor_si128(bits0, k); \ + bits1 = _mm_xor_si128(bits1, k); \ + bits2 = _mm_xor_si128(bits2, k); \ + bits3 = _mm_xor_si128(bits3, k); \ + bits4 = _mm_xor_si128(bits4, k); \ + bits5 = _mm_xor_si128(bits5, k); \ + } while (0) + +/* aes block update */ +#define AESECB6_UPDATE(i) \ + do { \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ + bits0 = _mm_aesenc_si128(bits0, k); \ + bits1 = _mm_aesenc_si128(bits1, k); \ + bits2 = _mm_aesenc_si128(bits2, k); \ + bits3 = _mm_aesenc_si128(bits3, k); \ + bits4 = _mm_aesenc_si128(bits4, k); \ + bits5 = _mm_aesenc_si128(bits5, k); \ + } while (0) + +/* aesenclast */ +#define AESECB6_FINAL(i) \ + do { \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ + bits0 = _mm_aesenclast_si128(bits0, k); \ + bits1 = _mm_aesenclast_si128(bits1, k); \ + bits2 = _mm_aesenclast_si128(bits2, k); \ + bits3 = _mm_aesenclast_si128(bits3, k); \ + bits4 = _mm_aesenclast_si128(bits4, k); \ + bits5 = _mm_aesenclast_si128(bits5, k); \ + } while (0) + + struct aesgcm_context *agctx = (void *)_ctx; + uint8_t *output = _output; + +#define STATE_EK0_READY 0x1 +#define STATE_COPY_128B 0x2 + int32_t state = 0; + + /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ + uint8_t encbuf[32 * 6] __attribute__((aligned(32))), *encp; + + /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is + * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ + PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 16 + 16); + + /* load unaligned data within same cache line preceding `output`, adjusting pointers accordingly */ + encp = load_preceding_unaligned(encbuf, &output); + + /* First write would be 128 bytes (32+6*16), if encbuf contains no less than 32 bytes already. */ + if (encp - encbuf >= 32) + state |= STATE_COPY_128B; + + /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ + __m128i ctr = calc_counter(agctx, seq); + ctr = _mm_insert_epi32(ctr, 1, 0); + __m128i ek0 = _mm_shuffle_epi8(ctr, byteswap128); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128); + + struct ptls_fusion_aesgcm_context128 *ctx = (void *)agctx->aesgcm; + __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); + struct ptls_fusion_gfmul_state128 gstate = {0}; + + /* find the first non-empty vec */ + const uint8_t *src = NULL; + size_t srclen = 0, src_vecleft = incnt; + while (srclen == 0 && src_vecleft != 0) { + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + + /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ + AESECB6_INIT(); + AESECB6_UPDATE(1); + AESECB6_UPDATE(2); + reduce_aad128(&gstate, ctx->ghash, aad, aadlen); + for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) + AESECB6_UPDATE(i); + AESECB6_FINAL(ctx->super.ecb.rounds); + + /* Main loop. This loop: + * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, + * 2. then if there is no more data to be encrypted, exit the loop, otherwise, + * 3. calculate ghash of the blocks being written to encbuf, + * 4. calculate next 6 * 16 bytes of keystream, + * 5. writes encbuf in 64-byte blocks + * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be + * calculated. */ + size_t remaining_ghash_from = encp - encbuf; + if (srclen != 0) { + while (1) { + /* apply the bit stream to input, writing to encbuf */ + if (PTLS_LIKELY(srclen >= 6 * 16)) { +#define APPLY(i) _mm_storeu_si128((void *)(encp + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encp += 6 * 16; + src += 6 * 16; + srclen -= 6 * 16; + if (PTLS_UNLIKELY(srclen == 0)) { + if (src_vecleft == 0) { + remaining_ghash_from = (encp - encbuf) - 96; + break; + } + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + } else { + /* slow path, load at most 6 * 16 bytes to encbuf then encrypt in-place */ + size_t bytes_copied = 0; + do { + if (srclen >= 16 && bytes_copied < 5 * 16) { + _mm_storeu_si128((void *)(encp + bytes_copied), _mm_loadu_si128((void *)src)); + bytes_copied += 16; + src += 16; + srclen -= 16; + } else { + encp[bytes_copied++] = *src++; + --srclen; + } + if (PTLS_UNLIKELY(srclen == 0)) { + do { + if (src_vecleft == 0) + break; + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } while (srclen == 0); + if (srclen == 0) + break; + } + } while (bytes_copied < 6 * 16); +#define APPLY(i) _mm_storeu_si128((void *)(encp + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(encp + i * 16)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encp += bytes_copied; + if (PTLS_UNLIKELY(srclen == 0)) { + /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it + * will be fed into ghash. */ + remaining_ghash_from = (encp - encbuf) - bytes_copied; + if ((bytes_copied & 15) != 0) + _mm_storeu_si128((void *)encp, _mm_setzero_si128()); + break; + } + } + + /* Next 96-byte block starts here. Run AES and ghash in while writing output using non-temporal stores in 64-byte + * blocks. */ + AESECB6_INIT(); + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + 6; + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encp - 6 * 16)), --ghash_precompute); + AESECB6_UPDATE(1); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 5 * 16)), --ghash_precompute); + AESECB6_UPDATE(2); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 4 * 16)), --ghash_precompute); + AESECB6_UPDATE(3); + _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); + _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); + AESECB6_UPDATE(4); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 3 * 16)), --ghash_precompute); + AESECB6_UPDATE(5); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 2 * 16)), --ghash_precompute); + AESECB6_UPDATE(6); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 1 * 16)), --ghash_precompute); + AESECB6_UPDATE(7); + if ((state & STATE_COPY_128B) != 0) { + _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); + _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); + output += 128; + encp -= 128; + AESECB6_UPDATE(8); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 128))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 160))); + } else { + output += 64; + encp -= 64; + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 64))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 96))); + AESECB6_UPDATE(8); + } + state ^= STATE_COPY_128B; + AESECB6_UPDATE(9); + if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { + for (size_t i = 10; PTLS_LIKELY(i < ctx->super.ecb.rounds); ++i) + AESECB6_UPDATE(i); + } + assert(ctx->ghash == ghash_precompute); + gfmul_reduce128(&gstate); + AESECB6_FINAL(ctx->super.ecb.rounds); + } + } + + /* Now, All the encrypted bits are built in encbuf. Calculate AEAD tag and append to encbuf. */ + + { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 + * blocks at once. */ + size_t ac_off = remaining_ghash_from + ((encp - encbuf) - remaining_ghash_from + 15) / 16 * 16; + _mm_storeu_si128((void *)(encbuf + ac_off), ac); + size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ + assert(blocks <= 7); + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + blocks; + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + remaining_ghash_from += 16; + while (ghash_precompute != ctx->ghash) { + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + remaining_ghash_from += 16; + } + gfmul_reduce128(&gstate); + } + + /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ + if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { + bits5 = _mm_xor_si128(ek0, ctx->super.ecb.keys.m128[0]); + for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) + bits5 = _mm_aesenc_si128(bits5, ctx->super.ecb.keys.m128[i]); + bits5 = _mm_aesenclast_si128(bits5, ctx->super.ecb.keys.m128[ctx->super.ecb.rounds]); + } + + /* append tag to encbuf */ + _mm_storeu_si128((void *)encp, gfmul_get_tag128(&gstate, bits5)); + encp += 16; + + /* write remaining bytes */ + write_remaining_bytes(output, encbuf, encp); + +#undef AESECB6_INIT +#undef AESECB6_UPDATE +#undef AESECB6_FINAL +#undef STATE_EK0_READY +#undef STATE_COPY_128B +} + +static size_t non_temporal_decrypt128(ptls_aead_context_t *_ctx, void *_output, const void *_input, size_t inlen, uint64_t seq, + const void *aad, size_t aadlen) +{ + /* Bail out if the input is too short, or remove tag from range. */ + if (inlen < 16) + return SIZE_MAX; + inlen -= 16; + size_t textlen = inlen; + +/* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ +#define AESECB6_INIT() \ + do { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ + if (PTLS_LIKELY(inlen > 16 * 5)) { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ + } else { \ + bits5 = ek0; \ + state |= STATE_EK0_READY; \ + } \ + __m128i k = ctx->super.ecb.keys.m128[0]; \ + bits0 = _mm_xor_si128(bits0, k); \ + bits1 = _mm_xor_si128(bits1, k); \ + bits2 = _mm_xor_si128(bits2, k); \ + bits3 = _mm_xor_si128(bits3, k); \ + bits4 = _mm_xor_si128(bits4, k); \ + bits5 = _mm_xor_si128(bits5, k); \ + } while (0) + +/* aes block update */ +#define AESECB6_UPDATE(i) \ + do { \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ + bits0 = _mm_aesenc_si128(bits0, k); \ + bits1 = _mm_aesenc_si128(bits1, k); \ + bits2 = _mm_aesenc_si128(bits2, k); \ + bits3 = _mm_aesenc_si128(bits3, k); \ + bits4 = _mm_aesenc_si128(bits4, k); \ + bits5 = _mm_aesenc_si128(bits5, k); \ + } while (0) + +/* aesenclast */ +#define AESECB6_FINAL(i) \ + do { \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ + bits0 = _mm_aesenclast_si128(bits0, k); \ + bits1 = _mm_aesenclast_si128(bits1, k); \ + bits2 = _mm_aesenclast_si128(bits2, k); \ + bits3 = _mm_aesenclast_si128(bits3, k); \ + bits4 = _mm_aesenclast_si128(bits4, k); \ + bits5 = _mm_aesenclast_si128(bits5, k); \ + } while (0) + + struct aesgcm_context *agctx = (void *)_ctx; + uint8_t *output = _output; + const uint8_t *input = _input; + +#define STATE_EK0_READY 0x1 + int32_t state = 0; + + /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ + __m128i ctr = calc_counter(agctx, seq); + ctr = _mm_insert_epi32(ctr, 1, 0); + __m128i ek0 = _mm_shuffle_epi8(ctr, byteswap128); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); + + struct ptls_fusion_aesgcm_context128 *ctx = (void *)agctx->aesgcm; + __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); + struct ptls_fusion_gfmul_state128 gstate = {0}; + + /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ + AESECB6_INIT(); + AESECB6_UPDATE(1); + AESECB6_UPDATE(2); + reduce_aad128(&gstate, ctx->ghash, aad, aadlen); + for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) + AESECB6_UPDATE(i); + AESECB6_FINAL(ctx->super.ecb.rounds); + + /* Main loop. Operate in full blocks (6 * 16 bytes). */ + while (PTLS_LIKELY(inlen >= 6 * 16)) { +#define DECRYPT(i) _mm_storeu_si128((void *)(output + i * 16), _mm_xor_si128(bits##i, _mm_loadu_si128((void *)(input + i * 16)))) + DECRYPT(0); + DECRYPT(1); + DECRYPT(2); + DECRYPT(3); + DECRYPT(4); + DECRYPT(5); +#undef DECRYPT +#define GFMUL_NEXT(i) gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(input + i * 16)), ctx->ghash + 5 - i) + AESECB6_INIT(); + AESECB6_UPDATE(1); + AESECB6_UPDATE(2); + AESECB6_UPDATE(3); + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)input), ctx->ghash + 5); + AESECB6_UPDATE(4); + GFMUL_NEXT(1); + AESECB6_UPDATE(5); + GFMUL_NEXT(2); + AESECB6_UPDATE(6); + GFMUL_NEXT(3); + AESECB6_UPDATE(7); + GFMUL_NEXT(4); + AESECB6_UPDATE(8); + GFMUL_NEXT(5); + AESECB6_UPDATE(9); + gfmul_reduce128(&gstate); + if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { + size_t i = 10; + do { + AESECB6_UPDATE(i); + } while (++i < ctx->super.ecb.rounds); + } + AESECB6_FINAL(ctx->super.ecb.rounds); + output += 6 * 16; + input += 6 * 16; + inlen -= 6 * 16; +#undef GFMUL_NEXT + } + + /* Decrypt the remainder as well as finishing GHASH calculation. */ + if (inlen != 0) { + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (inlen + 15) / 16 + 1; +#define ONEBLOCK(i) \ + do { \ + if (inlen != 0) { \ + __m128i b = inlen >= 16 ? _mm_loadu_si128((void *)input) : loadn128(input, inlen); \ + if (i == 0) { \ + gfmul_firststep128(&gstate, b, --ghash_precompute); \ + } else { \ + gfmul_nextstep128(&gstate, b, --ghash_precompute); \ + } \ + b = _mm_xor_si128(b, bits##i); \ + if (inlen >= 16) { \ + _mm_storeu_si128((void *)output, b); \ + output += 16; \ + input += 16; \ + inlen -= 16; \ + } else { \ + storen128(output, inlen, b); \ + output += inlen; \ + input += inlen; \ + inlen = 0; \ + } \ + } \ + } while (0) + ONEBLOCK(0); + ONEBLOCK(1); + ONEBLOCK(2); + ONEBLOCK(3); + ONEBLOCK(4); + ONEBLOCK(5); +#undef ONEBLOCK + gfmul_nextstep128(&gstate, ac, --ghash_precompute); + assert(ghash_precompute == ctx->ghash); + } else { + gfmul_firststep128(&gstate, ac, ctx->ghash); + } + gfmul_reduce128(&gstate); + + /* Calculate EK0 if not yet available in bits5. */ + if ((state & STATE_EK0_READY) == 0) { + bits5 = _mm_xor_si128(ek0, ctx->super.ecb.keys.m128[0]); + for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) + bits5 = _mm_aesenc_si128(bits5, ctx->super.ecb.keys.m128[i]); + bits5 = _mm_aesenclast_si128(bits5, ctx->super.ecb.keys.m128[ctx->super.ecb.rounds]); + } + + /* Calculate GCM tag and compare. */ + __m128i calctag = gfmul_get_tag128(&gstate, bits5); + __m128i recvtag = _mm_loadu_si128((void *)input); + if (_mm_movemask_epi8(_mm_cmpeq_epi8(calctag, recvtag)) != 0xffff) + return SIZE_MAX; + + return textlen; + +#undef AESECB6_INIT +#undef AESECB6_UPDATE +#undef AESECB6_FINAL +#undef STATE_EK0_READY +} + +NO_SANITIZE_ADDRESS +static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, + uint64_t seq, const void *_aad, size_t aadlen) +{ +/* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ +#define AESECB6_INIT() \ + do { \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits0 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits1 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits2 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits3 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits4 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits5 = _mm256_shuffle_epi8(ctr, byteswap256); \ + if (PTLS_UNLIKELY(srclen <= 32 * 6 - 16) && src_vecleft == 0) { \ + bits5 = _mm256_permute2f128_si256(bits5, ac_ek0, 0x30); \ + state |= STATE_EK0_READY; \ + } \ + __m256i k = ctx->super.ecb.keys.m256[0]; \ + bits0 = _mm256_xor_si256(bits0, k); \ + bits1 = _mm256_xor_si256(bits1, k); \ + bits2 = _mm256_xor_si256(bits2, k); \ + bits3 = _mm256_xor_si256(bits3, k); \ + bits4 = _mm256_xor_si256(bits4, k); \ + bits5 = _mm256_xor_si256(bits5, k); \ + } while (0) + +/* aes block update */ +#define AESECB6_UPDATE(i) \ + do { \ + __m256i k = ctx->super.ecb.keys.m256[i]; \ + bits0 = _mm256_aesenc_epi128(bits0, k); \ + bits1 = _mm256_aesenc_epi128(bits1, k); \ + bits2 = _mm256_aesenc_epi128(bits2, k); \ + bits3 = _mm256_aesenc_epi128(bits3, k); \ + bits4 = _mm256_aesenc_epi128(bits4, k); \ + bits5 = _mm256_aesenc_epi128(bits5, k); \ + } while (0) + +/* aesenclast */ +#define AESECB6_FINAL(i) \ + do { \ + __m256i k = ctx->super.ecb.keys.m256[i]; \ + bits0 = _mm256_aesenclast_epi128(bits0, k); \ + bits1 = _mm256_aesenclast_epi128(bits1, k); \ + bits2 = _mm256_aesenclast_epi128(bits2, k); \ + bits3 = _mm256_aesenclast_epi128(bits3, k); \ + bits4 = _mm256_aesenclast_epi128(bits4, k); \ + bits5 = _mm256_aesenclast_epi128(bits5, k); \ + } while (0) + + struct aesgcm_context *agctx = (void *)_ctx; + uint8_t *output = _output; + const uint8_t *aad = _aad; + +#define STATE_EK0_READY 0x1 + int32_t state = 0; + + /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ + uint8_t encbuf[32 * 9] __attribute__((aligned(32))), *encp; + + /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is + * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ + PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 32 + 16); + + /* load unaligned data within same cache line preceding `output`, adjusting pointers accordingly */ + encp = load_preceding_unaligned(encbuf, &output); + + /* setup ctr, retaining Ek(0), len(A) | len(C) to be fed into GCM */ + __m256i ctr = _mm256_broadcastsi128_si256(calc_counter(agctx, seq)); + ctr = _mm256_insert_epi32(ctr, 1, 4); + __m256i ac_ek0 = _mm256_permute2f128_si256( + /* first half: ac */ + _mm256_castsi128_si256( + _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128)), + /* second half: ek0 */ + _mm256_shuffle_epi8(ctr, byteswap256), 0x30); + + struct ptls_fusion_aesgcm_context256 *ctx = (void *)agctx->aesgcm; + __m256i bits0, bits1, bits2, bits3, bits4, bits5 = _mm256_setzero_si256(); + struct ptls_fusion_gfmul_state256 gstate = {0}; + + /* find the first non-empty vec */ + const uint8_t *src = NULL; + size_t srclen = 0, src_vecleft = incnt; + while (srclen == 0 && src_vecleft != 0) { + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + + /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ + AESECB6_INIT(); + AESECB6_UPDATE(1); + AESECB6_UPDATE(2); + if (PTLS_LIKELY(aadlen != 0)) { + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute; + while (PTLS_UNLIKELY(aadlen >= 6 * 32)) { + ghash_precompute = ctx->ghash + 6; + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); + aad += 32; + aadlen -= 32; + for (int i = 1; i < 6; ++i) { + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)aad), --ghash_precompute); + aad += 32; + aadlen -= 32; + } + gfmul_reduce256(&gstate); + } + if (PTLS_LIKELY(aadlen != 0)) { + ghash_precompute = ctx->ghash + (aadlen + 31) / 32; + if (PTLS_UNLIKELY(aadlen >= 32)) { + if (aadlen % 32 == 0 || aadlen % 32 > 16) { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); + aad += 32; + aadlen -= 32; + } else { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 1, --ghash_precompute); + aad += 16; + aadlen -= 16; + } + while (aadlen >= 32) { + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)aad), --ghash_precompute); + aad += 32; + aadlen -= 32; + } + if (PTLS_LIKELY(aadlen != 0)) { + assert(aadlen > 16); + gfmul_nextstep256(&gstate, loadn256(aad, aadlen), --ghash_precompute); + } + } else { + gfmul_firststep256(&gstate, loadn256(aad, aadlen), aadlen <= 16, --ghash_precompute); + } + assert(ctx->ghash == ghash_precompute); + gfmul_reduce256(&gstate); + } + } + for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) + AESECB6_UPDATE(i); + AESECB6_FINAL(ctx->super.ecb.rounds); + + /* Main loop. This loop: + * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, + * 2. then if there is no more data to be encrypted, exit the loop, otherwise, + * 3. calculate ghash of the blocks being written to encbuf, + * 4. calculate next 6 * 16 bytes of keystream, + * 5. writes encbuf in 64-byte blocks + * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be + * calculated. */ + size_t remaining_ghash_from = encp - encbuf; + if (srclen != 0) { + while (1) { + /* apply the bit stream to input, writing to encbuf */ + if (PTLS_LIKELY(srclen >= 6 * 32)) { +#define APPLY(i) _mm256_storeu_si256((void *)(encp + i * 32), _mm256_xor_si256(_mm256_loadu_si256((void *)(src + i * 32)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encp += 6 * 32; + src += 6 * 32; + srclen -= 6 * 32; + if (PTLS_UNLIKELY(srclen == 0)) { + if (src_vecleft == 0) { + remaining_ghash_from = (encp - encbuf) - 6 * 32; + break; + } + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + } else { + /* slow path, load at most 6 * 32 bytes to encbuf then encrypt in-place */ + size_t bytes_copied = 0; + do { + if (srclen >= 32 && bytes_copied < 5 * 32) { + _mm256_storeu_si256((void *)(encp + bytes_copied), _mm256_loadu_si256((void *)src)); + bytes_copied += 32; + src += 32; + srclen -= 32; + } else { + encp[bytes_copied++] = *src++; + --srclen; + } + if (PTLS_UNLIKELY(srclen == 0)) { + do { + if (src_vecleft == 0) + break; + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } while (srclen == 0); + if (srclen == 0) + break; + } + } while (bytes_copied < 6 * 32); +#define APPLY(i) \ + _mm256_storeu_si256((void *)(encp + i * 32), _mm256_xor_si256(_mm256_loadu_si256((void *)(encp + i * 32)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encp += bytes_copied; + if (PTLS_UNLIKELY(srclen == 0)) { + /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it + * will be fed into ghash. */ + remaining_ghash_from = (encp - encbuf) - bytes_copied; + if ((bytes_copied & 15) != 0) + _mm_storeu_si128((void *)encp, _mm_setzero_si128()); + break; + } + } + + /* Next 96-byte block starts here. Run AES and ghash in parallel while writing output using non-temporal store + * instructions. */ + AESECB6_INIT(); + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash + 6; + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encp - 6 * 32)), 0, --ghash_precompute); + AESECB6_UPDATE(1); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 5 * 32)), --ghash_precompute); + AESECB6_UPDATE(2); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 4 * 32)), --ghash_precompute); + AESECB6_UPDATE(3); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 3 * 32)), --ghash_precompute); + AESECB6_UPDATE(4); + _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); + _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); + _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); + _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); + _mm256_stream_si256((void *)(output + 128), _mm256_load_si256((void *)(encbuf + 128))); + _mm256_stream_si256((void *)(output + 160), _mm256_load_si256((void *)(encbuf + 160))); + AESECB6_UPDATE(5); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 2 * 32)), --ghash_precompute); + AESECB6_UPDATE(6); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 1 * 32)), --ghash_precompute); + output += 192; + encp -= 192; + AESECB6_UPDATE(7); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 192))); + AESECB6_UPDATE(8); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 224))); + AESECB6_UPDATE(9); + if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { + for (size_t i = 10; PTLS_LIKELY(i < ctx->super.ecb.rounds); ++i) + AESECB6_UPDATE(i); + } + assert(ctx->ghash == ghash_precompute); + gfmul_reduce256(&gstate); + AESECB6_FINAL(ctx->super.ecb.rounds); + } + } + + /* Now, All the encrypted bits are built in encbuf. Calculate AEAD tag and append to encbuf. */ + + { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 + * blocks at once. */ + size_t ac_off = remaining_ghash_from + ((encp - encbuf) - remaining_ghash_from + 15) / 16 * 16; + _mm_storeu_si128((void *)(encbuf + ac_off), _mm256_castsi256_si128(ac_ek0)); + size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ + assert(blocks <= 13); + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash + blocks / 2; + if (blocks % 2 != 0) { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 1, ghash_precompute); + remaining_ghash_from += 16; + } else { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 0, --ghash_precompute); + remaining_ghash_from += 32; + } + while (ghash_precompute != ctx->ghash) { + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + remaining_ghash_from += 32; + } + gfmul_reduce256(&gstate); + } + + /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ + if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { + bits5 = ac_ek0; + bits5 = _mm256_xor_si256(bits5, ctx->super.ecb.keys.m256[0]); + for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) + bits5 = _mm256_aesenc_epi128(bits5, ctx->super.ecb.keys.m256[i]); + bits5 = _mm256_aesenclast_epi128(bits5, ctx->super.ecb.keys.m256[ctx->super.ecb.rounds]); + } + + /* append tag to encbuf */ + _mm_storeu_si128((void *)encp, + gfmul_get_tag256(&gstate, _mm256_castsi256_si128(_mm256_permute2f128_si256(bits5, bits5, 0x11)))); + encp += 16; + + /* write remaining bytes */ + write_remaining_bytes(output, encbuf, encp); +} + +static int non_temporal_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) +{ + struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; + int aesni256 = is_enc && ptls_fusion_can_aesni256; + + ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); + ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); + if (key == NULL) + return 0; + + ctx->super.dispose_crypto = aesgcm_dispose_crypto; + ctx->super.do_xor_iv = aesgcm_xor_iv; + ctx->super.do_encrypt_init = NULL; + ctx->super.do_encrypt_update = NULL; + ctx->super.do_encrypt_final = NULL; + if (is_enc) { + ctx->super.do_encrypt = ptls_aead__do_encrypt; + ctx->super.do_encrypt_v = aesni256 ? non_temporal_encrypt_v256 : non_temporal_encrypt_v128; + ctx->super.do_decrypt = NULL; + } else { + assert(!aesni256); + ctx->super.do_encrypt = NULL; + ctx->super.do_encrypt_v = NULL; + ctx->super.do_decrypt = non_temporal_decrypt128; + } + + ctx->aesgcm = + new_aesgcm(key, key_size, + 7 * (ptls_fusion_can_aesni256 ? 32 : 16), // 6 blocks at once, plus len(A) | len(C) that we might append + aesni256); + + return 0; +} + +static int non_temporal_aes128gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) +{ + return non_temporal_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); +} + +static int non_temporal_aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) +{ + return non_temporal_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); +} + +ptls_aead_algorithm_t ptls_non_temporal_aes128gcm = {"AES128-GCM", + PTLS_AESGCM_CONFIDENTIALITY_LIMIT, + PTLS_AESGCM_INTEGRITY_LIMIT, + &ptls_fusion_aes128ctr, + NULL, // &ptls_fusion_aes128ecb, + PTLS_AES128_KEY_SIZE, + PTLS_AESGCM_IV_SIZE, + PTLS_AESGCM_TAG_SIZE, + 1, + sizeof(struct aesgcm_context), + non_temporal_aes128gcm_setup}; +ptls_aead_algorithm_t ptls_non_temporal_aes256gcm = {"AES256-GCM", + PTLS_AESGCM_CONFIDENTIALITY_LIMIT, + PTLS_AESGCM_INTEGRITY_LIMIT, + &ptls_fusion_aes256ctr, + NULL, // &ptls_fusion_aes128ecb, + PTLS_AES256_KEY_SIZE, + PTLS_AESGCM_IV_SIZE, + PTLS_AESGCM_TAG_SIZE, + 1, + sizeof(struct aesgcm_context), + non_temporal_aes256gcm_setup}; + #ifdef _WINDOWS /** * ptls_fusion_is_supported_by_cpu: @@ -1034,11 +2177,16 @@ int ptls_fusion_is_supported_by_cpu(void) leaf1_ecx = cpu_info[2]; if (/* PCLMUL */ (leaf1_ecx & (1 << 5)) != 0 && /* AES */ (leaf1_ecx & (1 << 25)) != 0) { - uint32_t leaf7_ebx; + uint32_t leaf7_ebx, leaf7_ecx; __cpuid(cpu_info, 7); leaf7_ebx = cpu_info[1]; + leaf7_ecx = cpu_info[2]; is_supported = /* AVX2 */ (leaf7_ebx & (1 << 5)) != 0; + + /* enable 256-bit mode if possible */ + if (is_supported && (leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_aesni256) + ptls_fusion_can_aesni256 = 1; } } @@ -1047,7 +2195,7 @@ int ptls_fusion_is_supported_by_cpu(void) #else int ptls_fusion_is_supported_by_cpu(void) { - unsigned leaf1_ecx, leaf7_ebx; + unsigned leaf1_ecx, leaf7_ebx, leaf7_ecx; { /* GCC-specific code to obtain CPU features */ unsigned leaf_cnt; @@ -1055,7 +2203,7 @@ int ptls_fusion_is_supported_by_cpu(void) if (leaf_cnt < 7) return 0; __asm__("cpuid" : "=c"(leaf1_ecx) : "a"(1) : "ebx", "edx"); - __asm__("cpuid" : "=b"(leaf7_ebx) : "a"(7), "c"(0) : "edx"); + __asm__("cpuid" : "=b"(leaf7_ebx), "=c"(leaf7_ecx) : "a"(7), "c"(0) : "edx"); } /* AVX2 */ @@ -1068,6 +2216,10 @@ int ptls_fusion_is_supported_by_cpu(void) if ((leaf1_ecx & (1 << 1)) == 0) return 0; + /* enable 256-bit mode if possible */ + if ((leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_aesni256) + ptls_fusion_can_aesni256 = 1; + return 1; } #endif diff --git a/lib/openssl.c b/lib/openssl.c index f891e0420..200763890 100644 --- a/lib/openssl.c +++ b/lib/openssl.c @@ -1594,6 +1594,7 @@ ptls_aead_algorithm_t ptls_openssl_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aead_crypto_context_t), aead_aes128gcm_setup_crypto}; ptls_cipher_algorithm_t ptls_openssl_aes256ecb = { @@ -1610,6 +1611,7 @@ ptls_aead_algorithm_t ptls_openssl_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aead_crypto_context_t), aead_aes256gcm_setup_crypto}; ptls_hash_algorithm_t ptls_openssl_sha256 = {PTLS_SHA256_BLOCK_SIZE, PTLS_SHA256_DIGEST_SIZE, sha256_create, @@ -1636,6 +1638,7 @@ ptls_aead_algorithm_t ptls_openssl_chacha20poly1305 = {"CHACHA20-POLY1305", PTLS_CHACHA20_KEY_SIZE, PTLS_CHACHA20POLY1305_IV_SIZE, PTLS_CHACHA20POLY1305_TAG_SIZE, + 0, sizeof(struct aead_crypto_context_t), aead_chacha20poly1305_setup_crypto}; ptls_cipher_suite_t ptls_openssl_chacha20poly1305sha256 = {.id = PTLS_CIPHER_SUITE_CHACHA20_POLY1305_SHA256, diff --git a/lib/picotls.c b/lib/picotls.c index 8bb5b136c..3ebd33a03 100644 --- a/lib/picotls.c +++ b/lib/picotls.c @@ -509,8 +509,13 @@ static uint64_t ntoh64(const uint8_t *src) void ptls_buffer__release_memory(ptls_buffer_t *buf) { ptls_clear_memory(buf->base, buf->off); - if (buf->is_allocated) + if (buf->is_allocated) { +#ifdef _WINDOWS + _aligned_free(buf->base); +#else free(buf->base); +#endif + } } int ptls_buffer_reserve(ptls_buffer_t *buf, size_t delta) @@ -519,15 +524,20 @@ int ptls_buffer_reserve(ptls_buffer_t *buf, size_t delta) return PTLS_ERROR_NO_MEMORY; if (PTLS_MEMORY_DEBUG || buf->capacity < buf->off + delta) { - uint8_t *newp; + void *newp; size_t new_capacity = buf->capacity; if (new_capacity < 1024) new_capacity = 1024; while (new_capacity < buf->off + delta) { new_capacity *= 2; } - if ((newp = malloc(new_capacity)) == NULL) +#ifdef _WINDOWS + if ((newp = _aligned_malloc(new_capacity, PTLS_SIZEOF_CACHE_LINE)) == NULL) + return PTLS_ERROR_NO_MEMORY; +#else + if (posix_memalign(&newp, PTLS_SIZEOF_CACHE_LINE, new_capacity) != 0) return PTLS_ERROR_NO_MEMORY; +#endif memcpy(newp, buf->base, buf->off); ptls_buffer__release_memory(buf); buf->base = newp; diff --git a/lib/ptlsbcrypt.c b/lib/ptlsbcrypt.c index 41e6a737d..da8dc16de 100644 --- a/lib/ptlsbcrypt.c +++ b/lib/ptlsbcrypt.c @@ -786,6 +786,7 @@ ptls_aead_algorithm_t ptls_bcrypt_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct ptls_bcrypt_aead_context_t), ptls_bcrypt_aead_setup_crypto_aesgcm}; @@ -797,6 +798,7 @@ ptls_aead_algorithm_t ptls_bcrypt_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct ptls_bcrypt_aead_context_t), ptls_bcrypt_aead_setup_crypto_aesgcm}; diff --git a/picotls.xcodeproj/project.pbxproj b/picotls.xcodeproj/project.pbxproj index 27453aa40..2887b4f7a 100644 --- a/picotls.xcodeproj/project.pbxproj +++ b/picotls.xcodeproj/project.pbxproj @@ -1286,7 +1286,11 @@ isa = XCBuildConfiguration; buildSettings = { GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; - OTHER_CFLAGS = "-march=native"; + OTHER_CFLAGS = ( + "-march=native", + "-mvaes", + "-mvpclmulqdq", + ); OTHER_LDFLAGS = ""; PRODUCT_NAME = "$(TARGET_NAME)"; }; @@ -1296,7 +1300,11 @@ isa = XCBuildConfiguration; buildSettings = { GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; - OTHER_CFLAGS = "-march=native"; + OTHER_CFLAGS = ( + "-march=native", + "-mvaes", + "-mvpclmulqdq", + ); OTHER_LDFLAGS = ""; PRODUCT_NAME = "$(TARGET_NAME)"; }; diff --git a/t/fusion.c b/t/fusion.c index 4c79153a8..d5c51daf4 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -49,14 +49,14 @@ static const char *tostr(const void *_p, size_t len) return buf; } -static void test_loadn(void) +static void test_loadn128(void) { uint8_t buf[8192] = {0}; for (size_t off = 0; off < 8192 - 15; ++off) { uint8_t *src = buf + off; memcpy(src, "hello world12345", 16); - __m128i v = loadn(src, 11); + __m128i v = loadn128(src, 11); if (memcmp(&v, "hello world\0\0\0\0\0", 16) != 0) { ok(!"fail"); return; @@ -73,17 +73,164 @@ static void test_ecb(void) ptls_fusion_aesecb_context_t ecb; uint8_t encrypted[16]; - ptls_fusion_aesecb_init(&ecb, 1, zero, 16); + ptls_fusion_aesecb_init(&ecb, 1, zero, 16, 0); ptls_fusion_aesecb_encrypt(&ecb, encrypted, "hello world!!!!!"); ptls_fusion_aesecb_dispose(&ecb); ok(strcmp(tostr(encrypted, 16), "172afecb50b5f1237814b2f7cb51d0f7") == 0); - ptls_fusion_aesecb_init(&ecb, 1, zero, 32); + ptls_fusion_aesecb_init(&ecb, 1, zero, 32, 0); ptls_fusion_aesecb_encrypt(&ecb, encrypted, "hello world!!!!!"); ptls_fusion_aesecb_dispose(&ecb); ok(strcmp(tostr(encrypted, 16), "2a033f0627b3554aa4fe5786550736ff") == 0); } +static void test_gfmul(void) +{ + ptls_fusion_aesgcm_context_t *ctx; + size_t ghash_cnt = 4; + + ctx = malloc(calc_aesgcm_context_size(&ghash_cnt, ptls_fusion_can_aesni256)); + *ctx = (ptls_fusion_aesgcm_context_t){ + .ecb = {.aesni256 = ptls_fusion_can_aesni256}, + .capacity = ghash_cnt * 16, + }; + + __m128i H0 = _mm_loadu_si128((void *)"hello world bye"); + if (ctx->ecb.aesni256) { + ((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash[0].H[1] = H0; + } else { + ((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash[0].H = H0; + } + + while (ctx->ghash_cnt < ghash_cnt) + setup_one_ghash_entry(ctx); + +#define GHASH128 (((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash) +#define GHASH256 (((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash) + + { /* one block */ + static const char input[32] = "deaddeadbeefbeef"; /* latter 16-byte is NUL */ + __m128i hash; + if (ctx->ecb.aesni256) { + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, GHASH256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\x12\xd9\xd9\x14\x8b\x3f\x20\xbd\x20\x2a\xa5\x9e\x17\xa8\xb0\x7b", 16) == 0); + } + + { /* two blocks */ + static const char input[32] = "Lorem ipsum dolor sit amet, con"; + __m128i hash; + if (ctx->ecb.aesni256) { + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\xda\xdf\xe8\x9b\xc7\x8c\xbd\x5c\xa7\xc1\x83\x9a\xa2\x9f\x80\x55", 16) == 0); + } + + { /* three blocks */ + static const char input[64] = "The quick brown fox jumps over the lazy dog."; + __m128i hash; + if (ctx->ecb.aesni256) { + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, GHASH256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 16)), GHASH256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), GHASH128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\xad\xdf\x91\x52\x38\x40\xf7\xc3\x85\xaf\x41\xb1\x7d\xed\x4b\x56", 16) == 0); + } + + { /* five blocks */ + static const char input[80] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor "; + __m128i hash; + if (ctx->ecb.aesni256) { + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), GHASH256); + gfmul_reduce256(&state); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 1, GHASH256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 3); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), GHASH128); + gfmul_reduce128(&state); + gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), GHASH128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\xb8\xab\x1b\xa8\xf2\x92\xf3\x89\x44\x9d\x39\xf6\xb6\x37\xca\x5d", 16) == 0); + } + + { /* six blocks */ + static const char input[96] = + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut la"; + __m128i hash; + if (ctx->ecb.aesni256) { + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), GHASH256); + gfmul_reduce256(&state); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 0, GHASH256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 3); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), GHASH128); + gfmul_reduce128(&state); + gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 80)), GHASH128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\x52\xce\x25\x22\x86\x2c\x91\xa4\xe7\x4e\xf9\x9a\x32\x77\xbd\x3e", 16) == 0); + } + + free(ctx); + +#undef GHASH128 +#undef GHASH256 +} + static void gcm_basic(void) { { @@ -230,7 +377,10 @@ static void gcm_iv96(void) ptls_aead_free(aead); } -static void test_generated(int aes256, int iv96) +static ptls_aead_algorithm_t *test_generated_encryptor, *test_generated_decryptor; +static int test_generated_iv96, test_generated_multivec; + +static void test_generated(void) { ptls_cipher_context_t *rand = ptls_cipher_new(&ptls_minicrypto_aes128ctr, 1, zero); ptls_cipher_init(rand, zero); @@ -262,32 +412,36 @@ static void test_generated(int aes256, int iv96) memset(decrypted, 0xcc, sizeof(decrypted)); { /* check using fusion */ - ptls_aead_context_t *fusion = - ptls_aead_new_direct(aes256 ? &ptls_fusion_aes256gcm : &ptls_fusion_aes128gcm, 1, key, iv); - if (iv96) { - ptls_aead_xor_iv(fusion, seq32, sizeof(seq32)); + ptls_aead_context_t *ctx = ptls_aead_new_direct(test_generated_encryptor, 1, key, iv); + if (test_generated_iv96) + ptls_aead_xor_iv(ctx, seq32, sizeof(seq32)); + if (test_generated_multivec) { + uint8_t pos1, pos2; + do { + ptls_cipher_encrypt(rand, &pos2, zero, sizeof(pos2)); + } while (pos2 > textlen); + do { + ptls_cipher_encrypt(rand, &pos1, zero, sizeof(pos1)); + } while (pos1 > pos2); + ptls_iovec_t textvec[3] = {{text, pos1}, {text + pos1, pos2 - pos1}, {text + pos2, textlen - pos2}}; + ptls_aead_encrypt_v(ctx, encrypted, textvec, 3, seq, aad, aadlen); + } else { + ptls_aead_encrypt(ctx, encrypted, text, textlen, seq, aad, aadlen); } - ptls_aead_encrypt(fusion, encrypted, text, textlen, seq, aad, aadlen); - if (ptls_aead_decrypt(fusion, decrypted, encrypted, textlen + 16, seq, aad, aadlen) != textlen) - goto Fail; - if (memcmp(decrypted, text, textlen) != 0) - goto Fail; - ptls_aead_free(fusion); + ptls_aead_free(ctx); } memset(decrypted, 0xcc, sizeof(decrypted)); - { /* check that the encrypted text can be decrypted by OpenSSL */ - ptls_aead_context_t *mc = - ptls_aead_new_direct(aes256 ? &ptls_minicrypto_aes256gcm : &ptls_minicrypto_aes128gcm, 0, key, iv); - if (iv96) { - ptls_aead_xor_iv(mc, seq32, sizeof(seq32)); - } - if (ptls_aead_decrypt(mc, decrypted, encrypted, textlen + 16, seq, aad, aadlen) != textlen) + { /* check that the encrypted text can be decrypted by the decryptor */ + ptls_aead_context_t *ctx = ptls_aead_new_direct(test_generated_decryptor, 0, key, iv); + if (test_generated_iv96) + ptls_aead_xor_iv(ctx, seq32, sizeof(seq32)); + if (ptls_aead_decrypt(ctx, decrypted, encrypted, textlen + 16, seq, aad, aadlen) != textlen) goto Fail; if (memcmp(decrypted, text, textlen) != 0) goto Fail; - ptls_aead_free(mc); + ptls_aead_free(ctx); } } @@ -300,42 +454,93 @@ static void test_generated(int aes256, int iv96) ok(0); } -static void test_generated_aes128(void) +static void test_generated_all(ptls_aead_algorithm_t *e1, ptls_aead_algorithm_t *e2, int can_multivec) +{ + test_generated_encryptor = e1; + test_generated_decryptor = e2; + + test_generated_iv96 = 0; + subtest("encrypt", test_generated); + test_generated_iv96 = 1; + subtest("encrypt-iv96", test_generated); + + if (can_multivec) { + test_generated_multivec = 1; + test_generated_iv96 = 0; + subtest("encrypt", test_generated); + test_generated_iv96 = 1; + subtest("encrypt-iv96", test_generated); + test_generated_multivec = 0; + } + + test_generated_encryptor = e2; + test_generated_decryptor = e1; + + test_generated_iv96 = 0; + subtest("decrypt", test_generated); + test_generated_iv96 = 1; + subtest("decrypt-iv96", test_generated); +} + +static void test_fusion_aes128gcm(void) +{ + test_generated_all(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm, 0); +} + +static void test_fusion_aes256gcm(void) { - test_generated(0, 0); + test_generated_all(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm, 0); } -static void test_generated_aes256(void) +static void test_fusion_aesgcm(void) { - test_generated(1, 0); + subtest("128", test_fusion_aes128gcm); + subtest("256", test_fusion_aes256gcm); } -static void test_generated_aes128_iv96(void) +static void test_non_temporal_aes128gcm(void) { - test_generated(0, 1); + test_generated_all(&ptls_non_temporal_aes128gcm, &ptls_minicrypto_aes128gcm, 1); } -static void test_generated_aes256_iv96(void) +static void test_non_temporal_aes256gcm(void) { - test_generated(1, 1); + test_generated_all(&ptls_non_temporal_aes256gcm, &ptls_minicrypto_aes256gcm, 1); } + +static void test_non_temporal(void) +{ + subtest("128", test_non_temporal_aes128gcm); + subtest("256", test_non_temporal_aes256gcm); +} + int main(int argc, char **argv) { if (!ptls_fusion_is_supported_by_cpu()) { note("CPU does have the necessary features (avx2, aes, pclmul)\n"); return done_testing(); } + int can256bit = ptls_fusion_can_aesni256; + ptls_fusion_can_aesni256 = 0; - subtest("loadn", test_loadn); + subtest("loadn128", test_loadn128); subtest("ecb", test_ecb); + subtest("gfmul128", test_gfmul); subtest("gcm-basic", gcm_basic); subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); - subtest("generated-128", test_generated_aes128); - subtest("generated-256", test_generated_aes256); - subtest("generated-128-iv96", test_generated_aes128_iv96); - subtest("generated-256-iv96", test_generated_aes256_iv96); + subtest("fusion-aesgcm", test_fusion_aesgcm); + subtest("non-temporal-avx128", test_non_temporal); + + if (can256bit) { + ptls_fusion_can_aesni256 = 1; + subtest("gfmul256", test_gfmul); + subtest("non-temporal-avx256", test_non_temporal); + ptls_fusion_can_aesni256 = 0; + } else { + note("gfmul256: skipping, CPU does not support 256-bit aes / clmul"); + } return done_testing(); }