From ed661b11ab85083be33cc0ba6c6aac61a40c7309 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sat, 30 Apr 2022 21:43:31 +0900 Subject: [PATCH 01/48] it works! implement aes-gcm that uses non-temporal stores --- include/picotls/fusion.h | 1 + lib/fusion.c | 394 +++++++++++++++++++++++++++++++++++++-- t/fusion.c | 60 +++--- 3 files changed, 415 insertions(+), 40 deletions(-) diff --git a/include/picotls/fusion.h b/include/picotls/fusion.h index 332dd93ab..59f96844b 100644 --- a/include/picotls/fusion.h +++ b/include/picotls/fusion.h @@ -86,6 +86,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, extern ptls_cipher_algorithm_t ptls_fusion_aes128ctr, ptls_fusion_aes256ctr; extern ptls_aead_algorithm_t ptls_fusion_aes128gcm, ptls_fusion_aes256gcm; +extern ptls_aead_algorithm_t ptls_fastls_aes128gcm, ptls_fastls_aes256gcm; /** * Returns a boolean indicating if fusion can be used. diff --git a/lib/fusion.c b/lib/fusion.c index 642de8fbb..b2226f5c9 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -148,10 +148,9 @@ struct ptls_fusion_gfmul_state { __m128i hi, lo, mid; }; -static inline void gfmul_onestep(struct ptls_fusion_gfmul_state *gstate, __m128i X, +static inline void gfmul_do_step(struct ptls_fusion_gfmul_state *gstate, __m128i X, struct ptls_fusion_aesgcm_ghash_precompute *precompute) { - X = _mm_shuffle_epi8(X, bswap8); __m128i t = _mm_clmulepi64_si128(precompute->H, X, 0x00); gstate->lo = _mm_xor_si128(gstate->lo, t); t = _mm_clmulepi64_si128(precompute->H, X, 0x11); @@ -162,7 +161,25 @@ static inline void gfmul_onestep(struct ptls_fusion_gfmul_state *gstate, __m128i gstate->mid = _mm_xor_si128(gstate->mid, t); } -static inline __m128i gfmul_final(struct ptls_fusion_gfmul_state *gstate, __m128i ek0) +static inline void gfmul_firststep(struct ptls_fusion_gfmul_state *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute *precompute) +{ + X = _mm_shuffle_epi8(X, bswap8); + X = _mm_xor_si128(gstate->lo, X); + gstate->lo = _mm_setzero_si128(); + gstate->hi = _mm_setzero_si128(); + gstate->mid = _mm_setzero_si128(); + gfmul_do_step(gstate, X, precompute); +} + +static inline void gfmul_nextstep(struct ptls_fusion_gfmul_state *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute *precompute) +{ + X = _mm_shuffle_epi8(X, bswap8); + gfmul_do_step(gstate, X, precompute); +} + +static inline void gfmul_reduce(struct ptls_fusion_gfmul_state *gstate) { /* finish multiplication */ gstate->mid = _mm_xor_si128(gstate->mid, gstate->hi); @@ -177,10 +194,13 @@ static inline __m128i gfmul_final(struct ptls_fusion_gfmul_state *gstate, __m128 r = _mm_clmulepi64_si128(gstate->lo, poly, 0x10); gstate->lo = _mm_shuffle_epi32(gstate->lo, 78); gstate->lo = _mm_xor_si128(gstate->lo, r); - __m128i tag = _mm_xor_si128(gstate->hi, gstate->lo); - tag = _mm_shuffle_epi8(tag, bswap8); - tag = _mm_xor_si128(tag, ek0); + gstate->lo = _mm_xor_si128(gstate->hi, gstate->lo); +} +static inline __m128i gfmul_get_tag(struct ptls_fusion_gfmul_state *gstate, __m128i ek0) +{ + __m128i tag = _mm_shuffle_epi8(gstate->lo, bswap8); + tag = _mm_xor_si128(tag, ek0); return tag; } @@ -354,7 +374,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, size_t i; for (i = 2; i < gdata_cnt + 2; ++i) { AESECB6_UPDATE(i); - gfmul_onestep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } for (; i < ctx->ecb.rounds; ++i) AESECB6_UPDATE(i); @@ -465,9 +485,10 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, */ assert(STATE_EK0_READY()); for (size_t i = 0; i < gdata_cnt; ++i) - gfmul_onestep(&gstate, gdatabuf[i], --ghash_precompute); + gfmul_nextstep(&gstate, gdatabuf[i], --ghash_precompute); - _mm_storeu_si128(dst, gfmul_final(&gstate, ek0)); + gfmul_reduce(&gstate); + _mm_storeu_si128(dst, gfmul_get_tag(&gstate, ek0)); /* Finish the calculation of supplemental vector. Done at the very last, because the sample might cover the GCM tag. */ if ((state & STATE_SUPP_USED) != 0) { @@ -600,7 +621,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, size_t aesi; for (aesi = 1; aesi <= gdata_cnt; ++aesi) { AESECB6_UPDATE(aesi); - gfmul_onestep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } for (; aesi < ctx->ecb.rounds; ++aesi) AESECB6_UPDATE(aesi); @@ -667,10 +688,11 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* the only case where AES operation is complete and GHASH is not is when the application of AC is remaining */ if ((state & STATE_GHASH_HAS_MORE) != 0) { assert(ghash_precompute - 1 == ctx->ghash); - gfmul_onestep(&gstate, ac, --ghash_precompute); + gfmul_nextstep(&gstate, ac, --ghash_precompute); } - __m128i calctag = gfmul_final(&gstate, ek0); + gfmul_reduce(&gstate); + __m128i calctag = gfmul_get_tag(&gstate, ek0); return _mm_movemask_epi8(_mm_cmpeq_epi8(calctag, _mm_loadu_si128(tag))) == 0xffff; @@ -991,6 +1013,354 @@ ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", sizeof(struct aesgcm_context), aes256gcm_setup}; +static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, ptls_iovec_t *input, size_t incnt, uint64_t seq, + ptls_iovec_t aad) +{ +/* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ +#define AESECB6_INIT() \ + do { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits0 = _mm_shuffle_epi8(ctr, bswap8); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits1 = _mm_shuffle_epi8(ctr, bswap8); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits2 = _mm_shuffle_epi8(ctr, bswap8); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits3 = _mm_shuffle_epi8(ctr, bswap8); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits4 = _mm_shuffle_epi8(ctr, bswap8); \ + if (PTLS_LIKELY(totlen > 16 * 5)) { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits5 = _mm_shuffle_epi8(ctr, bswap8); \ + } else { \ + if ((state & STATE_EK0_READY) == 0) { \ + bits5 = ek0; \ + state |= STATE_EK0_READY; \ + } \ + } \ + __m128i k = ctx->ecb.keys[0]; \ + bits0 = _mm_xor_si128(bits0, k); \ + bits1 = _mm_xor_si128(bits1, k); \ + bits2 = _mm_xor_si128(bits2, k); \ + bits3 = _mm_xor_si128(bits3, k); \ + bits4 = _mm_xor_si128(bits4, k); \ + bits5 = _mm_xor_si128(bits5, k); \ + } while (0) + +/* aes block update */ +#define AESECB6_UPDATE(i) \ + do { \ + __m128i k = ctx->ecb.keys[i]; \ + bits0 = _mm_aesenc_si128(bits0, k); \ + bits1 = _mm_aesenc_si128(bits1, k); \ + bits2 = _mm_aesenc_si128(bits2, k); \ + bits3 = _mm_aesenc_si128(bits3, k); \ + bits4 = _mm_aesenc_si128(bits4, k); \ + bits5 = _mm_aesenc_si128(bits5, k); \ + } while (0) + +/* aesenclast */ +#define AESECB6_FINAL(i) \ + do { \ + __m128i k = ctx->ecb.keys[i]; \ + bits0 = _mm_aesenclast_si128(bits0, k); \ + bits1 = _mm_aesenclast_si128(bits1, k); \ + bits2 = _mm_aesenclast_si128(bits2, k); \ + bits3 = _mm_aesenclast_si128(bits3, k); \ + bits4 = _mm_aesenclast_si128(bits4, k); \ + bits5 = _mm_aesenclast_si128(bits5, k); \ + } while (0) + + struct aesgcm_context *agctx = (void *)_ctx; + + size_t totlen = 0; + for (size_t i = 0; i < incnt; ++i) + totlen += input[i].len; + +#define STATE_EK0_READY 0x1 + int32_t state = 0; + + /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. Additional 16 bytes at the tail + * allows us to write the AEAD tag in addition to the 6 blocks of ciphertext before sending the bytes to main memory using NT + * store instructions. */ + char encbuf[64 + 6 * 16 + 16] __attribute__((aligned(32))); + size_t encbuf_size; + /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ + if ((encbuf_size = ((uintptr_t)output & 63)) != 0) { + _mm256_store_si256((void *)encbuf, _mm256_load_si256(output - encbuf_size)); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256(output - encbuf_size + 32)); + output = output - encbuf_size; + } + + /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ + __m128i ctr = calc_counter(agctx, seq); + ctr = _mm_insert_epi32(ctr, 1, 0); + __m128i ek0 = _mm_shuffle_epi8(ctr, bswap8); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aad.len * 8, 0, (int)totlen * 8), bswap8); + + ptls_fusion_aesgcm_context_t *ctx = agctx->aesgcm; + __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); + struct ptls_fusion_gfmul_state gstate = {0}; + + /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ + AESECB6_INIT(); + AESECB6_UPDATE(1); + AESECB6_UPDATE(2); + if (PTLS_LIKELY(aad.len != 0)) { + struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute; + while (PTLS_UNLIKELY(aad.len >= 6 * 16)) { + ghash_precompute = ctx->ghash + 6; + gfmul_firststep(&gstate, _mm_loadu_si128((void *)aad.base), --ghash_precompute); + aad.base += 16; + aad.len -= 16; + for (int i = 1; i < 6; ++i) { + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)aad.base), --ghash_precompute); + aad.base += 16; + aad.len -= 16; + } + gfmul_reduce(&gstate); + } + if (PTLS_LIKELY(aad.len != 0)) { + ghash_precompute = ctx->ghash + (aad.len + 15) / 16; + if (PTLS_UNLIKELY(aad.len >= 16)) { + gfmul_firststep(&gstate, _mm_loadu_si128((void *)aad.base), --ghash_precompute); + aad.base += 16; + aad.len -= 16; + while (aad.len >= 16) { + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)aad.base), --ghash_precompute); + aad.base += 16; + aad.len -= 16; + } + if (PTLS_LIKELY(aad.len != 0)) + gfmul_nextstep(&gstate, loadn(aad.base, aad.len), --ghash_precompute); + } else { + gfmul_firststep(&gstate, loadn(aad.base, aad.len), --ghash_precompute); + } + assert(ctx->ghash == ghash_precompute); + gfmul_reduce(&gstate); + } + } + for (size_t i = 3; i < ctx->ecb.rounds; ++i) + AESECB6_UPDATE(i); + AESECB6_FINAL(ctx->ecb.rounds); + + /* Main loop. This loop: + * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, + * 2. then if there is no more data to be encrypted, exit the loop, otherwise, + * 3. calculate ghash of the blocks being written to encbuf, + * 4. calculate next 6 * 16 bytes of keystream, + * 5. writes encbuf in 64-byte blocks + * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be + * calculated. */ + size_t remaining_ghash_from = encbuf_size; + if (totlen != 0) { + const uint8_t *src = (void *)input[0].base; + size_t srclen = input[0].len; + ++input; + size_t src_vecleft = incnt - 1; + + while (1) { + /* apply the bit stream to input, writing to encbuf */ + if (PTLS_LIKELY(srclen >= 6 * 16)) { +#define APPLY(i) \ + _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encbuf_size += 6 * 16; + src += 6 * 16; + srclen -= 6 * 16; + if (PTLS_UNLIKELY(srclen == 0)) { + if (src_vecleft == 0) { + remaining_ghash_from = encbuf_size - 96; + break; + } + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + } else { + /* slow path, load at most 6 * 16 bytes to encbuf then encrypt in-place */ + size_t bytes_copied = 0; + do { + if (srclen >= 16 && bytes_copied < 5 * 80) { + _mm_storeu_si128((void *)(encbuf + encbuf_size + bytes_copied), _mm_loadu_si128((void *)src)); + bytes_copied += 16; + src += 16; + srclen -= 16; + } else { + encbuf[encbuf_size + bytes_copied++] = *src++; + --srclen; + } + if (PTLS_UNLIKELY(srclen == 0)) { + if (src_vecleft == 0) { + break; + } else { + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + } + } while (bytes_copied < 6 * 16); +#define APPLY(i) \ + _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), \ + _mm_xor_si128(_mm_loadu_si128((void *)(encbuf + encbuf_size + i * 16)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encbuf_size += bytes_copied; + if (PTLS_UNLIKELY(srclen == 0)) { + /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it + * will be fed into ghash. */ + remaining_ghash_from = encbuf_size - bytes_copied; + if ((encbuf_size & 15) != 0) + _mm_storeu_si128((void *)(encbuf + encbuf_size), _mm_setzero_si128()); + break; + } + } + + /* Next 96-byte block starts here. Run AES and ghash in while writing output using non-temporal stores in 64-byte + * blocks. */ + AESECB6_INIT(); + struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + 6; + gfmul_firststep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 6 * 16), --ghash_precompute); + for (size_t i = 1; i <= 5; ++i) { + AESECB6_UPDATE(i); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); + } + AESECB6_UPDATE(6); + _mm256_store_si256((void *)output, _mm256_load_si256((void *)encbuf)); + _mm256_store_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); + AESECB6_UPDATE(7); + AESECB6_UPDATE(8); + if (encbuf_size >= 128) { + _mm256_store_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); + _mm256_store_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); + output += 128; + encbuf_size -= 128; + _mm256_store_si256((void *)encbuf, _mm256_loadu_si256((void *)encbuf + 128)); + _mm256_store_si256((void *)encbuf + 32, _mm256_loadu_si256((void *)encbuf + 160)); + } else { + output += 64; + encbuf_size -= 64; + _mm256_store_si256((void *)encbuf, _mm256_loadu_si256((void *)encbuf + 64)); + _mm256_store_si256((void *)encbuf + 32, _mm256_loadu_si256((void *)encbuf + 96)); + } + for (size_t i = 9; i < ctx->ecb.rounds; ++i) + AESECB6_UPDATE(i); + assert(ctx->ghash == ghash_precompute); + gfmul_reduce(&gstate); + AESECB6_FINAL(ctx->ecb.rounds); + } + } + + /* Now, All the encrypted bits are built in encbuf. Calculate AEAD tag and append to encbuf. */ + + { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 + * blocks at once. */ + _mm_store_si128((void *)(encbuf + (encbuf_size + 15) / 16 * 16), ac); + size_t blocks = (encbuf_size - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ + assert(blocks <= 7); + struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + blocks; + gfmul_firststep(&gstate, _mm_load_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + remaining_ghash_from += 16; + while (ghash_precompute != ctx->ghash) { + gfmul_nextstep(&gstate, _mm_load_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + remaining_ghash_from += 16; + } + gfmul_reduce(&gstate); + } + + /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ + if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { + bits5 = _mm_xor_si128(ek0, ctx->ecb.keys[0]); + for (size_t i = 1; i < ctx->ecb.rounds; ++i) + bits5 = _mm_aesenc_si128(bits5, ctx->ecb.keys[i]); + bits5 = _mm_aesenclast_si128(bits5, ctx->ecb.keys[ctx->ecb.rounds]); + } + + /* append tag to encbuf */ + _mm_storeu_si128((void *)(encbuf + encbuf_size), gfmul_get_tag(&gstate, bits5)); + encbuf_size += 16; + + { /* Write encbuf, using NT store instructions in 64-byte chunks. Last partial block, if any, is written to cache, as that cache + * line would likely be read when the next TLS record is being built. */ + size_t off = 0; + for (; off + 64 <= encbuf_size; off += 64) { + _mm256_stream_si256(output + off, _mm256_load_si256((void *)(encbuf + off))); + _mm256_stream_si256(output + off + 32, _mm256_load_si256((void *)(encbuf + off + 32))); + } + _mm_sfence(); /* weakly ordered writes have to be synced before being passed to NIC */ + if (off != encbuf_size) { + for (; off + 16 <= encbuf_size; off += 16) + _mm_store_si128(output + off, _mm_load_si128((void *)(encbuf + off))); + if (off != encbuf_size) + storen(output + off, encbuf_size - off, loadn(encbuf + off, encbuf_size - off)); + } + } +} + +static int fastls_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) +{ + struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; + + ctx->static_iv = loadn(iv, PTLS_AESGCM_IV_SIZE); + ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, bswap8); + if (key == NULL) + return 0; + + ctx->super.dispose_crypto = aesgcm_dispose_crypto; + ctx->super.do_xor_iv = aesgcm_xor_iv; + ctx->super.do_encrypt = ptls_aead__do_encrypt; + ctx->super.do_encrypt_v = fastls_encrypt_v; + ctx->super.do_decrypt = NULL; /* FIXME */ + + ctx->aesgcm = ptls_fusion_aesgcm_new(key, key_size, 7 * 16 /* 6 blocks at once, plus len(A) | len(C) that we might append */); + + return 0; +} + +static int fastls_aes128gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) +{ + return fastls_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); +} + +static int fastls_aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) +{ + return fastls_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); +} + +ptls_aead_algorithm_t ptls_fastls_aes128gcm = {"AES128-GCM", + PTLS_AESGCM_CONFIDENTIALITY_LIMIT, + PTLS_AESGCM_INTEGRITY_LIMIT, + &ptls_fusion_aes128ctr, + NULL, // &ptls_fusion_aes128ecb, + PTLS_AES128_KEY_SIZE, + PTLS_AESGCM_IV_SIZE, + PTLS_AESGCM_TAG_SIZE, + sizeof(struct aesgcm_context), + fastls_aes128gcm_setup}; +ptls_aead_algorithm_t ptls_fastls_aes256gcm = {"AES256-GCM", + PTLS_AESGCM_CONFIDENTIALITY_LIMIT, + PTLS_AESGCM_INTEGRITY_LIMIT, + &ptls_fusion_aes256ctr, + NULL, // &ptls_fusion_aes128ecb, + PTLS_AES256_KEY_SIZE, + PTLS_AESGCM_IV_SIZE, + PTLS_AESGCM_TAG_SIZE, + sizeof(struct aesgcm_context), + fastls_aes256gcm_setup}; + #ifdef _WINDOWS /** * ptls_fusion_is_supported_by_cpu: diff --git a/t/fusion.c b/t/fusion.c index 4c79153a8..57d2b45a0 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -230,7 +230,7 @@ static void gcm_iv96(void) ptls_aead_free(aead); } -static void test_generated(int aes256, int iv96) +static void test_generated(ptls_aead_algorithm_t *encryptor, ptls_aead_algorithm_t *decryptor, int iv96) { ptls_cipher_context_t *rand = ptls_cipher_new(&ptls_minicrypto_aes128ctr, 1, zero); ptls_cipher_init(rand, zero); @@ -262,32 +262,24 @@ static void test_generated(int aes256, int iv96) memset(decrypted, 0xcc, sizeof(decrypted)); { /* check using fusion */ - ptls_aead_context_t *fusion = - ptls_aead_new_direct(aes256 ? &ptls_fusion_aes256gcm : &ptls_fusion_aes128gcm, 1, key, iv); - if (iv96) { - ptls_aead_xor_iv(fusion, seq32, sizeof(seq32)); - } - ptls_aead_encrypt(fusion, encrypted, text, textlen, seq, aad, aadlen); - if (ptls_aead_decrypt(fusion, decrypted, encrypted, textlen + 16, seq, aad, aadlen) != textlen) - goto Fail; - if (memcmp(decrypted, text, textlen) != 0) - goto Fail; - ptls_aead_free(fusion); + ptls_aead_context_t *ctx = ptls_aead_new_direct(encryptor, 1, key, iv); + if (iv96) + ptls_aead_xor_iv(ctx, seq32, sizeof(seq32)); + ptls_aead_encrypt(ctx, encrypted, text, textlen, seq, aad, aadlen); + ptls_aead_free(ctx); } memset(decrypted, 0xcc, sizeof(decrypted)); - { /* check that the encrypted text can be decrypted by OpenSSL */ - ptls_aead_context_t *mc = - ptls_aead_new_direct(aes256 ? &ptls_minicrypto_aes256gcm : &ptls_minicrypto_aes128gcm, 0, key, iv); - if (iv96) { - ptls_aead_xor_iv(mc, seq32, sizeof(seq32)); - } - if (ptls_aead_decrypt(mc, decrypted, encrypted, textlen + 16, seq, aad, aadlen) != textlen) + { /* check that the encrypted text can be decrypted by the decryptor */ + ptls_aead_context_t *ctx = ptls_aead_new_direct(decryptor, 0, key, iv); + if (iv96) + ptls_aead_xor_iv(ctx, seq32, sizeof(seq32)); + if (ptls_aead_decrypt(ctx, decrypted, encrypted, textlen + 16, seq, aad, aadlen) != textlen) goto Fail; if (memcmp(decrypted, text, textlen) != 0) goto Fail; - ptls_aead_free(mc); + ptls_aead_free(ctx); } } @@ -302,23 +294,37 @@ static void test_generated(int aes256, int iv96) static void test_generated_aes128(void) { - test_generated(0, 0); + test_generated(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm, 0); } static void test_generated_aes256(void) { - test_generated(1, 0); + test_generated(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm, 0); } static void test_generated_aes128_iv96(void) { - test_generated(0, 1); + test_generated(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm, 1); } static void test_generated_aes256_iv96(void) { - test_generated(1, 1); + test_generated(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm, 1); +} + +static void test_aesgcm(void) +{ + subtest("aes128gcm", test_generated_aes128); + subtest("aes256gcm", test_generated_aes256); + subtest("aes128gcm-iv96", test_generated_aes128_iv96); + subtest("aes256gcm-iv96", test_generated_aes256_iv96); } + +static void test_fastls(void) +{ + test_generated(&ptls_fastls_aes128gcm, &ptls_minicrypto_aes128gcm, 0); +} + int main(int argc, char **argv) { if (!ptls_fusion_is_supported_by_cpu()) { @@ -332,10 +338,8 @@ int main(int argc, char **argv) subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); - subtest("generated-128", test_generated_aes128); - subtest("generated-256", test_generated_aes256); - subtest("generated-128-iv96", test_generated_aes128_iv96); - subtest("generated-256-iv96", test_generated_aes256_iv96); + subtest("aesgcm", test_aesgcm); + subtest("fastls", test_fastls); return done_testing(); } From 42a4264040da398b3b3094e5480362c600b00e98 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sat, 30 Apr 2022 22:58:20 +0900 Subject: [PATCH 02/48] aligned --- lib/fusion.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index b2226f5c9..c9e7edb7c 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1163,7 +1163,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, /* apply the bit stream to input, writing to encbuf */ if (PTLS_LIKELY(srclen >= 6 * 16)) { #define APPLY(i) \ - _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) + _mm_store_si128((void *)(encbuf + encbuf_size + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); @@ -1189,7 +1189,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, size_t bytes_copied = 0; do { if (srclen >= 16 && bytes_copied < 5 * 80) { - _mm_storeu_si128((void *)(encbuf + encbuf_size + bytes_copied), _mm_loadu_si128((void *)src)); + _mm_store_si128((void *)(encbuf + encbuf_size + bytes_copied), _mm_loadu_si128((void *)src)); bytes_copied += 16; src += 16; srclen -= 16; @@ -1209,8 +1209,8 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, } } while (bytes_copied < 6 * 16); #define APPLY(i) \ - _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), \ - _mm_xor_si128(_mm_loadu_si128((void *)(encbuf + encbuf_size + i * 16)), bits##i)) + _mm_store_si128((void *)(encbuf + encbuf_size + i * 16), \ + _mm_xor_si128(_mm_load_si128((void *)(encbuf + encbuf_size + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); @@ -1233,10 +1233,10 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, * blocks. */ AESECB6_INIT(); struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + 6; - gfmul_firststep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 6 * 16), --ghash_precompute); + gfmul_firststep(&gstate, _mm_load_si128((void *)encbuf + encbuf_size - 6 * 16), --ghash_precompute); for (size_t i = 1; i <= 5; ++i) { AESECB6_UPDATE(i); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_load_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); } AESECB6_UPDATE(6); _mm256_store_si256((void *)output, _mm256_load_si256((void *)encbuf)); @@ -1248,13 +1248,13 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, _mm256_store_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); output += 128; encbuf_size -= 128; - _mm256_store_si256((void *)encbuf, _mm256_loadu_si256((void *)encbuf + 128)); - _mm256_store_si256((void *)encbuf + 32, _mm256_loadu_si256((void *)encbuf + 160)); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 128)); + _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 160)); } else { output += 64; encbuf_size -= 64; - _mm256_store_si256((void *)encbuf, _mm256_loadu_si256((void *)encbuf + 64)); - _mm256_store_si256((void *)encbuf + 32, _mm256_loadu_si256((void *)encbuf + 96)); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 64)); + _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 96)); } for (size_t i = 9; i < ctx->ecb.rounds; ++i) AESECB6_UPDATE(i); From b854db9a3823f6da70b1d1bbccb675cd51427782 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sat, 30 Apr 2022 23:44:34 +0900 Subject: [PATCH 03/48] on 9th-gen Core 9% slower than OpenSSL, on Zen 2 4% --- lib/fusion.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index c9e7edb7c..b73080883 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1238,11 +1238,10 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, AESECB6_UPDATE(i); gfmul_nextstep(&gstate, _mm_load_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); } - AESECB6_UPDATE(6); _mm256_store_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_store_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); + AESECB6_UPDATE(6); AESECB6_UPDATE(7); - AESECB6_UPDATE(8); if (encbuf_size >= 128) { _mm256_store_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); _mm256_store_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); @@ -1256,7 +1255,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 64)); _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 96)); } - for (size_t i = 9; i < ctx->ecb.rounds; ++i) + for (size_t i = 8; i < ctx->ecb.rounds; ++i) AESECB6_UPDATE(i); assert(ctx->ghash == ghash_precompute); gfmul_reduce(&gstate); From 7da09175c703134e88e854ba0521b05ec6a96107 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sat, 30 Apr 2022 23:54:59 +0900 Subject: [PATCH 04/48] oops, use non-temporal --- lib/fusion.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index b73080883..89781d656 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1238,13 +1238,13 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, AESECB6_UPDATE(i); gfmul_nextstep(&gstate, _mm_load_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); } - _mm256_store_si256((void *)output, _mm256_load_si256((void *)encbuf)); - _mm256_store_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); + _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); + _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); AESECB6_UPDATE(6); AESECB6_UPDATE(7); if (encbuf_size >= 128) { - _mm256_store_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); - _mm256_store_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); + _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); + _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); output += 128; encbuf_size -= 128; _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 128)); From 7fb163f43ddb9c0327da8a1eb812e10e2df35e24 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 1 May 2022 14:09:56 +0900 Subject: [PATCH 05/48] output buffers might not be aligned --- lib/fusion.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 89781d656..4b2ea9a03 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1163,7 +1163,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, /* apply the bit stream to input, writing to encbuf */ if (PTLS_LIKELY(srclen >= 6 * 16)) { #define APPLY(i) \ - _mm_store_si128((void *)(encbuf + encbuf_size + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) + _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); @@ -1189,7 +1189,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, size_t bytes_copied = 0; do { if (srclen >= 16 && bytes_copied < 5 * 80) { - _mm_store_si128((void *)(encbuf + encbuf_size + bytes_copied), _mm_loadu_si128((void *)src)); + _mm_storeu_si128((void *)(encbuf + encbuf_size + bytes_copied), _mm_loadu_si128((void *)src)); bytes_copied += 16; src += 16; srclen -= 16; @@ -1209,8 +1209,8 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, } } while (bytes_copied < 6 * 16); #define APPLY(i) \ - _mm_store_si128((void *)(encbuf + encbuf_size + i * 16), \ - _mm_xor_si128(_mm_load_si128((void *)(encbuf + encbuf_size + i * 16)), bits##i)) + _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), \ + _mm_xor_si128(_mm_loadu_si128((void *)(encbuf + encbuf_size + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); @@ -1223,7 +1223,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it * will be fed into ghash. */ remaining_ghash_from = encbuf_size - bytes_copied; - if ((encbuf_size & 15) != 0) + if ((bytes_copied & 15) != 0) _mm_storeu_si128((void *)(encbuf + encbuf_size), _mm_setzero_si128()); break; } @@ -1233,10 +1233,10 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, * blocks. */ AESECB6_INIT(); struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + 6; - gfmul_firststep(&gstate, _mm_load_si128((void *)encbuf + encbuf_size - 6 * 16), --ghash_precompute); + gfmul_firststep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 6 * 16), --ghash_precompute); for (size_t i = 1; i <= 5; ++i) { AESECB6_UPDATE(i); - gfmul_nextstep(&gstate, _mm_load_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); } _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); @@ -1267,14 +1267,15 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 * blocks at once. */ - _mm_store_si128((void *)(encbuf + (encbuf_size + 15) / 16 * 16), ac); + size_t ac_off = remaining_ghash_from + (encbuf_size - remaining_ghash_from + 15) / 16 * 16; + _mm_storeu_si128((void *)(encbuf + ac_off), ac); size_t blocks = (encbuf_size - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 7); struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + blocks; - gfmul_firststep(&gstate, _mm_load_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + gfmul_firststep(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; while (ghash_precompute != ctx->ghash) { - gfmul_nextstep(&gstate, _mm_load_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; } gfmul_reduce(&gstate); From 791036a6e7bd1621a9350e12c3ad833be12d9034 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 2 May 2022 11:51:37 +0900 Subject: [PATCH 06/48] 9-th gen Core, slowdown is 6% compared to fusion, when mm256_store is used --- lib/fusion.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 4b2ea9a03..c5d8f83bd 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1234,19 +1234,26 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, AESECB6_INIT(); struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + 6; gfmul_firststep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 6 * 16), --ghash_precompute); - for (size_t i = 1; i <= 5; ++i) { - AESECB6_UPDATE(i); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encbuf + encbuf_size + (i - 6) * 16)), --ghash_precompute); - } + AESECB6_UPDATE(1); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 5 * 16), --ghash_precompute); + AESECB6_UPDATE(2); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 4 * 16), --ghash_precompute); + AESECB6_UPDATE(3); _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); + AESECB6_UPDATE(4); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 3 * 16), --ghash_precompute); + AESECB6_UPDATE(5); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 2 * 16), --ghash_precompute); AESECB6_UPDATE(6); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 1 * 16), --ghash_precompute); AESECB6_UPDATE(7); if (encbuf_size >= 128) { _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); output += 128; encbuf_size -= 128; + AESECB6_UPDATE(8); _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 128)); _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 160)); } else { @@ -1254,8 +1261,10 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, encbuf_size -= 64; _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 64)); _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 96)); + AESECB6_UPDATE(8); } - for (size_t i = 8; i < ctx->ecb.rounds; ++i) + AESECB6_UPDATE(9); + for (size_t i = 10; i < ctx->ecb.rounds; ++i) AESECB6_UPDATE(i); assert(ctx->ghash == ghash_precompute); gfmul_reduce(&gstate); From 59983e99efa32fe81c67c967ff63d2111723bbbe Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 2 May 2022 12:04:47 +0900 Subject: [PATCH 07/48] this helps on gcc --- lib/fusion.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index c5d8f83bd..0f3a0d370 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1264,8 +1264,10 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, AESECB6_UPDATE(8); } AESECB6_UPDATE(9); - for (size_t i = 10; i < ctx->ecb.rounds; ++i) - AESECB6_UPDATE(i); + if (PTLS_UNLIKELY(ctx->ecb.rounds != 10)) { + for (size_t i = 10; PTLS_LIKELY(i < ctx->ecb.rounds); ++i) + AESECB6_UPDATE(i); + } assert(ctx->ghash == ghash_precompute); gfmul_reduce(&gstate); AESECB6_FINAL(ctx->ecb.rounds); From e0caeccdae75dfbcb2515b5147eb00c458accaa1 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 4 May 2022 10:09:24 +0900 Subject: [PATCH 08/48] gcc 9.4 (ubuntu 20.04) cannot detect this transformation --- lib/fusion.c | 84 +++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 0f3a0d370..09ba8d1cc 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1078,19 +1078,22 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, totlen += input[i].len; #define STATE_EK0_READY 0x1 +#define STATE_COPY_128B 0x2 int32_t state = 0; /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. Additional 16 bytes at the tail * allows us to write the AEAD tag in addition to the 6 blocks of ciphertext before sending the bytes to main memory using NT * store instructions. */ - char encbuf[64 + 6 * 16 + 16] __attribute__((aligned(32))); - size_t encbuf_size; + char encbuf[64 + 6 * 16 + 16] __attribute__((aligned(32))), *encp; /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ - if ((encbuf_size = ((uintptr_t)output & 63)) != 0) { - _mm256_store_si256((void *)encbuf, _mm256_load_si256(output - encbuf_size)); - _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256(output - encbuf_size + 32)); - output = output - encbuf_size; + if ((encp = encbuf + ((uintptr_t)output & 63)) != encbuf) { + _mm256_store_si256((void *)encbuf, _mm256_load_si256(output - (encp - encbuf))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256(output - (encp - encbuf) + 32)); + output = output - (encp - encbuf); } + /* First write would be 128 bytes (32+6*16), if encbuf contains no less than 32 bytes already. */ + if (encp - encbuf >= 32) + state |= STATE_COPY_128B; /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ __m128i ctr = calc_counter(agctx, seq); @@ -1152,7 +1155,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, * 5. writes encbuf in 64-byte blocks * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be * calculated. */ - size_t remaining_ghash_from = encbuf_size; + size_t remaining_ghash_from = encp - encbuf; if (totlen != 0) { const uint8_t *src = (void *)input[0].base; size_t srclen = input[0].len; @@ -1162,8 +1165,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, while (1) { /* apply the bit stream to input, writing to encbuf */ if (PTLS_LIKELY(srclen >= 6 * 16)) { -#define APPLY(i) \ - _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) +#define APPLY(i) _mm_storeu_si128((void *)(encp + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(src + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); @@ -1171,12 +1173,12 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, APPLY(4); APPLY(5); #undef APPLY - encbuf_size += 6 * 16; + encp += 6 * 16; src += 6 * 16; srclen -= 6 * 16; if (PTLS_UNLIKELY(srclen == 0)) { if (src_vecleft == 0) { - remaining_ghash_from = encbuf_size - 96; + remaining_ghash_from = (encp - encbuf) - 96; break; } src = (void *)input[0].base; @@ -1189,12 +1191,12 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, size_t bytes_copied = 0; do { if (srclen >= 16 && bytes_copied < 5 * 80) { - _mm_storeu_si128((void *)(encbuf + encbuf_size + bytes_copied), _mm_loadu_si128((void *)src)); + _mm_storeu_si128((void *)(encp + bytes_copied), _mm_loadu_si128((void *)src)); bytes_copied += 16; src += 16; srclen -= 16; } else { - encbuf[encbuf_size + bytes_copied++] = *src++; + encp[bytes_copied++] = *src++; --srclen; } if (PTLS_UNLIKELY(srclen == 0)) { @@ -1208,9 +1210,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, } } } while (bytes_copied < 6 * 16); -#define APPLY(i) \ - _mm_storeu_si128((void *)(encbuf + encbuf_size + i * 16), \ - _mm_xor_si128(_mm_loadu_si128((void *)(encbuf + encbuf_size + i * 16)), bits##i)) +#define APPLY(i) _mm_storeu_si128((void *)(encp + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(encp + i * 16)), bits##i)) APPLY(0); APPLY(1); APPLY(2); @@ -1218,13 +1218,13 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, APPLY(4); APPLY(5); #undef APPLY - encbuf_size += bytes_copied; + encp += bytes_copied; if (PTLS_UNLIKELY(srclen == 0)) { /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it * will be fed into ghash. */ - remaining_ghash_from = encbuf_size - bytes_copied; + remaining_ghash_from = (encp - encbuf) - bytes_copied; if ((bytes_copied & 15) != 0) - _mm_storeu_si128((void *)(encbuf + encbuf_size), _mm_setzero_si128()); + _mm_storeu_si128((void *)encp, _mm_setzero_si128()); break; } } @@ -1233,36 +1233,37 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, * blocks. */ AESECB6_INIT(); struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + 6; - gfmul_firststep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 6 * 16), --ghash_precompute); + gfmul_firststep(&gstate, _mm_loadu_si128((void *)encp - 6 * 16), --ghash_precompute); AESECB6_UPDATE(1); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 5 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 5 * 16), --ghash_precompute); AESECB6_UPDATE(2); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 4 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 4 * 16), --ghash_precompute); AESECB6_UPDATE(3); _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); AESECB6_UPDATE(4); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 3 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 3 * 16), --ghash_precompute); AESECB6_UPDATE(5); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 2 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 2 * 16), --ghash_precompute); AESECB6_UPDATE(6); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encbuf + encbuf_size - 1 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 1 * 16), --ghash_precompute); AESECB6_UPDATE(7); - if (encbuf_size >= 128) { + if ((state & STATE_COPY_128B) != 0) { _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); output += 128; - encbuf_size -= 128; + encp -= 128; AESECB6_UPDATE(8); _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 128)); _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 160)); } else { output += 64; - encbuf_size -= 64; + encp -= 64; _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 64)); _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 96)); AESECB6_UPDATE(8); } + state ^= STATE_COPY_128B; AESECB6_UPDATE(9); if (PTLS_UNLIKELY(ctx->ecb.rounds != 10)) { for (size_t i = 10; PTLS_LIKELY(i < ctx->ecb.rounds); ++i) @@ -1278,9 +1279,9 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 * blocks at once. */ - size_t ac_off = remaining_ghash_from + (encbuf_size - remaining_ghash_from + 15) / 16 * 16; + size_t ac_off = remaining_ghash_from + ((encp - encbuf) - remaining_ghash_from + 15) / 16 * 16; _mm_storeu_si128((void *)(encbuf + ac_off), ac); - size_t blocks = (encbuf_size - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ + size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 7); struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + blocks; gfmul_firststep(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); @@ -1301,22 +1302,23 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, } /* append tag to encbuf */ - _mm_storeu_si128((void *)(encbuf + encbuf_size), gfmul_get_tag(&gstate, bits5)); - encbuf_size += 16; + _mm_storeu_si128((void *)encp, gfmul_get_tag(&gstate, bits5)); + encp += 16; { /* Write encbuf, using NT store instructions in 64-byte chunks. Last partial block, if any, is written to cache, as that cache * line would likely be read when the next TLS record is being built. */ - size_t off = 0; - for (; off + 64 <= encbuf_size; off += 64) { - _mm256_stream_si256(output + off, _mm256_load_si256((void *)(encbuf + off))); - _mm256_stream_si256(output + off + 32, _mm256_load_si256((void *)(encbuf + off + 32))); + const char *s = encbuf; + char *d = output; + for (; encp - s >= 64; d += 64, s += 64) { + _mm256_stream_si256((void *)d, _mm256_load_si256((void *)s)); + _mm256_stream_si256((void *)(d + 32), _mm256_load_si256((void *)(s + 32))); } _mm_sfence(); /* weakly ordered writes have to be synced before being passed to NIC */ - if (off != encbuf_size) { - for (; off + 16 <= encbuf_size; off += 16) - _mm_store_si128(output + off, _mm_load_si128((void *)(encbuf + off))); - if (off != encbuf_size) - storen(output + off, encbuf_size - off, loadn(encbuf + off, encbuf_size - off)); + if (s != encp) { + for (; encp - s >= 16; d += 16, s += 16) + _mm_store_si128((void *)d, _mm_load_si128((void *)s)); + if (s != encp) + storen((void *)d, encp - s, loadn((void *)s, encp - s)); } } } From 8b9cd576ad228b0904726d5c65a8add343742e82 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 4 May 2022 13:44:33 +0900 Subject: [PATCH 09/48] disable address sanitization of `fastly_encrypt_v`, as it overwrites 64B lines --- lib/fusion.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 9592866b8..83c6892e9 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -48,6 +48,17 @@ #include "picotls.h" #include "picotls/fusion.h" +#if defined(__clang__) +#if __has_feature(address_sanitizer) +#define NO_SANITIZE_ADDRESS __attribute__((no_sanitize("address"))) +#endif +#elif __SANITIZE_ADDRESS__ /* gcc */ +#define NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#endif +#ifndef NO_SANITIZE_ADDRESS +#define NO_SANITIZE_ADDRESS +#endif + struct ptls_fusion_aesgcm_context { ptls_fusion_aesecb_context_t ecb; size_t capacity; @@ -223,13 +234,7 @@ static const uint8_t loadn_shuffle[31] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; // latter 15 bytes map to zero -#if defined(__clang__) -#if __has_feature(address_sanitizer) -__attribute__((no_sanitize("address"))) -#endif -#elif __SANITIZE_ADDRESS__ /* gcc */ -__attribute__((no_sanitize_address)) -#endif +NO_SANITIZE_ADDRESS static inline __m128i loadn(const void *p, size_t l) { __m128i v, mask = _mm_loadu_si128((__m128i *)(loadn_mask + 16 - l)); @@ -1033,6 +1038,7 @@ ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", sizeof(struct aesgcm_context), aes256gcm_setup}; +NO_SANITIZE_ADDRESS static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *aad, size_t aadlen) { From 122a33428a8741687b123279d3d600b9a6bce45e Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 4 May 2022 14:08:47 +0900 Subject: [PATCH 10/48] wip --- lib/fusion.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 83c6892e9..44ccda2ce 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1107,10 +1107,17 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, #define STATE_COPY_128B 0x2 int32_t state = 0; - /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. Additional 16 bytes at the tail - * allows us to write the AEAD tag in addition to the 6 blocks of ciphertext before sending the bytes to main memory using NT - * store instructions. */ - char encbuf[64 + 6 * 16 + 16] __attribute__((aligned(32))), *encp; + /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ + char encbuf[32 * 6 /* >= 64 + 6 * 16 + 16 */] __attribute__((aligned(32))), *encp; + + /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is + * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ + PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 16 + 16); + + /* clear encbuf to suppress UB warning */ + for (size_t i = 0; i < sizeof(encbuf) / 32; i += 32) + _mm256_store_si256((void *)encbuf + i, _mm256_setzero_si256()); + /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ if ((encp = encbuf + ((uintptr_t)output & 63)) != encbuf) { _mm256_store_si256((void *)encbuf, _mm256_load_si256(output - (encp - encbuf))); From 700beb9dfcd377886d7e9301b61e314533c9d18e Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 4 May 2022 14:30:42 +0900 Subject: [PATCH 11/48] more space, to suppress out-of-bounds read warning --- lib/fusion.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 44ccda2ce..44f3d0e70 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1108,16 +1108,12 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, int32_t state = 0; /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ - char encbuf[32 * 6 /* >= 64 + 6 * 16 + 16 */] __attribute__((aligned(32))), *encp; + char encbuf[32 * 6] __attribute__((aligned(32))), *encp; /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 16 + 16); - /* clear encbuf to suppress UB warning */ - for (size_t i = 0; i < sizeof(encbuf) / 32; i += 32) - _mm256_store_si256((void *)encbuf + i, _mm256_setzero_si256()); - /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ if ((encp = encbuf + ((uintptr_t)output & 63)) != encbuf) { _mm256_store_si256((void *)encbuf, _mm256_load_si256(output - (encp - encbuf))); From 7a0685dfe58904fdf0d6d9fc582d2e0a9928ccb0 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 4 May 2022 15:39:57 +0900 Subject: [PATCH 12/48] no pointer arithmetic on `void *` --- lib/fusion.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 44f3d0e70..7b5488c89 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1039,8 +1039,8 @@ ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", aes256gcm_setup}; NO_SANITIZE_ADDRESS -static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, ptls_iovec_t *input, size_t incnt, uint64_t seq, - const void *aad, size_t aadlen) +static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, + const void *_aad, size_t aadlen) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ @@ -1098,6 +1098,8 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, } while (0) struct aesgcm_context *agctx = (void *)_ctx; + uint8_t *output = _output; + const uint8_t *aad = _aad; size_t totlen = 0; for (size_t i = 0; i < incnt; ++i) @@ -1108,7 +1110,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, int32_t state = 0; /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ - char encbuf[32 * 6] __attribute__((aligned(32))), *encp; + uint8_t encbuf[32 * 6] __attribute__((aligned(32))), *encp; /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ @@ -1116,9 +1118,9 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ if ((encp = encbuf + ((uintptr_t)output & 63)) != encbuf) { - _mm256_store_si256((void *)encbuf, _mm256_load_si256(output - (encp - encbuf))); - _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256(output - (encp - encbuf) + 32)); - output = output - (encp - encbuf); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(output - (encp - encbuf)))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(output - (encp - encbuf) + 32))); + output -= encp - encbuf; } /* First write would be 128 bytes (32+6*16), if encbuf contains no less than 32 bytes already. */ if (encp - encbuf >= 32) @@ -1262,20 +1264,20 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, * blocks. */ AESECB6_INIT(); struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + 6; - gfmul_firststep(&gstate, _mm_loadu_si128((void *)encp - 6 * 16), --ghash_precompute); + gfmul_firststep(&gstate, _mm_loadu_si128((void *)(encp - 6 * 16)), --ghash_precompute); AESECB6_UPDATE(1); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 5 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 5 * 16)), --ghash_precompute); AESECB6_UPDATE(2); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 4 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 4 * 16)), --ghash_precompute); AESECB6_UPDATE(3); _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); AESECB6_UPDATE(4); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 3 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 3 * 16)), --ghash_precompute); AESECB6_UPDATE(5); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 2 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 2 * 16)), --ghash_precompute); AESECB6_UPDATE(6); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)encp - 1 * 16), --ghash_precompute); + gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 1 * 16)), --ghash_precompute); AESECB6_UPDATE(7); if ((state & STATE_COPY_128B) != 0) { _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); @@ -1283,13 +1285,13 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, output += 128; encp -= 128; AESECB6_UPDATE(8); - _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 128)); - _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 160)); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 128))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 160))); } else { output += 64; encp -= 64; - _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)encbuf + 64)); - _mm256_store_si256((void *)encbuf + 32, _mm256_load_si256((void *)encbuf + 96)); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 64))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 96))); AESECB6_UPDATE(8); } state ^= STATE_COPY_128B; @@ -1336,8 +1338,8 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *output, { /* Write encbuf, using NT store instructions in 64-byte chunks. Last partial block, if any, is written to cache, as that cache * line would likely be read when the next TLS record is being built. */ - const char *s = encbuf; - char *d = output; + const uint8_t *s = encbuf; + uint8_t *d = output; for (; encp - s >= 64; d += 64, s += 64) { _mm256_stream_si256((void *)d, _mm256_load_si256((void *)s)); _mm256_stream_si256((void *)(d + 32), _mm256_load_si256((void *)(s + 32))); From 65f3b13b8f0526ce65116da1bdcc4a3ef35fd70c Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 8 May 2022 07:32:41 +0900 Subject: [PATCH 13/48] oops --- lib/fusion.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fusion.c b/lib/fusion.c index 7b5488c89..ecea5a279 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1221,7 +1221,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, /* slow path, load at most 6 * 16 bytes to encbuf then encrypt in-place */ size_t bytes_copied = 0; do { - if (srclen >= 16 && bytes_copied < 5 * 80) { + if (srclen >= 16 && bytes_copied < 5 * 16) { _mm_storeu_si128((void *)(encp + bytes_copied), _mm_loadu_si128((void *)src)); bytes_copied += 16; src += 16; From 86aa4f6de30a6c6bb670f7feaef9f47a8f9b75ae Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 8 May 2022 16:59:49 +0900 Subject: [PATCH 14/48] no need to check; bytes_copied + srclen is guaranteed to be less than 6 * 16 --- lib/fusion.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fusion.c b/lib/fusion.c index ecea5a279..073324684 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1221,7 +1221,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, /* slow path, load at most 6 * 16 bytes to encbuf then encrypt in-place */ size_t bytes_copied = 0; do { - if (srclen >= 16 && bytes_copied < 5 * 16) { + if (srclen >= 16) { _mm_storeu_si128((void *)(encp + bytes_copied), _mm_loadu_si128((void *)src)); bytes_copied += 16; src += 16; From fa3cd328f4cefea791d02f885aff2fdcea29e253 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 8 May 2022 17:05:17 +0900 Subject: [PATCH 15/48] We need this - srclen can increase when switching to the next vector (revert prev commit) This reverts commit 86aa4f6de30a6c6bb670f7feaef9f47a8f9b75ae. --- lib/fusion.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fusion.c b/lib/fusion.c index 073324684..ecea5a279 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1221,7 +1221,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, /* slow path, load at most 6 * 16 bytes to encbuf then encrypt in-place */ size_t bytes_copied = 0; do { - if (srclen >= 16) { + if (srclen >= 16 && bytes_copied < 5 * 16) { _mm_storeu_si128((void *)(encp + bytes_copied), _mm_loadu_si128((void *)src)); bytes_copied += 16; src += 16; From 3b2ab6173248d65448bf4dc34e63c7916a0f4527 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 06:41:48 +0900 Subject: [PATCH 16/48] make sure ek0 is encrypted in the main loop --- lib/fusion.c | 33 +++++++++++++++++++++------------ t/fusion.c | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index ecea5a279..641c8f640 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1038,6 +1038,14 @@ ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", sizeof(struct aesgcm_context), aes256gcm_setup}; +static inline size_t calc_total_length(ptls_iovec_t *input, size_t incnt) +{ + size_t totlen = 0; + for (size_t i = 0; i < incnt; ++i) + totlen += input[i].len; + return totlen; +} + NO_SANITIZE_ADDRESS static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *_aad, size_t aadlen) @@ -1055,7 +1063,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, bits3 = _mm_shuffle_epi8(ctr, bswap8); \ ctr = _mm_add_epi64(ctr, one8); \ bits4 = _mm_shuffle_epi8(ctr, bswap8); \ - if (PTLS_LIKELY(totlen > 16 * 5)) { \ + if (PTLS_LIKELY(srclen > 16 * 5) || src_vecleft != 0) { \ ctr = _mm_add_epi64(ctr, one8); \ bits5 = _mm_shuffle_epi8(ctr, bswap8); \ } else { \ @@ -1101,10 +1109,6 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, uint8_t *output = _output; const uint8_t *aad = _aad; - size_t totlen = 0; - for (size_t i = 0; i < incnt; ++i) - totlen += input[i].len; - #define STATE_EK0_READY 0x1 #define STATE_COPY_128B 0x2 int32_t state = 0; @@ -1130,12 +1134,22 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, __m128i ctr = calc_counter(agctx, seq); ctr = _mm_insert_epi32(ctr, 1, 0); __m128i ek0 = _mm_shuffle_epi8(ctr, bswap8); - __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)totlen * 8), bswap8); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), bswap8); ptls_fusion_aesgcm_context_t *ctx = agctx->aesgcm; __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); struct ptls_fusion_gfmul_state gstate = {0}; + /* find the first non-empty vec */ + const uint8_t *src = NULL; + size_t srclen = 0, src_vecleft = incnt; + while (srclen == 0 && src_vecleft != 0) { + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ AESECB6_INIT(); AESECB6_UPDATE(1); @@ -1187,12 +1201,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be * calculated. */ size_t remaining_ghash_from = encp - encbuf; - if (totlen != 0) { - const uint8_t *src = (void *)input[0].base; - size_t srclen = input[0].len; - ++input; - size_t src_vecleft = incnt - 1; - + if (srclen != 0) { while (1) { /* apply the bit stream to input, writing to encbuf */ if (PTLS_LIKELY(srclen >= 6 * 16)) { diff --git a/t/fusion.c b/t/fusion.c index 57d2b45a0..2441da1dc 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -338,7 +338,7 @@ int main(int argc, char **argv) subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); - subtest("aesgcm", test_aesgcm); + //subtest("aesgcm", test_aesgcm); subtest("fastls", test_fastls); return done_testing(); From 9430f846929eadbf7a6ad00c8ef418dc5d6300fe Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 06:56:43 +0900 Subject: [PATCH 17/48] oops --- t/fusion.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/fusion.c b/t/fusion.c index 2441da1dc..57d2b45a0 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -338,7 +338,7 @@ int main(int argc, char **argv) subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); - //subtest("aesgcm", test_aesgcm); + subtest("aesgcm", test_aesgcm); subtest("fastls", test_fastls); return done_testing(); From 4f6bcaed65c7360b18b60f841acbb739436e79c7 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 07:02:13 +0900 Subject: [PATCH 18/48] remove needless if --- lib/fusion.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 641c8f640..b889b24bb 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1067,10 +1067,8 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, ctr = _mm_add_epi64(ctr, one8); \ bits5 = _mm_shuffle_epi8(ctr, bswap8); \ } else { \ - if ((state & STATE_EK0_READY) == 0) { \ - bits5 = ek0; \ - state |= STATE_EK0_READY; \ - } \ + bits5 = ek0; \ + state |= STATE_EK0_READY; \ } \ __m128i k = ctx->ecb.keys[0]; \ bits0 = _mm_xor_si128(bits0, k); \ From 204e76552967a27b36e28b85a511419ba7318471 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 15:52:54 +0900 Subject: [PATCH 19/48] add compile options for 256-bit-wide aesgcm --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8bce0dff8..d2cb4c03b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,14 +148,14 @@ ENDIF () IF (WITH_FUSION) ADD_LIBRARY(picotls-fusion lib/fusion.c) - SET_TARGET_PROPERTIES(picotls-fusion PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul") + SET_TARGET_PROPERTIES(picotls-fusion PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul -mvaes -mvpclmulqdq") TARGET_LINK_LIBRARIES(picotls-fusion picotls-core) ADD_EXECUTABLE(test-fusion.t deps/picotest/picotest.c lib/picotls.c t/fusion.c) TARGET_LINK_LIBRARIES(test-fusion.t picotls-minicrypto) - SET_TARGET_PROPERTIES(test-fusion.t PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul") + SET_TARGET_PROPERTIES(test-fusion.t PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul -mvaes -mvpclmulqdq") IF (WITH_DTRACE) ADD_DEPENDENCIES(test-fusion.t generate-picotls-probes) ENDIF () From 680ce18e22b7e3cdf12b22be9a61c07ac4c11488 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 15:53:18 +0900 Subject: [PATCH 20/48] aes-gcm using 256-bit insns --- include/picotls/fusion.h | 17 +- lib/fusion.c | 859 +++++++++++++++++++++++------- picotls.xcodeproj/project.pbxproj | 12 +- t/fusion.c | 158 +++++- 4 files changed, 842 insertions(+), 204 deletions(-) diff --git a/include/picotls/fusion.h b/include/picotls/fusion.h index 59f96844b..675c2ee8c 100644 --- a/include/picotls/fusion.h +++ b/include/picotls/fusion.h @@ -28,19 +28,24 @@ extern "C" { #include #include +#include #include "../picotls.h" #define PTLS_FUSION_AES128_ROUNDS 10 #define PTLS_FUSION_AES256_ROUNDS 14 typedef struct ptls_fusion_aesecb_context { - __m128i keys[PTLS_FUSION_AES256_ROUNDS + 1]; + union { + __m128i m128[PTLS_FUSION_AES256_ROUNDS + 1]; + __m256i m256[PTLS_FUSION_AES256_ROUNDS + 1]; + } keys; unsigned rounds; -} ptls_fusion_aesecb_context_t; + uint8_t avx256; +} __attribute__((aligned(32))) ptls_fusion_aesecb_context_t; typedef struct ptls_fusion_aesgcm_context ptls_fusion_aesgcm_context_t; -void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size); +void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int avx256); void ptls_fusion_aesecb_dispose(ptls_fusion_aesecb_context_t *ctx); void ptls_fusion_aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, void *dst, const void *src); @@ -84,6 +89,12 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *aad, size_t aadlen, const void *tag); +/** + * A boolean flag indicating if vaes and vpclmulqdq (256-bit crypto instructions) should be used. This flag is set automatically + * when `ptls_fusion_is_supported_by_cpu` is called. Users can update the flag to enforce behavior. Engines that do not have support + * for these 256-bit instructions will continue using the 128-bit ones, even when this flag is set. + */ +extern int ptls_fusion_can_avx256; extern ptls_cipher_algorithm_t ptls_fusion_aes128ctr, ptls_fusion_aes256ctr; extern ptls_aead_algorithm_t ptls_fusion_aes128gcm, ptls_fusion_aes256gcm; extern ptls_aead_algorithm_t ptls_fastls_aes128gcm, ptls_fastls_aes256gcm; diff --git a/lib/fusion.c b/lib/fusion.c index b889b24bb..21dcacfa7 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -63,10 +63,20 @@ struct ptls_fusion_aesgcm_context { ptls_fusion_aesecb_context_t ecb; size_t capacity; size_t ghash_cnt; - struct ptls_fusion_aesgcm_ghash_precompute { + struct ptls_fusion_aesgcm_ghash_precompute128 { __m128i H; __m128i r; - } ghash[0]; + } ghash128[0]; + union ptls_fusion_aesgcm_ghash_precompute256 { + struct { + __m128i H[2]; + __m128i r[2]; + }; + struct { + __m256i Hx2; + __m256i rx2; + }; + } ghash256[0]; }; struct ctr_context { @@ -87,10 +97,14 @@ struct aesgcm_context { static const uint64_t poly_[2] __attribute__((aligned(16))) = {1, 0xc200000000000000}; #define poly (*(__m128i *)poly_) -static const uint8_t bswap8_[16] __attribute__((aligned(16))) = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; -#define bswap8 (*(__m128i *)bswap8_) -static const uint8_t one8_[16] __attribute__((aligned(16))) = {1}; -#define one8 (*(__m128i *)one8_) +static const uint8_t byteswap_[32] __attribute__((aligned(32))) = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; +#define byteswap128 (*(__m128i *)byteswap_) +#define byteswap256 (*(__m256i *)byteswap_) +static const uint8_t one_[16] __attribute__((aligned(16))) = {1}; +#define one8 (*(__m128i *)one_) +static const uint8_t incr128x2_[32] __attribute__((aligned(32))) = {2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}; +#define incr128x2 (*(__m256i *)incr128x2_) /* This function is covered by the Apache License and the MIT License. The origin is crypto/modes/asm/ghash-x86_64.pl of openssl * at commit 33388b4. */ @@ -155,12 +169,31 @@ static __m128i gfmul(__m128i x, __m128i y) return _mm_xor_si128(hi, lo); } -struct ptls_fusion_gfmul_state { +static inline __m128i gfmul_do_reduce(__m128i hi, __m128i lo, __m128i mid) +{ + mid = _mm_xor_si128(mid, hi); + mid = _mm_xor_si128(mid, lo); + lo = _mm_xor_si128(lo, _mm_slli_si128(mid, 8)); + hi = _mm_xor_si128(hi, _mm_srli_si128(mid, 8)); + + /* fast reduction, using https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf */ + __m128i r = _mm_clmulepi64_si128(lo, poly, 0x10); + lo = _mm_shuffle_epi32(lo, 78); + lo = _mm_xor_si128(lo, r); + r = _mm_clmulepi64_si128(lo, poly, 0x10); + lo = _mm_shuffle_epi32(lo, 78); + lo = _mm_xor_si128(lo, r); + lo = _mm_xor_si128(hi, lo); + + return lo; +} + +struct ptls_fusion_gfmul_state128 { __m128i hi, lo, mid; }; -static inline void gfmul_do_step(struct ptls_fusion_gfmul_state *gstate, __m128i X, - struct ptls_fusion_aesgcm_ghash_precompute *precompute) +static inline void gfmul_do_step128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { __m128i t = _mm_clmulepi64_si128(precompute->H, X, 0x00); gstate->lo = _mm_xor_si128(gstate->lo, t); @@ -172,86 +205,159 @@ static inline void gfmul_do_step(struct ptls_fusion_gfmul_state *gstate, __m128i gstate->mid = _mm_xor_si128(gstate->mid, t); } -static inline void gfmul_firststep(struct ptls_fusion_gfmul_state *gstate, __m128i X, - struct ptls_fusion_aesgcm_ghash_precompute *precompute) +static inline void gfmul_firststep128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { - X = _mm_shuffle_epi8(X, bswap8); + X = _mm_shuffle_epi8(X, byteswap128); X = _mm_xor_si128(gstate->lo, X); gstate->lo = _mm_setzero_si128(); gstate->hi = _mm_setzero_si128(); gstate->mid = _mm_setzero_si128(); - gfmul_do_step(gstate, X, precompute); + gfmul_do_step128(gstate, X, precompute); } -static inline void gfmul_nextstep(struct ptls_fusion_gfmul_state *gstate, __m128i X, - struct ptls_fusion_aesgcm_ghash_precompute *precompute) +static inline void gfmul_nextstep128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, + struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { - X = _mm_shuffle_epi8(X, bswap8); - gfmul_do_step(gstate, X, precompute); + X = _mm_shuffle_epi8(X, byteswap128); + gfmul_do_step128(gstate, X, precompute); } -static inline void gfmul_reduce(struct ptls_fusion_gfmul_state *gstate) +static inline void gfmul_reduce128(struct ptls_fusion_gfmul_state128 *gstate) { - /* finish multiplication */ - gstate->mid = _mm_xor_si128(gstate->mid, gstate->hi); - gstate->mid = _mm_xor_si128(gstate->mid, gstate->lo); - gstate->lo = _mm_xor_si128(gstate->lo, _mm_slli_si128(gstate->mid, 8)); - gstate->hi = _mm_xor_si128(gstate->hi, _mm_srli_si128(gstate->mid, 8)); + gstate->lo = gfmul_do_reduce(gstate->hi, gstate->lo, gstate->mid); +} - /* fast reduction, using https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf */ - __m128i r = _mm_clmulepi64_si128(gstate->lo, poly, 0x10); - gstate->lo = _mm_shuffle_epi32(gstate->lo, 78); - gstate->lo = _mm_xor_si128(gstate->lo, r); - r = _mm_clmulepi64_si128(gstate->lo, poly, 0x10); - gstate->lo = _mm_shuffle_epi32(gstate->lo, 78); - gstate->lo = _mm_xor_si128(gstate->lo, r); - gstate->lo = _mm_xor_si128(gstate->hi, gstate->lo); +static inline __m128i gfmul_get_tag128(struct ptls_fusion_gfmul_state128 *gstate, __m128i ek0) +{ + __m128i tag = _mm_shuffle_epi8(gstate->lo, byteswap128); + tag = _mm_xor_si128(tag, ek0); + return tag; +} + +struct ptls_fusion_gfmul_state256 { + __m256i hi, lo, mid; +}; + +static inline void gfmul_do_step256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, + union ptls_fusion_aesgcm_ghash_precompute256 *precompute) +{ + __m256i t = _mm256_clmulepi64_epi128(precompute->Hx2, X, 0x00); + gstate->lo = _mm256_xor_si256(gstate->lo, t); + t = _mm256_clmulepi64_epi128(precompute->Hx2, X, 0x11); + gstate->hi = _mm256_xor_si256(gstate->hi, t); + t = _mm256_shuffle_epi32(X, 78); + t = _mm256_xor_si256(t, X); + t = _mm256_clmulepi64_epi128(precompute->rx2, t, 0x00); + gstate->mid = _mm256_xor_si256(gstate->mid, t); +} + +static inline void gfmul_firststep256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, int half, + union ptls_fusion_aesgcm_ghash_precompute256 *precompute) +{ + X = _mm256_shuffle_epi8(X, byteswap256); + X = _mm256_xor_si256(gstate->lo, X); + if (half) + X = _mm256_permute2f128_si256(X, X, 0x08); + gstate->lo = _mm256_setzero_si256(); + gstate->hi = _mm256_setzero_si256(); + gstate->mid = _mm256_setzero_si256(); + gfmul_do_step256(gstate, X, precompute); +} + +static inline void gfmul_nextstep256(struct ptls_fusion_gfmul_state256 *gstate, __m256i X, + union ptls_fusion_aesgcm_ghash_precompute256 *precompute) +{ + X = _mm256_shuffle_epi8(X, byteswap256); + gfmul_do_step256(gstate, X, precompute); } -static inline __m128i gfmul_get_tag(struct ptls_fusion_gfmul_state *gstate, __m128i ek0) +static inline void gfmul_reduce256(struct ptls_fusion_gfmul_state256 *gstate) { - __m128i tag = _mm_shuffle_epi8(gstate->lo, bswap8); +#define XOR_256TO128(y) _mm_xor_si128(_mm256_castsi256_si128(y), _mm256_extractf128_si256((y), 1)) + __m128i hi = XOR_256TO128(gstate->hi); + __m128i lo = XOR_256TO128(gstate->lo); + __m128i mid = XOR_256TO128(gstate->mid); +#undef XOR_256TO128 + + lo = gfmul_do_reduce(hi, lo, mid); + gstate->lo = _mm256_castsi128_si256(lo); +} + +static inline __m128i gfmul_get_tag256(struct ptls_fusion_gfmul_state256 *gstate, __m128i ek0) +{ + __m128i tag = _mm_shuffle_epi8(_mm256_castsi256_si128(gstate->lo), byteswap128); tag = _mm_xor_si128(tag, ek0); return tag; } static inline __m128i aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, __m128i v) { - size_t i; +#define ROUNDKEY(i) (ctx->avx256 ? _mm256_castsi256_si128(ctx->keys.m256[i]) : ctx->keys.m128[i]) - v = _mm_xor_si128(v, ctx->keys[0]); - for (i = 1; i < ctx->rounds; ++i) - v = _mm_aesenc_si128(v, ctx->keys[i]); - v = _mm_aesenclast_si128(v, ctx->keys[i]); + v = _mm_xor_si128(v, ROUNDKEY(0)); + for (size_t i = 1; i < ctx->rounds; ++i) + v = _mm_aesenc_si128(v, ROUNDKEY(i)); + v = _mm_aesenclast_si128(v, ROUNDKEY(ctx->rounds)); return v; + +#undef ROUNDKEY } -static const uint8_t loadn_mask[31] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; +// 32-bytes of 0xff followed by 31-bytes of 0x00 +static const uint8_t loadn_mask[63] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; static const uint8_t loadn_shuffle[31] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, // first 16 bytes map to byte offsets 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; // latter 15 bytes map to zero NO_SANITIZE_ADDRESS -static inline __m128i loadn(const void *p, size_t l) +static inline __m128i loadn_end_of_page(const void *p, size_t l) { - __m128i v, mask = _mm_loadu_si128((__m128i *)(loadn_mask + 16 - l)); + uintptr_t shift = (uintptr_t)p & 15; + __m128i pattern = _mm_loadu_si128((const __m128i *)(loadn_shuffle + shift)); + return _mm_shuffle_epi8(_mm_load_si128((const __m128i *)((uintptr_t)p - shift)), pattern); +} + +static inline __m128i loadn128(const void *p, size_t l) +{ + __m128i v, mask = _mm_loadu_si128((__m128i *)(loadn_mask + 32 - l)); uintptr_t mod4k = (uintptr_t)p % 4096; - if (PTLS_LIKELY(mod4k <= 4080) || mod4k + l > 4096) { + if (PTLS_LIKELY(mod4k <= 4096 - 16) || mod4k + l > 4096) { v = _mm_loadu_si128(p); } else { - uintptr_t shift = (uintptr_t)p & 15; - __m128i pattern = _mm_loadu_si128((const __m128i *)(loadn_shuffle + shift)); - v = _mm_shuffle_epi8(_mm_load_si128((const __m128i *)((uintptr_t)p - shift)), pattern); + v = loadn_end_of_page(p, l); } v = _mm_and_si128(v, mask); + return v; } -static inline void storen(void *_p, size_t l, __m128i v) +static inline __m256i loadn256(const void *p, size_t l) +{ + __m256i v, mask = _mm256_loadu_si256((__m256i *)(loadn_mask + 32 - l)); + uintptr_t mod4k = (uintptr_t)p % 4096; + + if (PTLS_LIKELY(mod4k < 4096 - 32) || mod4k + l > 4096) { + v = _mm256_loadu_si256(p); + } else if (l > 16) { + __m128i first16 = _mm_loadu_si128(p), second16 = loadn128((uint8_t *)(p + 16), l - 16); + v = _mm256_permute2f128_si256(_mm256_castsi128_si256(first16), _mm256_castsi128_si256(second16), 0x20); + } else if (l == 16) { + v = _mm256_castsi128_si256(_mm_loadu_si128(p)); + } else { + v = _mm256_castsi128_si256(loadn_end_of_page(p, l)); + } + v = _mm256_and_si256(v, mask); + + return v; +} + +static inline void storen128(void *_p, size_t l, __m128i v) { uint8_t buf[16], *p = _p; @@ -268,18 +374,18 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, #define AESECB6_INIT() \ do { \ ctr = _mm_add_epi64(ctr, one8); \ - bits0 = _mm_shuffle_epi8(ctr, bswap8); \ + bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits1 = _mm_shuffle_epi8(ctr, bswap8); \ + bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits2 = _mm_shuffle_epi8(ctr, bswap8); \ + bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits3 = _mm_shuffle_epi8(ctr, bswap8); \ + bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits4 = _mm_shuffle_epi8(ctr, bswap8); \ + bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ if (PTLS_LIKELY(srclen > 16 * 5)) { \ ctr = _mm_add_epi64(ctr, one8); \ - bits5 = _mm_shuffle_epi8(ctr, bswap8); \ + bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ } else { \ if ((state & STATE_EK0_BEEN_FED) == 0) { \ bits5 = ek0; \ @@ -287,11 +393,11 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, } \ if ((state & STATE_SUPP_USED) != 0 && srclen <= 16 * 4 && (const __m128i *)supp->input + 1 <= dst_ghash) { \ bits4 = _mm_loadu_si128(supp->input); \ - bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys; \ + bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys.m128; \ state |= STATE_SUPP_IN_PROCESS; \ } \ } \ - __m128i k = ctx->ecb.keys[0]; \ + __m128i k = ctx->ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ @@ -303,7 +409,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -315,7 +421,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ @@ -325,10 +431,10 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, } while (0) __m128i ek0, bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); - const __m128i *bits4keys = ctx->ecb.keys; /* is changed to supp->ctx.keys when calcurating suppout */ - struct ptls_fusion_gfmul_state gstate = {0}; + const __m128i *bits4keys = ctx->ecb.keys.m128; /* is changed to supp->ctx.keys when calcurating suppout */ + struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; - __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), bswap8); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); // src and dst are updated after the chunk is processed const __m128i *src = input; @@ -338,7 +444,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const __m128i *aad = _aad, *dst_ghash = dst; size_t dst_ghashlen = srclen; - struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (srclen + 15) / 16 + 1; + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + (aadlen + 15) / 16 + (srclen + 15) / 16 + 1; #define STATE_EK0_BEEN_FED 0x3 #define STATE_EK0_INCOMPLETE 0x2 @@ -349,7 +455,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* build counter */ ctr = _mm_insert_epi32(ctr, 1, 0); - ek0 = _mm_shuffle_epi8(ctr, bswap8); + ek0 = _mm_shuffle_epi8(ctr, byteswap128); /* start preparing AES */ AESECB6_INIT(); @@ -362,7 +468,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (PTLS_LIKELY(aadlen < 16)) { if (aadlen != 0) { - gdatabuf[gdata_cnt++] = loadn(aad, aadlen); + gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; } goto MainLoop; @@ -379,7 +485,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, size_t i; for (i = 2; i < gdata_cnt + 2; ++i) { AESECB6_UPDATE(i); - gfmul_nextstep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } for (; i < ctx->ecb.rounds; ++i) AESECB6_UPDATE(i); @@ -429,7 +535,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, #undef APPLY goto ApplyEnd; ApplyRemainder: - storen(dst, srclen, _mm_xor_si128(loadn(src, srclen), bits0)); + storen128(dst, srclen, _mm_xor_si128(loadn128(src, srclen), bits0)); dst = (__m128i *)((uint8_t *)dst + srclen); srclen = 0; ApplyEnd:; @@ -447,7 +553,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (aadlen < 16) { if (aadlen != 0) { - gdatabuf[gdata_cnt++] = loadn(aad, aadlen); + gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; } goto GdataFillDST; @@ -467,7 +573,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (dst_ghashlen < 16) { if (dst_ghashlen != 0) { - gdatabuf[gdata_cnt++] = loadn(dst_ghash, dst_ghashlen); + gdatabuf[gdata_cnt++] = loadn128(dst_ghash, dst_ghashlen); dst_ghashlen = 0; } if (gdata_cnt < 6) @@ -490,16 +596,16 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, */ assert(STATE_EK0_READY()); for (size_t i = 0; i < gdata_cnt; ++i) - gfmul_nextstep(&gstate, gdatabuf[i], --ghash_precompute); + gfmul_nextstep128(&gstate, gdatabuf[i], --ghash_precompute); - gfmul_reduce(&gstate); - _mm_storeu_si128(dst, gfmul_get_tag(&gstate, ek0)); + gfmul_reduce128(&gstate); + _mm_storeu_si128(dst, gfmul_get_tag128(&gstate, ek0)); /* Finish the calculation of supplemental vector. Done at the very last, because the sample might cover the GCM tag. */ if ((state & STATE_SUPP_USED) != 0) { size_t i; if ((state & STATE_SUPP_IN_PROCESS) == 0) { - bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys; + bits4keys = ((struct ctr_context *)supp->ctx)->fusion.keys.m128; bits4 = _mm_xor_si128(_mm_loadu_si128(supp->input), bits4keys[0]); i = 1; } else { @@ -525,10 +631,10 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, { __m128i ek0 = _mm_setzero_si128(), bits0, bits1 = _mm_setzero_si128(), bits2 = _mm_setzero_si128(), bits3 = _mm_setzero_si128(), bits4 = _mm_setzero_si128(), bits5 = _mm_setzero_si128(); - struct ptls_fusion_gfmul_state gstate = {0}; + struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; - __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), bswap8); - struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (inlen + 15) / 16 + 1; + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + (aadlen + 15) / 16 + (inlen + 15) / 16 + 1; const __m128i *gdata; // points to the elements fed into GHASH size_t gdata_cnt; @@ -539,7 +645,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* schedule ek0 and suppkey */ ctr = _mm_add_epi64(ctr, one8); - bits0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, bswap8), ctx->ecb.keys[0]); + bits0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), ctx->ecb.keys.m128[0]); ++nondata_aes_cnt; #define STATE_IS_FIRST_RUN 0x1 @@ -556,7 +662,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (aadlen < 16) { if (aadlen != 0) { - gdatabuf[gdata_cnt++] = loadn(aad, aadlen); + gdatabuf[gdata_cnt++] = loadn128(aad, aadlen); aadlen = 0; ++nondata_aes_cnt; } @@ -578,7 +684,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, while (gdata_cnt < 6) { if (src_ghashlen < 16) { if (src_ghashlen != 0) { - gdatabuf[gdata_cnt++] = loadn(src_ghash, src_ghashlen); + gdatabuf[gdata_cnt++] = loadn128(src_ghash, src_ghashlen); src_ghash = (__m128i *)((uint8_t *)src_ghash + src_ghashlen); src_ghashlen = 0; } @@ -600,7 +706,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, #define INIT_BITS(n, keys) \ case n: \ ctr = _mm_add_epi64(ctr, one8); \ - bits##n = _mm_xor_si128(_mm_shuffle_epi8(ctr, bswap8), keys[0]); + bits##n = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), keys.m128[0]); InitAllBits: INIT_BITS(0, ctx->ecb.keys); INIT_BITS(1, ctx->ecb.keys); @@ -614,7 +720,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, { /* run aes and ghash */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -626,11 +732,11 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, size_t aesi; for (aesi = 1; aesi <= gdata_cnt; ++aesi) { AESECB6_UPDATE(aesi); - gfmul_nextstep(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } for (; aesi < ctx->ecb.rounds; ++aesi) AESECB6_UPDATE(aesi); - __m128i k = ctx->ecb.keys[aesi]; + __m128i k = ctx->ecb.keys.m128[aesi]; bits0 = _mm_aesenclast_si128(bits0, k); bits1 = _mm_aesenclast_si128(bits1, k); bits2 = _mm_aesenclast_si128(bits2, k); @@ -685,19 +791,19 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, if (src_aeslen == 16) { _mm_storeu_si128(dst, _mm_xor_si128(_mm_loadu_si128(src_aes), bits0)); } else if (src_aeslen != 0) { - storen(dst, src_aeslen, _mm_xor_si128(loadn(src_aes, src_aeslen), bits0)); + storen128(dst, src_aeslen, _mm_xor_si128(loadn128(src_aes, src_aeslen), bits0)); } assert((state & STATE_IS_FIRST_RUN) == 0); /* the only case where AES operation is complete and GHASH is not is when the application of AC is remaining */ if ((state & STATE_GHASH_HAS_MORE) != 0) { - assert(ghash_precompute - 1 == ctx->ghash); - gfmul_nextstep(&gstate, ac, --ghash_precompute); + assert(ghash_precompute - 1 == ctx->ghash128); + gfmul_nextstep128(&gstate, ac, --ghash_precompute); } - gfmul_reduce(&gstate); - __m128i calctag = gfmul_get_tag(&gstate, ek0); + gfmul_reduce128(&gstate); + __m128i calctag = gfmul_get_tag128(&gstate, ek0); return _mm_movemask_epi8(_mm_cmpeq_epi8(calctag, _mm_loadu_si128(tag))) == 0xffff; @@ -716,7 +822,7 @@ static __m128i expand_key(__m128i key, __m128i temp) return key; } -void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size) +void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int avx256) { assert(is_enc && "decryption is not supported (yet)"); @@ -733,37 +839,48 @@ void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, cons assert(!"invalid key size; AES128 / AES256 are supported"); break; } + ctx->avx256 = avx256; - ctx->keys[i++] = _mm_loadu_si128((__m128i *)key); + /* load and expand keys using keys.m128 */ + ctx->keys.m128[i++] = _mm_loadu_si128((__m128i *)key); if (key_size == 32) - ctx->keys[i++] = _mm_loadu_si128((__m128i *)key + 1); - + ctx->keys.m128[i++] = _mm_loadu_si128((__m128i *)key + 1); + while (1) { #define EXPAND(R) \ - do { \ - ctx->keys[i] = expand_key(ctx->keys[i - key_size / 16], \ - _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys[i - 1], R), _MM_SHUFFLE(3, 3, 3, 3))); \ + { \ + ctx->keys.m128[i] = \ + expand_key(ctx->keys.m128[i - key_size / 16], \ + _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys.m128[i - 1], R), _MM_SHUFFLE(3, 3, 3, 3))); \ if (i == ctx->rounds) \ - goto Done; \ + break; \ ++i; \ if (key_size > 24) { \ - ctx->keys[i] = expand_key(ctx->keys[i - key_size / 16], \ - _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys[i - 1], R), _MM_SHUFFLE(2, 2, 2, 2))); \ + ctx->keys.m128[i] = \ + expand_key(ctx->keys.m128[i - key_size / 16], \ + _mm_shuffle_epi32(_mm_aeskeygenassist_si128(ctx->keys.m128[i - 1], R), _MM_SHUFFLE(2, 2, 2, 2))); \ ++i; \ } \ - } while (0) - EXPAND(0x1); - EXPAND(0x2); - EXPAND(0x4); - EXPAND(0x8); - EXPAND(0x10); - EXPAND(0x20); - EXPAND(0x40); - EXPAND(0x80); - EXPAND(0x1b); - EXPAND(0x36); + } + EXPAND(0x1); + EXPAND(0x2); + EXPAND(0x4); + EXPAND(0x8); + EXPAND(0x10); + EXPAND(0x20); + EXPAND(0x40); + EXPAND(0x80); + EXPAND(0x1b); + EXPAND(0x36); #undef EXPAND -Done: - assert(i == ctx->rounds); + } + + /* convert to keys.m256 if avx256 is used */ + if (ctx->avx256) { + size_t i = ctx->rounds; + do { + ctx->keys.m256[i] = _mm256_broadcastsi128_si256(ctx->keys.m128[i]); + } while (i-- != 0); + } } void ptls_fusion_aesecb_dispose(ptls_fusion_aesecb_context_t *ctx) @@ -789,31 +906,55 @@ static size_t aesgcm_calc_ghash_cnt(size_t capacity) static void setup_one_ghash_entry(ptls_fusion_aesgcm_context_t *ctx) { - if (ctx->ghash_cnt != 0) - ctx->ghash[ctx->ghash_cnt].H = gfmul(ctx->ghash[ctx->ghash_cnt - 1].H, ctx->ghash[0].H); + __m128i *H, *r, *Hprev, H0; + + if (ctx->ecb.avx256) { +#define GET_SLOT(i, mem) (&ctx->ghash256[(i) / 2].mem[(i) % 2 == 0]) + H = GET_SLOT(ctx->ghash_cnt, H); + r = GET_SLOT(ctx->ghash_cnt, r); + Hprev = ctx->ghash_cnt == 0 ? NULL : GET_SLOT(ctx->ghash_cnt - 1, H); +#undef GET_SLOT + H0 = ctx->ghash256[0].H[1]; + } else { + H = &ctx->ghash128[ctx->ghash_cnt].H; + r = &ctx->ghash128[ctx->ghash_cnt].r; + Hprev = ctx->ghash_cnt == 0 ? NULL : &ctx->ghash128[ctx->ghash_cnt - 1].H; + H0 = ctx->ghash128[0].H; + } - __m128i r = _mm_shuffle_epi32(ctx->ghash[ctx->ghash_cnt].H, 78); - r = _mm_xor_si128(r, ctx->ghash[ctx->ghash_cnt].H); - ctx->ghash[ctx->ghash_cnt].r = r; + if (Hprev != NULL) + *H = gfmul(*Hprev, H0); + + *r = _mm_shuffle_epi32(*H, 78); + *r = _mm_xor_si128(*r, *H); ++ctx->ghash_cnt; } -ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_new(const void *key, size_t key_size, size_t capacity) +static ptls_fusion_aesgcm_context_t *new_aesgcm(const void *key, size_t key_size, size_t capacity, int avx256) { ptls_fusion_aesgcm_context_t *ctx; size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity); + if (avx256 && ghash_cnt % 2 != 0) + ++ghash_cnt; - if ((ctx = malloc(sizeof(*ctx) + sizeof(ctx->ghash[0]) * ghash_cnt)) == NULL) + PTLS_BUILD_ASSERT(sizeof(ctx->ghash128[0]) * 2 == sizeof(ctx->ghash256[0])); + if ((ctx = aligned_alloc(32, sizeof(*ctx) + sizeof(ctx->ghash128[0]) * ghash_cnt)) == NULL) return NULL; - ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size); + ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size, avx256); ctx->capacity = capacity; - ctx->ghash[0].H = aesecb_encrypt(&ctx->ecb, _mm_setzero_si128()); - ctx->ghash[0].H = _mm_shuffle_epi8(ctx->ghash[0].H, bswap8); - ctx->ghash[0].H = transformH(ctx->ghash[0].H); + __m128i H0 = aesecb_encrypt(&ctx->ecb, _mm_setzero_si128()); + H0 = _mm_shuffle_epi8(H0, byteswap128); + H0 = transformH(H0); + if (ctx->ecb.avx256) { + ctx->ghash256[0].H[1] = H0; + } else { + ctx->ghash128[0].H = H0; + } + ctx->ghash_cnt = 0; while (ctx->ghash_cnt < ghash_cnt) setup_one_ghash_entry(ctx); @@ -821,6 +962,11 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_new(const void *key, size_t key return ctx; } +ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_new(const void *key, size_t key_size, size_t capacity) +{ + return new_aesgcm(key, key_size, capacity, 0); +} + ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm_context_t *ctx, size_t capacity) { size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity); @@ -828,7 +974,7 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm if (ghash_cnt <= ctx->ghash_cnt) return ctx; - if ((ctx = realloc(ctx, sizeof(*ctx) + sizeof(ctx->ghash[0]) * ghash_cnt)) == NULL) + if ((ctx = realloc(ctx, sizeof(*ctx) + sizeof(ctx->ghash128[0]) * ghash_cnt)) == NULL) return NULL; ctx->capacity = capacity; @@ -840,7 +986,7 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm void ptls_fusion_aesgcm_free(ptls_fusion_aesgcm_context_t *ctx) { - ptls_clear_memory(ctx->ghash, sizeof(ctx->ghash[0]) * ctx->ghash_cnt); + ptls_clear_memory(ctx->ghash128, sizeof(ctx->ghash128[0]) * ctx->ghash_cnt); ctx->ghash_cnt = 0; ptls_fusion_aesecb_dispose(&ctx->ecb); free(ctx); @@ -869,7 +1015,7 @@ static void ctr_transform(ptls_cipher_context_t *_ctx, void *output, const void ctx->is_ready = 0; if (len < 16) { - storen(output, len, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), loadn(input, len))); + storen128(output, len, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), loadn128(input, len))); } else { _mm_storeu_si128(output, _mm_xor_si128(_mm_loadu_si128(&ctx->bits), _mm_loadu_si128(input))); } @@ -882,7 +1028,7 @@ static int aesctr_setup(ptls_cipher_context_t *_ctx, int is_enc, const void *key ctx->super.do_dispose = ctr_dispose; ctx->super.do_init = ctr_init; ctx->super.do_transform = ctr_transform; - ptls_fusion_aesecb_init(&ctx->fusion, 1, key, key_size); + ptls_fusion_aesecb_init(&ctx->fusion, 1, key, key_size, 0 /* probably we do not need avx256 for CTR? */); ctx->is_ready = 0; return 0; @@ -967,8 +1113,8 @@ static size_t aead_do_decrypt(ptls_aead_context_t *_ctx, void *output, const voi static inline void aesgcm_xor_iv(ptls_aead_context_t *_ctx, const void *_bytes, size_t len) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; - __m128i xor_mask = loadn(_bytes, len); - xor_mask = _mm_shuffle_epi8(xor_mask, bswap8); + __m128i xor_mask = loadn128(_bytes, len); + xor_mask = _mm_shuffle_epi8(xor_mask, byteswap128); ctx->static_iv = _mm_xor_si128(ctx->static_iv, xor_mask); } @@ -976,8 +1122,8 @@ static int aesgcm_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; - ctx->static_iv = loadn(iv, PTLS_AESGCM_IV_SIZE); - ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, bswap8); + ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); + ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); if (key == NULL) return 0; @@ -990,7 +1136,7 @@ static int aesgcm_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, ctx->super.do_encrypt_v = aead_do_encrypt_v; ctx->super.do_decrypt = aead_do_decrypt; - ctx->aesgcm = ptls_fusion_aesgcm_new(key, key_size, 1500 /* assume ordinary packet size */); + ctx->aesgcm = new_aesgcm(key, key_size, 1500 /* assume ordinary packet size */, 0 /* no support for avx256 yet */); return 0; } @@ -1005,6 +1151,7 @@ static int aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key return aesgcm_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); } +int ptls_fusion_can_avx256 = 0; ptls_cipher_algorithm_t ptls_fusion_aes128ctr = {"AES128-CTR", PTLS_AES128_KEY_SIZE, 1, // block size @@ -1046,31 +1193,49 @@ static inline size_t calc_total_length(ptls_iovec_t *input, size_t incnt) return totlen; } +static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const uint8_t *end) +{ + /* Write in 64-byte chunks, using NT store instructions. Last partial block, if any, is written to cache, as that cache line + * would likely be read when the next TLS record is being built. */ + + for (; end - src >= 64; dst += 64, src += 64) { + _mm256_stream_si256((void *)dst, _mm256_load_si256((void *)src)); + _mm256_stream_si256((void *)(dst + 32), _mm256_load_si256((void *)(src + 32))); + } + _mm_sfence(); /* weakly ordered writes have to be synced before being passed to NIC */ + if (src != end) { + for (; end - src >= 16; dst += 16, src += 16) + _mm_store_si128((void *)dst, _mm_load_si128((void *)src)); + if (src != end) + storen128((void *)dst, end - src, loadn128((void *)src, end - src)); + } +} + NO_SANITIZE_ADDRESS -static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, - const void *_aad, size_t aadlen) +static void fastls_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, + const void *_aad, size_t aadlen) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ do { \ ctr = _mm_add_epi64(ctr, one8); \ - bits0 = _mm_shuffle_epi8(ctr, bswap8); \ + bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits1 = _mm_shuffle_epi8(ctr, bswap8); \ + bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits2 = _mm_shuffle_epi8(ctr, bswap8); \ + bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits3 = _mm_shuffle_epi8(ctr, bswap8); \ + bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ ctr = _mm_add_epi64(ctr, one8); \ - bits4 = _mm_shuffle_epi8(ctr, bswap8); \ + bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ if (PTLS_LIKELY(srclen > 16 * 5) || src_vecleft != 0) { \ ctr = _mm_add_epi64(ctr, one8); \ - bits5 = _mm_shuffle_epi8(ctr, bswap8); \ + bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ } else { \ bits5 = ek0; \ state |= STATE_EK0_READY; \ } \ - __m128i k = ctx->ecb.keys[0]; \ + __m128i k = ctx->ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ @@ -1082,7 +1247,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -1094,7 +1259,7 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ - __m128i k = ctx->ecb.keys[i]; \ + __m128i k = ctx->ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ @@ -1131,12 +1296,12 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ __m128i ctr = calc_counter(agctx, seq); ctr = _mm_insert_epi32(ctr, 1, 0); - __m128i ek0 = _mm_shuffle_epi8(ctr, bswap8); - __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), bswap8); + __m128i ek0 = _mm_shuffle_epi8(ctr, byteswap128); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128); ptls_fusion_aesgcm_context_t *ctx = agctx->aesgcm; __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); - struct ptls_fusion_gfmul_state gstate = {0}; + struct ptls_fusion_gfmul_state128 gstate = {0}; /* find the first non-empty vec */ const uint8_t *src = NULL; @@ -1153,37 +1318,37 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, AESECB6_UPDATE(1); AESECB6_UPDATE(2); if (PTLS_LIKELY(aadlen != 0)) { - struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute; + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute; while (PTLS_UNLIKELY(aadlen >= 6 * 16)) { - ghash_precompute = ctx->ghash + 6; - gfmul_firststep(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + ghash_precompute = ctx->ghash128 + 6; + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; for (int i = 1; i < 6; ++i) { - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; } - gfmul_reduce(&gstate); + gfmul_reduce128(&gstate); } if (PTLS_LIKELY(aadlen != 0)) { - ghash_precompute = ctx->ghash + (aadlen + 15) / 16; + ghash_precompute = ctx->ghash128 + (aadlen + 15) / 16; if (PTLS_UNLIKELY(aadlen >= 16)) { - gfmul_firststep(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; while (aadlen >= 16) { - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); aad += 16; aadlen -= 16; } if (PTLS_LIKELY(aadlen != 0)) - gfmul_nextstep(&gstate, loadn(aad, aadlen), --ghash_precompute); + gfmul_nextstep128(&gstate, loadn128(aad, aadlen), --ghash_precompute); } else { - gfmul_firststep(&gstate, loadn(aad, aadlen), --ghash_precompute); + gfmul_firststep128(&gstate, loadn128(aad, aadlen), --ghash_precompute); } - assert(ctx->ghash == ghash_precompute); - gfmul_reduce(&gstate); + assert(ctx->ghash128 == ghash_precompute); + gfmul_reduce128(&gstate); } } for (size_t i = 3; i < ctx->ecb.rounds; ++i) @@ -1270,21 +1435,21 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, /* Next 96-byte block starts here. Run AES and ghash in while writing output using non-temporal stores in 64-byte * blocks. */ AESECB6_INIT(); - struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + 6; - gfmul_firststep(&gstate, _mm_loadu_si128((void *)(encp - 6 * 16)), --ghash_precompute); + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + 6; + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encp - 6 * 16)), --ghash_precompute); AESECB6_UPDATE(1); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 5 * 16)), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 5 * 16)), --ghash_precompute); AESECB6_UPDATE(2); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 4 * 16)), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 4 * 16)), --ghash_precompute); AESECB6_UPDATE(3); _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); AESECB6_UPDATE(4); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 3 * 16)), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 3 * 16)), --ghash_precompute); AESECB6_UPDATE(5); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 2 * 16)), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 2 * 16)), --ghash_precompute); AESECB6_UPDATE(6); - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encp - 1 * 16)), --ghash_precompute); + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 1 * 16)), --ghash_precompute); AESECB6_UPDATE(7); if ((state & STATE_COPY_128B) != 0) { _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); @@ -1307,8 +1472,8 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, for (size_t i = 10; PTLS_LIKELY(i < ctx->ecb.rounds); ++i) AESECB6_UPDATE(i); } - assert(ctx->ghash == ghash_precompute); - gfmul_reduce(&gstate); + assert(ctx->ghash128 == ghash_precompute); + gfmul_reduce128(&gstate); AESECB6_FINAL(ctx->ecb.rounds); } } @@ -1321,62 +1486,361 @@ static void fastls_encrypt_v(struct st_ptls_aead_context_t *_ctx, void *_output, _mm_storeu_si128((void *)(encbuf + ac_off), ac); size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 7); - struct ptls_fusion_aesgcm_ghash_precompute *ghash_precompute = ctx->ghash + blocks; - gfmul_firststep(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + blocks; + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; - while (ghash_precompute != ctx->ghash) { - gfmul_nextstep(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + while (ghash_precompute != ctx->ghash128) { + gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; } - gfmul_reduce(&gstate); + gfmul_reduce128(&gstate); } /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { - bits5 = _mm_xor_si128(ek0, ctx->ecb.keys[0]); + bits5 = _mm_xor_si128(ek0, ctx->ecb.keys.m128[0]); for (size_t i = 1; i < ctx->ecb.rounds; ++i) - bits5 = _mm_aesenc_si128(bits5, ctx->ecb.keys[i]); - bits5 = _mm_aesenclast_si128(bits5, ctx->ecb.keys[ctx->ecb.rounds]); + bits5 = _mm_aesenc_si128(bits5, ctx->ecb.keys.m128[i]); + bits5 = _mm_aesenclast_si128(bits5, ctx->ecb.keys.m128[ctx->ecb.rounds]); } /* append tag to encbuf */ - _mm_storeu_si128((void *)encp, gfmul_get_tag(&gstate, bits5)); + _mm_storeu_si128((void *)encp, gfmul_get_tag128(&gstate, bits5)); encp += 16; - { /* Write encbuf, using NT store instructions in 64-byte chunks. Last partial block, if any, is written to cache, as that cache - * line would likely be read when the next TLS record is being built. */ - const uint8_t *s = encbuf; - uint8_t *d = output; - for (; encp - s >= 64; d += 64, s += 64) { - _mm256_stream_si256((void *)d, _mm256_load_si256((void *)s)); - _mm256_stream_si256((void *)(d + 32), _mm256_load_si256((void *)(s + 32))); + /* write remaining bytes */ + write_remaining_bytes(output, encbuf, encp); + +#undef AESECB6_INIT +#undef AESECB6_UPDATE +#undef AESECB6_FINAL +#undef STATE_EK0_READY +#undef STATE_COPY_128B +} + +NO_SANITIZE_ADDRESS +static void fastls_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, + const void *_aad, size_t aadlen) +{ +/* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ +#define AESECB6_INIT() \ + do { \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits0 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits1 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits2 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits3 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits4 = _mm256_shuffle_epi8(ctr, byteswap256); \ + ctr = _mm256_add_epi64(ctr, incr128x2); \ + bits5 = _mm256_shuffle_epi8(ctr, byteswap256); \ + if (PTLS_UNLIKELY(srclen <= 32 * 6 - 16) && src_vecleft == 0) { \ + bits5 = _mm256_insertf128_si256(bits5, ek0, 1); \ + state |= STATE_EK0_READY; \ + } \ + __m256i k = ctx->ecb.keys.m256[0]; \ + bits0 = _mm256_xor_si256(bits0, k); \ + bits1 = _mm256_xor_si256(bits1, k); \ + bits2 = _mm256_xor_si256(bits2, k); \ + bits3 = _mm256_xor_si256(bits3, k); \ + bits4 = _mm256_xor_si256(bits4, k); \ + bits5 = _mm256_xor_si256(bits5, k); \ + } while (0) + +/* aes block update */ +#define AESECB6_UPDATE(i) \ + do { \ + __m256i k = ctx->ecb.keys.m256[i]; \ + bits0 = _mm256_aesenc_epi128(bits0, k); \ + bits1 = _mm256_aesenc_epi128(bits1, k); \ + bits2 = _mm256_aesenc_epi128(bits2, k); \ + bits3 = _mm256_aesenc_epi128(bits3, k); \ + bits4 = _mm256_aesenc_epi128(bits4, k); \ + bits5 = _mm256_aesenc_epi128(bits5, k); \ + } while (0) + +/* aesenclast */ +#define AESECB6_FINAL(i) \ + do { \ + __m256i k = ctx->ecb.keys.m256[i]; \ + bits0 = _mm256_aesenclast_epi128(bits0, k); \ + bits1 = _mm256_aesenclast_epi128(bits1, k); \ + bits2 = _mm256_aesenclast_epi128(bits2, k); \ + bits3 = _mm256_aesenclast_epi128(bits3, k); \ + bits4 = _mm256_aesenclast_epi128(bits4, k); \ + bits5 = _mm256_aesenclast_epi128(bits5, k); \ + } while (0) + + struct aesgcm_context *agctx = (void *)_ctx; + uint8_t *output = _output; + const uint8_t *aad = _aad; + +#define STATE_EK0_READY 0x1 + int32_t state = 0; + + /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ + uint8_t encbuf[32 * 9] __attribute__((aligned(32))), *encp; + + /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is + * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ + PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 32 + 16); + + /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ + if ((encp = encbuf + ((uintptr_t)output & 63)) != encbuf) { + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(output - (encp - encbuf)))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(output - (encp - encbuf) + 32))); + output -= encp - encbuf; + } + + /* setup ctr, retaining Ek(0), len(A) | len(C) to be fed into GCM */ + __m256i ctr = _mm256_broadcastsi128_si256(calc_counter(agctx, seq)); + ctr = _mm256_insert_epi32(ctr, 1, 4); + __m128i ek0 = _mm_shuffle_epi8(_mm256_castsi256_si128(_mm256_permute2f128_si256(ctr, ctr, 0x81)), byteswap128); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128); + + ptls_fusion_aesgcm_context_t *ctx = agctx->aesgcm; + __m256i bits0, bits1, bits2, bits3, bits4, bits5 = _mm256_setzero_si256(); + struct ptls_fusion_gfmul_state256 gstate = {0}; + + /* find the first non-empty vec */ + const uint8_t *src = NULL; + size_t srclen = 0, src_vecleft = incnt; + while (srclen == 0 && src_vecleft != 0) { + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + + /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ + AESECB6_INIT(); + AESECB6_UPDATE(1); + AESECB6_UPDATE(2); + if (PTLS_LIKELY(aadlen != 0)) { + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute; + while (PTLS_UNLIKELY(aadlen >= 6 * 32)) { + ghash_precompute = ctx->ghash256 + 6; + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); + aad += 32; + aadlen -= 32; + for (int i = 1; i < 6; ++i) { + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)aad), --ghash_precompute); + aad += 32; + aadlen -= 32; + } + gfmul_reduce256(&gstate); } - _mm_sfence(); /* weakly ordered writes have to be synced before being passed to NIC */ - if (s != encp) { - for (; encp - s >= 16; d += 16, s += 16) - _mm_store_si128((void *)d, _mm_load_si128((void *)s)); - if (s != encp) - storen((void *)d, encp - s, loadn((void *)s, encp - s)); + if (PTLS_LIKELY(aadlen != 0)) { + ghash_precompute = ctx->ghash256 + (aadlen + 31) / 32; + if (PTLS_UNLIKELY(aadlen >= 32)) { + if (aadlen % 32 == 0 || aadlen % 32 > 16) { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); + aad += 32; + aadlen -= 32; + } else { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 1, --ghash_precompute); + aad += 16; + aadlen -= 16; + } + while (aadlen >= 32) { + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)aad), --ghash_precompute); + aad += 32; + aadlen -= 32; + } + if (PTLS_LIKELY(aadlen != 0)) { + assert(aadlen > 16); + gfmul_nextstep256(&gstate, loadn256(aad, aadlen), --ghash_precompute); + } + } else { + gfmul_firststep256(&gstate, loadn256(aad, aadlen), aadlen <= 16, --ghash_precompute); + } + assert(ctx->ghash256 == ghash_precompute); + gfmul_reduce256(&gstate); } } + for (size_t i = 3; i < ctx->ecb.rounds; ++i) + AESECB6_UPDATE(i); + AESECB6_FINAL(ctx->ecb.rounds); + + /* Main loop. This loop: + * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, + * 2. then if there is no more data to be encrypted, exit the loop, otherwise, + * 3. calculate ghash of the blocks being written to encbuf, + * 4. calculate next 6 * 16 bytes of keystream, + * 5. writes encbuf in 64-byte blocks + * When exitting the loop, `remaining_ghash_from` represents the offset within `encbuf` from where ghash remains to be + * calculated. */ + size_t remaining_ghash_from = encp - encbuf; + if (srclen != 0) { + while (1) { + /* apply the bit stream to input, writing to encbuf */ + if (PTLS_LIKELY(srclen >= 6 * 32)) { +#define APPLY(i) _mm256_storeu_si256((void *)(encp + i * 32), _mm256_xor_si256(_mm256_loadu_si256((void *)(src + i * 32)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encp += 6 * 32; + src += 6 * 32; + srclen -= 6 * 32; + if (PTLS_UNLIKELY(srclen == 0)) { + if (src_vecleft == 0) { + remaining_ghash_from = (encp - encbuf) - 6 * 32; + break; + } + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + } else { + /* slow path, load at most 6 * 32 bytes to encbuf then encrypt in-place */ + size_t bytes_copied = 0; + do { + if (srclen >= 32 && bytes_copied < 5 * 32) { + _mm256_storeu_si256((void *)(encp + bytes_copied), _mm256_loadu_si256((void *)src)); + bytes_copied += 32; + src += 32; + srclen -= 32; + } else { + encp[bytes_copied++] = *src++; + --srclen; + } + if (PTLS_UNLIKELY(srclen == 0)) { + if (src_vecleft == 0) { + break; + } else { + src = (void *)input[0].base; + srclen = input[0].len; + ++input; + --src_vecleft; + } + } + } while (bytes_copied < 6 * 32); +#define APPLY(i) \ + _mm256_storeu_si256((void *)(encp + i * 32), _mm256_xor_si256(_mm256_loadu_si256((void *)(encp + i * 32)), bits##i)) + APPLY(0); + APPLY(1); + APPLY(2); + APPLY(3); + APPLY(4); + APPLY(5); +#undef APPLY + encp += bytes_copied; + if (PTLS_UNLIKELY(srclen == 0)) { + /* Calculate amonut of data left to be ghashed, as well as zero-clearing the remainedr of partial block, as it + * will be fed into ghash. */ + remaining_ghash_from = (encp - encbuf) - bytes_copied; + if ((bytes_copied & 15) != 0) + _mm_storeu_si128((void *)encp, _mm_setzero_si128()); + break; + } + } + + /* Next 96-byte block starts here. Run AES and ghash in parallel while writing output using non-temporal store + * instructions. */ + AESECB6_INIT(); + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash256 + 6; + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encp - 6 * 32)), 0, --ghash_precompute); + AESECB6_UPDATE(1); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 5 * 32)), --ghash_precompute); + AESECB6_UPDATE(2); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 4 * 32)), --ghash_precompute); + AESECB6_UPDATE(3); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 3 * 32)), --ghash_precompute); + AESECB6_UPDATE(4); + _mm256_stream_si256((void *)output, _mm256_load_si256((void *)encbuf)); + _mm256_stream_si256((void *)(output + 32), _mm256_load_si256((void *)(encbuf + 32))); + _mm256_stream_si256((void *)(output + 64), _mm256_load_si256((void *)(encbuf + 64))); + _mm256_stream_si256((void *)(output + 96), _mm256_load_si256((void *)(encbuf + 96))); + _mm256_stream_si256((void *)(output + 128), _mm256_load_si256((void *)(encbuf + 128))); + _mm256_stream_si256((void *)(output + 160), _mm256_load_si256((void *)(encbuf + 160))); + AESECB6_UPDATE(5); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 2 * 32)), --ghash_precompute); + AESECB6_UPDATE(6); + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 1 * 32)), --ghash_precompute); + output += 192; + encp -= 192; + AESECB6_UPDATE(7); + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(encbuf + 192))); + AESECB6_UPDATE(8); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 224))); + AESECB6_UPDATE(9); + if (PTLS_UNLIKELY(ctx->ecb.rounds != 10)) { + for (size_t i = 10; PTLS_LIKELY(i < ctx->ecb.rounds); ++i) + AESECB6_UPDATE(i); + } + assert(ctx->ghash256 == ghash_precompute); + gfmul_reduce256(&gstate); + AESECB6_FINAL(ctx->ecb.rounds); + } + } + + /* Now, All the encrypted bits are built in encbuf. Calculate AEAD tag and append to encbuf. */ + + { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 + * blocks at once. */ + size_t ac_off = remaining_ghash_from + ((encp - encbuf) - remaining_ghash_from + 15) / 16 * 16; + _mm_storeu_si128((void *)(encbuf + ac_off), ac); + size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ + assert(blocks <= 13); + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash256 + blocks / 2; + if (blocks % 2 != 0) { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 1, ghash_precompute); + remaining_ghash_from += 16; + } else { + gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 0, --ghash_precompute); + remaining_ghash_from += 32; + } + while (ghash_precompute != ctx->ghash256) { + gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); + remaining_ghash_from += 32; + } + gfmul_reduce256(&gstate); + } + + /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ + if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { + bits5 = _mm256_insertf128_si256(bits5, ek0, 1); + bits5 = _mm256_xor_si256(bits5, ctx->ecb.keys.m256[0]); + for (size_t i = 1; i < ctx->ecb.rounds; ++i) + bits5 = _mm256_aesenc_epi128(bits5, ctx->ecb.keys.m256[i]); + bits5 = _mm256_aesenclast_epi128(bits5, ctx->ecb.keys.m256[ctx->ecb.rounds]); + } + + /* append tag to encbuf */ + _mm_storeu_si128((void *)encp, + gfmul_get_tag256(&gstate, _mm256_castsi256_si128(_mm256_permute2f128_si256(bits5, bits5, 0x11)))); + encp += 16; + + /* write remaining bytes */ + write_remaining_bytes(output, encbuf, encp); } static int fastls_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; - ctx->static_iv = loadn(iv, PTLS_AESGCM_IV_SIZE); - ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, bswap8); + ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); + ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); if (key == NULL) return 0; ctx->super.dispose_crypto = aesgcm_dispose_crypto; ctx->super.do_xor_iv = aesgcm_xor_iv; ctx->super.do_encrypt = ptls_aead__do_encrypt; - ctx->super.do_encrypt_v = fastls_encrypt_v; + ctx->super.do_encrypt_v = ptls_fusion_can_avx256 ? fastls_encrypt_v256 : fastls_encrypt_v128; ctx->super.do_decrypt = NULL; /* FIXME */ - ctx->aesgcm = ptls_fusion_aesgcm_new(key, key_size, 7 * 16 /* 6 blocks at once, plus len(A) | len(C) that we might append */); + ctx->aesgcm = new_aesgcm(key, key_size, + 7 * (ptls_fusion_can_avx256 ? 32 : 16), // 6 blocks at once, plus len(A) | len(C) that we might append + ptls_fusion_can_avx256); return 0; } @@ -1435,11 +1899,16 @@ int ptls_fusion_is_supported_by_cpu(void) leaf1_ecx = cpu_info[2]; if (/* PCLMUL */ (leaf1_ecx & (1 << 5)) != 0 && /* AES */ (leaf1_ecx & (1 << 25)) != 0) { - uint32_t leaf7_ebx; + uint32_t leaf7_ebx, leaf7_ecx; __cpuid(cpu_info, 7); leaf7_ebx = cpu_info[1]; + leaf7_ecx = cpu_info[2]; is_supported = /* AVX2 */ (leaf7_ebx & (1 << 5)) != 0; + + /* enable 256-bit mode if possible */ + if (is_supported && (leaf7_ecx & 0x600) != 0 && !ptls_fusion_avx256) + ptls_fusion_can_avx256 = 1; } } @@ -1448,7 +1917,7 @@ int ptls_fusion_is_supported_by_cpu(void) #else int ptls_fusion_is_supported_by_cpu(void) { - unsigned leaf1_ecx, leaf7_ebx; + unsigned leaf1_ecx, leaf7_ebx, leaf7_ecx; { /* GCC-specific code to obtain CPU features */ unsigned leaf_cnt; @@ -1456,7 +1925,7 @@ int ptls_fusion_is_supported_by_cpu(void) if (leaf_cnt < 7) return 0; __asm__("cpuid" : "=c"(leaf1_ecx) : "a"(1) : "ebx", "edx"); - __asm__("cpuid" : "=b"(leaf7_ebx) : "a"(7), "c"(0) : "edx"); + __asm__("cpuid" : "=b"(leaf7_ebx), "=c"(leaf7_ecx) : "a"(7), "c"(0) : "edx"); } /* AVX2 */ @@ -1469,6 +1938,10 @@ int ptls_fusion_is_supported_by_cpu(void) if ((leaf1_ecx & (1 << 1)) == 0) return 0; + /* enable 256-bit mode if possible */ + if ((leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_avx256) + ptls_fusion_can_avx256 = 1; + return 1; } #endif diff --git a/picotls.xcodeproj/project.pbxproj b/picotls.xcodeproj/project.pbxproj index 27453aa40..2887b4f7a 100644 --- a/picotls.xcodeproj/project.pbxproj +++ b/picotls.xcodeproj/project.pbxproj @@ -1286,7 +1286,11 @@ isa = XCBuildConfiguration; buildSettings = { GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; - OTHER_CFLAGS = "-march=native"; + OTHER_CFLAGS = ( + "-march=native", + "-mvaes", + "-mvpclmulqdq", + ); OTHER_LDFLAGS = ""; PRODUCT_NAME = "$(TARGET_NAME)"; }; @@ -1296,7 +1300,11 @@ isa = XCBuildConfiguration; buildSettings = { GCC_PREPROCESSOR_DEFINITIONS = "$(inherited)"; - OTHER_CFLAGS = "-march=native"; + OTHER_CFLAGS = ( + "-march=native", + "-mvaes", + "-mvpclmulqdq", + ); OTHER_LDFLAGS = ""; PRODUCT_NAME = "$(TARGET_NAME)"; }; diff --git a/t/fusion.c b/t/fusion.c index 57d2b45a0..fba7ba40e 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -49,14 +49,14 @@ static const char *tostr(const void *_p, size_t len) return buf; } -static void test_loadn(void) +static void test_loadn128(void) { uint8_t buf[8192] = {0}; for (size_t off = 0; off < 8192 - 15; ++off) { uint8_t *src = buf + off; memcpy(src, "hello world12345", 16); - __m128i v = loadn(src, 11); + __m128i v = loadn128(src, 11); if (memcmp(&v, "hello world\0\0\0\0\0", 16) != 0) { ok(!"fail"); return; @@ -73,17 +73,151 @@ static void test_ecb(void) ptls_fusion_aesecb_context_t ecb; uint8_t encrypted[16]; - ptls_fusion_aesecb_init(&ecb, 1, zero, 16); + ptls_fusion_aesecb_init(&ecb, 1, zero, 16, 0); ptls_fusion_aesecb_encrypt(&ecb, encrypted, "hello world!!!!!"); ptls_fusion_aesecb_dispose(&ecb); ok(strcmp(tostr(encrypted, 16), "172afecb50b5f1237814b2f7cb51d0f7") == 0); - ptls_fusion_aesecb_init(&ecb, 1, zero, 32); + ptls_fusion_aesecb_init(&ecb, 1, zero, 32, 0); ptls_fusion_aesecb_encrypt(&ecb, encrypted, "hello world!!!!!"); ptls_fusion_aesecb_dispose(&ecb); ok(strcmp(tostr(encrypted, 16), "2a033f0627b3554aa4fe5786550736ff") == 0); } +static void test_gfmul(void) +{ + ptls_fusion_aesgcm_context_t *ctx; + PTLS_BUILD_ASSERT(sizeof(ctx->ghash128[0]) * 2 == sizeof(ctx->ghash256[0])); + +#define COUNT 4 + + ctx = malloc(offsetof(ptls_fusion_aesgcm_context_t, ghash128) + sizeof(ctx->ghash128[0]) * COUNT); + memset(ctx, 0, offsetof(ptls_fusion_aesgcm_context_t, ghash128)); + ctx->capacity = 4 * 16; + ctx->ecb.avx256 = ptls_fusion_can_avx256; + + __m128i H0 = _mm_loadu_si128((void *)"hello world bye"); + if (ctx->ecb.avx256) { + ctx->ghash256[0].H[1] = H0; + } else { + ctx->ghash128[0].H = H0; + } + + while (ctx->ghash_cnt < COUNT) + setup_one_ghash_entry(ctx); + + { /* one block */ + static const char input[32] = "deaddeadbeefbeef"; /* latter 16-byte is NUL */ + __m128i hash; + if (ctx->ecb.avx256) { + struct ptls_fusion_gfmul_state256 state = {}; + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, ctx->ghash256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state = {}; + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\x12\xd9\xd9\x14\x8b\x3f\x20\xbd\x20\x2a\xa5\x9e\x17\xa8\xb0\x7b", 16) == 0); + } + + { /* two blocks */ + static const char input[32] = "Lorem ipsum dolor sit amet, con"; + __m128i hash; + if (ctx->ecb.avx256) { + struct ptls_fusion_gfmul_state256 state = {}; + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state = {}; + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\xda\xdf\xe8\x9b\xc7\x8c\xbd\x5c\xa7\xc1\x83\x9a\xa2\x9f\x80\x55", 16) == 0); + } + + { /* three blocks */ + static const char input[64] = "The quick brown fox jumps over the lazy dog."; + __m128i hash; + if (ctx->ecb.avx256) { + struct ptls_fusion_gfmul_state256 state = {}; + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, ctx->ghash256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 16)), ctx->ghash256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state = {}; + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\xad\xdf\x91\x52\x38\x40\xf7\xc3\x85\xaf\x41\xb1\x7d\xed\x4b\x56", 16) == 0); + } + + { /* five blocks */ + static const char input[80] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor "; + __m128i hash; + if (ctx->ecb.avx256) { + struct ptls_fusion_gfmul_state256 state = {}; + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), ctx->ghash256); + gfmul_reduce256(&state); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 1, ctx->ghash256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state = {}; + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 3); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), ctx->ghash128); + gfmul_reduce128(&state); + gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), ctx->ghash128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\xb8\xab\x1b\xa8\xf2\x92\xf3\x89\x44\x9d\x39\xf6\xb6\x37\xca\x5d", 16) == 0); + } + + { /* six blocks */ + static const char input[96] = + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut la"; + __m128i hash; + if (ctx->ecb.avx256) { + struct ptls_fusion_gfmul_state256 state = {}; + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), ctx->ghash256); + gfmul_reduce256(&state); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 0, ctx->ghash256); + gfmul_reduce256(&state); + hash = _mm256_castsi256_si128(state.lo); + } else { + struct ptls_fusion_gfmul_state128 state = {}; + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 3); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), ctx->ghash128); + gfmul_reduce128(&state); + gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), ctx->ghash128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 80)), ctx->ghash128); + gfmul_reduce128(&state); + hash = state.lo; + } + ok(memcmp(&hash, "\x52\xce\x25\x22\x86\x2c\x91\xa4\xe7\x4e\xf9\x9a\x32\x77\xbd\x3e", 16) == 0); + } + + free(ctx); + +#undef COUNT +} + static void gcm_basic(void) { { @@ -331,15 +465,27 @@ int main(int argc, char **argv) note("CPU does have the necessary features (avx2, aes, pclmul)\n"); return done_testing(); } + int can256bit = ptls_fusion_can_avx256; + ptls_fusion_can_avx256 = 0; - subtest("loadn", test_loadn); + subtest("loadn128", test_loadn128); subtest("ecb", test_ecb); + subtest("gfmul128", test_gfmul); subtest("gcm-basic", gcm_basic); subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); subtest("aesgcm", test_aesgcm); - subtest("fastls", test_fastls); + subtest("fastls128", test_fastls); + + if (can256bit) { + ptls_fusion_can_avx256 = 1; + subtest("gfmul256", test_gfmul); + subtest("fastls256", test_fastls); + ptls_fusion_can_avx256 = 0; + } else { + note("gfmul256: skipping, CPU does not support 256-bit aes / clmul"); + } return done_testing(); } From d1a09127e56cd82639f6ef447e8ba1dd3f4a08a3 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 16:06:36 +0900 Subject: [PATCH 21/48] update copyright --- lib/fusion.c | 2 +- t/fusion.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 21dcacfa7..43f1344b4 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -17,7 +17,7 @@ * All other work, including modifications to the `transformH` function is * covered by the following MIT license: * - * Copyright (c) 2020 Fastly, Kazuho Oku + * Copyright (c) 2020-2022 Fastly, Kazuho Oku * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to diff --git a/t/fusion.c b/t/fusion.c index fba7ba40e..deb5b2718 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -475,7 +475,7 @@ int main(int argc, char **argv) subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); - subtest("aesgcm", test_aesgcm); + // subtest("aesgcm", test_aesgcm); subtest("fastls128", test_fastls); if (can256bit) { From 34e9b2d03c02723ac6be1995b0a2ac0194e72ccd Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 17:04:02 +0900 Subject: [PATCH 22/48] less sexy name --- include/picotls/fusion.h | 2 +- lib/fusion.c | 60 ++++++++++++++++++++-------------------- t/fusion.c | 2 +- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/include/picotls/fusion.h b/include/picotls/fusion.h index 675c2ee8c..dff299753 100644 --- a/include/picotls/fusion.h +++ b/include/picotls/fusion.h @@ -97,7 +97,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, extern int ptls_fusion_can_avx256; extern ptls_cipher_algorithm_t ptls_fusion_aes128ctr, ptls_fusion_aes256ctr; extern ptls_aead_algorithm_t ptls_fusion_aes128gcm, ptls_fusion_aes256gcm; -extern ptls_aead_algorithm_t ptls_fastls_aes128gcm, ptls_fastls_aes256gcm; +extern ptls_aead_algorithm_t ptls_non_temporal_aes128gcm, ptls_non_temporal_aes256gcm; /** * Returns a boolean indicating if fusion can be used. diff --git a/lib/fusion.c b/lib/fusion.c index 43f1344b4..d5f54d434 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1212,8 +1212,8 @@ static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const } NO_SANITIZE_ADDRESS -static void fastls_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, - const void *_aad, size_t aadlen) +static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, + uint64_t seq, const void *_aad, size_t aadlen) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ @@ -1519,8 +1519,8 @@ static void fastls_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_outp } NO_SANITIZE_ADDRESS -static void fastls_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, - const void *_aad, size_t aadlen) +static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, + uint64_t seq, const void *_aad, size_t aadlen) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ @@ -1823,7 +1823,7 @@ static void fastls_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void *_outp write_remaining_bytes(output, encbuf, encp); } -static int fastls_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) +static int nt_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; @@ -1835,7 +1835,7 @@ static int fastls_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, ctx->super.dispose_crypto = aesgcm_dispose_crypto; ctx->super.do_xor_iv = aesgcm_xor_iv; ctx->super.do_encrypt = ptls_aead__do_encrypt; - ctx->super.do_encrypt_v = ptls_fusion_can_avx256 ? fastls_encrypt_v256 : fastls_encrypt_v128; + ctx->super.do_encrypt_v = ptls_fusion_can_avx256 ? non_temporal_encrypt_v256 : non_temporal_encrypt_v128; ctx->super.do_decrypt = NULL; /* FIXME */ ctx->aesgcm = new_aesgcm(key, key_size, @@ -1845,36 +1845,36 @@ static int fastls_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, return 0; } -static int fastls_aes128gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) +static int non_temporal_aes128gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { - return fastls_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); + return nt_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); } -static int fastls_aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) +static int non_temporal_aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { - return fastls_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); + return nt_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); } -ptls_aead_algorithm_t ptls_fastls_aes128gcm = {"AES128-GCM", - PTLS_AESGCM_CONFIDENTIALITY_LIMIT, - PTLS_AESGCM_INTEGRITY_LIMIT, - &ptls_fusion_aes128ctr, - NULL, // &ptls_fusion_aes128ecb, - PTLS_AES128_KEY_SIZE, - PTLS_AESGCM_IV_SIZE, - PTLS_AESGCM_TAG_SIZE, - sizeof(struct aesgcm_context), - fastls_aes128gcm_setup}; -ptls_aead_algorithm_t ptls_fastls_aes256gcm = {"AES256-GCM", - PTLS_AESGCM_CONFIDENTIALITY_LIMIT, - PTLS_AESGCM_INTEGRITY_LIMIT, - &ptls_fusion_aes256ctr, - NULL, // &ptls_fusion_aes128ecb, - PTLS_AES256_KEY_SIZE, - PTLS_AESGCM_IV_SIZE, - PTLS_AESGCM_TAG_SIZE, - sizeof(struct aesgcm_context), - fastls_aes256gcm_setup}; +ptls_aead_algorithm_t ptls_non_temporal_aes128gcm = {"AES128-GCM", + PTLS_AESGCM_CONFIDENTIALITY_LIMIT, + PTLS_AESGCM_INTEGRITY_LIMIT, + &ptls_fusion_aes128ctr, + NULL, // &ptls_fusion_aes128ecb, + PTLS_AES128_KEY_SIZE, + PTLS_AESGCM_IV_SIZE, + PTLS_AESGCM_TAG_SIZE, + sizeof(struct aesgcm_context), + non_temporal_aes128gcm_setup}; +ptls_aead_algorithm_t ptls_non_temporal_aes256gcm = {"AES256-GCM", + PTLS_AESGCM_CONFIDENTIALITY_LIMIT, + PTLS_AESGCM_INTEGRITY_LIMIT, + &ptls_fusion_aes256ctr, + NULL, // &ptls_fusion_aes128ecb, + PTLS_AES256_KEY_SIZE, + PTLS_AESGCM_IV_SIZE, + PTLS_AESGCM_TAG_SIZE, + sizeof(struct aesgcm_context), + non_temporal_aes256gcm_setup}; #ifdef _WINDOWS /** diff --git a/t/fusion.c b/t/fusion.c index deb5b2718..b29cb8e14 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -456,7 +456,7 @@ static void test_aesgcm(void) static void test_fastls(void) { - test_generated(&ptls_fastls_aes128gcm, &ptls_minicrypto_aes128gcm, 0); + test_generated(&ptls_non_temporal_aes128gcm, &ptls_minicrypto_aes128gcm, 0); } int main(int argc, char **argv) From 0a8330b578caf088bf672b7abe73407f3335fde3 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 17:21:08 +0900 Subject: [PATCH 23/48] oops again --- t/fusion.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/fusion.c b/t/fusion.c index b29cb8e14..9ac67f3d5 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -475,7 +475,7 @@ int main(int argc, char **argv) subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); - // subtest("aesgcm", test_aesgcm); + subtest("aesgcm", test_aesgcm); subtest("fastls128", test_fastls); if (can256bit) { From a7006dc4db0d69412a9507af062ed05841478c26 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 17:24:31 +0900 Subject: [PATCH 24/48] refactor --- lib/fusion.c | 78 ++++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index d5f54d434..921bdd74b 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1193,6 +1193,46 @@ static inline size_t calc_total_length(ptls_iovec_t *input, size_t incnt) return totlen; } +static inline void reduce_aad128(struct ptls_fusion_gfmul_state128 *gstate, struct ptls_fusion_aesgcm_ghash_precompute128 *ghash, + const void *_aad, size_t aadlen) +{ + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute; + const uint8_t *aad = _aad; + + while (PTLS_UNLIKELY(aadlen >= 6 * 16)) { + ghash_precompute = ghash + 6; + gfmul_firststep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + for (int i = 1; i < 6; ++i) { + gfmul_nextstep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + } + gfmul_reduce128(gstate); + } + + if (PTLS_LIKELY(aadlen != 0)) { + ghash_precompute = ghash + (aadlen + 15) / 16; + if (PTLS_UNLIKELY(aadlen >= 16)) { + gfmul_firststep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + while (aadlen >= 16) { + gfmul_nextstep128(gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); + aad += 16; + aadlen -= 16; + } + if (PTLS_LIKELY(aadlen != 0)) + gfmul_nextstep128(gstate, loadn128(aad, aadlen), --ghash_precompute); + } else { + gfmul_firststep128(gstate, loadn128(aad, aadlen), --ghash_precompute); + } + assert(ghash == ghash_precompute); + gfmul_reduce128(gstate); + } +} + static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const uint8_t *end) { /* Write in 64-byte chunks, using NT store instructions. Last partial block, if any, is written to cache, as that cache line @@ -1213,7 +1253,7 @@ static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const NO_SANITIZE_ADDRESS static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, - uint64_t seq, const void *_aad, size_t aadlen) + uint64_t seq, const void *aad, size_t aadlen) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ #define AESECB6_INIT() \ @@ -1270,7 +1310,6 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void struct aesgcm_context *agctx = (void *)_ctx; uint8_t *output = _output; - const uint8_t *aad = _aad; #define STATE_EK0_READY 0x1 #define STATE_COPY_128B 0x2 @@ -1317,40 +1356,7 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void AESECB6_INIT(); AESECB6_UPDATE(1); AESECB6_UPDATE(2); - if (PTLS_LIKELY(aadlen != 0)) { - struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute; - while (PTLS_UNLIKELY(aadlen >= 6 * 16)) { - ghash_precompute = ctx->ghash128 + 6; - gfmul_firststep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); - aad += 16; - aadlen -= 16; - for (int i = 1; i < 6; ++i) { - gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); - aad += 16; - aadlen -= 16; - } - gfmul_reduce128(&gstate); - } - if (PTLS_LIKELY(aadlen != 0)) { - ghash_precompute = ctx->ghash128 + (aadlen + 15) / 16; - if (PTLS_UNLIKELY(aadlen >= 16)) { - gfmul_firststep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); - aad += 16; - aadlen -= 16; - while (aadlen >= 16) { - gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)aad), --ghash_precompute); - aad += 16; - aadlen -= 16; - } - if (PTLS_LIKELY(aadlen != 0)) - gfmul_nextstep128(&gstate, loadn128(aad, aadlen), --ghash_precompute); - } else { - gfmul_firststep128(&gstate, loadn128(aad, aadlen), --ghash_precompute); - } - assert(ctx->ghash128 == ghash_precompute); - gfmul_reduce128(&gstate); - } - } + reduce_aad128(&gstate, ctx->ghash128, aad, aadlen); for (size_t i = 3; i < ctx->ecb.rounds; ++i) AESECB6_UPDATE(i); AESECB6_FINAL(ctx->ecb.rounds); From fd7d5c16ef3a5e2df545cbc6260ebb95d468ec09 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 21:37:38 +0900 Subject: [PATCH 25/48] reduce register usage --- lib/fusion.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 921bdd74b..289e12a72 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1544,7 +1544,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void ctr = _mm256_add_epi64(ctr, incr128x2); \ bits5 = _mm256_shuffle_epi8(ctr, byteswap256); \ if (PTLS_UNLIKELY(srclen <= 32 * 6 - 16) && src_vecleft == 0) { \ - bits5 = _mm256_insertf128_si256(bits5, ek0, 1); \ + bits5 = _mm256_permute2f128_si256(bits5, ac_ek0, 0x30); \ state |= STATE_EK0_READY; \ } \ __m256i k = ctx->ecb.keys.m256[0]; \ @@ -1604,8 +1604,12 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void /* setup ctr, retaining Ek(0), len(A) | len(C) to be fed into GCM */ __m256i ctr = _mm256_broadcastsi128_si256(calc_counter(agctx, seq)); ctr = _mm256_insert_epi32(ctr, 1, 4); - __m128i ek0 = _mm_shuffle_epi8(_mm256_castsi256_si128(_mm256_permute2f128_si256(ctr, ctr, 0x81)), byteswap128); - __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128); + __m256i ac_ek0 = _mm256_permute2f128_si256( + /* first half: ac */ + _mm256_castsi128_si256( + _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128)), + /* second half: ek0 */ + _mm256_shuffle_epi8(ctr, byteswap256), 0x30); ptls_fusion_aesgcm_context_t *ctx = agctx->aesgcm; __m256i bits0, bits1, bits2, bits3, bits4, bits5 = _mm256_setzero_si256(); @@ -1793,7 +1797,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void { /* Run ghash against the remaining bytes, after appending `ac` (i.e., len(A) | len(C)). At this point, we might be ghashing 7 * blocks at once. */ size_t ac_off = remaining_ghash_from + ((encp - encbuf) - remaining_ghash_from + 15) / 16 * 16; - _mm_storeu_si128((void *)(encbuf + ac_off), ac); + _mm_storeu_si128((void *)(encbuf + ac_off), _mm256_castsi256_si128(ac_ek0)); size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 13); union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash256 + blocks / 2; @@ -1813,7 +1817,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { - bits5 = _mm256_insertf128_si256(bits5, ek0, 1); + bits5 = ac_ek0; bits5 = _mm256_xor_si256(bits5, ctx->ecb.keys.m256[0]); for (size_t i = 1; i < ctx->ecb.rounds; ++i) bits5 = _mm256_aesenc_epi128(bits5, ctx->ecb.keys.m256[i]); From 93b04e13bf8c489569fd9dbe33a38a6fa59b271c Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 9 May 2022 23:51:18 +0900 Subject: [PATCH 26/48] quick hack for GCC on Intel --- lib/fusion.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/fusion.c b/lib/fusion.c index 289e12a72..c25132f7b 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1315,7 +1315,13 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void #define STATE_COPY_128B 0x2 int32_t state = 0; - /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ + /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. + * Use of thread-local stroage is a hack. GCC spills a lot onto stack, and unless we move `encbuf` outside of stack, temporary + * variables end up being stored onto somewhere not as fast as places near the stack pointer. This provides 18% speed + * improvement on Core i5 9400, at the cost of 2% slowdown on Zen 3. */ +#if defined(__GNUC__) && !defined(__clang__) + static __thread +#endif uint8_t encbuf[32 * 6] __attribute__((aligned(32))), *encp; /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is From 224fa36dcd0e861da21c4fd97e4724abbcccae61 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 06:59:45 +0900 Subject: [PATCH 27/48] don't forget the union --- lib/fusion.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index c25132f7b..fe303bd3a 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -63,20 +63,22 @@ struct ptls_fusion_aesgcm_context { ptls_fusion_aesecb_context_t ecb; size_t capacity; size_t ghash_cnt; - struct ptls_fusion_aesgcm_ghash_precompute128 { - __m128i H; - __m128i r; - } ghash128[0]; - union ptls_fusion_aesgcm_ghash_precompute256 { - struct { - __m128i H[2]; - __m128i r[2]; - }; - struct { - __m256i Hx2; - __m256i rx2; - }; - } ghash256[0]; + union { + struct ptls_fusion_aesgcm_ghash_precompute128 { + __m128i H; + __m128i r; + } ghash128[0]; + union ptls_fusion_aesgcm_ghash_precompute256 { + struct { + __m128i H[2]; + __m128i r[2]; + }; + struct { + __m256i Hx2; + __m256i rx2; + }; + } ghash256[0]; + }; }; struct ctr_context { From 7f165e0463de7f716917ed077d12b4aef6e9c935 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 07:04:06 +0900 Subject: [PATCH 28/48] win32/64 fixes --- lib/fusion.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index fe303bd3a..7044bcf7f 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -347,7 +347,7 @@ static inline __m256i loadn256(const void *p, size_t l) if (PTLS_LIKELY(mod4k < 4096 - 32) || mod4k + l > 4096) { v = _mm256_loadu_si256(p); } else if (l > 16) { - __m128i first16 = _mm_loadu_si128(p), second16 = loadn128((uint8_t *)(p + 16), l - 16); + __m128i first16 = _mm_loadu_si128(p), second16 = loadn128((uint8_t *)p + 16, l - 16); v = _mm256_permute2f128_si256(_mm256_castsi128_si256(first16), _mm256_castsi128_si256(second16), 0x20); } else if (l == 16) { v = _mm256_castsi128_si256(_mm_loadu_si128(p)); @@ -1925,7 +1925,7 @@ int ptls_fusion_is_supported_by_cpu(void) is_supported = /* AVX2 */ (leaf7_ebx & (1 << 5)) != 0; /* enable 256-bit mode if possible */ - if (is_supported && (leaf7_ecx & 0x600) != 0 && !ptls_fusion_avx256) + if (is_supported && (leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_avx256) ptls_fusion_can_avx256 = 1; } } From eb3ebf420c042deaf02a3ae3f1048a183f3982bd Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 07:06:17 +0900 Subject: [PATCH 29/48] use _aligned_malloc on windows --- lib/fusion.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/fusion.c b/lib/fusion.c index 7044bcf7f..468ed75d8 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -59,6 +59,10 @@ #define NO_SANITIZE_ADDRESS #endif +#ifdef _WINDOWS +#define aligned_alloc(a, s) _aligned_malloc((s), (a)) +#endif + struct ptls_fusion_aesgcm_context { ptls_fusion_aesecb_context_t ecb; size_t capacity; From 6144b4f8e20224db875f5b329e12498279b437b5 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 08:11:51 +0900 Subject: [PATCH 30/48] windows does not like `= {}` --- t/fusion.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/t/fusion.c b/t/fusion.c index 9ac67f3d5..02d291847 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -110,12 +110,14 @@ static void test_gfmul(void) static const char input[32] = "deaddeadbeefbeef"; /* latter 16-byte is NUL */ __m128i hash; if (ctx->ecb.avx256) { - struct ptls_fusion_gfmul_state256 state = {}; + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, ctx->ghash256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { - struct ptls_fusion_gfmul_state128 state = {}; + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128); gfmul_reduce128(&state); hash = state.lo; @@ -127,12 +129,14 @@ static void test_gfmul(void) static const char input[32] = "Lorem ipsum dolor sit amet, con"; __m128i hash; if (ctx->ecb.avx256) { - struct ptls_fusion_gfmul_state256 state = {}; + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { - struct ptls_fusion_gfmul_state128 state = {}; + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 1); gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128); gfmul_reduce128(&state); @@ -145,13 +149,15 @@ static void test_gfmul(void) static const char input[64] = "The quick brown fox jumps over the lazy dog."; __m128i hash; if (ctx->ecb.avx256) { - struct ptls_fusion_gfmul_state256 state = {}; + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, ctx->ghash256 + 1); gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 16)), ctx->ghash256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { - struct ptls_fusion_gfmul_state128 state = {}; + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 2); gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 1); gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128); @@ -165,7 +171,8 @@ static void test_gfmul(void) static const char input[80] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor "; __m128i hash; if (ctx->ecb.avx256) { - struct ptls_fusion_gfmul_state256 state = {}; + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256 + 1); gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), ctx->ghash256); gfmul_reduce256(&state); @@ -173,7 +180,8 @@ static void test_gfmul(void) gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { - struct ptls_fusion_gfmul_state128 state = {}; + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 3); gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 2); gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128 + 1); @@ -191,7 +199,8 @@ static void test_gfmul(void) "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut la"; __m128i hash; if (ctx->ecb.avx256) { - struct ptls_fusion_gfmul_state256 state = {}; + struct ptls_fusion_gfmul_state256 state; + state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256 + 1); gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), ctx->ghash256); gfmul_reduce256(&state); @@ -199,7 +208,8 @@ static void test_gfmul(void) gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { - struct ptls_fusion_gfmul_state128 state = {}; + struct ptls_fusion_gfmul_state128 state; + state.lo = _mm_setzero_si128(); gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 3); gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 2); gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128 + 1); From 07f37c2eea5ae1cd366c9a194dc96aa69db44674 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 09:45:42 +0900 Subject: [PATCH 31/48] Use dedicated types, as ASAN does not understand flexibly arrays inside union. It's annoying, but using separate types makes the code safer --- lib/fusion.c | 211 ++++++++++++++++++++++++++++----------------------- t/fusion.c | 79 +++++++++---------- 2 files changed, 159 insertions(+), 131 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 468ed75d8..1b414e6c1 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -67,22 +67,28 @@ struct ptls_fusion_aesgcm_context { ptls_fusion_aesecb_context_t ecb; size_t capacity; size_t ghash_cnt; - union { - struct ptls_fusion_aesgcm_ghash_precompute128 { - __m128i H; - __m128i r; - } ghash128[0]; - union ptls_fusion_aesgcm_ghash_precompute256 { - struct { - __m128i H[2]; - __m128i r[2]; - }; - struct { - __m256i Hx2; - __m256i rx2; - }; - } ghash256[0]; - }; +}; + +struct ptls_fusion_aesgcm_context128 { + struct ptls_fusion_aesgcm_context super; + struct ptls_fusion_aesgcm_ghash_precompute128 { + __m128i H; + __m128i r; + } ghash[0]; +}; + +struct ptls_fusion_aesgcm_context256 { + struct ptls_fusion_aesgcm_context super; + union ptls_fusion_aesgcm_ghash_precompute256 { + struct { + __m128i H[2]; + __m128i r[2]; + }; + struct { + __m256i Hx2; + __m256i rx2; + }; + } ghash[0]; }; struct ctr_context { @@ -373,7 +379,7 @@ static inline void storen128(void *_p, size_t l, __m128i v) p[i] = buf[i]; } -void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const void *input, size_t inlen, __m128i ctr, +void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *_ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *_aad, size_t aadlen, ptls_aead_supplementary_encryption_t *supp) { /* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ @@ -403,7 +409,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, state |= STATE_SUPP_IN_PROCESS; \ } \ } \ - __m128i k = ctx->ecb.keys.m128[0]; \ + __m128i k = ctx->super.ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ @@ -415,7 +421,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys.m128[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -427,7 +433,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ - __m128i k = ctx->ecb.keys.m128[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ @@ -436,8 +442,9 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, bits5 = _mm_aesenclast_si128(bits5, k); \ } while (0) + struct ptls_fusion_aesgcm_context128 *ctx = (void *)_ctx; __m128i ek0, bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); - const __m128i *bits4keys = ctx->ecb.keys.m128; /* is changed to supp->ctx.keys when calcurating suppout */ + const __m128i *bits4keys = ctx->super.ecb.keys.m128; /* is changed to supp->ctx.keys when calcurating suppout */ struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); @@ -450,7 +457,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const __m128i *aad = _aad, *dst_ghash = dst; size_t dst_ghashlen = srclen; - struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + (aadlen + 15) / 16 + (srclen + 15) / 16 + 1; + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (srclen + 15) / 16 + 1; #define STATE_EK0_BEEN_FED 0x3 #define STATE_EK0_INCOMPLETE 0x2 @@ -493,7 +500,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, AESECB6_UPDATE(i); gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } - for (; i < ctx->ecb.rounds; ++i) + for (; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); AESECB6_FINAL(i); @@ -619,7 +626,7 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, } do { bits4 = _mm_aesenc_si128(bits4, bits4keys[i++]); - } while (i != ctx->ecb.rounds); + } while (i != ctx->super.ecb.rounds); bits4 = _mm_aesenclast_si128(bits4, bits4keys[i]); _mm_storeu_si128((__m128i *)supp->output, bits4); } @@ -632,15 +639,16 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, #undef STATE_SUPP_IN_PROCESS } -int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const void *input, size_t inlen, __m128i ctr, +int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *_ctx, void *output, const void *input, size_t inlen, __m128i ctr, const void *_aad, size_t aadlen, const void *tag) { + struct ptls_fusion_aesgcm_context128 *ctx = (void *)_ctx; __m128i ek0 = _mm_setzero_si128(), bits0, bits1 = _mm_setzero_si128(), bits2 = _mm_setzero_si128(), bits3 = _mm_setzero_si128(), bits4 = _mm_setzero_si128(), bits5 = _mm_setzero_si128(); struct ptls_fusion_gfmul_state128 gstate = {0}; __m128i gdatabuf[6]; __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); - struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + (aadlen + 15) / 16 + (inlen + 15) / 16 + 1; + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (aadlen + 15) / 16 + (inlen + 15) / 16 + 1; const __m128i *gdata; // points to the elements fed into GHASH size_t gdata_cnt; @@ -651,7 +659,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* schedule ek0 and suppkey */ ctr = _mm_add_epi64(ctr, one8); - bits0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), ctx->ecb.keys.m128[0]); + bits0 = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), ctx->super.ecb.keys.m128[0]); ++nondata_aes_cnt; #define STATE_IS_FIRST_RUN 0x1 @@ -714,19 +722,19 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, ctr = _mm_add_epi64(ctr, one8); \ bits##n = _mm_xor_si128(_mm_shuffle_epi8(ctr, byteswap128), keys.m128[0]); InitAllBits: - INIT_BITS(0, ctx->ecb.keys); - INIT_BITS(1, ctx->ecb.keys); - INIT_BITS(2, ctx->ecb.keys); - INIT_BITS(3, ctx->ecb.keys); - INIT_BITS(4, ctx->ecb.keys); - INIT_BITS(5, ctx->ecb.keys); + INIT_BITS(0, ctx->super.ecb.keys); + INIT_BITS(1, ctx->super.ecb.keys); + INIT_BITS(2, ctx->super.ecb.keys); + INIT_BITS(3, ctx->super.ecb.keys); + INIT_BITS(4, ctx->super.ecb.keys); + INIT_BITS(5, ctx->super.ecb.keys); #undef INIT_BITS } { /* run aes and ghash */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys.m128[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -740,9 +748,9 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, AESECB6_UPDATE(aesi); gfmul_nextstep128(&gstate, _mm_loadu_si128(gdata++), --ghash_precompute); } - for (; aesi < ctx->ecb.rounds; ++aesi) + for (; aesi < ctx->super.ecb.rounds; ++aesi) AESECB6_UPDATE(aesi); - __m128i k = ctx->ecb.keys.m128[aesi]; + __m128i k = ctx->super.ecb.keys.m128[aesi]; bits0 = _mm_aesenclast_si128(bits0, k); bits1 = _mm_aesenclast_si128(bits1, k); bits2 = _mm_aesenclast_si128(bits2, k); @@ -804,7 +812,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, /* the only case where AES operation is complete and GHASH is not is when the application of AC is remaining */ if ((state & STATE_GHASH_HAS_MORE) != 0) { - assert(ghash_precompute - 1 == ctx->ghash128); + assert(ghash_precompute - 1 == ctx->ghash); gfmul_nextstep128(&gstate, ac, --ghash_precompute); } @@ -915,17 +923,19 @@ static void setup_one_ghash_entry(ptls_fusion_aesgcm_context_t *ctx) __m128i *H, *r, *Hprev, H0; if (ctx->ecb.avx256) { -#define GET_SLOT(i, mem) (&ctx->ghash256[(i) / 2].mem[(i) % 2 == 0]) + struct ptls_fusion_aesgcm_context256 *ctx256 = (void *)ctx; +#define GET_SLOT(i, mem) (&ctx256->ghash[(i) / 2].mem[(i) % 2 == 0]) H = GET_SLOT(ctx->ghash_cnt, H); r = GET_SLOT(ctx->ghash_cnt, r); Hprev = ctx->ghash_cnt == 0 ? NULL : GET_SLOT(ctx->ghash_cnt - 1, H); #undef GET_SLOT - H0 = ctx->ghash256[0].H[1]; + H0 = ctx256->ghash[0].H[1]; } else { - H = &ctx->ghash128[ctx->ghash_cnt].H; - r = &ctx->ghash128[ctx->ghash_cnt].r; - Hprev = ctx->ghash_cnt == 0 ? NULL : &ctx->ghash128[ctx->ghash_cnt - 1].H; - H0 = ctx->ghash128[0].H; + struct ptls_fusion_aesgcm_context128 *ctx128 = (void *)ctx; + H = &ctx128->ghash[ctx->ghash_cnt].H; + r = &ctx128->ghash[ctx->ghash_cnt].r; + Hprev = ctx->ghash_cnt == 0 ? NULL : &ctx128->ghash[ctx->ghash_cnt - 1].H; + H0 = ctx128->ghash[0].H; } if (Hprev != NULL) @@ -937,15 +947,28 @@ static void setup_one_ghash_entry(ptls_fusion_aesgcm_context_t *ctx) ++ctx->ghash_cnt; } +static size_t calc_aesgcm_context_size(size_t *ghash_cnt, int avx256) +{ + size_t sz; + + if (avx256) { + if (*ghash_cnt % 2 != 0) + ++*ghash_cnt; + sz = offsetof(struct ptls_fusion_aesgcm_context256, ghash) + + sizeof(union ptls_fusion_aesgcm_ghash_precompute256) * *ghash_cnt / 2; + } else { + sz = offsetof(struct ptls_fusion_aesgcm_context128, ghash) + + sizeof(struct ptls_fusion_aesgcm_ghash_precompute128) * *ghash_cnt; + } + return sz; +} + static ptls_fusion_aesgcm_context_t *new_aesgcm(const void *key, size_t key_size, size_t capacity, int avx256) { ptls_fusion_aesgcm_context_t *ctx; - size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity); - if (avx256 && ghash_cnt % 2 != 0) - ++ghash_cnt; + size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity), ctx_size = calc_aesgcm_context_size(&ghash_cnt, avx256); - PTLS_BUILD_ASSERT(sizeof(ctx->ghash128[0]) * 2 == sizeof(ctx->ghash256[0])); - if ((ctx = aligned_alloc(32, sizeof(*ctx) + sizeof(ctx->ghash128[0]) * ghash_cnt)) == NULL) + if ((ctx = aligned_alloc(32, ctx_size)) == NULL) return NULL; ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size, avx256); @@ -956,9 +979,9 @@ static ptls_fusion_aesgcm_context_t *new_aesgcm(const void *key, size_t key_size H0 = _mm_shuffle_epi8(H0, byteswap128); H0 = transformH(H0); if (ctx->ecb.avx256) { - ctx->ghash256[0].H[1] = H0; + ((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash[0].H[1] = H0; } else { - ctx->ghash128[0].H = H0; + ((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash[0].H = H0; } ctx->ghash_cnt = 0; @@ -980,7 +1003,8 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm if (ghash_cnt <= ctx->ghash_cnt) return ctx; - if ((ctx = realloc(ctx, sizeof(*ctx) + sizeof(ctx->ghash128[0]) * ghash_cnt)) == NULL) + size_t ctx_size = calc_aesgcm_context_size(&ghash_cnt, ctx->ecb.avx256); + if ((ctx = realloc(ctx, ctx_size)) == NULL) return NULL; ctx->capacity = capacity; @@ -992,9 +1016,9 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm void ptls_fusion_aesgcm_free(ptls_fusion_aesgcm_context_t *ctx) { - ptls_clear_memory(ctx->ghash128, sizeof(ctx->ghash128[0]) * ctx->ghash_cnt); - ctx->ghash_cnt = 0; - ptls_fusion_aesecb_dispose(&ctx->ecb); + ptls_clear_memory(ctx, calc_aesgcm_context_size(&ctx->ghash_cnt, ctx->ecb.avx256)); + /* skip ptls_fusion_aesecb_dispose, based on the knowledge that it does not allocate memory elsewhere */ + free(ctx); } @@ -1281,7 +1305,7 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void bits5 = ek0; \ state |= STATE_EK0_READY; \ } \ - __m128i k = ctx->ecb.keys.m128[0]; \ + __m128i k = ctx->super.ecb.keys.m128[0]; \ bits0 = _mm_xor_si128(bits0, k); \ bits1 = _mm_xor_si128(bits1, k); \ bits2 = _mm_xor_si128(bits2, k); \ @@ -1293,7 +1317,7 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ - __m128i k = ctx->ecb.keys.m128[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenc_si128(bits0, k); \ bits1 = _mm_aesenc_si128(bits1, k); \ bits2 = _mm_aesenc_si128(bits2, k); \ @@ -1305,7 +1329,7 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ - __m128i k = ctx->ecb.keys.m128[i]; \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ bits0 = _mm_aesenclast_si128(bits0, k); \ bits1 = _mm_aesenclast_si128(bits1, k); \ bits2 = _mm_aesenclast_si128(bits2, k); \ @@ -1328,7 +1352,8 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void #if defined(__GNUC__) && !defined(__clang__) static __thread #endif - uint8_t encbuf[32 * 6] __attribute__((aligned(32))), *encp; + uint8_t encbuf[32 * 6] __attribute__((aligned(32))), + *encp; /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ @@ -1350,7 +1375,7 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void __m128i ek0 = _mm_shuffle_epi8(ctr, byteswap128); __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)calc_total_length(input, incnt) * 8), byteswap128); - ptls_fusion_aesgcm_context_t *ctx = agctx->aesgcm; + struct ptls_fusion_aesgcm_context128 *ctx = (void *)agctx->aesgcm; __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); struct ptls_fusion_gfmul_state128 gstate = {0}; @@ -1368,10 +1393,10 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void AESECB6_INIT(); AESECB6_UPDATE(1); AESECB6_UPDATE(2); - reduce_aad128(&gstate, ctx->ghash128, aad, aadlen); - for (size_t i = 3; i < ctx->ecb.rounds; ++i) + reduce_aad128(&gstate, ctx->ghash, aad, aadlen); + for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); - AESECB6_FINAL(ctx->ecb.rounds); + AESECB6_FINAL(ctx->super.ecb.rounds); /* Main loop. This loop: * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, @@ -1453,7 +1478,7 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void /* Next 96-byte block starts here. Run AES and ghash in while writing output using non-temporal stores in 64-byte * blocks. */ AESECB6_INIT(); - struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + 6; + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + 6; gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encp - 6 * 16)), --ghash_precompute); AESECB6_UPDATE(1); gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encp - 5 * 16)), --ghash_precompute); @@ -1486,13 +1511,13 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void } state ^= STATE_COPY_128B; AESECB6_UPDATE(9); - if (PTLS_UNLIKELY(ctx->ecb.rounds != 10)) { - for (size_t i = 10; PTLS_LIKELY(i < ctx->ecb.rounds); ++i) + if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { + for (size_t i = 10; PTLS_LIKELY(i < ctx->super.ecb.rounds); ++i) AESECB6_UPDATE(i); } - assert(ctx->ghash128 == ghash_precompute); + assert(ctx->ghash == ghash_precompute); gfmul_reduce128(&gstate); - AESECB6_FINAL(ctx->ecb.rounds); + AESECB6_FINAL(ctx->super.ecb.rounds); } } @@ -1504,10 +1529,10 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void _mm_storeu_si128((void *)(encbuf + ac_off), ac); size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 7); - struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash128 + blocks; + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + blocks; gfmul_firststep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; - while (ghash_precompute != ctx->ghash128) { + while (ghash_precompute != ctx->ghash) { gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 16; } @@ -1516,10 +1541,10 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { - bits5 = _mm_xor_si128(ek0, ctx->ecb.keys.m128[0]); - for (size_t i = 1; i < ctx->ecb.rounds; ++i) - bits5 = _mm_aesenc_si128(bits5, ctx->ecb.keys.m128[i]); - bits5 = _mm_aesenclast_si128(bits5, ctx->ecb.keys.m128[ctx->ecb.rounds]); + bits5 = _mm_xor_si128(ek0, ctx->super.ecb.keys.m128[0]); + for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) + bits5 = _mm_aesenc_si128(bits5, ctx->super.ecb.keys.m128[i]); + bits5 = _mm_aesenclast_si128(bits5, ctx->super.ecb.keys.m128[ctx->super.ecb.rounds]); } /* append tag to encbuf */ @@ -1559,7 +1584,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void bits5 = _mm256_permute2f128_si256(bits5, ac_ek0, 0x30); \ state |= STATE_EK0_READY; \ } \ - __m256i k = ctx->ecb.keys.m256[0]; \ + __m256i k = ctx->super.ecb.keys.m256[0]; \ bits0 = _mm256_xor_si256(bits0, k); \ bits1 = _mm256_xor_si256(bits1, k); \ bits2 = _mm256_xor_si256(bits2, k); \ @@ -1571,7 +1596,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void /* aes block update */ #define AESECB6_UPDATE(i) \ do { \ - __m256i k = ctx->ecb.keys.m256[i]; \ + __m256i k = ctx->super.ecb.keys.m256[i]; \ bits0 = _mm256_aesenc_epi128(bits0, k); \ bits1 = _mm256_aesenc_epi128(bits1, k); \ bits2 = _mm256_aesenc_epi128(bits2, k); \ @@ -1583,7 +1608,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void /* aesenclast */ #define AESECB6_FINAL(i) \ do { \ - __m256i k = ctx->ecb.keys.m256[i]; \ + __m256i k = ctx->super.ecb.keys.m256[i]; \ bits0 = _mm256_aesenclast_epi128(bits0, k); \ bits1 = _mm256_aesenclast_epi128(bits1, k); \ bits2 = _mm256_aesenclast_epi128(bits2, k); \ @@ -1623,7 +1648,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void /* second half: ek0 */ _mm256_shuffle_epi8(ctr, byteswap256), 0x30); - ptls_fusion_aesgcm_context_t *ctx = agctx->aesgcm; + struct ptls_fusion_aesgcm_context256 *ctx = (void *)agctx->aesgcm; __m256i bits0, bits1, bits2, bits3, bits4, bits5 = _mm256_setzero_si256(); struct ptls_fusion_gfmul_state256 gstate = {0}; @@ -1644,7 +1669,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void if (PTLS_LIKELY(aadlen != 0)) { union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute; while (PTLS_UNLIKELY(aadlen >= 6 * 32)) { - ghash_precompute = ctx->ghash256 + 6; + ghash_precompute = ctx->ghash + 6; gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); aad += 32; aadlen -= 32; @@ -1656,7 +1681,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void gfmul_reduce256(&gstate); } if (PTLS_LIKELY(aadlen != 0)) { - ghash_precompute = ctx->ghash256 + (aadlen + 31) / 32; + ghash_precompute = ctx->ghash + (aadlen + 31) / 32; if (PTLS_UNLIKELY(aadlen >= 32)) { if (aadlen % 32 == 0 || aadlen % 32 > 16) { gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)aad), 0, --ghash_precompute); @@ -1679,13 +1704,13 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void } else { gfmul_firststep256(&gstate, loadn256(aad, aadlen), aadlen <= 16, --ghash_precompute); } - assert(ctx->ghash256 == ghash_precompute); + assert(ctx->ghash == ghash_precompute); gfmul_reduce256(&gstate); } } - for (size_t i = 3; i < ctx->ecb.rounds; ++i) + for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) AESECB6_UPDATE(i); - AESECB6_FINAL(ctx->ecb.rounds); + AESECB6_FINAL(ctx->super.ecb.rounds); /* Main loop. This loop: * 1. using current keystream (bits0..bits5), xors a up to 6 * 16 bytes and writes to encbuf, @@ -1768,7 +1793,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void /* Next 96-byte block starts here. Run AES and ghash in parallel while writing output using non-temporal store * instructions. */ AESECB6_INIT(); - union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash256 + 6; + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash + 6; gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encp - 6 * 32)), 0, --ghash_precompute); AESECB6_UPDATE(1); gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encp - 5 * 32)), --ghash_precompute); @@ -1794,13 +1819,13 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void AESECB6_UPDATE(8); _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(encbuf + 224))); AESECB6_UPDATE(9); - if (PTLS_UNLIKELY(ctx->ecb.rounds != 10)) { - for (size_t i = 10; PTLS_LIKELY(i < ctx->ecb.rounds); ++i) + if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { + for (size_t i = 10; PTLS_LIKELY(i < ctx->super.ecb.rounds); ++i) AESECB6_UPDATE(i); } - assert(ctx->ghash256 == ghash_precompute); + assert(ctx->ghash == ghash_precompute); gfmul_reduce256(&gstate); - AESECB6_FINAL(ctx->ecb.rounds); + AESECB6_FINAL(ctx->super.ecb.rounds); } } @@ -1812,7 +1837,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void _mm_storeu_si128((void *)(encbuf + ac_off), _mm256_castsi256_si128(ac_ek0)); size_t blocks = ((encp - encbuf) - remaining_ghash_from + 15) / 16 + 1; /* round up, +1 for AC */ assert(blocks <= 13); - union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash256 + blocks / 2; + union ptls_fusion_aesgcm_ghash_precompute256 *ghash_precompute = ctx->ghash + blocks / 2; if (blocks % 2 != 0) { gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 1, ghash_precompute); remaining_ghash_from += 16; @@ -1820,7 +1845,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void gfmul_firststep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), 0, --ghash_precompute); remaining_ghash_from += 32; } - while (ghash_precompute != ctx->ghash256) { + while (ghash_precompute != ctx->ghash) { gfmul_nextstep256(&gstate, _mm256_loadu_si256((void *)(encbuf + remaining_ghash_from)), --ghash_precompute); remaining_ghash_from += 32; } @@ -1830,10 +1855,10 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void /* Calculate EK0, if in the unlikely case on not been done yet. When encoding in full size (16K), EK0 will be ready. */ if (PTLS_UNLIKELY((state & STATE_EK0_READY) == 0)) { bits5 = ac_ek0; - bits5 = _mm256_xor_si256(bits5, ctx->ecb.keys.m256[0]); - for (size_t i = 1; i < ctx->ecb.rounds; ++i) - bits5 = _mm256_aesenc_epi128(bits5, ctx->ecb.keys.m256[i]); - bits5 = _mm256_aesenclast_epi128(bits5, ctx->ecb.keys.m256[ctx->ecb.rounds]); + bits5 = _mm256_xor_si256(bits5, ctx->super.ecb.keys.m256[0]); + for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) + bits5 = _mm256_aesenc_epi128(bits5, ctx->super.ecb.keys.m256[i]); + bits5 = _mm256_aesenclast_epi128(bits5, ctx->super.ecb.keys.m256[ctx->super.ecb.rounds]); } /* append tag to encbuf */ diff --git a/t/fusion.c b/t/fusion.c index 02d291847..c5eec412e 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -87,38 +87,40 @@ static void test_ecb(void) static void test_gfmul(void) { ptls_fusion_aesgcm_context_t *ctx; - PTLS_BUILD_ASSERT(sizeof(ctx->ghash128[0]) * 2 == sizeof(ctx->ghash256[0])); + size_t ghash_cnt = 4; -#define COUNT 4 - - ctx = malloc(offsetof(ptls_fusion_aesgcm_context_t, ghash128) + sizeof(ctx->ghash128[0]) * COUNT); - memset(ctx, 0, offsetof(ptls_fusion_aesgcm_context_t, ghash128)); - ctx->capacity = 4 * 16; - ctx->ecb.avx256 = ptls_fusion_can_avx256; + ctx = malloc(calc_aesgcm_context_size(&ghash_cnt, ptls_fusion_can_avx256)); + *ctx = (ptls_fusion_aesgcm_context_t){ + .ecb = {.avx256 = ptls_fusion_can_avx256}, + .capacity = ghash_cnt * 16, + }; __m128i H0 = _mm_loadu_si128((void *)"hello world bye"); if (ctx->ecb.avx256) { - ctx->ghash256[0].H[1] = H0; + ((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash[0].H[1] = H0; } else { - ctx->ghash128[0].H = H0; + ((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash[0].H = H0; } - while (ctx->ghash_cnt < COUNT) + while (ctx->ghash_cnt < ghash_cnt) setup_one_ghash_entry(ctx); +#define GHASH128 (((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash) +#define GHASH256 (((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash) + { /* one block */ static const char input[32] = "deaddeadbeefbeef"; /* latter 16-byte is NUL */ __m128i hash; if (ctx->ecb.avx256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); - gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, ctx->ghash256); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, GHASH256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { struct ptls_fusion_gfmul_state128 state; state.lo = _mm_setzero_si128(); - gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128); gfmul_reduce128(&state); hash = state.lo; } @@ -131,14 +133,14 @@ static void test_gfmul(void) if (ctx->ecb.avx256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); - gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { struct ptls_fusion_gfmul_state128 state; state.lo = _mm_setzero_si128(); - gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 1); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128); gfmul_reduce128(&state); hash = state.lo; } @@ -151,16 +153,16 @@ static void test_gfmul(void) if (ctx->ecb.avx256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); - gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, ctx->ghash256 + 1); - gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 16)), ctx->ghash256); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, GHASH256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 16)), GHASH256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { struct ptls_fusion_gfmul_state128 state; state.lo = _mm_setzero_si128(); - gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 2); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 1); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), GHASH128); gfmul_reduce128(&state); hash = state.lo; } @@ -173,21 +175,21 @@ static void test_gfmul(void) if (ctx->ecb.avx256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); - gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256 + 1); - gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), ctx->ghash256); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), GHASH256); gfmul_reduce256(&state); - gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 1, ctx->ghash256); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 1, GHASH256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { struct ptls_fusion_gfmul_state128 state; state.lo = _mm_setzero_si128(); - gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 3); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 2); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128 + 1); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), ctx->ghash128); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 3); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), GHASH128); gfmul_reduce128(&state); - gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), ctx->ghash128); + gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), GHASH128); gfmul_reduce128(&state); hash = state.lo; } @@ -201,22 +203,22 @@ static void test_gfmul(void) if (ctx->ecb.avx256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); - gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, ctx->ghash256 + 1); - gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), ctx->ghash256); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256 + 1); + gfmul_nextstep256(&state, _mm256_loadu_si256((void *)(input + 32)), GHASH256); gfmul_reduce256(&state); - gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 0, ctx->ghash256); + gfmul_firststep256(&state, _mm256_loadu_si256((void *)(input + 64)), 0, GHASH256); gfmul_reduce256(&state); hash = _mm256_castsi256_si128(state.lo); } else { struct ptls_fusion_gfmul_state128 state; state.lo = _mm_setzero_si128(); - gfmul_firststep128(&state, _mm_loadu_si128((void *)input), ctx->ghash128 + 3); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), ctx->ghash128 + 2); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), ctx->ghash128 + 1); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), ctx->ghash128); + gfmul_firststep128(&state, _mm_loadu_si128((void *)input), GHASH128 + 3); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 16)), GHASH128 + 2); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 32)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 48)), GHASH128); gfmul_reduce128(&state); - gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), ctx->ghash128 + 1); - gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 80)), ctx->ghash128); + gfmul_firststep128(&state, _mm_loadu_si128((void *)(input + 64)), GHASH128 + 1); + gfmul_nextstep128(&state, _mm_loadu_si128((void *)(input + 80)), GHASH128); gfmul_reduce128(&state); hash = state.lo; } @@ -225,7 +227,8 @@ static void test_gfmul(void) free(ctx); -#undef COUNT +#undef GHASH128 +#undef GHASH256 } static void gcm_basic(void) From 196e477cae256ff61544c1bcc1ea5e1d725500ff Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 09:52:07 +0900 Subject: [PATCH 32/48] we do out-of-bounds access within the same page directly, which asan complains --- lib/fusion.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/fusion.c b/lib/fusion.c index 1b414e6c1..0d04772a6 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -334,6 +334,7 @@ static inline __m128i loadn_end_of_page(const void *p, size_t l) return _mm_shuffle_epi8(_mm_load_si128((const __m128i *)((uintptr_t)p - shift)), pattern); } +NO_SANITIZE_ADDRESS static inline __m128i loadn128(const void *p, size_t l) { __m128i v, mask = _mm_loadu_si128((__m128i *)(loadn_mask + 32 - l)); @@ -349,6 +350,7 @@ static inline __m128i loadn128(const void *p, size_t l) return v; } +NO_SANITIZE_ADDRESS static inline __m256i loadn256(const void *p, size_t l) { __m256i v, mask = _mm256_loadu_si256((__m256i *)(loadn_mask + 32 - l)); From 13ced8249559f769c6796fa14072d5ded6c12c0e Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 10:09:54 +0900 Subject: [PATCH 33/48] extract "unsafe" logic --- lib/fusion.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 0d04772a6..527becb78 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1265,6 +1265,21 @@ static inline void reduce_aad128(struct ptls_fusion_gfmul_state128 *gstate, stru } } +NO_SANITIZE_ADDRESS +static inline uint8_t *load_preceding_unaligned(uint8_t *encbuf, uint8_t **output) +{ + uint8_t *encp; + + if ((encp = encbuf + ((uintptr_t)*output & 63)) != encbuf) { + _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(*output - (encp - encbuf)))); + _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(*output - (encp - encbuf) + 32))); + *output -= encp - encbuf; + } + + return encp; +} + +NO_SANITIZE_ADDRESS static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const uint8_t *end) { /* Write in 64-byte chunks, using NT store instructions. Last partial block, if any, is written to cache, as that cache line @@ -1283,7 +1298,6 @@ static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const } } -NO_SANITIZE_ADDRESS static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *aad, size_t aadlen) { @@ -1361,12 +1375,9 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 16 + 16); - /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ - if ((encp = encbuf + ((uintptr_t)output & 63)) != encbuf) { - _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(output - (encp - encbuf)))); - _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(output - (encp - encbuf) + 32))); - output -= encp - encbuf; - } + /* load unaligned data within same cache line preceding `output`, adjusting pointers accordingly */ + encp = load_preceding_unaligned(encbuf, &output); + /* First write would be 128 bytes (32+6*16), if encbuf contains no less than 32 bytes already. */ if (encp - encbuf >= 32) state |= STATE_COPY_128B; @@ -1633,12 +1644,8 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ PTLS_BUILD_ASSERT(sizeof(encbuf) >= 64 + 6 * 32 + 16); - /* determine `num_bytes_write_delayed` as well as initializing `encbuf`, adjusting `output` */ - if ((encp = encbuf + ((uintptr_t)output & 63)) != encbuf) { - _mm256_store_si256((void *)encbuf, _mm256_load_si256((void *)(output - (encp - encbuf)))); - _mm256_store_si256((void *)(encbuf + 32), _mm256_load_si256((void *)(output - (encp - encbuf) + 32))); - output -= encp - encbuf; - } + /* load unaligned data within same cache line preceding `output`, adjusting pointers accordingly */ + encp = load_preceding_unaligned(encbuf, &output); /* setup ctr, retaining Ek(0), len(A) | len(C) to be fed into GCM */ __m256i ctr = _mm256_broadcastsi128_si256(calc_counter(agctx, seq)); From 454398292441ef3ffd4dee2ba612919675042dca Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 10:16:49 +0900 Subject: [PATCH 34/48] retain the attribute, as we write to `output` adjusted to out-of-bounds --- lib/fusion.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/fusion.c b/lib/fusion.c index 527becb78..67b4ce3a9 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1298,6 +1298,7 @@ static inline void write_remaining_bytes(uint8_t *dst, const uint8_t *src, const } } +NO_SANITIZE_ADDRESS static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *aad, size_t aadlen) { From 2344f276da397cecf634d05a2c317006c5e29ffd Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 11:40:26 +0900 Subject: [PATCH 35/48] turn fusion off on platform that does not have GCC with support for vaes, vpclmulqdq --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a3d736bc..5e91a12ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: matrix: include: - name: "Linux / OpenSSL 1.1.0" - command: make -f misc/docker-ci.mk CMAKE_ARGS='-DOPENSSL_ROOT_DIR=-DOPENSSL_ROOT_DIR=/opt/openssl-1.1.0' CONTAINER_NAME='h2oserver/h2o-ci:ubuntu1604' + command: make -f misc/docker-ci.mk CMAKE_ARGS='-DOPENSSL_ROOT_DIR=-DOPENSSL_ROOT_DIR=/opt/openssl-1.1.0 -DWITH_FUSION=OFF' CONTAINER_NAME='h2oserver/h2o-ci:ubuntu1604' - name: "Linux / OpenSSL 1.1.1" command: make -f misc/docker-ci.mk - name: "Linux / OpenSSL 1.1.1 + ASan & UBSan" From 2094f783f7c5b70ec79e17d76a4af1d8fba2c572 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 22:13:00 +0900 Subject: [PATCH 36/48] decoder works --- lib/fusion.c | 224 +++++++++++++++++++++++++++++++++++++++++++++++++-- t/fusion.c | 7 +- 2 files changed, 221 insertions(+), 10 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 67b4ce3a9..288d6460b 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1575,6 +1575,205 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void #undef STATE_COPY_128B } +static size_t non_temporal_decrypt128(ptls_aead_context_t *_ctx, void *_output, const void *_input, size_t inlen, uint64_t seq, + const void *aad, size_t aadlen) +{ + /* Bail out if the input is too short, or remove tag from range. */ + if (inlen < 16) + return SIZE_MAX; + inlen -= 16; + size_t textlen = inlen; + +/* init the bits (we can always run in full), but use the last slot for calculating ek0, if possible */ +#define AESECB6_INIT() \ + do { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits0 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits1 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits2 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits3 = _mm_shuffle_epi8(ctr, byteswap128); \ + ctr = _mm_add_epi64(ctr, one8); \ + bits4 = _mm_shuffle_epi8(ctr, byteswap128); \ + if (PTLS_LIKELY(inlen > 16 * 5)) { \ + ctr = _mm_add_epi64(ctr, one8); \ + bits5 = _mm_shuffle_epi8(ctr, byteswap128); \ + } else { \ + bits5 = ek0; \ + state |= STATE_EK0_READY; \ + } \ + __m128i k = ctx->super.ecb.keys.m128[0]; \ + bits0 = _mm_xor_si128(bits0, k); \ + bits1 = _mm_xor_si128(bits1, k); \ + bits2 = _mm_xor_si128(bits2, k); \ + bits3 = _mm_xor_si128(bits3, k); \ + bits4 = _mm_xor_si128(bits4, k); \ + bits5 = _mm_xor_si128(bits5, k); \ + } while (0) + +/* aes block update */ +#define AESECB6_UPDATE(i) \ + do { \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ + bits0 = _mm_aesenc_si128(bits0, k); \ + bits1 = _mm_aesenc_si128(bits1, k); \ + bits2 = _mm_aesenc_si128(bits2, k); \ + bits3 = _mm_aesenc_si128(bits3, k); \ + bits4 = _mm_aesenc_si128(bits4, k); \ + bits5 = _mm_aesenc_si128(bits5, k); \ + } while (0) + +/* aesenclast */ +#define AESECB6_FINAL(i) \ + do { \ + __m128i k = ctx->super.ecb.keys.m128[i]; \ + bits0 = _mm_aesenclast_si128(bits0, k); \ + bits1 = _mm_aesenclast_si128(bits1, k); \ + bits2 = _mm_aesenclast_si128(bits2, k); \ + bits3 = _mm_aesenclast_si128(bits3, k); \ + bits4 = _mm_aesenclast_si128(bits4, k); \ + bits5 = _mm_aesenclast_si128(bits5, k); \ + } while (0) + + struct aesgcm_context *agctx = (void *)_ctx; + uint8_t *output = _output; + const uint8_t *input = _input; + +#define STATE_EK0_READY 0x1 + int32_t state = 0; + + /* setup ctr, retain Ek(0), len(A) | len(C) to be fed into GCM */ + __m128i ctr = calc_counter(agctx, seq); + ctr = _mm_insert_epi32(ctr, 1, 0); + __m128i ek0 = _mm_shuffle_epi8(ctr, byteswap128); + __m128i ac = _mm_shuffle_epi8(_mm_set_epi32(0, (int)aadlen * 8, 0, (int)inlen * 8), byteswap128); + + struct ptls_fusion_aesgcm_context128 *ctx = (void *)agctx->aesgcm; + __m128i bits0, bits1, bits2, bits3, bits4, bits5 = _mm_setzero_si128(); + struct ptls_fusion_gfmul_state128 gstate = {0}; + + /* Prepare first 6 blocks of bit stream, at the same time calculating ghash of AAD. */ + AESECB6_INIT(); + AESECB6_UPDATE(1); + AESECB6_UPDATE(2); + reduce_aad128(&gstate, ctx->ghash, aad, aadlen); + for (size_t i = 3; i < ctx->super.ecb.rounds; ++i) + AESECB6_UPDATE(i); + AESECB6_FINAL(ctx->super.ecb.rounds); + + /* Main loop. Operate in full blocks (6 * 16 bytes). */ + while (PTLS_LIKELY(inlen >= 6 * 16)) { + uint8_t keystream[6 * 16] __attribute__((aligned(16))); + _mm_store_si128((void *)keystream, bits0); + _mm_store_si128((void *)(keystream + 16), bits1); + _mm_store_si128((void *)(keystream + 32), bits2); + _mm_store_si128((void *)(keystream + 48), bits3); + _mm_store_si128((void *)(keystream + 64), bits4); + _mm_store_si128((void *)(keystream + 80), bits5); +#define APPLY(i) \ + do { \ + __m128i b = _mm_loadu_si128((void *)input + i * 16); \ + _mm_storeu_si128((void *)(output + i * 16), _mm_xor_si128(_mm_load_si128((void *)(keystream + i * 16)), b)); \ + if (i == 0) { \ + gfmul_firststep128(&gstate, b, ctx->ghash + 5 - i); \ + } else { \ + gfmul_nextstep128(&gstate, b, ctx->ghash + 5 - i); \ + } \ + } while (0) + AESECB6_INIT(); + AESECB6_UPDATE(1); + APPLY(0); + AESECB6_UPDATE(2); + APPLY(1); + AESECB6_UPDATE(3); + APPLY(2); + AESECB6_UPDATE(4); + APPLY(3); + AESECB6_UPDATE(5); + APPLY(4); + AESECB6_UPDATE(6); + APPLY(5); + AESECB6_UPDATE(7); + gfmul_reduce128(&gstate); + AESECB6_UPDATE(8); + AESECB6_UPDATE(9); + if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { + size_t i = 10; + do { + AESECB6_UPDATE(i); + } while (++i < ctx->super.ecb.rounds); + } + AESECB6_FINAL(ctx->super.ecb.rounds); + output += 6 * 16; + input += 6 * 16; + inlen -= 6 * 16; +#undef APPLY + } + + /* Decrypt the remainder as well as finishing GHASH calculation. */ + if (inlen != 0) { + struct ptls_fusion_aesgcm_ghash_precompute128 *ghash_precompute = ctx->ghash + (inlen + 15) / 16 + 1; +#define ONEBLOCK(i) \ + do { \ + if (inlen != 0) { \ + __m128i b = inlen >= 16 ? _mm_loadu_si128((void *)input) : loadn128(input, inlen); \ + if (i == 0) { \ + gfmul_firststep128(&gstate, b, --ghash_precompute); \ + } else { \ + gfmul_nextstep128(&gstate, b, --ghash_precompute); \ + } \ + b = _mm_xor_si128(b, bits##i); \ + if (inlen >= 16) { \ + _mm_storeu_si128((void *)output, b); \ + output += 16; \ + input += 16; \ + inlen -= 16; \ + } else { \ + storen128(output, inlen, b); \ + output += inlen; \ + input += inlen; \ + inlen = 0; \ + } \ + } \ + } while (0) + ONEBLOCK(0); + ONEBLOCK(1); + ONEBLOCK(2); + ONEBLOCK(3); + ONEBLOCK(4); + ONEBLOCK(5); +#undef ONEBLOCK + gfmul_nextstep128(&gstate, ac, --ghash_precompute); + assert(ghash_precompute == ctx->ghash); + } else { + gfmul_firststep128(&gstate, ac, ctx->ghash); + } + gfmul_reduce128(&gstate); + + /* Calculate EK0 if not yet available in bits5. */ + if ((state & STATE_EK0_READY) == 0) { + bits5 = _mm_xor_si128(ek0, ctx->super.ecb.keys.m128[0]); + for (size_t i = 1; i < ctx->super.ecb.rounds; ++i) + bits5 = _mm_aesenc_si128(bits5, ctx->super.ecb.keys.m128[i]); + bits5 = _mm_aesenclast_si128(bits5, ctx->super.ecb.keys.m128[ctx->super.ecb.rounds]); + } + + /* Calculate GCM tag and compare. */ + __m128i calctag = gfmul_get_tag128(&gstate, bits5); + __m128i recvtag = _mm_loadu_si128((void *)input); + if (_mm_movemask_epi8(_mm_cmpeq_epi8(calctag, recvtag)) != 0xffff) + return SIZE_MAX; + + return textlen; + +#undef AESECB6_INIT +#undef AESECB6_UPDATE +#undef AESECB6_FINAL +#undef STATE_EK0_READY +} + NO_SANITIZE_ADDRESS static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void *_output, ptls_iovec_t *input, size_t incnt, uint64_t seq, const void *_aad, size_t aadlen) @@ -1880,9 +2079,10 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void write_remaining_bytes(output, encbuf, encp); } -static int nt_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) +static int non_temporal_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; + int avx256 = is_enc && ptls_fusion_can_avx256; ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); @@ -1891,25 +2091,35 @@ static int nt_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, cons ctx->super.dispose_crypto = aesgcm_dispose_crypto; ctx->super.do_xor_iv = aesgcm_xor_iv; - ctx->super.do_encrypt = ptls_aead__do_encrypt; - ctx->super.do_encrypt_v = ptls_fusion_can_avx256 ? non_temporal_encrypt_v256 : non_temporal_encrypt_v128; - ctx->super.do_decrypt = NULL; /* FIXME */ + ctx->super.do_encrypt_init = NULL; + ctx->super.do_encrypt_update = NULL; + ctx->super.do_encrypt_final = NULL; + if (is_enc) { + ctx->super.do_encrypt = ptls_aead__do_encrypt; + ctx->super.do_encrypt_v = avx256 ? non_temporal_encrypt_v256 : non_temporal_encrypt_v128; + ctx->super.do_decrypt = NULL; + } else { + assert(!avx256); + ctx->super.do_encrypt = NULL; + ctx->super.do_encrypt_v = NULL; + ctx->super.do_decrypt = non_temporal_decrypt128; + } ctx->aesgcm = new_aesgcm(key, key_size, 7 * (ptls_fusion_can_avx256 ? 32 : 16), // 6 blocks at once, plus len(A) | len(C) that we might append - ptls_fusion_can_avx256); + avx256); return 0; } static int non_temporal_aes128gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { - return nt_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); + return non_temporal_setup(ctx, is_enc, key, iv, PTLS_AES128_KEY_SIZE); } static int non_temporal_aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key, const void *iv) { - return nt_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); + return non_temporal_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); } ptls_aead_algorithm_t ptls_non_temporal_aes128gcm = {"AES128-GCM", diff --git a/t/fusion.c b/t/fusion.c index c5eec412e..fbea21395 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -467,9 +467,10 @@ static void test_aesgcm(void) subtest("aes256gcm-iv96", test_generated_aes256_iv96); } -static void test_fastls(void) +static void test_non_temporal(void) { test_generated(&ptls_non_temporal_aes128gcm, &ptls_minicrypto_aes128gcm, 0); + test_generated(&ptls_minicrypto_aes128gcm, &ptls_non_temporal_aes128gcm, 0); } int main(int argc, char **argv) @@ -489,12 +490,12 @@ int main(int argc, char **argv) subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); subtest("aesgcm", test_aesgcm); - subtest("fastls128", test_fastls); + subtest("non-temporal128", test_non_temporal); if (can256bit) { ptls_fusion_can_avx256 = 1; subtest("gfmul256", test_gfmul); - subtest("fastls256", test_fastls); + subtest("non-temporal256", test_non_temporal); ptls_fusion_can_avx256 = 0; } else { note("gfmul256: skipping, CPU does not support 256-bit aes / clmul"); From 5f76ffc11e76d5c4b26969719f9c2c8cd821bb25 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Tue, 10 May 2022 23:19:54 +0900 Subject: [PATCH 37/48] use avx256 instructions for load, store, xor --- lib/fusion.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 288d6460b..682370049 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1665,36 +1665,31 @@ static size_t non_temporal_decrypt128(ptls_aead_context_t *_ctx, void *_output, /* Main loop. Operate in full blocks (6 * 16 bytes). */ while (PTLS_LIKELY(inlen >= 6 * 16)) { - uint8_t keystream[6 * 16] __attribute__((aligned(16))); - _mm_store_si128((void *)keystream, bits0); - _mm_store_si128((void *)(keystream + 16), bits1); - _mm_store_si128((void *)(keystream + 32), bits2); - _mm_store_si128((void *)(keystream + 48), bits3); - _mm_store_si128((void *)(keystream + 64), bits4); - _mm_store_si128((void *)(keystream + 80), bits5); +#define MERGE_BITS(x, y) _mm256_permute2f128_si256(_mm256_castsi128_si256(x), _mm256_castsi128_si256(y), 0x20) + __m256i ks0 = MERGE_BITS(bits0, bits1), ks2 = MERGE_BITS(bits2, bits3), ks4 = MERGE_BITS(bits4, bits5); +#undef MERGE_BITS #define APPLY(i) \ do { \ - __m128i b = _mm_loadu_si128((void *)input + i * 16); \ - _mm_storeu_si128((void *)(output + i * 16), _mm_xor_si128(_mm_load_si128((void *)(keystream + i * 16)), b)); \ + __m256i bb = _mm256_loadu_si256((void *)(input + i * 16)); \ + _mm256_storeu_si256((void *)(output + i * 16), _mm256_xor_si256(ks##i, bb)); \ + __m128i b0 = _mm256_castsi256_si128(bb), b1 = _mm256_castsi256_si128(_mm256_permute2f128_si256(bb, bb, 0x81)); \ if (i == 0) { \ - gfmul_firststep128(&gstate, b, ctx->ghash + 5 - i); \ + gfmul_firststep128(&gstate, b0, ctx->ghash + 5 - i); \ } else { \ - gfmul_nextstep128(&gstate, b, ctx->ghash + 5 - i); \ + gfmul_nextstep128(&gstate, b0, ctx->ghash + 5 - i); \ } \ + gfmul_nextstep128(&gstate, b1, ctx->ghash + 4 - i); \ } while (0) AESECB6_INIT(); AESECB6_UPDATE(1); APPLY(0); AESECB6_UPDATE(2); - APPLY(1); AESECB6_UPDATE(3); APPLY(2); AESECB6_UPDATE(4); - APPLY(3); AESECB6_UPDATE(5); APPLY(4); AESECB6_UPDATE(6); - APPLY(5); AESECB6_UPDATE(7); gfmul_reduce128(&gstate); AESECB6_UPDATE(8); From b93d856d218e306a170ee75dcb787f9a8c97af71 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 11 May 2022 07:54:11 +0900 Subject: [PATCH 38/48] full matrix testing --- t/fusion.c | 64 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/t/fusion.c b/t/fusion.c index fbea21395..429736971 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -377,7 +377,10 @@ static void gcm_iv96(void) ptls_aead_free(aead); } -static void test_generated(ptls_aead_algorithm_t *encryptor, ptls_aead_algorithm_t *decryptor, int iv96) +static ptls_aead_algorithm_t *test_generated_encryptor, *test_generated_decryptor; +static int test_generated_iv96; + +static void test_generated(void) { ptls_cipher_context_t *rand = ptls_cipher_new(&ptls_minicrypto_aes128ctr, 1, zero); ptls_cipher_init(rand, zero); @@ -409,8 +412,8 @@ static void test_generated(ptls_aead_algorithm_t *encryptor, ptls_aead_algorithm memset(decrypted, 0xcc, sizeof(decrypted)); { /* check using fusion */ - ptls_aead_context_t *ctx = ptls_aead_new_direct(encryptor, 1, key, iv); - if (iv96) + ptls_aead_context_t *ctx = ptls_aead_new_direct(test_generated_encryptor, 1, key, iv); + if (test_generated_iv96) ptls_aead_xor_iv(ctx, seq32, sizeof(seq32)); ptls_aead_encrypt(ctx, encrypted, text, textlen, seq, aad, aadlen); ptls_aead_free(ctx); @@ -419,8 +422,8 @@ static void test_generated(ptls_aead_algorithm_t *encryptor, ptls_aead_algorithm memset(decrypted, 0xcc, sizeof(decrypted)); { /* check that the encrypted text can be decrypted by the decryptor */ - ptls_aead_context_t *ctx = ptls_aead_new_direct(decryptor, 0, key, iv); - if (iv96) + ptls_aead_context_t *ctx = ptls_aead_new_direct(test_generated_decryptor, 0, key, iv); + if (test_generated_iv96) ptls_aead_xor_iv(ctx, seq32, sizeof(seq32)); if (ptls_aead_decrypt(ctx, decrypted, encrypted, textlen + 16, seq, aad, aadlen) != textlen) goto Fail; @@ -439,38 +442,53 @@ static void test_generated(ptls_aead_algorithm_t *encryptor, ptls_aead_algorithm ok(0); } -static void test_generated_aes128(void) +static void test_generated_all(ptls_aead_algorithm_t *e1, ptls_aead_algorithm_t *e2) +{ + test_generated_encryptor = e1; + test_generated_decryptor = e2; + test_generated_iv96 = 0; + subtest("encrypt", test_generated); + test_generated_iv96 = 1; + subtest("encrypt-iv96", test_generated); + + test_generated_encryptor = e2; + test_generated_decryptor = e1; + test_generated_iv96 = 0; + subtest("decrypt", test_generated); + test_generated_iv96 = 1; + subtest("decrypt-iv96", test_generated); +} + +static void test_fusion_aes128gcm(void) { - test_generated(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm, 0); + test_generated_all(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm); } -static void test_generated_aes256(void) +static void test_fusion_aes256gcm(void) { - test_generated(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm, 0); + test_generated_all(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm); } -static void test_generated_aes128_iv96(void) +static void test_fusion_aesgcm(void) { - test_generated(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm, 1); + subtest("128", test_fusion_aes128gcm); + subtest("256", test_fusion_aes256gcm); } -static void test_generated_aes256_iv96(void) +static void test_non_temporal_aes128gcm(void) { - test_generated(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm, 1); + test_generated_all(&ptls_non_temporal_aes128gcm, &ptls_minicrypto_aes128gcm); } -static void test_aesgcm(void) +static void test_non_temporal_aes256gcm(void) { - subtest("aes128gcm", test_generated_aes128); - subtest("aes256gcm", test_generated_aes256); - subtest("aes128gcm-iv96", test_generated_aes128_iv96); - subtest("aes256gcm-iv96", test_generated_aes256_iv96); + test_generated_all(&ptls_non_temporal_aes256gcm, &ptls_minicrypto_aes256gcm); } static void test_non_temporal(void) { - test_generated(&ptls_non_temporal_aes128gcm, &ptls_minicrypto_aes128gcm, 0); - test_generated(&ptls_minicrypto_aes128gcm, &ptls_non_temporal_aes128gcm, 0); + subtest("128", test_non_temporal_aes128gcm); + subtest("256", test_non_temporal_aes256gcm); } int main(int argc, char **argv) @@ -489,13 +507,13 @@ int main(int argc, char **argv) subtest("gcm-capacity", gcm_capacity); subtest("gcm-test-vectors", gcm_test_vectors); subtest("gcm-iv96", gcm_iv96); - subtest("aesgcm", test_aesgcm); - subtest("non-temporal128", test_non_temporal); + subtest("fusion-aesgcm", test_fusion_aesgcm); + subtest("non-temporal-avx128", test_non_temporal); if (can256bit) { ptls_fusion_can_avx256 = 1; subtest("gfmul256", test_gfmul); - subtest("non-temporal256", test_non_temporal); + subtest("non-temporal-avx256", test_non_temporal); ptls_fusion_can_avx256 = 0; } else { note("gfmul256: skipping, CPU does not support 256-bit aes / clmul"); From 9f8e12aedfb5c53e9fe49b40d9266dcb30a6ac8a Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 11 May 2022 08:21:45 +0900 Subject: [PATCH 39/48] teach GCC that xor should not be delayed across multiple steps of gfmul, as that causes spills. Works better for encrypt_v128 --- lib/fusion.c | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 682370049..57dbabe70 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -204,19 +204,32 @@ struct ptls_fusion_gfmul_state128 { __m128i hi, lo, mid; }; +#if defined(__GNUC__) && !defined(__clang__) +static inline __m128i xor128(__m128i x, __m128i y) +{ + __m128i ret; + __asm__("vpxor %2, %1, %0" : "=x"(ret) : "x"(x), "xm"(y)); + return ret; +} +#else +#define xor128 _mm_xor_si128 +#endif + static inline void gfmul_do_step128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { - __m128i t = _mm_clmulepi64_si128(precompute->H, X, 0x00); - gstate->lo = _mm_xor_si128(gstate->lo, t); - t = _mm_clmulepi64_si128(precompute->H, X, 0x11); - gstate->hi = _mm_xor_si128(gstate->hi, t); - t = _mm_shuffle_epi32(X, 78); - t = _mm_xor_si128(t, X); - t = _mm_clmulepi64_si128(precompute->r, t, 0x00); - gstate->mid = _mm_xor_si128(gstate->mid, t); + __m128i t1 = _mm_clmulepi64_si128(precompute->H, X, 0x00); + __m128i t2 = _mm_clmulepi64_si128(precompute->H, X, 0x11); + __m128i t3 = _mm_shuffle_epi32(X, 78); + t3 = _mm_xor_si128(t3, X); + t3 = _mm_clmulepi64_si128(precompute->r, t3, 0x00); + gstate->lo = xor128(gstate->lo, t1); + gstate->hi = xor128(gstate->hi, t2); + gstate->mid = xor128(gstate->mid, t3); } +#undef xor128 + static inline void gfmul_firststep128(struct ptls_fusion_gfmul_state128 *gstate, __m128i X, struct ptls_fusion_aesgcm_ghash_precompute128 *precompute) { @@ -1362,15 +1375,8 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void #define STATE_COPY_128B 0x2 int32_t state = 0; - /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. - * Use of thread-local stroage is a hack. GCC spills a lot onto stack, and unless we move `encbuf` outside of stack, temporary - * variables end up being stored onto somewhere not as fast as places near the stack pointer. This provides 18% speed - * improvement on Core i5 9400, at the cost of 2% slowdown on Zen 3. */ -#if defined(__GNUC__) && !defined(__clang__) - static __thread -#endif - uint8_t encbuf[32 * 6] __attribute__((aligned(32))), - *encp; + /* Bytes are written here first then written using NT store instructions, 64 bytes at a time. */ + uint8_t encbuf[32 * 6] __attribute__((aligned(32))), *encp; /* `encbuf` should be large enough to store up to 63-bytes of unaligned bytes, 6 16-byte AES blocks, plus AEAD tag that is * append to the ciphertext before writing the bytes to main memory using NT store instructions. */ From 908f00a247ce973540677470ba7da595ca6a848c Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 11 May 2022 08:52:16 +0900 Subject: [PATCH 40/48] revert to the simple method; previous method (using 256-bit registers for xor as well as retaining 96-bytes of encrypted bytes) was better but GCC can no longer reorder much --- lib/fusion.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 57dbabe70..ad1d96dfc 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1671,35 +1671,32 @@ static size_t non_temporal_decrypt128(ptls_aead_context_t *_ctx, void *_output, /* Main loop. Operate in full blocks (6 * 16 bytes). */ while (PTLS_LIKELY(inlen >= 6 * 16)) { -#define MERGE_BITS(x, y) _mm256_permute2f128_si256(_mm256_castsi128_si256(x), _mm256_castsi128_si256(y), 0x20) - __m256i ks0 = MERGE_BITS(bits0, bits1), ks2 = MERGE_BITS(bits2, bits3), ks4 = MERGE_BITS(bits4, bits5); -#undef MERGE_BITS -#define APPLY(i) \ - do { \ - __m256i bb = _mm256_loadu_si256((void *)(input + i * 16)); \ - _mm256_storeu_si256((void *)(output + i * 16), _mm256_xor_si256(ks##i, bb)); \ - __m128i b0 = _mm256_castsi256_si128(bb), b1 = _mm256_castsi256_si128(_mm256_permute2f128_si256(bb, bb, 0x81)); \ - if (i == 0) { \ - gfmul_firststep128(&gstate, b0, ctx->ghash + 5 - i); \ - } else { \ - gfmul_nextstep128(&gstate, b0, ctx->ghash + 5 - i); \ - } \ - gfmul_nextstep128(&gstate, b1, ctx->ghash + 4 - i); \ - } while (0) +#define DECRYPT(i) _mm_storeu_si128((void *)(output + i * 16), _mm_xor_si128(bits##i, _mm_loadu_si128((void *)(input + i * 16)))) + DECRYPT(0); + DECRYPT(1); + DECRYPT(2); + DECRYPT(3); + DECRYPT(4); + DECRYPT(5); +#undef DECRYPT +#define GFMUL_NEXT(i) gfmul_nextstep128(&gstate, _mm_loadu_si128((void *)(input + i * 16)), ctx->ghash + 5 - i) AESECB6_INIT(); AESECB6_UPDATE(1); - APPLY(0); AESECB6_UPDATE(2); AESECB6_UPDATE(3); - APPLY(2); + gfmul_firststep128(&gstate, _mm_loadu_si128((void *)input), ctx->ghash + 5); AESECB6_UPDATE(4); + GFMUL_NEXT(1); AESECB6_UPDATE(5); - APPLY(4); + GFMUL_NEXT(2); AESECB6_UPDATE(6); + GFMUL_NEXT(3); AESECB6_UPDATE(7); - gfmul_reduce128(&gstate); + GFMUL_NEXT(4); AESECB6_UPDATE(8); + GFMUL_NEXT(5); AESECB6_UPDATE(9); + gfmul_reduce128(&gstate); if (PTLS_UNLIKELY(ctx->super.ecb.rounds != 10)) { size_t i = 10; do { @@ -1710,7 +1707,7 @@ static size_t non_temporal_decrypt128(ptls_aead_context_t *_ctx, void *_output, output += 6 * 16; input += 6 * 16; inlen -= 6 * 16; -#undef APPLY +#undef GFMUL_NEXT } /* Decrypt the remainder as well as finishing GHASH calculation. */ From 688d70cf0b16a522dd3a7d274868601b4e9d9414 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 11 May 2022 13:46:22 +0900 Subject: [PATCH 41/48] expansion should retain 32-byte alignment --- lib/fusion.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/fusion.c b/lib/fusion.c index ad1d96dfc..76df00217 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1019,8 +1019,12 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm return ctx; size_t ctx_size = calc_aesgcm_context_size(&ghash_cnt, ctx->ecb.avx256); - if ((ctx = realloc(ctx, ctx_size)) == NULL) + ptls_fusion_aesgcm_context_t *newp; + if ((newp = aligned_alloc(32, ctx_size)) == NULL) return NULL; + memcpy(newp, ctx, ctx_size); + free(ctx); + ctx = newp; ctx->capacity = capacity; while (ghash_cnt < ctx->ghash_cnt) From 6e6d72b74c5945f81b70561ca6b16648b6ded0af Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 6 Jun 2022 08:41:21 +0900 Subject: [PATCH 42/48] for non-temporal aesgcm, buffer allocation should be cache-line-aware --- include/picotls.h | 4 ++++ lib/picotls.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/picotls.h b/include/picotls.h index cad1206cb..03d5e39c0 100644 --- a/include/picotls.h +++ b/include/picotls.h @@ -70,6 +70,10 @@ extern "C" { #define PTLS_FUZZ_HANDSHAKE 0 #endif +#ifndef PTLS_SIZEOF_CACHE_LINE +#define PTLS_SIZEOF_CACHE_LINE 64 +#endif + #define PTLS_HELLO_RANDOM_SIZE 32 #define PTLS_AES128_KEY_SIZE 16 diff --git a/lib/picotls.c b/lib/picotls.c index 8bb5b136c..033f9be83 100644 --- a/lib/picotls.c +++ b/lib/picotls.c @@ -519,14 +519,14 @@ int ptls_buffer_reserve(ptls_buffer_t *buf, size_t delta) return PTLS_ERROR_NO_MEMORY; if (PTLS_MEMORY_DEBUG || buf->capacity < buf->off + delta) { - uint8_t *newp; + void *newp; size_t new_capacity = buf->capacity; if (new_capacity < 1024) new_capacity = 1024; while (new_capacity < buf->off + delta) { new_capacity *= 2; } - if ((newp = malloc(new_capacity)) == NULL) + if (posix_memalign(&newp, PTLS_SIZEOF_CACHE_LINE, new_capacity) != 0) return PTLS_ERROR_NO_MEMORY; memcpy(newp, buf->base, buf->off); ptls_buffer__release_memory(buf); From f9c85cbc8f5aeb45182e464097bd5b1bb9c59661 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 6 Jun 2022 16:46:16 +0900 Subject: [PATCH 43/48] on windows, use _aligned_malloc / _aligned_free --- lib/picotls.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/picotls.c b/lib/picotls.c index 033f9be83..3ebd33a03 100644 --- a/lib/picotls.c +++ b/lib/picotls.c @@ -509,8 +509,13 @@ static uint64_t ntoh64(const uint8_t *src) void ptls_buffer__release_memory(ptls_buffer_t *buf) { ptls_clear_memory(buf->base, buf->off); - if (buf->is_allocated) + if (buf->is_allocated) { +#ifdef _WINDOWS + _aligned_free(buf->base); +#else free(buf->base); +#endif + } } int ptls_buffer_reserve(ptls_buffer_t *buf, size_t delta) @@ -526,8 +531,13 @@ int ptls_buffer_reserve(ptls_buffer_t *buf, size_t delta) while (new_capacity < buf->off + delta) { new_capacity *= 2; } +#ifdef _WINDOWS + if ((newp = _aligned_malloc(new_capacity, PTLS_SIZEOF_CACHE_LINE)) == NULL) + return PTLS_ERROR_NO_MEMORY; +#else if (posix_memalign(&newp, PTLS_SIZEOF_CACHE_LINE, new_capacity) != 0) return PTLS_ERROR_NO_MEMORY; +#endif memcpy(newp, buf->base, buf->off); ptls_buffer__release_memory(buf); buf->base = newp; From 9dc6982d27a37af9a39140cb7211aefd444f39ed Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 8 Jun 2022 13:25:52 +0900 Subject: [PATCH 44/48] add flag indicating if the AEAD engine uses non-temporal store instructions --- include/picotls.h | 4 ++++ lib/cifra/aes128.c | 1 + lib/cifra/aes256.c | 1 + lib/cifra/chacha20.c | 1 + lib/fusion.c | 4 ++++ lib/openssl.c | 3 +++ lib/ptlsbcrypt.c | 2 ++ 7 files changed, 16 insertions(+) diff --git a/include/picotls.h b/include/picotls.h index 03d5e39c0..f7e36b10b 100644 --- a/include/picotls.h +++ b/include/picotls.h @@ -396,6 +396,10 @@ typedef const struct st_ptls_aead_algorithm_t { * size of the tag */ size_t tag_size; + /** + * if encrypted bytes are going to be written using non-temporal store instructions (i.e., skip cache) + */ + unsigned non_temporal : 1; /** * size of memory allocated for ptls_aead_context_t. AEAD implementations can set this value to something greater than * sizeof(ptls_aead_context_t) and stuff additional data at the bottom of the struct. diff --git a/lib/cifra/aes128.c b/lib/cifra/aes128.c index 9fac84507..34d353b39 100644 --- a/lib/cifra/aes128.c +++ b/lib/cifra/aes128.c @@ -55,6 +55,7 @@ ptls_aead_algorithm_t ptls_minicrypto_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context_t), aead_aes128gcm_setup_crypto}; ptls_cipher_suite_t ptls_minicrypto_aes128gcmsha256 = {.id = PTLS_CIPHER_SUITE_AES_128_GCM_SHA256, diff --git a/lib/cifra/aes256.c b/lib/cifra/aes256.c index 1d604fc23..ae3ac7c9b 100644 --- a/lib/cifra/aes256.c +++ b/lib/cifra/aes256.c @@ -55,6 +55,7 @@ ptls_aead_algorithm_t ptls_minicrypto_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context_t), aead_aes256gcm_setup_crypto}; ptls_cipher_suite_t ptls_minicrypto_aes256gcmsha384 = {.id = PTLS_CIPHER_SUITE_AES_256_GCM_SHA384, diff --git a/lib/cifra/chacha20.c b/lib/cifra/chacha20.c index 2e3584348..4a3f5098f 100644 --- a/lib/cifra/chacha20.c +++ b/lib/cifra/chacha20.c @@ -226,6 +226,7 @@ ptls_aead_algorithm_t ptls_minicrypto_chacha20poly1305 = {"CHACHA20-POLY1305", PTLS_CHACHA20_KEY_SIZE, PTLS_CHACHA20POLY1305_IV_SIZE, PTLS_CHACHA20POLY1305_TAG_SIZE, + 0, sizeof(struct chacha20poly1305_context_t), aead_chacha20poly1305_setup_crypto}; ptls_cipher_suite_t ptls_minicrypto_chacha20poly1305sha256 = {.id = PTLS_CIPHER_SUITE_CHACHA20_POLY1305_SHA256, diff --git a/lib/fusion.c b/lib/fusion.c index 76df00217..766d451ba 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1221,6 +1221,7 @@ ptls_aead_algorithm_t ptls_fusion_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context), aes128gcm_setup}; ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", @@ -1231,6 +1232,7 @@ ptls_aead_algorithm_t ptls_fusion_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aesgcm_context), aes256gcm_setup}; @@ -2132,6 +2134,7 @@ ptls_aead_algorithm_t ptls_non_temporal_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 1, sizeof(struct aesgcm_context), non_temporal_aes128gcm_setup}; ptls_aead_algorithm_t ptls_non_temporal_aes256gcm = {"AES256-GCM", @@ -2142,6 +2145,7 @@ ptls_aead_algorithm_t ptls_non_temporal_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 1, sizeof(struct aesgcm_context), non_temporal_aes256gcm_setup}; diff --git a/lib/openssl.c b/lib/openssl.c index 95356a176..1bccbbbca 100644 --- a/lib/openssl.c +++ b/lib/openssl.c @@ -1593,6 +1593,7 @@ ptls_aead_algorithm_t ptls_openssl_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aead_crypto_context_t), aead_aes128gcm_setup_crypto}; ptls_cipher_algorithm_t ptls_openssl_aes256ecb = { @@ -1609,6 +1610,7 @@ ptls_aead_algorithm_t ptls_openssl_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct aead_crypto_context_t), aead_aes256gcm_setup_crypto}; ptls_hash_algorithm_t ptls_openssl_sha256 = {PTLS_SHA256_BLOCK_SIZE, PTLS_SHA256_DIGEST_SIZE, sha256_create, @@ -1635,6 +1637,7 @@ ptls_aead_algorithm_t ptls_openssl_chacha20poly1305 = {"CHACHA20-POLY1305", PTLS_CHACHA20_KEY_SIZE, PTLS_CHACHA20POLY1305_IV_SIZE, PTLS_CHACHA20POLY1305_TAG_SIZE, + 0, sizeof(struct aead_crypto_context_t), aead_chacha20poly1305_setup_crypto}; ptls_cipher_suite_t ptls_openssl_chacha20poly1305sha256 = {.id = PTLS_CIPHER_SUITE_CHACHA20_POLY1305_SHA256, diff --git a/lib/ptlsbcrypt.c b/lib/ptlsbcrypt.c index 41e6a737d..da8dc16de 100644 --- a/lib/ptlsbcrypt.c +++ b/lib/ptlsbcrypt.c @@ -786,6 +786,7 @@ ptls_aead_algorithm_t ptls_bcrypt_aes128gcm = {"AES128-GCM", PTLS_AES128_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct ptls_bcrypt_aead_context_t), ptls_bcrypt_aead_setup_crypto_aesgcm}; @@ -797,6 +798,7 @@ ptls_aead_algorithm_t ptls_bcrypt_aes256gcm = {"AES256-GCM", PTLS_AES256_KEY_SIZE, PTLS_AESGCM_IV_SIZE, PTLS_AESGCM_TAG_SIZE, + 0, sizeof(struct ptls_bcrypt_aead_context_t), ptls_bcrypt_aead_setup_crypto_aesgcm}; From d155972c533ea8f0ffab0e46d0e755ef91793c31 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 29 Jun 2022 11:15:30 +0900 Subject: [PATCH 45/48] add multivec test --- t/fusion.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/t/fusion.c b/t/fusion.c index 429736971..7c6747f9b 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -378,7 +378,7 @@ static void gcm_iv96(void) } static ptls_aead_algorithm_t *test_generated_encryptor, *test_generated_decryptor; -static int test_generated_iv96; +static int test_generated_iv96, test_generated_multivec; static void test_generated(void) { @@ -415,7 +415,19 @@ static void test_generated(void) ptls_aead_context_t *ctx = ptls_aead_new_direct(test_generated_encryptor, 1, key, iv); if (test_generated_iv96) ptls_aead_xor_iv(ctx, seq32, sizeof(seq32)); - ptls_aead_encrypt(ctx, encrypted, text, textlen, seq, aad, aadlen); + if (test_generated_multivec) { + uint8_t pos1, pos2; + do { + ptls_cipher_encrypt(rand, &pos2, zero, sizeof(pos2)); + } while (pos2 > textlen); + do { + ptls_cipher_encrypt(rand, &pos1, zero, sizeof(pos1)); + } while (pos1 > pos2); + ptls_iovec_t textvec[3] = {{text, pos1}, {text + pos1, pos2 - pos1}, {text + pos2, textlen - pos2}}; + ptls_aead_encrypt_v(ctx, encrypted, textvec, 3, seq, aad, aadlen); + } else { + ptls_aead_encrypt(ctx, encrypted, text, textlen, seq, aad, aadlen); + } ptls_aead_free(ctx); } @@ -442,17 +454,28 @@ static void test_generated(void) ok(0); } -static void test_generated_all(ptls_aead_algorithm_t *e1, ptls_aead_algorithm_t *e2) +static void test_generated_all(ptls_aead_algorithm_t *e1, ptls_aead_algorithm_t *e2, int can_multivec) { test_generated_encryptor = e1; test_generated_decryptor = e2; + test_generated_iv96 = 0; subtest("encrypt", test_generated); test_generated_iv96 = 1; subtest("encrypt-iv96", test_generated); + if (can_multivec) { + test_generated_multivec = 1; + test_generated_iv96 = 0; + subtest("encrypt", test_generated); + test_generated_iv96 = 1; + subtest("encrypt-iv96", test_generated); + test_generated_multivec = 0; + } + test_generated_encryptor = e2; test_generated_decryptor = e1; + test_generated_iv96 = 0; subtest("decrypt", test_generated); test_generated_iv96 = 1; @@ -461,12 +484,12 @@ static void test_generated_all(ptls_aead_algorithm_t *e1, ptls_aead_algorithm_t static void test_fusion_aes128gcm(void) { - test_generated_all(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm); + test_generated_all(&ptls_fusion_aes128gcm, &ptls_minicrypto_aes128gcm, 0); } static void test_fusion_aes256gcm(void) { - test_generated_all(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm); + test_generated_all(&ptls_fusion_aes256gcm, &ptls_minicrypto_aes256gcm, 0); } static void test_fusion_aesgcm(void) @@ -477,12 +500,12 @@ static void test_fusion_aesgcm(void) static void test_non_temporal_aes128gcm(void) { - test_generated_all(&ptls_non_temporal_aes128gcm, &ptls_minicrypto_aes128gcm); + test_generated_all(&ptls_non_temporal_aes128gcm, &ptls_minicrypto_aes128gcm, 1); } static void test_non_temporal_aes256gcm(void) { - test_generated_all(&ptls_non_temporal_aes256gcm, &ptls_minicrypto_aes256gcm); + test_generated_all(&ptls_non_temporal_aes256gcm, &ptls_minicrypto_aes256gcm, 1); } static void test_non_temporal(void) From 78e6c3a71a026864a346b7f23195dd4950dcbfac Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 29 Jun 2022 11:17:06 +0900 Subject: [PATCH 46/48] fix handling of zero-length vec --- lib/fusion.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index 766d451ba..da8e6029c 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1472,14 +1472,16 @@ static void non_temporal_encrypt_v128(struct st_ptls_aead_context_t *_ctx, void --srclen; } if (PTLS_UNLIKELY(srclen == 0)) { - if (src_vecleft == 0) { - break; - } else { + do { + if (src_vecleft == 0) + break; src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; - } + } while (srclen == 0); + if (srclen == 0) + break; } } while (bytes_copied < 6 * 16); #define APPLY(i) _mm_storeu_si128((void *)(encp + i * 16), _mm_xor_si128(_mm_loadu_si128((void *)(encp + i * 16)), bits##i)) From 629b800d0210ce6c9000daac1e0d80fc74bdf0ac Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 29 Jun 2022 12:31:35 +0900 Subject: [PATCH 47/48] ditto for aesni256-side --- lib/fusion.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/fusion.c b/lib/fusion.c index da8e6029c..a73bcf900 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -1975,14 +1975,16 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void --srclen; } if (PTLS_UNLIKELY(srclen == 0)) { - if (src_vecleft == 0) { - break; - } else { + do { + if (src_vecleft == 0) + break; src = (void *)input[0].base; srclen = input[0].len; ++input; --src_vecleft; - } + } while (srclen == 0); + if (srclen == 0) + break; } } while (bytes_copied < 6 * 32); #define APPLY(i) \ From 65d3e799269e095156acd582dca083d2e6a4e89e Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 29 Jun 2022 12:04:02 +0900 Subject: [PATCH 48/48] s/avx256/aesni256/, as we use avx256 instructions even when not using 256-bit variant of aesni --- include/picotls/fusion.h | 4 +-- lib/fusion.c | 55 ++++++++++++++++++++-------------------- t/fusion.c | 24 +++++++++--------- 3 files changed, 42 insertions(+), 41 deletions(-) diff --git a/include/picotls/fusion.h b/include/picotls/fusion.h index dff299753..0c7f211f5 100644 --- a/include/picotls/fusion.h +++ b/include/picotls/fusion.h @@ -40,7 +40,7 @@ typedef struct ptls_fusion_aesecb_context { __m256i m256[PTLS_FUSION_AES256_ROUNDS + 1]; } keys; unsigned rounds; - uint8_t avx256; + uint8_t aesni256; } __attribute__((aligned(32))) ptls_fusion_aesecb_context_t; typedef struct ptls_fusion_aesgcm_context ptls_fusion_aesgcm_context_t; @@ -94,7 +94,7 @@ int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, * when `ptls_fusion_is_supported_by_cpu` is called. Users can update the flag to enforce behavior. Engines that do not have support * for these 256-bit instructions will continue using the 128-bit ones, even when this flag is set. */ -extern int ptls_fusion_can_avx256; +extern int ptls_fusion_can_aesni256; extern ptls_cipher_algorithm_t ptls_fusion_aes128ctr, ptls_fusion_aes256ctr; extern ptls_aead_algorithm_t ptls_fusion_aes128gcm, ptls_fusion_aes256gcm; extern ptls_aead_algorithm_t ptls_non_temporal_aes128gcm, ptls_non_temporal_aes256gcm; diff --git a/lib/fusion.c b/lib/fusion.c index a73bcf900..6c1afa104 100644 --- a/lib/fusion.c +++ b/lib/fusion.c @@ -318,7 +318,7 @@ static inline __m128i gfmul_get_tag256(struct ptls_fusion_gfmul_state256 *gstate static inline __m128i aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, __m128i v) { -#define ROUNDKEY(i) (ctx->avx256 ? _mm256_castsi256_si128(ctx->keys.m256[i]) : ctx->keys.m128[i]) +#define ROUNDKEY(i) (ctx->aesni256 ? _mm256_castsi256_si128(ctx->keys.m256[i]) : ctx->keys.m128[i]) v = _mm_xor_si128(v, ROUNDKEY(0)); for (size_t i = 1; i < ctx->rounds; ++i) @@ -851,7 +851,7 @@ static __m128i expand_key(__m128i key, __m128i temp) return key; } -void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int avx256) +void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int aesni256) { assert(is_enc && "decryption is not supported (yet)"); @@ -868,7 +868,7 @@ void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, cons assert(!"invalid key size; AES128 / AES256 are supported"); break; } - ctx->avx256 = avx256; + ctx->aesni256 = aesni256; /* load and expand keys using keys.m128 */ ctx->keys.m128[i++] = _mm_loadu_si128((__m128i *)key); @@ -903,8 +903,8 @@ void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, cons #undef EXPAND } - /* convert to keys.m256 if avx256 is used */ - if (ctx->avx256) { + /* convert to keys.m256 if aesni256 is used */ + if (ctx->aesni256) { size_t i = ctx->rounds; do { ctx->keys.m256[i] = _mm256_broadcastsi128_si256(ctx->keys.m128[i]); @@ -937,7 +937,7 @@ static void setup_one_ghash_entry(ptls_fusion_aesgcm_context_t *ctx) { __m128i *H, *r, *Hprev, H0; - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { struct ptls_fusion_aesgcm_context256 *ctx256 = (void *)ctx; #define GET_SLOT(i, mem) (&ctx256->ghash[(i) / 2].mem[(i) % 2 == 0]) H = GET_SLOT(ctx->ghash_cnt, H); @@ -962,11 +962,11 @@ static void setup_one_ghash_entry(ptls_fusion_aesgcm_context_t *ctx) ++ctx->ghash_cnt; } -static size_t calc_aesgcm_context_size(size_t *ghash_cnt, int avx256) +static size_t calc_aesgcm_context_size(size_t *ghash_cnt, int aesni256) { size_t sz; - if (avx256) { + if (aesni256) { if (*ghash_cnt % 2 != 0) ++*ghash_cnt; sz = offsetof(struct ptls_fusion_aesgcm_context256, ghash) + @@ -978,22 +978,22 @@ static size_t calc_aesgcm_context_size(size_t *ghash_cnt, int avx256) return sz; } -static ptls_fusion_aesgcm_context_t *new_aesgcm(const void *key, size_t key_size, size_t capacity, int avx256) +static ptls_fusion_aesgcm_context_t *new_aesgcm(const void *key, size_t key_size, size_t capacity, int aesni256) { ptls_fusion_aesgcm_context_t *ctx; - size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity), ctx_size = calc_aesgcm_context_size(&ghash_cnt, avx256); + size_t ghash_cnt = aesgcm_calc_ghash_cnt(capacity), ctx_size = calc_aesgcm_context_size(&ghash_cnt, aesni256); if ((ctx = aligned_alloc(32, ctx_size)) == NULL) return NULL; - ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size, avx256); + ptls_fusion_aesecb_init(&ctx->ecb, 1, key, key_size, aesni256); ctx->capacity = capacity; __m128i H0 = aesecb_encrypt(&ctx->ecb, _mm_setzero_si128()); H0 = _mm_shuffle_epi8(H0, byteswap128); H0 = transformH(H0); - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { ((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash[0].H[1] = H0; } else { ((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash[0].H = H0; @@ -1018,7 +1018,7 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm if (ghash_cnt <= ctx->ghash_cnt) return ctx; - size_t ctx_size = calc_aesgcm_context_size(&ghash_cnt, ctx->ecb.avx256); + size_t ctx_size = calc_aesgcm_context_size(&ghash_cnt, ctx->ecb.aesni256); ptls_fusion_aesgcm_context_t *newp; if ((newp = aligned_alloc(32, ctx_size)) == NULL) return NULL; @@ -1035,7 +1035,7 @@ ptls_fusion_aesgcm_context_t *ptls_fusion_aesgcm_set_capacity(ptls_fusion_aesgcm void ptls_fusion_aesgcm_free(ptls_fusion_aesgcm_context_t *ctx) { - ptls_clear_memory(ctx, calc_aesgcm_context_size(&ctx->ghash_cnt, ctx->ecb.avx256)); + ptls_clear_memory(ctx, calc_aesgcm_context_size(&ctx->ghash_cnt, ctx->ecb.aesni256)); /* skip ptls_fusion_aesecb_dispose, based on the knowledge that it does not allocate memory elsewhere */ free(ctx); @@ -1077,7 +1077,7 @@ static int aesctr_setup(ptls_cipher_context_t *_ctx, int is_enc, const void *key ctx->super.do_dispose = ctr_dispose; ctx->super.do_init = ctr_init; ctx->super.do_transform = ctr_transform; - ptls_fusion_aesecb_init(&ctx->fusion, 1, key, key_size, 0 /* probably we do not need avx256 for CTR? */); + ptls_fusion_aesecb_init(&ctx->fusion, 1, key, key_size, 0 /* probably we do not need aesni256 for CTR? */); ctx->is_ready = 0; return 0; @@ -1185,7 +1185,7 @@ static int aesgcm_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, ctx->super.do_encrypt_v = aead_do_encrypt_v; ctx->super.do_decrypt = aead_do_decrypt; - ctx->aesgcm = new_aesgcm(key, key_size, 1500 /* assume ordinary packet size */, 0 /* no support for avx256 yet */); + ctx->aesgcm = new_aesgcm(key, key_size, 1500 /* assume ordinary packet size */, 0 /* no support for aesni256 yet */); return 0; } @@ -1200,7 +1200,7 @@ static int aes256gcm_setup(ptls_aead_context_t *ctx, int is_enc, const void *key return aesgcm_setup(ctx, is_enc, key, iv, PTLS_AES256_KEY_SIZE); } -int ptls_fusion_can_avx256 = 0; +int ptls_fusion_can_aesni256 = 0; ptls_cipher_algorithm_t ptls_fusion_aes128ctr = {"AES128-CTR", PTLS_AES128_KEY_SIZE, 1, // block size @@ -2090,7 +2090,7 @@ static void non_temporal_encrypt_v256(struct st_ptls_aead_context_t *_ctx, void static int non_temporal_setup(ptls_aead_context_t *_ctx, int is_enc, const void *key, const void *iv, size_t key_size) { struct aesgcm_context *ctx = (struct aesgcm_context *)_ctx; - int avx256 = is_enc && ptls_fusion_can_avx256; + int aesni256 = is_enc && ptls_fusion_can_aesni256; ctx->static_iv = loadn128(iv, PTLS_AESGCM_IV_SIZE); ctx->static_iv = _mm_shuffle_epi8(ctx->static_iv, byteswap128); @@ -2104,18 +2104,19 @@ static int non_temporal_setup(ptls_aead_context_t *_ctx, int is_enc, const void ctx->super.do_encrypt_final = NULL; if (is_enc) { ctx->super.do_encrypt = ptls_aead__do_encrypt; - ctx->super.do_encrypt_v = avx256 ? non_temporal_encrypt_v256 : non_temporal_encrypt_v128; + ctx->super.do_encrypt_v = aesni256 ? non_temporal_encrypt_v256 : non_temporal_encrypt_v128; ctx->super.do_decrypt = NULL; } else { - assert(!avx256); + assert(!aesni256); ctx->super.do_encrypt = NULL; ctx->super.do_encrypt_v = NULL; ctx->super.do_decrypt = non_temporal_decrypt128; } - ctx->aesgcm = new_aesgcm(key, key_size, - 7 * (ptls_fusion_can_avx256 ? 32 : 16), // 6 blocks at once, plus len(A) | len(C) that we might append - avx256); + ctx->aesgcm = + new_aesgcm(key, key_size, + 7 * (ptls_fusion_can_aesni256 ? 32 : 16), // 6 blocks at once, plus len(A) | len(C) that we might append + aesni256); return 0; } @@ -2184,8 +2185,8 @@ int ptls_fusion_is_supported_by_cpu(void) is_supported = /* AVX2 */ (leaf7_ebx & (1 << 5)) != 0; /* enable 256-bit mode if possible */ - if (is_supported && (leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_avx256) - ptls_fusion_can_avx256 = 1; + if (is_supported && (leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_aesni256) + ptls_fusion_can_aesni256 = 1; } } @@ -2216,8 +2217,8 @@ int ptls_fusion_is_supported_by_cpu(void) return 0; /* enable 256-bit mode if possible */ - if ((leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_avx256) - ptls_fusion_can_avx256 = 1; + if ((leaf7_ecx & 0x600) != 0 && !ptls_fusion_can_aesni256) + ptls_fusion_can_aesni256 = 1; return 1; } diff --git a/t/fusion.c b/t/fusion.c index 7c6747f9b..d5c51daf4 100644 --- a/t/fusion.c +++ b/t/fusion.c @@ -89,14 +89,14 @@ static void test_gfmul(void) ptls_fusion_aesgcm_context_t *ctx; size_t ghash_cnt = 4; - ctx = malloc(calc_aesgcm_context_size(&ghash_cnt, ptls_fusion_can_avx256)); + ctx = malloc(calc_aesgcm_context_size(&ghash_cnt, ptls_fusion_can_aesni256)); *ctx = (ptls_fusion_aesgcm_context_t){ - .ecb = {.avx256 = ptls_fusion_can_avx256}, + .ecb = {.aesni256 = ptls_fusion_can_aesni256}, .capacity = ghash_cnt * 16, }; __m128i H0 = _mm_loadu_si128((void *)"hello world bye"); - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { ((struct ptls_fusion_aesgcm_context256 *)ctx)->ghash[0].H[1] = H0; } else { ((struct ptls_fusion_aesgcm_context128 *)ctx)->ghash[0].H = H0; @@ -111,7 +111,7 @@ static void test_gfmul(void) { /* one block */ static const char input[32] = "deaddeadbeefbeef"; /* latter 16-byte is NUL */ __m128i hash; - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, GHASH256); @@ -130,7 +130,7 @@ static void test_gfmul(void) { /* two blocks */ static const char input[32] = "Lorem ipsum dolor sit amet, con"; __m128i hash; - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256); @@ -150,7 +150,7 @@ static void test_gfmul(void) { /* three blocks */ static const char input[64] = "The quick brown fox jumps over the lazy dog."; __m128i hash; - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 1, GHASH256 + 1); @@ -172,7 +172,7 @@ static void test_gfmul(void) { /* five blocks */ static const char input[80] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor "; __m128i hash; - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256 + 1); @@ -200,7 +200,7 @@ static void test_gfmul(void) static const char input[96] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut la"; __m128i hash; - if (ctx->ecb.avx256) { + if (ctx->ecb.aesni256) { struct ptls_fusion_gfmul_state256 state; state.lo = _mm256_setzero_si256(); gfmul_firststep256(&state, _mm256_loadu_si256((void *)input), 0, GHASH256 + 1); @@ -520,8 +520,8 @@ int main(int argc, char **argv) note("CPU does have the necessary features (avx2, aes, pclmul)\n"); return done_testing(); } - int can256bit = ptls_fusion_can_avx256; - ptls_fusion_can_avx256 = 0; + int can256bit = ptls_fusion_can_aesni256; + ptls_fusion_can_aesni256 = 0; subtest("loadn128", test_loadn128); subtest("ecb", test_ecb); @@ -534,10 +534,10 @@ int main(int argc, char **argv) subtest("non-temporal-avx128", test_non_temporal); if (can256bit) { - ptls_fusion_can_avx256 = 1; + ptls_fusion_can_aesni256 = 1; subtest("gfmul256", test_gfmul); subtest("non-temporal-avx256", test_non_temporal); - ptls_fusion_can_avx256 = 0; + ptls_fusion_can_aesni256 = 0; } else { note("gfmul256: skipping, CPU does not support 256-bit aes / clmul"); }