New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
enable avx512 support for base64 encoding. Reuse WojciechMula/base64-… #102
Closed
Closed
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #include <stdint.h> | ||
| #include <stddef.h> | ||
| #include <stdlib.h> | ||
|
|
||
| #include "../../../include/libbase64.h" | ||
| #include "../../tables/tables.h" | ||
| #include "../../codecs.h" | ||
| #include "config.h" | ||
| #include "../../env.h" | ||
|
|
||
| #if HAVE_AVX512 | ||
| #include <immintrin.h> | ||
|
|
||
| #include "dec_reshuffle.c" | ||
| #include "dec_loop.c" | ||
| #include "enc_reshuffle_translate.c" | ||
| #include "enc_loop.c" | ||
|
|
||
| #endif // HAVE_AVX512 | ||
|
|
||
| BASE64_ENC_FUNCTION(avx512) | ||
| { | ||
| #if HAVE_AVX2 | ||
| #include "../generic/enc_head.c" | ||
| enc_loop_avx512(&s, &slen, &o, &olen); | ||
| #include "../generic/enc_tail.c" | ||
| #else | ||
| BASE64_ENC_STUB | ||
| #endif | ||
| } | ||
|
|
||
| // Reuse AVX2 decoding. Not supporting AVX512 at present | ||
| BASE64_DEC_FUNCTION(avx512) | ||
| { | ||
| #if HAVE_AVX2 | ||
| #include "../generic/dec_head.c" | ||
| dec_loop_avx2(&s, &slen, &o, &olen); | ||
| #include "../generic/dec_tail.c" | ||
| #else | ||
| BASE64_DEC_STUB | ||
| #endif | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| static inline int | ||
| dec_loop_avx2_inner (const uint8_t **s, uint8_t **o, size_t *rounds) | ||
| { | ||
| const __m256i lut_lo = _mm256_setr_epi8( | ||
| 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, | ||
| 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A, | ||
| 0x15, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, | ||
| 0x11, 0x11, 0x13, 0x1A, 0x1B, 0x1B, 0x1B, 0x1A); | ||
|
|
||
| const __m256i lut_hi = _mm256_setr_epi8( | ||
| 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, | ||
| 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, | ||
| 0x10, 0x10, 0x01, 0x02, 0x04, 0x08, 0x04, 0x08, | ||
| 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10); | ||
|
|
||
| const __m256i lut_roll = _mm256_setr_epi8( | ||
| 0, 16, 19, 4, -65, -65, -71, -71, | ||
| 0, 0, 0, 0, 0, 0, 0, 0, | ||
| 0, 16, 19, 4, -65, -65, -71, -71, | ||
| 0, 0, 0, 0, 0, 0, 0, 0); | ||
|
|
||
| const __m256i mask_2F = _mm256_set1_epi8(0x2F); | ||
|
|
||
| // Load input: | ||
| __m256i str = _mm256_loadu_si256((__m256i *) *s); | ||
|
|
||
| // See the SSSE3 decoder for an explanation of the algorithm. | ||
| const __m256i hi_nibbles = _mm256_and_si256(_mm256_srli_epi32(str, 4), mask_2F); | ||
| const __m256i lo_nibbles = _mm256_and_si256(str, mask_2F); | ||
| const __m256i hi = _mm256_shuffle_epi8(lut_hi, hi_nibbles); | ||
| const __m256i lo = _mm256_shuffle_epi8(lut_lo, lo_nibbles); | ||
|
|
||
| if (!_mm256_testz_si256(lo, hi)) { | ||
| return 0; | ||
| } | ||
|
|
||
| const __m256i eq_2F = _mm256_cmpeq_epi8(str, mask_2F); | ||
| const __m256i roll = _mm256_shuffle_epi8(lut_roll, _mm256_add_epi8(eq_2F, hi_nibbles)); | ||
|
|
||
| // Now simply add the delta values to the input: | ||
| str = _mm256_add_epi8(str, roll); | ||
|
|
||
| // Reshuffle the input to packed 12-byte output format: | ||
| str = dec_reshuffle(str); | ||
|
|
||
| // Store the output: | ||
| _mm256_storeu_si256((__m256i *) *o, str); | ||
|
|
||
| *s += 32; | ||
| *o += 24; | ||
| *rounds -= 1; | ||
|
|
||
| return 1; | ||
| } | ||
|
|
||
| static inline void | ||
| dec_loop_avx2 (const uint8_t **s, size_t *slen, uint8_t **o, size_t *olen) | ||
| { | ||
| if (*slen < 45) { | ||
| return; | ||
| } | ||
|
|
||
| // Process blocks of 32 bytes per round. Because 8 extra zero bytes are | ||
| // written after the output, ensure that there will be at least 13 | ||
| // bytes of input data left to cover the gap. (11 data bytes and up to | ||
| // two end-of-string markers.) | ||
| size_t rounds = (*slen - 13) / 32; | ||
|
|
||
| *slen -= rounds * 32; // 32 bytes consumed per round | ||
| *olen += rounds * 24; // 24 bytes produced per round | ||
|
|
||
| do { | ||
| if (rounds >= 8) { | ||
| if (dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds)) { | ||
| continue; | ||
| } | ||
| break; | ||
| } | ||
| if (rounds >= 4) { | ||
| if (dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds)) { | ||
| continue; | ||
| } | ||
| break; | ||
| } | ||
| if (rounds >= 2) { | ||
| if (dec_loop_avx2_inner(s, o, &rounds) && | ||
| dec_loop_avx2_inner(s, o, &rounds)) { | ||
| continue; | ||
| } | ||
| break; | ||
| } | ||
| dec_loop_avx2_inner(s, o, &rounds); | ||
| break; | ||
|
|
||
| } while (rounds > 0); | ||
|
|
||
| // Adjust for any rounds that were skipped: | ||
| *slen += rounds * 32; | ||
| *olen -= rounds * 24; | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| static inline __m256i | ||
| dec_reshuffle (const __m256i in) | ||
| { | ||
| // in, lower lane, bits, upper case are most significant bits, lower | ||
| // case are least significant bits: | ||
| // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ | ||
| // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG | ||
| // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD | ||
| // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA | ||
|
|
||
| const __m256i merge_ab_and_bc = _mm256_maddubs_epi16(in, _mm256_set1_epi32(0x01400140)); | ||
| // 0000kkkk LLllllll 0000JJJJ JJjjKKKK | ||
| // 0000hhhh IIiiiiii 0000GGGG GGggHHHH | ||
| // 0000eeee FFffffff 0000DDDD DDddEEEE | ||
| // 0000bbbb CCcccccc 0000AAAA AAaaBBBB | ||
|
|
||
| __m256i out = _mm256_madd_epi16(merge_ab_and_bc, _mm256_set1_epi32(0x00011000)); | ||
| // 00000000 JJJJJJjj KKKKkkkk LLllllll | ||
| // 00000000 GGGGGGgg HHHHhhhh IIiiiiii | ||
| // 00000000 DDDDDDdd EEEEeeee FFffffff | ||
| // 00000000 AAAAAAaa BBBBbbbb CCcccccc | ||
|
|
||
| // Pack bytes together in each lane: | ||
| out = _mm256_shuffle_epi8(out, _mm256_setr_epi8( | ||
| 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1, | ||
| 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1)); | ||
| // 00000000 00000000 00000000 00000000 | ||
| // LLllllll KKKKkkkk JJJJJJjj IIiiiiii | ||
| // HHHHhhhh GGGGGGgg FFffffff EEEEeeee | ||
| // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa | ||
|
|
||
| // Pack lanes: | ||
| return _mm256_permutevar8x32_epi32(out, _mm256_setr_epi32(0, 1, 2, 4, 5, 6, -1, -1)); | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Typo? Should be
dec_loop_avx512There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No. This PR is only for the encoding part for AVX512 because Node.js only depends on base64 SIMD encoding. In general Base64 decoding cannot be vectorized when there are space chars in input. To not break your project in general, I reuse the AVX2 for decoding part in my PR.