|
22 | 22 | * (of length KYBER_POLYCOMPRESSEDBYTES) |
23 | 23 | * - const poly *a: pointer to input polynomial |
24 | 24 | **************************************************/ |
25 | | -#if (KYBER_POLYCOMPRESSEDBYTES == 96) |
26 | | -void poly_compress(uint8_t r[96], const poly * restrict a) |
27 | | -{ |
28 | | - unsigned int i; |
29 | | - __m256i f0, f1, f2, f3; |
30 | | - __m128i t0, t1; |
31 | | - const __m256i v = _mm256_load_si256(&qdata.vec[_16XV/16]); |
32 | | - const __m256i shift1 = _mm256_set1_epi16(1 << 8); |
33 | | - const __m256i mask = _mm256_set1_epi16(7); |
34 | | - const __m256i shift2 = _mm256_set1_epi16((8 << 8) + 1); |
35 | | - const __m256i shift3 = _mm256_set1_epi32((64 << 16) + 1); |
36 | | - const __m256i sllvdidx = _mm256_set1_epi64x(12LL << 32); |
37 | | - const __m256i shufbidx = _mm256_set_epi8( 8, 2, 1, 0,-1,-1,-1,-1,14,13,12, 6, 5, 4,10, 9, |
38 | | - -1,-1,-1,-1,14,13,12, 6, 5, 4,10, 9, 8, 2, 1, 0); |
39 | | - |
40 | | - for(i=0;i<KYBER_N/64;i++) { |
41 | | - f0 = _mm256_load_si256(&a->vec[4*i+0]); |
42 | | - f1 = _mm256_load_si256(&a->vec[4*i+1]); |
43 | | - f2 = _mm256_load_si256(&a->vec[4*i+2]); |
44 | | - f3 = _mm256_load_si256(&a->vec[4*i+3]); |
45 | | - f0 = _mm256_mulhi_epi16(f0,v); |
46 | | - f1 = _mm256_mulhi_epi16(f1,v); |
47 | | - f2 = _mm256_mulhi_epi16(f2,v); |
48 | | - f3 = _mm256_mulhi_epi16(f3,v); |
49 | | - f0 = _mm256_mulhrs_epi16(f0,shift1); |
50 | | - f1 = _mm256_mulhrs_epi16(f1,shift1); |
51 | | - f2 = _mm256_mulhrs_epi16(f2,shift1); |
52 | | - f3 = _mm256_mulhrs_epi16(f3,shift1); |
53 | | - f0 = _mm256_and_si256(f0,mask); |
54 | | - f1 = _mm256_and_si256(f1,mask); |
55 | | - f2 = _mm256_and_si256(f2,mask); |
56 | | - f3 = _mm256_and_si256(f3,mask); |
57 | | - f0 = _mm256_packus_epi16(f0,f1); |
58 | | - f2 = _mm256_packus_epi16(f2,f3); |
59 | | - f0 = _mm256_maddubs_epi16(f0,shift2); // a0 a1 a2 a3 b0 b1 b2 b3 a4 a5 a6 a7 b4 b5 b6 b7 |
60 | | - f2 = _mm256_maddubs_epi16(f2,shift2); // c0 c1 c2 c3 d0 d1 d2 d3 c4 c5 c6 c7 d4 d5 d6 d7 |
61 | | - f0 = _mm256_madd_epi16(f0,shift3); // a0 a1 b0 b1 a2 a3 b2 b3 |
62 | | - f2 = _mm256_madd_epi16(f2,shift3); // c0 c1 d0 d1 c2 c3 d2 d3 |
63 | | - f0 = _mm256_sllv_epi32(f0,sllvdidx); |
64 | | - f2 = _mm256_sllv_epi32(f2,sllvdidx); |
65 | | - f0 = _mm256_hadd_epi32(f0,f2); // a0 c0 c0 d0 a1 b1 c1 d1 |
66 | | - f0 = _mm256_permute4x64_epi64(f0,0xD8); // a0 b0 a1 b1 c0 d0 c1 d1 |
67 | | - f0 = _mm256_shuffle_epi8(f0,shufbidx); |
68 | | - t0 = _mm256_castsi256_si128(f0); |
69 | | - t1 = _mm256_extracti128_si256(f0,1); |
70 | | - t0 = _mm_blend_epi32(t0,t1,0x08); |
71 | | - _mm_storeu_si128((__m128i *)&r[24*i+ 0],t0); |
72 | | - _mm_storel_epi64((__m128i *)&r[24*i+16],t1); |
73 | | - } |
74 | | -} |
75 | | - |
76 | | -/************************************************* |
77 | | -* Name: poly_decompress |
78 | | -* |
79 | | -* Description: De-serialization and subsequent decompression of a polynomial; |
80 | | -* approximate inverse of poly_compress |
81 | | -* |
82 | | -* Arguments: - poly *r: pointer to output polynomial |
83 | | -* - const uint8_t *a: pointer to input byte array |
84 | | -* (of length KYBER_POLYCOMPRESSEDBYTES bytes) |
85 | | -**************************************************/ |
86 | | -void poly_decompress(poly * restrict r, const uint8_t a[96]) |
87 | | -{ |
88 | | - unsigned int i; |
89 | | - __m128i t; |
90 | | - __m256i f; |
91 | | - const __m256i q = _mm256_load_si256(&qdata.vec[_16XQ/16]); |
92 | | - const __m256i shufbidx = _mm256_set_epi8(5,5,5,5,5,4,4,4,4,4,4,3,3,3,3,3, |
93 | | - 2,2,2,2,2,1,1,1,1,1,1,0,0,0,0,0); |
94 | | - const __m256i mask = _mm256_set_epi16(224,28,896,112,14,448,56,7, |
95 | | - 224,28,896,112,14,448,56,7); |
96 | | - const __m256i shift = _mm256_set_epi16(128,1024,32,256,2048,64,512,4096, |
97 | | - 128,1024,32,256,2048,64,512,4096); |
98 | | - |
99 | | - for(i=0;i<KYBER_N/16;i++) { |
100 | | - t = _mm_castps_si128(_mm_load_ss((float *)&a[6*i+0]))); |
101 | | - t = _mm_insert_epi16(t,*(int16_t *)&a[6*i+4],2); |
102 | | - f = _mm256_broadcastsi128_si256(t); |
103 | | - f = _mm256_blend_epi16(f,g,0x); |
104 | | - f = _mm256_shuffle_epi8(f,shufbidx); |
105 | | - f = _mm256_and_si256(f,mask); |
106 | | - f = _mm256_mullo_epi16(f,shift); |
107 | | - f = _mm256_mulhrs_epi16(f,q); |
108 | | - _mm256_store_si256(&r->vec[i],f); |
109 | | - } |
110 | | -} |
111 | | - |
112 | | -#elif (KYBER_POLYCOMPRESSEDBYTES == 128) |
| 25 | +#if (KYBER_POLYCOMPRESSEDBYTES == 128) |
113 | 26 | void poly_compress(uint8_t r[128], const poly * restrict a) |
114 | 27 | { |
115 | 28 | unsigned int i; |
|
0 commit comments