Skip to content

Commit 9ad58b4

Browse files
ardbiesheuvelherbertx
ardbiesheuvel
authored and
herbertx
committed
crypto: x86/serpent - drop dependency on glue helper
Replace the glue helper dependency with implementations of ECB and CBC based on the new CPP macros, which avoid the need for indirect calls. Acked-by: Eric Biggers <ebiggers@google.com> Signed-off-by: Ard Biesheuvel <ardb@kernel.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
1 parent 407d409 commit 9ad58b4

File tree

4 files changed

+61
-157
lines changed

4 files changed

+61
-157
lines changed

‎arch/x86/crypto/serpent_avx2_glue.c‎

Lines changed: 20 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@
1212
#include <crypto/algapi.h>
1313
#include <crypto/internal/simd.h>
1414
#include <crypto/serpent.h>
15-
#include <asm/crypto/glue_helper.h>
1615
#include <asm/crypto/serpent-avx.h>
1716

17+
#include "ecb_cbc_helpers.h"
18+
1819
#define SERPENT_AVX2_PARALLEL_BLOCKS 16
1920

2021
/* 16-way AVX2 parallel cipher functions */
@@ -28,72 +29,38 @@ static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
2829
return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
2930
}
3031

31-
static const struct common_glue_ctx serpent_enc = {
32-
.num_funcs = 3,
33-
.fpu_blocks_limit = 8,
34-
35-
.funcs = { {
36-
.num_blocks = 16,
37-
.fn_u = { .ecb = serpent_ecb_enc_16way }
38-
}, {
39-
.num_blocks = 8,
40-
.fn_u = { .ecb = serpent_ecb_enc_8way_avx }
41-
}, {
42-
.num_blocks = 1,
43-
.fn_u = { .ecb = __serpent_encrypt }
44-
} }
45-
};
46-
47-
static const struct common_glue_ctx serpent_dec = {
48-
.num_funcs = 3,
49-
.fpu_blocks_limit = 8,
50-
51-
.funcs = { {
52-
.num_blocks = 16,
53-
.fn_u = { .ecb = serpent_ecb_dec_16way }
54-
}, {
55-
.num_blocks = 8,
56-
.fn_u = { .ecb = serpent_ecb_dec_8way_avx }
57-
}, {
58-
.num_blocks = 1,
59-
.fn_u = { .ecb = __serpent_decrypt }
60-
} }
61-
};
62-
63-
static const struct common_glue_ctx serpent_dec_cbc = {
64-
.num_funcs = 3,
65-
.fpu_blocks_limit = 8,
66-
67-
.funcs = { {
68-
.num_blocks = 16,
69-
.fn_u = { .cbc = serpent_cbc_dec_16way }
70-
}, {
71-
.num_blocks = 8,
72-
.fn_u = { .cbc = serpent_cbc_dec_8way_avx }
73-
}, {
74-
.num_blocks = 1,
75-
.fn_u = { .cbc = __serpent_decrypt }
76-
} }
77-
};
78-
7932
static int ecb_encrypt(struct skcipher_request *req)
8033
{
81-
return glue_ecb_req_128bit(&serpent_enc, req);
34+
ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
35+
ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way);
36+
ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx);
37+
ECB_BLOCK(1, __serpent_encrypt);
38+
ECB_WALK_END();
8239
}
8340

8441
static int ecb_decrypt(struct skcipher_request *req)
8542
{
86-
return glue_ecb_req_128bit(&serpent_dec, req);
43+
ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
44+
ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way);
45+
ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx);
46+
ECB_BLOCK(1, __serpent_decrypt);
47+
ECB_WALK_END();
8748
}
8849

8950
static int cbc_encrypt(struct skcipher_request *req)
9051
{
91-
return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req);
52+
CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1);
53+
CBC_ENC_BLOCK(__serpent_encrypt);
54+
CBC_WALK_END();
9255
}
9356

9457
static int cbc_decrypt(struct skcipher_request *req)
9558
{
96-
return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
59+
CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
60+
CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way);
61+
CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx);
62+
CBC_DEC_BLOCK(1, __serpent_decrypt);
63+
CBC_WALK_END();
9764
}
9865

9966
static struct skcipher_alg serpent_algs[] = {

‎arch/x86/crypto/serpent_avx_glue.c‎

Lines changed: 17 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@
1515
#include <crypto/algapi.h>
1616
#include <crypto/internal/simd.h>
1717
#include <crypto/serpent.h>
18-
#include <asm/crypto/glue_helper.h>
1918
#include <asm/crypto/serpent-avx.h>
2019

20+
#include "ecb_cbc_helpers.h"
21+
2122
/* 8-way parallel cipher functions */
2223
asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst,
2324
const u8 *src);
@@ -37,63 +38,35 @@ static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
3738
return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
3839
}
3940

40-
static const struct common_glue_ctx serpent_enc = {
41-
.num_funcs = 2,
42-
.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
43-
44-
.funcs = { {
45-
.num_blocks = SERPENT_PARALLEL_BLOCKS,
46-
.fn_u = { .ecb = serpent_ecb_enc_8way_avx }
47-
}, {
48-
.num_blocks = 1,
49-
.fn_u = { .ecb = __serpent_encrypt }
50-
} }
51-
};
52-
53-
static const struct common_glue_ctx serpent_dec = {
54-
.num_funcs = 2,
55-
.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
56-
57-
.funcs = { {
58-
.num_blocks = SERPENT_PARALLEL_BLOCKS,
59-
.fn_u = { .ecb = serpent_ecb_dec_8way_avx }
60-
}, {
61-
.num_blocks = 1,
62-
.fn_u = { .ecb = __serpent_decrypt }
63-
} }
64-
};
65-
66-
static const struct common_glue_ctx serpent_dec_cbc = {
67-
.num_funcs = 2,
68-
.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
69-
70-
.funcs = { {
71-
.num_blocks = SERPENT_PARALLEL_BLOCKS,
72-
.fn_u = { .cbc = serpent_cbc_dec_8way_avx }
73-
}, {
74-
.num_blocks = 1,
75-
.fn_u = { .cbc = __serpent_decrypt }
76-
} }
77-
};
78-
7941
static int ecb_encrypt(struct skcipher_request *req)
8042
{
81-
return glue_ecb_req_128bit(&serpent_enc, req);
43+
ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
44+
ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx);
45+
ECB_BLOCK(1, __serpent_encrypt);
46+
ECB_WALK_END();
8247
}
8348

8449
static int ecb_decrypt(struct skcipher_request *req)
8550
{
86-
return glue_ecb_req_128bit(&serpent_dec, req);
51+
ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
52+
ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx);
53+
ECB_BLOCK(1, __serpent_decrypt);
54+
ECB_WALK_END();
8755
}
8856

8957
static int cbc_encrypt(struct skcipher_request *req)
9058
{
91-
return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req);
59+
CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1);
60+
CBC_ENC_BLOCK(__serpent_encrypt);
61+
CBC_WALK_END();
9262
}
9363

9464
static int cbc_decrypt(struct skcipher_request *req)
9565
{
96-
return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
66+
CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
67+
CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx);
68+
CBC_DEC_BLOCK(1, __serpent_decrypt);
69+
CBC_WALK_END();
9770
}
9871

9972
static struct skcipher_alg serpent_algs[] = {

‎arch/x86/crypto/serpent_sse2_glue.c‎

Lines changed: 24 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -21,88 +21,55 @@
2121
#include <crypto/internal/simd.h>
2222
#include <crypto/serpent.h>
2323
#include <asm/crypto/serpent-sse2.h>
24-
#include <asm/crypto/glue_helper.h>
24+
25+
#include "ecb_cbc_helpers.h"
2526

2627
static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
2728
const u8 *key, unsigned int keylen)
2829
{
2930
return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
3031
}
3132

32-
static void serpent_decrypt_cbc_xway(const void *ctx, u8 *d, const u8 *s)
33+
static void serpent_decrypt_cbc_xway(const void *ctx, u8 *dst, const u8 *src)
3334
{
34-
u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
35-
u128 *dst = (u128 *)d;
36-
const u128 *src = (const u128 *)s;
37-
unsigned int j;
38-
39-
for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
40-
ivs[j] = src[j];
35+
u8 buf[SERPENT_PARALLEL_BLOCKS - 1][SERPENT_BLOCK_SIZE];
36+
const u8 *s = src;
4137

42-
serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
43-
44-
for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
45-
u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
38+
if (dst == src)
39+
s = memcpy(buf, src, sizeof(buf));
40+
serpent_dec_blk_xway(ctx, dst, src);
41+
crypto_xor(dst + SERPENT_BLOCK_SIZE, s, sizeof(buf));
4642
}
4743

48-
static const struct common_glue_ctx serpent_enc = {
49-
.num_funcs = 2,
50-
.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
51-
52-
.funcs = { {
53-
.num_blocks = SERPENT_PARALLEL_BLOCKS,
54-
.fn_u = { .ecb = serpent_enc_blk_xway }
55-
}, {
56-
.num_blocks = 1,
57-
.fn_u = { .ecb = __serpent_encrypt }
58-
} }
59-
};
60-
61-
static const struct common_glue_ctx serpent_dec = {
62-
.num_funcs = 2,
63-
.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
64-
65-
.funcs = { {
66-
.num_blocks = SERPENT_PARALLEL_BLOCKS,
67-
.fn_u = { .ecb = serpent_dec_blk_xway }
68-
}, {
69-
.num_blocks = 1,
70-
.fn_u = { .ecb = __serpent_decrypt }
71-
} }
72-
};
73-
74-
static const struct common_glue_ctx serpent_dec_cbc = {
75-
.num_funcs = 2,
76-
.fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
77-
78-
.funcs = { {
79-
.num_blocks = SERPENT_PARALLEL_BLOCKS,
80-
.fn_u = { .cbc = serpent_decrypt_cbc_xway }
81-
}, {
82-
.num_blocks = 1,
83-
.fn_u = { .cbc = __serpent_decrypt }
84-
} }
85-
};
86-
8744
static int ecb_encrypt(struct skcipher_request *req)
8845
{
89-
return glue_ecb_req_128bit(&serpent_enc, req);
46+
ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
47+
ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_enc_blk_xway);
48+
ECB_BLOCK(1, __serpent_encrypt);
49+
ECB_WALK_END();
9050
}
9151

9252
static int ecb_decrypt(struct skcipher_request *req)
9353
{
94-
return glue_ecb_req_128bit(&serpent_dec, req);
54+
ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
55+
ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_dec_blk_xway);
56+
ECB_BLOCK(1, __serpent_decrypt);
57+
ECB_WALK_END();
9558
}
9659

9760
static int cbc_encrypt(struct skcipher_request *req)
9861
{
99-
return glue_cbc_encrypt_req_128bit(__serpent_encrypt,
100-
req);
62+
CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1);
63+
CBC_ENC_BLOCK(__serpent_encrypt);
64+
CBC_WALK_END();
10165
}
10266

10367
static int cbc_decrypt(struct skcipher_request *req)
10468
{
105-
return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
69+
CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS);
70+
CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_decrypt_cbc_xway);
71+
CBC_DEC_BLOCK(1, __serpent_decrypt);
72+
CBC_WALK_END();
10673
}
10774

10875
static struct skcipher_alg serpent_algs[] = {

‎crypto/Kconfig‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1538,7 +1538,6 @@ config CRYPTO_SERPENT_SSE2_X86_64
15381538
tristate "Serpent cipher algorithm (x86_64/SSE2)"
15391539
depends on X86 && 64BIT
15401540
select CRYPTO_SKCIPHER
1541-
select CRYPTO_GLUE_HELPER_X86
15421541
select CRYPTO_SERPENT
15431542
select CRYPTO_SIMD
15441543
imply CRYPTO_CTR
@@ -1558,7 +1557,6 @@ config CRYPTO_SERPENT_SSE2_586
15581557
tristate "Serpent cipher algorithm (i586/SSE2)"
15591558
depends on X86 && !64BIT
15601559
select CRYPTO_SKCIPHER
1561-
select CRYPTO_GLUE_HELPER_X86
15621560
select CRYPTO_SERPENT
15631561
select CRYPTO_SIMD
15641562
imply CRYPTO_CTR
@@ -1578,7 +1576,6 @@ config CRYPTO_SERPENT_AVX_X86_64
15781576
tristate "Serpent cipher algorithm (x86_64/AVX)"
15791577
depends on X86 && 64BIT
15801578
select CRYPTO_SKCIPHER
1581-
select CRYPTO_GLUE_HELPER_X86
15821579
select CRYPTO_SERPENT
15831580
select CRYPTO_SIMD
15841581
imply CRYPTO_XTS

0 commit comments

Comments
 (0)