Skip to content

Commit 010d124

Browse files
teknoman117behlendorf
teknoman117
authored and
behlendorf
committed
Add support for selecting encryption backend
- Add two new module parameters to icp (icp_aes_impl, icp_gcm_impl) that control the crypto implementation. At the moment there is a choice between generic and aesni (on platforms that support it). - This enables support for AES-NI and PCLMULQDQ-NI on AMD Family 15h (bulldozer) and newer CPUs (zen). - Modify aes_key_t to track what implementation it was generated with as key schedules generated with various implementations are not necessarily interchangable. Reviewed by: Gvozden Neskovic <neskovic@gmail.com> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Reviewed-by: Tom Caputi <tcaputi@datto.com> Reviewed-by: Richard Laager <rlaager@wiktel.com> Signed-off-by: Nathaniel R. Lewis <linux.robotdude@gmail.com> Closes #7102 Closes #7103
1 parent 3d503a7 commit 010d124

File tree

18 files changed

+2296
-1586
lines changed

18 files changed

+2296
-1586
lines changed

‎cmd/ztest/ztest.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@
129129
#include <zfs_fletcher.h>
130130
#include <libnvpair.h>
131131
#include <libzfs.h>
132+
#include <sys/crypto/icp.h>
132133
#ifdef __GLIBC__
133134
#include <execinfo.h> /* for backtrace() */
134135
#endif
@@ -3836,6 +3837,13 @@ ztest_dataset_create(char *dsname)
38363837
VERIFY0(dsl_crypto_params_create_nvlist(DCP_CMD_NONE, props,
38373838
crypto_args, &dcp));
38383839

3840+
/*
3841+
* Cycle through all available encryption implementations
3842+
* to verify interoperability.
3843+
*/
3844+
VERIFY0(gcm_impl_set("cycle"));
3845+
VERIFY0(aes_impl_set("cycle"));
3846+
38393847
fnvlist_free(crypto_args);
38403848
fnvlist_free(props);
38413849
}

‎config/toolchain-simd.m4‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
2121
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512PF
2222
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512ER
2323
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VL
24+
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES
25+
ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ
2426
;;
2527
esac
2628
])
@@ -359,3 +361,43 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AVX512VL], [
359361
AC_MSG_RESULT([no])
360362
])
361363
])
364+
365+
dnl #
366+
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES
367+
dnl #
368+
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES], [
369+
AC_MSG_CHECKING([whether host toolchain supports AES])
370+
371+
AC_LINK_IFELSE([AC_LANG_SOURCE([
372+
[
373+
void main()
374+
{
375+
__asm__ __volatile__("aesenc %xmm0, %xmm1");
376+
}
377+
]])], [
378+
AC_MSG_RESULT([yes])
379+
AC_DEFINE([HAVE_AES], 1, [Define if host toolchain supports AES])
380+
], [
381+
AC_MSG_RESULT([no])
382+
])
383+
])
384+
385+
dnl #
386+
dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ
387+
dnl #
388+
AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ], [
389+
AC_MSG_CHECKING([whether host toolchain supports PCLMULQDQ])
390+
391+
AC_LINK_IFELSE([AC_LANG_SOURCE([
392+
[
393+
void main()
394+
{
395+
__asm__ __volatile__("pclmulqdq %0, %%xmm0, %%xmm1" :: "i"(0));
396+
}
397+
]])], [
398+
AC_MSG_RESULT([yes])
399+
AC_DEFINE([HAVE_PCLMULQDQ], 1, [Define if host toolchain supports PCLMULQDQ])
400+
], [
401+
AC_MSG_RESULT([no])
402+
])
403+
])

‎include/linux/simd_x86.h‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,9 @@ typedef enum cpuid_inst_sets {
148148
AVX512VBMI,
149149
AVX512PF,
150150
AVX512ER,
151-
AVX512VL
151+
AVX512VL,
152+
AES,
153+
PCLMULQDQ
152154
} cpuid_inst_sets_t;
153155

154156
/*
@@ -170,6 +172,8 @@ typedef struct cpuid_feature_desc {
170172
#define _AVX512PF_BIT (_AVX512F_BIT | (1U << 26))
171173
#define _AVX512ER_BIT (_AVX512F_BIT | (1U << 27))
172174
#define _AVX512VL_BIT (1U << 31) /* if used also check other levels */
175+
#define _AES_BIT (1U << 25)
176+
#define _PCLMULQDQ_BIT (1U << 1)
173177

174178
/*
175179
* Descriptions of supported instruction sets
@@ -194,7 +198,9 @@ static const cpuid_feature_desc_t cpuid_features[] = {
194198
[AVX512VBMI] = {7U, 0U, _AVX512VBMI_BIT, ECX },
195199
[AVX512PF] = {7U, 0U, _AVX512PF_BIT, EBX },
196200
[AVX512ER] = {7U, 0U, _AVX512ER_BIT, EBX },
197-
[AVX512VL] = {7U, 0U, _AVX512ER_BIT, EBX }
201+
[AVX512VL] = {7U, 0U, _AVX512ER_BIT, EBX },
202+
[AES] = {1U, 0U, _AES_BIT, ECX },
203+
[PCLMULQDQ] = {1U, 0U, _PCLMULQDQ_BIT, ECX },
198204
};
199205

200206
/*
@@ -265,6 +271,8 @@ CPUID_FEATURE_CHECK(avx512vbmi, AVX512VBMI);
265271
CPUID_FEATURE_CHECK(avx512pf, AVX512PF);
266272
CPUID_FEATURE_CHECK(avx512er, AVX512ER);
267273
CPUID_FEATURE_CHECK(avx512vl, AVX512VL);
274+
CPUID_FEATURE_CHECK(aes, AES);
275+
CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);
268276

269277
#endif /* !defined(_KERNEL) */
270278

@@ -442,6 +450,35 @@ zfs_bmi2_available(void)
442450
#endif
443451
}
444452

453+
/*
454+
* Check if AES instruction set is available
455+
*/
456+
static inline boolean_t
457+
zfs_aes_available(void)
458+
{
459+
#if defined(_KERNEL) && defined(X86_FEATURE_AES)
460+
return (!!boot_cpu_has(X86_FEATURE_AES));
461+
#elif defined(_KERNEL) && !defined(X86_FEATURE_AES)
462+
return (B_FALSE);
463+
#else
464+
return (__cpuid_has_aes());
465+
#endif
466+
}
467+
468+
/*
469+
* Check if PCLMULQDQ instruction set is available
470+
*/
471+
static inline boolean_t
472+
zfs_pclmulqdq_available(void)
473+
{
474+
#if defined(_KERNEL) && defined(X86_FEATURE_PCLMULQDQ)
475+
return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ));
476+
#elif defined(_KERNEL) && !defined(X86_FEATURE_PCLMULQDQ)
477+
return (B_FALSE);
478+
#else
479+
return (__cpuid_has_pclmulqdq());
480+
#endif
481+
}
445482

446483
/*
447484
* AVX-512 family of instruction sets:

‎include/sys/crypto/icp.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,7 @@ int skein_mod_fini(void);
4444
int icp_init(void);
4545
void icp_fini(void);
4646

47+
int aes_impl_set(const char *);
48+
int gcm_impl_set(const char *);
49+
4750
#endif /* _SYS_CRYPTO_ALGS_H */

‎lib/libicp/Makefile.am‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ if TARGET_ASM_X86_64
1818
ASM_SOURCES_C = asm-x86_64/aes/aeskey.c
1919
ASM_SOURCES_AS = \
2020
asm-x86_64/aes/aes_amd64.S \
21-
asm-x86_64/aes/aes_intel.S \
22-
asm-x86_64/modes/gcm_intel.S \
21+
asm-x86_64/aes/aes_aesni.S \
22+
asm-x86_64/modes/gcm_pclmulqdq.S \
2323
asm-x86_64/sha1/sha1-x86_64.S \
2424
asm-x86_64/sha2/sha256_impl.S \
2525
asm-x86_64/sha2/sha512_impl.S
@@ -46,11 +46,16 @@ KERNEL_C = \
4646
api/kcf_cipher.c \
4747
api/kcf_miscapi.c \
4848
api/kcf_mac.c \
49+
algs/aes/aes_impl_aesni.c \
50+
algs/aes/aes_impl_generic.c \
51+
algs/aes/aes_impl_x86-64.c \
4952
algs/aes/aes_impl.c \
5053
algs/aes/aes_modes.c \
5154
algs/edonr/edonr.c \
5255
algs/modes/modes.c \
5356
algs/modes/cbc.c \
57+
algs/modes/gcm_generic.c \
58+
algs/modes/gcm_pclmulqdq.c \
5459
algs/modes/gcm.c \
5560
algs/modes/ctr.c \
5661
algs/modes/ccm.c \

‎module/icp/Makefile.in‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ TARGET_ASM_DIR = @TARGET_ASM_DIR@
88
ifeq ($(TARGET_ASM_DIR), asm-x86_64)
99
ASM_SOURCES := asm-x86_64/aes/aeskey.o
1010
ASM_SOURCES += asm-x86_64/aes/aes_amd64.o
11-
ASM_SOURCES += asm-x86_64/aes/aes_intel.o
12-
ASM_SOURCES += asm-x86_64/modes/gcm_intel.o
11+
ASM_SOURCES += asm-x86_64/aes/aes_aesni.o
12+
ASM_SOURCES += asm-x86_64/modes/gcm_pclmulqdq.o
1313
ASM_SOURCES += asm-x86_64/sha1/sha1-x86_64.o
1414
ASM_SOURCES += asm-x86_64/sha2/sha256_impl.o
1515
ASM_SOURCES += asm-x86_64/sha2/sha512_impl.o
@@ -53,8 +53,10 @@ $(MODULE)-objs += algs/modes/cbc.o
5353
$(MODULE)-objs += algs/modes/ccm.o
5454
$(MODULE)-objs += algs/modes/ctr.o
5555
$(MODULE)-objs += algs/modes/ecb.o
56+
$(MODULE)-objs += algs/modes/gcm_generic.o
5657
$(MODULE)-objs += algs/modes/gcm.o
5758
$(MODULE)-objs += algs/modes/modes.o
59+
$(MODULE)-objs += algs/aes/aes_impl_generic.o
5860
$(MODULE)-objs += algs/aes/aes_impl.o
5961
$(MODULE)-objs += algs/aes/aes_modes.o
6062
$(MODULE)-objs += algs/edonr/edonr.o
@@ -66,6 +68,10 @@ $(MODULE)-objs += algs/skein/skein_block.o
6668
$(MODULE)-objs += algs/skein/skein_iv.o
6769
$(MODULE)-objs += $(ASM_SOURCES)
6870

71+
$(MODULE)-$(CONFIG_X86) += algs/modes/gcm_pclmulqdq.o
72+
$(MODULE)-$(CONFIG_X86) += algs/aes/aes_impl_aesni.o
73+
$(MODULE)-$(CONFIG_X86) += algs/aes/aes_impl_x86-64.o
74+
6975
ICP_DIRS = \
7076
api \
7177
core \

0 commit comments

Comments
 (0)