forked from linux-riscv/linux
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge patch series "RISC-V crypto with reworked asm files"
Eric Biggers <ebiggers@kernel.org> says: This patchset, which applies to v6.8-rc1, adds cryptographic algorithm implementations accelerated using the RISC-V vector crypto extensions (https://github.com/riscv/riscv-crypto/releases/download/v1.0.0/riscv-crypto-spec-vector.pdf) and RISC-V vector extension (https://github.com/riscv/riscv-v-spec/releases/download/v1.0/riscv-v-spec-1.0.pdf). The following algorithms are included: AES in ECB, CBC, CTR, and XTS modes; ChaCha20; GHASH; SHA-2; SM3; and SM4. In general, the assembly code requires a 64-bit RISC-V CPU with VLEN >= 128, little endian byte order, and vector unaligned access support. The ECB, CTR, XTS, and ChaCha20 code is designed to naturally scale up to larger VLEN values. Building the assembly code requires tip-of-tree binutils (future 2.42) or tip-of-tree clang (future 18.x). All algorithms pass testing in QEMU, using CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y. Much of the assembly code is derived from OpenSSL code that was added by openssl/openssl#21923. It's been cleaned up for integration with the kernel, e.g. reducing code duplication, eliminating use of .inst and perlasm, and fixing a few bugs. This patchset incorporates the work of multiple people, including Jerry Shih, Heiko Stuebner, Christoph Müllner, Phoebe Chen, Charalampos Mitrodimas, and myself. This patchset went through several versions from Heiko (last version https://lore.kernel.org/linux-crypto/20230711153743.1970625-1-heiko@sntech.de), then several versions from Jerry (last version: https://lore.kernel.org/linux-crypto/20231231152743.6304-1-jerry.shih@sifive.com), then finally several versions from me. Thanks to everyone who has contributed to this patchset or its prerequisites. * b4-shazam-merge: crypto: riscv - add vector crypto accelerated SM4 crypto: riscv - add vector crypto accelerated SM3 crypto: riscv - add vector crypto accelerated SHA-{512,384} crypto: riscv - add vector crypto accelerated SHA-{256,224} crypto: riscv - add vector crypto accelerated GHASH crypto: riscv - add vector crypto accelerated ChaCha20 crypto: riscv - add vector crypto accelerated AES-{ECB,CBC,CTR,XTS} RISC-V: hook new crypto subdir into build-system RISC-V: add TOOLCHAIN_HAS_VECTOR_CRYPTO RISC-V: add helper function to read the vector VLEN Link: https://lore.kernel.org/r/20240122002024.27477-1-ebiggers@kernel.org Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
- Loading branch information
Showing
23 changed files
with
3,274 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
# SPDX-License-Identifier: GPL-2.0 | ||
|
||
menu "Accelerated Cryptographic Algorithms for CPU (riscv)" | ||
|
||
config CRYPTO_AES_RISCV64 | ||
tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS" | ||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO | ||
select CRYPTO_ALGAPI | ||
select CRYPTO_LIB_AES | ||
select CRYPTO_SKCIPHER | ||
help | ||
Block cipher: AES cipher algorithms | ||
Length-preserving ciphers: AES with ECB, CBC, CTR, XTS | ||
|
||
Architecture: riscv64 using: | ||
- Zvkned vector crypto extension | ||
- Zvbb vector extension (XTS) | ||
- Zvkb vector crypto extension (CTR) | ||
- Zvkg vector crypto extension (XTS) | ||
|
||
config CRYPTO_CHACHA_RISCV64 | ||
tristate "Ciphers: ChaCha" | ||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO | ||
select CRYPTO_SKCIPHER | ||
select CRYPTO_LIB_CHACHA_GENERIC | ||
help | ||
Length-preserving ciphers: ChaCha20 stream cipher algorithm | ||
|
||
Architecture: riscv64 using: | ||
- Zvkb vector crypto extension | ||
|
||
config CRYPTO_GHASH_RISCV64 | ||
tristate "Hash functions: GHASH" | ||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO | ||
select CRYPTO_GCM | ||
help | ||
GCM GHASH function (NIST SP 800-38D) | ||
|
||
Architecture: riscv64 using: | ||
- Zvkg vector crypto extension | ||
|
||
config CRYPTO_SHA256_RISCV64 | ||
tristate "Hash functions: SHA-224 and SHA-256" | ||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO | ||
select CRYPTO_SHA256 | ||
help | ||
SHA-224 and SHA-256 secure hash algorithm (FIPS 180) | ||
|
||
Architecture: riscv64 using: | ||
- Zvknha or Zvknhb vector crypto extensions | ||
- Zvkb vector crypto extension | ||
|
||
config CRYPTO_SHA512_RISCV64 | ||
tristate "Hash functions: SHA-384 and SHA-512" | ||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO | ||
select CRYPTO_SHA512 | ||
help | ||
SHA-384 and SHA-512 secure hash algorithm (FIPS 180) | ||
|
||
Architecture: riscv64 using: | ||
- Zvknhb vector crypto extension | ||
- Zvkb vector crypto extension | ||
|
||
config CRYPTO_SM3_RISCV64 | ||
tristate "Hash functions: SM3 (ShangMi 3)" | ||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO | ||
select CRYPTO_HASH | ||
select CRYPTO_SM3 | ||
help | ||
SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) | ||
|
||
Architecture: riscv64 using: | ||
- Zvksh vector crypto extension | ||
- Zvkb vector crypto extension | ||
|
||
config CRYPTO_SM4_RISCV64 | ||
tristate "Ciphers: SM4 (ShangMi 4)" | ||
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO | ||
select CRYPTO_ALGAPI | ||
select CRYPTO_SM4 | ||
help | ||
SM4 block cipher algorithm (OSCCA GB/T 32907-2016, | ||
ISO/IEC 18033-3:2010/Amd 1:2021) | ||
|
||
SM4 (GBT.32907-2016) is a cryptographic standard issued by the | ||
Organization of State Commercial Administration of China (OSCCA) | ||
as an authorized cryptographic algorithm for use within China. | ||
|
||
Architecture: riscv64 using: | ||
- Zvksed vector crypto extension | ||
- Zvkb vector crypto extension | ||
|
||
endmenu |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
# SPDX-License-Identifier: GPL-2.0-only | ||
|
||
obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o | ||
aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ | ||
aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o | ||
|
||
obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o | ||
chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o | ||
|
||
obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o | ||
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o | ||
|
||
obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o | ||
sha256-riscv64-y := sha256-riscv64-glue.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o | ||
|
||
obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o | ||
sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o | ||
|
||
obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o | ||
sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o | ||
|
||
obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o | ||
sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,156 @@ | ||
/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ | ||
// | ||
// This file is dual-licensed, meaning that you can use it under your | ||
// choice of either of the following two licenses: | ||
// | ||
// Copyright 2023 The OpenSSL Project Authors. All Rights Reserved. | ||
// | ||
// Licensed under the Apache License 2.0 (the "License"). You can obtain | ||
// a copy in the file LICENSE in the source distribution or at | ||
// https://www.openssl.org/source/license.html | ||
// | ||
// or | ||
// | ||
// Copyright (c) 2023, Christoph Müllner <christoph.muellner@vrull.eu> | ||
// Copyright (c) 2023, Phoebe Chen <phoebe.chen@sifive.com> | ||
// Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com> | ||
// Copyright 2024 Google LLC | ||
// All rights reserved. | ||
// | ||
// Redistribution and use in source and binary forms, with or without | ||
// modification, are permitted provided that the following conditions | ||
// are met: | ||
// 1. Redistributions of source code must retain the above copyright | ||
// notice, this list of conditions and the following disclaimer. | ||
// 2. Redistributions in binary form must reproduce the above copyright | ||
// notice, this list of conditions and the following disclaimer in the | ||
// documentation and/or other materials provided with the distribution. | ||
// | ||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | ||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | ||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | ||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | ||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | ||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | ||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | ||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | ||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | ||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
|
||
// This file contains macros that are shared by the other aes-*.S files. The | ||
// generated code of these macros depends on the following RISC-V extensions: | ||
// - RV64I | ||
// - RISC-V Vector ('V') with VLEN >= 128 | ||
// - RISC-V Vector AES block cipher extension ('Zvkned') | ||
|
||
// Loads the AES round keys from \keyp into vector registers and jumps to code | ||
// specific to the length of the key. Specifically: | ||
// - If AES-128, loads round keys into v1-v11 and jumps to \label128. | ||
// - If AES-192, loads round keys into v1-v13 and jumps to \label192. | ||
// - If AES-256, loads round keys into v1-v15 and continues onwards. | ||
// | ||
// Also sets vl=4 and vtype=e32,m1,ta,ma. Clobbers t0 and t1. | ||
.macro aes_begin keyp, label128, label192 | ||
lwu t0, 480(\keyp) // t0 = key length in bytes | ||
li t1, 24 // t1 = key length for AES-192 | ||
vsetivli zero, 4, e32, m1, ta, ma | ||
vle32.v v1, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v2, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v3, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v4, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v5, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v6, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v7, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v8, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v9, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v10, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v11, (\keyp) | ||
blt t0, t1, \label128 // If AES-128, goto label128. | ||
addi \keyp, \keyp, 16 | ||
vle32.v v12, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v13, (\keyp) | ||
beq t0, t1, \label192 // If AES-192, goto label192. | ||
// Else, it's AES-256. | ||
addi \keyp, \keyp, 16 | ||
vle32.v v14, (\keyp) | ||
addi \keyp, \keyp, 16 | ||
vle32.v v15, (\keyp) | ||
.endm | ||
|
||
// Encrypts \data using zvkned instructions, using the round keys loaded into | ||
// v1-v11 (for AES-128), v1-v13 (for AES-192), or v1-v15 (for AES-256). \keylen | ||
// is the AES key length in bits. vl and vtype must already be set | ||
// appropriately. Note that if vl > 4, multiple blocks are encrypted. | ||
.macro aes_encrypt data, keylen | ||
vaesz.vs \data, v1 | ||
vaesem.vs \data, v2 | ||
vaesem.vs \data, v3 | ||
vaesem.vs \data, v4 | ||
vaesem.vs \data, v5 | ||
vaesem.vs \data, v6 | ||
vaesem.vs \data, v7 | ||
vaesem.vs \data, v8 | ||
vaesem.vs \data, v9 | ||
vaesem.vs \data, v10 | ||
.if \keylen == 128 | ||
vaesef.vs \data, v11 | ||
.elseif \keylen == 192 | ||
vaesem.vs \data, v11 | ||
vaesem.vs \data, v12 | ||
vaesef.vs \data, v13 | ||
.else | ||
vaesem.vs \data, v11 | ||
vaesem.vs \data, v12 | ||
vaesem.vs \data, v13 | ||
vaesem.vs \data, v14 | ||
vaesef.vs \data, v15 | ||
.endif | ||
.endm | ||
|
||
// Same as aes_encrypt, but decrypts instead of encrypts. | ||
.macro aes_decrypt data, keylen | ||
.if \keylen == 128 | ||
vaesz.vs \data, v11 | ||
.elseif \keylen == 192 | ||
vaesz.vs \data, v13 | ||
vaesdm.vs \data, v12 | ||
vaesdm.vs \data, v11 | ||
.else | ||
vaesz.vs \data, v15 | ||
vaesdm.vs \data, v14 | ||
vaesdm.vs \data, v13 | ||
vaesdm.vs \data, v12 | ||
vaesdm.vs \data, v11 | ||
.endif | ||
vaesdm.vs \data, v10 | ||
vaesdm.vs \data, v9 | ||
vaesdm.vs \data, v8 | ||
vaesdm.vs \data, v7 | ||
vaesdm.vs \data, v6 | ||
vaesdm.vs \data, v5 | ||
vaesdm.vs \data, v4 | ||
vaesdm.vs \data, v3 | ||
vaesdm.vs \data, v2 | ||
vaesdf.vs \data, v1 | ||
.endm | ||
|
||
// Expands to aes_encrypt or aes_decrypt according to \enc, which is 1 or 0. | ||
.macro aes_crypt data, enc, keylen | ||
.if \enc | ||
aes_encrypt \data, \keylen | ||
.else | ||
aes_decrypt \data, \keylen | ||
.endif | ||
.endm |
Oops, something went wrong.