diff --git a/.github/workflows/fips-check-pr.yml b/.github/workflows/fips-check-pr.yml new file mode 100644 index 0000000000000..87e38a6575337 --- /dev/null +++ b/.github/workflows/fips-check-pr.yml @@ -0,0 +1,158 @@ +# FIPS Protected Directory Check +# +# This workflow runs on PRs targeting ciq-*-next branches and checks +# whether the new upstream commits (the stable release delta) touch +# any FIPS protected directories. If so, it posts a comment on the +# PR alerting reviewers to involve the FIPS / Security team. +# +# How it works: +# PR base branch: ciq-X.Y.y-next (created from stable_X.Y.y) +# Old branch: ciq-X.Y.y (previous CIQ branch, derived by stripping "-next") +# merge-base(old, next) = last common upstream commit +# merge-base..ciq-X.Y.y-next = new upstream commits to check +# +# TODO: remove ref: clk-fips-check once kernel-src-tree-tools merges that branch + +name: FIPS Protected Directory Check + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - 'ciq-*-next' + +permissions: + contents: read + pull-requests: write + +jobs: + fips-check: + name: FIPS Directory Check + runs-on: ubuntu-latest + + steps: + - name: Generate GitHub App token + id: generate_token + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + repositories: | + kernel-src-tree + kernel-src-tree-tools + + - name: Checkout kernel source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + token: ${{ steps.generate_token.outputs.token }} + + - name: Checkout kernel-src-tree-tools + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ctrliq/kernel-src-tree-tools + ref: clk-fips-check + path: kernel-src-tree-tools + token: ${{ steps.generate_token.outputs.token }} + + - name: Install Python dependencies + run: pip install gitpython + + - name: Fetch branches and run FIPS check + id: fips_check + env: + PYTHONPATH: ${{ github.workspace }}/kernel-src-tree-tools + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + OLD_BRANCH="${BASE_REF%-next}" + echo "Derived old branch: $OLD_BRANCH from PR base: $BASE_REF" + + # Fetch both branches (full history needed for merge-base) + git fetch origin "$BASE_REF:refs/remotes/origin/$BASE_REF" + git fetch origin "$OLD_BRANCH:refs/remotes/origin/$OLD_BRANCH" 2>/dev/null || { + echo "::warning::Could not fetch branch $OLD_BRANCH — skipping FIPS check" + echo "result=skip" >> "$GITHUB_OUTPUT" + exit 0 + } + + MERGE_BASE=$(git merge-base "origin/$OLD_BRANCH" "origin/$BASE_REF" 2>/dev/null) || { + echo "::warning::Could not compute merge-base between $OLD_BRANCH and $BASE_REF" + echo "result=skip" >> "$GITHUB_OUTPUT" + exit 0 + } + + echo "Merge base: $MERGE_BASE" + echo "Checking upstream commits: $MERGE_BASE..origin/$BASE_REF" + + python3 kernel-src-tree-tools/check_fips_changes.py \ + --repo . \ + --base-ref "$MERGE_BASE" \ + --target-ref "origin/$BASE_REF" \ + --fips-override 2>&1 | tee fips_report.txt + + if grep -q "FIPS protected changes detected" fips_report.txt; then + echo "result=found" >> "$GITHUB_OUTPUT" + else + echo "result=clean" >> "$GITHUB_OUTPUT" + fi + + - name: Post or update PR comment + if: always() + env: + GH_TOKEN: ${{ steps.generate_token.outputs.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RESULT: ${{ steps.fips_check.outputs.result }} + run: | + MARKER="" + REPO="${{ github.repository }}" + + # Find any existing FIPS check comment + EXISTING_ID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" \ + --jq ".[] | select(.body | contains(\"$MARKER\")) | .id" | head -1) + + if [ "$RESULT" = "found" ]; then + REPORT=$(cat fips_report.txt) + BODY=$(cat < + Details + + \`\`\` + $REPORT + \`\`\` + + + EOF + ) + elif [ "$RESULT" = "clean" ]; then + BODY=$(cat <Vbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL); if (!drbg->Vbuf) { ret = -ENOMEM; @@ -1495,13 +1501,14 @@ static int drbg_generate(struct drbg_state *drbg, * Wrapper around drbg_generate which can pull arbitrary long strings * from the DRBG without hitting the maximum request limitation. * - * Parameters: see drbg_generate + * Parameters: see drbg_generate, except @reseed, which triggers reseeding * Return codes: see drbg_generate -- if one drbg_generate request fails, * the entire drbg_generate_long request fails */ static int drbg_generate_long(struct drbg_state *drbg, unsigned char *buf, unsigned int buflen, - struct drbg_string *addtl) + struct drbg_string *addtl, + bool reseed) { unsigned int len = 0; unsigned int slice = 0; @@ -1511,6 +1518,8 @@ static int drbg_generate_long(struct drbg_state *drbg, slice = ((buflen - len) / drbg_max_request_bytes(drbg)); chunk = slice ? drbg_max_request_bytes(drbg) : (buflen - len); mutex_lock(&drbg->drbg_mutex); + if (reseed) + drbg->seeded = DRBG_SEED_STATE_UNSEEDED; err = drbg_generate(drbg, buf + len, chunk, addtl); mutex_unlock(&drbg->drbg_mutex); if (0 > err) @@ -1937,6 +1946,7 @@ static int drbg_kcapi_random(struct crypto_rng *tfm, struct drbg_state *drbg = crypto_rng_ctx(tfm); struct drbg_string *addtl = NULL; struct drbg_string string; + int err; if (slen) { /* linked list variable is now local to allow modification */ @@ -1944,7 +1954,15 @@ static int drbg_kcapi_random(struct crypto_rng *tfm, addtl = &string; } - return drbg_generate_long(drbg, dst, dlen, addtl); + err = drbg_generate_long(drbg, dst, dlen, addtl, + (crypto_tfm_get_flags(crypto_rng_tfm(tfm)) & + CRYPTO_TFM_REQ_NEED_RESEED) == + CRYPTO_TFM_REQ_NEED_RESEED); + + crypto_tfm_clear_flags(crypto_rng_tfm(tfm), + CRYPTO_TFM_REQ_NEED_RESEED); + + return err; } /* diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 9f0b93b3166d8..07729930229a2 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -11,6 +11,7 @@ #include #include #include +#include struct ecdh_ctx { unsigned int curve_id; @@ -99,6 +100,36 @@ static int ecdh_compute_value(struct kpp_request *req) ctx->private_key, public_key); buf = public_key; nbytes = public_key_sz; + + /* + * SP800-56Arev3, 5.6.2.1.4: ("Owner Assurance of + * Pair-wise Consistency"): recompute the public key + * and check if the results match. + */ + if (fips_enabled) { + u64 *public_key_pct; + + if (ret < 0) + goto free_all; + + public_key_pct = kmalloc(public_key_sz, GFP_KERNEL); + if (!public_key_pct) { + ret = -ENOMEM; + goto free_all; + } + + ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits, + ctx->private_key, + public_key_pct); + if (ret < 0) { + kfree(public_key_pct); + goto free_all; + } + + if (memcmp(public_key, public_key_pct, public_key_sz)) + panic("ECDH PCT failed in FIPS mode"); + kfree(public_key_pct); + } } if (ret < 0) diff --git a/crypto/essiv.c b/crypto/essiv.c index a47a3eab69351..6627de4036196 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -114,13 +114,16 @@ static int essiv_aead_setkey(struct crypto_aead *tfm, const u8 *key, crypto_shash_update(desc, keys.enckey, keys.enckeylen) ?: crypto_shash_finup(desc, keys.authkey, keys.authkeylen, salt); if (err) - return err; + goto out; crypto_cipher_clear_flags(tctx->essiv_cipher, CRYPTO_TFM_REQ_MASK); crypto_cipher_set_flags(tctx->essiv_cipher, crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); - return crypto_cipher_setkey(tctx->essiv_cipher, salt, + err = crypto_cipher_setkey(tctx->essiv_cipher, salt, crypto_shash_digestsize(tctx->hash)); +out: + memzero_explicit(&keys, sizeof(keys)); + return err; } static int essiv_aead_setauthsize(struct crypto_aead *tfm, diff --git a/crypto/kdf_sp800108.c b/crypto/kdf_sp800108.c index b7a6bf9da773a..251badf7bf3f9 100644 --- a/crypto/kdf_sp800108.c +++ b/crypto/kdf_sp800108.c @@ -25,6 +25,10 @@ int crypto_kdf108_ctr_generate(struct crypto_shash *kmd, int err = 0; u8 *dst_orig = dst; + /* SP800-131Arev2: minimum 112-bit output length */ + if (fips_enabled && dlen < 112 / 8) + return -EINVAL; + desc->tfm = kmd; while (dlen) { diff --git a/crypto/rng.c b/crypto/rng.c index ee1768c5a4005..2836eac9738ac 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -6,16 +6,22 @@ * * Copyright (c) 2008 Neil Horman * Copyright (c) 2015 Herbert Xu + * + * Copyright (C) 2025 Ctrl IQ, Inc. + * Author: Sultan Alsawaf */ #include #include #include #include +#include #include #include -#include +#include #include +#include +#include #include #include #include @@ -23,10 +29,38 @@ #include "internal.h" -static DEFINE_MUTEX(crypto_default_rng_lock); +static ____cacheline_aligned_in_smp DEFINE_RT_MUTEX(crypto_default_rng_lock); struct crypto_rng *crypto_default_rng; EXPORT_SYMBOL_GPL(crypto_default_rng); -static int crypto_default_rng_refcnt; +static unsigned int crypto_default_rng_refcnt; + +/* + * Per-CPU RNG instances are only used by crypto_devrandom_rng. The global RNG, + * crypto_default_rng, is only used directly by other drivers. + * + * Per-CPU instances of the DRBG are efficient because the DRBG itself supports + * an arbitrary number of instances and can be seeded on a per-CPU basis. + * + * Specifically, the DRBG is seeded by the CRNG and the Jitter RNG. The CRNG is + * globally accessible and is already per-CPU. And while the Jitter RNG _isn't_ + * per-CPU, creating a DRBG instance also creates a Jitter RNG instance; + * therefore, per-CPU DRBG instances implies per-CPU Jitter RNG instances. + */ +struct cpu_rng_inst { + local_lock_t lock; + struct rt_mutex mlock; + struct crypto_rng *rng; + void *page; +}; + +static DEFINE_PER_CPU_ALIGNED(struct cpu_rng_inst, pcpu_default_rng) = { + .lock = INIT_LOCAL_LOCK(pcpu_default_rng.lock), + .mlock = __RT_MUTEX_INITIALIZER(pcpu_default_rng.mlock) +}; +static DEFINE_PER_CPU_ALIGNED(struct cpu_rng_inst, pcpu_reseed_rng) = { + /* The reseed instances don't use the local lock */ + .mlock = __RT_MUTEX_INITIALIZER(pcpu_reseed_rng.mlock) +}; int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen) { @@ -107,32 +141,38 @@ struct crypto_rng *crypto_alloc_rng(const char *alg_name, u32 type, u32 mask) } EXPORT_SYMBOL_GPL(crypto_alloc_rng); -int crypto_get_default_rng(void) +static int crypto_get_rng(struct crypto_rng **rngp) { struct crypto_rng *rng; int err; - mutex_lock(&crypto_default_rng_lock); - if (!crypto_default_rng) { + if (!*rngp) { rng = crypto_alloc_rng("stdrng", 0, 0); err = PTR_ERR(rng); if (IS_ERR(rng)) - goto unlock; + return err; err = crypto_rng_reset(rng, NULL, crypto_rng_seedsize(rng)); if (err) { crypto_free_rng(rng); - goto unlock; + return err; } - crypto_default_rng = rng; + *rngp = rng; } - crypto_default_rng_refcnt++; - err = 0; + return 0; +} -unlock: - mutex_unlock(&crypto_default_rng_lock); +int crypto_get_default_rng(void) +{ + int err; + + rt_mutex_lock(&crypto_default_rng_lock); + err = crypto_get_rng(&crypto_default_rng); + if (!err) + crypto_default_rng_refcnt++; + rt_mutex_unlock(&crypto_default_rng_lock); return err; } @@ -140,32 +180,61 @@ EXPORT_SYMBOL_GPL(crypto_get_default_rng); void crypto_put_default_rng(void) { - mutex_lock(&crypto_default_rng_lock); + rt_mutex_lock(&crypto_default_rng_lock); crypto_default_rng_refcnt--; - mutex_unlock(&crypto_default_rng_lock); + rt_mutex_unlock(&crypto_default_rng_lock); } EXPORT_SYMBOL_GPL(crypto_put_default_rng); #if defined(CONFIG_CRYPTO_RNG) || defined(CONFIG_CRYPTO_RNG_MODULE) -int crypto_del_default_rng(void) +#define down_read_del_pcpu_rwsem() down_read(&del_pcpu_rwsem) +#define up_read_del_pcpu_rwsem() up_read(&del_pcpu_rwsem) +static DECLARE_RWSEM(del_pcpu_rwsem); + +static void crypto_del_pcpu_rng(struct cpu_rng_inst __percpu *pcri) { - int err = -EBUSY; + int cpu; - mutex_lock(&crypto_default_rng_lock); - if (crypto_default_rng_refcnt) - goto out; + for_each_possible_cpu(cpu) { + struct cpu_rng_inst *cri = per_cpu_ptr(pcri, cpu); - crypto_free_rng(crypto_default_rng); - crypto_default_rng = NULL; + if (cri->rng) { + crypto_free_rng(cri->rng); + cri->rng = NULL; + } + } +} - err = 0; +int crypto_del_default_rng(void) +{ + bool busy; -out: - mutex_unlock(&crypto_default_rng_lock); + rt_mutex_lock(&crypto_default_rng_lock); + if (!(busy = crypto_default_rng_refcnt)) { + crypto_free_rng(crypto_default_rng); + crypto_default_rng = NULL; + } + rt_mutex_unlock(&crypto_default_rng_lock); + if (busy) + return -EBUSY; - return err; + if (!down_write_trylock(&del_pcpu_rwsem)) + return -EBUSY; + + crypto_del_pcpu_rng(&pcpu_default_rng); + crypto_del_pcpu_rng(&pcpu_reseed_rng); + up_write(&del_pcpu_rwsem); + + return 0; } EXPORT_SYMBOL_GPL(crypto_del_default_rng); +#else +static inline void down_read_del_pcpu_rwsem(void) +{ +} +static inline void up_read_del_pcpu_rwsem(void) +{ +} #endif static void rng_default_set_ent(struct crypto_rng *tfm, const u8 *data, @@ -226,5 +295,420 @@ void crypto_unregister_rngs(struct rng_alg *algs, int count) } EXPORT_SYMBOL_GPL(crypto_unregister_rngs); +/* + * On non-PREEMPT_RT kernels, local locks disable preemption. When there's no + * rng allocated, one must be allocated by calling crypto_get_rng(), which can + * sleep. Therefore, crypto_get_rng() cannot be called under local_lock(), so if + * our CPU's RNG instance doesn't have an rng allocated, we drop the local lock + * and take a mutex lock instead. After the local lock is dropped, the current + * task can be freely migrated to another CPU, which means that calling + * local_lock() again might not result in the same instance getting locked as + * before. That's why this function exists: to loop on calling local_lock() and + * allocating an rng as needed with crypto_get_rng() until the current CPU's + * instance is found to have an rng allocated. If crypto_get_rng() ever fails, + * this function returns an error even if there are instances for other CPUs + * which _do_ have an rng allocated. + */ +static __always_inline struct cpu_rng_inst * +lock_default_rng(struct crypto_rng **rng) __acquires(&cri->lock) +{ + struct cpu_rng_inst __percpu *pcri = &pcpu_default_rng; + struct cpu_rng_inst *cri; + int ret; + + while (1) { + local_lock(&pcri->lock); + cri = this_cpu_ptr(pcri); + /* + * cri->rng may have transitioned from non-NULL to NULL, but + * underneath down_read_del_pcpu_rwsem() it can only transition + * from NULL to non-NULL. This may occur on a different CPU, + * thus cri->rng must be read atomically to prevent data races; + * this elides mlock by pairing with the WRITE_ONCE() in the + * slow path below. + * + * + * And if cri->rng is non-NULL, then it is good to go. To avoid + * data races due to load speculation on torn cri->rng loads + * _after_ the NULL check, one of the following is required: + * 1. smp_acquire__after_ctrl_dep() in the if-statement + * 2. All cri->rng reads are performed with READ_ONCE() + * 3. cri->rng is never read again outside this function + * + * Option #3 yields the best performance, so this function + * provides the rng pointer as an output for the caller to use. + */ + *rng = READ_ONCE(cri->rng); + if (likely(*rng)) + return cri; + + /* + * Slow path: there's no rng currently allocated to this instance. + * Release the local lock and acquire this instance's mlock to + * perform the allocation. + * + * Note that this task may be migrated to a different CPU now! + */ + local_unlock(&cri->lock); + rt_mutex_lock(&cri->mlock); + if (!cri->rng) { + struct crypto_rng *new_rng = NULL; + + ret = crypto_get_rng(&new_rng); + if (ret) { + rt_mutex_unlock(&cri->mlock); + break; + } + + /* + * Pairs with READ_ONCE() above, because we might not be + * on the same CPU anymore as when we first got `cri`. + */ + WRITE_ONCE(cri->rng, new_rng); + } + rt_mutex_unlock(&cri->mlock); + } + + /* + * Even if this task got migrated to another CPU that _does_ have an rng + * allocated, just bail out if crypto_get_rng() ever fails in order to + * avoid looping forever. + */ + return ERR_PTR(ret); +} + +static __always_inline struct cpu_rng_inst * +lock_reseed_rng(struct crypto_rng **rng) __acquires(&cri->mlock) +{ + struct cpu_rng_inst __percpu *pcri = &pcpu_reseed_rng; + struct cpu_rng_inst *cri; + int ret; + + /* + * Use whichever CPU this task is currently running on, knowing full + * well that the task can freely migrate to other CPUs. The reseed RNG + * requires holding a lock across the entire devrandom read, so that + * another task cannot extract entropy from the same seed. In other + * words, when reseeding is requested, reseeding must be done every time + * every time mlock is acquired. + */ + cri = raw_cpu_ptr(pcri); + rt_mutex_lock(&cri->mlock); + if (likely(cri->rng)) { + /* + * Since this rng instance wasn't just allocated, it needs to be + * explicitly reseeded. New rng instances are seeded on creation + * in crypto_get_rng() and thus don't need explicit reseeding. + */ + crypto_tfm_set_flags(crypto_rng_tfm(cri->rng), + CRYPTO_TFM_REQ_NEED_RESEED); + } else { + ret = crypto_get_rng(&cri->rng); + if (ret) { + rt_mutex_unlock(&cri->mlock); + return ERR_PTR(ret); + } + } + + *rng = cri->rng; + return cri; +} + +#define lock_local_rng(rng, reseed) \ + ({ (reseed) ? lock_reseed_rng(rng) : lock_default_rng(rng); }) + +#define unlock_local_rng(cri, reseed) \ +do { \ + if (reseed) \ + rt_mutex_unlock(&(cri)->mlock); \ + else \ + local_unlock(&(cri)->lock); \ +} while (0) + +static __always_inline void +clear_rng_page(struct cpu_rng_inst *cri, size_t count) +{ + /* For zeroing a whole page, clear_page() is faster than memset() */ + count < PAGE_SIZE ? memset(cri->page, 0, count) : clear_page(cri->page); +} + +static ssize_t crypto_devrandom_read_iter(struct iov_iter *iter, bool reseed) +{ + /* lock_local_rng() puts us in atomic context for !reseed on non-RT */ + const bool atomic = !reseed && !IS_ENABLED(CONFIG_PREEMPT_RT); + const bool user_no_reseed = !reseed && user_backed_iter(iter); + size_t ulen, page_dirty_len = 0; + struct cpu_rng_inst *cri; + struct crypto_rng *rng; + void __user *uaddr; + struct page *upage; + ssize_t ret = 0; + + if (unlikely(!iov_iter_count(iter))) + return 0; + + /* Set up the starting user destination address and length */ + if (user_no_reseed) { + if (iter_is_ubuf(iter)) { + uaddr = iter->ubuf + iter->iov_offset; + ulen = iov_iter_count(iter); + } else if (iter_is_iovec(iter)) { + uaddr = iter_iov_addr(iter); + ulen = iter_iov_len(iter); + } else { + /* + * ITER_UBUF and ITER_IOVEC are the only user-backed + * iters. Bug out if a new user-backed iter appears. + */ + BUG(); + } + } + + /* Prevent rngs from getting deleted from per-CPU RNG instances */ + down_read_del_pcpu_rwsem(); +restart: + /* + * Pin the user page backing the current user destination address, + * potentially prefaulting to allocate a page for the destination. By + * prefaulting without the RNG lock held, the DRBG won't be blocked by + * time spent on page faults for this task, and thus the DRBG can still + * be used by other tasks. + */ + if (user_no_reseed && pin_user_pages_fast((unsigned long)uaddr, 1, + FOLL_WRITE, &upage) != 1) + goto up_rwsem; + + cri = lock_local_rng(&rng, reseed); + if (IS_ERR(cri)) { + if (!ret) + ret = PTR_ERR(cri); + goto unpin_upage; + } + + while (1) { + size_t copied, i = min(iov_iter_count(iter), PAGE_SIZE); + bool resched_without_lock = false; + int err; + + /* + * Generate up to one page at a time, and align to a page + * boundary so we only need to pin one user page at a time. + */ + if (user_no_reseed) + i = min3(i, PAGE_SIZE - offset_in_page(uaddr), ulen); + + /* + * On non-PREEMPT_RT kernels, local locks disable preemption. + * The DRBG's generate() function has a mutex lock, which could + * mean that we'll schedule while atomic if the mutex lock + * sleeps. However, that will never happen if we ensure that + * there's never any contention on the DRBG's mutex lock while + * we're atomic! Our local lock ensures calls to the DRBG are + * always serialized, so there's no contention from here. And + * the DRBG only uses its mutex lock from one other path, when + * an instance of the DRBG is freshly allocated, which we only + * do from crypto_get_rng(). So the DRBG's mutex lock is + * guaranteed to not have contention when we call generate() and + * thus it'll never sleep here. And of course, nothing else in + * generate() ever sleeps. + */ + err = crypto_rng_get_bytes(rng, cri->page, i); + if (err) { + if (!ret) + ret = err; + break; + } + + /* + * Record the number of bytes used in cri->page and either copy + * directly to the user address without faulting, or copy to the + * iter which is always backed by kernel memory when !reseed && + * !user_backed_iter(). When reseed == true, the iter may be + * backed by user memory, but we copy to it with the possibility + * of page faults anyway because we need to hold the lock across + * the entire call; this is why a mutex is used instead of a + * local lock for the reseed RNG, to permit sleeping without + * yielding the DRBG instance. + */ + page_dirty_len = max(i, page_dirty_len); + if (user_no_reseed) { + err = copy_to_user_nofault(uaddr, cri->page, i); + if (err >= 0) { + iov_iter_advance(iter, i - err); + ret += i - err; + } + if (err) + break; + } else { + /* + * We know that copying from cri->page is safe, so use + * _copy_to_iter() directly to skip check_copy_size(). + */ + copied = _copy_to_iter(cri->page, i, iter); + ret += copied; + if (copied != i) + break; + } + + /* + * Quit when either the requested number of bytes have been + * generated or there is a pending signal. + */ + if (!iov_iter_count(iter) || signal_pending(current)) + break; + + /* Compute the next user destination address and length */ + if (user_no_reseed) { + ulen -= i; + if (likely(ulen)) { + uaddr += i; + } else { + /* + * This path is only reachable by ITER_IOVEC + * because ulen is initialized to the request + * size for ITER_UBUF, and therefore ITER_UBUF + * will always quit at the iov_iter_count() + * check above before ulen can become zero. + * + * iter->iov_offset is guaranteed to be zero + * here, so iter_iov_{addr|len}() isn't needed. + */ + uaddr = iter_iov(iter)->iov_base; + ulen = iter_iov(iter)->iov_len; + } + + unpin_user_page(upage); + } + + /* + * Reschedule right now if needed and we're not atomic. If we're + * atomic, then we must first drop the lock to reschedule. + */ + if (need_resched()) { + if (atomic) + resched_without_lock = true; + else + cond_resched(); + } + + /* + * Optimistically try to pin the next user page without + * faulting, so we don't need to clear cri->page and drop the + * lock on every iteration. If this fails, we fall back to + * pinning with the option to prefault. + */ + if (user_no_reseed && !resched_without_lock && + pin_user_pages_fast_only((unsigned long)uaddr, 1, + FOLL_WRITE, &upage) == 1) + continue; + + /* + * Restart if either rescheduling is needed (and requires + * dropping the lock since we're atomic) or the optimistic page + * pinning attempt failed. + * + * This always implies `reseed == false`, so unlock_local_rng() + * can just be passed `false` for reseed to eliminate a branch. + */ + if (resched_without_lock || user_no_reseed) { + /* + * Clear the buffer of our latest random bytes before + * unlocking and potentially migrating CPUs, in which + * case we wouldn't have the same `cri` anymore. + */ + clear_rng_page(cri, page_dirty_len); + unlock_local_rng(cri, false); + page_dirty_len = 0; + if (resched_without_lock) + cond_resched(); + goto restart; + } + } + + if (page_dirty_len) + clear_rng_page(cri, page_dirty_len); + unlock_local_rng(cri, reseed); +unpin_upage: + if (user_no_reseed) + unpin_user_page(upage); +up_rwsem: + up_read_del_pcpu_rwsem(); + return ret ? ret : -EFAULT; +} + +static const struct random_extrng crypto_devrandom_rng = { + .extrng_read_iter = crypto_devrandom_read_iter, + .owner = THIS_MODULE, +}; + +static void free_pcpu_inst(struct cpu_rng_inst __percpu *pcri) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct cpu_rng_inst *cri = per_cpu_ptr(pcri, cpu); + + if (cri->rng) + crypto_free_rng(cri->rng); + + free_page((unsigned long)cri->page); + } +} + +static int __init alloc_pcpu_inst(struct cpu_rng_inst __percpu *pcri) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct cpu_rng_inst *cri = per_cpu_ptr(pcri, cpu); + + cri->page = (void *)__get_free_page(GFP_KERNEL); + if (!cri->page) + goto err_page_alloc; + + local_lock_init(&cri->lock); + } + + return 0; + +err_page_alloc: + while (cpu--) + free_page((unsigned long)per_cpu_ptr(pcri, cpu)->page); + return -ENOMEM; +} + +static int __init crypto_rng_init(void) +{ + int ret; + + if (!fips_enabled) + return 0; + + ret = alloc_pcpu_inst(&pcpu_default_rng); + if (ret) + return ret; + + ret = alloc_pcpu_inst(&pcpu_reseed_rng); + if (ret) + goto free_pcpu_default; + + random_register_extrng(&crypto_devrandom_rng); + return 0; + +free_pcpu_default: + free_pcpu_inst(&pcpu_default_rng); + return ret; +} + +static void __exit crypto_rng_exit(void) +{ + random_unregister_extrng(); + free_pcpu_inst(&pcpu_default_rng); + free_pcpu_inst(&pcpu_reseed_rng); +} + +late_initcall(crypto_rng_init); +module_exit(crypto_rng_exit); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Random Number Generator"); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 678bb4145d783..fcc059e7d3db5 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -123,6 +123,19 @@ static int seqiv_aead_decrypt(struct aead_request *req) return crypto_aead_decrypt(subreq); } +static int aead_init_seqiv(struct crypto_aead *aead) +{ + int err; + + err = aead_init_geniv(aead); + if (err) + return err; + + crypto_aead_set_flags(aead, CRYPTO_TFM_FIPS_COMPLIANCE); + + return 0; +} + static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) { struct aead_instance *inst; @@ -140,7 +153,7 @@ static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.encrypt = seqiv_aead_encrypt; inst->alg.decrypt = seqiv_aead_decrypt; - inst->alg.init = aead_init_geniv; + inst->alg.init = aead_init_seqiv; inst->alg.exit = aead_exit_geniv; inst->alg.base.cra_ctxsize = sizeof(struct aead_geniv_ctx); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 6a490aaa71b9a..06a77e118cd0b 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5673,7 +5673,6 @@ static const struct alg_test_desc alg_test_descs[] = { #endif .alg = "xxhash64", .test = alg_test_hash, - .fips_allowed = 1, .suite = { .hash = __VECS(xxhash64_tv_template) } diff --git a/drivers/char/random.c b/drivers/char/random.c index f344f67c83f4e..f079327d5352e 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -54,6 +54,8 @@ #include #include #include +#include +#include #include #include #ifdef CONFIG_VDSO_GETRANDOM @@ -323,6 +325,11 @@ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE], memzero_explicit(first_block, sizeof(first_block)); } +/* + * Hook for external RNG. + */ +static const struct random_extrng __rcu *extrng; + /* * This function returns a ChaCha state that you may use for generating * random data. It also returns up to 32 bytes on its own of random data @@ -735,7 +742,8 @@ static void __cold _credit_init_bits(size_t bits) queue_work(system_unbound_wq, &set_ready); atomic_notifier_call_chain(&random_ready_notifier, 0, NULL); #ifdef CONFIG_VDSO_GETRANDOM - WRITE_ONCE(vdso_k_rng_data->is_ready, true); + if (!fips_enabled) + WRITE_ONCE(vdso_k_rng_data->is_ready, true); #endif wake_up_interruptible(&crng_init_wait); kill_fasync(&fasync, SIGIO, POLL_IN); @@ -754,6 +762,8 @@ static void __cold _credit_init_bits(size_t bits) } } +static const struct file_operations extrng_random_fops; +static const struct file_operations extrng_urandom_fops; /********************************************************************** * @@ -972,6 +982,19 @@ void __init add_bootloader_randomness(const void *buf, size_t len) credit_init_bits(len * 8); } +void random_register_extrng(const struct random_extrng *rng) +{ + rcu_assign_pointer(extrng, rng); +} +EXPORT_SYMBOL_GPL(random_register_extrng); + +void random_unregister_extrng(void) +{ + RCU_INIT_POINTER(extrng, NULL); + synchronize_rcu(); +} +EXPORT_SYMBOL_GPL(random_unregister_extrng); + #if IS_ENABLED(CONFIG_VMGENID) static BLOCKING_NOTIFIER_HEAD(vmfork_chain); @@ -1386,6 +1409,7 @@ SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags { struct iov_iter iter; int ret; + const struct random_extrng *rng; if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE)) return -EINVAL; @@ -1397,6 +1421,21 @@ SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM)) return -EINVAL; + rcu_read_lock(); + rng = rcu_dereference(extrng); + if (rng && !try_module_get(rng->owner)) + rng = NULL; + rcu_read_unlock(); + + if (rng) { + ret = import_ubuf(ITER_DEST, ubuf, len, &iter); + if (unlikely(ret)) + return ret; + ret = rng->extrng_read_iter(&iter, !!(flags & GRND_RANDOM)); + module_put(rng->owner); + return ret; + } + if (!crng_ready() && !(flags & GRND_INSECURE)) { if (flags & GRND_NONBLOCK) return -EAGAIN; @@ -1417,6 +1456,12 @@ static __poll_t random_poll(struct file *file, poll_table *wait) return crng_ready() ? EPOLLIN | EPOLLRDNORM : EPOLLOUT | EPOLLWRNORM; } +static __poll_t extrng_poll(struct file *file, poll_table * wait) +{ + /* extrng pool is always full, always read, no writes */ + return EPOLLIN | EPOLLRDNORM; +} + static ssize_t write_pool_user(struct iov_iter *iter) { u8 block[BLAKE2S_BLOCK_SIZE]; @@ -1557,7 +1602,58 @@ static int random_fasync(int fd, struct file *filp, int on) return fasync_helper(fd, filp, on, &fasync); } +static int random_open(struct inode *inode, struct file *filp) +{ + const struct random_extrng *rng; + + rcu_read_lock(); + rng = rcu_dereference(extrng); + if (rng && !try_module_get(rng->owner)) + rng = NULL; + rcu_read_unlock(); + + if (!rng) + return 0; + + filp->f_op = &extrng_random_fops; + filp->private_data = rng->owner; + + return 0; +} + +static int urandom_open(struct inode *inode, struct file *filp) +{ + const struct random_extrng *rng; + + rcu_read_lock(); + rng = rcu_dereference(extrng); + if (rng && !try_module_get(rng->owner)) + rng = NULL; + rcu_read_unlock(); + + if (!rng) + return 0; + + filp->f_op = &extrng_urandom_fops; + filp->private_data = rng->owner; + + return 0; +} + +static int extrng_release(struct inode *inode, struct file *filp) +{ + module_put(filp->private_data); + return 0; +} + +static ssize_t +extrng_read_iter(struct kiocb *kiocb, struct iov_iter *iter) +{ + return rcu_dereference_raw(extrng)->extrng_read_iter(iter, false); +} + const struct file_operations random_fops = { + .open = random_open, .read_iter = random_read_iter, .write_iter = random_write_iter, .poll = random_poll, @@ -1570,6 +1666,7 @@ const struct file_operations random_fops = { }; const struct file_operations urandom_fops = { + .open = urandom_open, .read_iter = urandom_read_iter, .write_iter = random_write_iter, .unlocked_ioctl = random_ioctl, @@ -1580,6 +1677,32 @@ const struct file_operations urandom_fops = { .splice_write = iter_file_splice_write, }; +static const struct file_operations extrng_random_fops = { + .open = random_open, + .read_iter = extrng_read_iter, + .write_iter = random_write_iter, + .poll = extrng_poll, + .unlocked_ioctl = random_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = random_fasync, + .llseek = noop_llseek, + .release = extrng_release, + .splice_read = copy_splice_read, + .splice_write = iter_file_splice_write, +}; + +static const struct file_operations extrng_urandom_fops = { + .open = urandom_open, + .read_iter = extrng_read_iter, + .write_iter = random_write_iter, + .unlocked_ioctl = random_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fasync = random_fasync, + .llseek = noop_llseek, + .release = extrng_release, + .splice_read = copy_splice_read, + .splice_write = iter_file_splice_write, +}; /******************************************************************** * diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 4438637c8900c..bf7feff2fe44d 100644 --- a/fs/btrfs/Kconfig +++ b/fs/btrfs/Kconfig @@ -4,11 +4,8 @@ config BTRFS_FS tristate "Btrfs filesystem support" select BLK_CGROUP_PUNT_BIO select CRC32 - select CRYPTO - select CRYPTO_CRC32C - select CRYPTO_XXHASH - select CRYPTO_SHA256 - select CRYPTO_BLAKE2B + select CRYPTO_LIB_BLAKE2B + select CRYPTO_LIB_SHA256 select ZLIB_INFLATE select ZLIB_DEFLATE select LZO_COMPRESS @@ -18,6 +15,7 @@ config BTRFS_FS select FS_IOMAP select RAID6_PQ select XOR_BLOCKS + select XXHASH depends on PAGE_SIZE_LESS_THAN_256KB help diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 8c3899832a1aa..358466bdf0cd9 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -21,7 +21,6 @@ #include #include #include -#include #include "misc.h" #include "ctree.h" #include "fs.h" diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 655eed981078b..16c22a1ac97ef 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -18,7 +18,6 @@ #include #include #include -#include #include "ctree.h" #include "disk-io.h" #include "transaction.h" @@ -62,12 +61,6 @@ static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info); static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info); -static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info) -{ - if (fs_info->csum_shash) - crypto_free_shash(fs_info->csum_shash); -} - /* * Compute the csum of a btree block and store the result to provided buffer. */ @@ -76,12 +69,11 @@ static void csum_tree_block(struct extent_buffer *buf, u8 *result) struct btrfs_fs_info *fs_info = buf->fs_info; int num_pages; u32 first_page_part; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); + struct btrfs_csum_ctx csum; char *kaddr; int i; - shash->tfm = fs_info->csum_shash; - crypto_shash_init(shash); + btrfs_csum_init(&csum, fs_info->csum_type); if (buf->addr) { /* Pages are contiguous, handle them as a big one. */ @@ -94,21 +86,21 @@ static void csum_tree_block(struct extent_buffer *buf, u8 *result) num_pages = num_extent_pages(buf); } - crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE, - first_page_part - BTRFS_CSUM_SIZE); + btrfs_csum_update(&csum, kaddr + BTRFS_CSUM_SIZE, + first_page_part - BTRFS_CSUM_SIZE); /* * Multiple single-page folios case would reach here. * * nodesize <= PAGE_SIZE and large folio all handled by above - * crypto_shash_update() already. + * btrfs_csum_update() already. */ for (i = 1; i < num_pages && INLINE_EXTENT_BUFFER_PAGES > 1; i++) { kaddr = folio_address(buf->folios[i]); - crypto_shash_update(shash, kaddr, PAGE_SIZE); + btrfs_csum_update(&csum, kaddr, PAGE_SIZE); } memset(result, 0, BTRFS_CSUM_SIZE); - crypto_shash_final(shash, result); + btrfs_csum_final(&csum, result); } /* @@ -160,18 +152,15 @@ static bool btrfs_supported_super_csum(u16 csum_type) int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, const struct btrfs_super_block *disk_sb) { - char result[BTRFS_CSUM_SIZE]; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); - - shash->tfm = fs_info->csum_shash; + u8 result[BTRFS_CSUM_SIZE]; /* * The super_block structure does not span the whole * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is * filled with zeros and is included in the checksum. */ - crypto_shash_digest(shash, (const u8 *)disk_sb + BTRFS_CSUM_SIZE, - BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result); + btrfs_csum(fs_info->csum_type, (const u8 *)disk_sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result); if (memcmp(disk_sb->csum, result, fs_info->csum_size)) return 1; @@ -1235,7 +1224,6 @@ void btrfs_free_fs_info(struct btrfs_fs_info *fs_info) ASSERT(percpu_counter_sum_positive(em_counter) == 0); percpu_counter_destroy(em_counter); percpu_counter_destroy(&fs_info->dev_replace.bio_counter); - btrfs_free_csum_hash(fs_info); btrfs_free_stripe_hash_table(fs_info); btrfs_free_ref_cache(fs_info); kfree(fs_info->balance_ctl); @@ -1994,21 +1982,8 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) return 0; } -static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type) +static void btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type) { - struct crypto_shash *csum_shash; - const char *csum_driver = btrfs_super_csum_driver(csum_type); - - csum_shash = crypto_alloc_shash(csum_driver, 0, 0); - - if (IS_ERR(csum_shash)) { - btrfs_err(fs_info, "error allocating %s hash for checksum", - csum_driver); - return PTR_ERR(csum_shash); - } - - fs_info->csum_shash = csum_shash; - /* Check if the checksum implementation is a fast accelerated one. */ switch (csum_type) { case BTRFS_CSUM_TYPE_CRC32: @@ -2022,10 +1997,8 @@ static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type) break; } - btrfs_info(fs_info, "using %s (%s) checksum algorithm", - btrfs_super_csum_name(csum_type), - crypto_shash_driver_name(csum_shash)); - return 0; + btrfs_info(fs_info, "using %s checksum algorithm", + btrfs_super_csum_name(csum_type)); } static int btrfs_replay_log(struct btrfs_fs_info *fs_info, @@ -3318,12 +3291,9 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device } fs_info->csum_size = btrfs_super_csum_size(disk_super); + fs_info->csum_type = csum_type; - ret = btrfs_init_csum_hash(fs_info, csum_type); - if (ret) { - btrfs_release_disk_super(disk_super); - goto fail_alloc; - } + btrfs_init_csum_hash(fs_info, csum_type); /* * We want to check superblock checksum, the type is stored inside. @@ -3721,7 +3691,6 @@ static int write_dev_supers(struct btrfs_device *device, { struct btrfs_fs_info *fs_info = device->fs_info; struct address_space *mapping = device->bdev->bd_mapping; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); int i; int ret; u64 bytenr, bytenr_orig; @@ -3731,8 +3700,6 @@ static int write_dev_supers(struct btrfs_device *device, if (max_mirrors == 0) max_mirrors = BTRFS_SUPER_MIRROR_MAX; - shash->tfm = fs_info->csum_shash; - for (i = 0; i < max_mirrors; i++) { struct folio *folio; struct bio *bio; @@ -3756,9 +3723,8 @@ static int write_dev_supers(struct btrfs_device *device, btrfs_set_super_bytenr(sb, bytenr_orig); - crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE, - BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, - sb->csum); + btrfs_csum(fs_info->csum_type, (const u8 *)sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, sb->csum); folio = __filemap_get_folio(mapping, bytenr >> PAGE_SHIFT, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 4b7c40f05e8f9..90cf12073a52d 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -8,7 +8,6 @@ #include #include #include -#include #include "messages.h" #include "ctree.h" #include "disk-io.h" @@ -769,7 +768,6 @@ static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src) { struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); struct bio *bio = &bbio->bio; struct btrfs_ordered_sum *sums = bbio->sums; struct bvec_iter iter = *src; @@ -777,8 +775,6 @@ static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src) const u32 blocksize = fs_info->sectorsize; int index = 0; - shash->tfm = fs_info->csum_shash; - btrfs_bio_for_each_block(paddr, bio, &iter, blocksize) { btrfs_calculate_block_csum(fs_info, paddr, sums->sums + index); index += fs_info->csum_size; diff --git a/fs/btrfs/fs.c b/fs/btrfs/fs.c index feb0a2faa8379..14d83565cdee1 100644 --- a/fs/btrfs/fs.c +++ b/fs/btrfs/fs.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include "messages.h" #include "fs.h" #include "accessors.h" @@ -8,13 +9,11 @@ static const struct btrfs_csums { u16 size; const char name[10]; - const char driver[12]; } btrfs_csums[] = { [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, - [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b", - .driver = "blake2b-256" }, + [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b" }, }; /* This exists for btrfs-progs usages. */ @@ -37,21 +36,94 @@ const char *btrfs_super_csum_name(u16 csum_type) return btrfs_csums[csum_type].name; } -/* - * Return driver name if defined, otherwise the name that's also a valid driver - * name. - */ -const char *btrfs_super_csum_driver(u16 csum_type) +size_t __attribute_const__ btrfs_get_num_csums(void) { - /* csum type is validated at mount time */ - return btrfs_csums[csum_type].driver[0] ? - btrfs_csums[csum_type].driver : - btrfs_csums[csum_type].name; + return ARRAY_SIZE(btrfs_csums); } -size_t __attribute_const__ btrfs_get_num_csums(void) +void btrfs_csum(u16 csum_type, const u8 *data, size_t len, u8 *out) { - return ARRAY_SIZE(btrfs_csums); + switch (csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + put_unaligned_le32(~crc32c(~0, data, len), out); + break; + case BTRFS_CSUM_TYPE_XXHASH: + put_unaligned_le64(xxh64(data, len, 0), out); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256(data, len, out); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b(NULL, 0, data, len, out, 32); + break; + default: + /* Checksum type is validated at mount time. */ + BUG(); + } +} + +void btrfs_csum_init(struct btrfs_csum_ctx *ctx, u16 csum_type) +{ + ctx->csum_type = csum_type; + switch (ctx->csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + ctx->crc32 = ~0; + break; + case BTRFS_CSUM_TYPE_XXHASH: + xxh64_reset(&ctx->xxh64, 0); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256_init(&ctx->sha256); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b_init(&ctx->blake2b, 32); + break; + default: + /* Checksume type is validated at mount time. */ + BUG(); + } +} + +void btrfs_csum_update(struct btrfs_csum_ctx *ctx, const u8 *data, size_t len) +{ + switch (ctx->csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + ctx->crc32 = crc32c(ctx->crc32, data, len); + break; + case BTRFS_CSUM_TYPE_XXHASH: + xxh64_update(&ctx->xxh64, data, len); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256_update(&ctx->sha256, data, len); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b_update(&ctx->blake2b, data, len); + break; + default: + /* Checksum type is validated at mount time. */ + BUG(); + } +} + +void btrfs_csum_final(struct btrfs_csum_ctx *ctx, u8 *out) +{ + switch (ctx->csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + put_unaligned_le32(~ctx->crc32, out); + break; + case BTRFS_CSUM_TYPE_XXHASH: + put_unaligned_le64(xxh64_digest(&ctx->xxh64), out); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256_final(&ctx->sha256, out); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b_final(&ctx->blake2b, out); + break; + default: + /* Checksum type is validated at mount time. */ + BUG(); + } } /* diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h index 37aa8d141a83d..04424d9486416 100644 --- a/fs/btrfs/fs.h +++ b/fs/btrfs/fs.h @@ -3,6 +3,8 @@ #ifndef BTRFS_FS_H #define BTRFS_FS_H +#include +#include #include #include #include @@ -24,6 +26,7 @@ #include #include #include +#include #include #include #include "extent-io-tree.h" @@ -34,7 +37,6 @@ struct inode; struct super_block; struct kobject; struct reloc_control; -struct crypto_shash; struct ulist; struct btrfs_device; struct btrfs_block_group; @@ -837,9 +839,10 @@ struct btrfs_fs_info { u32 sectorsize_bits; u32 block_min_order; u32 block_max_order; + u32 stripesize; u32 csum_size; u32 csums_per_leaf; - u32 stripesize; + u32 csum_type; /* * Maximum size of an extent. BTRFS_MAX_EXTENT_SIZE on regular @@ -851,8 +854,6 @@ struct btrfs_fs_info { spinlock_t swapfile_pins_lock; struct rb_root swapfile_pins; - struct crypto_shash *csum_shash; - /* Type of exclusive operation running, protected by super_lock */ enum btrfs_exclusive_operation exclusive_operation; @@ -1044,8 +1045,20 @@ int btrfs_check_ioctl_vol_args_path(const struct btrfs_ioctl_vol_args *vol_args) u16 btrfs_csum_type_size(u16 type); int btrfs_super_csum_size(const struct btrfs_super_block *s); const char *btrfs_super_csum_name(u16 csum_type); -const char *btrfs_super_csum_driver(u16 csum_type); size_t __attribute_const__ btrfs_get_num_csums(void); +struct btrfs_csum_ctx { + u16 csum_type; + union { + u32 crc32; + struct xxh64_state xxh64; + struct sha256_ctx sha256; + struct blake2b_ctx blake2b; + }; +}; +void btrfs_csum(u16 csum_type, const u8 *data, size_t len, u8 *out); +void btrfs_csum_init(struct btrfs_csum_ctx *ctx, u16 csum_type); +void btrfs_csum_update(struct btrfs_csum_ctx *ctx, const u8 *data, size_t len); +void btrfs_csum_final(struct btrfs_csum_ctx *ctx, u8 *out); static inline bool btrfs_is_empty_uuid(const u8 *uuid) { diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 2c361e0691fc5..f2893fe6b215b 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3,7 +3,6 @@ * Copyright (C) 2007 Oracle. All rights reserved. */ -#include #include #include #include @@ -3353,29 +3352,28 @@ void btrfs_calculate_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr { struct folio *folio = page_folio(phys_to_page(paddr)); const u32 blocksize = fs_info->sectorsize; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); + struct btrfs_csum_ctx csum; - shash->tfm = fs_info->csum_shash; /* The full block must be inside the folio. */ ASSERT(offset_in_folio(folio, paddr) + blocksize <= folio_size(folio)); if (folio_test_partial_kmap(folio)) { size_t cur = paddr; - crypto_shash_init(shash); + btrfs_csum_init(&csum, fs_info->csum_type); while (cur < paddr + blocksize) { void *kaddr; size_t len = min(paddr + blocksize - cur, PAGE_SIZE - offset_in_page(cur)); kaddr = kmap_local_folio(folio, offset_in_folio(folio, cur)); - crypto_shash_update(shash, kaddr, len); + btrfs_csum_update(&csum, kaddr, len); kunmap_local(kaddr); cur += len; } - crypto_shash_final(shash, dest); + btrfs_csum_final(&csum, dest); } else { - crypto_shash_digest(shash, phys_to_virt(paddr), blocksize, dest); + btrfs_csum(fs_info->csum_type, phys_to_virt(paddr), blocksize, dest); } } /* diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 16936d17166ee..c4241014fe114 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -6,7 +6,6 @@ #include #include #include -#include #include "ctree.h" #include "discard.h" #include "volumes.h" @@ -720,7 +719,7 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits); void *first_kaddr = scrub_stripe_get_kaddr(stripe, sector_nr); struct btrfs_header *header = first_kaddr; - SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); + struct btrfs_csum_ctx csum; u8 on_disk_csum[BTRFS_CSUM_SIZE]; u8 calculated_csum[BTRFS_CSUM_SIZE]; @@ -762,17 +761,16 @@ static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr } /* Now check tree block csum. */ - shash->tfm = fs_info->csum_shash; - crypto_shash_init(shash); - crypto_shash_update(shash, first_kaddr + BTRFS_CSUM_SIZE, - fs_info->sectorsize - BTRFS_CSUM_SIZE); + btrfs_csum_init(&csum, fs_info->csum_type); + btrfs_csum_update(&csum, first_kaddr + BTRFS_CSUM_SIZE, + fs_info->sectorsize - BTRFS_CSUM_SIZE); for (int i = sector_nr + 1; i < sector_nr + sectors_per_tree; i++) { - crypto_shash_update(shash, scrub_stripe_get_kaddr(stripe, i), - fs_info->sectorsize); + btrfs_csum_update(&csum, scrub_stripe_get_kaddr(stripe, i), + fs_info->sectorsize); } - crypto_shash_final(shash, calculated_csum); + btrfs_csum_final(&csum, calculated_csum); if (memcmp(calculated_csum, on_disk_csum, fs_info->csum_size) != 0) { scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index c40944ca7b948..5c01637bcc9dc 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -2641,7 +2641,3 @@ module_exit(exit_btrfs_fs) MODULE_DESCRIPTION("B-Tree File System (BTRFS)"); MODULE_LICENSE("GPL"); -MODULE_SOFTDEP("pre: crc32c"); -MODULE_SOFTDEP("pre: xxhash64"); -MODULE_SOFTDEP("pre: sha256"); -MODULE_SOFTDEP("pre: blake2b-256"); diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index d66681ce2b3da..3afcea340c083 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -10,7 +10,6 @@ #include #include #include -#include #include "messages.h" #include "ctree.h" #include "discard.h" @@ -1252,10 +1251,9 @@ static ssize_t btrfs_checksum_show(struct kobject *kobj, { struct btrfs_fs_info *fs_info = to_fs_info(kobj); u16 csum_type = btrfs_super_csum_type(fs_info->super_copy); + const char *csum_name = btrfs_super_csum_name(csum_type); - return sysfs_emit(buf, "%s (%s)\n", - btrfs_super_csum_name(csum_type), - crypto_shash_driver_name(fs_info->csum_shash)); + return sysfs_emit(buf, "%s (%s-lib)\n", csum_name, csum_name); } BTRFS_ATTR(, checksum, btrfs_checksum_show); diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 9339da7c20a8b..0f540a1cebff3 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -26,6 +26,7 @@ struct crypto_aes_ctx { u32 key_enc[AES_MAX_KEYLENGTH_U32]; u32 key_dec[AES_MAX_KEYLENGTH_U32]; u32 key_length; + bool fips_approved; }; extern const u32 crypto_ft_tab[4][256] ____cacheline_aligned; diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h index dd7694477e50f..4879e2ec26867 100644 --- a/include/crypto/blake2b.h +++ b/include/crypto/blake2b.h @@ -28,6 +28,25 @@ enum blake2b_lengths { BLAKE2B_512_HASH_SIZE = 64, }; +/** + * struct blake2b_ctx - Context for hashing a message with BLAKE2b + * @h: compression function state + * @t: block counter + * @f: finalization indicator + * @buf: partial block buffer; 'buflen' bytes are valid + * @buflen: number of bytes buffered in @buf + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + */ +struct blake2b_ctx { + /* 'h', 't', and 'f' are used in assembly code, so keep them as-is. */ + u64 h[8]; + u64 t[2]; + u64 f[2]; + u8 buf[BLAKE2B_BLOCK_SIZE]; + unsigned int buflen; + unsigned int outlen; +}; + enum blake2b_iv { BLAKE2B_IV0 = 0x6A09E667F3BCC908ULL, BLAKE2B_IV1 = 0xBB67AE8584CAA73BULL, @@ -39,19 +58,109 @@ enum blake2b_iv { BLAKE2B_IV7 = 0x5BE0CD19137E2179ULL, }; -static inline void __blake2b_init(struct blake2b_state *state, size_t outlen, - size_t keylen) +static inline void __blake2b_init(struct blake2b_ctx *ctx, size_t outlen, + const void *key, size_t keylen) +{ + ctx->h[0] = BLAKE2B_IV0 ^ (0x01010000 | keylen << 8 | outlen); + ctx->h[1] = BLAKE2B_IV1; + ctx->h[2] = BLAKE2B_IV2; + ctx->h[3] = BLAKE2B_IV3; + ctx->h[4] = BLAKE2B_IV4; + ctx->h[5] = BLAKE2B_IV5; + ctx->h[6] = BLAKE2B_IV6; + ctx->h[7] = BLAKE2B_IV7; + ctx->t[0] = 0; + ctx->t[1] = 0; + ctx->f[0] = 0; + ctx->f[1] = 0; + ctx->buflen = 0; + ctx->outlen = outlen; + if (keylen) { + memcpy(ctx->buf, key, keylen); + memset(&ctx->buf[keylen], 0, BLAKE2B_BLOCK_SIZE - keylen); + ctx->buflen = BLAKE2B_BLOCK_SIZE; + } +} + +/** + * blake2b_init() - Initialize a BLAKE2b context for a new message (unkeyed) + * @ctx: the context to initialize + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + * + * Context: Any context. + */ +static inline void blake2b_init(struct blake2b_ctx *ctx, size_t outlen) +{ + __blake2b_init(ctx, outlen, NULL, 0); +} + +/** + * blake2b_init_key() - Initialize a BLAKE2b context for a new message (keyed) + * @ctx: the context to initialize + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + * @key: the key + * @keylen: the key length in bytes, at most BLAKE2B_KEY_SIZE + * + * Context: Any context. + */ +static inline void blake2b_init_key(struct blake2b_ctx *ctx, size_t outlen, + const void *key, size_t keylen) +{ + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2B_HASH_SIZE || + !key || !keylen || keylen > BLAKE2B_KEY_SIZE)); + + __blake2b_init(ctx, outlen, key, keylen); +} + +/** + * blake2b_update() - Update a BLAKE2b context with message data + * @ctx: the context to update; must have been initialized + * @in: the message data + * @inlen: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +void blake2b_update(struct blake2b_ctx *ctx, const u8 *in, size_t inlen); + +/** + * blake2b_final() - Finish computing a BLAKE2b hash + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting BLAKE2b hash. Its length will be equal to the + * @outlen that was passed to blake2b_init() or blake2b_init_key(). + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void blake2b_final(struct blake2b_ctx *ctx, u8 *out); + +/** + * blake2b() - Compute BLAKE2b hash in one shot + * @key: the key, or NULL for an unkeyed hash + * @keylen: the key length in bytes (at most BLAKE2B_KEY_SIZE), or 0 for an + * unkeyed hash + * @in: the message data + * @inlen: the data length in bytes + * @out: (output) the resulting BLAKE2b hash, with length @outlen + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + * + * Context: Any context. + */ +static inline void blake2b(const u8 *key, size_t keylen, + const u8 *in, size_t inlen, + u8 *out, size_t outlen) { - state->h[0] = BLAKE2B_IV0 ^ (0x01010000 | keylen << 8 | outlen); - state->h[1] = BLAKE2B_IV1; - state->h[2] = BLAKE2B_IV2; - state->h[3] = BLAKE2B_IV3; - state->h[4] = BLAKE2B_IV4; - state->h[5] = BLAKE2B_IV5; - state->h[6] = BLAKE2B_IV6; - state->h[7] = BLAKE2B_IV7; - state->t[0] = 0; - state->t[1] = 0; + struct blake2b_ctx ctx; + + WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen || + outlen > BLAKE2B_HASH_SIZE || keylen > BLAKE2B_KEY_SIZE || + (!key && keylen))); + + __blake2b_init(&ctx, outlen, key, keylen); + blake2b_update(&ctx, in, inlen); + blake2b_final(&ctx, out); } #endif /* _CRYPTO_BLAKE2B_H */ diff --git a/include/crypto/internal/blake2b.h b/include/crypto/internal/blake2b.h index 3e09e24853060..3712df69def18 100644 --- a/include/crypto/internal/blake2b.h +++ b/include/crypto/internal/blake2b.h @@ -57,13 +57,28 @@ static inline int crypto_blake2b_setkey(struct crypto_shash *tfm, return 0; } +static inline void __crypto_blake2b_init(struct blake2b_state *state, + size_t outlen, size_t keylen) +{ + state->h[0] = BLAKE2B_IV0 ^ (0x01010000 | keylen << 8 | outlen); + state->h[1] = BLAKE2B_IV1; + state->h[2] = BLAKE2B_IV2; + state->h[3] = BLAKE2B_IV3; + state->h[4] = BLAKE2B_IV4; + state->h[5] = BLAKE2B_IV5; + state->h[6] = BLAKE2B_IV6; + state->h[7] = BLAKE2B_IV7; + state->t[0] = 0; + state->t[1] = 0; +} + static inline int crypto_blake2b_init(struct shash_desc *desc) { const struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); struct blake2b_state *state = shash_desc_ctx(desc); unsigned int outlen = crypto_shash_digestsize(desc->tfm); - __blake2b_init(state, outlen, tctx->keylen); + __crypto_blake2b_init(state, outlen, tctx->keylen); return tctx->keylen ? crypto_shash_update(desc, tctx->key, BLAKE2B_BLOCK_SIZE) : 0; } diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index 162a529ec8413..7dc0bff8d6be6 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -24,6 +24,7 @@ struct sha1_state { u32 state[SHA1_DIGEST_SIZE / 4]; u64 count; u8 buffer[SHA1_BLOCK_SIZE]; + bool fips_approved; }; /* @@ -51,6 +52,7 @@ struct sha1_ctx { struct sha1_block_state state; u64 bytecount; u8 buf[SHA1_BLOCK_SIZE]; + bool fips_approved; }; /** @@ -104,6 +106,7 @@ void sha1(const u8 *data, size_t len, u8 out[SHA1_DIGEST_SIZE]); struct hmac_sha1_key { struct sha1_block_state istate; struct sha1_block_state ostate; + bool fips_approved; }; /** @@ -114,6 +117,7 @@ struct hmac_sha1_key { struct hmac_sha1_ctx { struct sha1_ctx sha_ctx; struct sha1_block_state ostate; + bool fips_approved; }; /** diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index e5dafb935cc88..2dc60205e671f 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -153,6 +153,7 @@ void __hmac_sha256_init(struct __hmac_sha256_ctx *ctx, */ struct sha224_ctx { struct __sha256_ctx ctx; + bool fips_approved; }; /** @@ -208,6 +209,7 @@ void sha224(const u8 *data, size_t len, u8 out[SHA224_DIGEST_SIZE]); */ struct hmac_sha224_key { struct __hmac_sha256_key key; + bool fips_approved; }; /** @@ -216,6 +218,7 @@ struct hmac_sha224_key { */ struct hmac_sha224_ctx { struct __hmac_sha256_ctx ctx; + bool fips_approved; }; /** @@ -245,6 +248,7 @@ static inline void hmac_sha224_init(struct hmac_sha224_ctx *ctx, const struct hmac_sha224_key *key) { __hmac_sha256_init(&ctx->ctx, &key->key); + ctx->fips_approved = key->fips_approved; } /** @@ -326,6 +330,7 @@ void hmac_sha224_usingrawkey(const u8 *raw_key, size_t raw_key_len, */ struct sha256_ctx { struct __sha256_ctx ctx; + bool fips_approved; }; /** @@ -409,6 +414,7 @@ bool sha256_finup_2x_is_optimized(void); */ struct hmac_sha256_key { struct __hmac_sha256_key key; + bool fips_approved; }; /** @@ -417,6 +423,7 @@ struct hmac_sha256_key { */ struct hmac_sha256_ctx { struct __hmac_sha256_ctx ctx; + bool fips_approved; }; /** @@ -446,6 +453,7 @@ static inline void hmac_sha256_init(struct hmac_sha256_ctx *ctx, const struct hmac_sha256_key *key) { __hmac_sha256_init(&ctx->ctx, &key->key); + ctx->fips_approved = key->fips_approved; } /** @@ -561,6 +569,7 @@ void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx, */ struct sha384_ctx { struct __sha512_ctx ctx; + bool fips_approved; }; /** @@ -616,6 +625,7 @@ void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE]); */ struct hmac_sha384_key { struct __hmac_sha512_key key; + bool fips_approved; }; /** @@ -624,6 +634,7 @@ struct hmac_sha384_key { */ struct hmac_sha384_ctx { struct __hmac_sha512_ctx ctx; + bool fips_approved; }; /** @@ -653,6 +664,7 @@ static inline void hmac_sha384_init(struct hmac_sha384_ctx *ctx, const struct hmac_sha384_key *key) { __hmac_sha512_init(&ctx->ctx, &key->key); + ctx->fips_approved = key->fips_approved; } /** @@ -734,6 +746,7 @@ void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len, */ struct sha512_ctx { struct __sha512_ctx ctx; + bool fips_approved; }; /** @@ -789,6 +802,7 @@ void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE]); */ struct hmac_sha512_key { struct __hmac_sha512_key key; + bool fips_approved; }; /** @@ -797,6 +811,7 @@ struct hmac_sha512_key { */ struct hmac_sha512_ctx { struct __hmac_sha512_ctx ctx; + bool fips_approved; }; /** @@ -826,6 +841,7 @@ static inline void hmac_sha512_init(struct hmac_sha512_ctx *ctx, const struct hmac_sha512_key *key) { __hmac_sha512_init(&ctx->ctx, &key->key); + ctx->fips_approved = key->fips_approved; } /** diff --git a/include/linux/byteorder/generic.h b/include/linux/byteorder/generic.h index b3705e8bbe2b8..55a44199de872 100644 --- a/include/linux/byteorder/generic.h +++ b/include/linux/byteorder/generic.h @@ -173,6 +173,22 @@ static inline void cpu_to_le32_array(u32 *buf, unsigned int words) } } +static inline void le64_to_cpu_array(u64 *buf, unsigned int words) +{ + while (words--) { + __le64_to_cpus(buf); + buf++; + } +} + +static inline void cpu_to_le64_array(u64 *buf, unsigned int words) +{ + while (words--) { + __cpu_to_le64s(buf); + buf++; + } +} + static inline void memcpy_from_le32(u32 *dst, const __le32 *src, size_t words) { size_t i; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index a2137e19be7d8..df268ca70170e 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -151,6 +151,9 @@ #define CRYPTO_TFM_REQ_MAY_SLEEP 0x00000200 #define CRYPTO_TFM_REQ_MAY_BACKLOG 0x00000400 #define CRYPTO_TFM_REQ_ON_STACK 0x00000800 +#define CRYPTO_TFM_REQ_NEED_RESEED 0x00001000 + +#define CRYPTO_TFM_FIPS_COMPLIANCE 0x80000000 /* * Miscellaneous stuff. diff --git a/include/linux/mm.h b/include/linux/mm.h index 1e74eb7267acb..82700934149ea 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2625,6 +2625,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages); int pin_user_pages_fast(unsigned long start, int nr_pages, unsigned int gup_flags, struct page **pages); +int pin_user_pages_fast_only(unsigned long start, int nr_pages, + unsigned int gup_flags, struct page **pages); void folio_add_pin(struct folio *folio); int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc); diff --git a/include/linux/random.h b/include/linux/random.h index 333cecfca93fd..6421bf2415a53 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -9,6 +9,13 @@ #include +struct iov_iter; + +struct random_extrng { + ssize_t (*extrng_read_iter)(struct iov_iter *iter, bool reseed); + struct module *owner; +}; + struct notifier_block; void add_device_randomness(const void *buf, size_t len); @@ -150,6 +157,9 @@ int random_prepare_cpu(unsigned int cpu); int random_online_cpu(unsigned int cpu); #endif +void random_register_extrng(const struct random_extrng *rng); +void random_unregister_extrng(void); + #ifndef MODULE extern const struct file_operations random_fops, urandom_fops; #endif diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index fa882108ba62c..8efbb0b9f2458 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -28,6 +28,16 @@ config CRYPTO_LIB_ARC4 config CRYPTO_LIB_GF128MUL tristate +config CRYPTO_LIB_BLAKE2B + tristate + help + The BLAKE2b library functions. Select this if your module uses any of + the functions from . + +config CRYPTO_LIB_BLAKE2B_ARCH + bool + depends on CRYPTO_LIB_BLAKE2B && !UML + # BLAKE2s support is always built-in, so there's no CRYPTO_LIB_BLAKE2S option. config CRYPTO_LIB_BLAKE2S_ARCH diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index d2845b2145858..5fe37b5c9916b 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -31,6 +31,15 @@ obj-$(CONFIG_CRYPTO_LIB_GF128MUL) += gf128mul.o ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_BLAKE2B) += libblake2b.o +libblake2b-y := blake2b.o +CFLAGS_blake2b.o := -Wframe-larger-than=4096 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105930 +ifeq ($(CONFIG_CRYPTO_LIB_BLAKE2B_ARCH),y) +CFLAGS_blake2b.o += -I$(src)/$(SRCARCH) +endif # CONFIG_CRYPTO_LIB_BLAKE2B_ARCH + +################################################################################ + # blake2s is used by the /dev/random driver which is always builtin obj-y += blake2s.o ifeq ($(CONFIG_CRYPTO_LIB_BLAKE2S_ARCH),y) diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index 102aaa76bc8d7..34c5752fc8bef 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -246,6 +247,7 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, ctx->key_dec[i + 2] = ctx->key_enc[2]; ctx->key_dec[i + 3] = ctx->key_enc[3]; + ctx->fips_approved = fips_enabled; return 0; } EXPORT_SYMBOL(aes_expandkey); diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 0f294c8cbf3c2..b6370857a98e9 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -10,6 +10,7 @@ #include #include #include +#include static void aescfb_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst, const void *src) @@ -219,6 +220,8 @@ static int __init libaescfb_init(void) if (aes_expandkey(&ctx, aescfb_tv[i].key, aescfb_tv[i].klen)) { pr_err("aes_expandkey() failed on vector %d\n", i); + if (fips_enabled) + panic("aes_expandkey() failed on vector %d\n", i); return -ENODEV; } @@ -226,6 +229,8 @@ static int __init libaescfb_init(void) aescfb_tv[i].iv); if (memcmp(buf, aescfb_tv[i].ctext, aescfb_tv[i].len)) { pr_err("aescfb_encrypt() #1 failed on vector %d\n", i); + if (fips_enabled) + panic("aescfb_encrypt() #1 failed on vector %d\n", i); return -ENODEV; } @@ -233,6 +238,8 @@ static int __init libaescfb_init(void) aescfb_decrypt(&ctx, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv); if (memcmp(buf, aescfb_tv[i].ptext, aescfb_tv[i].len)) { pr_err("aescfb_decrypt() failed on vector %d\n", i); + if (fips_enabled) + panic("aescfb_decrypt() failed on vector %d\n", i); return -ENODEV; } @@ -240,6 +247,8 @@ static int __init libaescfb_init(void) aescfb_encrypt(&ctx, buf, buf, aescfb_tv[i].len, aescfb_tv[i].iv); if (memcmp(buf, aescfb_tv[i].ctext, aescfb_tv[i].len)) { pr_err("aescfb_encrypt() #2 failed on vector %d\n", i); + if (fips_enabled) + panic("aescfb_encrypt() #2 failed on vector %d\n", i); return -ENODEV; } diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index ac0b2fcfd6069..63945fec130ea 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -11,6 +11,7 @@ #include #include #include +#include static void aesgcm_encrypt_block(const struct crypto_aes_ctx *ctx, void *dst, const void *src) @@ -701,6 +702,8 @@ static int __init libaesgcm_init(void) if (aesgcm_expandkey(&ctx, aesgcm_tv[i].key, aesgcm_tv[i].klen, aesgcm_tv[i].clen - plen)) { pr_err("aesgcm_expandkey() failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_expandkey() failed on vector %d\n", i); return -ENODEV; } @@ -709,6 +712,8 @@ static int __init libaesgcm_init(void) aesgcm_tv[i].iv, aesgcm_tv[i].ctext + plen) || memcmp(buf, aesgcm_tv[i].ptext, plen)) { pr_err("aesgcm_decrypt() #1 failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_decrypt() #1 failed on vector %d\n", i); return -ENODEV; } @@ -717,6 +722,8 @@ static int __init libaesgcm_init(void) aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf); if (memcmp(buf, aesgcm_tv[i].ctext, plen)) { pr_err("aesgcm_encrypt() failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_encrypt() failed on vector %d\n", i); return -ENODEV; } @@ -725,6 +732,8 @@ static int __init libaesgcm_init(void) aesgcm_tv[i].alen, aesgcm_tv[i].iv, tagbuf) || memcmp(buf, aesgcm_tv[i].ptext, plen)) { pr_err("aesgcm_decrypt() #2 failed on vector %d\n", i); + if (fips_enabled) + panic("aesgcm_decrypt() #2 failed on vector %d\n", i); return -ENODEV; } } diff --git a/lib/crypto/blake2b.c b/lib/crypto/blake2b.c new file mode 100644 index 0000000000000..09c6d65d8a6e6 --- /dev/null +++ b/lib/crypto/blake2b.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright 2025 Google LLC + * + * This is an implementation of the BLAKE2b hash and PRF functions. + * + * Information: https://blake2.net/ + */ + +#include +#include +#include +#include +#include +#include +#include + +static const u8 blake2b_sigma[12][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + +static inline void blake2b_increment_counter(struct blake2b_ctx *ctx, u32 inc) +{ + ctx->t[0] += inc; + ctx->t[1] += (ctx->t[0] < inc); +} + +static void __maybe_unused +blake2b_compress_generic(struct blake2b_ctx *ctx, + const u8 *data, size_t nblocks, u32 inc) +{ + u64 m[16]; + u64 v[16]; + int i; + + WARN_ON(IS_ENABLED(DEBUG) && + (nblocks > 1 && inc != BLAKE2B_BLOCK_SIZE)); + + while (nblocks > 0) { + blake2b_increment_counter(ctx, inc); + memcpy(m, data, BLAKE2B_BLOCK_SIZE); + le64_to_cpu_array(m, ARRAY_SIZE(m)); + memcpy(v, ctx->h, 64); + v[ 8] = BLAKE2B_IV0; + v[ 9] = BLAKE2B_IV1; + v[10] = BLAKE2B_IV2; + v[11] = BLAKE2B_IV3; + v[12] = BLAKE2B_IV4 ^ ctx->t[0]; + v[13] = BLAKE2B_IV5 ^ ctx->t[1]; + v[14] = BLAKE2B_IV6 ^ ctx->f[0]; + v[15] = BLAKE2B_IV7 ^ ctx->f[1]; + +#define G(r, i, a, b, c, d) do { \ + a += b + m[blake2b_sigma[r][2 * i + 0]]; \ + d = ror64(d ^ a, 32); \ + c += d; \ + b = ror64(b ^ c, 24); \ + a += b + m[blake2b_sigma[r][2 * i + 1]]; \ + d = ror64(d ^ a, 16); \ + c += d; \ + b = ror64(b ^ c, 63); \ +} while (0) + +#define ROUND(r) do { \ + G(r, 0, v[0], v[ 4], v[ 8], v[12]); \ + G(r, 1, v[1], v[ 5], v[ 9], v[13]); \ + G(r, 2, v[2], v[ 6], v[10], v[14]); \ + G(r, 3, v[3], v[ 7], v[11], v[15]); \ + G(r, 4, v[0], v[ 5], v[10], v[15]); \ + G(r, 5, v[1], v[ 6], v[11], v[12]); \ + G(r, 6, v[2], v[ 7], v[ 8], v[13]); \ + G(r, 7, v[3], v[ 4], v[ 9], v[14]); \ +} while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + ROUND(10); + ROUND(11); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + ctx->h[i] ^= v[i] ^ v[i + 8]; + + data += BLAKE2B_BLOCK_SIZE; + --nblocks; + } +} + +#ifdef CONFIG_CRYPTO_LIB_BLAKE2B_ARCH +#include "blake2b.h" /* $(SRCARCH)/blake2b.h */ +#else +#define blake2b_compress blake2b_compress_generic +#endif + +static inline void blake2b_set_lastblock(struct blake2b_ctx *ctx) +{ + ctx->f[0] = -1; +} + +void blake2b_update(struct blake2b_ctx *ctx, const u8 *in, size_t inlen) +{ + const size_t fill = BLAKE2B_BLOCK_SIZE - ctx->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(ctx->buf + ctx->buflen, in, fill); + blake2b_compress(ctx, ctx->buf, 1, BLAKE2B_BLOCK_SIZE); + ctx->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2B_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2B_BLOCK_SIZE); + + blake2b_compress(ctx, in, nblocks - 1, BLAKE2B_BLOCK_SIZE); + in += BLAKE2B_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2B_BLOCK_SIZE * (nblocks - 1); + } + memcpy(ctx->buf + ctx->buflen, in, inlen); + ctx->buflen += inlen; +} +EXPORT_SYMBOL(blake2b_update); + +void blake2b_final(struct blake2b_ctx *ctx, u8 *out) +{ + WARN_ON(IS_ENABLED(DEBUG) && !out); + blake2b_set_lastblock(ctx); + memset(ctx->buf + ctx->buflen, 0, + BLAKE2B_BLOCK_SIZE - ctx->buflen); /* Padding */ + blake2b_compress(ctx, ctx->buf, 1, ctx->buflen); + cpu_to_le64_array(ctx->h, ARRAY_SIZE(ctx->h)); + memcpy(out, ctx->h, ctx->outlen); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL(blake2b_final); + +#ifdef blake2b_mod_init_arch +static int __init blake2b_mod_init(void) +{ + blake2b_mod_init_arch(); + return 0; +} +subsys_initcall(blake2b_mod_init); + +static void __exit blake2b_mod_exit(void) +{ +} +module_exit(blake2b_mod_exit); +#endif + +MODULE_DESCRIPTION("BLAKE2b hash function"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/fips.h b/lib/crypto/fips.h new file mode 100644 index 0000000000000..78a1bdd33a151 --- /dev/null +++ b/lib/crypto/fips.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: gen-fips-testvecs.py */ + +#include + +static const u8 fips_test_data[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, +}; + +static const u8 fips_test_key[] __initconst __maybe_unused = { + 0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73, + 0x74, 0x20, 0x6b, 0x65, 0x79, 0x00, 0x00, 0x00, +}; + +static const u8 fips_test_hmac_sha1_value[] __initconst __maybe_unused = { + 0x29, 0xa9, 0x88, 0xb8, 0x5c, 0xb4, 0xaf, 0x4b, + 0x97, 0x2a, 0xee, 0x87, 0x5b, 0x0a, 0x02, 0x55, + 0x99, 0xbf, 0x86, 0x78, +}; + +static const u8 fips_test_hmac_sha256_value[] __initconst __maybe_unused = { + 0x59, 0x25, 0x85, 0xcc, 0x40, 0xe9, 0x64, 0x2f, + 0xe9, 0xbf, 0x82, 0xb7, 0xd3, 0x15, 0x3d, 0x43, + 0x22, 0x0b, 0x4c, 0x00, 0x90, 0x14, 0x25, 0xcf, + 0x9e, 0x13, 0x2b, 0xc2, 0x30, 0xe6, 0xe8, 0x93, +}; + +static const u8 fips_test_hmac_sha512_value[] __initconst __maybe_unused = { + 0x6b, 0xea, 0x5d, 0x27, 0x49, 0x5b, 0x3f, 0xea, + 0xde, 0x2d, 0xfa, 0x32, 0x75, 0xdb, 0x77, 0xc8, + 0x26, 0xe9, 0x4e, 0x95, 0x4d, 0xad, 0x88, 0x02, + 0x87, 0xf9, 0x52, 0x0a, 0xd1, 0x92, 0x80, 0x1d, + 0x92, 0x7e, 0x3c, 0xbd, 0xb1, 0x3c, 0x49, 0x98, + 0x44, 0x9c, 0x8f, 0xee, 0x3f, 0x02, 0x71, 0x51, + 0x57, 0x0b, 0x15, 0x38, 0x95, 0xd8, 0xa3, 0x81, + 0xba, 0xb3, 0x15, 0x37, 0x5c, 0x6d, 0x57, 0x2b, +}; diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c index 5904e4ae85d24..59bb5b67cf787 100644 --- a/lib/crypto/sha1.c +++ b/lib/crypto/sha1.c @@ -12,6 +12,7 @@ #include #include #include +#include "fips.h" static const struct sha1_block_state sha1_iv = { .h = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, @@ -163,6 +164,7 @@ void sha1_init(struct sha1_ctx *ctx) { ctx->state = sha1_iv; ctx->bytecount = 0; + ctx->fips_approved = fips_enabled; } EXPORT_SYMBOL_GPL(sha1_init); @@ -268,6 +270,9 @@ void hmac_sha1_preparekey(struct hmac_sha1_key *key, { __hmac_sha1_preparekey(&key->istate, &key->ostate, raw_key, raw_key_len); + key->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + key->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha1_preparekey); @@ -275,6 +280,8 @@ void hmac_sha1_init(struct hmac_sha1_ctx *ctx, const struct hmac_sha1_key *key) { ctx->sha_ctx.state = key->istate; ctx->sha_ctx.bytecount = SHA1_BLOCK_SIZE; + ctx->sha_ctx.fips_approved = key->fips_approved; + ctx->fips_approved = key->fips_approved; ctx->ostate = key->ostate; } EXPORT_SYMBOL_GPL(hmac_sha1_init); @@ -285,6 +292,10 @@ void hmac_sha1_init_usingrawkey(struct hmac_sha1_ctx *ctx, __hmac_sha1_preparekey(&ctx->sha_ctx.state, &ctx->ostate, raw_key, raw_key_len); ctx->sha_ctx.bytecount = SHA1_BLOCK_SIZE; + ctx->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + ctx->fips_approved = false; + ctx->sha_ctx.fips_approved = ctx->fips_approved; } EXPORT_SYMBOL_GPL(hmac_sha1_init_usingrawkey); @@ -330,10 +341,26 @@ void hmac_sha1_usingrawkey(const u8 *raw_key, size_t raw_key_len, } EXPORT_SYMBOL_GPL(hmac_sha1_usingrawkey); -#ifdef sha1_mod_init_arch +#if defined(sha1_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS) static int __init sha1_mod_init(void) { +#ifdef sha1_mod_init_arch sha1_mod_init_arch(); +#endif + if (fips_enabled) { + /* + * FIPS cryptographic algorithm self-test. As per the FIPS + * Implementation Guidance, testing HMAC-SHA1 satisfies the test + * requirement for SHA-1 too. + */ + u8 mac[SHA1_DIGEST_SIZE]; + + hmac_sha1_usingrawkey(fips_test_key, sizeof(fips_test_key), + fips_test_data, sizeof(fips_test_data), + mac); + if (memcmp(fips_test_hmac_sha1_value, mac, sizeof(mac)) != 0) + panic("sha1: FIPS self-test failed\n"); + } return 0; } subsys_initcall(sha1_mod_init); diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 881b935418cea..789dbc36895ee 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -17,6 +17,7 @@ #include #include #include +#include "fips.h" static const struct sha256_block_state sha224_iv = { .h = { @@ -172,12 +173,22 @@ static void __sha256_init(struct __sha256_ctx *ctx, void sha224_init(struct sha224_ctx *ctx) { __sha256_init(&ctx->ctx, &sha224_iv, 0); +#ifndef __DISABLE_EXPORTS + ctx->fips_approved = fips_enabled; +#else + ctx->fips_approved = false; +#endif } EXPORT_SYMBOL_GPL(sha224_init); void sha256_init(struct sha256_ctx *ctx) { __sha256_init(&ctx->ctx, &sha256_iv, 0); +#ifndef __DISABLE_EXPORTS + ctx->fips_approved = fips_enabled; +#else + ctx->fips_approved = false; +#endif } EXPORT_SYMBOL_GPL(sha256_init); @@ -269,8 +280,8 @@ void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) EXPORT_SYMBOL(sha256); /* - * Pre-boot environment (as indicated by __DISABLE_EXPORTS being defined) - * doesn't need either HMAC support or interleaved hashing support + * Pre-boot environments (as indicated by __DISABLE_EXPORTS being defined) just + * need the generic SHA-256 code. Omit all other features from them. */ #ifndef __DISABLE_EXPORTS @@ -362,6 +373,9 @@ void hmac_sha224_preparekey(struct hmac_sha224_key *key, { __hmac_sha256_preparekey(&key->key.istate, &key->key.ostate, raw_key, raw_key_len, &sha224_iv); + key->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + key->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha224_preparekey); @@ -370,6 +384,9 @@ void hmac_sha256_preparekey(struct hmac_sha256_key *key, { __hmac_sha256_preparekey(&key->key.istate, &key->key.ostate, raw_key, raw_key_len, &sha256_iv); + key->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + key->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha256_preparekey); @@ -387,6 +404,9 @@ void hmac_sha224_init_usingrawkey(struct hmac_sha224_ctx *ctx, __hmac_sha256_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate, raw_key, raw_key_len, &sha224_iv); ctx->ctx.sha_ctx.bytecount = SHA256_BLOCK_SIZE; + ctx->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + ctx->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha224_init_usingrawkey); @@ -396,6 +416,9 @@ void hmac_sha256_init_usingrawkey(struct hmac_sha256_ctx *ctx, __hmac_sha256_preparekey(&ctx->ctx.sha_ctx.state, &ctx->ctx.ostate, raw_key, raw_key_len, &sha256_iv); ctx->ctx.sha_ctx.bytecount = SHA256_BLOCK_SIZE; + ctx->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + ctx->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha256_init_usingrawkey); @@ -477,12 +500,27 @@ void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len, hmac_sha256_final(&ctx, out); } EXPORT_SYMBOL_GPL(hmac_sha256_usingrawkey); -#endif /* !__DISABLE_EXPORTS */ -#ifdef sha256_mod_init_arch +#if defined(sha256_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS) static int __init sha256_mod_init(void) { +#ifdef sha256_mod_init_arch sha256_mod_init_arch(); +#endif + if (fips_enabled) { + /* + * FIPS cryptographic algorithm self-test. As per the FIPS + * Implementation Guidance, testing HMAC-SHA256 satisfies the + * test requirement for SHA-224, SHA-256, and HMAC-SHA224 too. + */ + u8 mac[SHA256_DIGEST_SIZE]; + + hmac_sha256_usingrawkey(fips_test_key, sizeof(fips_test_key), + fips_test_data, sizeof(fips_test_data), + mac); + if (memcmp(fips_test_hmac_sha256_value, mac, sizeof(mac)) != 0) + panic("sha256: FIPS self-test failed\n"); + } return 0; } subsys_initcall(sha256_mod_init); @@ -493,5 +531,7 @@ static void __exit sha256_mod_exit(void) module_exit(sha256_mod_exit); #endif +#endif /* !__DISABLE_EXPORTS */ + MODULE_DESCRIPTION("SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c index d8062188be98a..9afc873f37197 100644 --- a/lib/crypto/sha512.c +++ b/lib/crypto/sha512.c @@ -17,6 +17,7 @@ #include #include #include +#include "fips.h" static const struct sha512_block_state sha384_iv = { .h = { @@ -150,12 +151,14 @@ static void __sha512_init(struct __sha512_ctx *ctx, void sha384_init(struct sha384_ctx *ctx) { __sha512_init(&ctx->ctx, &sha384_iv, 0); + ctx->fips_approved = fips_enabled; } EXPORT_SYMBOL_GPL(sha384_init); void sha512_init(struct sha512_ctx *ctx) { __sha512_init(&ctx->ctx, &sha512_iv, 0); + ctx->fips_approved = fips_enabled; } EXPORT_SYMBOL_GPL(sha512_init); @@ -287,6 +290,9 @@ void hmac_sha384_preparekey(struct hmac_sha384_key *key, { __hmac_sha512_preparekey(&key->key.istate, &key->key.ostate, raw_key, raw_key_len, &sha384_iv); + key->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + key->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha384_preparekey); @@ -295,6 +301,9 @@ void hmac_sha512_preparekey(struct hmac_sha512_key *key, { __hmac_sha512_preparekey(&key->key.istate, &key->key.ostate, raw_key, raw_key_len, &sha512_iv); + key->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + key->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha512_preparekey); @@ -313,6 +322,9 @@ void hmac_sha384_init_usingrawkey(struct hmac_sha384_ctx *ctx, raw_key, raw_key_len, &sha384_iv); ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE; ctx->ctx.sha_ctx.bytecount_hi = 0; + ctx->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + ctx->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha384_init_usingrawkey); @@ -323,6 +335,9 @@ void hmac_sha512_init_usingrawkey(struct hmac_sha512_ctx *ctx, raw_key, raw_key_len, &sha512_iv); ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE; ctx->ctx.sha_ctx.bytecount_hi = 0; + ctx->fips_approved = fips_enabled; + if (fips_enabled && raw_key_len < 112 / 8) + ctx->fips_approved = false; } EXPORT_SYMBOL_GPL(hmac_sha512_init_usingrawkey); @@ -405,10 +420,26 @@ void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len, } EXPORT_SYMBOL_GPL(hmac_sha512_usingrawkey); -#ifdef sha512_mod_init_arch +#if defined(sha512_mod_init_arch) || defined(CONFIG_CRYPTO_FIPS) static int __init sha512_mod_init(void) { +#ifdef sha512_mod_init_arch sha512_mod_init_arch(); +#endif + if (fips_enabled) { + /* + * FIPS cryptographic algorithm self-test. As per the FIPS + * Implementation Guidance, testing HMAC-SHA512 satisfies the + * test requirement for SHA-384, SHA-512, and HMAC-SHA384 too. + */ + u8 mac[SHA512_DIGEST_SIZE]; + + hmac_sha512_usingrawkey(fips_test_key, sizeof(fips_test_key), + fips_test_data, sizeof(fips_test_data), + mac); + if (memcmp(fips_test_hmac_sha512_value, mac, sizeof(mac)) != 0) + panic("sha512: FIPS self-test failed\n"); + } return 0; } subsys_initcall(sha512_mod_init); diff --git a/mm/gup.c b/mm/gup.c index 95d948c8e86c9..53732b743722b 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3327,6 +3327,34 @@ int pin_user_pages_fast(unsigned long start, int nr_pages, } EXPORT_SYMBOL_GPL(pin_user_pages_fast); +/** + * pin_user_pages_fast_only() - pin user pages in memory + * @start: starting user address + * @nr_pages: number of pages from start to pin + * @gup_flags: flags modifying pin behaviour + * @pages: array that receives pointers to the pages pinned. + * Should be at least nr_pages long. + * + * Like pin_user_pages_fast() except it's IRQ-safe in that it won't fall back to + * the regular GUP. + * + * If the architecture does not support this function, simply return with no + * pages pinned. + * + * Careful, careful! COW breaking can go either way, so a non-write + * access can get ambiguous page results. If you call this function without + * 'write' set, you'd better be sure that you're ok with that ambiguity. + */ +int pin_user_pages_fast_only(unsigned long start, int nr_pages, + unsigned int gup_flags, struct page **pages) +{ + if (!is_valid_gup_args(pages, NULL, &gup_flags, + FOLL_PIN | FOLL_FAST_ONLY)) + return -EINVAL; + return gup_fast_fallback(start, nr_pages, gup_flags, pages); +} +EXPORT_SYMBOL_GPL(pin_user_pages_fast_only); + /** * pin_user_pages_remote() - pin pages of a remote process * diff --git a/scripts/crypto/gen-fips-testvecs.py b/scripts/crypto/gen-fips-testvecs.py new file mode 100755 index 0000000000000..2956f88b764ae --- /dev/null +++ b/scripts/crypto/gen-fips-testvecs.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Script that generates lib/crypto/fips.h +# +# Copyright 2025 Google LLC + +import hmac + +fips_test_data = b"fips test data\0\0" +fips_test_key = b"fips test key\0\0\0" + +def print_static_u8_array_definition(name, value): + print('') + print(f'static const u8 {name}[] __initconst __maybe_unused = {{') + for i in range(0, len(value), 8): + line = '\t' + ''.join(f'0x{b:02x}, ' for b in value[i:i+8]) + print(f'{line.rstrip()}') + print('};') + +print('/* SPDX-License-Identifier: GPL-2.0-or-later */') +print(f'/* This file was generated by: gen-fips-testvecs.py */') +print() +print('#include ') + +print_static_u8_array_definition("fips_test_data", fips_test_data) +print_static_u8_array_definition("fips_test_key", fips_test_key) + +for alg in 'sha1', 'sha256', 'sha512': + ctx = hmac.new(fips_test_key, digestmod=alg) + ctx.update(fips_test_data) + print_static_u8_array_definition(f'fips_test_hmac_{alg}_value', ctx.digest())