lkml.org 
[lkml]   [2022]   [Oct]   [6]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[RFC PATCH 6/7] crypto: x86/ghash - limit FPU preemption
    Date
    As done by the ECB and CBC helpers in arch/x86/crypt/ecb_cbc_helpers.h,
    limit the number of bytes processed between kernel_fpu_begin() and
    kernel_fpu_end() calls.

    Those functions call preempt_disable() and preempt_enable(), so
    the CPU core is unavailable for scheduling while running, leading to:
    rcu: INFO: rcu_preempt detected expedited stalls on CPUs/tasks: {12-... } 22 jiffies s: 277 root: 0x1/.

    Fixes: 0e1227d356e9 ("crypto: ghash - Add PCLMULQDQ accelerated implementation")
    Suggested-by: Herbert Xu <herbert@gondor.apana.org.au>
    Signed-off-by: Robert Elliott <elliott@hpe.com>
    ---
    arch/x86/crypto/ghash-clmulni-intel_glue.c | 27 ++++++++++++++++------
    1 file changed, 20 insertions(+), 7 deletions(-)

    diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c
    index b25730c5b267..b9a3e2187f5b 100644
    --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c
    +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c
    @@ -25,6 +25,8 @@
    #define GHASH_BLOCK_SIZE 16
    #define GHASH_DIGEST_SIZE 16

    +#define FPU_BYTES 4096U /* avoid kernel_fpu_begin/end scheduler/rcu stalls */
    +
    void clmul_ghash_mul(char *dst, const u128 *shash);

    void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
    @@ -81,10 +83,11 @@ static int ghash_update(struct shash_desc *desc,
    struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
    struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
    u8 *dst = dctx->buffer;
    + unsigned int fpulen;

    if (dctx->bytes) {
    int n = min(srclen, dctx->bytes);
    - u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
    + u8 *pos = dst + GHASH_BLOCK_SIZE - dctx->bytes;

    dctx->bytes -= n;
    srclen -= n;
    @@ -99,13 +102,23 @@ static int ghash_update(struct shash_desc *desc,
    }
    }

    - kernel_fpu_begin();
    - clmul_ghash_update(dst, src, srclen, &ctx->shash);
    - kernel_fpu_end();
    + while (srclen >= GHASH_BLOCK_SIZE) {
    + fpulen = min(srclen, FPU_BYTES);
    +
    + kernel_fpu_begin();
    + while (fpulen >= GHASH_BLOCK_SIZE) {
    + int n = min_t(unsigned int, fpulen, GHASH_BLOCK_SIZE);
    +
    + clmul_ghash_update(dst, src, n, &ctx->shash);
    +
    + srclen -= n;
    + fpulen -= n;
    + src += n;
    + }
    + kernel_fpu_end();
    + }

    - if (srclen & 0xf) {
    - src += srclen - (srclen & 0xf);
    - srclen &= 0xf;
    + if (srclen) {
    dctx->bytes = GHASH_BLOCK_SIZE - srclen;
    while (srclen--)
    *dst++ ^= *src++;
    --
    2.37.3
    \
     
     \ /
      Last update: 2022-10-07 00:33    [W:7.270 / U:0.036 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site