lkml.org 
[lkml]   [2022]   [Jun]   [23]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 4.9 116/264] random: remove batched entropy locking
    Date
    From: "Jason A. Donenfeld" <Jason@zx2c4.com>

    commit 77760fd7f7ae3dfd03668204e708d1568d75447d upstream.

    Rather than use spinlocks to protect batched entropy, we can instead
    disable interrupts locally, since we're dealing with per-cpu data, and
    manage resets with a basic generation counter. At the same time, we
    can't quite do this on PREEMPT_RT, where we still want spinlocks-as-
    mutexes semantics. So we use a local_lock_t, which provides the right
    behavior for each. Because this is a per-cpu lock, that generation
    counter is still doing the necessary CPU-to-CPU communication.

    This should improve performance a bit. It will also fix the linked splat
    that Jonathan received with a PROVE_RAW_LOCK_NESTING=y.

    Reviewed-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
    Reviewed-by: Dominik Brodowski <linux@dominikbrodowski.net>
    Reviewed-by: Eric Biggers <ebiggers@google.com>
    Suggested-by: Andy Lutomirski <luto@kernel.org>
    Reported-by: Jonathan Neuschäfer <j.neuschaefer@gmx.net>
    Tested-by: Jonathan Neuschäfer <j.neuschaefer@gmx.net>
    Link: https://lore.kernel.org/lkml/YfMa0QgsjCVdRAvJ@latitude/
    Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
    Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
    ---
    drivers/char/random.c | 58 +++++++++++++++++++++++---------------------------
    1 file changed, 27 insertions(+), 31 deletions(-)

    --- a/drivers/char/random.c
    +++ b/drivers/char/random.c
    @@ -1770,13 +1770,15 @@ struct ctl_table random_table[] = {
    };
    #endif /* CONFIG_SYSCTL */

    +static atomic_t batch_generation = ATOMIC_INIT(0);
    +
    struct batched_entropy {
    union {
    u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)];
    u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)];
    };
    unsigned int position;
    - spinlock_t batch_lock;
    + int generation;
    };

    /*
    @@ -1787,9 +1789,7 @@ struct batched_entropy {
    * wait_for_random_bytes() should be called and return 0 at least once at any
    * point prior.
    */
    -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
    - .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock),
    -};
    +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64);

    u64 get_random_u64(void)
    {
    @@ -1797,67 +1797,63 @@ u64 get_random_u64(void)
    unsigned long flags;
    struct batched_entropy *batch;
    static void *previous;
    + int next_gen;

    warn_unseeded_randomness(&previous);

    + local_irq_save(flags);
    batch = raw_cpu_ptr(&batched_entropy_u64);
    - spin_lock_irqsave(&batch->batch_lock, flags);
    - if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
    +
    + next_gen = atomic_read(&batch_generation);
    + if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0 ||
    + next_gen != batch->generation) {
    extract_crng((u8 *)batch->entropy_u64);
    batch->position = 0;
    + batch->generation = next_gen;
    }
    +
    ret = batch->entropy_u64[batch->position++];
    - spin_unlock_irqrestore(&batch->batch_lock, flags);
    + local_irq_restore(flags);
    return ret;
    }
    EXPORT_SYMBOL(get_random_u64);

    -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
    - .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock),
    -};
    +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32);
    +
    u32 get_random_u32(void)
    {
    u32 ret;
    unsigned long flags;
    struct batched_entropy *batch;
    static void *previous;
    + int next_gen;

    warn_unseeded_randomness(&previous);

    + local_irq_save(flags);
    batch = raw_cpu_ptr(&batched_entropy_u32);
    - spin_lock_irqsave(&batch->batch_lock, flags);
    - if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
    +
    + next_gen = atomic_read(&batch_generation);
    + if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0 ||
    + next_gen != batch->generation) {
    extract_crng((u8 *)batch->entropy_u32);
    batch->position = 0;
    + batch->generation = next_gen;
    }
    +
    ret = batch->entropy_u32[batch->position++];
    - spin_unlock_irqrestore(&batch->batch_lock, flags);
    + local_irq_restore(flags);
    return ret;
    }
    EXPORT_SYMBOL(get_random_u32);

    /* It's important to invalidate all potential batched entropy that might
    * be stored before the crng is initialized, which we can do lazily by
    - * simply resetting the counter to zero so that it's re-extracted on the
    - * next usage. */
    + * bumping the generation counter.
    + */
    static void invalidate_batched_entropy(void)
    {
    - int cpu;
    - unsigned long flags;
    -
    - for_each_possible_cpu(cpu) {
    - struct batched_entropy *batched_entropy;
    -
    - batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu);
    - spin_lock_irqsave(&batched_entropy->batch_lock, flags);
    - batched_entropy->position = 0;
    - spin_unlock(&batched_entropy->batch_lock);
    -
    - batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu);
    - spin_lock(&batched_entropy->batch_lock);
    - batched_entropy->position = 0;
    - spin_unlock_irqrestore(&batched_entropy->batch_lock, flags);
    - }
    + atomic_inc(&batch_generation);
    }

    /**

    \
     
     \ /
      Last update: 2022-06-23 18:55    [W:4.178 / U:0.160 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site