lkml.org 
[lkml]   [2019]   [Nov]   [13]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[PATCH 1/5] locking/percpu-rwsem, lockdep: Make percpu-rwsem use its own lockdep_map
    As preparation for replacing the embedded rwsem, give percpu-rwsem its
    own lockdep_map.

    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    ---
    include/linux/percpu-rwsem.h | 29 +++++++++++++++++++----------
    kernel/cpu.c | 4 ++--
    kernel/locking/percpu-rwsem.c | 14 +++++++++++---
    kernel/locking/rwsem.c | 4 ++--
    kernel/locking/rwsem.h | 2 ++
    5 files changed, 36 insertions(+), 17 deletions(-)

    --- a/include/linux/percpu-rwsem.h
    +++ b/include/linux/percpu-rwsem.h
    @@ -15,8 +15,17 @@ struct percpu_rw_semaphore {
    struct rw_semaphore rw_sem; /* slowpath */
    struct rcuwait writer; /* blocked writer */
    int readers_block;
    +#ifdef CONFIG_DEBUG_LOCK_ALLOC
    + struct lockdep_map dep_map;
    +#endif
    };

    +#ifdef CONFIG_DEBUG_LOCK_ALLOC
    +#define __PERCPU_RWSEM_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname },
    +#else
    +#define __PERCPU_RWSEM_DEP_MAP_INIT(lockname)
    +#endif
    +
    #define __DEFINE_PERCPU_RWSEM(name, is_static) \
    static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \
    is_static struct percpu_rw_semaphore name = { \
    @@ -24,7 +33,9 @@ is_static struct percpu_rw_semaphore nam
    .read_count = &__percpu_rwsem_rc_##name, \
    .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \
    .writer = __RCUWAIT_INITIALIZER(name.writer), \
    + __PERCPU_RWSEM_DEP_MAP_INIT(name) \
    }
    +
    #define DEFINE_PERCPU_RWSEM(name) \
    __DEFINE_PERCPU_RWSEM(name, /* not static */)
    #define DEFINE_STATIC_PERCPU_RWSEM(name) \
    @@ -37,7 +48,7 @@ static inline void percpu_down_read(stru
    {
    might_sleep();

    - rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_);
    + rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);

    preempt_disable();
    /*
    @@ -76,13 +87,15 @@ static inline int percpu_down_read_trylo
    */

    if (ret)
    - rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_);
    + rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);

    return ret;
    }

    static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
    {
    + rwsem_release(&sem->dep_map, _RET_IP_);
    +
    preempt_disable();
    /*
    * Same as in percpu_down_read().
    @@ -92,8 +105,6 @@ static inline void percpu_up_read(struct
    else
    __percpu_up_read(sem); /* Unconditional memory barrier */
    preempt_enable();
    -
    - rwsem_release(&sem->rw_sem.dep_map, _RET_IP_);
    }

    extern void percpu_down_write(struct percpu_rw_semaphore *);
    @@ -110,15 +121,13 @@ extern void percpu_free_rwsem(struct per
    __percpu_init_rwsem(sem, #sem, &rwsem_key); \
    })

    -#define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem)
    -
    -#define percpu_rwsem_assert_held(sem) \
    - lockdep_assert_held(&(sem)->rw_sem)
    +#define percpu_rwsem_is_held(sem) lockdep_is_held(sem)
    +#define percpu_rwsem_assert_held(sem) lockdep_assert_held(sem)

    static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem,
    bool read, unsigned long ip)
    {
    - lock_release(&sem->rw_sem.dep_map, ip);
    + lock_release(&sem->dep_map, ip);
    #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
    if (!read)
    atomic_long_set(&sem->rw_sem.owner, RWSEM_OWNER_UNKNOWN);
    @@ -128,7 +137,7 @@ static inline void percpu_rwsem_release(
    static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem,
    bool read, unsigned long ip)
    {
    - lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip);
    + lock_acquire(&sem->dep_map, 0, 1, read, 1, NULL, ip);
    #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
    if (!read)
    atomic_long_set(&sem->rw_sem.owner, (long)current);
    --- a/kernel/cpu.c
    +++ b/kernel/cpu.c
    @@ -331,12 +331,12 @@ void lockdep_assert_cpus_held(void)

    static void lockdep_acquire_cpus_lock(void)
    {
    - rwsem_acquire(&cpu_hotplug_lock.rw_sem.dep_map, 0, 0, _THIS_IP_);
    + rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
    }

    static void lockdep_release_cpus_lock(void)
    {
    - rwsem_release(&cpu_hotplug_lock.rw_sem.dep_map, _THIS_IP_);
    + rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
    }

    /*
    --- a/kernel/locking/percpu-rwsem.c
    +++ b/kernel/locking/percpu-rwsem.c
    @@ -11,7 +11,7 @@
    #include "rwsem.h"

    int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
    - const char *name, struct lock_class_key *rwsem_key)
    + const char *name, struct lock_class_key *key)
    {
    sem->read_count = alloc_percpu(int);
    if (unlikely(!sem->read_count))
    @@ -22,6 +22,10 @@ int __percpu_init_rwsem(struct percpu_rw
    __init_rwsem(&sem->rw_sem, name, rwsem_key);
    rcuwait_init(&sem->writer);
    sem->readers_block = 0;
    +#ifdef CONFIG_DEBUG_LOCK_ALLOC
    + debug_check_no_locks_freed((void *)sem, sizeof(*sem));
    + lockdep_init_map(&sem->dep_map, name, key, 0);
    +#endif
    return 0;
    }
    EXPORT_SYMBOL_GPL(__percpu_init_rwsem);
    @@ -142,10 +146,12 @@ static bool readers_active_check(struct

    void percpu_down_write(struct percpu_rw_semaphore *sem)
    {
    + rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
    +
    /* Notify readers to take the slow path. */
    rcu_sync_enter(&sem->rss);

    - down_write(&sem->rw_sem);
    + __down_write(&sem->rw_sem);

    /*
    * Notify new readers to block; up until now, and thus throughout the
    @@ -168,6 +174,8 @@ EXPORT_SYMBOL_GPL(percpu_down_write);

    void percpu_up_write(struct percpu_rw_semaphore *sem)
    {
    + rwsem_release(&sem->dep_map, _RET_IP_);
    +
    /*
    * Signal the writer is done, no fast path yet.
    *
    @@ -183,7 +191,7 @@ void percpu_up_write(struct percpu_rw_se
    /*
    * Release the write lock, this will allow readers back in the game.
    */
    - up_write(&sem->rw_sem);
    + __up_write(&sem->rw_sem);

    /*
    * Once this completes (at least one RCU-sched grace period hence) the
    --- a/kernel/locking/rwsem.c
    +++ b/kernel/locking/rwsem.c
    @@ -1383,7 +1383,7 @@ static inline int __down_read_trylock(st
    /*
    * lock for writing
    */
    -static inline void __down_write(struct rw_semaphore *sem)
    +inline void __down_write(struct rw_semaphore *sem)
    {
    long tmp = RWSEM_UNLOCKED_VALUE;

    @@ -1446,7 +1446,7 @@ inline void __up_read(struct rw_semaphor
    /*
    * unlock after writing
    */
    -static inline void __up_write(struct rw_semaphore *sem)
    +inline void __up_write(struct rw_semaphore *sem)
    {
    long tmp;

    --- a/kernel/locking/rwsem.h
    +++ b/kernel/locking/rwsem.h
    @@ -6,5 +6,7 @@

    extern void __down_read(struct rw_semaphore *sem);
    extern void __up_read(struct rw_semaphore *sem);
    +extern void __down_write(struct rw_semaphore *sem);
    +extern void __up_write(struct rw_semaphore *sem);

    #endif /* __INTERNAL_RWSEM_H */

    \
     
     \ /
      Last update: 2019-11-13 11:30    [W:4.065 / U:0.016 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site