lkml.org 
[lkml]   [2019]   [Aug]   [1]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH tip/core/rcu 3/8] rcu: Add support for consolidated-RCU reader checking
    Date
    From: "Joel Fernandes (Google)" <joel@joelfernandes.org>

    This commit adds RCU-reader checks to list_for_each_entry_rcu() and
    hlist_for_each_entry_rcu(). These checks are optional, and are indicated
    by a lockdep expression passed to a new optional argument to these two
    macros. If this optional lockdep expression is omitted, these two macros
    act as before, checking for an RCU read-side critical section.

    Signed-off-by: Joel Fernandes (Google) <joel@joelfernandes.org>
    [ paulmck: Update to eliminate return within macro. ]
    Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
    ---
    include/linux/rculist.h | 32 +++++++++++++++++----
    include/linux/rcupdate.h | 7 +++++
    kernel/rcu/Kconfig.debug | 11 ++++++++
    kernel/rcu/update.c | 61 ++++++++++++++++++++++++++++------------
    4 files changed, 88 insertions(+), 23 deletions(-)

    diff --git a/include/linux/rculist.h b/include/linux/rculist.h
    index 932296144131..4158b7212936 100644
    --- a/include/linux/rculist.h
    +++ b/include/linux/rculist.h
    @@ -40,6 +40,24 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
    */
    #define list_next_rcu(list) (*((struct list_head __rcu **)(&(list)->next)))

    +/*
    + * Check during list traversal that we are within an RCU reader
    + */
    +
    +#define check_arg_count_one(dummy)
    +
    +#ifdef CONFIG_PROVE_RCU_LIST
    +#define __list_check_rcu(dummy, cond, extra...) \
    + ({ \
    + check_arg_count_one(extra); \
    + RCU_LOCKDEP_WARN(!cond && !rcu_read_lock_any_held(), \
    + "RCU-list traversed in non-reader section!"); \
    + })
    +#else
    +#define __list_check_rcu(dummy, cond, extra...) \
    + ({ check_arg_count_one(extra); })
    +#endif
    +
    /*
    * Insert a new entry between two known consecutive entries.
    *
    @@ -343,14 +361,16 @@ static inline void list_splice_tail_init_rcu(struct list_head *list,
    * @pos: the type * to use as a loop cursor.
    * @head: the head for your list.
    * @member: the name of the list_head within the struct.
    + * @cond: optional lockdep expression if called from non-RCU protection.
    *
    * This list-traversal primitive may safely run concurrently with
    * the _rcu list-mutation primitives such as list_add_rcu()
    * as long as the traversal is guarded by rcu_read_lock().
    */
    -#define list_for_each_entry_rcu(pos, head, member) \
    - for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
    - &pos->member != (head); \
    +#define list_for_each_entry_rcu(pos, head, member, cond...) \
    + for (__list_check_rcu(dummy, ## cond, 0), \
    + pos = list_entry_rcu((head)->next, typeof(*pos), member); \
    + &pos->member != (head); \
    pos = list_entry_rcu(pos->member.next, typeof(*pos), member))

    /**
    @@ -616,13 +636,15 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
    * @pos: the type * to use as a loop cursor.
    * @head: the head for your list.
    * @member: the name of the hlist_node within the struct.
    + * @cond: optional lockdep expression if called from non-RCU protection.
    *
    * This list-traversal primitive may safely run concurrently with
    * the _rcu list-mutation primitives such as hlist_add_head_rcu()
    * as long as the traversal is guarded by rcu_read_lock().
    */
    -#define hlist_for_each_entry_rcu(pos, head, member) \
    - for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
    +#define hlist_for_each_entry_rcu(pos, head, member, cond...) \
    + for (__list_check_rcu(dummy, ## cond, 0), \
    + pos = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)),\
    typeof(*(pos)), member); \
    pos; \
    pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
    diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
    index bfcafbc1e301..80d6056f5855 100644
    --- a/include/linux/rcupdate.h
    +++ b/include/linux/rcupdate.h
    @@ -221,6 +221,7 @@ int debug_lockdep_rcu_enabled(void);
    int rcu_read_lock_held(void);
    int rcu_read_lock_bh_held(void);
    int rcu_read_lock_sched_held(void);
    +int rcu_read_lock_any_held(void);

    #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

    @@ -241,6 +242,12 @@ static inline int rcu_read_lock_sched_held(void)
    {
    return !preemptible();
    }
    +
    +static inline int rcu_read_lock_any_held(void)
    +{
    + return !preemptible();
    +}
    +
    #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */

    #ifdef CONFIG_PROVE_RCU
    diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug
    index 5ec3ea4028e2..4aa02eee8f6c 100644
    --- a/kernel/rcu/Kconfig.debug
    +++ b/kernel/rcu/Kconfig.debug
    @@ -8,6 +8,17 @@ menu "RCU Debugging"
    config PROVE_RCU
    def_bool PROVE_LOCKING

    +config PROVE_RCU_LIST
    + bool "RCU list lockdep debugging"
    + depends on PROVE_RCU && RCU_EXPERT
    + default n
    + help
    + Enable RCU lockdep checking for list usages. By default it is
    + turned off since there are several list RCU users that still
    + need to be converted to pass a lockdep expression. To prevent
    + false-positive splats, we keep it default disabled but once all
    + users are converted, we can remove this config option.
    +
    config TORTURE_TEST
    tristate
    default n
    diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
    index 9dd5aeef6e70..aaa08d62ceb6 100644
    --- a/kernel/rcu/update.c
    +++ b/kernel/rcu/update.c
    @@ -91,14 +91,29 @@ module_param(rcu_normal_after_boot, int, 0);
    * Similarly, we avoid claiming an SRCU read lock held if the current
    * CPU is offline.
    */
    +static bool rcu_read_lock_held_common(bool *ret)
    +{
    + if (!debug_lockdep_rcu_enabled()) {
    + *ret = 1;
    + return true;
    + }
    + if (!rcu_is_watching()) {
    + *ret = 0;
    + return true;
    + }
    + if (!rcu_lockdep_current_cpu_online()) {
    + *ret = 0;
    + return true;
    + }
    + return false;
    +}
    +
    int rcu_read_lock_sched_held(void)
    {
    - if (!debug_lockdep_rcu_enabled())
    - return 1;
    - if (!rcu_is_watching())
    - return 0;
    - if (!rcu_lockdep_current_cpu_online())
    - return 0;
    + bool ret;
    +
    + if (rcu_read_lock_held_common(&ret))
    + return ret;
    return lock_is_held(&rcu_sched_lock_map) || !preemptible();
    }
    EXPORT_SYMBOL(rcu_read_lock_sched_held);
    @@ -257,12 +272,10 @@ NOKPROBE_SYMBOL(debug_lockdep_rcu_enabled);
    */
    int rcu_read_lock_held(void)
    {
    - if (!debug_lockdep_rcu_enabled())
    - return 1;
    - if (!rcu_is_watching())
    - return 0;
    - if (!rcu_lockdep_current_cpu_online())
    - return 0;
    + bool ret;
    +
    + if (rcu_read_lock_held_common(&ret))
    + return ret;
    return lock_is_held(&rcu_lock_map);
    }
    EXPORT_SYMBOL_GPL(rcu_read_lock_held);
    @@ -284,16 +297,28 @@ EXPORT_SYMBOL_GPL(rcu_read_lock_held);
    */
    int rcu_read_lock_bh_held(void)
    {
    - if (!debug_lockdep_rcu_enabled())
    - return 1;
    - if (!rcu_is_watching())
    - return 0;
    - if (!rcu_lockdep_current_cpu_online())
    - return 0;
    + bool ret;
    +
    + if (rcu_read_lock_held_common(&ret))
    + return ret;
    return in_softirq() || irqs_disabled();
    }
    EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);

    +int rcu_read_lock_any_held(void)
    +{
    + bool ret;
    +
    + if (rcu_read_lock_held_common(&ret))
    + return ret;
    + if (lock_is_held(&rcu_lock_map) ||
    + lock_is_held(&rcu_bh_lock_map) ||
    + lock_is_held(&rcu_sched_lock_map))
    + return 1;
    + return !preemptible();
    +}
    +EXPORT_SYMBOL_GPL(rcu_read_lock_any_held);
    +
    #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

    /**
    --
    2.17.1
    \
     
     \ /
      Last update: 2019-08-02 00:45    [W:6.402 / U:0.048 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site