lkml.org 
[lkml]   [2010]   [Feb]   [24]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 02/10] rcu: annotated list rcu code
    Date
    The listrcu implementation now defines new rcu_list_head,
    rcu_hlist_head and rcu_hlist_entry structures that are
    annotated with __rcu. Only these can now be passed into
    rcu_list_for_each and related interfaces.

    When not running sparse, the types are defined to the
    original list_head etc in order to not break working
    setups that are lacking annotation.

    Signed-off-by: Arnd Bergmann <arnd@arndb.de>
    ---
    include/linux/rculist.h | 152 ++++++++++++++++++++++++++++++++++------------
    1 files changed, 112 insertions(+), 40 deletions(-)

    diff --git a/include/linux/rculist.h b/include/linux/rculist.h
    index 1bf0f70..dfbc6ea 100644
    --- a/include/linux/rculist.h
    +++ b/include/linux/rculist.h
    @@ -9,16 +9,67 @@
    #include <linux/list.h>
    #include <linux/rcupdate.h>

    +#ifdef __CHECKER__
    +struct rcu_list_head {
    + struct rcu_list_head __rcu *next;
    + struct rcu_list_head *prev;
    +};
    +#define LIST_HEAD_INIT_RCU(name) { (struct rcu_list_head __force __rcu *)&(name), &(name) }
    +
    +#define LIST_HEAD_RCU(name) \
    + struct rcu_list_head name = LIST_HEAD_INIT_RCU(name)
    +
    +static inline void INIT_LIST_HEAD_RCU(struct rcu_list_head *list)
    +{
    + __rcu_assign_pointer(list->next, list);
    + list->prev = list;
    +}
    +
    +struct rcu_hlist_head {
    + struct rcu_hlist_node __rcu *first;
    +};
    +
    +struct rcu_hlist_node {
    + struct rcu_hlist_node __rcu *next, **pprev;
    +};
    +
    +#define HLIST_HEAD_INIT_RCU { .first = (void __rcu __force *)NULL }
    +#define HLIST_HEAD_RCU(name) struct rcu_hlist_head name = \
    + { .first = (void __rcu __force *)NULL }
    +#define INIT_HLIST_HEAD_RCU(ptr) ((ptr)->first = (void __rcu __force *)NULL)
    +static inline void INIT_HLIST_NODE_RCU(struct rcu_hlist_node *h)
    +{
    + __rcu_assign_pointer(h->next, NULL);
    + h->pprev = NULL;
    +}
    +
    +#else /* !__CHECKER__ */
    +
    +#define rcu_list_head list_head
    +#define LIST_HEAD_INIT_RCU LIST_HEAD_INIT
    +#define LIST_HEAD_RCU LIST_HEAD
    +#define INIT_LIST_HEAD_RCU INIT_LIST_HEAD
    +
    +#define rcu_hlist_head hlist_head
    +#define rcu_hlist_node hlist_node
    +#define HLIST_HEAD_INIT_RCU HLIST_HEAD_INIT
    +#define HLIST_HEAD_RCU HLIST_HEAD
    +#define INIT_HLIST_HEAD_RCU INIT_HLIST_HEAD
    +#define INIT_HLIST_NODE_RCU INIT_HLIST_NODE
    +
    +#endif /* !__CHECKER__ */
    +
    +
    /*
    * Insert a new entry between two known consecutive entries.
    *
    * This is only for internal list manipulation where we know
    * the prev/next entries already!
    */
    -static inline void __list_add_rcu(struct list_head *new,
    - struct list_head *prev, struct list_head *next)
    +static inline void __list_add_rcu(struct rcu_list_head *new,
    + struct rcu_list_head *prev, struct rcu_list_head *next)
    {
    - new->next = next;
    + __rcu_assign_pointer(new->next, next);
    new->prev = prev;
    rcu_assign_pointer(prev->next, new);
    next->prev = new;
    @@ -40,9 +91,9 @@ static inline void __list_add_rcu(struct list_head *new,
    * the _rcu list-traversal primitives, such as
    * list_for_each_entry_rcu().
    */
    -static inline void list_add_rcu(struct list_head *new, struct list_head *head)
    +static inline void list_add_rcu(struct rcu_list_head *new, struct rcu_list_head *head)
    {
    - __list_add_rcu(new, head, head->next);
    + __list_add_rcu(new, head, __rcu_dereference(head->next));
    }

    /**
    @@ -61,8 +112,8 @@ static inline void list_add_rcu(struct list_head *new, struct list_head *head)
    * the _rcu list-traversal primitives, such as
    * list_for_each_entry_rcu().
    */
    -static inline void list_add_tail_rcu(struct list_head *new,
    - struct list_head *head)
    +static inline void list_add_tail_rcu(struct rcu_list_head *new,
    + struct rcu_list_head *head)
    {
    __list_add_rcu(new, head->prev, head);
    }
    @@ -91,13 +142,29 @@ static inline void list_add_tail_rcu(struct list_head *new,
    * or call_rcu() must be used to defer freeing until an RCU
    * grace period has elapsed.
    */
    -static inline void list_del_rcu(struct list_head *entry)
    +static inline void __list_del_rcu(struct rcu_list_head *prev, struct rcu_list_head *next)
    {
    - __list_del(entry->prev, entry->next);
    + next->prev = prev;
    + __rcu_assign_pointer(prev->next, next);
    +}
    +
    +static inline void list_del_rcu(struct rcu_list_head *entry)
    +{
    + __list_del_rcu(entry->prev, __rcu_dereference(entry->next));
    entry->prev = LIST_POISON2;
    }

    /**
    + * list_empty - tests whether a list is empty
    + * @head: the list to test.
    + */
    +static inline int list_empty_rcu(const struct rcu_list_head *head)
    +{
    + return rcu_dereference(head->next) == head;
    +}
    +
    +
    +/**
    * hlist_del_init_rcu - deletes entry from hash list with re-initialization
    * @n: the element to delete from the hash list.
    *
    @@ -117,7 +184,7 @@ static inline void list_del_rcu(struct list_head *entry)
    * perfectly legal to run concurrently with the _rcu list-traversal
    * primitives, such as hlist_for_each_entry_rcu().
    */
    -static inline void hlist_del_init_rcu(struct hlist_node *n)
    +static inline void hlist_del_init_rcu(struct rcu_hlist_node *n)
    {
    if (!hlist_unhashed(n)) {
    __hlist_del(n);
    @@ -133,13 +200,13 @@ static inline void hlist_del_init_rcu(struct hlist_node *n)
    * The @old entry will be replaced with the @new entry atomically.
    * Note: @old should not be empty.
    */
    -static inline void list_replace_rcu(struct list_head *old,
    - struct list_head *new)
    +static inline void list_replace_rcu(struct rcu_list_head *old,
    + struct rcu_list_head *new)
    {
    new->next = old->next;
    new->prev = old->prev;
    - rcu_assign_pointer(new->prev->next, new);
    - new->next->prev = new;
    + __rcu_assign_pointer(new->prev->next, new);
    + rcu_dereference(new->next)->prev = new;
    old->prev = LIST_POISON2;
    }

    @@ -160,13 +227,13 @@ static inline void list_replace_rcu(struct list_head *old,
    * based on call_rcu() could be created. But only if -really-
    * needed -- there is no shortage of RCU API members.
    */
    -static inline void list_splice_init_rcu(struct list_head *list,
    - struct list_head *head,
    +static inline void list_splice_init_rcu(struct rcu_list_head *list,
    + struct rcu_list_head *head,
    void (*sync)(void))
    {
    - struct list_head *first = list->next;
    - struct list_head *last = list->prev;
    - struct list_head *at = head->next;
    + struct rcu_list_head *first = __rcu_dereference(list->next);
    + struct rcu_list_head *last = list->prev;
    + struct rcu_list_head *at = __rcu_dereference(head->next);

    if (list_empty(head))
    return;
    @@ -192,7 +259,7 @@ static inline void list_splice_init_rcu(struct list_head *list,
    * this function.
    */

    - last->next = at;
    + __rcu_assign_pointer(last->next,at);
    rcu_assign_pointer(head->next, first);
    first->prev = head;
    at->prev = last;
    @@ -200,7 +267,7 @@ static inline void list_splice_init_rcu(struct list_head *list,

    /**
    * list_entry_rcu - get the struct for this entry
    - * @ptr: the &struct list_head pointer.
    + * @ptr: the &struct rcu_list_head pointer.
    * @type: the type of the struct this is embedded in.
    * @member: the name of the list_struct within the struct.
    *
    @@ -241,13 +308,13 @@ static inline void list_splice_init_rcu(struct list_head *list,
    */
    #define list_for_each_entry_rcu(pos, head, member) \
    for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
    - prefetch(pos->member.next), &pos->member != (head); \
    + prefetch(__rcu_dereference(pos->member.next)), &pos->member != (head); \
    pos = list_entry_rcu(pos->member.next, typeof(*pos), member))


    /**
    * list_for_each_continue_rcu
    - * @pos: the &struct list_head to use as a loop cursor.
    + * @pos: the &struct rcu_list_head to use as a loop cursor.
    * @head: the head for your list.
    *
    * Iterate over an rcu-protected list, continuing after current point.
    @@ -294,9 +361,9 @@ static inline void list_splice_init_rcu(struct list_head *list,
    * the _rcu list-traversal primitives, such as
    * hlist_for_each_entry().
    */
    -static inline void hlist_del_rcu(struct hlist_node *n)
    +static inline void hlist_del_rcu(struct rcu_hlist_node *n)
    {
    - __hlist_del(n);
    + __hlist_del((struct hlist_node *)n);
    n->pprev = LIST_POISON2;
    }

    @@ -307,16 +374,16 @@ static inline void hlist_del_rcu(struct hlist_node *n)
    *
    * The @old entry will be replaced with the @new entry atomically.
    */
    -static inline void hlist_replace_rcu(struct hlist_node *old,
    - struct hlist_node *new)
    +static inline void hlist_replace_rcu(struct rcu_hlist_node *old,
    + struct rcu_hlist_node *new)
    {
    - struct hlist_node *next = old->next;
    + struct rcu_hlist_node __rcu *next = old->next;

    new->next = next;
    new->pprev = old->pprev;
    rcu_assign_pointer(*new->pprev, new);
    if (next)
    - new->next->pprev = &new->next;
    + __rcu_dereference(new->next)->pprev = &new->next;
    old->pprev = LIST_POISON2;
    }

    @@ -339,12 +406,12 @@ static inline void hlist_replace_rcu(struct hlist_node *old,
    * problems on Alpha CPUs. Regardless of the type of CPU, the
    * list-traversal primitive must be guarded by rcu_read_lock().
    */
    -static inline void hlist_add_head_rcu(struct hlist_node *n,
    - struct hlist_head *h)
    +static inline void hlist_add_head_rcu(struct rcu_hlist_node *n,
    + struct rcu_hlist_head *h)
    {
    - struct hlist_node *first = h->first;
    + struct rcu_hlist_node *first = __rcu_dereference(h->first);

    - n->next = first;
    + __rcu_assign_pointer(n->next, first);
    n->pprev = &h->first;
    rcu_assign_pointer(h->first, n);
    if (first)
    @@ -369,8 +436,8 @@ static inline void hlist_add_head_rcu(struct hlist_node *n,
    * hlist_for_each_entry_rcu(), used to prevent memory-consistency
    * problems on Alpha CPUs.
    */
    -static inline void hlist_add_before_rcu(struct hlist_node *n,
    - struct hlist_node *next)
    +static inline void hlist_add_before_rcu(struct rcu_hlist_node *n,
    + struct rcu_hlist_node *next)
    {
    n->pprev = next->pprev;
    n->next = next;
    @@ -396,8 +463,8 @@ static inline void hlist_add_before_rcu(struct hlist_node *n,
    * hlist_for_each_entry_rcu(), used to prevent memory-consistency
    * problems on Alpha CPUs.
    */
    -static inline void hlist_add_after_rcu(struct hlist_node *prev,
    - struct hlist_node *n)
    +static inline void hlist_add_after_rcu(struct rcu_hlist_node *prev,
    + struct rcu_hlist_node *n)
    {
    n->next = prev->next;
    n->pprev = &prev->next;
    @@ -406,12 +473,17 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
    n->next->pprev = &n->next;
    }

    +static inline int hlist_empty_rcu(const struct rcu_hlist_head *h)
    +{
    + return !__rcu_dereference(h->first);
    +}
    +
    /**
    * hlist_for_each_entry_rcu - iterate over rcu list of given type
    * @tpos: the type * to use as a loop cursor.
    - * @pos: the &struct hlist_node to use as a loop cursor.
    + * @pos: the &struct rcu_hlist_node to use as a loop cursor.
    * @head: the head for your list.
    - * @member: the name of the hlist_node within the struct.
    + * @member: the name of the rcu_hlist_node within the struct.
    *
    * This list-traversal primitive may safely run concurrently with
    * the _rcu list-mutation primitives such as hlist_add_head_rcu()
    @@ -419,7 +491,7 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
    */
    #define hlist_for_each_entry_rcu(tpos, pos, head, member) \
    for (pos = rcu_dereference((head)->first); \
    - pos && ({ prefetch(pos->next); 1; }) && \
    + pos && ({ prefetch(__rcu_dereference(pos->next)); 1; }) && \
    ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \
    pos = rcu_dereference(pos->next))

    --
    1.6.3.3


    \
     
     \ /
      Last update: 2010-02-24 21:07    [W:5.063 / U:0.268 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site