Messages in this thread Patch in this message | | | Date | Tue, 21 May 2013 22:12:27 +0400 | From | Roman Gushchin <> | Subject | Re: [PATCH v2] rcu: fix a race in hlist_nulls_for_each_entry_rcu macro |
| |
> If other rcu accessors have the same problem, a more complete patch is > needed.
[PATCH] rcu: fix a race in rcu lists traverse macros
Some network functions (udp4_lib_lookup2(), for instance) use the rcu lists traverse macros (hlist_nulls_for_each_entry_rcu, for instance) in a way that assumes restarting of a loop. In this case, it is strictly necessary to reread the head->first value from the memory before each scan. Without additional hints, gcc caches this value in a register. In this case, if a cached node is moved to another chain during the scan, we can loop forever getting wrong nulls values and restarting the loop uninterruptedly.
Signed-off-by: Roman Gushchin <klamm@yandex-team.ru> Reported-by: Boris Zhmurov <zhmurov@yandex-team.ru> --- include/linux/compiler.h | 6 ++++++ include/linux/rculist.h | 6 ++++-- include/linux/rculist_nulls.h | 3 ++- 3 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 92669cd..0e05d7c 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -351,6 +351,12 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); */ #define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
+/* + * Prevent the compiler from optimizing accesses to structure member. + */ +#define GET_STRUCT_FIELD_VOLATILE(struct_ptr, field) \ + (((volatile typeof(*struct_ptr) *)struct_ptr)->field) + /* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */ #ifdef CONFIG_KPROBES # define __kprobes __attribute__((__section__(".kprobes.text"))) diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 8089e35..6c3eea7 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h @@ -282,7 +282,8 @@ static inline void list_splice_init_rcu(struct list_head *list, * as long as the traversal is guarded by rcu_read_lock(). */ #define list_for_each_entry_rcu(pos, head, member) \ - for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \ + for (pos = list_entry_rcu(GET_STRUCT_FIELD_VOLATILE(head, next), \ + typeof(*pos), member); \ &pos->member != (head); \ pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
@@ -348,7 +349,8 @@ static inline void hlist_replace_rcu(struct hlist_node *old, /* * return the first or the next element in an RCU protected hlist */ -#define hlist_first_rcu(head) (*((struct hlist_node __rcu **)(&(head)->first))) +#define hlist_first_rcu(head) (*((struct hlist_node __rcu **) \ + (&GET_STRUCT_FIELD_VOLATILE(head, first)))) #define hlist_next_rcu(node) (*((struct hlist_node __rcu **)(&(node)->next))) #define hlist_pprev_rcu(node) (*((struct hlist_node __rcu **)((node)->pprev)))
diff --git a/include/linux/rculist_nulls.h b/include/linux/rculist_nulls.h index 2ae1371..b04fd0a 100644 --- a/include/linux/rculist_nulls.h +++ b/include/linux/rculist_nulls.h @@ -38,7 +38,8 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n) }
#define hlist_nulls_first_rcu(head) \ - (*((struct hlist_nulls_node __rcu __force **)&(head)->first)) + (*((struct hlist_nulls_node __rcu __force **) \ + &(GET_STRUCT_FIELD_VOLATILE(head, first))))
#define hlist_nulls_next_rcu(node) \ (*((struct hlist_nulls_node __rcu __force **)&(node)->next)) -- 1.8.1.2
| |