1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_RCULIST_H
3 #define _LINUX_RCULIST_H
4 
5 #ifdef __KERNEL__
6 
7 /*
8  * RCU-protected list version
9  */
10 #include <linux/list.h>
11 #include <linux/rcupdate.h>
12 
13 /*
14  * Why is there no list_empty_rcu()?  Because list_empty() serves this
15  * purpose.  The list_empty() function fetches the RCU-protected pointer
16  * and compares it to the address of the list head, but neither dereferences
17  * this pointer itself nor provides this pointer to the caller.  Therefore,
18  * it is not necessary to use rcu_dereference(), so that list_empty() can
19  * be used anywhere you would want to use a list_empty_rcu().
20  */
21 
22 /*
23  * INIT_LIST_HEAD_RCU - Initialize a list_head visible to RCU readers
24  * @list: list to be initialized
25  *
26  * You should instead use INIT_LIST_HEAD() for normal initialization and
27  * cleanup tasks, when readers have no access to the list being initialized.
28  * However, if the list being initialized is visible to readers, you
29  * need to keep the compiler from being too mischievous.
30  */
INIT_LIST_HEAD_RCU(struct list_head * list)31 static inline void INIT_LIST_HEAD_RCU(struct list_head *list)
32 {
33 	WRITE_ONCE(list->next, list);
34 	WRITE_ONCE(list->prev, list);
35 }
36 
37 /*
38  * return the ->next pointer of a list_head in an rcu safe
39  * way, we must not access it directly
40  */
41 #define list_next_rcu(list)	(*((struct list_head __rcu **)(&(list)->next)))
42 
43 /*
44  * Insert a new entry between two known consecutive entries.
45  *
46  * This is only for internal list manipulation where we know
47  * the prev/next entries already!
48  */
__list_add_rcu(struct list_head * new,struct list_head * prev,struct list_head * next)49 static inline void __list_add_rcu(struct list_head *new,
50 		struct list_head *prev, struct list_head *next)
51 {
52 	if (!__list_add_valid(new, prev, next))
53 		return;
54 
55 	new->next = next;
56 	new->prev = prev;
57 	rcu_assign_pointer(list_next_rcu(prev), new);
58 	next->prev = new;
59 }
60 
61 /**
62  * list_add_rcu - add a new entry to rcu-protected list
63  * @new: new entry to be added
64  * @head: list head to add it after
65  *
66  * Insert a new entry after the specified head.
67  * This is good for implementing stacks.
68  *
69  * The caller must take whatever precautions are necessary
70  * (such as holding appropriate locks) to avoid racing
71  * with another list-mutation primitive, such as list_add_rcu()
72  * or list_del_rcu(), running on this same list.
73  * However, it is perfectly legal to run concurrently with
74  * the _rcu list-traversal primitives, such as
75  * list_for_each_entry_rcu().
76  */
list_add_rcu(struct list_head * new,struct list_head * head)77 static inline void list_add_rcu(struct list_head *new, struct list_head *head)
78 {
79 	__list_add_rcu(new, head, head->next);
80 }
81 
82 /**
83  * list_add_tail_rcu - add a new entry to rcu-protected list
84  * @new: new entry to be added
85  * @head: list head to add it before
86  *
87  * Insert a new entry before the specified head.
88  * This is useful for implementing queues.
89  *
90  * The caller must take whatever precautions are necessary
91  * (such as holding appropriate locks) to avoid racing
92  * with another list-mutation primitive, such as list_add_tail_rcu()
93  * or list_del_rcu(), running on this same list.
94  * However, it is perfectly legal to run concurrently with
95  * the _rcu list-traversal primitives, such as
96  * list_for_each_entry_rcu().
97  */
list_add_tail_rcu(struct list_head * new,struct list_head * head)98 static inline void list_add_tail_rcu(struct list_head *new,
99 					struct list_head *head)
100 {
101 	__list_add_rcu(new, head->prev, head);
102 }
103 
104 /**
105  * list_del_rcu - deletes entry from list without re-initialization
106  * @entry: the element to delete from the list.
107  *
108  * Note: list_empty() on entry does not return true after this,
109  * the entry is in an undefined state. It is useful for RCU based
110  * lockfree traversal.
111  *
112  * In particular, it means that we can not poison the forward
113  * pointers that may still be used for walking the list.
114  *
115  * The caller must take whatever precautions are necessary
116  * (such as holding appropriate locks) to avoid racing
117  * with another list-mutation primitive, such as list_del_rcu()
118  * or list_add_rcu(), running on this same list.
119  * However, it is perfectly legal to run concurrently with
120  * the _rcu list-traversal primitives, such as
121  * list_for_each_entry_rcu().
122  *
123  * Note that the caller is not permitted to immediately free
124  * the newly deleted entry.  Instead, either synchronize_rcu()
125  * or call_rcu() must be used to defer freeing until an RCU
126  * grace period has elapsed.
127  */
list_del_rcu(struct list_head * entry)128 static inline void list_del_rcu(struct list_head *entry)
129 {
130 	__list_del_entry(entry);
131 	entry->prev = LIST_POISON2;
132 }
133 
134 /**
135  * hlist_del_init_rcu - deletes entry from hash list with re-initialization
136  * @n: the element to delete from the hash list.
137  *
138  * Note: list_unhashed() on the node return true after this. It is
139  * useful for RCU based read lockfree traversal if the writer side
140  * must know if the list entry is still hashed or already unhashed.
141  *
142  * In particular, it means that we can not poison the forward pointers
143  * that may still be used for walking the hash list and we can only
144  * zero the pprev pointer so list_unhashed() will return true after
145  * this.
146  *
147  * The caller must take whatever precautions are necessary (such as
148  * holding appropriate locks) to avoid racing with another
149  * list-mutation primitive, such as hlist_add_head_rcu() or
150  * hlist_del_rcu(), running on this same list.  However, it is
151  * perfectly legal to run concurrently with the _rcu list-traversal
152  * primitives, such as hlist_for_each_entry_rcu().
153  */
hlist_del_init_rcu(struct hlist_node * n)154 static inline void hlist_del_init_rcu(struct hlist_node *n)
155 {
156 	if (!hlist_unhashed(n)) {
157 		__hlist_del(n);
158 		n->pprev = NULL;
159 	}
160 }
161 
162 /**
163  * list_replace_rcu - replace old entry by new one
164  * @old : the element to be replaced
165  * @new : the new element to insert
166  *
167  * The @old entry will be replaced with the @new entry atomically.
168  * Note: @old should not be empty.
169  */
list_replace_rcu(struct list_head * old,struct list_head * new)170 static inline void list_replace_rcu(struct list_head *old,
171 				struct list_head *new)
172 {
173 	new->next = old->next;
174 	new->prev = old->prev;
175 	rcu_assign_pointer(list_next_rcu(new->prev), new);
176 	new->next->prev = new;
177 	old->prev = LIST_POISON2;
178 }
179 
180 /**
181  * __list_splice_init_rcu - join an RCU-protected list into an existing list.
182  * @list:	the RCU-protected list to splice
183  * @prev:	points to the last element of the existing list
184  * @next:	points to the first element of the existing list
185  * @sync:	function to sync: synchronize_rcu(), synchronize_sched(), ...
186  *
187  * The list pointed to by @prev and @next can be RCU-read traversed
188  * concurrently with this function.
189  *
190  * Note that this function blocks.
191  *
192  * Important note: the caller must take whatever action is necessary to prevent
193  * any other updates to the existing list.  In principle, it is possible to
194  * modify the list as soon as sync() begins execution. If this sort of thing
195  * becomes necessary, an alternative version based on call_rcu() could be
196  * created.  But only if -really- needed -- there is no shortage of RCU API
197  * members.
198  */
__list_splice_init_rcu(struct list_head * list,struct list_head * prev,struct list_head * next,void (* sync)(void))199 static inline void __list_splice_init_rcu(struct list_head *list,
200 					  struct list_head *prev,
201 					  struct list_head *next,
202 					  void (*sync)(void))
203 {
204 	struct list_head *first = list->next;
205 	struct list_head *last = list->prev;
206 
207 	/*
208 	 * "first" and "last" tracking list, so initialize it.  RCU readers
209 	 * have access to this list, so we must use INIT_LIST_HEAD_RCU()
210 	 * instead of INIT_LIST_HEAD().
211 	 */
212 
213 	INIT_LIST_HEAD_RCU(list);
214 
215 	/*
216 	 * At this point, the list body still points to the source list.
217 	 * Wait for any readers to finish using the list before splicing
218 	 * the list body into the new list.  Any new readers will see
219 	 * an empty list.
220 	 */
221 
222 	sync();
223 
224 	/*
225 	 * Readers are finished with the source list, so perform splice.
226 	 * The order is important if the new list is global and accessible
227 	 * to concurrent RCU readers.  Note that RCU readers are not
228 	 * permitted to traverse the prev pointers without excluding
229 	 * this function.
230 	 */
231 
232 	last->next = next;
233 	rcu_assign_pointer(list_next_rcu(prev), first);
234 	first->prev = prev;
235 	next->prev = last;
236 }
237 
238 /**
239  * list_splice_init_rcu - splice an RCU-protected list into an existing list,
240  *                        designed for stacks.
241  * @list:	the RCU-protected list to splice
242  * @head:	the place in the existing list to splice the first list into
243  * @sync:	function to sync: synchronize_rcu(), synchronize_sched(), ...
244  */
list_splice_init_rcu(struct list_head * list,struct list_head * head,void (* sync)(void))245 static inline void list_splice_init_rcu(struct list_head *list,
246 					struct list_head *head,
247 					void (*sync)(void))
248 {
249 	if (!list_empty(list))
250 		__list_splice_init_rcu(list, head, head->next, sync);
251 }
252 
253 /**
254  * list_splice_tail_init_rcu - splice an RCU-protected list into an existing
255  *                             list, designed for queues.
256  * @list:	the RCU-protected list to splice
257  * @head:	the place in the existing list to splice the first list into
258  * @sync:	function to sync: synchronize_rcu(), synchronize_sched(), ...
259  */
list_splice_tail_init_rcu(struct list_head * list,struct list_head * head,void (* sync)(void))260 static inline void list_splice_tail_init_rcu(struct list_head *list,
261 					     struct list_head *head,
262 					     void (*sync)(void))
263 {
264 	if (!list_empty(list))
265 		__list_splice_init_rcu(list, head->prev, head, sync);
266 }
267 
268 /**
269  * list_entry_rcu - get the struct for this entry
270  * @ptr:        the &struct list_head pointer.
271  * @type:       the type of the struct this is embedded in.
272  * @member:     the name of the list_head within the struct.
273  *
274  * This primitive may safely run concurrently with the _rcu list-mutation
275  * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
276  */
277 #define list_entry_rcu(ptr, type, member) \
278 	container_of(READ_ONCE(ptr), type, member)
279 
280 /*
281  * Where are list_empty_rcu() and list_first_entry_rcu()?
282  *
283  * Implementing those functions following their counterparts list_empty() and
284  * list_first_entry() is not advisable because they lead to subtle race
285  * conditions as the following snippet shows:
286  *
287  * if (!list_empty_rcu(mylist)) {
288  *	struct foo *bar = list_first_entry_rcu(mylist, struct foo, list_member);
289  *	do_something(bar);
290  * }
291  *
292  * The list may not be empty when list_empty_rcu checks it, but it may be when
293  * list_first_entry_rcu rereads the ->next pointer.
294  *
295  * Rereading the ->next pointer is not a problem for list_empty() and
296  * list_first_entry() because they would be protected by a lock that blocks
297  * writers.
298  *
299  * See list_first_or_null_rcu for an alternative.
300  */
301 
302 /**
303  * list_first_or_null_rcu - get the first element from a list
304  * @ptr:        the list head to take the element from.
305  * @type:       the type of the struct this is embedded in.
306  * @member:     the name of the list_head within the struct.
307  *
308  * Note that if the list is empty, it returns NULL.
309  *
310  * This primitive may safely run concurrently with the _rcu list-mutation
311  * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
312  */
313 #define list_first_or_null_rcu(ptr, type, member) \
314 ({ \
315 	struct list_head *__ptr = (ptr); \
316 	struct list_head *__next = READ_ONCE(__ptr->next); \
317 	likely(__ptr != __next) ? list_entry_rcu(__next, type, member) : NULL; \
318 })
319 
320 /**
321  * list_next_or_null_rcu - get the first element from a list
322  * @head:	the head for the list.
323  * @ptr:        the list head to take the next element from.
324  * @type:       the type of the struct this is embedded in.
325  * @member:     the name of the list_head within the struct.
326  *
327  * Note that if the ptr is at the end of the list, NULL is returned.
328  *
329  * This primitive may safely run concurrently with the _rcu list-mutation
330  * primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
331  */
332 #define list_next_or_null_rcu(head, ptr, type, member) \
333 ({ \
334 	struct list_head *__head = (head); \
335 	struct list_head *__ptr = (ptr); \
336 	struct list_head *__next = READ_ONCE(__ptr->next); \
337 	likely(__next != __head) ? list_entry_rcu(__next, type, \
338 						  member) : NULL; \
339 })
340 
341 /**
342  * list_for_each_entry_rcu	-	iterate over rcu list of given type
343  * @pos:	the type * to use as a loop cursor.
344  * @head:	the head for your list.
345  * @member:	the name of the list_head within the struct.
346  *
347  * This list-traversal primitive may safely run concurrently with
348  * the _rcu list-mutation primitives such as list_add_rcu()
349  * as long as the traversal is guarded by rcu_read_lock().
350  */
351 #define list_for_each_entry_rcu(pos, head, member) \
352 	for (pos = list_entry_rcu((head)->next, typeof(*pos), member); \
353 		&pos->member != (head); \
354 		pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
355 
356 /**
357  * list_entry_lockless - get the struct for this entry
358  * @ptr:        the &struct list_head pointer.
359  * @type:       the type of the struct this is embedded in.
360  * @member:     the name of the list_head within the struct.
361  *
362  * This primitive may safely run concurrently with the _rcu list-mutation
363  * primitives such as list_add_rcu(), but requires some implicit RCU
364  * read-side guarding.  One example is running within a special
365  * exception-time environment where preemption is disabled and where
366  * lockdep cannot be invoked (in which case updaters must use RCU-sched,
367  * as in synchronize_sched(), call_rcu_sched(), and friends).  Another
368  * example is when items are added to the list, but never deleted.
369  */
370 #define list_entry_lockless(ptr, type, member) \
371 	container_of((typeof(ptr))READ_ONCE(ptr), type, member)
372 
373 /**
374  * list_for_each_entry_lockless - iterate over rcu list of given type
375  * @pos:	the type * to use as a loop cursor.
376  * @head:	the head for your list.
377  * @member:	the name of the list_struct within the struct.
378  *
379  * This primitive may safely run concurrently with the _rcu list-mutation
380  * primitives such as list_add_rcu(), but requires some implicit RCU
381  * read-side guarding.  One example is running within a special
382  * exception-time environment where preemption is disabled and where
383  * lockdep cannot be invoked (in which case updaters must use RCU-sched,
384  * as in synchronize_sched(), call_rcu_sched(), and friends).  Another
385  * example is when items are added to the list, but never deleted.
386  */
387 #define list_for_each_entry_lockless(pos, head, member) \
388 	for (pos = list_entry_lockless((head)->next, typeof(*pos), member); \
389 	     &pos->member != (head); \
390 	     pos = list_entry_lockless(pos->member.next, typeof(*pos), member))
391 
392 /**
393  * list_for_each_entry_continue_rcu - continue iteration over list of given type
394  * @pos:	the type * to use as a loop cursor.
395  * @head:	the head for your list.
396  * @member:	the name of the list_head within the struct.
397  *
398  * Continue to iterate over list of given type, continuing after
399  * the current position which must have been in the list when the RCU read
400  * lock was taken.
401  * This would typically require either that you obtained the node from a
402  * previous walk of the list in the same RCU read-side critical section, or
403  * that you held some sort of non-RCU reference (such as a reference count)
404  * to keep the node alive *and* in the list.
405  *
406  * This iterator is similar to list_for_each_entry_from_rcu() except
407  * this starts after the given position and that one starts at the given
408  * position.
409  */
410 #define list_for_each_entry_continue_rcu(pos, head, member) 		\
411 	for (pos = list_entry_rcu(pos->member.next, typeof(*pos), member); \
412 	     &pos->member != (head);	\
413 	     pos = list_entry_rcu(pos->member.next, typeof(*pos), member))
414 
415 /**
416  * list_for_each_entry_from_rcu - iterate over a list from current point
417  * @pos:	the type * to use as a loop cursor.
418  * @head:	the head for your list.
419  * @member:	the name of the list_node within the struct.
420  *
421  * Iterate over the tail of a list starting from a given position,
422  * which must have been in the list when the RCU read lock was taken.
423  * This would typically require either that you obtained the node from a
424  * previous walk of the list in the same RCU read-side critical section, or
425  * that you held some sort of non-RCU reference (such as a reference count)
426  * to keep the node alive *and* in the list.
427  *
428  * This iterator is similar to list_for_each_entry_continue_rcu() except
429  * this starts from the given position and that one starts from the position
430  * after the given position.
431  */
432 #define list_for_each_entry_from_rcu(pos, head, member)			\
433 	for (; &(pos)->member != (head);					\
434 		pos = list_entry_rcu(pos->member.next, typeof(*(pos)), member))
435 
436 /**
437  * hlist_del_rcu - deletes entry from hash list without re-initialization
438  * @n: the element to delete from the hash list.
439  *
440  * Note: list_unhashed() on entry does not return true after this,
441  * the entry is in an undefined state. It is useful for RCU based
442  * lockfree traversal.
443  *
444  * In particular, it means that we can not poison the forward
445  * pointers that may still be used for walking the hash list.
446  *
447  * The caller must take whatever precautions are necessary
448  * (such as holding appropriate locks) to avoid racing
449  * with another list-mutation primitive, such as hlist_add_head_rcu()
450  * or hlist_del_rcu(), running on this same list.
451  * However, it is perfectly legal to run concurrently with
452  * the _rcu list-traversal primitives, such as
453  * hlist_for_each_entry().
454  */
hlist_del_rcu(struct hlist_node * n)455 static inline void hlist_del_rcu(struct hlist_node *n)
456 {
457 	__hlist_del(n);
458 	n->pprev = LIST_POISON2;
459 }
460 
461 /**
462  * hlist_replace_rcu - replace old entry by new one
463  * @old : the element to be replaced
464  * @new : the new element to insert
465  *
466  * The @old entry will be replaced with the @new entry atomically.
467  */
hlist_replace_rcu(struct hlist_node * old,struct hlist_node * new)468 static inline void hlist_replace_rcu(struct hlist_node *old,
469 					struct hlist_node *new)
470 {
471 	struct hlist_node *next = old->next;
472 
473 	new->next = next;
474 	new->pprev = old->pprev;
475 	rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new);
476 	if (next)
477 		new->next->pprev = &new->next;
478 	old->pprev = LIST_POISON2;
479 }
480 
481 /*
482  * return the first or the next element in an RCU protected hlist
483  */
484 #define hlist_first_rcu(head)	(*((struct hlist_node __rcu **)(&(head)->first)))
485 #define hlist_next_rcu(node)	(*((struct hlist_node __rcu **)(&(node)->next)))
486 #define hlist_pprev_rcu(node)	(*((struct hlist_node __rcu **)((node)->pprev)))
487 
488 /**
489  * hlist_add_head_rcu
490  * @n: the element to add to the hash list.
491  * @h: the list to add to.
492  *
493  * Description:
494  * Adds the specified element to the specified hlist,
495  * while permitting racing traversals.
496  *
497  * The caller must take whatever precautions are necessary
498  * (such as holding appropriate locks) to avoid racing
499  * with another list-mutation primitive, such as hlist_add_head_rcu()
500  * or hlist_del_rcu(), running on this same list.
501  * However, it is perfectly legal to run concurrently with
502  * the _rcu list-traversal primitives, such as
503  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
504  * problems on Alpha CPUs.  Regardless of the type of CPU, the
505  * list-traversal primitive must be guarded by rcu_read_lock().
506  */
hlist_add_head_rcu(struct hlist_node * n,struct hlist_head * h)507 static inline void hlist_add_head_rcu(struct hlist_node *n,
508 					struct hlist_head *h)
509 {
510 	struct hlist_node *first = h->first;
511 
512 	n->next = first;
513 	n->pprev = &h->first;
514 	rcu_assign_pointer(hlist_first_rcu(h), n);
515 	if (first)
516 		first->pprev = &n->next;
517 }
518 
519 /**
520  * hlist_add_tail_rcu
521  * @n: the element to add to the hash list.
522  * @h: the list to add to.
523  *
524  * Description:
525  * Adds the specified element to the specified hlist,
526  * while permitting racing traversals.
527  *
528  * The caller must take whatever precautions are necessary
529  * (such as holding appropriate locks) to avoid racing
530  * with another list-mutation primitive, such as hlist_add_head_rcu()
531  * or hlist_del_rcu(), running on this same list.
532  * However, it is perfectly legal to run concurrently with
533  * the _rcu list-traversal primitives, such as
534  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
535  * problems on Alpha CPUs.  Regardless of the type of CPU, the
536  * list-traversal primitive must be guarded by rcu_read_lock().
537  */
hlist_add_tail_rcu(struct hlist_node * n,struct hlist_head * h)538 static inline void hlist_add_tail_rcu(struct hlist_node *n,
539 				      struct hlist_head *h)
540 {
541 	struct hlist_node *i, *last = NULL;
542 
543 	/* Note: write side code, so rcu accessors are not needed. */
544 	for (i = h->first; i; i = i->next)
545 		last = i;
546 
547 	if (last) {
548 		n->next = last->next;
549 		n->pprev = &last->next;
550 		rcu_assign_pointer(hlist_next_rcu(last), n);
551 	} else {
552 		hlist_add_head_rcu(n, h);
553 	}
554 }
555 
556 /**
557  * hlist_add_before_rcu
558  * @n: the new element to add to the hash list.
559  * @next: the existing element to add the new element before.
560  *
561  * Description:
562  * Adds the specified element to the specified hlist
563  * before the specified node while permitting racing traversals.
564  *
565  * The caller must take whatever precautions are necessary
566  * (such as holding appropriate locks) to avoid racing
567  * with another list-mutation primitive, such as hlist_add_head_rcu()
568  * or hlist_del_rcu(), running on this same list.
569  * However, it is perfectly legal to run concurrently with
570  * the _rcu list-traversal primitives, such as
571  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
572  * problems on Alpha CPUs.
573  */
hlist_add_before_rcu(struct hlist_node * n,struct hlist_node * next)574 static inline void hlist_add_before_rcu(struct hlist_node *n,
575 					struct hlist_node *next)
576 {
577 	n->pprev = next->pprev;
578 	n->next = next;
579 	rcu_assign_pointer(hlist_pprev_rcu(n), n);
580 	next->pprev = &n->next;
581 }
582 
583 /**
584  * hlist_add_behind_rcu
585  * @n: the new element to add to the hash list.
586  * @prev: the existing element to add the new element after.
587  *
588  * Description:
589  * Adds the specified element to the specified hlist
590  * after the specified node while permitting racing traversals.
591  *
592  * The caller must take whatever precautions are necessary
593  * (such as holding appropriate locks) to avoid racing
594  * with another list-mutation primitive, such as hlist_add_head_rcu()
595  * or hlist_del_rcu(), running on this same list.
596  * However, it is perfectly legal to run concurrently with
597  * the _rcu list-traversal primitives, such as
598  * hlist_for_each_entry_rcu(), used to prevent memory-consistency
599  * problems on Alpha CPUs.
600  */
hlist_add_behind_rcu(struct hlist_node * n,struct hlist_node * prev)601 static inline void hlist_add_behind_rcu(struct hlist_node *n,
602 					struct hlist_node *prev)
603 {
604 	n->next = prev->next;
605 	n->pprev = &prev->next;
606 	rcu_assign_pointer(hlist_next_rcu(prev), n);
607 	if (n->next)
608 		n->next->pprev = &n->next;
609 }
610 
611 #define __hlist_for_each_rcu(pos, head)				\
612 	for (pos = rcu_dereference(hlist_first_rcu(head));	\
613 	     pos;						\
614 	     pos = rcu_dereference(hlist_next_rcu(pos)))
615 
616 /**
617  * hlist_for_each_entry_rcu - iterate over rcu list of given type
618  * @pos:	the type * to use as a loop cursor.
619  * @head:	the head for your list.
620  * @member:	the name of the hlist_node within the struct.
621  *
622  * This list-traversal primitive may safely run concurrently with
623  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
624  * as long as the traversal is guarded by rcu_read_lock().
625  */
626 #define hlist_for_each_entry_rcu(pos, head, member)			\
627 	for (pos = hlist_entry_safe (rcu_dereference_raw(hlist_first_rcu(head)),\
628 			typeof(*(pos)), member);			\
629 		pos;							\
630 		pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(\
631 			&(pos)->member)), typeof(*(pos)), member))
632 
633 /**
634  * hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing)
635  * @pos:	the type * to use as a loop cursor.
636  * @head:	the head for your list.
637  * @member:	the name of the hlist_node within the struct.
638  *
639  * This list-traversal primitive may safely run concurrently with
640  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
641  * as long as the traversal is guarded by rcu_read_lock().
642  *
643  * This is the same as hlist_for_each_entry_rcu() except that it does
644  * not do any RCU debugging or tracing.
645  */
646 #define hlist_for_each_entry_rcu_notrace(pos, head, member)			\
647 	for (pos = hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\
648 			typeof(*(pos)), member);			\
649 		pos;							\
650 		pos = hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\
651 			&(pos)->member)), typeof(*(pos)), member))
652 
653 /**
654  * hlist_for_each_entry_rcu_bh - iterate over rcu list of given type
655  * @pos:	the type * to use as a loop cursor.
656  * @head:	the head for your list.
657  * @member:	the name of the hlist_node within the struct.
658  *
659  * This list-traversal primitive may safely run concurrently with
660  * the _rcu list-mutation primitives such as hlist_add_head_rcu()
661  * as long as the traversal is guarded by rcu_read_lock().
662  */
663 #define hlist_for_each_entry_rcu_bh(pos, head, member)			\
664 	for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_first_rcu(head)),\
665 			typeof(*(pos)), member);			\
666 		pos;							\
667 		pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(\
668 			&(pos)->member)), typeof(*(pos)), member))
669 
670 /**
671  * hlist_for_each_entry_continue_rcu - iterate over a hlist continuing after current point
672  * @pos:	the type * to use as a loop cursor.
673  * @member:	the name of the hlist_node within the struct.
674  */
675 #define hlist_for_each_entry_continue_rcu(pos, member)			\
676 	for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
677 			&(pos)->member)), typeof(*(pos)), member);	\
678 	     pos;							\
679 	     pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(	\
680 			&(pos)->member)), typeof(*(pos)), member))
681 
682 /**
683  * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
684  * @pos:	the type * to use as a loop cursor.
685  * @member:	the name of the hlist_node within the struct.
686  */
687 #define hlist_for_each_entry_continue_rcu_bh(pos, member)		\
688 	for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(  \
689 			&(pos)->member)), typeof(*(pos)), member);	\
690 	     pos;							\
691 	     pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu(	\
692 			&(pos)->member)), typeof(*(pos)), member))
693 
694 /**
695  * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
696  * @pos:	the type * to use as a loop cursor.
697  * @member:	the name of the hlist_node within the struct.
698  */
699 #define hlist_for_each_entry_from_rcu(pos, member)			\
700 	for (; pos;							\
701 	     pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(	\
702 			&(pos)->member)), typeof(*(pos)), member))
703 
704 #endif	/* __KERNEL__ */
705 #endif
706