1 /*
2  * Generic barrier definitions.
3  *
4  * It should be possible to use these on really simple architectures,
5  * but it serves more as a starting point for new ports.
6  *
7  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
8  * Written by David Howells (dhowells@redhat.com)
9  *
10  * This program is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU General Public Licence
12  * as published by the Free Software Foundation; either version
13  * 2 of the Licence, or (at your option) any later version.
14  */
15 #ifndef __ASM_GENERIC_BARRIER_H
16 #define __ASM_GENERIC_BARRIER_H
17 
18 #ifndef __ASSEMBLY__
19 
20 #include <linux/compiler.h>
21 
22 #ifndef nop
23 #define nop()	asm volatile ("nop")
24 #endif
25 
26 /*
27  * Force strict CPU ordering. And yes, this is required on UP too when we're
28  * talking to devices.
29  *
30  * Fall back to compiler barriers if nothing better is provided.
31  */
32 
33 #ifndef mb
34 #define mb()	barrier()
35 #endif
36 
37 #ifndef rmb
38 #define rmb()	mb()
39 #endif
40 
41 #ifndef wmb
42 #define wmb()	mb()
43 #endif
44 
45 #ifndef dma_rmb
46 #define dma_rmb()	rmb()
47 #endif
48 
49 #ifndef dma_wmb
50 #define dma_wmb()	wmb()
51 #endif
52 
53 #ifndef read_barrier_depends
54 #define read_barrier_depends()		do { } while (0)
55 #endif
56 
57 #ifndef __smp_mb
58 #define __smp_mb()	mb()
59 #endif
60 
61 #ifndef __smp_rmb
62 #define __smp_rmb()	rmb()
63 #endif
64 
65 #ifndef __smp_wmb
66 #define __smp_wmb()	wmb()
67 #endif
68 
69 #ifndef __smp_read_barrier_depends
70 #define __smp_read_barrier_depends()	read_barrier_depends()
71 #endif
72 
73 #ifdef CONFIG_SMP
74 
75 #ifndef smp_mb
76 #define smp_mb()	__smp_mb()
77 #endif
78 
79 #ifndef smp_rmb
80 #define smp_rmb()	__smp_rmb()
81 #endif
82 
83 #ifndef smp_wmb
84 #define smp_wmb()	__smp_wmb()
85 #endif
86 
87 #ifndef smp_read_barrier_depends
88 #define smp_read_barrier_depends()	__smp_read_barrier_depends()
89 #endif
90 
91 #else	/* !CONFIG_SMP */
92 
93 #ifndef smp_mb
94 #define smp_mb()	barrier()
95 #endif
96 
97 #ifndef smp_rmb
98 #define smp_rmb()	barrier()
99 #endif
100 
101 #ifndef smp_wmb
102 #define smp_wmb()	barrier()
103 #endif
104 
105 #ifndef smp_read_barrier_depends
106 #define smp_read_barrier_depends()	do { } while (0)
107 #endif
108 
109 #endif	/* CONFIG_SMP */
110 
111 #ifndef __smp_store_mb
112 #define __smp_store_mb(var, value)  do { WRITE_ONCE(var, value); __smp_mb(); } while (0)
113 #endif
114 
115 #ifndef __smp_mb__before_atomic
116 #define __smp_mb__before_atomic()	__smp_mb()
117 #endif
118 
119 #ifndef __smp_mb__after_atomic
120 #define __smp_mb__after_atomic()	__smp_mb()
121 #endif
122 
123 #ifndef __smp_store_release
124 #define __smp_store_release(p, v)					\
125 do {									\
126 	compiletime_assert_atomic_type(*p);				\
127 	__smp_mb();							\
128 	WRITE_ONCE(*p, v);						\
129 } while (0)
130 #endif
131 
132 #ifndef __smp_load_acquire
133 #define __smp_load_acquire(p)						\
134 ({									\
135 	typeof(*p) ___p1 = READ_ONCE(*p);				\
136 	compiletime_assert_atomic_type(*p);				\
137 	__smp_mb();							\
138 	___p1;								\
139 })
140 #endif
141 
142 #ifdef CONFIG_SMP
143 
144 #ifndef smp_store_mb
145 #define smp_store_mb(var, value)  __smp_store_mb(var, value)
146 #endif
147 
148 #ifndef smp_mb__before_atomic
149 #define smp_mb__before_atomic()	__smp_mb__before_atomic()
150 #endif
151 
152 #ifndef smp_mb__after_atomic
153 #define smp_mb__after_atomic()	__smp_mb__after_atomic()
154 #endif
155 
156 #ifndef smp_store_release
157 #define smp_store_release(p, v) __smp_store_release(p, v)
158 #endif
159 
160 #ifndef smp_load_acquire
161 #define smp_load_acquire(p) __smp_load_acquire(p)
162 #endif
163 
164 #else	/* !CONFIG_SMP */
165 
166 #ifndef smp_store_mb
167 #define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); barrier(); } while (0)
168 #endif
169 
170 #ifndef smp_mb__before_atomic
171 #define smp_mb__before_atomic()	barrier()
172 #endif
173 
174 #ifndef smp_mb__after_atomic
175 #define smp_mb__after_atomic()	barrier()
176 #endif
177 
178 #ifndef smp_store_release
179 #define smp_store_release(p, v)						\
180 do {									\
181 	compiletime_assert_atomic_type(*p);				\
182 	barrier();							\
183 	WRITE_ONCE(*p, v);						\
184 } while (0)
185 #endif
186 
187 #ifndef smp_load_acquire
188 #define smp_load_acquire(p)						\
189 ({									\
190 	typeof(*p) ___p1 = READ_ONCE(*p);				\
191 	compiletime_assert_atomic_type(*p);				\
192 	barrier();							\
193 	___p1;								\
194 })
195 #endif
196 
197 #endif	/* CONFIG_SMP */
198 
199 /* Barriers for virtual machine guests when talking to an SMP host */
200 #define virt_mb() __smp_mb()
201 #define virt_rmb() __smp_rmb()
202 #define virt_wmb() __smp_wmb()
203 #define virt_read_barrier_depends() __smp_read_barrier_depends()
204 #define virt_store_mb(var, value) __smp_store_mb(var, value)
205 #define virt_mb__before_atomic() __smp_mb__before_atomic()
206 #define virt_mb__after_atomic()	__smp_mb__after_atomic()
207 #define virt_store_release(p, v) __smp_store_release(p, v)
208 #define virt_load_acquire(p) __smp_load_acquire(p)
209 
210 /**
211  * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency
212  *
213  * A control dependency provides a LOAD->STORE order, the additional RMB
214  * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order,
215  * aka. (load)-ACQUIRE.
216  *
217  * Architectures that do not do load speculation can have this be barrier().
218  */
219 #ifndef smp_acquire__after_ctrl_dep
220 #define smp_acquire__after_ctrl_dep()		smp_rmb()
221 #endif
222 
223 /**
224  * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
225  * @ptr: pointer to the variable to wait on
226  * @cond: boolean expression to wait for
227  *
228  * Equivalent to using READ_ONCE() on the condition variable.
229  *
230  * Due to C lacking lambda expressions we load the value of *ptr into a
231  * pre-named variable @VAL to be used in @cond.
232  */
233 #ifndef smp_cond_load_relaxed
234 #define smp_cond_load_relaxed(ptr, cond_expr) ({		\
235 	typeof(ptr) __PTR = (ptr);				\
236 	typeof(*ptr) VAL;					\
237 	for (;;) {						\
238 		VAL = READ_ONCE(*__PTR);			\
239 		if (cond_expr)					\
240 			break;					\
241 		cpu_relax();					\
242 	}							\
243 	VAL;							\
244 })
245 #endif
246 
247 /**
248  * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
249  * @ptr: pointer to the variable to wait on
250  * @cond: boolean expression to wait for
251  *
252  * Equivalent to using smp_load_acquire() on the condition variable but employs
253  * the control dependency of the wait to reduce the barrier on many platforms.
254  */
255 #ifndef smp_cond_load_acquire
256 #define smp_cond_load_acquire(ptr, cond_expr) ({		\
257 	typeof(*ptr) _val;					\
258 	_val = smp_cond_load_relaxed(ptr, cond_expr);		\
259 	smp_acquire__after_ctrl_dep();				\
260 	_val;							\
261 })
262 #endif
263 
264 #endif /* !__ASSEMBLY__ */
265 #endif /* __ASM_GENERIC_BARRIER_H */
266