1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_GENERIC_PERCPU_H_
3 #define _ASM_GENERIC_PERCPU_H_
4 
5 #include <linux/compiler.h>
6 #include <linux/threads.h>
7 #include <linux/percpu-defs.h>
8 
9 #ifdef CONFIG_SMP
10 
11 /*
12  * per_cpu_offset() is the offset that has to be added to a
13  * percpu variable to get to the instance for a certain processor.
14  *
15  * Most arches use the __per_cpu_offset array for those offsets but
16  * some arches have their own ways of determining the offset (x86_64, s390).
17  */
18 #ifndef __per_cpu_offset
19 extern unsigned long __per_cpu_offset[NR_CPUS];
20 
21 #define per_cpu_offset(x) (__per_cpu_offset[x])
22 #endif
23 
24 /*
25  * Determine the offset for the currently active processor.
26  * An arch may define __my_cpu_offset to provide a more effective
27  * means of obtaining the offset to the per cpu variables of the
28  * current processor.
29  */
30 #ifndef __my_cpu_offset
31 #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
32 #endif
33 #ifdef CONFIG_DEBUG_PREEMPT
34 #define my_cpu_offset per_cpu_offset(smp_processor_id())
35 #else
36 #define my_cpu_offset __my_cpu_offset
37 #endif
38 
39 /*
40  * Arch may define arch_raw_cpu_ptr() to provide more efficient address
41  * translations for raw_cpu_ptr().
42  */
43 #ifndef arch_raw_cpu_ptr
44 #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
45 #endif
46 
47 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
48 extern void setup_per_cpu_areas(void);
49 #endif
50 
51 #endif	/* SMP */
52 
53 #ifndef PER_CPU_BASE_SECTION
54 #ifdef CONFIG_SMP
55 #define PER_CPU_BASE_SECTION ".data..percpu"
56 #else
57 #define PER_CPU_BASE_SECTION ".data"
58 #endif
59 #endif
60 
61 #ifndef PER_CPU_ATTRIBUTES
62 #define PER_CPU_ATTRIBUTES
63 #endif
64 
65 #ifndef PER_CPU_DEF_ATTRIBUTES
66 #define PER_CPU_DEF_ATTRIBUTES
67 #endif
68 
69 #define raw_cpu_generic_read(pcp)					\
70 ({									\
71 	*raw_cpu_ptr(&(pcp));						\
72 })
73 
74 #define raw_cpu_generic_to_op(pcp, val, op)				\
75 do {									\
76 	*raw_cpu_ptr(&(pcp)) op val;					\
77 } while (0)
78 
79 #define raw_cpu_generic_add_return(pcp, val)				\
80 ({									\
81 	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));			\
82 									\
83 	*__p += val;							\
84 	*__p;								\
85 })
86 
87 #define raw_cpu_generic_xchg(pcp, nval)					\
88 ({									\
89 	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));			\
90 	typeof(pcp) __ret;						\
91 	__ret = *__p;							\
92 	*__p = nval;							\
93 	__ret;								\
94 })
95 
96 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)			\
97 ({									\
98 	typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp));			\
99 	typeof(pcp) __ret;						\
100 	__ret = *__p;							\
101 	if (__ret == (oval))						\
102 		*__p = nval;						\
103 	__ret;								\
104 })
105 
106 #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
107 ({									\
108 	typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1));			\
109 	typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2));			\
110 	int __ret = 0;							\
111 	if (*__p1 == (oval1) && *__p2  == (oval2)) {			\
112 		*__p1 = nval1;						\
113 		*__p2 = nval2;						\
114 		__ret = 1;						\
115 	}								\
116 	(__ret);							\
117 })
118 
119 #define __this_cpu_generic_read_nopreempt(pcp)				\
120 ({									\
121 	typeof(pcp) __ret;						\
122 	preempt_disable_notrace();					\
123 	__ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));			\
124 	preempt_enable_notrace();					\
125 	__ret;								\
126 })
127 
128 #define __this_cpu_generic_read_noirq(pcp)				\
129 ({									\
130 	typeof(pcp) __ret;						\
131 	unsigned long __flags;						\
132 	raw_local_irq_save(__flags);					\
133 	__ret = raw_cpu_generic_read(pcp);				\
134 	raw_local_irq_restore(__flags);					\
135 	__ret;								\
136 })
137 
138 #define this_cpu_generic_read(pcp)					\
139 ({									\
140 	typeof(pcp) __ret;						\
141 	if (__native_word(pcp))						\
142 		__ret = __this_cpu_generic_read_nopreempt(pcp);		\
143 	else								\
144 		__ret = __this_cpu_generic_read_noirq(pcp);		\
145 	__ret;								\
146 })
147 
148 #define this_cpu_generic_to_op(pcp, val, op)				\
149 do {									\
150 	unsigned long __flags;						\
151 	raw_local_irq_save(__flags);					\
152 	raw_cpu_generic_to_op(pcp, val, op);				\
153 	raw_local_irq_restore(__flags);					\
154 } while (0)
155 
156 
157 #define this_cpu_generic_add_return(pcp, val)				\
158 ({									\
159 	typeof(pcp) __ret;						\
160 	unsigned long __flags;						\
161 	raw_local_irq_save(__flags);					\
162 	__ret = raw_cpu_generic_add_return(pcp, val);			\
163 	raw_local_irq_restore(__flags);					\
164 	__ret;								\
165 })
166 
167 #define this_cpu_generic_xchg(pcp, nval)				\
168 ({									\
169 	typeof(pcp) __ret;						\
170 	unsigned long __flags;						\
171 	raw_local_irq_save(__flags);					\
172 	__ret = raw_cpu_generic_xchg(pcp, nval);			\
173 	raw_local_irq_restore(__flags);					\
174 	__ret;								\
175 })
176 
177 #define this_cpu_generic_cmpxchg(pcp, oval, nval)			\
178 ({									\
179 	typeof(pcp) __ret;						\
180 	unsigned long __flags;						\
181 	raw_local_irq_save(__flags);					\
182 	__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);		\
183 	raw_local_irq_restore(__flags);					\
184 	__ret;								\
185 })
186 
187 #define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)	\
188 ({									\
189 	int __ret;							\
190 	unsigned long __flags;						\
191 	raw_local_irq_save(__flags);					\
192 	__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2,		\
193 			oval1, oval2, nval1, nval2);			\
194 	raw_local_irq_restore(__flags);					\
195 	__ret;								\
196 })
197 
198 #ifndef raw_cpu_read_1
199 #define raw_cpu_read_1(pcp)		raw_cpu_generic_read(pcp)
200 #endif
201 #ifndef raw_cpu_read_2
202 #define raw_cpu_read_2(pcp)		raw_cpu_generic_read(pcp)
203 #endif
204 #ifndef raw_cpu_read_4
205 #define raw_cpu_read_4(pcp)		raw_cpu_generic_read(pcp)
206 #endif
207 #ifndef raw_cpu_read_8
208 #define raw_cpu_read_8(pcp)		raw_cpu_generic_read(pcp)
209 #endif
210 
211 #ifndef raw_cpu_write_1
212 #define raw_cpu_write_1(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
213 #endif
214 #ifndef raw_cpu_write_2
215 #define raw_cpu_write_2(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
216 #endif
217 #ifndef raw_cpu_write_4
218 #define raw_cpu_write_4(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
219 #endif
220 #ifndef raw_cpu_write_8
221 #define raw_cpu_write_8(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
222 #endif
223 
224 #ifndef raw_cpu_add_1
225 #define raw_cpu_add_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
226 #endif
227 #ifndef raw_cpu_add_2
228 #define raw_cpu_add_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
229 #endif
230 #ifndef raw_cpu_add_4
231 #define raw_cpu_add_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
232 #endif
233 #ifndef raw_cpu_add_8
234 #define raw_cpu_add_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
235 #endif
236 
237 #ifndef raw_cpu_and_1
238 #define raw_cpu_and_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
239 #endif
240 #ifndef raw_cpu_and_2
241 #define raw_cpu_and_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
242 #endif
243 #ifndef raw_cpu_and_4
244 #define raw_cpu_and_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
245 #endif
246 #ifndef raw_cpu_and_8
247 #define raw_cpu_and_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
248 #endif
249 
250 #ifndef raw_cpu_or_1
251 #define raw_cpu_or_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
252 #endif
253 #ifndef raw_cpu_or_2
254 #define raw_cpu_or_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
255 #endif
256 #ifndef raw_cpu_or_4
257 #define raw_cpu_or_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
258 #endif
259 #ifndef raw_cpu_or_8
260 #define raw_cpu_or_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
261 #endif
262 
263 #ifndef raw_cpu_add_return_1
264 #define raw_cpu_add_return_1(pcp, val)	raw_cpu_generic_add_return(pcp, val)
265 #endif
266 #ifndef raw_cpu_add_return_2
267 #define raw_cpu_add_return_2(pcp, val)	raw_cpu_generic_add_return(pcp, val)
268 #endif
269 #ifndef raw_cpu_add_return_4
270 #define raw_cpu_add_return_4(pcp, val)	raw_cpu_generic_add_return(pcp, val)
271 #endif
272 #ifndef raw_cpu_add_return_8
273 #define raw_cpu_add_return_8(pcp, val)	raw_cpu_generic_add_return(pcp, val)
274 #endif
275 
276 #ifndef raw_cpu_xchg_1
277 #define raw_cpu_xchg_1(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
278 #endif
279 #ifndef raw_cpu_xchg_2
280 #define raw_cpu_xchg_2(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
281 #endif
282 #ifndef raw_cpu_xchg_4
283 #define raw_cpu_xchg_4(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
284 #endif
285 #ifndef raw_cpu_xchg_8
286 #define raw_cpu_xchg_8(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
287 #endif
288 
289 #ifndef raw_cpu_cmpxchg_1
290 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
291 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
292 #endif
293 #ifndef raw_cpu_cmpxchg_2
294 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
295 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
296 #endif
297 #ifndef raw_cpu_cmpxchg_4
298 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
299 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
300 #endif
301 #ifndef raw_cpu_cmpxchg_8
302 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
303 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
304 #endif
305 
306 #ifndef raw_cpu_cmpxchg_double_1
307 #define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
308 	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
309 #endif
310 #ifndef raw_cpu_cmpxchg_double_2
311 #define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
312 	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
313 #endif
314 #ifndef raw_cpu_cmpxchg_double_4
315 #define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
316 	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
317 #endif
318 #ifndef raw_cpu_cmpxchg_double_8
319 #define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
320 	raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
321 #endif
322 
323 #ifndef this_cpu_read_1
324 #define this_cpu_read_1(pcp)		this_cpu_generic_read(pcp)
325 #endif
326 #ifndef this_cpu_read_2
327 #define this_cpu_read_2(pcp)		this_cpu_generic_read(pcp)
328 #endif
329 #ifndef this_cpu_read_4
330 #define this_cpu_read_4(pcp)		this_cpu_generic_read(pcp)
331 #endif
332 #ifndef this_cpu_read_8
333 #define this_cpu_read_8(pcp)		this_cpu_generic_read(pcp)
334 #endif
335 
336 #ifndef this_cpu_write_1
337 #define this_cpu_write_1(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
338 #endif
339 #ifndef this_cpu_write_2
340 #define this_cpu_write_2(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
341 #endif
342 #ifndef this_cpu_write_4
343 #define this_cpu_write_4(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
344 #endif
345 #ifndef this_cpu_write_8
346 #define this_cpu_write_8(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
347 #endif
348 
349 #ifndef this_cpu_add_1
350 #define this_cpu_add_1(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
351 #endif
352 #ifndef this_cpu_add_2
353 #define this_cpu_add_2(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
354 #endif
355 #ifndef this_cpu_add_4
356 #define this_cpu_add_4(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
357 #endif
358 #ifndef this_cpu_add_8
359 #define this_cpu_add_8(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
360 #endif
361 
362 #ifndef this_cpu_and_1
363 #define this_cpu_and_1(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
364 #endif
365 #ifndef this_cpu_and_2
366 #define this_cpu_and_2(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
367 #endif
368 #ifndef this_cpu_and_4
369 #define this_cpu_and_4(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
370 #endif
371 #ifndef this_cpu_and_8
372 #define this_cpu_and_8(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
373 #endif
374 
375 #ifndef this_cpu_or_1
376 #define this_cpu_or_1(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
377 #endif
378 #ifndef this_cpu_or_2
379 #define this_cpu_or_2(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
380 #endif
381 #ifndef this_cpu_or_4
382 #define this_cpu_or_4(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
383 #endif
384 #ifndef this_cpu_or_8
385 #define this_cpu_or_8(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
386 #endif
387 
388 #ifndef this_cpu_add_return_1
389 #define this_cpu_add_return_1(pcp, val)	this_cpu_generic_add_return(pcp, val)
390 #endif
391 #ifndef this_cpu_add_return_2
392 #define this_cpu_add_return_2(pcp, val)	this_cpu_generic_add_return(pcp, val)
393 #endif
394 #ifndef this_cpu_add_return_4
395 #define this_cpu_add_return_4(pcp, val)	this_cpu_generic_add_return(pcp, val)
396 #endif
397 #ifndef this_cpu_add_return_8
398 #define this_cpu_add_return_8(pcp, val)	this_cpu_generic_add_return(pcp, val)
399 #endif
400 
401 #ifndef this_cpu_xchg_1
402 #define this_cpu_xchg_1(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
403 #endif
404 #ifndef this_cpu_xchg_2
405 #define this_cpu_xchg_2(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
406 #endif
407 #ifndef this_cpu_xchg_4
408 #define this_cpu_xchg_4(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
409 #endif
410 #ifndef this_cpu_xchg_8
411 #define this_cpu_xchg_8(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
412 #endif
413 
414 #ifndef this_cpu_cmpxchg_1
415 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
416 	this_cpu_generic_cmpxchg(pcp, oval, nval)
417 #endif
418 #ifndef this_cpu_cmpxchg_2
419 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
420 	this_cpu_generic_cmpxchg(pcp, oval, nval)
421 #endif
422 #ifndef this_cpu_cmpxchg_4
423 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
424 	this_cpu_generic_cmpxchg(pcp, oval, nval)
425 #endif
426 #ifndef this_cpu_cmpxchg_8
427 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
428 	this_cpu_generic_cmpxchg(pcp, oval, nval)
429 #endif
430 
431 #ifndef this_cpu_cmpxchg_double_1
432 #define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
433 	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
434 #endif
435 #ifndef this_cpu_cmpxchg_double_2
436 #define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
437 	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
438 #endif
439 #ifndef this_cpu_cmpxchg_double_4
440 #define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
441 	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
442 #endif
443 #ifndef this_cpu_cmpxchg_double_8
444 #define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
445 	this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
446 #endif
447 
448 #endif /* _ASM_GENERIC_PERCPU_H_ */
449