1 /*
2  *  include/linux/hrtimer.h
3  *
4  *  hrtimers - High-resolution kernel timers
5  *
6  *   Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
7  *   Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
8  *
9  *  data type definitions, declarations, prototypes
10  *
11  *  Started by: Thomas Gleixner and Ingo Molnar
12  *
13  *  For licencing details see kernel-base/COPYING
14  */
15 #ifndef _LINUX_HRTIMER_H
16 #define _LINUX_HRTIMER_H
17 
18 #include <linux/rbtree.h>
19 #include <linux/ktime.h>
20 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/percpu.h>
23 #include <linux/timer.h>
24 #include <linux/timerqueue.h>
25 
26 struct hrtimer_clock_base;
27 struct hrtimer_cpu_base;
28 
29 /*
30  * Mode arguments of xxx_hrtimer functions:
31  *
32  * HRTIMER_MODE_ABS		- Time value is absolute
33  * HRTIMER_MODE_REL		- Time value is relative to now
34  * HRTIMER_MODE_PINNED		- Timer is bound to CPU (is only considered
35  *				  when starting the timer)
36  * HRTIMER_MODE_SOFT		- Timer callback function will be executed in
37  *				  soft irq context
38  */
39 enum hrtimer_mode {
40 	HRTIMER_MODE_ABS	= 0x00,
41 	HRTIMER_MODE_REL	= 0x01,
42 	HRTIMER_MODE_PINNED	= 0x02,
43 	HRTIMER_MODE_SOFT	= 0x04,
44 
45 	HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
46 	HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
47 
48 	HRTIMER_MODE_ABS_SOFT	= HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
49 	HRTIMER_MODE_REL_SOFT	= HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,
50 
51 	HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
52 	HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
53 
54 };
55 
56 /*
57  * Return values for the callback function
58  */
59 enum hrtimer_restart {
60 	HRTIMER_NORESTART,	/* Timer is not restarted */
61 	HRTIMER_RESTART,	/* Timer must be restarted */
62 };
63 
64 /*
65  * Values to track state of the timer
66  *
67  * Possible states:
68  *
69  * 0x00		inactive
70  * 0x01		enqueued into rbtree
71  *
72  * The callback state is not part of the timer->state because clearing it would
73  * mean touching the timer after the callback, this makes it impossible to free
74  * the timer from the callback function.
75  *
76  * Therefore we track the callback state in:
77  *
78  *	timer->base->cpu_base->running == timer
79  *
80  * On SMP it is possible to have a "callback function running and enqueued"
81  * status. It happens for example when a posix timer expired and the callback
82  * queued a signal. Between dropping the lock which protects the posix timer
83  * and reacquiring the base lock of the hrtimer, another CPU can deliver the
84  * signal and rearm the timer.
85  *
86  * All state transitions are protected by cpu_base->lock.
87  */
88 #define HRTIMER_STATE_INACTIVE	0x00
89 #define HRTIMER_STATE_ENQUEUED	0x01
90 
91 /**
92  * struct hrtimer - the basic hrtimer structure
93  * @node:	timerqueue node, which also manages node.expires,
94  *		the absolute expiry time in the hrtimers internal
95  *		representation. The time is related to the clock on
96  *		which the timer is based. Is setup by adding
97  *		slack to the _softexpires value. For non range timers
98  *		identical to _softexpires.
99  * @_softexpires: the absolute earliest expiry time of the hrtimer.
100  *		The time which was given as expiry time when the timer
101  *		was armed.
102  * @function:	timer expiry callback function
103  * @base:	pointer to the timer base (per cpu and per clock)
104  * @state:	state information (See bit values above)
105  * @is_rel:	Set if the timer was armed relative
106  * @is_soft:	Set if hrtimer will be expired in soft interrupt context.
107  *
108  * The hrtimer structure must be initialized by hrtimer_init()
109  */
110 struct hrtimer {
111 	struct timerqueue_node		node;
112 	ktime_t				_softexpires;
113 	enum hrtimer_restart		(*function)(struct hrtimer *);
114 	struct hrtimer_clock_base	*base;
115 	u8				state;
116 	u8				is_rel;
117 	u8				is_soft;
118 };
119 
120 /**
121  * struct hrtimer_sleeper - simple sleeper structure
122  * @timer:	embedded timer structure
123  * @task:	task to wake up
124  *
125  * task is set to NULL, when the timer expires.
126  */
127 struct hrtimer_sleeper {
128 	struct hrtimer timer;
129 	struct task_struct *task;
130 };
131 
132 #ifdef CONFIG_64BIT
133 # define __hrtimer_clock_base_align	____cacheline_aligned
134 #else
135 # define __hrtimer_clock_base_align
136 #endif
137 
138 /**
139  * struct hrtimer_clock_base - the timer base for a specific clock
140  * @cpu_base:		per cpu clock base
141  * @index:		clock type index for per_cpu support when moving a
142  *			timer to a base on another cpu.
143  * @clockid:		clock id for per_cpu support
144  * @seq:		seqcount around __run_hrtimer
145  * @running:		pointer to the currently running hrtimer
146  * @active:		red black tree root node for the active timers
147  * @get_time:		function to retrieve the current time of the clock
148  * @offset:		offset of this clock to the monotonic base
149  */
150 struct hrtimer_clock_base {
151 	struct hrtimer_cpu_base	*cpu_base;
152 	unsigned int		index;
153 	clockid_t		clockid;
154 	seqcount_t		seq;
155 	struct hrtimer		*running;
156 	struct timerqueue_head	active;
157 	ktime_t			(*get_time)(void);
158 	ktime_t			offset;
159 } __hrtimer_clock_base_align;
160 
161 enum  hrtimer_base_type {
162 	HRTIMER_BASE_MONOTONIC,
163 	HRTIMER_BASE_REALTIME,
164 	HRTIMER_BASE_BOOTTIME,
165 	HRTIMER_BASE_TAI,
166 	HRTIMER_BASE_MONOTONIC_SOFT,
167 	HRTIMER_BASE_REALTIME_SOFT,
168 	HRTIMER_BASE_BOOTTIME_SOFT,
169 	HRTIMER_BASE_TAI_SOFT,
170 	HRTIMER_MAX_CLOCK_BASES,
171 };
172 
173 /**
174  * struct hrtimer_cpu_base - the per cpu clock bases
175  * @lock:		lock protecting the base and associated clock bases
176  *			and timers
177  * @cpu:		cpu number
178  * @active_bases:	Bitfield to mark bases with active timers
179  * @clock_was_set_seq:	Sequence counter of clock was set events
180  * @hres_active:	State of high resolution mode
181  * @in_hrtirq:		hrtimer_interrupt() is currently executing
182  * @hang_detected:	The last hrtimer interrupt detected a hang
183  * @softirq_activated:	displays, if the softirq is raised - update of softirq
184  *			related settings is not required then.
185  * @nr_events:		Total number of hrtimer interrupt events
186  * @nr_retries:		Total number of hrtimer interrupt retries
187  * @nr_hangs:		Total number of hrtimer interrupt hangs
188  * @max_hang_time:	Maximum time spent in hrtimer_interrupt
189  * @expires_next:	absolute time of the next event, is required for remote
190  *			hrtimer enqueue; it is the total first expiry time (hard
191  *			and soft hrtimer are taken into account)
192  * @next_timer:		Pointer to the first expiring timer
193  * @softirq_expires_next: Time to check, if soft queues needs also to be expired
194  * @softirq_next_timer: Pointer to the first expiring softirq based timer
195  * @clock_base:		array of clock bases for this cpu
196  *
197  * Note: next_timer is just an optimization for __remove_hrtimer().
198  *	 Do not dereference the pointer because it is not reliable on
199  *	 cross cpu removals.
200  */
201 struct hrtimer_cpu_base {
202 	raw_spinlock_t			lock;
203 	unsigned int			cpu;
204 	unsigned int			active_bases;
205 	unsigned int			clock_was_set_seq;
206 	unsigned int			hres_active		: 1,
207 					in_hrtirq		: 1,
208 					hang_detected		: 1,
209 					softirq_activated       : 1;
210 #ifdef CONFIG_HIGH_RES_TIMERS
211 	unsigned int			nr_events;
212 	unsigned short			nr_retries;
213 	unsigned short			nr_hangs;
214 	unsigned int			max_hang_time;
215 #endif
216 	ktime_t				expires_next;
217 	struct hrtimer			*next_timer;
218 	ktime_t				softirq_expires_next;
219 	struct hrtimer			*softirq_next_timer;
220 	struct hrtimer_clock_base	clock_base[HRTIMER_MAX_CLOCK_BASES];
221 } ____cacheline_aligned;
222 
hrtimer_set_expires(struct hrtimer * timer,ktime_t time)223 static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
224 {
225 	timer->node.expires = time;
226 	timer->_softexpires = time;
227 }
228 
hrtimer_set_expires_range(struct hrtimer * timer,ktime_t time,ktime_t delta)229 static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta)
230 {
231 	timer->_softexpires = time;
232 	timer->node.expires = ktime_add_safe(time, delta);
233 }
234 
hrtimer_set_expires_range_ns(struct hrtimer * timer,ktime_t time,u64 delta)235 static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
236 {
237 	timer->_softexpires = time;
238 	timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
239 }
240 
hrtimer_set_expires_tv64(struct hrtimer * timer,s64 tv64)241 static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64)
242 {
243 	timer->node.expires = tv64;
244 	timer->_softexpires = tv64;
245 }
246 
hrtimer_add_expires(struct hrtimer * timer,ktime_t time)247 static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time)
248 {
249 	timer->node.expires = ktime_add_safe(timer->node.expires, time);
250 	timer->_softexpires = ktime_add_safe(timer->_softexpires, time);
251 }
252 
hrtimer_add_expires_ns(struct hrtimer * timer,u64 ns)253 static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns)
254 {
255 	timer->node.expires = ktime_add_ns(timer->node.expires, ns);
256 	timer->_softexpires = ktime_add_ns(timer->_softexpires, ns);
257 }
258 
hrtimer_get_expires(const struct hrtimer * timer)259 static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
260 {
261 	return timer->node.expires;
262 }
263 
hrtimer_get_softexpires(const struct hrtimer * timer)264 static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
265 {
266 	return timer->_softexpires;
267 }
268 
hrtimer_get_expires_tv64(const struct hrtimer * timer)269 static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer)
270 {
271 	return timer->node.expires;
272 }
hrtimer_get_softexpires_tv64(const struct hrtimer * timer)273 static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer)
274 {
275 	return timer->_softexpires;
276 }
277 
hrtimer_get_expires_ns(const struct hrtimer * timer)278 static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
279 {
280 	return ktime_to_ns(timer->node.expires);
281 }
282 
hrtimer_expires_remaining(const struct hrtimer * timer)283 static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
284 {
285 	return ktime_sub(timer->node.expires, timer->base->get_time());
286 }
287 
hrtimer_cb_get_time(struct hrtimer * timer)288 static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
289 {
290 	return timer->base->get_time();
291 }
292 
hrtimer_is_hres_active(struct hrtimer * timer)293 static inline int hrtimer_is_hres_active(struct hrtimer *timer)
294 {
295 	return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
296 		timer->base->cpu_base->hres_active : 0;
297 }
298 
299 #ifdef CONFIG_HIGH_RES_TIMERS
300 struct clock_event_device;
301 
302 extern void hrtimer_interrupt(struct clock_event_device *dev);
303 
304 /*
305  * The resolution of the clocks. The resolution value is returned in
306  * the clock_getres() system call to give application programmers an
307  * idea of the (in)accuracy of timers. Timer values are rounded up to
308  * this resolution values.
309  */
310 # define HIGH_RES_NSEC		1
311 # define KTIME_HIGH_RES		(HIGH_RES_NSEC)
312 # define MONOTONIC_RES_NSEC	HIGH_RES_NSEC
313 # define KTIME_MONOTONIC_RES	KTIME_HIGH_RES
314 
315 extern void clock_was_set_delayed(void);
316 
317 extern unsigned int hrtimer_resolution;
318 
319 #else
320 
321 # define MONOTONIC_RES_NSEC	LOW_RES_NSEC
322 # define KTIME_MONOTONIC_RES	KTIME_LOW_RES
323 
324 #define hrtimer_resolution	(unsigned int)LOW_RES_NSEC
325 
clock_was_set_delayed(void)326 static inline void clock_was_set_delayed(void) { }
327 
328 #endif
329 
330 static inline ktime_t
__hrtimer_expires_remaining_adjusted(const struct hrtimer * timer,ktime_t now)331 __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
332 {
333 	ktime_t rem = ktime_sub(timer->node.expires, now);
334 
335 	/*
336 	 * Adjust relative timers for the extra we added in
337 	 * hrtimer_start_range_ns() to prevent short timeouts.
338 	 */
339 	if (IS_ENABLED(CONFIG_TIME_LOW_RES) && timer->is_rel)
340 		rem -= hrtimer_resolution;
341 	return rem;
342 }
343 
344 static inline ktime_t
hrtimer_expires_remaining_adjusted(const struct hrtimer * timer)345 hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
346 {
347 	return __hrtimer_expires_remaining_adjusted(timer,
348 						    timer->base->get_time());
349 }
350 
351 extern void clock_was_set(void);
352 #ifdef CONFIG_TIMERFD
353 extern void timerfd_clock_was_set(void);
354 #else
timerfd_clock_was_set(void)355 static inline void timerfd_clock_was_set(void) { }
356 #endif
357 extern void hrtimers_resume(void);
358 
359 DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
360 
361 
362 /* Exported timer functions: */
363 
364 /* Initialize timers: */
365 extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
366 			 enum hrtimer_mode mode);
367 
368 #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
369 extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
370 				  enum hrtimer_mode mode);
371 
372 extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
373 #else
hrtimer_init_on_stack(struct hrtimer * timer,clockid_t which_clock,enum hrtimer_mode mode)374 static inline void hrtimer_init_on_stack(struct hrtimer *timer,
375 					 clockid_t which_clock,
376 					 enum hrtimer_mode mode)
377 {
378 	hrtimer_init(timer, which_clock, mode);
379 }
destroy_hrtimer_on_stack(struct hrtimer * timer)380 static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
381 #endif
382 
383 /* Basic timer operations: */
384 extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
385 				   u64 range_ns, const enum hrtimer_mode mode);
386 
387 /**
388  * hrtimer_start - (re)start an hrtimer
389  * @timer:	the timer to be added
390  * @tim:	expiry time
391  * @mode:	timer mode: absolute (HRTIMER_MODE_ABS) or
392  *		relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
393  *		softirq based mode is considered for debug purpose only!
394  */
hrtimer_start(struct hrtimer * timer,ktime_t tim,const enum hrtimer_mode mode)395 static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
396 				 const enum hrtimer_mode mode)
397 {
398 	hrtimer_start_range_ns(timer, tim, 0, mode);
399 }
400 
401 extern int hrtimer_cancel(struct hrtimer *timer);
402 extern int hrtimer_try_to_cancel(struct hrtimer *timer);
403 
hrtimer_start_expires(struct hrtimer * timer,enum hrtimer_mode mode)404 static inline void hrtimer_start_expires(struct hrtimer *timer,
405 					 enum hrtimer_mode mode)
406 {
407 	u64 delta;
408 	ktime_t soft, hard;
409 	soft = hrtimer_get_softexpires(timer);
410 	hard = hrtimer_get_expires(timer);
411 	delta = ktime_to_ns(ktime_sub(hard, soft));
412 	hrtimer_start_range_ns(timer, soft, delta, mode);
413 }
414 
hrtimer_restart(struct hrtimer * timer)415 static inline void hrtimer_restart(struct hrtimer *timer)
416 {
417 	hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
418 }
419 
420 /* Query timers: */
421 extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);
422 
hrtimer_get_remaining(const struct hrtimer * timer)423 static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
424 {
425 	return __hrtimer_get_remaining(timer, false);
426 }
427 
428 extern u64 hrtimer_get_next_event(void);
429 extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);
430 
431 extern bool hrtimer_active(const struct hrtimer *timer);
432 
433 /**
434  * hrtimer_is_queued = check, whether the timer is on one of the queues
435  * @timer:	Timer to check
436  *
437  * Returns: True if the timer is queued, false otherwise
438  *
439  * The function can be used lockless, but it gives only a current snapshot.
440  */
hrtimer_is_queued(struct hrtimer * timer)441 static inline bool hrtimer_is_queued(struct hrtimer *timer)
442 {
443 	/* The READ_ONCE pairs with the update functions of timer->state */
444 	return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED);
445 }
446 
447 /*
448  * Helper function to check, whether the timer is running the callback
449  * function
450  */
hrtimer_callback_running(struct hrtimer * timer)451 static inline int hrtimer_callback_running(struct hrtimer *timer)
452 {
453 	return timer->base->running == timer;
454 }
455 
456 /* Forward a hrtimer so it expires after now: */
457 extern u64
458 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
459 
460 /**
461  * hrtimer_forward_now - forward the timer expiry so it expires after now
462  * @timer:	hrtimer to forward
463  * @interval:	the interval to forward
464  *
465  * Forward the timer expiry so it will expire after the current time
466  * of the hrtimer clock base. Returns the number of overruns.
467  *
468  * Can be safely called from the callback function of @timer. If
469  * called from other contexts @timer must neither be enqueued nor
470  * running the callback and the caller needs to take care of
471  * serialization.
472  *
473  * Note: This only updates the timer expiry value and does not requeue
474  * the timer.
475  */
hrtimer_forward_now(struct hrtimer * timer,ktime_t interval)476 static inline u64 hrtimer_forward_now(struct hrtimer *timer,
477 				      ktime_t interval)
478 {
479 	return hrtimer_forward(timer, timer->base->get_time(), interval);
480 }
481 
482 /* Precise sleep: */
483 
484 extern int nanosleep_copyout(struct restart_block *, struct timespec64 *);
485 extern long hrtimer_nanosleep(const struct timespec64 *rqtp,
486 			      const enum hrtimer_mode mode,
487 			      const clockid_t clockid);
488 
489 extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
490 				 struct task_struct *tsk);
491 
492 extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
493 						const enum hrtimer_mode mode);
494 extern int schedule_hrtimeout_range_clock(ktime_t *expires,
495 					  u64 delta,
496 					  const enum hrtimer_mode mode,
497 					  clockid_t clock_id);
498 extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
499 
500 /* Soft interrupt function to run the hrtimer queues: */
501 extern void hrtimer_run_queues(void);
502 
503 /* Bootup initialization: */
504 extern void __init hrtimers_init(void);
505 
506 /* Show pending timers: */
507 extern void sysrq_timer_list_show(void);
508 
509 int hrtimers_prepare_cpu(unsigned int cpu);
510 #ifdef CONFIG_HOTPLUG_CPU
511 int hrtimers_dead_cpu(unsigned int cpu);
512 #else
513 #define hrtimers_dead_cpu	NULL
514 #endif
515 
516 #endif
517