xref: /wlan-driver/qca-wifi-host-cmn/hif/src/hif_irq_affinity.c (revision 5113495b16420b49004c444715d2daae2066e7dc)
1 /*
2  * Copyright (c) 2015-2021 The Linux Foundation. All rights reserved.
3  * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
4  *
5  * Permission to use, copy, modify, and/or distribute this software for
6  * any purpose with or without fee is hereby granted, provided that the
7  * above copyright notice and this permission notice appear in all
8  * copies.
9  *
10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
11  * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
12  * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
13  * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
14  * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
15  * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
16  * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
17  * PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 /**
21  * DOC: hif_irq_affinity.c
22  *
23  * This irq affinity implementation is os dependent, so this can be treated as
24  * an abstraction layer...  Should this be moved into a /linux folder?
25  */
26 
27 #include <linux/string.h> /* memset */
28 
29 /* Linux headers */
30 #include <linux/cpumask.h>
31 #include <linux/cpufreq.h>
32 #include <linux/cpu.h>
33 #include <linux/topology.h>
34 #include <linux/interrupt.h>
35 #include <linux/pm.h>
36 #include <hif_napi.h>
37 #include <hif_irq_affinity.h>
38 #include <hif_exec.h>
39 #include <hif_main.h>
40 #include "qdf_irq.h"
41 
42 #if defined(FEATURE_NAPI_DEBUG) && defined(HIF_IRQ_AFFINITY)
43 /*
44  * Local functions
45  * - no argument checks, all internal/trusted callers
46  */
hnc_dump_cpus(struct qca_napi_data * napid)47 static void hnc_dump_cpus(struct qca_napi_data *napid)
48 {
49 	hif_napi_stats(napid);
50 }
51 #else
hnc_dump_cpus(struct qca_napi_data * napid)52 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ };
53 #endif /* FEATURE_NAPI_DEBUG */
54 
55 #ifdef HIF_IRQ_AFFINITY
56 /**
57  * hif_exec_event() - reacts to events that impact irq affinity
58  * @hif_ctx: pointer to hif context
59  * @event: event that has been detected
60  * @data: more data regarding the event
61  *
62  * Description:
63  *   This function handles two types of events:
64  *   1- Events that change the state of NAPI (enabled/disabled):
65  *      {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE}
66  *      The state is retrievable by "hdd_napi_enabled(-1)"
67  *    - NAPI will be on if either INI file is on and it has not been disabled
68  *                                by a subsequent vendor CMD,
69  *                         or     it has been enabled by a vendor CMD.
70  *   2- Events that change the CPU affinity of a NAPI instance/IRQ:
71  *      {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE}
72  *    - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode
73  *    - NAPI will switch throughput mode based on hdd_napi_throughput_policy()
74  *    - In LO tput mode, NAPI will yield control if its interrupts to the system
75  *      management functions. However in HI throughput mode, NAPI will actively
76  *      manage its interrupts/instances (by trying to disperse them out to
77  *      separate performance cores).
78  *    - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events.
79  *
80  *    + In some cases (roaming peer management is the only case so far), a
81  *      a client can trigger a "SERIALIZE" event. Basically, this means that the
82  *      users is asking NAPI to go into a truly single execution context state.
83  *      So, NAPI indicates to msm-irqbalancer that it wants to be denylisted,
84  *      (if called for the first time) and then moves all IRQs (for NAPI
85  *      instances) to be collapsed to a single core. If called multiple times,
86  *      it will just re-collapse the CPUs. This is because denylist-on() API
87  *      is reference-counted, and because the API has already been called.
88  *
89  *      Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go
90  *      to its "normal" operation. Optionally, they can give a timeout value (in
91  *      multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this
92  *      case, NAPI will just set the current throughput state to uninitialized
93  *      and set the delay period. Once policy handler is called, it would skip
94  *      applying the policy delay period times, and otherwise apply the policy.
95  *
96  * Return:
97  *  < 0: some error
98  *  = 0: event handled successfully
99  */
hif_exec_event(struct hif_opaque_softc * hif_ctx,enum qca_napi_event event,void * data)100 int hif_exec_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event,
101 		   void *data)
102 {
103 	int      rc = 0;
104 	uint32_t prev_state;
105 	struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
106 	struct qca_napi_data *napid = &(hif->napi_data);
107 	enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED;
108 	enum {
109 		DENYLIST_NOT_PENDING,
110 		DENYLIST_ON_PENDING,
111 		DENYLIST_OFF_PENDING
112 	     } denylist_pending = DENYLIST_NOT_PENDING;
113 
114 	NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data);
115 
116 	qdf_spin_lock_bh(&(napid->lock));
117 	prev_state = napid->state;
118 	switch (event) {
119 	case NAPI_EVT_INI_FILE:
120 	case NAPI_EVT_CMD_STATE:
121 	case NAPI_EVT_INT_STATE:
122 		/* deprecated */
123 		break;
124 
125 	case NAPI_EVT_CPU_STATE: {
126 		int cpu = ((unsigned long int)data >> 16);
127 		int val = ((unsigned long int)data & 0x0ff);
128 
129 		NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d",
130 			   __func__, cpu, val);
131 
132 		/* state has already been set by hnc_cpu_notify_cb */
133 		if ((val == QCA_NAPI_CPU_DOWN) &&
134 		    (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */
135 		    (napid->napi_cpu[cpu].napis != 0)) {
136 			NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d",
137 				   __func__, cpu);
138 			rc = hif_exec_cpu_migrate(napid,
139 						  cpu,
140 						  HNC_ACT_RELOCATE);
141 			napid->napi_cpu[cpu].napis = 0;
142 		}
143 		/* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */
144 		break;
145 	}
146 
147 	case NAPI_EVT_TPUT_STATE: {
148 		tput_mode = (enum qca_napi_tput_state)data;
149 		if (tput_mode == QCA_NAPI_TPUT_LO) {
150 			/* from TPUT_HI -> TPUT_LO */
151 			NAPI_DEBUG("%s: Moving to napi_tput_LO state",
152 				   __func__);
153 			denylist_pending = DENYLIST_OFF_PENDING;
154 			/*
155 			 * Ideally we should "collapse" interrupts here, since
156 			 * we are "dispersing" interrupts in the "else" case.
157 			 * This allows the possibility that our interrupts may
158 			 * still be on the perf cluster the next time we enter
159 			 * high tput mode. However, the irq_balancer is free
160 			 * to move our interrupts to power cluster once
161 			 * denylisting has been turned off in the "else" case.
162 			 */
163 		} else {
164 			/* from TPUT_LO -> TPUT->HI */
165 			NAPI_DEBUG("%s: Moving to napi_tput_HI state",
166 				   __func__);
167 			rc = hif_exec_cpu_migrate(napid,
168 						  HNC_ANY_CPU,
169 						  HNC_ACT_DISPERSE);
170 
171 			denylist_pending = DENYLIST_ON_PENDING;
172 		}
173 		napid->napi_mode = tput_mode;
174 		break;
175 	}
176 
177 	case NAPI_EVT_USR_SERIAL: {
178 		unsigned long users = (unsigned long)data;
179 
180 		NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld",
181 			   __func__, users);
182 
183 		rc = hif_exec_cpu_migrate(napid,
184 					  HNC_ANY_CPU,
185 					  HNC_ACT_COLLAPSE);
186 		if ((users == 0) && (rc == 0))
187 			denylist_pending = DENYLIST_ON_PENDING;
188 		break;
189 	}
190 	case NAPI_EVT_USR_NORMAL: {
191 		NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__);
192 		if (!napid->user_cpu_affin_mask)
193 			denylist_pending = DENYLIST_OFF_PENDING;
194 		/*
195 		 * Deserialization timeout is handled at hdd layer;
196 		 * just mark current mode to uninitialized to ensure
197 		 * it will be set when the delay is over
198 		 */
199 		napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED;
200 		break;
201 	}
202 	default: {
203 		hif_err("Unknown event: %d (data=0x%0lx)",
204 			event, (unsigned long) data);
205 		break;
206 	} /* default */
207 	}; /* switch */
208 
209 
210 	switch (denylist_pending) {
211 	case DENYLIST_ON_PENDING:
212 		/* assume the control of WLAN IRQs */
213 		hif_napi_cpu_denylist(napid, DENYLIST_ON);
214 		break;
215 	case DENYLIST_OFF_PENDING:
216 		/* yield the control of WLAN IRQs */
217 		hif_napi_cpu_denylist(napid, DENYLIST_OFF);
218 		break;
219 	default: /* nothing to do */
220 		break;
221 	} /* switch denylist_pending */
222 
223 	qdf_spin_unlock_bh(&(napid->lock));
224 
225 	NAPI_DEBUG("<--[rc=%d]", rc);
226 	return rc;
227 }
228 
229 #endif
230 
231 /**
232  * hncm_exec_migrate_to() - migrates a NAPI to a CPU
233  * @napid: pointer to NAPI block
234  * @ctx_id: CE_id of the NAPI instance
235  * @didx: index in the CPU topology table for the CPU to migrate to
236  *
237  * Migrates NAPI (identified by the CE_id) to the destination core
238  * Updates the napi_map of the destination entry
239  *
240  * Return:
241  *  =0 : success
242  *  <0 : error
243  */
hncm_exec_migrate_to(struct qca_napi_data * napid,uint8_t ctx_id,int didx)244 static int hncm_exec_migrate_to(struct qca_napi_data *napid, uint8_t ctx_id,
245 				int didx)
246 {
247 	struct hif_exec_context *exec_ctx;
248 	struct qdf_cpu_mask *cpumask;
249 	int rc = 0;
250 	int status = 0;
251 	int ind;
252 
253 	NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, ctx_id, didx);
254 
255 	exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, ctx_id);
256 	if (!exec_ctx)
257 		return -EINVAL;
258 
259 	exec_ctx->cpumask.bits[0] = (1 << didx);
260 
261 	for (ind = 0; ind < exec_ctx->numirq; ind++) {
262 		if (exec_ctx->os_irq[ind]) {
263 			qdf_dev_modify_irq_status(exec_ctx->os_irq[ind],
264 						  QDF_IRQ_NO_BALANCING, 0);
265 			cpumask = (struct qdf_cpu_mask *)&exec_ctx->cpumask;
266 			rc = qdf_dev_set_irq_affinity(exec_ctx->os_irq[ind],
267 						      cpumask);
268 			if (rc)
269 				status = rc;
270 		}
271 	}
272 
273 	/* unmark the napis bitmap in the cpu table */
274 	napid->napi_cpu[exec_ctx->cpu].napis &= ~(0x01 << ctx_id);
275 	/* mark the napis bitmap for the new designated cpu */
276 	napid->napi_cpu[didx].napis |= (0x01 << ctx_id);
277 	exec_ctx->cpu = didx;
278 
279 	NAPI_DEBUG("<--%s[%d]", __func__, rc);
280 	return status;
281 }
282 
283 /**
284  * hncm_dest_cpu() - finds a destination CPU for NAPI
285  * @napid: pointer to NAPI block
286  * @act  : RELOCATE | COLLAPSE | DISPERSE
287  *
288  * Finds the designated destination for the next IRQ.
289  * RELOCATE: translated to either COLLAPSE or DISPERSE based
290  *           on napid->napi_mode (throughput state)
291  * COLLAPSE: All have the same destination: the first online CPU in lilcl
292  * DISPERSE: One of the CPU in bigcl, which has the smallest number of
293  *           NAPIs on it
294  *
295  * Return: >=0 : index in the cpu topology table
296  *       : < 0 : error
297  */
hncm_dest_cpu(struct qca_napi_data * napid,int act)298 static int hncm_dest_cpu(struct qca_napi_data *napid, int act)
299 {
300 	int destidx = -1;
301 	int head, i;
302 
303 	NAPI_DEBUG("-->%s(act=%d)", __func__, act);
304 	if (act == HNC_ACT_RELOCATE) {
305 		if (napid->napi_mode == QCA_NAPI_TPUT_LO)
306 			act = HNC_ACT_COLLAPSE;
307 		else
308 			act = HNC_ACT_DISPERSE;
309 		NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d",
310 			   __func__, act);
311 	}
312 	if (act == HNC_ACT_COLLAPSE) {
313 		head = i = napid->lilcl_head;
314 retry_collapse:
315 		while (i >= 0) {
316 			if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) {
317 				destidx = i;
318 				break;
319 			}
320 			i = napid->napi_cpu[i].cluster_nxt;
321 		}
322 		if ((destidx < 0) && (head == napid->lilcl_head)) {
323 			NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl",
324 				__func__);
325 			head = i = napid->bigcl_head;
326 			goto retry_collapse;
327 		}
328 	} else { /* HNC_ACT_DISPERSE */
329 		int smallest = 99; /* all 32 bits full */
330 		int smallidx = -1;
331 
332 		head = i = napid->bigcl_head;
333 retry_disperse:
334 		while (i >= 0) {
335 			if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) &&
336 			    (hweight32(napid->napi_cpu[i].napis) <= smallest)) {
337 				smallest = napid->napi_cpu[i].napis;
338 				smallidx = i;
339 			}
340 			i = napid->napi_cpu[i].cluster_nxt;
341 		}
342 		destidx = smallidx;
343 		if ((destidx < 0) && (head == napid->bigcl_head)) {
344 			NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl",
345 				__func__);
346 			head = i = napid->lilcl_head;
347 			goto retry_disperse;
348 		}
349 	}
350 	NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx);
351 	return destidx;
352 }
353 
354 /**
355  * hif_exec_cpu_migrate() - migrate IRQs away
356  * @napid: pointer to qca_napi_data structure
357  * @cpu: -1: all CPUs <n> specific CPU
358  * @action: COLLAPSE | DISPERSE
359  *
360  * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible
361  * cores. Eligible cores are:
362  * act=COLLAPSE -> the first online core of the little cluster
363  * act=DISPERSE -> separate cores of the big cluster, so that each core will
364  *                 host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis)
365  *
366  * Note that this function is called with a spinlock acquired already.
367  *
368  * Return: =0: success
369  *         <0: error
370  */
hif_exec_cpu_migrate(struct qca_napi_data * napid,int cpu,int action)371 int hif_exec_cpu_migrate(struct qca_napi_data *napid, int cpu, int action)
372 {
373 	int      rc = 0;
374 	struct qca_napi_cpu *cpup;
375 	int      i, dind;
376 	uint32_t napis;
377 
378 
379 	NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)",
380 		   __func__, cpu, action);
381 
382 	if (napid->exec_map == 0) {
383 		NAPI_DEBUG("%s: datapath contexts to disperse", __func__);
384 		goto hncm_return;
385 	}
386 	cpup = napid->napi_cpu;
387 
388 	switch (action) {
389 	case HNC_ACT_RELOCATE:
390 	case HNC_ACT_DISPERSE:
391 	case HNC_ACT_COLLAPSE: {
392 		/* first find the src napi set */
393 		if (cpu == HNC_ANY_CPU)
394 			napis = napid->exec_map;
395 		else
396 			napis = cpup[cpu].napis;
397 		/* then clear the napi bitmap on each CPU */
398 		for (i = 0; i < NR_CPUS; i++)
399 			cpup[i].napis = 0;
400 		/* then for each of the NAPIs to disperse: */
401 		for (i = 0; i < HIF_MAX_GROUP; i++)
402 			if (napis & (1 << i)) {
403 				/* find a destination CPU */
404 				dind = hncm_dest_cpu(napid, action);
405 				if (dind >= 0) {
406 					rc = hncm_exec_migrate_to(napid, i,
407 								  dind);
408 				} else {
409 					NAPI_DEBUG("No dest for NAPI ce%d", i);
410 					hnc_dump_cpus(napid);
411 					rc = -1;
412 				}
413 			}
414 		break;
415 	}
416 	default: {
417 		NAPI_DEBUG("%s: bad action: %d\n", __func__, action);
418 		QDF_BUG(0);
419 		break;
420 	}
421 	} /* switch action */
422 
423 hncm_return:
424 	hnc_dump_cpus(napid);
425 	return rc;
426 }
427 
428 
429 /**
430  * hif_exec_dl_irq() - calls irq_modify_status to enable/disable denylisting
431  * @napid: pointer to qca_napi_data structure
432  * @dl_flag: denylist flag to enable/disable denylisting
433  *
434  * The function enables/disables denylisting for all the copy engine
435  * interrupts on which NAPI is enabled.
436  *
437  * Return: None
438  */
hif_exec_dl_irq(struct qca_napi_data * napid,bool dl_flag)439 static inline void hif_exec_dl_irq(struct qca_napi_data *napid, bool dl_flag)
440 {
441 	int i, j;
442 	struct hif_exec_context *exec_ctx;
443 
444 	for (i = 0; i < HIF_MAX_GROUP; i++) {
445 		/* check if NAPI is enabled on the CE */
446 		if (!(napid->exec_map & (0x01 << i)))
447 			continue;
448 
449 		/*double check that NAPI is allocated for the CE */
450 		exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, i);
451 		if (!(exec_ctx))
452 			continue;
453 
454 		if (dl_flag == true)
455 			for (j = 0; j < exec_ctx->numirq; j++)
456 				qdf_dev_modify_irq_status(exec_ctx->os_irq[j],
457 							  0,
458 							  QDF_IRQ_NO_BALANCING);
459 		else
460 			for (j = 0; j < exec_ctx->numirq; j++)
461 				qdf_dev_modify_irq_status(exec_ctx->os_irq[j],
462 							  QDF_IRQ_NO_BALANCING,
463 							  0);
464 		hif_debug("dl_flag %d CE %d", dl_flag, i);
465 	}
466 }
467 
468 /**
469  * hif_exec_cpu_denylist() - en(dis)ables denylisting for NAPI RX interrupts.
470  * @napid: pointer to qca_napi_data structure
471  * @op: denylist operation to perform
472  *
473  * The function enables/disables/queries denylisting for all CE RX
474  * interrupts with NAPI enabled. Besides denylisting, it also enables/disables
475  * core_ctl_set_boost.
476  * Once denylisting is enabled, the interrupts will not be managed by the IRQ
477  * balancer.
478  *
479  * Return: -EINVAL, in case IRQ_DENYLISTING and CORE_CTL_BOOST is not enabled
480  *         for DENYLIST_QUERY op - denylist refcount
481  *         for DENYLIST_ON op    - return value from core_ctl_set_boost API
482  *         for DENYLIST_OFF op   - return value from core_ctl_set_boost API
483  */
hif_exec_cpu_denylist(struct qca_napi_data * napid,enum qca_denylist_op op)484 int hif_exec_cpu_denylist(struct qca_napi_data *napid,
485 			  enum qca_denylist_op op)
486 {
487 	int rc = 0;
488 	static int ref_count; /* = 0 by the compiler */
489 	uint8_t flags = napid->flags;
490 	bool dl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING;
491 	bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST;
492 
493 	NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op);
494 
495 	if (!(dl_en && ccb_en)) {
496 		rc = -EINVAL;
497 		goto out;
498 	}
499 
500 	switch (op) {
501 	case DENYLIST_QUERY:
502 		rc = ref_count;
503 		break;
504 	case DENYLIST_ON:
505 		ref_count++;
506 		rc = 0;
507 		if (ref_count == 1) {
508 			rc = hif_napi_core_ctl_set_boost(true);
509 			NAPI_DEBUG("boost_on() returns %d - refcnt=%d",
510 				rc, ref_count);
511 			hif_exec_dl_irq(napid, true);
512 		}
513 		break;
514 	case DENYLIST_OFF:
515 		if (ref_count)
516 			ref_count--;
517 		rc = 0;
518 		if (ref_count == 0) {
519 			rc = hif_napi_core_ctl_set_boost(false);
520 			NAPI_DEBUG("boost_off() returns %d - refcnt=%d",
521 				   rc, ref_count);
522 			hif_exec_dl_irq(napid, false);
523 		}
524 		break;
525 	default:
526 		NAPI_DEBUG("Invalid denylist op: %d", op);
527 		rc = -EINVAL;
528 	} /* switch */
529 out:
530 	NAPI_DEBUG("<--%s[%d]", __func__, rc);
531 	return rc;
532 }
533 
534