1 /*
2 * Copyright (c) 2015-2021 The Linux Foundation. All rights reserved.
3 * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
4 *
5 * Permission to use, copy, modify, and/or distribute this software for
6 * any purpose with or without fee is hereby granted, provided that the
7 * above copyright notice and this permission notice appear in all
8 * copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
17 * PERFORMANCE OF THIS SOFTWARE.
18 */
19
20 /**
21 * DOC: hif_irq_affinity.c
22 *
23 * This irq affinity implementation is os dependent, so this can be treated as
24 * an abstraction layer... Should this be moved into a /linux folder?
25 */
26
27 #include <linux/string.h> /* memset */
28
29 /* Linux headers */
30 #include <linux/cpumask.h>
31 #include <linux/cpufreq.h>
32 #include <linux/cpu.h>
33 #include <linux/topology.h>
34 #include <linux/interrupt.h>
35 #include <linux/pm.h>
36 #include <hif_napi.h>
37 #include <hif_irq_affinity.h>
38 #include <hif_exec.h>
39 #include <hif_main.h>
40 #include "qdf_irq.h"
41
42 #if defined(FEATURE_NAPI_DEBUG) && defined(HIF_IRQ_AFFINITY)
43 /*
44 * Local functions
45 * - no argument checks, all internal/trusted callers
46 */
hnc_dump_cpus(struct qca_napi_data * napid)47 static void hnc_dump_cpus(struct qca_napi_data *napid)
48 {
49 hif_napi_stats(napid);
50 }
51 #else
hnc_dump_cpus(struct qca_napi_data * napid)52 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ };
53 #endif /* FEATURE_NAPI_DEBUG */
54
55 #ifdef HIF_IRQ_AFFINITY
56 /**
57 * hif_exec_event() - reacts to events that impact irq affinity
58 * @hif_ctx: pointer to hif context
59 * @event: event that has been detected
60 * @data: more data regarding the event
61 *
62 * Description:
63 * This function handles two types of events:
64 * 1- Events that change the state of NAPI (enabled/disabled):
65 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE}
66 * The state is retrievable by "hdd_napi_enabled(-1)"
67 * - NAPI will be on if either INI file is on and it has not been disabled
68 * by a subsequent vendor CMD,
69 * or it has been enabled by a vendor CMD.
70 * 2- Events that change the CPU affinity of a NAPI instance/IRQ:
71 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE}
72 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode
73 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy()
74 * - In LO tput mode, NAPI will yield control if its interrupts to the system
75 * management functions. However in HI throughput mode, NAPI will actively
76 * manage its interrupts/instances (by trying to disperse them out to
77 * separate performance cores).
78 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events.
79 *
80 * + In some cases (roaming peer management is the only case so far), a
81 * a client can trigger a "SERIALIZE" event. Basically, this means that the
82 * users is asking NAPI to go into a truly single execution context state.
83 * So, NAPI indicates to msm-irqbalancer that it wants to be denylisted,
84 * (if called for the first time) and then moves all IRQs (for NAPI
85 * instances) to be collapsed to a single core. If called multiple times,
86 * it will just re-collapse the CPUs. This is because denylist-on() API
87 * is reference-counted, and because the API has already been called.
88 *
89 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go
90 * to its "normal" operation. Optionally, they can give a timeout value (in
91 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this
92 * case, NAPI will just set the current throughput state to uninitialized
93 * and set the delay period. Once policy handler is called, it would skip
94 * applying the policy delay period times, and otherwise apply the policy.
95 *
96 * Return:
97 * < 0: some error
98 * = 0: event handled successfully
99 */
hif_exec_event(struct hif_opaque_softc * hif_ctx,enum qca_napi_event event,void * data)100 int hif_exec_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event,
101 void *data)
102 {
103 int rc = 0;
104 uint32_t prev_state;
105 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
106 struct qca_napi_data *napid = &(hif->napi_data);
107 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED;
108 enum {
109 DENYLIST_NOT_PENDING,
110 DENYLIST_ON_PENDING,
111 DENYLIST_OFF_PENDING
112 } denylist_pending = DENYLIST_NOT_PENDING;
113
114 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data);
115
116 qdf_spin_lock_bh(&(napid->lock));
117 prev_state = napid->state;
118 switch (event) {
119 case NAPI_EVT_INI_FILE:
120 case NAPI_EVT_CMD_STATE:
121 case NAPI_EVT_INT_STATE:
122 /* deprecated */
123 break;
124
125 case NAPI_EVT_CPU_STATE: {
126 int cpu = ((unsigned long int)data >> 16);
127 int val = ((unsigned long int)data & 0x0ff);
128
129 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d",
130 __func__, cpu, val);
131
132 /* state has already been set by hnc_cpu_notify_cb */
133 if ((val == QCA_NAPI_CPU_DOWN) &&
134 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */
135 (napid->napi_cpu[cpu].napis != 0)) {
136 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d",
137 __func__, cpu);
138 rc = hif_exec_cpu_migrate(napid,
139 cpu,
140 HNC_ACT_RELOCATE);
141 napid->napi_cpu[cpu].napis = 0;
142 }
143 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */
144 break;
145 }
146
147 case NAPI_EVT_TPUT_STATE: {
148 tput_mode = (enum qca_napi_tput_state)data;
149 if (tput_mode == QCA_NAPI_TPUT_LO) {
150 /* from TPUT_HI -> TPUT_LO */
151 NAPI_DEBUG("%s: Moving to napi_tput_LO state",
152 __func__);
153 denylist_pending = DENYLIST_OFF_PENDING;
154 /*
155 * Ideally we should "collapse" interrupts here, since
156 * we are "dispersing" interrupts in the "else" case.
157 * This allows the possibility that our interrupts may
158 * still be on the perf cluster the next time we enter
159 * high tput mode. However, the irq_balancer is free
160 * to move our interrupts to power cluster once
161 * denylisting has been turned off in the "else" case.
162 */
163 } else {
164 /* from TPUT_LO -> TPUT->HI */
165 NAPI_DEBUG("%s: Moving to napi_tput_HI state",
166 __func__);
167 rc = hif_exec_cpu_migrate(napid,
168 HNC_ANY_CPU,
169 HNC_ACT_DISPERSE);
170
171 denylist_pending = DENYLIST_ON_PENDING;
172 }
173 napid->napi_mode = tput_mode;
174 break;
175 }
176
177 case NAPI_EVT_USR_SERIAL: {
178 unsigned long users = (unsigned long)data;
179
180 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld",
181 __func__, users);
182
183 rc = hif_exec_cpu_migrate(napid,
184 HNC_ANY_CPU,
185 HNC_ACT_COLLAPSE);
186 if ((users == 0) && (rc == 0))
187 denylist_pending = DENYLIST_ON_PENDING;
188 break;
189 }
190 case NAPI_EVT_USR_NORMAL: {
191 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__);
192 if (!napid->user_cpu_affin_mask)
193 denylist_pending = DENYLIST_OFF_PENDING;
194 /*
195 * Deserialization timeout is handled at hdd layer;
196 * just mark current mode to uninitialized to ensure
197 * it will be set when the delay is over
198 */
199 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED;
200 break;
201 }
202 default: {
203 hif_err("Unknown event: %d (data=0x%0lx)",
204 event, (unsigned long) data);
205 break;
206 } /* default */
207 }; /* switch */
208
209
210 switch (denylist_pending) {
211 case DENYLIST_ON_PENDING:
212 /* assume the control of WLAN IRQs */
213 hif_napi_cpu_denylist(napid, DENYLIST_ON);
214 break;
215 case DENYLIST_OFF_PENDING:
216 /* yield the control of WLAN IRQs */
217 hif_napi_cpu_denylist(napid, DENYLIST_OFF);
218 break;
219 default: /* nothing to do */
220 break;
221 } /* switch denylist_pending */
222
223 qdf_spin_unlock_bh(&(napid->lock));
224
225 NAPI_DEBUG("<--[rc=%d]", rc);
226 return rc;
227 }
228
229 #endif
230
231 /**
232 * hncm_exec_migrate_to() - migrates a NAPI to a CPU
233 * @napid: pointer to NAPI block
234 * @ctx_id: CE_id of the NAPI instance
235 * @didx: index in the CPU topology table for the CPU to migrate to
236 *
237 * Migrates NAPI (identified by the CE_id) to the destination core
238 * Updates the napi_map of the destination entry
239 *
240 * Return:
241 * =0 : success
242 * <0 : error
243 */
hncm_exec_migrate_to(struct qca_napi_data * napid,uint8_t ctx_id,int didx)244 static int hncm_exec_migrate_to(struct qca_napi_data *napid, uint8_t ctx_id,
245 int didx)
246 {
247 struct hif_exec_context *exec_ctx;
248 struct qdf_cpu_mask *cpumask;
249 int rc = 0;
250 int status = 0;
251 int ind;
252
253 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, ctx_id, didx);
254
255 exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, ctx_id);
256 if (!exec_ctx)
257 return -EINVAL;
258
259 exec_ctx->cpumask.bits[0] = (1 << didx);
260
261 for (ind = 0; ind < exec_ctx->numirq; ind++) {
262 if (exec_ctx->os_irq[ind]) {
263 qdf_dev_modify_irq_status(exec_ctx->os_irq[ind],
264 QDF_IRQ_NO_BALANCING, 0);
265 cpumask = (struct qdf_cpu_mask *)&exec_ctx->cpumask;
266 rc = qdf_dev_set_irq_affinity(exec_ctx->os_irq[ind],
267 cpumask);
268 if (rc)
269 status = rc;
270 }
271 }
272
273 /* unmark the napis bitmap in the cpu table */
274 napid->napi_cpu[exec_ctx->cpu].napis &= ~(0x01 << ctx_id);
275 /* mark the napis bitmap for the new designated cpu */
276 napid->napi_cpu[didx].napis |= (0x01 << ctx_id);
277 exec_ctx->cpu = didx;
278
279 NAPI_DEBUG("<--%s[%d]", __func__, rc);
280 return status;
281 }
282
283 /**
284 * hncm_dest_cpu() - finds a destination CPU for NAPI
285 * @napid: pointer to NAPI block
286 * @act : RELOCATE | COLLAPSE | DISPERSE
287 *
288 * Finds the designated destination for the next IRQ.
289 * RELOCATE: translated to either COLLAPSE or DISPERSE based
290 * on napid->napi_mode (throughput state)
291 * COLLAPSE: All have the same destination: the first online CPU in lilcl
292 * DISPERSE: One of the CPU in bigcl, which has the smallest number of
293 * NAPIs on it
294 *
295 * Return: >=0 : index in the cpu topology table
296 * : < 0 : error
297 */
hncm_dest_cpu(struct qca_napi_data * napid,int act)298 static int hncm_dest_cpu(struct qca_napi_data *napid, int act)
299 {
300 int destidx = -1;
301 int head, i;
302
303 NAPI_DEBUG("-->%s(act=%d)", __func__, act);
304 if (act == HNC_ACT_RELOCATE) {
305 if (napid->napi_mode == QCA_NAPI_TPUT_LO)
306 act = HNC_ACT_COLLAPSE;
307 else
308 act = HNC_ACT_DISPERSE;
309 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d",
310 __func__, act);
311 }
312 if (act == HNC_ACT_COLLAPSE) {
313 head = i = napid->lilcl_head;
314 retry_collapse:
315 while (i >= 0) {
316 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) {
317 destidx = i;
318 break;
319 }
320 i = napid->napi_cpu[i].cluster_nxt;
321 }
322 if ((destidx < 0) && (head == napid->lilcl_head)) {
323 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl",
324 __func__);
325 head = i = napid->bigcl_head;
326 goto retry_collapse;
327 }
328 } else { /* HNC_ACT_DISPERSE */
329 int smallest = 99; /* all 32 bits full */
330 int smallidx = -1;
331
332 head = i = napid->bigcl_head;
333 retry_disperse:
334 while (i >= 0) {
335 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) &&
336 (hweight32(napid->napi_cpu[i].napis) <= smallest)) {
337 smallest = napid->napi_cpu[i].napis;
338 smallidx = i;
339 }
340 i = napid->napi_cpu[i].cluster_nxt;
341 }
342 destidx = smallidx;
343 if ((destidx < 0) && (head == napid->bigcl_head)) {
344 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl",
345 __func__);
346 head = i = napid->lilcl_head;
347 goto retry_disperse;
348 }
349 }
350 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx);
351 return destidx;
352 }
353
354 /**
355 * hif_exec_cpu_migrate() - migrate IRQs away
356 * @napid: pointer to qca_napi_data structure
357 * @cpu: -1: all CPUs <n> specific CPU
358 * @action: COLLAPSE | DISPERSE
359 *
360 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible
361 * cores. Eligible cores are:
362 * act=COLLAPSE -> the first online core of the little cluster
363 * act=DISPERSE -> separate cores of the big cluster, so that each core will
364 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis)
365 *
366 * Note that this function is called with a spinlock acquired already.
367 *
368 * Return: =0: success
369 * <0: error
370 */
hif_exec_cpu_migrate(struct qca_napi_data * napid,int cpu,int action)371 int hif_exec_cpu_migrate(struct qca_napi_data *napid, int cpu, int action)
372 {
373 int rc = 0;
374 struct qca_napi_cpu *cpup;
375 int i, dind;
376 uint32_t napis;
377
378
379 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)",
380 __func__, cpu, action);
381
382 if (napid->exec_map == 0) {
383 NAPI_DEBUG("%s: datapath contexts to disperse", __func__);
384 goto hncm_return;
385 }
386 cpup = napid->napi_cpu;
387
388 switch (action) {
389 case HNC_ACT_RELOCATE:
390 case HNC_ACT_DISPERSE:
391 case HNC_ACT_COLLAPSE: {
392 /* first find the src napi set */
393 if (cpu == HNC_ANY_CPU)
394 napis = napid->exec_map;
395 else
396 napis = cpup[cpu].napis;
397 /* then clear the napi bitmap on each CPU */
398 for (i = 0; i < NR_CPUS; i++)
399 cpup[i].napis = 0;
400 /* then for each of the NAPIs to disperse: */
401 for (i = 0; i < HIF_MAX_GROUP; i++)
402 if (napis & (1 << i)) {
403 /* find a destination CPU */
404 dind = hncm_dest_cpu(napid, action);
405 if (dind >= 0) {
406 rc = hncm_exec_migrate_to(napid, i,
407 dind);
408 } else {
409 NAPI_DEBUG("No dest for NAPI ce%d", i);
410 hnc_dump_cpus(napid);
411 rc = -1;
412 }
413 }
414 break;
415 }
416 default: {
417 NAPI_DEBUG("%s: bad action: %d\n", __func__, action);
418 QDF_BUG(0);
419 break;
420 }
421 } /* switch action */
422
423 hncm_return:
424 hnc_dump_cpus(napid);
425 return rc;
426 }
427
428
429 /**
430 * hif_exec_dl_irq() - calls irq_modify_status to enable/disable denylisting
431 * @napid: pointer to qca_napi_data structure
432 * @dl_flag: denylist flag to enable/disable denylisting
433 *
434 * The function enables/disables denylisting for all the copy engine
435 * interrupts on which NAPI is enabled.
436 *
437 * Return: None
438 */
hif_exec_dl_irq(struct qca_napi_data * napid,bool dl_flag)439 static inline void hif_exec_dl_irq(struct qca_napi_data *napid, bool dl_flag)
440 {
441 int i, j;
442 struct hif_exec_context *exec_ctx;
443
444 for (i = 0; i < HIF_MAX_GROUP; i++) {
445 /* check if NAPI is enabled on the CE */
446 if (!(napid->exec_map & (0x01 << i)))
447 continue;
448
449 /*double check that NAPI is allocated for the CE */
450 exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, i);
451 if (!(exec_ctx))
452 continue;
453
454 if (dl_flag == true)
455 for (j = 0; j < exec_ctx->numirq; j++)
456 qdf_dev_modify_irq_status(exec_ctx->os_irq[j],
457 0,
458 QDF_IRQ_NO_BALANCING);
459 else
460 for (j = 0; j < exec_ctx->numirq; j++)
461 qdf_dev_modify_irq_status(exec_ctx->os_irq[j],
462 QDF_IRQ_NO_BALANCING,
463 0);
464 hif_debug("dl_flag %d CE %d", dl_flag, i);
465 }
466 }
467
468 /**
469 * hif_exec_cpu_denylist() - en(dis)ables denylisting for NAPI RX interrupts.
470 * @napid: pointer to qca_napi_data structure
471 * @op: denylist operation to perform
472 *
473 * The function enables/disables/queries denylisting for all CE RX
474 * interrupts with NAPI enabled. Besides denylisting, it also enables/disables
475 * core_ctl_set_boost.
476 * Once denylisting is enabled, the interrupts will not be managed by the IRQ
477 * balancer.
478 *
479 * Return: -EINVAL, in case IRQ_DENYLISTING and CORE_CTL_BOOST is not enabled
480 * for DENYLIST_QUERY op - denylist refcount
481 * for DENYLIST_ON op - return value from core_ctl_set_boost API
482 * for DENYLIST_OFF op - return value from core_ctl_set_boost API
483 */
hif_exec_cpu_denylist(struct qca_napi_data * napid,enum qca_denylist_op op)484 int hif_exec_cpu_denylist(struct qca_napi_data *napid,
485 enum qca_denylist_op op)
486 {
487 int rc = 0;
488 static int ref_count; /* = 0 by the compiler */
489 uint8_t flags = napid->flags;
490 bool dl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING;
491 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST;
492
493 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op);
494
495 if (!(dl_en && ccb_en)) {
496 rc = -EINVAL;
497 goto out;
498 }
499
500 switch (op) {
501 case DENYLIST_QUERY:
502 rc = ref_count;
503 break;
504 case DENYLIST_ON:
505 ref_count++;
506 rc = 0;
507 if (ref_count == 1) {
508 rc = hif_napi_core_ctl_set_boost(true);
509 NAPI_DEBUG("boost_on() returns %d - refcnt=%d",
510 rc, ref_count);
511 hif_exec_dl_irq(napid, true);
512 }
513 break;
514 case DENYLIST_OFF:
515 if (ref_count)
516 ref_count--;
517 rc = 0;
518 if (ref_count == 0) {
519 rc = hif_napi_core_ctl_set_boost(false);
520 NAPI_DEBUG("boost_off() returns %d - refcnt=%d",
521 rc, ref_count);
522 hif_exec_dl_irq(napid, false);
523 }
524 break;
525 default:
526 NAPI_DEBUG("Invalid denylist op: %d", op);
527 rc = -EINVAL;
528 } /* switch */
529 out:
530 NAPI_DEBUG("<--%s[%d]", __func__, rc);
531 return rc;
532 }
533
534