1 /*
2 * Copyright (c) 2015-2021 The Linux Foundation. All rights reserved.
3 * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
4 *
5 * Permission to use, copy, modify, and/or distribute this software for
6 * any purpose with or without fee is hereby granted, provided that the
7 * above copyright notice and this permission notice appear in all
8 * copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
17 * PERFORMANCE OF THIS SOFTWARE.
18 */
19
20 /**
21 * DOC: hif_napi.c
22 *
23 * HIF NAPI interface implementation
24 */
25
26 #include <linux/string.h> /* memset */
27
28 /* Linux headers */
29 #include <linux/cpumask.h>
30 #include <linux/cpufreq.h>
31 #include <linux/cpu.h>
32 #include <linux/topology.h>
33 #include <linux/interrupt.h>
34 #ifdef CONFIG_SCHED_CORE_CTL
35 #include <linux/sched/core_ctl.h>
36 #endif
37 #include <pld_common.h>
38 #include <linux/pm.h>
39
40 /* Driver headers */
41 #include <hif_napi.h>
42 #include <hif_debug.h>
43 #include <hif_io32.h>
44 #include <ce_api.h>
45 #include <ce_internal.h>
46 #include <hif_irq_affinity.h>
47 #include "qdf_cpuhp.h"
48 #include "qdf_module.h"
49 #include "qdf_net_if.h"
50 #include "qdf_dev.h"
51 #include "qdf_irq.h"
52
53 enum napi_decision_vector {
54 HIF_NAPI_NOEVENT = 0,
55 HIF_NAPI_INITED = 1,
56 HIF_NAPI_CONF_UP = 2
57 };
58 #define ENABLE_NAPI_MASK (HIF_NAPI_INITED | HIF_NAPI_CONF_UP)
59
60 #ifdef RECEIVE_OFFLOAD
61 /**
62 * hif_rxthread_napi_poll() - dummy napi poll for rx_thread NAPI
63 * @napi: Rx_thread NAPI
64 * @budget: NAPI BUDGET
65 *
66 * Return: 0 as it is not supposed to be polled at all as it is not scheduled.
67 */
hif_rxthread_napi_poll(struct napi_struct * napi,int budget)68 static int hif_rxthread_napi_poll(struct napi_struct *napi, int budget)
69 {
70 hif_err("This napi_poll should not be polled as we don't schedule it");
71 QDF_ASSERT(0);
72 return 0;
73 }
74
75 /**
76 * hif_init_rx_thread_napi() - Initialize dummy Rx_thread NAPI
77 * @napii: Handle to napi_info holding rx_thread napi
78 *
79 * Return: None
80 */
hif_init_rx_thread_napi(struct qca_napi_info * napii)81 static void hif_init_rx_thread_napi(struct qca_napi_info *napii)
82 {
83 struct qdf_net_if *nd = (struct qdf_net_if *)&napii->rx_thread_netdev;
84
85 qdf_net_if_create_dummy_if(nd);
86 qdf_netif_napi_add(&napii->rx_thread_netdev, &napii->rx_thread_napi,
87 hif_rxthread_napi_poll, 64);
88 qdf_napi_enable(&napii->rx_thread_napi);
89 }
90
91 /**
92 * hif_deinit_rx_thread_napi() - Deinitialize dummy Rx_thread NAPI
93 * @napii: Handle to napi_info holding rx_thread napi
94 *
95 * Return: None
96 */
hif_deinit_rx_thread_napi(struct qca_napi_info * napii)97 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii)
98 {
99 qdf_netif_napi_del(&napii->rx_thread_napi);
100 }
101 #else /* RECEIVE_OFFLOAD */
hif_init_rx_thread_napi(struct qca_napi_info * napii)102 static void hif_init_rx_thread_napi(struct qca_napi_info *napii)
103 {
104 }
105
hif_deinit_rx_thread_napi(struct qca_napi_info * napii)106 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii)
107 {
108 }
109 #endif
110
111 /**
112 * hif_napi_create() - creates the NAPI structures for a given CE
113 * @hif_ctx: pointer to hif context
114 * @poll: poll function to be used for this NAPI instance
115 * @budget: budget to be registered with the NAPI instance
116 * @scale: scale factor on the weight (to scaler budget to 1000)
117 * @flags: feature flags
118 *
119 * Description:
120 * Creates NAPI instances. This function is called
121 * unconditionally during initialization. It creates
122 * napi structures through the proper HTC/HIF calls.
123 * The structures are disabled on creation.
124 * Note that for each NAPI instance a separate dummy netdev is used
125 *
126 * Return:
127 * < 0: error
128 * = 0: <should never happen>
129 * > 0: id of the created object (for multi-NAPI, number of objects created)
130 */
hif_napi_create(struct hif_opaque_softc * hif_ctx,int (* poll)(struct napi_struct *,int),int budget,int scale,uint8_t flags)131 int hif_napi_create(struct hif_opaque_softc *hif_ctx,
132 int (*poll)(struct napi_struct *, int),
133 int budget,
134 int scale,
135 uint8_t flags)
136 {
137 int i;
138 struct qca_napi_data *napid;
139 struct qca_napi_info *napii;
140 struct CE_state *ce_state;
141 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
142 int rc = 0;
143
144 NAPI_DEBUG("-->(budget=%d, scale=%d)",
145 budget, scale);
146 NAPI_DEBUG("hif->napi_data.state = 0x%08x",
147 hif->napi_data.state);
148 NAPI_DEBUG("hif->napi_data.ce_map = 0x%08x",
149 hif->napi_data.ce_map);
150
151 napid = &(hif->napi_data);
152 if (0 == (napid->state & HIF_NAPI_INITED)) {
153 memset(napid, 0, sizeof(struct qca_napi_data));
154 qdf_spinlock_create(&(napid->lock));
155
156 napid->state |= HIF_NAPI_INITED;
157 napid->flags = flags;
158
159 rc = hif_napi_cpu_init(hif_ctx);
160 if (rc != 0 && rc != -EALREADY) {
161 hif_err("NAPI_initialization failed(rc=%d)", rc);
162 rc = napid->ce_map;
163 goto hnc_err;
164 } else
165 rc = 0;
166
167 hif_debug("NAPI structures initialized, rc=%d", rc);
168 }
169 for (i = 0; i < hif->ce_count; i++) {
170 ce_state = hif->ce_id_to_state[i];
171 NAPI_DEBUG("ce %d: htt_rx=%d htt_tx=%d",
172 i, ce_state->htt_rx_data,
173 ce_state->htt_tx_data);
174 if (ce_srng_based(hif))
175 continue;
176
177 if (!ce_state->htt_rx_data)
178 continue;
179
180 /* Now this is a CE where we need NAPI on */
181 NAPI_DEBUG("Creating NAPI on pipe %d", i);
182 napii = qdf_mem_malloc(sizeof(*napii));
183 napid->napis[i] = napii;
184 if (!napii) {
185 rc = -ENOMEM;
186 goto napii_free;
187 }
188 }
189
190 for (i = 0; i < hif->ce_count; i++) {
191 napii = napid->napis[i];
192 if (!napii)
193 continue;
194
195 NAPI_DEBUG("initializing NAPI for pipe %d", i);
196 memset(napii, 0, sizeof(struct qca_napi_info));
197 napii->scale = scale;
198 napii->id = NAPI_PIPE2ID(i);
199 napii->hif_ctx = hif_ctx;
200 napii->irq = pld_get_irq(hif->qdf_dev->dev, i);
201
202 if (napii->irq < 0)
203 hif_warn("bad IRQ value for CE %d: %d", i, napii->irq);
204
205 qdf_net_if_create_dummy_if((struct qdf_net_if *)&napii->netdev);
206
207 NAPI_DEBUG("adding napi=%pK to netdev=%pK (poll=%pK, bdgt=%d)",
208 &(napii->napi), &(napii->netdev), poll, budget);
209 qdf_netif_napi_add(&(napii->netdev), &(napii->napi),
210 poll, budget);
211
212 NAPI_DEBUG("after napi_add");
213 NAPI_DEBUG("napi=0x%pK, netdev=0x%pK",
214 &(napii->napi), &(napii->netdev));
215 NAPI_DEBUG("napi.dev_list.prev=0x%pK, next=0x%pK",
216 napii->napi.dev_list.prev,
217 napii->napi.dev_list.next);
218 NAPI_DEBUG("dev.napi_list.prev=0x%pK, next=0x%pK",
219 napii->netdev.napi_list.prev,
220 napii->netdev.napi_list.next);
221
222 hif_init_rx_thread_napi(napii);
223 napii->lro_ctx = qdf_lro_init();
224 NAPI_DEBUG("Registering LRO for ce_id %d NAPI callback for %d lro_ctx %pK\n",
225 i, napii->id, napii->lro_ctx);
226
227 /* It is OK to change the state variable below without
228 * protection as there should be no-one around yet
229 */
230 napid->ce_map |= (0x01 << i);
231 hif_debug("NAPI id %d created for pipe %d", napii->id, i);
232 }
233
234 /* no ces registered with the napi */
235 if (!ce_srng_based(hif) && napid->ce_map == 0) {
236 hif_warn("no napis created for copy engines");
237 rc = -EFAULT;
238 goto napii_free;
239 }
240
241 NAPI_DEBUG("napi map = %x", napid->ce_map);
242 NAPI_DEBUG("NAPI ids created for all applicable pipes");
243 return napid->ce_map;
244
245 napii_free:
246 for (i = 0; i < hif->ce_count; i++) {
247 napii = napid->napis[i];
248 napid->napis[i] = NULL;
249 if (napii)
250 qdf_mem_free(napii);
251 }
252
253 hnc_err:
254 NAPI_DEBUG("<--napi_instances_map=%x]", napid->ce_map);
255 return rc;
256 }
257 qdf_export_symbol(hif_napi_create);
258
259 #ifdef RECEIVE_OFFLOAD
hif_napi_rx_offld_flush_cb_register(struct hif_opaque_softc * hif_hdl,void (offld_flush_handler)(void *))260 void hif_napi_rx_offld_flush_cb_register(struct hif_opaque_softc *hif_hdl,
261 void (offld_flush_handler)(void *))
262 {
263 int i;
264 struct CE_state *ce_state;
265 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl);
266 struct qca_napi_data *napid;
267 struct qca_napi_info *napii;
268
269 if (!scn) {
270 hif_err("hif_state NULL!");
271 QDF_ASSERT(0);
272 return;
273 }
274
275 napid = hif_napi_get_all(hif_hdl);
276 for (i = 0; i < scn->ce_count; i++) {
277 ce_state = scn->ce_id_to_state[i];
278 if (ce_state && (ce_state->htt_rx_data)) {
279 napii = napid->napis[i];
280 napii->offld_flush_cb = offld_flush_handler;
281 hif_debug("Registering offload for ce_id %d NAPI callback for %d flush_cb %pK",
282 i, napii->id, napii->offld_flush_cb);
283 }
284 }
285 }
286
hif_napi_rx_offld_flush_cb_deregister(struct hif_opaque_softc * hif_hdl)287 void hif_napi_rx_offld_flush_cb_deregister(struct hif_opaque_softc *hif_hdl)
288 {
289 int i;
290 struct CE_state *ce_state;
291 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl);
292 struct qca_napi_data *napid;
293 struct qca_napi_info *napii;
294
295 if (!scn) {
296 hif_err("hif_state NULL!");
297 QDF_ASSERT(0);
298 return;
299 }
300
301 napid = hif_napi_get_all(hif_hdl);
302 for (i = 0; i < scn->ce_count; i++) {
303 ce_state = scn->ce_id_to_state[i];
304 if (ce_state && (ce_state->htt_rx_data)) {
305 napii = napid->napis[i];
306 hif_debug("deRegistering offld for ce_id %d NAPI callback for %d flush_cb %pK",
307 i, napii->id, napii->offld_flush_cb);
308 /* Not required */
309 napii->offld_flush_cb = NULL;
310 }
311 }
312 }
313 #endif /* RECEIVE_OFFLOAD */
314
315 /**
316 * hif_napi_destroy() - destroys the NAPI structures for a given instance
317 * @hif_ctx: pointer to hif context
318 * @id: the CE id whose napi instance will be destroyed
319 * @force: if set, will destroy even if entry is active (de-activates)
320 *
321 * Description:
322 * Destroy a given NAPI instance. This function is called
323 * unconditionally during cleanup.
324 * Refuses to destroy an entry of it is still enabled (unless force=1)
325 * Marks the whole napi_data invalid if all instances are destroyed.
326 *
327 * Return:
328 * -EINVAL: specific entry has not been created
329 * -EPERM : specific entry is still active
330 * 0 < : error
331 * 0 = : success
332 */
hif_napi_destroy(struct hif_opaque_softc * hif_ctx,uint8_t id,int force)333 int hif_napi_destroy(struct hif_opaque_softc *hif_ctx,
334 uint8_t id,
335 int force)
336 {
337 uint8_t ce = NAPI_ID2PIPE(id);
338 int rc = 0;
339 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
340
341 NAPI_DEBUG("-->(id=%d, force=%d)", id, force);
342
343 if (0 == (hif->napi_data.state & HIF_NAPI_INITED)) {
344 hif_err("NAPI not initialized or entry %d not created", id);
345 rc = -EINVAL;
346 } else if (0 == (hif->napi_data.ce_map & (0x01 << ce))) {
347 hif_err("NAPI instance %d (pipe %d) not created", id, ce);
348 if (hif->napi_data.napis[ce])
349 hif_err("memory allocated but ce_map not set %d (pipe %d)",
350 id, ce);
351 rc = -EINVAL;
352 } else {
353 struct qca_napi_data *napid;
354 struct qca_napi_info *napii;
355
356 napid = &(hif->napi_data);
357 napii = napid->napis[ce];
358 if (!napii) {
359 if (napid->ce_map & (0x01 << ce))
360 hif_err("napii & ce_map out of sync(ce %d)", ce);
361 return -EINVAL;
362 }
363
364
365 if (hif->napi_data.state == HIF_NAPI_CONF_UP) {
366 if (force) {
367 qdf_napi_disable(&(napii->napi));
368 hif_debug("NAPI entry %d force disabled", id);
369 NAPI_DEBUG("NAPI %d force disabled", id);
370 } else {
371 hif_err("Cannot destroy active NAPI %d", id);
372 rc = -EPERM;
373 }
374 }
375 if (0 == rc) {
376 NAPI_DEBUG("before napi_del");
377 NAPI_DEBUG("napi.dlist.prv=0x%pK, next=0x%pK",
378 napii->napi.dev_list.prev,
379 napii->napi.dev_list.next);
380 NAPI_DEBUG("dev.napi_l.prv=0x%pK, next=0x%pK",
381 napii->netdev.napi_list.prev,
382 napii->netdev.napi_list.next);
383
384 qdf_lro_deinit(napii->lro_ctx);
385 qdf_netif_napi_del(&(napii->napi));
386 hif_deinit_rx_thread_napi(napii);
387
388 napid->ce_map &= ~(0x01 << ce);
389 napid->napis[ce] = NULL;
390 napii->scale = 0;
391 qdf_mem_free(napii);
392 hif_debug("NAPI %d destroyed", id);
393
394 /* if there are no active instances and
395 * if they are all destroyed,
396 * set the whole structure to uninitialized state
397 */
398 if (napid->ce_map == 0) {
399 rc = hif_napi_cpu_deinit(hif_ctx);
400 /* caller is tolerant to receiving !=0 rc */
401
402 qdf_spinlock_destroy(&(napid->lock));
403 memset(napid,
404 0, sizeof(struct qca_napi_data));
405 hif_debug("no NAPI instances. Zapped");
406 }
407 }
408 }
409
410 return rc;
411 }
412 qdf_export_symbol(hif_napi_destroy);
413
414 #ifdef FEATURE_LRO
hif_napi_get_lro_info(struct hif_opaque_softc * hif_hdl,int napi_id)415 void *hif_napi_get_lro_info(struct hif_opaque_softc *hif_hdl, int napi_id)
416 {
417 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl);
418 struct qca_napi_data *napid;
419 struct qca_napi_info *napii;
420
421 napid = &(scn->napi_data);
422 napii = napid->napis[NAPI_ID2PIPE(napi_id)];
423
424 if (napii)
425 return napii->lro_ctx;
426 return 0;
427 }
428 #endif
429
430 /**
431 * hif_napi_get_all() - returns the address of the whole HIF NAPI structure
432 * @hif_ctx: pointer to hif context
433 *
434 * Description:
435 * Returns the address of the whole structure
436 *
437 * Return:
438 * <addr>: address of the whole HIF NAPI structure
439 */
hif_napi_get_all(struct hif_opaque_softc * hif_ctx)440 inline struct qca_napi_data *hif_napi_get_all(struct hif_opaque_softc *hif_ctx)
441 {
442 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
443
444 return &(hif->napi_data);
445 }
446
hif_get_napi(int napi_id,struct qca_napi_data * napid)447 struct qca_napi_info *hif_get_napi(int napi_id, struct qca_napi_data *napid)
448 {
449 int id = NAPI_ID2PIPE(napi_id);
450
451 return napid->napis[id];
452 }
453
454 /**
455 * hif_napi_event() - reacts to events that impact NAPI
456 * @hif_ctx: pointer to hif context
457 * @event: event that has been detected
458 * @data: more data regarding the event
459 *
460 * Description:
461 * This function handles two types of events:
462 * 1- Events that change the state of NAPI (enabled/disabled):
463 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE}
464 * The state is retrievable by "hdd_napi_enabled(-1)"
465 * - NAPI will be on if either INI file is on and it has not been disabled
466 * by a subsequent vendor CMD,
467 * or it has been enabled by a vendor CMD.
468 * 2- Events that change the CPU affinity of a NAPI instance/IRQ:
469 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE}
470 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode
471 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy()
472 * - In LO tput mode, NAPI will yield control if its interrupts to the system
473 * management functions. However in HI throughput mode, NAPI will actively
474 * manage its interrupts/instances (by trying to disperse them out to
475 * separate performance cores).
476 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events.
477 *
478 * + In some cases (roaming peer management is the only case so far), a
479 * a client can trigger a "SERIALIZE" event. Basically, this means that the
480 * users is asking NAPI to go into a truly single execution context state.
481 * So, NAPI indicates to msm-irqbalancer that it wants to be denylisted,
482 * (if called for the first time) and then moves all IRQs (for NAPI
483 * instances) to be collapsed to a single core. If called multiple times,
484 * it will just re-collapse the CPUs. This is because denylist-on() API
485 * is reference-counted, and because the API has already been called.
486 *
487 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go
488 * to its "normal" operation. Optionally, they can give a timeout value (in
489 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this
490 * case, NAPI will just set the current throughput state to uninitialized
491 * and set the delay period. Once policy handler is called, it would skip
492 * applying the policy delay period times, and otherwise apply the policy.
493 *
494 * Return:
495 * < 0: some error
496 * = 0: event handled successfully
497 */
hif_napi_event(struct hif_opaque_softc * hif_ctx,enum qca_napi_event event,void * data)498 int hif_napi_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event,
499 void *data)
500 {
501 int rc = 0;
502 uint32_t prev_state;
503 int i;
504 bool state_changed;
505 struct napi_struct *napi;
506 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
507 struct qca_napi_data *napid = &(hif->napi_data);
508 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED;
509 enum {
510 DENYLIST_NOT_PENDING,
511 DENYLIST_ON_PENDING,
512 DENYLIST_OFF_PENDING
513 } denylist_pending = DENYLIST_NOT_PENDING;
514
515 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data);
516
517 if (ce_srng_based(hif))
518 return hif_exec_event(hif_ctx, event, data);
519
520 if ((napid->state & HIF_NAPI_INITED) == 0) {
521 NAPI_DEBUG("%s: got event when NAPI not initialized",
522 __func__);
523 return -EINVAL;
524 }
525 qdf_spin_lock_bh(&(napid->lock));
526 prev_state = napid->state;
527 switch (event) {
528 case NAPI_EVT_INI_FILE:
529 case NAPI_EVT_CMD_STATE:
530 case NAPI_EVT_INT_STATE: {
531 int on = (data != ((void *)0));
532
533 hif_debug("recved evnt: STATE_CMD %d; v = %d (state=0x%0x)",
534 event, on, prev_state);
535 if (on)
536 if (prev_state & HIF_NAPI_CONF_UP) {
537 hif_debug("Duplicate NAPI conf ON msg");
538 } else {
539 hif_debug("Setting state to ON");
540 napid->state |= HIF_NAPI_CONF_UP;
541 }
542 else /* off request */
543 if (prev_state & HIF_NAPI_CONF_UP) {
544 hif_debug("Setting state to OFF");
545 napid->state &= ~HIF_NAPI_CONF_UP;
546 } else {
547 hif_debug("Duplicate NAPI conf OFF msg");
548 }
549 break;
550 }
551 /* case NAPI_INIT_FILE/CMD_STATE */
552
553 case NAPI_EVT_CPU_STATE: {
554 int cpu = ((unsigned long int)data >> 16);
555 int val = ((unsigned long int)data & 0x0ff);
556
557 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d",
558 __func__, cpu, val);
559
560 /* state has already been set by hnc_cpu_notify_cb */
561 if ((val == QCA_NAPI_CPU_DOWN) &&
562 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */
563 (napid->napi_cpu[cpu].napis != 0)) {
564 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d",
565 __func__, cpu);
566 rc = hif_napi_cpu_migrate(napid,
567 cpu,
568 HNC_ACT_RELOCATE);
569 napid->napi_cpu[cpu].napis = 0;
570 }
571 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */
572 break;
573 }
574
575 case NAPI_EVT_TPUT_STATE: {
576 tput_mode = (enum qca_napi_tput_state)data;
577 if (tput_mode == QCA_NAPI_TPUT_LO) {
578 /* from TPUT_HI -> TPUT_LO */
579 NAPI_DEBUG("%s: Moving to napi_tput_LO state",
580 __func__);
581 denylist_pending = DENYLIST_OFF_PENDING;
582 /*
583 * Ideally we should "collapse" interrupts here, since
584 * we are "dispersing" interrupts in the "else" case.
585 * This allows the possibility that our interrupts may
586 * still be on the perf cluster the next time we enter
587 * high tput mode. However, the irq_balancer is free
588 * to move our interrupts to power cluster once
589 * denylisting has been turned off in the "else" case.
590 */
591 } else {
592 /* from TPUT_LO -> TPUT->HI */
593 NAPI_DEBUG("%s: Moving to napi_tput_HI state",
594 __func__);
595 rc = hif_napi_cpu_migrate(napid,
596 HNC_ANY_CPU,
597 HNC_ACT_DISPERSE);
598
599 denylist_pending = DENYLIST_ON_PENDING;
600 }
601 napid->napi_mode = tput_mode;
602 break;
603 }
604
605 case NAPI_EVT_USR_SERIAL: {
606 unsigned long users = (unsigned long)data;
607
608 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld",
609 __func__, users);
610
611 rc = hif_napi_cpu_migrate(napid,
612 HNC_ANY_CPU,
613 HNC_ACT_COLLAPSE);
614 if ((users == 0) && (rc == 0))
615 denylist_pending = DENYLIST_ON_PENDING;
616 break;
617 }
618 case NAPI_EVT_USR_NORMAL: {
619 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__);
620 if (!napid->user_cpu_affin_mask)
621 denylist_pending = DENYLIST_OFF_PENDING;
622 /*
623 * Deserialization timeout is handled at hdd layer;
624 * just mark current mode to uninitialized to ensure
625 * it will be set when the delay is over
626 */
627 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED;
628 break;
629 }
630 default: {
631 hif_err("Unknown event: %d (data=0x%0lx)",
632 event, (unsigned long) data);
633 break;
634 } /* default */
635 }; /* switch */
636
637
638 switch (denylist_pending) {
639 case DENYLIST_ON_PENDING:
640 /* assume the control of WLAN IRQs */
641 hif_napi_cpu_denylist(napid, DENYLIST_ON);
642 break;
643 case DENYLIST_OFF_PENDING:
644 /* yield the control of WLAN IRQs */
645 hif_napi_cpu_denylist(napid, DENYLIST_OFF);
646 break;
647 default: /* nothing to do */
648 break;
649 } /* switch denylist_pending */
650
651 /* we want to perform the comparison in lock:
652 * there is a possibility of hif_napi_event get called
653 * from two different contexts (driver unload and cpu hotplug
654 * notification) and napid->state get changed
655 * in driver unload context and can lead to race condition
656 * in cpu hotplug context. Therefore, perform the napid->state
657 * comparison before releasing lock.
658 */
659 state_changed = (prev_state != napid->state);
660 qdf_spin_unlock_bh(&(napid->lock));
661
662 if (state_changed) {
663 if (napid->state == ENABLE_NAPI_MASK) {
664 rc = 1;
665 for (i = 0; i < CE_COUNT_MAX; i++) {
666 struct qca_napi_info *napii = napid->napis[i];
667 if (napii) {
668 napi = &(napii->napi);
669 NAPI_DEBUG("%s: enabling NAPI %d",
670 __func__, i);
671 qdf_napi_enable(napi);
672 }
673 }
674 } else {
675 rc = 0;
676 for (i = 0; i < CE_COUNT_MAX; i++) {
677 struct qca_napi_info *napii = napid->napis[i];
678 if (napii) {
679 napi = &(napii->napi);
680 NAPI_DEBUG("%s: disabling NAPI %d",
681 __func__, i);
682 qdf_napi_disable(napi);
683 /* in case it is affined, remove it */
684 qdf_dev_set_irq_affinity(napii->irq,
685 NULL);
686 }
687 }
688 }
689 } else {
690 hif_debug("no change in hif napi state (still %d)", prev_state);
691 }
692
693 NAPI_DEBUG("<--[rc=%d]", rc);
694 return rc;
695 }
696 qdf_export_symbol(hif_napi_event);
697
698 /**
699 * hif_napi_enabled() - checks whether NAPI is enabled for given ce or not
700 * @hif_ctx: hif context
701 * @ce: CE instance (or -1, to check if any CEs are enabled)
702 *
703 * Return: bool
704 */
hif_napi_enabled(struct hif_opaque_softc * hif_ctx,int ce)705 int hif_napi_enabled(struct hif_opaque_softc *hif_ctx, int ce)
706 {
707 int rc;
708 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
709
710 if (-1 == ce)
711 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK));
712 else
713 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK) &&
714 (hif->napi_data.ce_map & (0x01 << ce)));
715 return rc;
716 }
717 qdf_export_symbol(hif_napi_enabled);
718
719 /**
720 * hif_napi_created() - checks whether NAPI is created for given ce or not
721 * @hif_ctx: hif context
722 * @ce: CE instance
723 *
724 * Return: bool
725 */
hif_napi_created(struct hif_opaque_softc * hif_ctx,int ce)726 bool hif_napi_created(struct hif_opaque_softc *hif_ctx, int ce)
727 {
728 int rc;
729 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
730
731 rc = (hif->napi_data.ce_map & (0x01 << ce));
732
733 return !!rc;
734 }
735 qdf_export_symbol(hif_napi_created);
736
737 /**
738 * hif_napi_enable_irq() - enables bus interrupts after napi_complete
739 *
740 * @hif: hif context
741 * @id: id of NAPI instance calling this (used to determine the CE)
742 *
743 * Return: void
744 */
hif_napi_enable_irq(struct hif_opaque_softc * hif,int id)745 inline void hif_napi_enable_irq(struct hif_opaque_softc *hif, int id)
746 {
747 struct hif_softc *scn = HIF_GET_SOFTC(hif);
748
749 hif_irq_enable(scn, NAPI_ID2PIPE(id));
750 }
751
752 #if defined(QCA_WIFI_WCN6450) && defined(HIF_LATENCY_PROFILE_ENABLE)
753 /*
754 * hif_napi_latency_profile_start() - update the schedule start timestamp
755 *
756 * @scn: HIF context
757 * ce_id: Copyengine id
758 *
759 * Return: None
760 */
hif_napi_latency_profile_start(struct hif_softc * scn,int ce_id)761 static inline void hif_napi_latency_profile_start(struct hif_softc *scn,
762 int ce_id)
763 {
764 struct qca_napi_info *napii;
765
766 napii = scn->napi_data.napis[ce_id];
767 if (napii)
768 napii->tstamp = qdf_ktime_to_ms(qdf_ktime_get());
769 }
770
771 /*
772 * hif_napi_latency_profile_measure() - calculate the NAPI schedule latency
773 * and update histogram
774 *
775 * @napi_info: pointer to qca_napi_info for the napi instance
776 *
777 * Return: None
778 */
hif_napi_latency_profile_measure(struct qca_napi_info * napi_info)779 static void hif_napi_latency_profile_measure(struct qca_napi_info *napi_info)
780 {
781 int64_t cur_tstamp;
782 int64_t time_elapsed;
783
784 cur_tstamp = qdf_ktime_to_ms(qdf_ktime_get());
785
786 if (cur_tstamp > napi_info->tstamp)
787 time_elapsed = (cur_tstamp - napi_info->tstamp);
788 else
789 time_elapsed = ~0x0 - (napi_info->tstamp - cur_tstamp);
790
791 napi_info->tstamp = cur_tstamp;
792
793 if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_0_2)
794 napi_info->sched_latency_stats[0]++;
795 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_3_10)
796 napi_info->sched_latency_stats[1]++;
797 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_11_20)
798 napi_info->sched_latency_stats[2]++;
799 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_21_50)
800 napi_info->sched_latency_stats[3]++;
801 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_51_100)
802 napi_info->sched_latency_stats[4]++;
803 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_101_250)
804 napi_info->sched_latency_stats[5]++;
805 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_251_500)
806 napi_info->sched_latency_stats[6]++;
807 else
808 napi_info->sched_latency_stats[7]++;
809 }
810
hif_print_napi_latency_stats(struct qca_napi_info * napii,int ce_id)811 static void hif_print_napi_latency_stats(struct qca_napi_info *napii, int ce_id)
812 {
813 int i;
814 int64_t cur_tstamp;
815
816 const char time_str[HIF_SCHED_LATENCY_BUCKETS][15] = {
817 "0-2 ms",
818 "3-10 ms",
819 "11-20 ms",
820 "21-50 ms",
821 "51-100 ms",
822 "101-250 ms",
823 "251-500 ms",
824 "> 500 ms"
825 };
826
827 cur_tstamp = qdf_ktime_to_ms(qdf_ktime_get());
828
829 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH,
830 "Current timestamp: %lld", cur_tstamp);
831
832 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH,
833 "ce id %d Last serviced timestamp: %lld",
834 ce_id, napii->tstamp);
835
836 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH,
837 "Latency Bucket | Time elapsed");
838
839 for (i = 0; i < HIF_SCHED_LATENCY_BUCKETS; i++)
840 QDF_TRACE(QDF_MODULE_ID_HIF,
841 QDF_TRACE_LEVEL_INFO_HIGH,
842 "%s | %lld",
843 time_str[i],
844 napii->sched_latency_stats[i]);
845 }
846 #else
847 static inline void
hif_napi_latency_profile_start(struct hif_softc * scn,int ce_id)848 hif_napi_latency_profile_start(struct hif_softc *scn, int ce_id)
849 {
850 }
851
852 static inline void
hif_napi_latency_profile_measure(struct qca_napi_info * napi_info)853 hif_napi_latency_profile_measure(struct qca_napi_info *napi_info)
854 {
855 }
856
857 static inline void
hif_print_napi_latency_stats(struct qca_napi_info * napii,int ce_id)858 hif_print_napi_latency_stats(struct qca_napi_info *napii, int ce_id)
859 {
860 }
861 #endif
862
863 #ifdef QCA_WIFI_WCN6450
864 #ifdef WLAN_FEATURE_RX_SOFTIRQ_TIME_LIMIT
865 /**
866 * hif_napi_update_service_start_time() - Update NAPI poll start time
867 *
868 * @napi_info: per NAPI instance data structure
869 *
870 * The function is called at the beginning of a NAPI poll to record the poll
871 * start time.
872 *
873 * Return: None
874 */
875 static inline void
hif_napi_update_service_start_time(struct qca_napi_info * napi_info)876 hif_napi_update_service_start_time(struct qca_napi_info *napi_info)
877 {
878 napi_info->poll_start_time = qdf_time_sched_clock();
879 }
880
881 /**
882 * hif_napi_fill_poll_time_histogram() - fills poll time histogram for a NAPI
883 *
884 * @napi_info: per NAPI instance data structure
885 *
886 * The function is called at the end of a NAPI poll to calculate poll time
887 * buckets.
888 *
889 * Return: void
890 */
hif_napi_fill_poll_time_histogram(struct qca_napi_info * napi_info)891 static void hif_napi_fill_poll_time_histogram(struct qca_napi_info *napi_info)
892 {
893 struct qca_napi_stat *napi_stat;
894 unsigned long long poll_time_ns;
895 uint32_t poll_time_us;
896 uint32_t bucket_size_us = 500;
897 uint32_t bucket;
898 uint32_t cpu_id = qdf_get_cpu();
899
900 poll_time_ns = qdf_time_sched_clock() - napi_info->poll_start_time;
901 poll_time_us = qdf_do_div(poll_time_ns, 1000);
902
903 napi_stat = &napi_info->stats[cpu_id];
904 if (poll_time_ns > napi_info->stats[cpu_id].napi_max_poll_time)
905 napi_info->stats[cpu_id].napi_max_poll_time = poll_time_ns;
906
907 bucket = poll_time_us / bucket_size_us;
908 if (bucket >= QCA_NAPI_NUM_BUCKETS)
909 bucket = QCA_NAPI_NUM_BUCKETS - 1;
910
911 ++napi_stat->poll_time_buckets[bucket];
912 }
913
914 /*
915 * hif_get_poll_times_hist_str() - Get HIF poll times histogram string
916 * @stats: NAPI stats to get poll time buckets
917 * @buf: buffer to fill histogram string
918 * @buf_len: length of the buffer
919 *
920 * Return: void
921 */
hif_get_poll_times_hist_str(struct qca_napi_stat * stats,char * buf,uint8_t buf_len)922 static void hif_get_poll_times_hist_str(struct qca_napi_stat *stats, char *buf,
923 uint8_t buf_len)
924 {
925 int i;
926 int str_index = 0;
927
928 for (i = 0; i < QCA_NAPI_NUM_BUCKETS; i++)
929 str_index += qdf_scnprintf(buf + str_index, buf_len - str_index,
930 "%u|", stats->poll_time_buckets[i]);
931 }
932
hif_print_napi_stats(struct hif_opaque_softc * hif_ctx)933 void hif_print_napi_stats(struct hif_opaque_softc *hif_ctx)
934 {
935 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx);
936 struct qca_napi_info *napii;
937 struct qca_napi_stat *napi_stats;
938 int ce_id, cpu;
939
940 /*
941 * Max value of uint_32 (poll_time_bucket) = 4294967295
942 * Thus we need 10 chars + 1 space =11 chars for each bucket value.
943 * +1 space for '\0'.
944 */
945 char hist_str[(QCA_NAPI_NUM_BUCKETS * 11) + 1] = {'\0'};
946
947 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH,
948 "NAPI[#]CPU[#] |scheds |polls |comps |dones |t-lim |max(us)|hist(500us buckets)");
949
950 for (ce_id = 0; ce_id < CE_COUNT_MAX; ce_id++) {
951 if (!hif_napi_enabled(hif_ctx, ce_id))
952 continue;
953
954 napii = scn->napi_data.napis[ce_id];
955 if (napii) {
956 for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
957 napi_stats = &napii->stats[cpu];
958
959 hif_get_poll_times_hist_str(napi_stats,
960 hist_str,
961 sizeof(hist_str));
962
963 if (napi_stats->napi_schedules != 0)
964 QDF_TRACE(QDF_MODULE_ID_HIF,
965 QDF_TRACE_LEVEL_INFO_HIGH,
966 "NAPI[%d]CPU[%d]: %7u %7u %7u %7u %7u %7llu %s",
967 ce_id, cpu,
968 napi_stats->napi_schedules,
969 napi_stats->napi_polls,
970 napi_stats->napi_completes,
971 napi_stats->napi_workdone,
972 napi_stats->time_limit_reached,
973 qdf_do_div(napi_stats->napi_max_poll_time, 1000),
974 hist_str);
975 }
976
977 hif_print_napi_latency_stats(napii, ce_id);
978 }
979 }
980 }
981 #else
982 static inline void
hif_napi_update_service_start_time(struct qca_napi_info * napi_info)983 hif_napi_update_service_start_time(struct qca_napi_info *napi_info)
984 {
985 }
986
987 static inline void
hif_napi_fill_poll_time_histogram(struct qca_napi_info * napi_info)988 hif_napi_fill_poll_time_histogram(struct qca_napi_info *napi_info)
989 {
990 }
991
hif_print_napi_stats(struct hif_opaque_softc * hif_ctx)992 void hif_print_napi_stats(struct hif_opaque_softc *hif_ctx)
993 {
994 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx);
995 struct qca_napi_info *napii;
996 struct qca_napi_stat *napi_stats;
997 int ce_id, cpu;
998
999 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_FATAL,
1000 "NAPI[#ctx]CPU[#] |schedules |polls |completes |workdone");
1001
1002 for (ce_id = 0; ce_id < CE_COUNT_MAX; ce_id++) {
1003 if (!hif_napi_enabled(hif_ctx, ce_id))
1004 continue;
1005
1006 napii = scn->napi_data.napis[ce_id];
1007 if (napii) {
1008 for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
1009 napi_stats = &napii->stats[cpu];
1010
1011 if (napi_stats->napi_schedules != 0)
1012 QDF_TRACE(QDF_MODULE_ID_HIF,
1013 QDF_TRACE_LEVEL_FATAL,
1014 "NAPI[%2d]CPU[%d]: "
1015 "%7d %7d %7d %7d ",
1016 ce_id, cpu,
1017 napi_stats->napi_schedules,
1018 napi_stats->napi_polls,
1019 napi_stats->napi_completes,
1020 napi_stats->napi_workdone);
1021 }
1022
1023 hif_print_napi_latency_stats(napii, ce_id);
1024 }
1025 }
1026 }
1027 #endif
1028
1029 #ifdef HIF_LATENCY_PROFILE_ENABLE
hif_clear_napi_stats(struct hif_opaque_softc * hif_ctx)1030 void hif_clear_napi_stats(struct hif_opaque_softc *hif_ctx)
1031 {
1032 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx);
1033 struct qca_napi_info *napii;
1034 int ce_id;
1035
1036 for (ce_id = 0; ce_id < CE_COUNT_MAX; ce_id++) {
1037 if (!hif_napi_enabled(hif_ctx, ce_id))
1038 continue;
1039
1040 napii = scn->napi_data.napis[ce_id];
1041 if (napii)
1042 qdf_mem_set(napii->sched_latency_stats,
1043 sizeof(napii->sched_latency_stats), 0);
1044 }
1045 }
1046 #else
hif_clear_napi_stats(struct hif_opaque_softc * hif_ctx)1047 inline void hif_clear_napi_stats(struct hif_opaque_softc *hif_ctx)
1048 {
1049 }
1050 #endif /* HIF_LATENCY_PROFILE_ENABLE */
1051
1052 #else
1053 static inline void
hif_napi_update_service_start_time(struct qca_napi_info * napi_info)1054 hif_napi_update_service_start_time(struct qca_napi_info *napi_info)
1055 {
1056 }
1057
1058 static inline void
hif_napi_fill_poll_time_histogram(struct qca_napi_info * napi_info)1059 hif_napi_fill_poll_time_histogram(struct qca_napi_info *napi_info)
1060 {
1061 }
1062 #endif
1063
1064 /**
1065 * hif_napi_schedule() - schedules napi, updates stats
1066 * @hif_ctx: hif context
1067 * @ce_id: index of napi instance
1068 *
1069 * Return: false if napi didn't enable or already scheduled, otherwise true
1070 */
hif_napi_schedule(struct hif_opaque_softc * hif_ctx,int ce_id)1071 bool hif_napi_schedule(struct hif_opaque_softc *hif_ctx, int ce_id)
1072 {
1073 int cpu = smp_processor_id();
1074 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx);
1075 struct qca_napi_info *napii;
1076
1077 napii = scn->napi_data.napis[ce_id];
1078 if (qdf_unlikely(!napii)) {
1079 hif_err("scheduling unallocated napi (ce:%d)", ce_id);
1080 qdf_atomic_dec(&scn->active_tasklet_cnt);
1081 return false;
1082 }
1083
1084 if (test_bit(NAPI_STATE_SCHED, &napii->napi.state)) {
1085 NAPI_DEBUG("napi scheduled, return");
1086 qdf_atomic_dec(&scn->active_tasklet_cnt);
1087 return false;
1088 }
1089
1090 hif_record_ce_desc_event(scn, ce_id, NAPI_SCHEDULE,
1091 NULL, NULL, 0, 0);
1092 napii->stats[cpu].napi_schedules++;
1093 NAPI_DEBUG("scheduling napi %d (ce:%d)", napii->id, ce_id);
1094 hif_napi_latency_profile_start(scn, ce_id);
1095 napi_schedule(&(napii->napi));
1096
1097 return true;
1098 }
1099 qdf_export_symbol(hif_napi_schedule);
1100
1101 /**
1102 * hif_napi_correct_cpu() - correct the interrupt affinity for napi if needed
1103 * @napi_info: pointer to qca_napi_info for the napi instance
1104 *
1105 * Return: true => interrupt already on correct cpu, no correction needed
1106 * false => interrupt on wrong cpu, correction done for cpu affinity
1107 * of the interrupt
1108 */
1109 static inline
hif_napi_correct_cpu(struct qca_napi_info * napi_info)1110 bool hif_napi_correct_cpu(struct qca_napi_info *napi_info)
1111 {
1112 bool right_cpu = true;
1113 int rc = 0;
1114 int cpu;
1115 struct qca_napi_data *napid;
1116 QDF_STATUS ret;
1117
1118 napid = hif_napi_get_all(GET_HIF_OPAQUE_HDL(napi_info->hif_ctx));
1119
1120 if (napid->flags & QCA_NAPI_FEATURE_CPU_CORRECTION) {
1121
1122 cpu = qdf_get_cpu();
1123 if (unlikely((hif_napi_cpu_denylist(napid,
1124 DENYLIST_QUERY) > 0) &&
1125 cpu != napi_info->cpu)) {
1126 right_cpu = false;
1127
1128 NAPI_DEBUG("interrupt on wrong CPU, correcting");
1129 napi_info->cpumask.bits[0] = (0x01 << napi_info->cpu);
1130
1131 qdf_dev_modify_irq_status(napi_info->irq,
1132 QDF_IRQ_NO_BALANCING, 0);
1133 ret = qdf_dev_set_irq_affinity(napi_info->irq,
1134 (struct qdf_cpu_mask *)
1135 &napi_info->cpumask);
1136 rc = qdf_status_to_os_return(ret);
1137 qdf_dev_modify_irq_status(napi_info->irq, 0,
1138 QDF_IRQ_NO_BALANCING);
1139
1140 if (rc)
1141 hif_err("Setting irq affinity hint: %d", rc);
1142 else
1143 napi_info->stats[cpu].cpu_corrected++;
1144 }
1145 }
1146 return right_cpu;
1147 }
1148
1149 #ifdef RECEIVE_OFFLOAD
1150 /**
1151 * hif_napi_offld_flush_cb() - Call upper layer flush callback
1152 * @napi_info: Handle to hif_napi_info
1153 *
1154 * Return: None
1155 */
hif_napi_offld_flush_cb(struct qca_napi_info * napi_info)1156 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info)
1157 {
1158 if (napi_info->offld_flush_cb)
1159 napi_info->offld_flush_cb(napi_info);
1160 }
1161 #else
hif_napi_offld_flush_cb(struct qca_napi_info * napi_info)1162 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info)
1163 {
1164 }
1165 #endif
1166
1167 /**
1168 * hif_napi_poll() - NAPI poll routine
1169 * @hif_ctx: HIF context
1170 * @napi: pointer to NAPI struct as kernel holds it
1171 * @budget:
1172 *
1173 * This is the body of the poll function.
1174 * The poll function is called by kernel. So, there is a wrapper
1175 * function in HDD, which in turn calls this function.
1176 * Two main reasons why the whole thing is not implemented in HDD:
1177 * a) references to things like ce_service that HDD is not aware of
1178 * b) proximity to the implementation of ce_tasklet, which the body
1179 * of this function should be very close to.
1180 *
1181 * NOTE TO THE MAINTAINER:
1182 * Consider this function and ce_tasklet very tightly coupled pairs.
1183 * Any changes to ce_tasklet or this function may likely need to be
1184 * reflected in the counterpart.
1185 *
1186 * Returns:
1187 * int: the amount of work done in this poll (<= budget)
1188 */
hif_napi_poll(struct hif_opaque_softc * hif_ctx,struct napi_struct * napi,int budget)1189 int hif_napi_poll(struct hif_opaque_softc *hif_ctx,
1190 struct napi_struct *napi,
1191 int budget)
1192 {
1193 int rc = 0; /* default: no work done, also takes care of error */
1194 int normalized = 0;
1195 int bucket;
1196 int cpu = smp_processor_id();
1197 bool poll_on_right_cpu;
1198 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
1199 struct qca_napi_info *napi_info;
1200 struct CE_state *ce_state = NULL;
1201
1202 if (unlikely(!hif)) {
1203 hif_err("hif context is NULL");
1204 QDF_ASSERT(0);
1205 goto out;
1206 }
1207
1208 napi_info = (struct qca_napi_info *)
1209 container_of(napi, struct qca_napi_info, napi);
1210
1211 hif_napi_update_service_start_time(napi_info);
1212 hif_napi_latency_profile_measure(napi_info);
1213
1214 NAPI_DEBUG("%s -->(napi(%d, irq=%d), budget=%d)",
1215 __func__, napi_info->id, napi_info->irq, budget);
1216
1217 napi_info->stats[cpu].napi_polls++;
1218
1219 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id),
1220 NAPI_POLL_ENTER, NULL, NULL, cpu, 0);
1221
1222 rc = ce_per_engine_service(hif, NAPI_ID2PIPE(napi_info->id));
1223 NAPI_DEBUG("%s: ce_per_engine_service processed %d msgs",
1224 __func__, rc);
1225
1226 hif_napi_offld_flush_cb(napi_info);
1227
1228 /* do not return 0, if there was some work done,
1229 * even if it is below the scale
1230 */
1231 if (rc) {
1232 napi_info->stats[cpu].napi_workdone += rc;
1233 normalized = (rc / napi_info->scale);
1234 if (normalized == 0)
1235 normalized++;
1236 bucket = (normalized - 1) /
1237 (QCA_NAPI_BUDGET / QCA_NAPI_NUM_BUCKETS);
1238 if (bucket >= QCA_NAPI_NUM_BUCKETS) {
1239 bucket = QCA_NAPI_NUM_BUCKETS - 1;
1240 hif_err("Bad bucket#(%d) > QCA_NAPI_NUM_BUCKETS(%d)"
1241 " normalized %d, napi budget %d",
1242 bucket, QCA_NAPI_NUM_BUCKETS,
1243 normalized, QCA_NAPI_BUDGET);
1244 }
1245 napi_info->stats[cpu].napi_budget_uses[bucket]++;
1246 } else {
1247 /* if ce_per engine reports 0, then poll should be terminated */
1248 NAPI_DEBUG("%s:%d: nothing processed by CE. Completing NAPI",
1249 __func__, __LINE__);
1250 }
1251
1252 ce_state = hif->ce_id_to_state[NAPI_ID2PIPE(napi_info->id)];
1253
1254 /*
1255 * Not using the API hif_napi_correct_cpu directly in the if statement
1256 * below since the API may not get evaluated if put at the end if any
1257 * prior condition would evaluate to be true. The CPU correction
1258 * check should kick in every poll.
1259 */
1260 #ifdef NAPI_YIELD_BUDGET_BASED
1261 if (ce_state && (ce_state->force_break || 0 == rc)) {
1262 #else
1263 poll_on_right_cpu = hif_napi_correct_cpu(napi_info);
1264 if ((ce_state) &&
1265 (!ce_check_rx_pending(ce_state) || (0 == rc) ||
1266 !poll_on_right_cpu)) {
1267 #endif
1268 napi_info->stats[cpu].napi_completes++;
1269 #ifdef NAPI_YIELD_BUDGET_BASED
1270 ce_state->force_break = 0;
1271 #endif
1272
1273 hif_record_ce_desc_event(hif, ce_state->id, NAPI_COMPLETE,
1274 NULL, NULL, 0, 0);
1275 if (normalized >= budget)
1276 normalized = budget - 1;
1277
1278 napi_complete(napi);
1279 /* enable interrupts */
1280 hif_napi_enable_irq(hif_ctx, napi_info->id);
1281 /* support suspend/resume */
1282 qdf_atomic_dec(&(hif->active_tasklet_cnt));
1283
1284 NAPI_DEBUG("%s:%d: napi_complete + enabling the interrupts",
1285 __func__, __LINE__);
1286 } else {
1287 /* 4.4 kernel NAPI implementation requires drivers to
1288 * return full work when they ask to be re-scheduled,
1289 * or napi_complete and re-start with a fresh interrupt
1290 */
1291 normalized = budget;
1292 }
1293
1294 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id),
1295 NAPI_POLL_EXIT, NULL, NULL, normalized, 0);
1296
1297 hif_napi_fill_poll_time_histogram(napi_info);
1298
1299 NAPI_DEBUG("%s <--[normalized=%d]", __func__, normalized);
1300 return normalized;
1301 out:
1302 return rc;
1303 }
1304 qdf_export_symbol(hif_napi_poll);
1305
1306 void hif_update_napi_max_poll_time(struct CE_state *ce_state,
1307 int ce_id,
1308 int cpu_id)
1309 {
1310 struct hif_softc *hif;
1311 struct qca_napi_info *napi_info;
1312 unsigned long long napi_poll_time = qdf_time_sched_clock() -
1313 ce_state->ce_service_start_time;
1314
1315 hif = ce_state->scn;
1316 napi_info = hif->napi_data.napis[ce_id];
1317 if (napi_poll_time >
1318 napi_info->stats[cpu_id].napi_max_poll_time)
1319 napi_info->stats[cpu_id].napi_max_poll_time = napi_poll_time;
1320 }
1321 qdf_export_symbol(hif_update_napi_max_poll_time);
1322
1323 #ifdef HIF_IRQ_AFFINITY
1324 /**
1325 * hif_napi_update_yield_stats() - update NAPI yield related stats
1326 * @ce_state: CE state structure
1327 * @time_limit_reached: indicates whether the time limit was reached
1328 * @rxpkt_thresh_reached: indicates whether rx packet threshold was reached
1329 *
1330 * Return: None
1331 */
1332 void hif_napi_update_yield_stats(struct CE_state *ce_state,
1333 bool time_limit_reached,
1334 bool rxpkt_thresh_reached)
1335 {
1336 struct hif_softc *hif;
1337 struct qca_napi_data *napi_data = NULL;
1338 int ce_id = 0;
1339 int cpu_id = 0;
1340
1341 if (unlikely(!ce_state)) {
1342 QDF_ASSERT(ce_state);
1343 return;
1344 }
1345
1346 hif = ce_state->scn;
1347
1348 if (unlikely(!hif)) {
1349 QDF_ASSERT(hif);
1350 return;
1351 }
1352 napi_data = &(hif->napi_data);
1353 if (unlikely(!napi_data)) {
1354 QDF_ASSERT(napi_data);
1355 return;
1356 }
1357
1358 ce_id = ce_state->id;
1359 cpu_id = qdf_get_cpu();
1360
1361 if (unlikely(!napi_data->napis[ce_id])) {
1362 return;
1363 }
1364
1365 if (time_limit_reached)
1366 napi_data->napis[ce_id]->stats[cpu_id].time_limit_reached++;
1367 else
1368 napi_data->napis[ce_id]->stats[cpu_id].rxpkt_thresh_reached++;
1369
1370 hif_update_napi_max_poll_time(ce_state, ce_id,
1371 cpu_id);
1372 }
1373
1374 /**
1375 * hif_napi_stats() - display NAPI CPU statistics
1376 * @napid: pointer to qca_napi_data
1377 *
1378 * Description:
1379 * Prints the various CPU cores on which the NAPI instances /CEs interrupts
1380 * are being executed. Can be called from outside NAPI layer.
1381 *
1382 * Return: None
1383 */
1384 void hif_napi_stats(struct qca_napi_data *napid)
1385 {
1386 int i;
1387 struct qca_napi_cpu *cpu;
1388
1389 if (!napid) {
1390 qdf_debug("%s: napiid struct is null", __func__);
1391 return;
1392 }
1393
1394 cpu = napid->napi_cpu;
1395 qdf_debug("NAPI CPU TABLE");
1396 qdf_debug("lilclhead=%d, bigclhead=%d",
1397 napid->lilcl_head, napid->bigcl_head);
1398 for (i = 0; i < NR_CPUS; i++) {
1399 qdf_debug("CPU[%02d]: state:%d crid=%02d clid=%02d crmk:0x%0lx thmk:0x%0lx frq:%d napi = 0x%08x lnk:%d",
1400 i,
1401 cpu[i].state, cpu[i].core_id, cpu[i].cluster_id,
1402 cpu[i].core_mask.bits[0],
1403 cpu[i].thread_mask.bits[0],
1404 cpu[i].max_freq, cpu[i].napis,
1405 cpu[i].cluster_nxt);
1406 }
1407 }
1408
1409 #ifdef FEATURE_NAPI_DEBUG
1410 /*
1411 * Local functions
1412 * - no argument checks, all internal/trusted callers
1413 */
1414 static void hnc_dump_cpus(struct qca_napi_data *napid)
1415 {
1416 hif_napi_stats(napid);
1417 }
1418 #else
1419 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ };
1420 #endif /* FEATURE_NAPI_DEBUG */
1421
1422 #define HNC_MIN_CLUSTER 0
1423 #define HNC_MAX_CLUSTER 1
1424
1425 /**
1426 * hnc_link_clusters() - partitions to cpu table into clusters
1427 * @napid: pointer to NAPI data
1428 *
1429 * Takes in a CPU topology table and builds two linked lists
1430 * (big cluster cores, list-head at bigcl_head, and little cluster
1431 * cores, list-head at lilcl_head) out of it.
1432 *
1433 * If there are more than two clusters:
1434 * - bigcl_head and lilcl_head will be different,
1435 * - the cluster with highest cpufreq will be considered the "big" cluster.
1436 * If there are more than one with the highest frequency, the *last* of such
1437 * clusters will be designated as the "big cluster"
1438 * - the cluster with lowest cpufreq will be considered the "li'l" cluster.
1439 * If there are more than one clusters with the lowest cpu freq, the *first*
1440 * of such clusters will be designated as the "little cluster"
1441 * - We only support up to 32 clusters
1442 * Return: 0 : OK
1443 * !0: error (at least one of lil/big clusters could not be found)
1444 */
1445 static int hnc_link_clusters(struct qca_napi_data *napid)
1446 {
1447 int rc = 0;
1448
1449 int i;
1450 int it = 0;
1451 uint32_t cl_done = 0x0;
1452 int cl, curcl, curclhead = 0;
1453 int more;
1454 unsigned int lilfrq = INT_MAX;
1455 unsigned int bigfrq = 0;
1456 unsigned int clfrq = 0;
1457 int prev = 0;
1458 struct qca_napi_cpu *cpus = napid->napi_cpu;
1459
1460 napid->lilcl_head = napid->bigcl_head = -1;
1461
1462 do {
1463 more = 0;
1464 it++; curcl = -1;
1465 for (i = 0; i < NR_CPUS; i++) {
1466 cl = cpus[i].cluster_id;
1467 NAPI_DEBUG("Processing cpu[%d], cluster=%d\n",
1468 i, cl);
1469 if ((cl < HNC_MIN_CLUSTER) || (cl > HNC_MAX_CLUSTER)) {
1470 NAPI_DEBUG("Bad cluster (%d). SKIPPED\n", cl);
1471 /* continue if ASSERTs are disabled */
1472 continue;
1473 };
1474 if (cpumask_weight(&(cpus[i].core_mask)) == 0) {
1475 NAPI_DEBUG("Core mask 0. SKIPPED\n");
1476 continue;
1477 }
1478 if (cl_done & (0x01 << cl)) {
1479 NAPI_DEBUG("Cluster already processed. SKIPPED\n");
1480 continue;
1481 } else {
1482 if (more == 0) {
1483 more = 1;
1484 curcl = cl;
1485 curclhead = i; /* row */
1486 clfrq = cpus[i].max_freq;
1487 prev = -1;
1488 };
1489 if ((curcl >= 0) && (curcl != cl)) {
1490 NAPI_DEBUG("Entry cl(%d) != curcl(%d). SKIPPED\n",
1491 cl, curcl);
1492 continue;
1493 }
1494 if (cpus[i].max_freq != clfrq)
1495 NAPI_DEBUG("WARN: frq(%d)!=clfrq(%d)\n",
1496 cpus[i].max_freq, clfrq);
1497 if (clfrq >= bigfrq) {
1498 bigfrq = clfrq;
1499 napid->bigcl_head = curclhead;
1500 NAPI_DEBUG("bigcl=%d\n", curclhead);
1501 }
1502 if (clfrq < lilfrq) {
1503 lilfrq = clfrq;
1504 napid->lilcl_head = curclhead;
1505 NAPI_DEBUG("lilcl=%d\n", curclhead);
1506 }
1507 if (prev != -1)
1508 cpus[prev].cluster_nxt = i;
1509
1510 prev = i;
1511 }
1512 }
1513 if (curcl >= 0)
1514 cl_done |= (0x01 << curcl);
1515
1516 } while (more);
1517
1518 if (qdf_unlikely((napid->lilcl_head < 0) && (napid->bigcl_head < 0)))
1519 rc = -EFAULT;
1520
1521 hnc_dump_cpus(napid); /* if NAPI_DEBUG */
1522 return rc;
1523 }
1524 #undef HNC_MIN_CLUSTER
1525 #undef HNC_MAX_CLUSTER
1526
1527 /*
1528 * hotplug function group
1529 */
1530
1531 /**
1532 * hnc_cpu_online_cb() - handles CPU hotplug "up" events
1533 * @context: the associated HIF context
1534 * @cpu: the CPU Id of the CPU the event happened on
1535 *
1536 * Return: None
1537 */
1538 static void hnc_cpu_online_cb(void *context, uint32_t cpu)
1539 {
1540 struct hif_softc *hif = context;
1541 struct qca_napi_data *napid = &hif->napi_data;
1542
1543 if (cpu >= NR_CPUS)
1544 return;
1545
1546 NAPI_DEBUG("-->%s(act=online, cpu=%u)", __func__, cpu);
1547
1548 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_UP;
1549 NAPI_DEBUG("%s: CPU %u marked %d",
1550 __func__, cpu, napid->napi_cpu[cpu].state);
1551
1552 NAPI_DEBUG("<--%s", __func__);
1553 }
1554
1555 /**
1556 * hnc_cpu_before_offline_cb() - handles CPU hotplug "prepare down" events
1557 * @context: the associated HIF context
1558 * @cpu: the CPU Id of the CPU the event happened on
1559 *
1560 * On transition to offline, we act on PREP events, because we may need to move
1561 * the irqs/NAPIs to another CPU before it is actually off-lined.
1562 *
1563 * Return: None
1564 */
1565 static void hnc_cpu_before_offline_cb(void *context, uint32_t cpu)
1566 {
1567 struct hif_softc *hif = context;
1568 struct qca_napi_data *napid = &hif->napi_data;
1569
1570 if (cpu >= NR_CPUS)
1571 return;
1572
1573 NAPI_DEBUG("-->%s(act=before_offline, cpu=%u)", __func__, cpu);
1574
1575 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_DOWN;
1576
1577 NAPI_DEBUG("%s: CPU %u marked %d; updating affinity",
1578 __func__, cpu, napid->napi_cpu[cpu].state);
1579
1580 /**
1581 * we need to move any NAPIs on this CPU out.
1582 * if we are in LO throughput mode, then this is valid
1583 * if the CPU is the the low designated CPU.
1584 */
1585 hif_napi_event(GET_HIF_OPAQUE_HDL(hif),
1586 NAPI_EVT_CPU_STATE,
1587 (void *)
1588 ((size_t)cpu << 16 | napid->napi_cpu[cpu].state));
1589
1590 NAPI_DEBUG("<--%s", __func__);
1591 }
1592
1593 static int hnc_hotplug_register(struct hif_softc *hif_sc)
1594 {
1595 QDF_STATUS status;
1596
1597 NAPI_DEBUG("-->%s", __func__);
1598
1599 status = qdf_cpuhp_register(&hif_sc->napi_data.cpuhp_handler,
1600 hif_sc,
1601 hnc_cpu_online_cb,
1602 hnc_cpu_before_offline_cb);
1603
1604 NAPI_DEBUG("<--%s [%d]", __func__, status);
1605
1606 return qdf_status_to_os_return(status);
1607 }
1608
1609 static void hnc_hotplug_unregister(struct hif_softc *hif_sc)
1610 {
1611 NAPI_DEBUG("-->%s", __func__);
1612
1613 if (hif_sc->napi_data.cpuhp_handler)
1614 qdf_cpuhp_unregister(&hif_sc->napi_data.cpuhp_handler);
1615
1616 NAPI_DEBUG("<--%s", __func__);
1617 }
1618
1619 /**
1620 * hnc_tput_hook() - installs a callback in the throughput detector
1621 * @install: !0 => install; =0: uninstall
1622 *
1623 * installs a callback to be called when wifi driver throughput (tx+rx)
1624 * crosses a threshold. Currently, we are using the same criteria as
1625 * TCP ack suppression (500 packets/100ms by default).
1626 *
1627 * Return: 0 : success
1628 * <0: failure
1629 */
1630
1631 static int hnc_tput_hook(int install)
1632 {
1633 int rc = 0;
1634
1635 /*
1636 * Nothing, until the bw_calculation accepts registration
1637 * it is now hardcoded in the wlan_hdd_main.c::hdd_bus_bw_compute_cbk
1638 * hdd_napi_throughput_policy(...)
1639 */
1640 return rc;
1641 }
1642
1643 /*
1644 * Implementation of hif_napi_cpu API
1645 */
1646
1647 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
1648 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i)
1649 {
1650 cpumask_copy(&(cpus[i].thread_mask),
1651 topology_sibling_cpumask(i));
1652 }
1653 #else
1654 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i)
1655 {
1656 }
1657 #endif
1658
1659
1660 /**
1661 * hif_napi_cpu_init() - initialization of irq affinity block
1662 * @hif: HIF context
1663 *
1664 * called by hif_napi_create, after the first instance is called
1665 * - builds napi_rss_cpus table from cpu topology
1666 * - links cores of the same clusters together
1667 * - installs hot-plug notifier
1668 * - installs throughput trigger notifier (when such mechanism exists)
1669 *
1670 * Return: 0: OK
1671 * <0: error code
1672 */
1673 int hif_napi_cpu_init(struct hif_opaque_softc *hif)
1674 {
1675 int rc = 0;
1676 int i;
1677 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data;
1678 struct qca_napi_cpu *cpus = napid->napi_cpu;
1679
1680 NAPI_DEBUG("--> ");
1681
1682 if (cpus[0].state != QCA_NAPI_CPU_UNINITIALIZED) {
1683 NAPI_DEBUG("NAPI RSS table already initialized.\n");
1684 rc = -EALREADY;
1685 goto lab_rss_init;
1686 }
1687
1688 /* build CPU topology table */
1689 for_each_possible_cpu(i) {
1690 cpus[i].state = ((cpumask_test_cpu(i, cpu_online_mask)
1691 ? QCA_NAPI_CPU_UP
1692 : QCA_NAPI_CPU_DOWN));
1693 cpus[i].core_id = topology_core_id(i);
1694 cpus[i].cluster_id = topology_physical_package_id(i);
1695 cpumask_copy(&(cpus[i].core_mask),
1696 topology_core_cpumask(i));
1697 record_sibling_cpumask(cpus, i);
1698 cpus[i].max_freq = cpufreq_quick_get_max(i);
1699 cpus[i].napis = 0x0;
1700 cpus[i].cluster_nxt = -1; /* invalid */
1701 }
1702
1703 /* link clusters together */
1704 rc = hnc_link_clusters(napid);
1705 if (0 != rc)
1706 goto lab_err_topology;
1707
1708 /* install hotplug notifier */
1709 rc = hnc_hotplug_register(HIF_GET_SOFTC(hif));
1710 if (0 != rc)
1711 goto lab_err_hotplug;
1712
1713 /* install throughput notifier */
1714 rc = hnc_tput_hook(1);
1715 if (0 == rc)
1716 goto lab_rss_init;
1717
1718 lab_err_hotplug:
1719 hnc_tput_hook(0);
1720 hnc_hotplug_unregister(HIF_GET_SOFTC(hif));
1721 lab_err_topology:
1722 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS);
1723 lab_rss_init:
1724 NAPI_DEBUG("<-- [rc=%d]", rc);
1725 return rc;
1726 }
1727
1728 /**
1729 * hif_napi_cpu_deinit() - clean-up of irq affinity block
1730 * @hif: HIF context
1731 *
1732 * called by hif_napi_destroy, when the last instance is removed
1733 * - uninstalls throughput and hotplug notifiers
1734 * - clears cpu topology table
1735 * Return: 0: OK
1736 */
1737 int hif_napi_cpu_deinit(struct hif_opaque_softc *hif)
1738 {
1739 int rc = 0;
1740 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data;
1741
1742 NAPI_DEBUG("-->%s(...)", __func__);
1743
1744 /* uninstall tput notifier */
1745 rc = hnc_tput_hook(0);
1746
1747 /* uninstall hotplug notifier */
1748 hnc_hotplug_unregister(HIF_GET_SOFTC(hif));
1749
1750 /* clear the topology table */
1751 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS);
1752
1753 NAPI_DEBUG("<--%s[rc=%d]", __func__, rc);
1754
1755 return rc;
1756 }
1757
1758 /**
1759 * hncm_migrate_to() - migrates a NAPI to a CPU
1760 * @napid: pointer to NAPI block
1761 * @napi_ce: CE_id of the NAPI instance
1762 * @didx: index in the CPU topology table for the CPU to migrate to
1763 *
1764 * Migrates NAPI (identified by the CE_id) to the destination core
1765 * Updates the napi_map of the destination entry
1766 *
1767 * Return:
1768 * =0 : success
1769 * <0 : error
1770 */
1771 static int hncm_migrate_to(struct qca_napi_data *napid,
1772 int napi_ce,
1773 int didx)
1774 {
1775 int rc = 0;
1776 QDF_STATUS status;
1777
1778 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, napi_ce, didx);
1779
1780 if (!napid->napis[napi_ce])
1781 return -EINVAL;
1782
1783 napid->napis[napi_ce]->cpumask.bits[0] = (1 << didx);
1784
1785 qdf_dev_modify_irq_status(napid->napis[napi_ce]->irq,
1786 QDF_IRQ_NO_BALANCING, 0);
1787 status = qdf_dev_set_irq_affinity(napid->napis[napi_ce]->irq,
1788 (struct qdf_cpu_mask *)
1789 &napid->napis[napi_ce]->cpumask);
1790 rc = qdf_status_to_os_return(status);
1791
1792 /* unmark the napis bitmap in the cpu table */
1793 napid->napi_cpu[napid->napis[napi_ce]->cpu].napis &= ~(0x01 << napi_ce);
1794 /* mark the napis bitmap for the new designated cpu */
1795 napid->napi_cpu[didx].napis |= (0x01 << napi_ce);
1796 napid->napis[napi_ce]->cpu = didx;
1797
1798 NAPI_DEBUG("<--%s[%d]", __func__, rc);
1799 return rc;
1800 }
1801 /**
1802 * hncm_dest_cpu() - finds a destination CPU for NAPI
1803 * @napid: pointer to NAPI block
1804 * @act: RELOCATE | COLLAPSE | DISPERSE
1805 *
1806 * Finds the designated destination for the next IRQ.
1807 * RELOCATE: translated to either COLLAPSE or DISPERSE based
1808 * on napid->napi_mode (throughput state)
1809 * COLLAPSE: All have the same destination: the first online CPU in lilcl
1810 * DISPERSE: One of the CPU in bigcl, which has the smallest number of
1811 * NAPIs on it
1812 *
1813 * Return: >=0 : index in the cpu topology table
1814 * : < 0 : error
1815 */
1816 static int hncm_dest_cpu(struct qca_napi_data *napid, int act)
1817 {
1818 int destidx = -1;
1819 int head, i;
1820
1821 NAPI_DEBUG("-->%s(act=%d)", __func__, act);
1822 if (act == HNC_ACT_RELOCATE) {
1823 if (napid->napi_mode == QCA_NAPI_TPUT_LO)
1824 act = HNC_ACT_COLLAPSE;
1825 else
1826 act = HNC_ACT_DISPERSE;
1827 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d",
1828 __func__, act);
1829 }
1830 if (act == HNC_ACT_COLLAPSE) {
1831 head = i = napid->lilcl_head;
1832 retry_collapse:
1833 while (i >= 0) {
1834 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) {
1835 destidx = i;
1836 break;
1837 }
1838 i = napid->napi_cpu[i].cluster_nxt;
1839 }
1840 if ((destidx < 0) && (head == napid->lilcl_head)) {
1841 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl",
1842 __func__);
1843 head = i = napid->bigcl_head;
1844 goto retry_collapse;
1845 }
1846 } else { /* HNC_ACT_DISPERSE */
1847 int smallest = 99; /* all 32 bits full */
1848 int smallidx = -1;
1849
1850 head = i = napid->bigcl_head;
1851 retry_disperse:
1852 while (i >= 0) {
1853 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) &&
1854 (hweight32(napid->napi_cpu[i].napis) <= smallest)) {
1855 smallest = napid->napi_cpu[i].napis;
1856 smallidx = i;
1857 }
1858 i = napid->napi_cpu[i].cluster_nxt;
1859 }
1860 /* Check if matches with user specified CPU mask */
1861 smallidx = ((1 << smallidx) & napid->user_cpu_affin_mask) ?
1862 smallidx : -1;
1863
1864 if ((smallidx < 0) && (head == napid->bigcl_head)) {
1865 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl",
1866 __func__);
1867 head = i = napid->lilcl_head;
1868 goto retry_disperse;
1869 }
1870 destidx = smallidx;
1871 }
1872 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx);
1873 return destidx;
1874 }
1875 /**
1876 * hif_napi_cpu_migrate() - migrate IRQs away
1877 * @napid: pointer to NAPI block
1878 * @cpu: -1: all CPUs <n> specific CPU
1879 * @action: COLLAPSE | DISPERSE
1880 *
1881 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible
1882 * cores. Eligible cores are:
1883 * act=COLLAPSE -> the first online core of the little cluster
1884 * act=DISPERSE -> separate cores of the big cluster, so that each core will
1885 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis)
1886 *
1887 * Note that this function is called with a spinlock acquired already.
1888 *
1889 * Return: =0: success
1890 * <0: error
1891 */
1892
1893 int hif_napi_cpu_migrate(struct qca_napi_data *napid, int cpu, int action)
1894 {
1895 int rc = 0;
1896 struct qca_napi_cpu *cpup;
1897 int i, dind;
1898 uint32_t napis;
1899
1900 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)",
1901 __func__, cpu, action);
1902 /* the following is really: hif_napi_enabled() with less overhead */
1903 if (napid->ce_map == 0) {
1904 NAPI_DEBUG("%s: NAPI disabled. Not migrating.", __func__);
1905 goto hncm_return;
1906 }
1907
1908 cpup = napid->napi_cpu;
1909
1910 switch (action) {
1911 case HNC_ACT_RELOCATE:
1912 case HNC_ACT_DISPERSE:
1913 case HNC_ACT_COLLAPSE: {
1914 /* first find the src napi set */
1915 if (cpu == HNC_ANY_CPU)
1916 napis = napid->ce_map;
1917 else
1918 napis = cpup[cpu].napis;
1919 /* then clear the napi bitmap on each CPU */
1920 for (i = 0; i < NR_CPUS; i++)
1921 cpup[i].napis = 0;
1922 /* then for each of the NAPIs to disperse: */
1923 for (i = 0; i < CE_COUNT_MAX; i++)
1924 if (napis & (1 << i)) {
1925 /* find a destination CPU */
1926 dind = hncm_dest_cpu(napid, action);
1927 if (dind >= 0) {
1928 NAPI_DEBUG("Migrating NAPI ce%d to %d",
1929 i, dind);
1930 rc = hncm_migrate_to(napid, i, dind);
1931 } else {
1932 NAPI_DEBUG("No dest for NAPI ce%d", i);
1933 hnc_dump_cpus(napid);
1934 rc = -1;
1935 }
1936 }
1937 break;
1938 }
1939 default: {
1940 NAPI_DEBUG("%s: bad action: %d\n", __func__, action);
1941 QDF_BUG(0);
1942 break;
1943 }
1944 } /* switch action */
1945
1946 hncm_return:
1947 hnc_dump_cpus(napid);
1948 return rc;
1949 }
1950
1951
1952 /**
1953 * hif_napi_dl_irq() - calls irq_modify_status to enable/disable denylisting
1954 * @napid: pointer to qca_napi_data structure
1955 * @dl_flag: denylist flag to enable/disable denylisting
1956 *
1957 * The function enables/disables denylisting for all the copy engine
1958 * interrupts on which NAPI is enabled.
1959 *
1960 * Return: None
1961 */
1962 static inline void hif_napi_dl_irq(struct qca_napi_data *napid, bool dl_flag)
1963 {
1964 int i;
1965 struct qca_napi_info *napii;
1966
1967 for (i = 0; i < CE_COUNT_MAX; i++) {
1968 /* check if NAPI is enabled on the CE */
1969 if (!(napid->ce_map & (0x01 << i)))
1970 continue;
1971
1972 /*double check that NAPI is allocated for the CE */
1973 napii = napid->napis[i];
1974 if (!(napii))
1975 continue;
1976
1977 if (dl_flag == true)
1978 qdf_dev_modify_irq_status(napii->irq,
1979 0, QDF_IRQ_NO_BALANCING);
1980 else
1981 qdf_dev_modify_irq_status(napii->irq,
1982 QDF_IRQ_NO_BALANCING, 0);
1983 hif_debug("dl_flag %d CE %d", dl_flag, i);
1984 }
1985 }
1986
1987 /**
1988 * hif_napi_cpu_denylist() - en(dis)ables denylisting for NAPI RX interrupts.
1989 * @napid: pointer to qca_napi_data structure
1990 * @op: denylist operation to perform
1991 *
1992 * The function enables/disables/queries denylisting for all CE RX
1993 * interrupts with NAPI enabled. Besides denylisting, it also enables/disables
1994 * core_ctl_set_boost.
1995 * Once denylisting is enabled, the interrupts will not be managed by the IRQ
1996 * balancer.
1997 *
1998 * Return: -EINVAL, in case IRQ_DENYLISTING and CORE_CTL_BOOST is not enabled
1999 * for DENYLIST_QUERY op - denylist refcount
2000 * for DENYLIST_ON op - return value from core_ctl_set_boost API
2001 * for DENYLIST_OFF op - return value from core_ctl_set_boost API
2002 */
2003 int hif_napi_cpu_denylist(struct qca_napi_data *napid,
2004 enum qca_denylist_op op)
2005 {
2006 int rc = 0;
2007 static int ref_count; /* = 0 by the compiler */
2008 uint8_t flags = napid->flags;
2009 bool dl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING;
2010 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST;
2011
2012 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op);
2013
2014 if (!(dl_en && ccb_en)) {
2015 rc = -EINVAL;
2016 goto out;
2017 }
2018
2019 switch (op) {
2020 case DENYLIST_QUERY:
2021 rc = ref_count;
2022 break;
2023 case DENYLIST_ON:
2024 ref_count++;
2025 rc = 0;
2026 if (ref_count == 1) {
2027 rc = hif_napi_core_ctl_set_boost(true);
2028 NAPI_DEBUG("boost_on() returns %d - refcnt=%d",
2029 rc, ref_count);
2030 hif_napi_dl_irq(napid, true);
2031 }
2032 break;
2033 case DENYLIST_OFF:
2034 if (ref_count) {
2035 ref_count--;
2036 rc = 0;
2037 if (ref_count == 0) {
2038 rc = hif_napi_core_ctl_set_boost(false);
2039 NAPI_DEBUG("boost_off() returns %d - refcnt=%d",
2040 rc, ref_count);
2041 hif_napi_dl_irq(napid, false);
2042 }
2043 }
2044 break;
2045 default:
2046 NAPI_DEBUG("Invalid denylist op: %d", op);
2047 rc = -EINVAL;
2048 } /* switch */
2049 out:
2050 NAPI_DEBUG("<--%s[%d]", __func__, rc);
2051 return rc;
2052 }
2053
2054 static unsigned long napi_serialize_reqs;
2055 /**
2056 * hif_napi_serialize() - [de-]serialize NAPI operations
2057 * @hif: context
2058 * @is_on: 1: serialize, 0: deserialize
2059 *
2060 * hif_napi_serialize(hif, 1) can be called multiple times. It will perform the
2061 * following steps (see hif_napi_event for code):
2062 * - put irqs of all NAPI instances on the same CPU
2063 * - only for the first serialize call: denylist
2064 *
2065 * hif_napi_serialize(hif, 0):
2066 * - start a timer (multiple of BusBandwidthTimer -- default: 100 msec)
2067 * - at the end of the timer, check the current throughput state and
2068 * implement it.
2069 */
2070 int hif_napi_serialize(struct hif_opaque_softc *hif, int is_on)
2071 {
2072 int rc = -EINVAL;
2073
2074 if (hif)
2075 switch (is_on) {
2076 case 0: { /* de-serialize */
2077 rc = hif_napi_event(hif, NAPI_EVT_USR_NORMAL,
2078 (void *) 0);
2079 napi_serialize_reqs = 0;
2080 break;
2081 } /* end de-serialize */
2082 case 1: { /* serialize */
2083 rc = hif_napi_event(hif, NAPI_EVT_USR_SERIAL,
2084 (void *)napi_serialize_reqs++);
2085 break;
2086 } /* end serialize */
2087 default:
2088 break; /* no-op */
2089 } /* switch */
2090 return rc;
2091 }
2092
2093 #endif /* ifdef HIF_IRQ_AFFINITY */
2094